def test_merging_topics_and_comments(self):
topics_schema = [{
'id': {
'*': 'topic-id'
},
'type': {
'*':
'|lambda x: {0: "NEED", 1: "GOAL", 2: "IDEA", 3: "PLAN", 4: "STEP", 5: "TASK"}.get(x)'
},
'owner': {
'username': {
'*': ''
},
'id': {
'*': 'user-id'
}
},
'blockchain': {
'*': '|lambda x: x and True or False'
},
}]
comments_schema = [{
'id': {
'*': 'comment-id'
},
'topic': {
'*': 'topic-url'
},
'text': {
'*': 'body'
},
'owner': {
'username': {
'*': ''
},
'id': {
'*': 'user-id'
}
},
'blockchain': {
'*': '|lambda x: x and True or False'
},
}]
normal_topics = normalize(self.topics, topics_schema)
normal_comments = normalize(self.comments, comments_schema)
with open('metaform/tests/data/topics+comments.json', 'r') as f:
expect = json.load(f)
self.assertEqual(normal_topics + normal_comments, expect)
def test_apply_lambdas(self):
schema = {
'*':
'greeting',
'hello': {
'*': 'length|lambda x: x+5.'
},
'world': {
'*': 'atoms|lambda x: str(x)+"ABC"'
},
'how': [{
'*': 'method',
'are': {
'*': 'yup',
'you': {
'*': 'me|lambda x: "-".join(list(x))'
}
}
}]
}
expect = {
'atoms': '2ABC',
'length': 6.0,
'method': ['is', {
'yup': {
'me': 'd-o-i-n-g'
}
}]
}
self.assertEqual(normalize(self.data, schema), expect)
def test_rename_keys(self):
schema = {
'*': 'greeting',
'hello': {
'*': 'length'
},
'world': {
'*': 'atoms'
},
'how': [{
'*': 'method',
'are': {
'*': 'yup',
'you': {
'*': 'me'
}
}
}]
}
expect = {
'atoms': 2,
'length': 1.0,
'method': ['is', {
'yup': {
'me': 'doing'
}
}]
}
self.assertEqual(normalize(self.data, schema), expect)
def readin(cls, location, schema=None, limit=None):
from tqdm import tqdm #noqa
if os.path.exists(location):
if os.path.isdir(location):
n = 0
for fname in tqdm(os.listdir(location)):
record = yaml.load(open(os.path.join(location,
fname)).read(),
Loader=loader)
if schema is not None:
import metaform #noqa
yield metaform.formatize(
metaform.normalize(record, schema))
else:
yield record
if limit is not None:
n += 1
if n >= limit:
break
elif os.path.isfile(location):
yield yaml.load(open(location).read(), Loader=loader)
def test_custom_converters(self):
def some_func(x):
a = 123
b = 345
return (b - a) * x
converters.func = some_func
schema = {
'*':
'greeting',
'hello': {
'*': 'length|converters.func'
},
'world': {
'*': 'atoms|lambda x: str(x)+"ABC"'
},
'how': [{
'*': 'method',
'are': {
'*': 'yup',
'you': {
'*': 'me|lambda x: "-".join(list(x))'
}
}
}]
}
expect = {
'atoms': '2ABC',
'length': 222.0,
'method': ['is', {
'yup': {
'me': 'd-o-i-n-g'
}
}]
}
self.assertEqual(normalize(self.data, schema), expect)
def test_make_and_apply_template(self):
data = {'a': [{'b': 'c'}, {'e': 'f'}, {'g': 'h'}], 'b': 'something'}
tpl = template(data)
answer = {
'*': '',
'a': [{
'*': '',
'b': {
'*': ''
},
'e': {
'*': ''
},
'g': {
'*': ''
}
}],
'b': {
'*': ''
}
}
self.assertEqual(tpl, answer)
# apply:
tpl['a'][0]['b']['*'] = 'hello'
result = normalize(data, tpl)
answer = deepcopy(data)
del answer['a'][0]['b']
answer['a'][0]['hello'] = 'c'
self.assertEqual(result, answer)
def test_make_and_apply_complex_template(self):
data = {
'fields': {
'blockchain': 0,
'body':
'.:en\nThe estimated cost of the laser array is based on extrapolation from the past two decades...', # noqa
'categories': [],
'comment_count': 37,
'created_date': '2012-10-14T11:43:11',
'data': None,
'editors': [],
'is_draft': False,
'languages': '["en"]',
'owner': 120,
'parents': [32],
'source': '',
'title': '.:en:Cost',
'type': 5,
'unsubscribed': [],
'updated_date': '2017-03-21T11:31:53.338'
},
'model': 'core.topic',
'pk': 1
}
tpl = template(data)
tpl['fields']['blockchain']['*'] = 'HELLO'
ndata = normalize(data, tpl, slugify=True)
answer = deepcopy(data)
del answer['fields']['blockchain']
answer['fields']['hello'] = 0
self.assertEqual(ndata, answer)
def test_hyphenation_slugify(self):
self.assertEqual(
normalize({'A': 1}, {'A': {
'*': 'a_b'
}}, slugify=True), {'a-b': 1})
def test_keys_renaming_slugify(self):
self.assertEqual(normalize({'A': 1}, {'A': {
'*': 'B'
}}, slugify=True), {'b': 1})
def test_hyphenation(self):
self.assertEqual(normalize({'A': 1}, {'A': {'*': 'a_b'}}), {'a_b': 1})
def test_keys_renaming(self):
self.assertEqual(normalize({'A': 1}, {'A': {'*': 'B'}}), {'B': 1})
def test_keys_pickout_alignment(self):
topics_schema = [{
'id': {
'*': 'topic-id'
},
'type': {
'*':
'|lambda x: {0: "NEED", 1: "GOAL", 2: "IDEA", 3: "PLAN", 4: "STEP", 5: "TASK"}.get(x)'
},
'owner': {
'username': {
'*': ''
},
'id': {
'*': 'user-id'
}
},
'blockchain': {
'*': '|lambda x: x and True or False'
},
}]
comments_schema = [{
'id': {
'*': 'comment-id'
},
'topic': {
'*': 'topic-url'
},
'text': {
'*': 'body'
},
'owner': {
'username': {
'*': ''
},
'id': {
'*': 'user-id'
}
},
'blockchain': {
'*': '|lambda x: x and True or False'
},
}]
normal_topics = normalize(self.topics, topics_schema)
normal_comments = normalize(self.comments, comments_schema)
abnormal_comments = [
dict(
comment, **{
"some": {
"place": {
"deep": comment["owner"]
}
},
"owner": None
}) for comment in normal_comments
]
with open('metaform/tests/data/topics+comments-pickout.json',
'r') as f:
expect = json.load(f)
self.assertEqual(
json.loads(
json.dumps(
list(align([normal_topics[:1], abnormal_comments[:1]])))),
expect)
def test_normalization(self):
data = [{
'address': {
'number': 14,
'street': 'Leonardo str.'
},
'children': [{
'age': 1,
'name': 'Mike'
}, {
'age': 15,
'name': 'Tom'
}],
'name':
'Max'
}, {
'address': {
'number': 1,
'street': 'Nexus str.'
},
'children': [{
'age': 1,
'name': 'Deli'
}, {
'age': 7,
'name': 'Miki'
}],
'name':
'Dim'
}]
schema = [{
'_version':
'domain.com/parents-0.1',
'*':
'https://github.com/infamily/_/wiki/address',
'address': {
'*': 'https://github.com/infamily/_/wiki/address',
'number': {
'*':
'https://www.wikidata.org/wiki/Q1413235|lambda _: int(_)'
},
'street': {
'*': 'https://www.wikidata.org/wiki/Q24574749'
}
},
'children': [{
'*': 'https://www.wikidata.org/wiki/Q7569',
'age': {
'*':
'https://www.wikidata.org/wiki/Q185836#time-years|lambda _: float(_)'
},
'name': {
'*': 'https://www.wikidata.org/wiki/Q82799'
}
}],
'name': {
'*': 'https://www.wikidata.org/wiki/Q82799'
}
}]
result = [{
'GH:infamily/_/address': {
'WD:Q1413235': 14,
'WD:Q24574749': 'Leonardo str.'
},
'WD:Q7569': [{
'WD:Q185836#time-years': 1.0,
'WD:Q82799': 'Mike'
}, {
'WD:Q185836#time-years': 15.0,
'WD:Q82799': 'Tom'
}],
'WD:Q82799':
'Max'
}, {
'GH:infamily/_/address': {
'WD:Q1413235': 1,
'WD:Q24574749': 'Nexus str.'
},
'WD:Q7569': [{
'WD:Q185836#time-years': 1.0,
'WD:Q82799': 'Deli'
}, {
'WD:Q185836#time-years': 7.0,
'WD:Q82799': 'Miki'
}],
'WD:Q82799':
'Dim'
}]
self.assertEqual(normalize(data, schema, namespace=True), result)
