def test_merging_topics_and_comments(self): topics_schema = [{ 'id': { '*': 'topic-id' }, 'type': { '*': '|lambda x: {0: "NEED", 1: "GOAL", 2: "IDEA", 3: "PLAN", 4: "STEP", 5: "TASK"}.get(x)' }, 'owner': { 'username': { '*': '' }, 'id': { '*': 'user-id' } }, 'blockchain': { '*': '|lambda x: x and True or False' }, }] comments_schema = [{ 'id': { '*': 'comment-id' }, 'topic': { '*': 'topic-url' }, 'text': { '*': 'body' }, 'owner': { 'username': { '*': '' }, 'id': { '*': 'user-id' } }, 'blockchain': { '*': '|lambda x: x and True or False' }, }] normal_topics = normalize(self.topics, topics_schema) normal_comments = normalize(self.comments, comments_schema) with open('metaform/tests/data/topics+comments.json', 'r') as f: expect = json.load(f) self.assertEqual(normal_topics + normal_comments, expect)
def test_apply_lambdas(self): schema = { '*': 'greeting', 'hello': { '*': 'length|lambda x: x+5.' }, 'world': { '*': 'atoms|lambda x: str(x)+"ABC"' }, 'how': [{ '*': 'method', 'are': { '*': 'yup', 'you': { '*': 'me|lambda x: "-".join(list(x))' } } }] } expect = { 'atoms': '2ABC', 'length': 6.0, 'method': ['is', { 'yup': { 'me': 'd-o-i-n-g' } }] } self.assertEqual(normalize(self.data, schema), expect)
def test_rename_keys(self): schema = { '*': 'greeting', 'hello': { '*': 'length' }, 'world': { '*': 'atoms' }, 'how': [{ '*': 'method', 'are': { '*': 'yup', 'you': { '*': 'me' } } }] } expect = { 'atoms': 2, 'length': 1.0, 'method': ['is', { 'yup': { 'me': 'doing' } }] } self.assertEqual(normalize(self.data, schema), expect)
def readin(cls, location, schema=None, limit=None): from tqdm import tqdm #noqa if os.path.exists(location): if os.path.isdir(location): n = 0 for fname in tqdm(os.listdir(location)): record = yaml.load(open(os.path.join(location, fname)).read(), Loader=loader) if schema is not None: import metaform #noqa yield metaform.formatize( metaform.normalize(record, schema)) else: yield record if limit is not None: n += 1 if n >= limit: break elif os.path.isfile(location): yield yaml.load(open(location).read(), Loader=loader)
def test_custom_converters(self): def some_func(x): a = 123 b = 345 return (b - a) * x converters.func = some_func schema = { '*': 'greeting', 'hello': { '*': 'length|converters.func' }, 'world': { '*': 'atoms|lambda x: str(x)+"ABC"' }, 'how': [{ '*': 'method', 'are': { '*': 'yup', 'you': { '*': 'me|lambda x: "-".join(list(x))' } } }] } expect = { 'atoms': '2ABC', 'length': 222.0, 'method': ['is', { 'yup': { 'me': 'd-o-i-n-g' } }] } self.assertEqual(normalize(self.data, schema), expect)
def test_make_and_apply_template(self): data = {'a': [{'b': 'c'}, {'e': 'f'}, {'g': 'h'}], 'b': 'something'} tpl = template(data) answer = { '*': '', 'a': [{ '*': '', 'b': { '*': '' }, 'e': { '*': '' }, 'g': { '*': '' } }], 'b': { '*': '' } } self.assertEqual(tpl, answer) # apply: tpl['a'][0]['b']['*'] = 'hello' result = normalize(data, tpl) answer = deepcopy(data) del answer['a'][0]['b'] answer['a'][0]['hello'] = 'c' self.assertEqual(result, answer)
def test_make_and_apply_complex_template(self): data = { 'fields': { 'blockchain': 0, 'body': '.:en\nThe estimated cost of the laser array is based on extrapolation from the past two decades...', # noqa 'categories': [], 'comment_count': 37, 'created_date': '2012-10-14T11:43:11', 'data': None, 'editors': [], 'is_draft': False, 'languages': '["en"]', 'owner': 120, 'parents': [32], 'source': '', 'title': '.:en:Cost', 'type': 5, 'unsubscribed': [], 'updated_date': '2017-03-21T11:31:53.338' }, 'model': 'core.topic', 'pk': 1 } tpl = template(data) tpl['fields']['blockchain']['*'] = 'HELLO' ndata = normalize(data, tpl, slugify=True) answer = deepcopy(data) del answer['fields']['blockchain'] answer['fields']['hello'] = 0 self.assertEqual(ndata, answer)
def test_hyphenation_slugify(self): self.assertEqual( normalize({'A': 1}, {'A': { '*': 'a_b' }}, slugify=True), {'a-b': 1})
def test_keys_renaming_slugify(self): self.assertEqual(normalize({'A': 1}, {'A': { '*': 'B' }}, slugify=True), {'b': 1})
def test_hyphenation(self): self.assertEqual(normalize({'A': 1}, {'A': {'*': 'a_b'}}), {'a_b': 1})
def test_keys_renaming(self): self.assertEqual(normalize({'A': 1}, {'A': {'*': 'B'}}), {'B': 1})
def test_keys_pickout_alignment(self): topics_schema = [{ 'id': { '*': 'topic-id' }, 'type': { '*': '|lambda x: {0: "NEED", 1: "GOAL", 2: "IDEA", 3: "PLAN", 4: "STEP", 5: "TASK"}.get(x)' }, 'owner': { 'username': { '*': '' }, 'id': { '*': 'user-id' } }, 'blockchain': { '*': '|lambda x: x and True or False' }, }] comments_schema = [{ 'id': { '*': 'comment-id' }, 'topic': { '*': 'topic-url' }, 'text': { '*': 'body' }, 'owner': { 'username': { '*': '' }, 'id': { '*': 'user-id' } }, 'blockchain': { '*': '|lambda x: x and True or False' }, }] normal_topics = normalize(self.topics, topics_schema) normal_comments = normalize(self.comments, comments_schema) abnormal_comments = [ dict( comment, **{ "some": { "place": { "deep": comment["owner"] } }, "owner": None }) for comment in normal_comments ] with open('metaform/tests/data/topics+comments-pickout.json', 'r') as f: expect = json.load(f) self.assertEqual( json.loads( json.dumps( list(align([normal_topics[:1], abnormal_comments[:1]])))), expect)
def test_normalization(self): data = [{ 'address': { 'number': 14, 'street': 'Leonardo str.' }, 'children': [{ 'age': 1, 'name': 'Mike' }, { 'age': 15, 'name': 'Tom' }], 'name': 'Max' }, { 'address': { 'number': 1, 'street': 'Nexus str.' }, 'children': [{ 'age': 1, 'name': 'Deli' }, { 'age': 7, 'name': 'Miki' }], 'name': 'Dim' }] schema = [{ '_version': 'domain.com/parents-0.1', '*': 'https://github.com/infamily/_/wiki/address', 'address': { '*': 'https://github.com/infamily/_/wiki/address', 'number': { '*': 'https://www.wikidata.org/wiki/Q1413235|lambda _: int(_)' }, 'street': { '*': 'https://www.wikidata.org/wiki/Q24574749' } }, 'children': [{ '*': 'https://www.wikidata.org/wiki/Q7569', 'age': { '*': 'https://www.wikidata.org/wiki/Q185836#time-years|lambda _: float(_)' }, 'name': { '*': 'https://www.wikidata.org/wiki/Q82799' } }], 'name': { '*': 'https://www.wikidata.org/wiki/Q82799' } }] result = [{ 'GH:infamily/_/address': { 'WD:Q1413235': 14, 'WD:Q24574749': 'Leonardo str.' }, 'WD:Q7569': [{ 'WD:Q185836#time-years': 1.0, 'WD:Q82799': 'Mike' }, { 'WD:Q185836#time-years': 15.0, 'WD:Q82799': 'Tom' }], 'WD:Q82799': 'Max' }, { 'GH:infamily/_/address': { 'WD:Q1413235': 1, 'WD:Q24574749': 'Nexus str.' }, 'WD:Q7569': [{ 'WD:Q185836#time-years': 1.0, 'WD:Q82799': 'Deli' }, { 'WD:Q185836#time-years': 7.0, 'WD:Q82799': 'Miki' }], 'WD:Q82799': 'Dim' }] self.assertEqual(normalize(data, schema, namespace=True), result)