def test_mfs_json(self): """MultiFieldSelector can work on JsonRecordList objects""" class Thing(JsonRecord): flintstone = JsonProperty() element = JsonProperty() class Things(JsonRecordList): itemtype = Thing flintstones = ("dino", "bammbamm", "wilma", "fred") elements = ("Rb", "At", "Pm", "Fl") data = list( {"flintstone": x[0], "element": x[1]} for x in zip(flintstones, elements) ) all_the_things = Things(data) mfs = MultiFieldSelector([None, "flintstone"]) self.assertEqual( mfs.get(all_the_things).json_data(), list(dict(flintstone=x) for x in flintstones), ) mfs = MultiFieldSelector([None, "flintstone"], [None, "element"]) self.assertEqual(mfs.get(all_the_things), all_the_things)
def test_mfs_json(self): """MultiFieldSelector can work on JsonRecordList objects""" class Thing(JsonRecord): flintstone = JsonProperty() element = JsonProperty() class Things(JsonRecordList): itemtype = Thing flintstones = ("dino", "bammbamm", "wilma", "fred") elements = ("Rb", "At", "Pm", "Fl") data = list({ "flintstone": x[0], "element": x[1] } for x in zip(flintstones, elements)) all_the_things = Things(data) mfs = MultiFieldSelector([None, "flintstone"]) self.assertEqual( mfs.get(all_the_things).json_data(), list(dict(flintstone=x) for x in flintstones), ) mfs = MultiFieldSelector([None, "flintstone"], [None, "element"]) self.assertEqual(mfs.get(all_the_things), all_the_things)
def test_filtered_coll_items_diff(self): strip_ids_mfs = MultiFieldSelector( ["name", "family"], ["date_of_birth"], ["friends", None, "name"], ["friends", None, "date_of_birth"], ) person = get_person(0, 2, 5, 6, 3) filtered_person = strip_ids_mfs.get(person) # not terribly useful! self.assertDifferences( person.diff_iter(filtered_person), { "REMOVED .ssn", "REMOVED .phone_number", "REMOVED .name.given", "REMOVED .friends[0]", "ADDED .friends[0]", "REMOVED .friends[1]", "ADDED .friends[1]", "REMOVED .friends[2]", "ADDED .friends[2]", "REMOVED .friends[3]", "ADDED .friends[3]", }, ) # however, pass the filter into diff, and it gets it right! self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs), {}, ) filtered_person.friends.append(get_person(1)) del filtered_person.friends[0] self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs), {"ADDED .friends[3]", "REMOVED .friends[0]"}, )
def test_filtered_coll_items_diff(self): strip_ids_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["date_of_birth"], ["friends", None, "given_name"], ["friends", None, "family_name"], ["friends", None, "date_of_birth"], ) person = get_person(0, 2, 5, 6, 3) filtered_person = strip_ids_mfs.get(person) # not terribly useful! self.assertDifferences( person.diff_iter(filtered_person), { "REMOVED .ssn", "REMOVED .phone_number", "REMOVED .friends[0]", "ADDED .friends[0]", "REMOVED .friends[1]", "ADDED .friends[1]", "REMOVED .friends[2]", "ADDED .friends[2]", "REMOVED .friends[3]", "ADDED .friends[3]", }, ) # however, pass the filter into diff, and it gets it right! self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs), {}, ) filtered_person.friends.append(get_person(1)) del filtered_person.friends.values[0] # FIXME :) self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs), {"ADDED .friends[3]", "REMOVED .friends[0]"}, )
def test_ignore_empty_and_coll(self): person = get_person(6, 0, 3, 4, 5) strip_ids_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["description"], ["friends", None, "given_name"], ["friends", None, "family_name"], ["friends", None, "description"], ) filtered_person = strip_ids_mfs.get(person) person.description = "" person.friends[0].description = "" self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs), { "REMOVED .description", "REMOVED .friends[0]", "ADDED .friends[0]", }, ) self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs, ignore_empty_slots=True), {}, )
def test_normalize_slot(self): person = get_person(3, 0, 2, 4, 6) strip_ids_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["phone_number"], ["friends", None, "given_name"], ["friends", None, "family_name"], ["friends", None, "phone_number"], ) filtered_person = strip_ids_mfs.get(person) class MyDiffOptions(DiffOptions): def normalize_slot(self, val, prop): if "phone" in prop.name and isinstance(val, basestring): val = normalize_phone(val) return super(MyDiffOptions, self).normalize_slot(val, prop) person.phone_number = '5309225668' person.friends[0].phone_number = '+1 239.978.5912' self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs, ignore_empty_slots=True), { "MODIFIED .phone_number", "REMOVED .friends[0]", "ADDED .friends[0]", }, ) my_options = MyDiffOptions( ignore_empty_slots=True, compare_filter=strip_ids_mfs, ) self.assertDifferences( person.diff_iter(filtered_person, options=my_options), {}, ) friendless = get_person(3) self.assertDifferences( person.diff_iter(friendless, options=my_options), {"REMOVED .friends"}, ) ignore_friends = MyDiffOptions(ignore_empty_slots=True, compare_filter=MultiFieldSelector( ["given_name"], ["family_name"], ["phone_number"], )) self.assertDifferences( person.diff_iter(friendless, options=ignore_friends), {}, )
def test_filtered_diff(self): """Test that diff notices when fields are removed""" name_mfs = MultiFieldSelector(["name", "given"], ["name", "family"]) person = get_person(1) filtered_person = name_mfs.get(person) self.assertDifferences( person.diff_iter(filtered_person), {"REMOVED .date_of_birth", "REMOVED .ssn", "REMOVED .phone_number"}, )
def test_filtered_diff(self): """Test that diff notices when fields are removed""" name_mfs = MultiFieldSelector(["given_name"], ["family_name"]) person = get_person(1) filtered_person = name_mfs.get(person) self.assertDifferences( person.diff_iter(filtered_person), {"REMOVED .date_of_birth", "REMOVED .ssn", "REMOVED .phone_number"}, )
def test_normalize_slot(self): person = get_person(3, 0, 2, 4, 6) strip_ids_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["phone_number"], ["friends", None, "given_name"], ["friends", None, "family_name"], ["friends", None, "phone_number"], ) filtered_person = strip_ids_mfs.get(person) class MyDiffOptions(DiffOptions): def normalize_slot(self, val, prop): if "phone" in prop.name and isinstance(val, basestring): val = normalize_phone(val) return super(MyDiffOptions, self).normalize_slot(val, prop) person.phone_number = '5309225668' person.friends[0].phone_number = '+1 239.978.5912' self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs, ignore_empty_slots=True), { "MODIFIED .phone_number", "REMOVED .friends[0]", "ADDED .friends[0]", }, ) my_options = MyDiffOptions( ignore_empty_slots=True, compare_filter=strip_ids_mfs, ) self.assertDifferences( person.diff_iter(filtered_person, options=my_options), {}, ) friendless = get_person(3) self.assertDifferences( person.diff_iter(friendless, options=my_options), {"REMOVED .friends"}, ) ignore_friends = MyDiffOptions( ignore_empty_slots=True, compare_filter=MultiFieldSelector( ["given_name"], ["family_name"], ["phone_number"], ) ) self.assertDifferences( person.diff_iter(friendless, options=ignore_friends), {}, )
def test_normalize_slot(self): person = get_person(3, 0, 2, 4, 6) strip_ids_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["phone_number"], ["friends", None, "given_name"], ["friends", None, "family_name"], ["friends", None, "phone_number"], ) filtered_person = strip_ids_mfs.get(person) # simplified NANP regex phone = re.compile( r"^(?:\+?1\s*(?:[.-]\s*)?)?(\d{3})\s*" r"(?:[.-]\s*)?(\d{3})\s*(?:[.-]\s*)?" r"(\d{4})" ) class MyDiffOptions(DiffOptions): def normalize_phone(self, phoney): m = re.match(phone, phoney) if m: return "(%s) %s-%s" % m.groups() else: return phoney def normalize_slot(self, val, prop): if "phone" in prop.name and isinstance(val, basestring): newval = self.normalize_phone(val) if val != newval: val = newval return super(MyDiffOptions, self).normalize_slot(val, prop) person.phone_number = '5309225668' person.friends[0].phone_number = '+1 239.978.5912' self.assertDifferences( person.diff_iter(filtered_person, compare_filter=strip_ids_mfs, ignore_empty_slots=True), { "MODIFIED .phone_number", "REMOVED .friends[0]", "ADDED .friends[0]", }, ) my_options = MyDiffOptions( ignore_empty_slots=True, compare_filter=strip_ids_mfs, ) self.assertDifferences( person.diff_iter(filtered_person, options=my_options), {}, )
def test_filtered_coll_diff(self): name_and_friends_mfs = MultiFieldSelector( ["name"], ["friends", 0], ["friends", 2], ) person = get_person(0, 2, 5, 6, 3) filtered_person = name_and_friends_mfs.get(person) self.assertDifferences( person.diff_iter(filtered_person), {"REMOVED .date_of_birth", "REMOVED .ssn", "REMOVED .phone_number", "REMOVED .friends[1]", "REMOVED .friends[3]"}, )
def test_filtered_coll_diff(self): name_and_friends_mfs = MultiFieldSelector( ["given_name"], ["family_name"], ["friends", 0], ["friends", 2], ) person = get_person(0, 2, 5, 6, 3) filtered_person = name_and_friends_mfs.get(person) self.assertDifferences( person.diff_iter(filtered_person), {"REMOVED .date_of_birth", "REMOVED .ssn", "REMOVED .phone_number", "REMOVED .friends[1]", "REMOVED .friends[3]"}, )
def test_mfs_apply_ops(self): from copy import deepcopy from testclasses import wall_one from normalize.diff import DiffTypes selectors = ( ("owner", ), ("posts", 0, "comments", 0, "poster"), ("posts", 0, "comments", 1, "content"), ) required_fields = ( ("id", ), ("posts", 0, "edited"), ("posts", 0, "post_id"), ("posts", 0, "wall_id"), ("posts", 0, "comments", 0, "edited"), ("posts", 0, "comments", 0, "id"), ("posts", 0, "comments", 1, "edited"), ("posts", 0, "comments", 1, "id"), ) deletable_mfs = MultiFieldSelector(*selectors) skeleton_mfs = MultiFieldSelector(*(required_fields + selectors)) scratch_wall = deepcopy(wall_one) saved_fields = skeleton_mfs.get(scratch_wall) deletable_mfs.delete(scratch_wall) removed = set( tuple(x.base) for x in wall_one.diff_iter(scratch_wall) if x.diff_type == DiffTypes.REMOVED) self.assertEqual( removed, set(selectors), "MultiFieldSelector.delete() can delete named attributes", ) deletable_mfs.patch(scratch_wall, saved_fields) self.assertFalse( scratch_wall.diff(wall_one), "MultiFieldSelector.patch() can copy named attributes", ) del saved_fields.owner deletable_mfs.patch(scratch_wall, saved_fields) self.assertFalse( hasattr(scratch_wall, "owner"), "MultiFieldSelector.patch() can delete missing attributes", )
def test_mfs_apply_ops(self): from copy import deepcopy from testclasses import wall_one from normalize.diff import DiffTypes selectors = ( ("owner",), ("posts", 0, "comments", 0, "poster"), ("posts", 0, "comments", 1, "content"), ) required_fields = ( ("id",), ("posts", 0, "edited"), ("posts", 0, "post_id"), ("posts", 0, "wall_id"), ("posts", 0, "comments", 0, "edited"), ("posts", 0, "comments", 0, "id"), ("posts", 0, "comments", 1, "edited"), ("posts", 0, "comments", 1, "id"), ) deletable_mfs = MultiFieldSelector(*selectors) skeleton_mfs = MultiFieldSelector(*(required_fields + selectors)) scratch_wall = deepcopy(wall_one) saved_fields = skeleton_mfs.get(scratch_wall) deletable_mfs.delete(scratch_wall) removed = set( tuple(x.base) for x in wall_one.diff_iter(scratch_wall) if x.diff_type == DiffTypes.REMOVED ) self.assertEqual( removed, set(selectors), "MultiFieldSelector.delete() can delete named attributes", ) deletable_mfs.patch(scratch_wall, saved_fields) self.assertFalse( scratch_wall.diff(wall_one), "MultiFieldSelector.patch() can copy named attributes", ) del saved_fields.owner deletable_mfs.patch(scratch_wall, saved_fields) self.assertFalse( hasattr(scratch_wall, "owner"), "MultiFieldSelector.patch() can delete missing attributes", )
def test_multi_selector(self): selectors = set(( ("bar", ), ("foo", "bar", 0, "boo"), ("foo", "bar", 0, "hiss"), ("foo", "bar", 1), )) mfs = MultiFieldSelector(*selectors) emitted = set(tuple(x.selectors) for x in mfs) self.assertEqual(emitted, selectors) # match, eg <MultiFieldSelector: (.foo.bar([0](.hiss|.boo)|[1])|.bar)> # but also <MultiFieldSelector: (.bar|.foo.bar([1]|[0](.boo|.hiss)))> regexp = re.compile( r"""<MultiFieldSelector:\s+\( (?: (?: .foo.bar \( (?: (?: \[0\] \( (?: (?: .hiss | .boo ) \|? ){2} \) | \[1\] ) \|? ){2} \) | .bar ) \|? ){2} \)>""", re.X, ) self.assertRegexpMatches(str(mfs), regexp) mfs_dupe = eval(repr(mfs)) emitted = set(tuple(x.selectors) for x in mfs_dupe) self.assertEqual(emitted, selectors) # test various dict-like functions self.assertIn("foo", mfs) self.assertIn("bar", mfs) self.assertNotIn("baz", mfs) self.assertIn('bar', mfs['foo']) self.assertIn(0, mfs['foo']['bar']) self.assertIn('hiss', mfs['foo']['bar'][0]) self.assertNotIn('miss', mfs['foo']['bar'][0]) self.assertIn('baz', mfs['bar']) self.assertIn('baz', mfs['bar']['frop']['quux']['fred']) # if you add a higher level selector, then more specific paths # disappear from the MFS mfs2 = MultiFieldSelector(mfs, ["foo", "bar"]) emitted = set(tuple(x.selectors) for x in mfs2) self.assertEqual(emitted, set((("bar", ), ("foo", "bar")))) data = { "bar": [1, 2, 3], "foo": { "bar": [ { "boo": "waa", "frop": "quux" }, { "waldo": "grault" }, { "fubar": "corge" }, ], }, } selected = mfs.get(data) self.assertEqual( selected, { "bar": [1, 2, 3], "foo": { "bar": [ { "boo": "waa" }, { "waldo": "grault" }, ], }, }) class Octothorpe(Record): name = Property() boo = Property() hiss = Property() class Caret(Record): bar = ListProperty(of=Octothorpe) class Pilcrow(Record): bar = ListProperty(of=Octothorpe) foo = Property(isa=Caret) baz = Property() quux = DictProperty(of=str) frop = DictProperty(of=list_of(unicode)) full = Pilcrow( bar=[dict(name="Heffalump"), dict(name="Uncle Robert")], foo=dict( bar=[dict(name="Owl", hiss="Hunny Bee"), dict(name="Piglet")]), baz="Wizzle", quux={ "protagonist": "Winnie_the_Pooh", "antagonist": "Alexander_Beetle" }, frop={ "lighting": ["Uncle_Robert", "Kanga", "Small"], "story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"] }, ) selectors.add(("quux", "protagonist")) self.assertEqual( FieldSelector(("quux", "protagonist")).get(full), "Winnie_the_Pooh", ) selectors.add(("frop", "story")) mfs = MultiFieldSelector(*selectors) filtered = mfs.get(full) expected = Pilcrow( bar=[dict(name="Heffalump"), dict(name="Uncle Robert")], foo=dict(bar=[dict( hiss="Hunny Bee"), dict(name="Piglet")]), quux={"protagonist": "Winnie_the_Pooh"}, frop={"story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"]}, ) self.assertEqual(filtered, expected)
def test_multi_selector(self): selectors = set( ( ("bar", ), ("foo", "bar", 0, "boo"), ("foo", "bar", 0, "hiss"), ("foo", "bar", 1), ) ) mfs = MultiFieldSelector(*selectors) emitted = set(tuple(x.selectors) for x in mfs) self.assertEqual(emitted, selectors) # match, eg <MultiFieldSelector: (.foo.bar([0](.hiss|.boo)|[1])|.bar)> # but also <MultiFieldSelector: (.bar|.foo.bar([1]|[0](.boo|.hiss)))> regexp = re.compile( r"""<MultiFieldSelector:\s+\( (?: (?: .foo.bar \( (?: (?: \[0\] \( (?: (?: .hiss | .boo ) \|? ){2} \) | \[1\] ) \|? ){2} \) | .bar ) \|? ){2} \)>""", re.X, ) self.assertRegexpMatches(str(mfs), regexp) mfs_dupe = eval(repr(mfs)) emitted = set(tuple(x.selectors) for x in mfs_dupe) self.assertEqual(emitted, selectors) # test various dict-like functions self.assertIn("foo", mfs) self.assertIn("bar", mfs) self.assertNotIn("baz", mfs) self.assertIn('bar', mfs['foo']) self.assertIn(0, mfs['foo']['bar']) self.assertIn('hiss', mfs['foo']['bar'][0]) self.assertNotIn('miss', mfs['foo']['bar'][0]) self.assertIn('baz', mfs['bar']) self.assertIn('baz', mfs['bar']['frop']['quux']['fred']) # if you add a higher level selector, then more specific paths # disappear from the MFS mfs2 = MultiFieldSelector(mfs, ["foo", "bar"]) emitted = set(tuple(x.selectors) for x in mfs2) self.assertEqual(emitted, set((("bar",), ("foo", "bar")))) data = { "bar": [1, 2, 3], "foo": { "bar": [ {"boo": "waa", "frop": "quux"}, {"waldo": "grault"}, {"fubar": "corge"}, ], }, } selected = mfs.get(data) self.assertEqual( selected, { "bar": [1, 2, 3], "foo": { "bar": [ {"boo": "waa"}, {"waldo": "grault"}, ], }, } ) class Octothorpe(Record): name = Property() boo = Property() hiss = Property() class Caret(Record): bar = ListProperty(of=Octothorpe) class Pilcrow(Record): bar = ListProperty(of=Octothorpe) foo = Property(isa=Caret) baz = Property() full = Pilcrow( bar=[dict(name="Heffalump"), dict(name="Uncle Robert")], foo=dict(bar=[dict(name="Owl", hiss="Hunny Bee"), dict(name="Piglet")]), baz="Wizzle", ) filtered = mfs.get(full) expected = Pilcrow( bar=[dict(name="Heffalump"), dict(name="Uncle Robert")], foo=dict(bar=[dict(hiss="Hunny Bee"), dict(name="Piglet")]), ) self.assertEqual(filtered, expected)