def test_mfs_json(self):
"""MultiFieldSelector can work on JsonRecordList objects"""
class Thing(JsonRecord):
flintstone = JsonProperty()
element = JsonProperty()
class Things(JsonRecordList):
itemtype = Thing
flintstones = ("dino", "bammbamm", "wilma", "fred")
elements = ("Rb", "At", "Pm", "Fl")
data = list(
{"flintstone": x[0], "element": x[1]} for x in
zip(flintstones, elements)
)
all_the_things = Things(data)
mfs = MultiFieldSelector([None, "flintstone"])
self.assertEqual(
mfs.get(all_the_things).json_data(),
list(dict(flintstone=x) for x in flintstones),
)
mfs = MultiFieldSelector([None, "flintstone"], [None, "element"])
self.assertEqual(mfs.get(all_the_things), all_the_things)
def test_mfs_json(self):
"""MultiFieldSelector can work on JsonRecordList objects"""
class Thing(JsonRecord):
flintstone = JsonProperty()
element = JsonProperty()
class Things(JsonRecordList):
itemtype = Thing
flintstones = ("dino", "bammbamm", "wilma", "fred")
elements = ("Rb", "At", "Pm", "Fl")
data = list({
"flintstone": x[0],
"element": x[1]
} for x in zip(flintstones, elements))
all_the_things = Things(data)
mfs = MultiFieldSelector([None, "flintstone"])
self.assertEqual(
mfs.get(all_the_things).json_data(),
list(dict(flintstone=x) for x in flintstones),
)
mfs = MultiFieldSelector([None, "flintstone"], [None, "element"])
self.assertEqual(mfs.get(all_the_things), all_the_things)
def test_filtered_coll_items_diff(self):
strip_ids_mfs = MultiFieldSelector(
["name", "family"], ["date_of_birth"],
["friends", None, "name"],
["friends", None, "date_of_birth"],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = strip_ids_mfs.get(person)
# not terribly useful!
self.assertDifferences(
person.diff_iter(filtered_person), {
"REMOVED .ssn", "REMOVED .phone_number",
"REMOVED .name.given",
"REMOVED .friends[0]", "ADDED .friends[0]",
"REMOVED .friends[1]", "ADDED .friends[1]",
"REMOVED .friends[2]", "ADDED .friends[2]",
"REMOVED .friends[3]", "ADDED .friends[3]",
},
)
# however, pass the filter into diff, and it gets it right!
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs), {},
)
filtered_person.friends.append(get_person(1))
del filtered_person.friends[0]
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs),
{"ADDED .friends[3]", "REMOVED .friends[0]"},
)
def test_filtered_coll_items_diff(self):
strip_ids_mfs = MultiFieldSelector(
["given_name"], ["family_name"], ["date_of_birth"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "date_of_birth"],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = strip_ids_mfs.get(person)
# not terribly useful!
self.assertDifferences(
person.diff_iter(filtered_person), {
"REMOVED .ssn", "REMOVED .phone_number",
"REMOVED .friends[0]", "ADDED .friends[0]",
"REMOVED .friends[1]", "ADDED .friends[1]",
"REMOVED .friends[2]", "ADDED .friends[2]",
"REMOVED .friends[3]", "ADDED .friends[3]",
},
)
# however, pass the filter into diff, and it gets it right!
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs), {},
)
filtered_person.friends.append(get_person(1))
del filtered_person.friends.values[0] # FIXME :)
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs),
{"ADDED .friends[3]", "REMOVED .friends[0]"},
)
def test_ignore_empty_and_coll(self):
person = get_person(6, 0, 3, 4, 5)
strip_ids_mfs = MultiFieldSelector(
["given_name"],
["family_name"],
["description"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "description"],
)
filtered_person = strip_ids_mfs.get(person)
person.description = ""
person.friends[0].description = ""
self.assertDifferences(
person.diff_iter(filtered_person, compare_filter=strip_ids_mfs),
{
"REMOVED .description",
"REMOVED .friends[0]",
"ADDED .friends[0]",
},
)
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{},
)
def test_ignore_empty_and_coll(self):
person = get_person(6, 0, 3, 4, 5)
strip_ids_mfs = MultiFieldSelector(
["given_name"], ["family_name"], ["description"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "description"],
)
filtered_person = strip_ids_mfs.get(person)
person.description = ""
person.friends[0].description = ""
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs),
{
"REMOVED .description",
"REMOVED .friends[0]", "ADDED .friends[0]",
},
)
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True), {},
)
def test_normalize_slot(self):
person = get_person(3, 0, 2, 4, 6)
strip_ids_mfs = MultiFieldSelector(
["given_name"],
["family_name"],
["phone_number"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "phone_number"],
)
filtered_person = strip_ids_mfs.get(person)
class MyDiffOptions(DiffOptions):
def normalize_slot(self, val, prop):
if "phone" in prop.name and isinstance(val, basestring):
val = normalize_phone(val)
return super(MyDiffOptions, self).normalize_slot(val, prop)
person.phone_number = '5309225668'
person.friends[0].phone_number = '+1 239.978.5912'
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{
"MODIFIED .phone_number",
"REMOVED .friends[0]",
"ADDED .friends[0]",
},
)
my_options = MyDiffOptions(
ignore_empty_slots=True,
compare_filter=strip_ids_mfs,
)
self.assertDifferences(
person.diff_iter(filtered_person, options=my_options),
{},
)
friendless = get_person(3)
self.assertDifferences(
person.diff_iter(friendless, options=my_options),
{"REMOVED .friends"},
)
ignore_friends = MyDiffOptions(ignore_empty_slots=True,
compare_filter=MultiFieldSelector(
["given_name"],
["family_name"],
["phone_number"],
))
self.assertDifferences(
person.diff_iter(friendless, options=ignore_friends),
{},
)
def test_filtered_diff(self):
"""Test that diff notices when fields are removed"""
name_mfs = MultiFieldSelector(["name", "given"], ["name", "family"])
person = get_person(1)
filtered_person = name_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number"},
)
def test_filtered_diff(self):
"""Test that diff notices when fields are removed"""
name_mfs = MultiFieldSelector(["given_name"], ["family_name"])
person = get_person(1)
filtered_person = name_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number"},
)
def test_normalize_slot(self):
person = get_person(3, 0, 2, 4, 6)
strip_ids_mfs = MultiFieldSelector(
["given_name"], ["family_name"], ["phone_number"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "phone_number"],
)
filtered_person = strip_ids_mfs.get(person)
class MyDiffOptions(DiffOptions):
def normalize_slot(self, val, prop):
if "phone" in prop.name and isinstance(val, basestring):
val = normalize_phone(val)
return super(MyDiffOptions, self).normalize_slot(val, prop)
person.phone_number = '5309225668'
person.friends[0].phone_number = '+1 239.978.5912'
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{
"MODIFIED .phone_number",
"REMOVED .friends[0]", "ADDED .friends[0]",
},
)
my_options = MyDiffOptions(
ignore_empty_slots=True,
compare_filter=strip_ids_mfs,
)
self.assertDifferences(
person.diff_iter(filtered_person, options=my_options), {},
)
friendless = get_person(3)
self.assertDifferences(
person.diff_iter(friendless, options=my_options),
{"REMOVED .friends"},
)
ignore_friends = MyDiffOptions(
ignore_empty_slots=True,
compare_filter=MultiFieldSelector(
["given_name"], ["family_name"], ["phone_number"],
)
)
self.assertDifferences(
person.diff_iter(friendless, options=ignore_friends), {},
)
def test_normalize_slot(self):
person = get_person(3, 0, 2, 4, 6)
strip_ids_mfs = MultiFieldSelector(
["given_name"], ["family_name"], ["phone_number"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "phone_number"],
)
filtered_person = strip_ids_mfs.get(person)
# simplified NANP regex
phone = re.compile(
r"^(?:\+?1\s*(?:[.-]\s*)?)?(\d{3})\s*"
r"(?:[.-]\s*)?(\d{3})\s*(?:[.-]\s*)?"
r"(\d{4})"
)
class MyDiffOptions(DiffOptions):
def normalize_phone(self, phoney):
m = re.match(phone, phoney)
if m:
return "(%s) %s-%s" % m.groups()
else:
return phoney
def normalize_slot(self, val, prop):
if "phone" in prop.name and isinstance(val, basestring):
newval = self.normalize_phone(val)
if val != newval:
val = newval
return super(MyDiffOptions, self).normalize_slot(val, prop)
person.phone_number = '5309225668'
person.friends[0].phone_number = '+1 239.978.5912'
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{
"MODIFIED .phone_number",
"REMOVED .friends[0]", "ADDED .friends[0]",
},
)
my_options = MyDiffOptions(
ignore_empty_slots=True,
compare_filter=strip_ids_mfs,
)
self.assertDifferences(
person.diff_iter(filtered_person, options=my_options), {},
)
def test_filtered_coll_diff(self):
name_and_friends_mfs = MultiFieldSelector(
["name"],
["friends", 0],
["friends", 2],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = name_and_friends_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number",
"REMOVED .friends[1]", "REMOVED .friends[3]"},
)
def test_filtered_coll_diff(self):
name_and_friends_mfs = MultiFieldSelector(
["given_name"], ["family_name"],
["friends", 0],
["friends", 2],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = name_and_friends_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number",
"REMOVED .friends[1]", "REMOVED .friends[3]"},
)
def test_mfs_apply_ops(self):
from copy import deepcopy
from testclasses import wall_one
from normalize.diff import DiffTypes
selectors = (
("owner", ),
("posts", 0, "comments", 0, "poster"),
("posts", 0, "comments", 1, "content"),
)
required_fields = (
("id", ),
("posts", 0, "edited"),
("posts", 0, "post_id"),
("posts", 0, "wall_id"),
("posts", 0, "comments", 0, "edited"),
("posts", 0, "comments", 0, "id"),
("posts", 0, "comments", 1, "edited"),
("posts", 0, "comments", 1, "id"),
)
deletable_mfs = MultiFieldSelector(*selectors)
skeleton_mfs = MultiFieldSelector(*(required_fields + selectors))
scratch_wall = deepcopy(wall_one)
saved_fields = skeleton_mfs.get(scratch_wall)
deletable_mfs.delete(scratch_wall)
removed = set(
tuple(x.base) for x in wall_one.diff_iter(scratch_wall)
if x.diff_type == DiffTypes.REMOVED)
self.assertEqual(
removed,
set(selectors),
"MultiFieldSelector.delete() can delete named attributes",
)
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
scratch_wall.diff(wall_one),
"MultiFieldSelector.patch() can copy named attributes",
)
del saved_fields.owner
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
hasattr(scratch_wall, "owner"),
"MultiFieldSelector.patch() can delete missing attributes",
)
def test_mfs_apply_ops(self):
from copy import deepcopy
from testclasses import wall_one
from normalize.diff import DiffTypes
selectors = (
("owner",),
("posts", 0, "comments", 0, "poster"),
("posts", 0, "comments", 1, "content"),
)
required_fields = (
("id",),
("posts", 0, "edited"),
("posts", 0, "post_id"),
("posts", 0, "wall_id"),
("posts", 0, "comments", 0, "edited"),
("posts", 0, "comments", 0, "id"),
("posts", 0, "comments", 1, "edited"),
("posts", 0, "comments", 1, "id"),
)
deletable_mfs = MultiFieldSelector(*selectors)
skeleton_mfs = MultiFieldSelector(*(required_fields + selectors))
scratch_wall = deepcopy(wall_one)
saved_fields = skeleton_mfs.get(scratch_wall)
deletable_mfs.delete(scratch_wall)
removed = set(
tuple(x.base) for x in wall_one.diff_iter(scratch_wall)
if x.diff_type == DiffTypes.REMOVED
)
self.assertEqual(
removed, set(selectors),
"MultiFieldSelector.delete() can delete named attributes",
)
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
scratch_wall.diff(wall_one),
"MultiFieldSelector.patch() can copy named attributes",
)
del saved_fields.owner
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
hasattr(scratch_wall, "owner"),
"MultiFieldSelector.patch() can delete missing attributes",
)
def test_multi_selector(self):
selectors = set((
("bar", ),
("foo", "bar", 0, "boo"),
("foo", "bar", 0, "hiss"),
("foo", "bar", 1),
))
mfs = MultiFieldSelector(*selectors)
emitted = set(tuple(x.selectors) for x in mfs)
self.assertEqual(emitted, selectors)
# match, eg <MultiFieldSelector: (.foo.bar([0](.hiss|.boo)|[1])|.bar)>
# but also <MultiFieldSelector: (.bar|.foo.bar([1]|[0](.boo|.hiss)))>
regexp = re.compile(
r"""<MultiFieldSelector:\s+\(
(?:
(?: .foo.bar \(
(?:
(?: \[0\] \(
(?:
(?: .hiss | .boo ) \|?
){2} \)
| \[1\] ) \|?
){2} \)
| .bar ) \|?
){2}
\)>""",
re.X,
)
self.assertRegexpMatches(str(mfs), regexp)
mfs_dupe = eval(repr(mfs))
emitted = set(tuple(x.selectors) for x in mfs_dupe)
self.assertEqual(emitted, selectors)
# test various dict-like functions
self.assertIn("foo", mfs)
self.assertIn("bar", mfs)
self.assertNotIn("baz", mfs)
self.assertIn('bar', mfs['foo'])
self.assertIn(0, mfs['foo']['bar'])
self.assertIn('hiss', mfs['foo']['bar'][0])
self.assertNotIn('miss', mfs['foo']['bar'][0])
self.assertIn('baz', mfs['bar'])
self.assertIn('baz', mfs['bar']['frop']['quux']['fred'])
# if you add a higher level selector, then more specific paths
# disappear from the MFS
mfs2 = MultiFieldSelector(mfs, ["foo", "bar"])
emitted = set(tuple(x.selectors) for x in mfs2)
self.assertEqual(emitted, set((("bar", ), ("foo", "bar"))))
data = {
"bar": [1, 2, 3],
"foo": {
"bar": [
{
"boo": "waa",
"frop": "quux"
},
{
"waldo": "grault"
},
{
"fubar": "corge"
},
],
},
}
selected = mfs.get(data)
self.assertEqual(
selected, {
"bar": [1, 2, 3],
"foo": {
"bar": [
{
"boo": "waa"
},
{
"waldo": "grault"
},
],
},
})
class Octothorpe(Record):
name = Property()
boo = Property()
hiss = Property()
class Caret(Record):
bar = ListProperty(of=Octothorpe)
class Pilcrow(Record):
bar = ListProperty(of=Octothorpe)
foo = Property(isa=Caret)
baz = Property()
quux = DictProperty(of=str)
frop = DictProperty(of=list_of(unicode))
full = Pilcrow(
bar=[dict(name="Heffalump"),
dict(name="Uncle Robert")],
foo=dict(
bar=[dict(name="Owl", hiss="Hunny Bee"),
dict(name="Piglet")]),
baz="Wizzle",
quux={
"protagonist": "Winnie_the_Pooh",
"antagonist": "Alexander_Beetle"
},
frop={
"lighting": ["Uncle_Robert", "Kanga", "Small"],
"story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"]
},
)
selectors.add(("quux", "protagonist"))
self.assertEqual(
FieldSelector(("quux", "protagonist")).get(full),
"Winnie_the_Pooh",
)
selectors.add(("frop", "story"))
mfs = MultiFieldSelector(*selectors)
filtered = mfs.get(full)
expected = Pilcrow(
bar=[dict(name="Heffalump"),
dict(name="Uncle Robert")],
foo=dict(bar=[dict(
hiss="Hunny Bee"), dict(name="Piglet")]),
quux={"protagonist": "Winnie_the_Pooh"},
frop={"story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"]},
)
self.assertEqual(filtered, expected)
def test_multi_selector(self):
selectors = set(
(
("bar", ),
("foo", "bar", 0, "boo"),
("foo", "bar", 0, "hiss"),
("foo", "bar", 1),
)
)
mfs = MultiFieldSelector(*selectors)
emitted = set(tuple(x.selectors) for x in mfs)
self.assertEqual(emitted, selectors)
# match, eg <MultiFieldSelector: (.foo.bar([0](.hiss|.boo)|[1])|.bar)>
# but also <MultiFieldSelector: (.bar|.foo.bar([1]|[0](.boo|.hiss)))>
regexp = re.compile(
r"""<MultiFieldSelector:\s+\(
(?:
(?: .foo.bar \(
(?:
(?: \[0\] \(
(?:
(?: .hiss | .boo ) \|?
){2} \)
| \[1\] ) \|?
){2} \)
| .bar ) \|?
){2}
\)>""", re.X,
)
self.assertRegexpMatches(str(mfs), regexp)
mfs_dupe = eval(repr(mfs))
emitted = set(tuple(x.selectors) for x in mfs_dupe)
self.assertEqual(emitted, selectors)
# test various dict-like functions
self.assertIn("foo", mfs)
self.assertIn("bar", mfs)
self.assertNotIn("baz", mfs)
self.assertIn('bar', mfs['foo'])
self.assertIn(0, mfs['foo']['bar'])
self.assertIn('hiss', mfs['foo']['bar'][0])
self.assertNotIn('miss', mfs['foo']['bar'][0])
self.assertIn('baz', mfs['bar'])
self.assertIn('baz', mfs['bar']['frop']['quux']['fred'])
# if you add a higher level selector, then more specific paths
# disappear from the MFS
mfs2 = MultiFieldSelector(mfs, ["foo", "bar"])
emitted = set(tuple(x.selectors) for x in mfs2)
self.assertEqual(emitted, set((("bar",), ("foo", "bar"))))
data = {
"bar": [1, 2, 3],
"foo": {
"bar": [
{"boo": "waa", "frop": "quux"},
{"waldo": "grault"},
{"fubar": "corge"},
],
},
}
selected = mfs.get(data)
self.assertEqual(
selected, {
"bar": [1, 2, 3],
"foo": {
"bar": [
{"boo": "waa"},
{"waldo": "grault"},
],
},
}
)
class Octothorpe(Record):
name = Property()
boo = Property()
hiss = Property()
class Caret(Record):
bar = ListProperty(of=Octothorpe)
class Pilcrow(Record):
bar = ListProperty(of=Octothorpe)
foo = Property(isa=Caret)
baz = Property()
full = Pilcrow(
bar=[dict(name="Heffalump"), dict(name="Uncle Robert")],
foo=dict(bar=[dict(name="Owl", hiss="Hunny Bee"),
dict(name="Piglet")]),
baz="Wizzle",
)
filtered = mfs.get(full)
expected = Pilcrow(
bar=[dict(name="Heffalump"), dict(name="Uncle Robert")],
foo=dict(bar=[dict(hiss="Hunny Bee"),
dict(name="Piglet")]),
)
self.assertEqual(filtered, expected)
