def test_mfs_json(self):
"""MultiFieldSelector can work on JsonRecordList objects"""
class Thing(JsonRecord):
flintstone = JsonProperty()
element = JsonProperty()
class Things(JsonRecordList):
itemtype = Thing
flintstones = ("dino", "bammbamm", "wilma", "fred")
elements = ("Rb", "At", "Pm", "Fl")
data = list({
"flintstone": x[0],
"element": x[1]
} for x in zip(flintstones, elements))
all_the_things = Things(data)
mfs = MultiFieldSelector([None, "flintstone"])
self.assertEqual(
mfs.get(all_the_things).json_data(),
list(dict(flintstone=x) for x in flintstones),
)
mfs = MultiFieldSelector([None, "flintstone"], [None, "element"])
self.assertEqual(mfs.get(all_the_things), all_the_things)
def test_normalize_slot(self):
person = get_person(3, 0, 2, 4, 6)
strip_ids_mfs = MultiFieldSelector(
["given_name"],
["family_name"],
["phone_number"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "phone_number"],
)
filtered_person = strip_ids_mfs.get(person)
class MyDiffOptions(DiffOptions):
def normalize_slot(self, val, prop):
if "phone" in prop.name and isinstance(val, basestring):
val = normalize_phone(val)
return super(MyDiffOptions, self).normalize_slot(val, prop)
person.phone_number = '5309225668'
person.friends[0].phone_number = '+1 239.978.5912'
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{
"MODIFIED .phone_number",
"REMOVED .friends[0]",
"ADDED .friends[0]",
},
)
my_options = MyDiffOptions(
ignore_empty_slots=True,
compare_filter=strip_ids_mfs,
)
self.assertDifferences(
person.diff_iter(filtered_person, options=my_options),
{},
)
friendless = get_person(3)
self.assertDifferences(
person.diff_iter(friendless, options=my_options),
{"REMOVED .friends"},
)
ignore_friends = MyDiffOptions(ignore_empty_slots=True,
compare_filter=MultiFieldSelector(
["given_name"],
["family_name"],
["phone_number"],
))
self.assertDifferences(
person.diff_iter(friendless, options=ignore_friends),
{},
)
def __str__(self):
what = ("%s vs %s" % (self.base_type_name, self.other_type_name)
if self.base_type_name != self.other_type_name else
self.base_type_name)
diffstate = collections.defaultdict(list)
for diff in self:
if diff.diff_type == DiffTypes.ADDED:
diffstate["+NEW"].append(diff.other)
elif diff.diff_type == DiffTypes.REMOVED:
diffstate["-OLD"].append(diff.base)
elif diff.diff_type == DiffTypes.MODIFIED:
if diff.base.path == diff.other.path:
diffstate['<>X'].append(diff.base)
else:
diffstate['<->OLD'].append(diff.base)
diffstate['<+>NEW'].append(diff.other)
elif diff.diff_type == DiffTypes.NO_CHANGE:
diffstate['==X'].append(diff.base)
prefix_paths = []
for k, v in diffstate.items():
prefix_paths.append("{prefix}({paths})".format(
prefix=k,
paths=MultiFieldSelector(*v).path,
))
return "<Diff [{what}]; {n} diff(s){summary}>".format(
n=len(self),
what=what,
summary=(": " + "; ".join("{prefix}({paths})".format(
prefix=k,
paths=MultiFieldSelector(*v).path,
) for (k, v) in diffstate.items()) if diffstate else ""),
)
def test_mfs_subscript_by_selector(self):
"""MultiFieldSelector subscript using FieldSelector"""
mfs = MultiFieldSelector([None, "foo"])
self.assertEqual(mfs.path, "[*].foo")
x = mfs[(1, "none")]
self.assertEqual(x, None)
self.assertEqual(mfs[FieldSelector((1, "none"))], None)
def test_multi_selector_in(self):
"""Test FieldSelectors can be checked against MultiFieldSelectors"""
mfs = MultiFieldSelector(
["rakkk", None, "awkkkkkk"],
["rakkk", None, "zgruppp"],
["cr_r_a_a_ck", "rip"],
["cr_r_a_a_ck", "aiieee"],
)
self.assertIn("rakkk", mfs)
self.assertNotIn("ouch", mfs)
self.assertIn(any, mfs)
fs_in = tuple(
FieldSelector(x) for x in (
("rakkk", 1, "zgruppp"),
("rakkk", None, "zgruppp"),
("rakkk", 2, "awkkkkkk", "bap"),
("cr_r_a_a_ck", "rip"),
("cr_r_a_a_ck", "rip", "spla_a_t"),
))
for fs in fs_in:
self.assertIn(fs, mfs, fs.path)
fs_not_in = tuple(
FieldSelector(x) for x in (
("rakkk", ),
("rakkk", 0),
("rakkk", None),
("rakkk", 0, "aiee"),
("rakkk", "clank"),
("ouch", ),
("cr_r_a_a_ck", ),
("cr_r_a_a_ck", "zlopp"),
("rakkk", 1, "pow"),
))
for fs in fs_not_in:
self.assertNotIn(fs, mfs, fs.path)
fs_some = fs_in + tuple(
FieldSelector(x) for x in (
("rakkk", ),
("rakkk", 0),
("cr_r_a_a_ck", ),
))
for fs in fs_some:
self.assertIsNotNone(mfs[fs], fs.path)
fs_not_any = tuple(
FieldSelector(x) for x in (
("ouch", ),
("cr_r_a_a_ck", "zlopp"),
("rakkk", 1, "pow"),
))
for fs in fs_not_any:
self.assertIsNone(mfs[fs], fs.path)
def test_filtered_coll_items_diff(self):
strip_ids_mfs = MultiFieldSelector(
["name", "family"], ["date_of_birth"],
["friends", None, "name"],
["friends", None, "date_of_birth"],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = strip_ids_mfs.get(person)
# not terribly useful!
self.assertDifferences(
person.diff_iter(filtered_person), {
"REMOVED .ssn", "REMOVED .phone_number",
"REMOVED .name.given",
"REMOVED .friends[0]", "ADDED .friends[0]",
"REMOVED .friends[1]", "ADDED .friends[1]",
"REMOVED .friends[2]", "ADDED .friends[2]",
"REMOVED .friends[3]", "ADDED .friends[3]",
},
)
# however, pass the filter into diff, and it gets it right!
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs), {},
)
filtered_person.friends.append(get_person(1))
del filtered_person.friends[0]
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs),
{"ADDED .friends[3]", "REMOVED .friends[0]"},
)
def __init__(self,
unpack_func,
apply_func,
collect_func,
reduce_func,
apply_empty_slots=False,
extraneous=False,
ignore_empty_string=False,
ignore_none=True,
visit_filter=None,
filter=None):
"""Create a new Visitor object. Generally called by a front-end class
method of :py:class:`VisitorPattern`
There are four positional arguments, which specify the particular
functions to be used during the visit. The important options from a
user of a visitor are the keyword arguments:
``apply_empty_slots=``\ *bool*
If set, then your ``apply`` method (or ``reverse``, etc) will
be called even if there is no corresponding value in the input.
Your method will receive the Exception as if it were the value.
``extraneous=``\ *bool*
Also call the apply method on properties marked *extraneous*.
False by default.
``ignore_empty_string=``\ *bool*
If the 'apply' function returns the empty string, treat it as
if the slot or object did not exist. ``False`` by default.
``ignore_none=``\ *bool*
If the 'apply' function returns ``None``, treat it as if the
slot or object did not exist. ``True`` by default.
``visit_filter=``\ *MultiFieldSelector*
This supplies an instance of
:py:class:`normalize.selector.MultiFieldSelector`, and
restricts the operation to the matched object fields. Can also
be specified as just ``filter=``
"""
self.unpack = unpack_func
self.apply = apply_func
self.collect = collect_func
self.reduce = reduce_func
self.apply_empty_slots = apply_empty_slots
self.extraneous = extraneous
self.ignore_empty_string = ignore_empty_string
self.ignore_none = ignore_none
if visit_filter is None:
visit_filter = filter
if isinstance(visit_filter, (MultiFieldSelector, type(None))):
self.visit_filter = visit_filter
else:
self.visit_filter = MultiFieldSelector(*visit_filter)
self.seen = set() # TODO
self.cue = list()
def test_ignore_empty_and_coll(self):
person = get_person(6, 0, 3, 4, 5)
strip_ids_mfs = MultiFieldSelector(
["given_name"],
["family_name"],
["description"],
["friends", None, "given_name"],
["friends", None, "family_name"],
["friends", None, "description"],
)
filtered_person = strip_ids_mfs.get(person)
person.description = ""
person.friends[0].description = ""
self.assertDifferences(
person.diff_iter(filtered_person, compare_filter=strip_ids_mfs),
{
"REMOVED .description",
"REMOVED .friends[0]",
"ADDED .friends[0]",
},
)
self.assertDifferences(
person.diff_iter(filtered_person,
compare_filter=strip_ids_mfs,
ignore_empty_slots=True),
{},
)
def test_mfs_apply_ops(self):
from copy import deepcopy
from testclasses import wall_one
from normalize.diff import DiffTypes
selectors = (
("owner", ),
("posts", 0, "comments", 0, "poster"),
("posts", 0, "comments", 1, "content"),
)
required_fields = (
("id", ),
("posts", 0, "edited"),
("posts", 0, "post_id"),
("posts", 0, "wall_id"),
("posts", 0, "comments", 0, "edited"),
("posts", 0, "comments", 0, "id"),
("posts", 0, "comments", 1, "edited"),
("posts", 0, "comments", 1, "id"),
)
deletable_mfs = MultiFieldSelector(*selectors)
skeleton_mfs = MultiFieldSelector(*(required_fields + selectors))
scratch_wall = deepcopy(wall_one)
saved_fields = skeleton_mfs.get(scratch_wall)
deletable_mfs.delete(scratch_wall)
removed = set(
tuple(x.base) for x in wall_one.diff_iter(scratch_wall)
if x.diff_type == DiffTypes.REMOVED)
self.assertEqual(
removed,
set(selectors),
"MultiFieldSelector.delete() can delete named attributes",
)
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
scratch_wall.diff(wall_one),
"MultiFieldSelector.patch() can copy named attributes",
)
del saved_fields.owner
deletable_mfs.patch(scratch_wall, saved_fields)
self.assertFalse(
hasattr(scratch_wall, "owner"),
"MultiFieldSelector.patch() can delete missing attributes",
)
def __init__(self, ignore_ws=True, ignore_case=False,
unicode_normal=True, unchanged=False,
ignore_empty_slots=False,
duck_type=False, extraneous=False,
compare_filter=None):
"""Create a new ``DiffOptions`` instance.
args:
``ignore_ws=``\ *BOOL*
Ignore whitespace in strings (beginning, end and middle).
True by default.
``ignore_case=``\ *BOOL*
Ignore case differences in strings. False by default.
``unicode_normal=``\ *BOOL*
Ignore unicode normal form differences in strings by
normalizing to NFC before comparison. True by default.
``unchanged=``\ *BOOL*
Yields ``DiffInfo`` objects for every comparison, not just
those which found a difference. Defaults to False. Useful for
testing.
``ignore_empty_slots=``\ *BOOL*
If true, slots containing typical 'empty' values (by default,
just ``''`` and ``None``) are treated as if they were not set.
False by default.
``duck_type=``\ *BOOL*
Normally, types must match or the result will always be
:py:attr:`normalize.diff.DiffTypes.MODIFIED` and the comparison
will not descend further.
However, setting this option bypasses this check, and just
checks that the 'other' object has all of the properties
defined on the 'base' type. This can be used to check progress
when porting from other object systems to normalize.
``compare_filter=``\ *MULTIFIELDSELECTOR*\ \|\ *LIST_OF_LISTS*
Restrict comparison to the fields described by the passed
:py:class:`MultiFieldSelector` (or list of FieldSelector
lists/objects)
"""
self.ignore_ws = ignore_ws
self.ignore_case = ignore_case
self.ignore_empty_slots = ignore_empty_slots
self.unicode_normal = unicode_normal
self.unchanged = unchanged
self.duck_type = duck_type
self.extraneous = extraneous
if isinstance(compare_filter, (MultiFieldSelector, types.NoneType)):
self.compare_filter = compare_filter
else:
self.compare_filter = MultiFieldSelector(*compare_filter)
def test_filtered_diff(self):
"""Test that diff notices when fields are removed"""
name_mfs = MultiFieldSelector(["name", "given"], ["name", "family"])
person = get_person(1)
filtered_person = name_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number"},
)
def test_filtered_collection_compare(self):
class Foo(Record):
bar = Property()
class Foos(RecordList):
itemtype = Foo
self.assertDifferences(
Foos().diff_iter(Foos(), compare_filter=MultiFieldSelector()),
{},
)
def test_mfs_marshal(self):
mfs = MultiFieldSelector(
["rakkk", None, "awkkkkkk"],
["rakkk", None, "zgruppp"],
["cr_r_a_a_ck", "rip"],
["cr_r_a_a_ck", "aiieee"],
)
path = mfs.path
new_mfs = MultiFieldSelector.from_path(path)
for fs in mfs:
self.assertIn(fs, new_mfs)
parts = list(fs)
self.assertIsNotNone(parts[-1])
for fs in new_mfs:
self.assertIn(fs, mfs)
self.assertEqual(len(mfs.path), len(new_mfs.path))
for path in (".foo", ".foo[*]", ".foo.bar[*]"):
mfs = MultiFieldSelector.from_path(path)
self.assertEqual(mfs.path, path)
for mfs_fs in (
((), ),
(("foo", ), ),
((1, ), (2, )),
((None, )),
(("foo", "bar", None), ),
):
mfs = MultiFieldSelector(*mfs_fs)
path = mfs.path
mfs_loop = MultiFieldSelector.from_path(path)
self.assertEqual(mfs_loop.path, path)
self.assertEqual(list(fs.path for fs in mfs),
list(fs.path for fs in mfs_loop))
def test_ignore_empty_items(self):
person = get_person(3)
person.friends = []
person2 = get_person(3, 0, 4, 2, 6)
no_populated_subfields_mfs = MultiFieldSelector(
["name"], ["description"],
["phone_number"],
["friends", None, "description"],
)
self.assertDifferences(
person.diff_iter(
person2,
compare_filter=no_populated_subfields_mfs,
ignore_empty_items=True,
),
set(),
)
some_populated_subfields_mfs = MultiFieldSelector(
["name"], ["description"],
["phone_number"],
["friends", None, "name", "family"],
)
self.assertDifferences(
person.diff_iter(
person2,
compare_filter=some_populated_subfields_mfs,
ignore_empty_items=True,
),
{
"ADDED .friends[0]",
"ADDED .friends[1]",
"ADDED .friends[2]",
"ADDED .friends[3]",
}
)
def test_filtered_coll_diff(self):
name_and_friends_mfs = MultiFieldSelector(
["name"],
["friends", 0],
["friends", 2],
)
person = get_person(0, 2, 5, 6, 3)
filtered_person = name_and_friends_mfs.get(person)
self.assertDifferences(
person.diff_iter(filtered_person),
{"REMOVED .date_of_birth", "REMOVED .ssn",
"REMOVED .phone_number",
"REMOVED .friends[1]", "REMOVED .friends[3]"},
)
def test_mfs_marshal(self):
mfs = MultiFieldSelector(
["rakkk", None, "awkkkkkk"],
["rakkk", None, "zgruppp"],
["cr_r_a_a_ck", "rip"],
["cr_r_a_a_ck", "aiieee"],
)
path = mfs.path
new_mfs = MultiFieldSelector.from_path(path)
for fs in mfs:
self.assertIn(fs, new_mfs)
for fs in new_mfs:
self.assertIn(fs, mfs)
self.assertEqual(len(mfs.path), len(new_mfs.path))
def test_fuzzy_compare(self):
person = get_person(3, 0, 2, 4, 6, 4, 1)
person2 = get_person(3, 0, 1, 2, 3, 4, 5, 6)
strip_ssn_mfs = MultiFieldSelector(
["name"], ["description"],
["phone_number"],
["friends", None, "name"],
["friends", None, "description"],
["friends", None, "phone_number"],
)
basic_differences = {
"REMOVED .friends[4]",
"ADDED .friends[5]",
"ADDED .friends[3]",
}
self.assertDifferences(
person.diff_iter(person2, compare_filter=strip_ssn_mfs),
basic_differences,
)
person2.friends[0].name.given = "Jim"
del person2.friends[1].phone_number
person2.friends[2].name.family = "Woolf"
person2.friends[6].name.family = "Haines"
self.assertDifferences(
person.diff_iter(person2, compare_filter=strip_ssn_mfs),
basic_differences | {
'MODIFIED .friends[0].name.given',
'MODIFIED (.friends[3].name.family/.friends[6].name.family)',
'REMOVED (.friends[5].phone_number/.friends[1].phone_number)',
'MODIFIED (.friends[1].name.family/.friends[2].name.family)',
}
)
def test_null_mfs(self):
null_mfs = MultiFieldSelector()
self.assertNotIn(any, null_mfs)
self.assertFalse(null_mfs)
self.assertFalse(null_mfs[any])
def __init__(self, ignore_ws=True, ignore_case=False,
unicode_normal=True, unchanged=False,
ignore_empty_slots=False, ignore_empty_items=False,
duck_type=False, extraneous=False,
compare_filter=None, fuzzy_match=True, moved=False,
recurse=False):
"""Create a new ``DiffOptions`` instance.
args:
``ignore_ws=``\ *BOOL*
Ignore whitespace in strings (beginning, end and middle).
True by default.
``ignore_case=``\ *BOOL*
Ignore case differences in strings. False by default.
``unicode_normal=``\ *BOOL*
Ignore unicode normal form differences in strings by
normalizing to NFC before comparison. True by default.
``unchanged=``\ *BOOL*
Yields ``DiffInfo`` objects for every comparison, not just
those which found a difference. Defaults to False. Useful for
testing.
``moved=``\ *BOOL*
Yields ``DiffInfo`` objects for comparisons where the values
matched, but keys or indexes were different. Defaults to
False.
``ignore_empty_slots=``\ *BOOL*
If true, slots containing typical 'empty' values (by default,
just ``''`` and ``None``) are treated as if they were not set.
False by default.
``ignore_empty_items=``\ *BOOL*
If true, items are considered to be absent from collections if
they have all ``None``, not set, or ``''`` in their primary key
fields (all compared fields in the absence of a primary key
definition). False by default.
``duck_type=``\ *BOOL*
Normally, types must match or the result will always be
:py:attr:`normalize.diff.DiffTypes.MODIFIED` and the comparison
will not descend further.
However, setting this option bypasses this check, and just
checks that the 'other' object has all of the properties
defined on the 'base' type. This can be used to check progress
when porting from other object systems to normalize.
``fuzzy_match=``\ *BOOL*
Enable approximate matching of items in collections, so that
finer granularity of changes are available.
``compare_filter=``\ *MULTIFIELDSELECTOR*\ \|\ *LIST_OF_LISTS*
Restrict comparison to the fields described by the passed
:py:class:`MultiFieldSelector` (or list of FieldSelector
lists/objects)
``recurse=``\ *BOOL* During diff operations, do a deeper
comparison via recursion. This may be potentially very
expensive computationally if your records are large or
very nested.
"""
self.ignore_ws = ignore_ws
self.ignore_case = ignore_case
self.ignore_empty_slots = ignore_empty_slots
self.ignore_empty_items = ignore_empty_items
self.unicode_normal = unicode_normal
self.fuzzy_match = fuzzy_match
self.unchanged = unchanged
self.moved = moved
self.duck_type = duck_type
self.extraneous = extraneous
self.recurse = recurse
if isinstance(compare_filter, (MultiFieldSelector, types.NoneType)):
self.compare_filter = compare_filter
else:
self.compare_filter = MultiFieldSelector(*compare_filter)
def test_mfs_subscript_identity(self):
"""MultiFieldSelector subscript has an identity value"""
mfs = MultiFieldSelector([None, "foo"])
self.assertEqual(mfs.path, "[*].foo")
self.assertEqual(mfs[FieldSelector(())].path, mfs.path)
self.assertEqual(mfs[()].path, mfs.path)
def test_multi_selector(self):
selectors = set((
("bar", ),
("foo", "bar", 0, "boo"),
("foo", "bar", 0, "hiss"),
("foo", "bar", 1),
))
mfs = MultiFieldSelector(*selectors)
emitted = set(tuple(x.selectors) for x in mfs)
self.assertEqual(emitted, selectors)
# match, eg <MultiFieldSelector: (.foo.bar([0](.hiss|.boo)|[1])|.bar)>
# but also <MultiFieldSelector: (.bar|.foo.bar([1]|[0](.boo|.hiss)))>
regexp = re.compile(
r"""<MultiFieldSelector:\s+\(
(?:
(?: .foo.bar \(
(?:
(?: \[0\] \(
(?:
(?: .hiss | .boo ) \|?
){2} \)
| \[1\] ) \|?
){2} \)
| .bar ) \|?
){2}
\)>""",
re.X,
)
self.assertRegexpMatches(str(mfs), regexp)
mfs_dupe = eval(repr(mfs))
emitted = set(tuple(x.selectors) for x in mfs_dupe)
self.assertEqual(emitted, selectors)
# test various dict-like functions
self.assertIn("foo", mfs)
self.assertIn("bar", mfs)
self.assertNotIn("baz", mfs)
self.assertIn('bar', mfs['foo'])
self.assertIn(0, mfs['foo']['bar'])
self.assertIn('hiss', mfs['foo']['bar'][0])
self.assertNotIn('miss', mfs['foo']['bar'][0])
self.assertIn('baz', mfs['bar'])
self.assertIn('baz', mfs['bar']['frop']['quux']['fred'])
# if you add a higher level selector, then more specific paths
# disappear from the MFS
mfs2 = MultiFieldSelector(mfs, ["foo", "bar"])
emitted = set(tuple(x.selectors) for x in mfs2)
self.assertEqual(emitted, set((("bar", ), ("foo", "bar"))))
data = {
"bar": [1, 2, 3],
"foo": {
"bar": [
{
"boo": "waa",
"frop": "quux"
},
{
"waldo": "grault"
},
{
"fubar": "corge"
},
],
},
}
selected = mfs.get(data)
self.assertEqual(
selected, {
"bar": [1, 2, 3],
"foo": {
"bar": [
{
"boo": "waa"
},
{
"waldo": "grault"
},
],
},
})
class Octothorpe(Record):
name = Property()
boo = Property()
hiss = Property()
class Caret(Record):
bar = ListProperty(of=Octothorpe)
class Pilcrow(Record):
bar = ListProperty(of=Octothorpe)
foo = Property(isa=Caret)
baz = Property()
quux = DictProperty(of=str)
frop = DictProperty(of=list_of(unicode))
full = Pilcrow(
bar=[dict(name="Heffalump"),
dict(name="Uncle Robert")],
foo=dict(
bar=[dict(name="Owl", hiss="Hunny Bee"),
dict(name="Piglet")]),
baz="Wizzle",
quux={
"protagonist": "Winnie_the_Pooh",
"antagonist": "Alexander_Beetle"
},
frop={
"lighting": ["Uncle_Robert", "Kanga", "Small"],
"story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"]
},
)
selectors.add(("quux", "protagonist"))
self.assertEqual(
FieldSelector(("quux", "protagonist")).get(full),
"Winnie_the_Pooh",
)
selectors.add(("frop", "story"))
mfs = MultiFieldSelector(*selectors)
filtered = mfs.get(full)
expected = Pilcrow(
bar=[dict(name="Heffalump"),
dict(name="Uncle Robert")],
foo=dict(bar=[dict(
hiss="Hunny Bee"), dict(name="Piglet")]),
quux={"protagonist": "Winnie_the_Pooh"},
frop={"story": ["Smallest_of_all", "Eeyore", "Christopher_Robin"]},
)
self.assertEqual(filtered, expected)