def _box_ingress_error(context, exception):
error_fs = FieldSelector([context])
if hasattr(exception, "error_fs"):
error_fs.extend(exception.error_fs)
if hasattr(exception, "sub_exception"):
exception = exception.sub_exception
return exc.JsonConversionError(
error_fs=error_fs,
sub_exception=exception,
)
def _box_ingress_error(context, exception):
error_fs = FieldSelector([context])
if hasattr(exception, "error_fs"):
error_fs.extend(exception.error_fs)
if hasattr(exception, "sub_exception"):
exception = exception.sub_exception
return exc.JsonConversionError(
error_fs=error_fs,
sub_exception=exception,
)
def diff_iter(base, other, options=None, **kwargs):
"""Compare a Record with another object (usually a record of the same
type), and yield differences as :py:class:`DiffInfo` instances.
args:
``base=``\ *Record*
The 'base' object to compare against. The enumeration in
:py:class:`DiffTypes` is relative to this object.
``other=``\ *Record*\ \|\ *<object>*
The 'other' object to compare against. If ``duck_type`` is not
true, then it must be of the same type as the ``base``.
``**kwargs``
Specify comparison options: ``duck_type``, ``ignore_ws``, etc. See
:py:meth:`normalize.diff.DiffOptions.__init__` for the complete
list.
``options=``\ *DiffOptions instance*
Pass in a pre-constructed :py:class:`DiffOptions` instance. This
may not be specified along with ``**kwargs``.
"""
if options is None:
options = DiffOptions(**kwargs)
elif len(kwargs):
raise exc.DiffOptionsException()
null_fs = FieldSelector(tuple())
return _diff_iter(base, other, null_fs, null_fs, options)
def compare_dict_iter(propval_a, propval_b, fs_a=None, fs_b=None,
options=None):
"""Generator for comparing 'simple' dicts when they are encountered. This
does not currently recurse further. Arguments are as per other
``compare_``\ *X* functions.
"""
if fs_a is None:
fs_a = FieldSelector(tuple())
fs_b = FieldSelector(tuple())
if not options:
options = DiffOptions()
propvals = dict(a=propval_a, b=propval_b)
values = dict()
rev_keys = dict()
for x in "a", "b":
propval_x = propvals[x]
vals = values[x] = set()
rev_key = rev_keys[x] = dict()
seen = collections.Counter()
for k, v in collection_generator(propval_x):
v = options.normalize_item(
v, propval_a if options.duck_type else propval_x
)
if not v.__hash__:
v = repr(v)
if v is not _nothing or not options.ignore_empty_slots:
vals.add((v, seen[v]))
rev_key[(v, seen[v])] = k
seen[v] += 1
removed = values['a'] - values['b']
added = values['b'] - values['a']
if options.unchanged:
unchanged = values['a'] & values['b']
for v, seq in unchanged:
a_key = rev_keys['a'][v, seq]
b_key = rev_keys['b'][v, seq]
yield DiffInfo(
diff_type=DiffTypes.NO_CHANGE,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
for v, seq in removed:
a_key = rev_keys['a'][v, seq]
selector = fs_a + [a_key]
yield DiffInfo(
diff_type=DiffTypes.REMOVED,
base=selector,
other=fs_b,
)
for v, seq in added:
b_key = rev_keys['b'][v, seq]
selector = fs_b + [b_key]
yield DiffInfo(
diff_type=DiffTypes.ADDED,
base=fs_a,
other=selector,
)
def compare_collection_iter(propval_a, propval_b, fs_a=None, fs_b=None,
options=None):
"""Generator function to compare two collections, and yield differences.
This function does not currently report moved items in collections, and
uses the :py:meth:`DiffOptions.record_id` method to decide if objects are
to be considered the same, and differences within returned.
Arguments are the same as :py:func:`compare_record_iter`.
Note that ``diff_iter`` and ``compare_record_iter`` will call *both* this
function and ``compare_record_iter`` on ``RecordList`` types. However, as
most ``RecordList`` types have no extra properties, no differences are
yielded by the ``compare_record_iter`` method.
"""
if fs_a is None:
fs_a = FieldSelector(tuple())
fs_b = FieldSelector(tuple())
if options is None:
options = DiffOptions()
propvals = dict(a=propval_a, b=propval_b)
values = dict()
rev_keys = dict()
compare_values = None
coll_type = (
type(propval_a) if propval_a is not _nothing else type(propval_b)
)
force_descent = (propval_a is _nothing) or (propval_b is _nothing)
id_args = options.id_args(coll_type.itemtype, fs_a)
for x in "a", "b":
propval_x = propvals[x]
vals = values[x] = set()
rev_key = rev_keys[x] = dict()
seen = collections.Counter()
for k, v in collection_generator(propval_x):
pk = options.record_id(v, **id_args)
if compare_values is None:
# the primary key being a tuple is taken to imply that
# the value type is a Record, and hence descent is
# possible.
compare_values = isinstance(pk, tuple)
vals.add((pk, seen[pk]))
rev_key[(pk, seen[pk])] = k
seen[pk] += 1
removed = values['a'] - values['b']
added = values['b'] - values['a']
if compare_values or force_descent:
descendable = (removed | added) if force_descent else (
values['a'].intersection(values['b'])
)
for pk, seq in descendable:
if not force_descent or propval_a is not _nothing:
a_key = rev_keys['a'][pk, seq]
a_val = propval_a[a_key]
if not force_descent or propval_b is not _nothing:
b_key = rev_keys['b'][pk, seq]
b_val = propval_b[b_key]
if force_descent:
if propval_a is _nothing:
a_key = b_key
a_val = _nothing
else:
b_key = a_key
b_val = _nothing
selector_a = fs_a + a_key
selector_b = fs_b + b_key
for diff in compare_record_iter(
a_val, b_val, selector_a, selector_b, options,
):
yield diff
if options.unchanged:
unchanged = values['a'] & values['b']
for pk, seq in unchanged:
a_key = rev_keys['a'][pk, seq]
b_key = rev_keys['b'][pk, seq]
yield DiffInfo(
diff_type=DiffTypes.NO_CHANGE,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
if not force_descent:
for pk, seq in removed:
a_key = rev_keys['a'][pk, seq]
selector = fs_a + [a_key]
yield DiffInfo(
diff_type=DiffTypes.REMOVED,
base=selector,
other=fs_b,
)
for pk, seq in added:
b_key = rev_keys['b'][pk, seq]
selector = fs_b + [b_key]
yield DiffInfo(
diff_type=DiffTypes.ADDED,
base=fs_a,
other=selector,
)
def compare_record_iter(a, b, fs_a=None, fs_b=None, options=None):
"""This generator function compares a record, slot by slot, and yields
differences found as ``DiffInfo`` objects.
args:
``a=``\ *Record*
The base object
``b=``\ *Record*\ \|\ *object*
The 'other' object, which must be the same type as ``a``, unless
``options.duck_type`` is set.
``fs_a=``\ *FieldSelector\*
The current diff context, prefixed to any returned ``base`` field
in yielded ``DiffInfo`` objects. Defaults to an empty
FieldSelector.
``fs_b=``\ *FieldSelector\*
The ``other`` object context. This will differ from ``fs_a`` in
the case of collections, where a value has moved slots. Defaults
to an empty FieldSelector.
``options=``\ *DiffOptions\*
A constructed ``DiffOptions`` object; a default one is created if
not passed in.
"""
if not options:
options = DiffOptions()
if not options.duck_type and type(a) != type(b) and not (
a is _nothing or b is _nothing
):
raise TypeError(
"cannot compare %s with %s" % (type(a).__name__, type(b).__name__)
)
if fs_a is None:
fs_a = FieldSelector(tuple())
fs_b = FieldSelector(tuple())
properties = (
type(a).properties if a is not _nothing else type(b).properties
)
for propname in sorted(properties):
prop = properties[propname]
if options.is_filtered(prop, fs_a + propname):
continue
propval_a = options.normalize_object_slot(
getattr(a, propname, _nothing), prop, a,
)
propval_b = options.normalize_object_slot(
getattr(b, propname, _nothing), prop, b,
)
if propval_a is _nothing and propval_b is _nothing:
# don't yield NO_CHANGE for fields missing on both sides
continue
one_side_nothing = (propval_a is _nothing) != (propval_b is _nothing)
types_match = type(propval_a) == type(propval_b)
comparable = (
isinstance(propval_a, COMPARABLE) or
isinstance(propval_b, COMPARABLE)
)
prop_fs_a = fs_a + [propname]
prop_fs_b = fs_b + [propname]
if comparable and (
types_match or options.duck_type or (
options.ignore_empty_slots and one_side_nothing
)
):
if one_side_nothing:
diff_types_found = set()
for type_union, func in COMPARE_FUNCTIONS.iteritems():
if isinstance(propval_a, type_union) or one_side_nothing and (
isinstance(propval_b, type_union)
):
for diff in func(
propval_a, propval_b, prop_fs_a,
prop_fs_b, options,
):
if one_side_nothing:
if diff.diff_type != DiffTypes.NO_CHANGE:
diff_types_found.add(diff.diff_type)
else:
yield diff
if one_side_nothing:
net_diff = None
if diff_types_found:
assert(len(diff_types_found) == 1)
net_diff = tuple(diff_types_found)[0]
elif options.unchanged:
net_diff = DiffTypes.NO_CHANGE
if net_diff:
yield DiffInfo(
diff_type=net_diff,
base=prop_fs_a,
other=prop_fs_b,
)
elif one_side_nothing:
yield DiffInfo(
diff_type=(
DiffTypes.ADDED if propval_a is _nothing else
DiffTypes.REMOVED
),
base=fs_a + [propname],
other=fs_b + [propname],
)
elif not options.items_equal(propval_a, propval_b):
yield DiffInfo(
diff_type=DiffTypes.MODIFIED,
base=fs_a + [propname],
other=fs_b + [propname],
)
elif options.unchanged:
yield DiffInfo(
diff_type=DiffTypes.NO_CHANGE,
base=fs_a + [propname],
other=fs_b + [propname],
)
def compare_collection_iter(propval_a, propval_b, fs_a=None, fs_b=None,
options=None):
"""Generator function to compare two collections, and yield differences.
This function does not currently report moved items in collections, and
uses the :py:meth:`DiffOptions.record_id` method to decide if objects are
to be considered the same, and differences within returned.
Arguments are the same as :py:func:`compare_record_iter`.
Note that ``diff_iter`` and ``compare_record_iter`` will call *both* this
function and ``compare_record_iter`` on ``RecordList`` types. However, as
most ``RecordList`` types have no extra properties, no differences are
yielded by the ``compare_record_iter`` method.
"""
if fs_a is None:
fs_a = FieldSelector(tuple())
fs_b = FieldSelector(tuple())
if options is None:
options = DiffOptions()
propvals = dict(a=propval_a, b=propval_b)
values = dict()
rev_keys = dict()
compare_values = None
coll_type = (
type(propval_a) if propval_a is not _nothing else type(propval_b)
)
force_descent = (propval_a is _nothing) or (propval_b is _nothing)
if isinstance(coll_type.itemtype, tuple):
raise exc.CollectionItemTypeUnsupported()
elif not issubclass(coll_type.itemtype, Record):
iter_func = (
compare_dict_iter if issubclass(coll_type, DictCollection) else
compare_list_iter if issubclass(coll_type, ListCollection) else
None
)
if not iter_func:
raise exc.CollectionDiffUnsupported(
coll_type=coll_type,
item_type=coll_type.itemtype,
item_type_name=coll_type.itemtype.__name__,
)
for diff in iter_func(propval_a, propval_b, fs_a, fs_b, options):
yield diff
return
id_args = options.id_args(coll_type.itemtype, fs_a)
if not callable(id_args) and 'selector' in id_args and \
not id_args['selector']:
# early exit shortcut
return
for x in "a", "b":
propval_x = propvals[x]
vals = values[x] = set()
rev_key = rev_keys[x] = dict()
seen = collections.Counter()
for k, v in collection_generator(propval_x):
if callable(id_args):
if fs_a + [k] not in options.compare_filter:
continue
pk = options.record_id(
v, **(id_args(k) if callable(id_args) else id_args))
if options.ignore_empty_items and _nested_empty(pk):
continue
if compare_values is None:
# the primary key being a tuple is taken to imply that
# the value type is a Record, and hence descent is
# possible.
compare_values = isinstance(pk, tuple)
vals.add((pk, seen[pk]))
rev_key[(pk, seen[pk])] = k
seen[pk] += 1
if options.recurse:
# we can be sure that both records have these keys
set_a = set(rev_keys["a"].values())
set_b = set(rev_keys["b"].values())
shared_keys = set_a.intersection(set_b)
removed = set_a - set_b
added = set_b - set_a
for key in shared_keys:
if (isinstance(propval_a, collections.Iterable) and
isinstance(propval_b, collections.Iterable)):
diffs = _diff_iter(propval_a[key], propval_b[key],
fs_a + [key], fs_b + [key], options)
for diff in diffs:
yield diff
for key in removed:
yield DiffInfo(diff_type=DiffTypes.REMOVED,
base=fs_a + [key],
other=fs_b)
for key in added:
yield DiffInfo(diff_type=DiffTypes.ADDED,
base=fs_a,
other=fs_b + [key])
else:
removed = values['a'] - values['b']
added = values['b'] - values['a']
common = values['a'].intersection(values['b'])
if compare_values or force_descent:
descendable = (removed | added) if force_descent else common
for pk, seq in descendable:
if not force_descent or propval_a is not _nothing:
a_key = rev_keys['a'][pk, seq]
a_val = propval_a[a_key]
if not force_descent or propval_b is not _nothing:
b_key = rev_keys['b'][pk, seq]
b_val = propval_b[b_key]
if force_descent:
if propval_a is _nothing:
a_key = b_key
a_val = _nothing
else:
b_key = a_key
b_val = _nothing
selector_a = fs_a + a_key
selector_b = fs_b + b_key
for diff in _diff_iter(
a_val, b_val, selector_a, selector_b, options,
):
yield diff
if not force_descent and options.fuzzy_match:
for a_pk_seq, b_pk_seq in _fuzzy_match(removed, added):
removed.remove(a_pk_seq)
added.remove(b_pk_seq)
a_key = rev_keys['a'][a_pk_seq]
a_val = propval_a[a_key]
b_key = rev_keys['b'][b_pk_seq]
b_val = propval_b[b_key]
selector_a = fs_a + a_key
selector_b = fs_b + b_key
any_diffs = False
for diff in _diff_iter(
a_val, b_val, selector_a, selector_b, options,
):
if diff.diff_type != DiffTypes.NO_CHANGE:
any_diffs = True
yield diff
if options.moved and a_key != b_key:
yield DiffInfo(
diff_type=DiffTypes.MOVED,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
elif options.unchanged and not any_diffs:
yield DiffInfo(
diff_type=DiffTypes.NO_CHANGE,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
if options.unchanged or options.moved:
unchanged = values['a'] & values['b']
for pk, seq in unchanged:
a_key = rev_keys['a'][pk, seq]
b_key = rev_keys['b'][pk, seq]
if options.moved and a_key != b_key:
yield DiffInfo(
diff_type=DiffTypes.MOVED,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
elif options.unchanged:
yield DiffInfo(
diff_type=DiffTypes.NO_CHANGE,
base=fs_a + [a_key],
other=fs_b + [b_key],
)
if not force_descent:
for pk, seq in removed:
a_key = rev_keys['a'][pk, seq]
selector = fs_a + [a_key]
yield DiffInfo(
diff_type=DiffTypes.REMOVED,
base=selector,
other=fs_b,
)
for pk, seq in added:
b_key = rev_keys['b'][pk, seq]
selector = fs_b + [b_key]
yield DiffInfo(
diff_type=DiffTypes.ADDED,
base=fs_a,
other=selector,
)
def field_selector(self):
return FieldSelector(self.cue)
