def _box_ingress_error(context, exception): error_fs = FieldSelector([context]) if hasattr(exception, "error_fs"): error_fs.extend(exception.error_fs) if hasattr(exception, "sub_exception"): exception = exception.sub_exception return exc.JsonConversionError( error_fs=error_fs, sub_exception=exception, )
def diff_iter(base, other, options=None, **kwargs): """Compare a Record with another object (usually a record of the same type), and yield differences as :py:class:`DiffInfo` instances. args: ``base=``\ *Record* The 'base' object to compare against. The enumeration in :py:class:`DiffTypes` is relative to this object. ``other=``\ *Record*\ \|\ *<object>* The 'other' object to compare against. If ``duck_type`` is not true, then it must be of the same type as the ``base``. ``**kwargs`` Specify comparison options: ``duck_type``, ``ignore_ws``, etc. See :py:meth:`normalize.diff.DiffOptions.__init__` for the complete list. ``options=``\ *DiffOptions instance* Pass in a pre-constructed :py:class:`DiffOptions` instance. This may not be specified along with ``**kwargs``. """ if options is None: options = DiffOptions(**kwargs) elif len(kwargs): raise exc.DiffOptionsException() null_fs = FieldSelector(tuple()) return _diff_iter(base, other, null_fs, null_fs, options)
def compare_dict_iter(propval_a, propval_b, fs_a=None, fs_b=None, options=None): """Generator for comparing 'simple' dicts when they are encountered. This does not currently recurse further. Arguments are as per other ``compare_``\ *X* functions. """ if fs_a is None: fs_a = FieldSelector(tuple()) fs_b = FieldSelector(tuple()) if not options: options = DiffOptions() propvals = dict(a=propval_a, b=propval_b) values = dict() rev_keys = dict() for x in "a", "b": propval_x = propvals[x] vals = values[x] = set() rev_key = rev_keys[x] = dict() seen = collections.Counter() for k, v in collection_generator(propval_x): v = options.normalize_item( v, propval_a if options.duck_type else propval_x ) if not v.__hash__: v = repr(v) if v is not _nothing or not options.ignore_empty_slots: vals.add((v, seen[v])) rev_key[(v, seen[v])] = k seen[v] += 1 removed = values['a'] - values['b'] added = values['b'] - values['a'] if options.unchanged: unchanged = values['a'] & values['b'] for v, seq in unchanged: a_key = rev_keys['a'][v, seq] b_key = rev_keys['b'][v, seq] yield DiffInfo( diff_type=DiffTypes.NO_CHANGE, base=fs_a + [a_key], other=fs_b + [b_key], ) for v, seq in removed: a_key = rev_keys['a'][v, seq] selector = fs_a + [a_key] yield DiffInfo( diff_type=DiffTypes.REMOVED, base=selector, other=fs_b, ) for v, seq in added: b_key = rev_keys['b'][v, seq] selector = fs_b + [b_key] yield DiffInfo( diff_type=DiffTypes.ADDED, base=fs_a, other=selector, )
def compare_collection_iter(propval_a, propval_b, fs_a=None, fs_b=None, options=None): """Generator function to compare two collections, and yield differences. This function does not currently report moved items in collections, and uses the :py:meth:`DiffOptions.record_id` method to decide if objects are to be considered the same, and differences within returned. Arguments are the same as :py:func:`compare_record_iter`. Note that ``diff_iter`` and ``compare_record_iter`` will call *both* this function and ``compare_record_iter`` on ``RecordList`` types. However, as most ``RecordList`` types have no extra properties, no differences are yielded by the ``compare_record_iter`` method. """ if fs_a is None: fs_a = FieldSelector(tuple()) fs_b = FieldSelector(tuple()) if options is None: options = DiffOptions() propvals = dict(a=propval_a, b=propval_b) values = dict() rev_keys = dict() compare_values = None coll_type = ( type(propval_a) if propval_a is not _nothing else type(propval_b) ) force_descent = (propval_a is _nothing) or (propval_b is _nothing) id_args = options.id_args(coll_type.itemtype, fs_a) for x in "a", "b": propval_x = propvals[x] vals = values[x] = set() rev_key = rev_keys[x] = dict() seen = collections.Counter() for k, v in collection_generator(propval_x): pk = options.record_id(v, **id_args) if compare_values is None: # the primary key being a tuple is taken to imply that # the value type is a Record, and hence descent is # possible. compare_values = isinstance(pk, tuple) vals.add((pk, seen[pk])) rev_key[(pk, seen[pk])] = k seen[pk] += 1 removed = values['a'] - values['b'] added = values['b'] - values['a'] if compare_values or force_descent: descendable = (removed | added) if force_descent else ( values['a'].intersection(values['b']) ) for pk, seq in descendable: if not force_descent or propval_a is not _nothing: a_key = rev_keys['a'][pk, seq] a_val = propval_a[a_key] if not force_descent or propval_b is not _nothing: b_key = rev_keys['b'][pk, seq] b_val = propval_b[b_key] if force_descent: if propval_a is _nothing: a_key = b_key a_val = _nothing else: b_key = a_key b_val = _nothing selector_a = fs_a + a_key selector_b = fs_b + b_key for diff in compare_record_iter( a_val, b_val, selector_a, selector_b, options, ): yield diff if options.unchanged: unchanged = values['a'] & values['b'] for pk, seq in unchanged: a_key = rev_keys['a'][pk, seq] b_key = rev_keys['b'][pk, seq] yield DiffInfo( diff_type=DiffTypes.NO_CHANGE, base=fs_a + [a_key], other=fs_b + [b_key], ) if not force_descent: for pk, seq in removed: a_key = rev_keys['a'][pk, seq] selector = fs_a + [a_key] yield DiffInfo( diff_type=DiffTypes.REMOVED, base=selector, other=fs_b, ) for pk, seq in added: b_key = rev_keys['b'][pk, seq] selector = fs_b + [b_key] yield DiffInfo( diff_type=DiffTypes.ADDED, base=fs_a, other=selector, )
def compare_record_iter(a, b, fs_a=None, fs_b=None, options=None): """This generator function compares a record, slot by slot, and yields differences found as ``DiffInfo`` objects. args: ``a=``\ *Record* The base object ``b=``\ *Record*\ \|\ *object* The 'other' object, which must be the same type as ``a``, unless ``options.duck_type`` is set. ``fs_a=``\ *FieldSelector\* The current diff context, prefixed to any returned ``base`` field in yielded ``DiffInfo`` objects. Defaults to an empty FieldSelector. ``fs_b=``\ *FieldSelector\* The ``other`` object context. This will differ from ``fs_a`` in the case of collections, where a value has moved slots. Defaults to an empty FieldSelector. ``options=``\ *DiffOptions\* A constructed ``DiffOptions`` object; a default one is created if not passed in. """ if not options: options = DiffOptions() if not options.duck_type and type(a) != type(b) and not ( a is _nothing or b is _nothing ): raise TypeError( "cannot compare %s with %s" % (type(a).__name__, type(b).__name__) ) if fs_a is None: fs_a = FieldSelector(tuple()) fs_b = FieldSelector(tuple()) properties = ( type(a).properties if a is not _nothing else type(b).properties ) for propname in sorted(properties): prop = properties[propname] if options.is_filtered(prop, fs_a + propname): continue propval_a = options.normalize_object_slot( getattr(a, propname, _nothing), prop, a, ) propval_b = options.normalize_object_slot( getattr(b, propname, _nothing), prop, b, ) if propval_a is _nothing and propval_b is _nothing: # don't yield NO_CHANGE for fields missing on both sides continue one_side_nothing = (propval_a is _nothing) != (propval_b is _nothing) types_match = type(propval_a) == type(propval_b) comparable = ( isinstance(propval_a, COMPARABLE) or isinstance(propval_b, COMPARABLE) ) prop_fs_a = fs_a + [propname] prop_fs_b = fs_b + [propname] if comparable and ( types_match or options.duck_type or ( options.ignore_empty_slots and one_side_nothing ) ): if one_side_nothing: diff_types_found = set() for type_union, func in COMPARE_FUNCTIONS.iteritems(): if isinstance(propval_a, type_union) or one_side_nothing and ( isinstance(propval_b, type_union) ): for diff in func( propval_a, propval_b, prop_fs_a, prop_fs_b, options, ): if one_side_nothing: if diff.diff_type != DiffTypes.NO_CHANGE: diff_types_found.add(diff.diff_type) else: yield diff if one_side_nothing: net_diff = None if diff_types_found: assert(len(diff_types_found) == 1) net_diff = tuple(diff_types_found)[0] elif options.unchanged: net_diff = DiffTypes.NO_CHANGE if net_diff: yield DiffInfo( diff_type=net_diff, base=prop_fs_a, other=prop_fs_b, ) elif one_side_nothing: yield DiffInfo( diff_type=( DiffTypes.ADDED if propval_a is _nothing else DiffTypes.REMOVED ), base=fs_a + [propname], other=fs_b + [propname], ) elif not options.items_equal(propval_a, propval_b): yield DiffInfo( diff_type=DiffTypes.MODIFIED, base=fs_a + [propname], other=fs_b + [propname], ) elif options.unchanged: yield DiffInfo( diff_type=DiffTypes.NO_CHANGE, base=fs_a + [propname], other=fs_b + [propname], )
def compare_collection_iter(propval_a, propval_b, fs_a=None, fs_b=None, options=None): """Generator function to compare two collections, and yield differences. This function does not currently report moved items in collections, and uses the :py:meth:`DiffOptions.record_id` method to decide if objects are to be considered the same, and differences within returned. Arguments are the same as :py:func:`compare_record_iter`. Note that ``diff_iter`` and ``compare_record_iter`` will call *both* this function and ``compare_record_iter`` on ``RecordList`` types. However, as most ``RecordList`` types have no extra properties, no differences are yielded by the ``compare_record_iter`` method. """ if fs_a is None: fs_a = FieldSelector(tuple()) fs_b = FieldSelector(tuple()) if options is None: options = DiffOptions() propvals = dict(a=propval_a, b=propval_b) values = dict() rev_keys = dict() compare_values = None coll_type = ( type(propval_a) if propval_a is not _nothing else type(propval_b) ) force_descent = (propval_a is _nothing) or (propval_b is _nothing) if isinstance(coll_type.itemtype, tuple): raise exc.CollectionItemTypeUnsupported() elif not issubclass(coll_type.itemtype, Record): iter_func = ( compare_dict_iter if issubclass(coll_type, DictCollection) else compare_list_iter if issubclass(coll_type, ListCollection) else None ) if not iter_func: raise exc.CollectionDiffUnsupported( coll_type=coll_type, item_type=coll_type.itemtype, item_type_name=coll_type.itemtype.__name__, ) for diff in iter_func(propval_a, propval_b, fs_a, fs_b, options): yield diff return id_args = options.id_args(coll_type.itemtype, fs_a) if not callable(id_args) and 'selector' in id_args and \ not id_args['selector']: # early exit shortcut return for x in "a", "b": propval_x = propvals[x] vals = values[x] = set() rev_key = rev_keys[x] = dict() seen = collections.Counter() for k, v in collection_generator(propval_x): if callable(id_args): if fs_a + [k] not in options.compare_filter: continue pk = options.record_id( v, **(id_args(k) if callable(id_args) else id_args)) if options.ignore_empty_items and _nested_empty(pk): continue if compare_values is None: # the primary key being a tuple is taken to imply that # the value type is a Record, and hence descent is # possible. compare_values = isinstance(pk, tuple) vals.add((pk, seen[pk])) rev_key[(pk, seen[pk])] = k seen[pk] += 1 if options.recurse: # we can be sure that both records have these keys set_a = set(rev_keys["a"].values()) set_b = set(rev_keys["b"].values()) shared_keys = set_a.intersection(set_b) removed = set_a - set_b added = set_b - set_a for key in shared_keys: if (isinstance(propval_a, collections.Iterable) and isinstance(propval_b, collections.Iterable)): diffs = _diff_iter(propval_a[key], propval_b[key], fs_a + [key], fs_b + [key], options) for diff in diffs: yield diff for key in removed: yield DiffInfo(diff_type=DiffTypes.REMOVED, base=fs_a + [key], other=fs_b) for key in added: yield DiffInfo(diff_type=DiffTypes.ADDED, base=fs_a, other=fs_b + [key]) else: removed = values['a'] - values['b'] added = values['b'] - values['a'] common = values['a'].intersection(values['b']) if compare_values or force_descent: descendable = (removed | added) if force_descent else common for pk, seq in descendable: if not force_descent or propval_a is not _nothing: a_key = rev_keys['a'][pk, seq] a_val = propval_a[a_key] if not force_descent or propval_b is not _nothing: b_key = rev_keys['b'][pk, seq] b_val = propval_b[b_key] if force_descent: if propval_a is _nothing: a_key = b_key a_val = _nothing else: b_key = a_key b_val = _nothing selector_a = fs_a + a_key selector_b = fs_b + b_key for diff in _diff_iter( a_val, b_val, selector_a, selector_b, options, ): yield diff if not force_descent and options.fuzzy_match: for a_pk_seq, b_pk_seq in _fuzzy_match(removed, added): removed.remove(a_pk_seq) added.remove(b_pk_seq) a_key = rev_keys['a'][a_pk_seq] a_val = propval_a[a_key] b_key = rev_keys['b'][b_pk_seq] b_val = propval_b[b_key] selector_a = fs_a + a_key selector_b = fs_b + b_key any_diffs = False for diff in _diff_iter( a_val, b_val, selector_a, selector_b, options, ): if diff.diff_type != DiffTypes.NO_CHANGE: any_diffs = True yield diff if options.moved and a_key != b_key: yield DiffInfo( diff_type=DiffTypes.MOVED, base=fs_a + [a_key], other=fs_b + [b_key], ) elif options.unchanged and not any_diffs: yield DiffInfo( diff_type=DiffTypes.NO_CHANGE, base=fs_a + [a_key], other=fs_b + [b_key], ) if options.unchanged or options.moved: unchanged = values['a'] & values['b'] for pk, seq in unchanged: a_key = rev_keys['a'][pk, seq] b_key = rev_keys['b'][pk, seq] if options.moved and a_key != b_key: yield DiffInfo( diff_type=DiffTypes.MOVED, base=fs_a + [a_key], other=fs_b + [b_key], ) elif options.unchanged: yield DiffInfo( diff_type=DiffTypes.NO_CHANGE, base=fs_a + [a_key], other=fs_b + [b_key], ) if not force_descent: for pk, seq in removed: a_key = rev_keys['a'][pk, seq] selector = fs_a + [a_key] yield DiffInfo( diff_type=DiffTypes.REMOVED, base=selector, other=fs_b, ) for pk, seq in added: b_key = rev_keys['b'][pk, seq] selector = fs_b + [b_key] yield DiffInfo( diff_type=DiffTypes.ADDED, base=fs_a, other=selector, )
def field_selector(self): return FieldSelector(self.cue)