def diff(*args, **kwargs):
"""
Deep Diff Commandline
Deep Difference of content in files.
It can read csv, tsv, json, yaml, and toml files.
T1 and T2 are the path to the files to be compared with each other.
"""
debug = kwargs.pop('debug')
kwargs['ignore_private_variables'] = not kwargs.pop(
'include_private_variables')
kwargs['progress_logger'] = logger.info if kwargs[
'progress_logger'] == 'info' else logger.error
create_patch = kwargs.pop('create_patch')
t1_path = kwargs.pop("t1")
t2_path = kwargs.pop("t2")
t1_extension = t1_path.split('.')[-1]
t2_extension = t2_path.split('.')[-1]
for name, t_path, t_extension in [('t1', t1_path, t1_extension),
('t2', t2_path, t2_extension)]:
try:
kwargs[name] = load_path_content(t_path, file_type=t_extension)
except Exception as e: # pragma: no cover.
if debug: # pragma: no cover.
raise # pragma: no cover.
else: # pragma: no cover.
sys.exit(str(
f"Error when loading {name}: {e}")) # pragma: no cover.
# if (t1_extension != t2_extension):
if t1_extension in {'csv', 'tsv'}:
kwargs['t1'] = [dict(i) for i in kwargs['t1']]
if t2_extension in {'csv', 'tsv'}:
kwargs['t2'] = [dict(i) for i in kwargs['t2']]
if create_patch:
# Disabling logging progress since it will leak into stdout
kwargs['log_frequency_in_sec'] = 0
try:
diff = DeepDiff(**kwargs)
except Exception as e: # pragma: no cover. No need to test this.
sys.exit(str(e)) # pragma: no cover. No need to test this.
if create_patch:
try:
delta = Delta(diff)
except Exception as e: # pragma: no cover.
if debug: # pragma: no cover.
raise # pragma: no cover.
else: # pragma: no cover.
sys.exit(f"Error when loading the patch (aka delta): {e}"
) # pragma: no cover.
# printing into stdout
sys.stdout.buffer.write(delta.dumps())
else:
pprint(diff, indent=2)
def test_repeated_timer(self, MockDiffLevel):
t1 = [1, [2]]
t2 = [1, [3]]
progress_logger = mock.Mock()
DeepDiff(t1, t2, log_frequency_in_sec=0.02, progress_logger=progress_logger)
assert PROGRESS_MSG.format(0, 0, 0) == progress_logger.call_args[0][0]
def test_truncate_datetime(self):
d1 = {'a': datetime.datetime(2020, 5, 17, 22, 15, 34, 913070)}
d2 = {'a': datetime.datetime(2020, 5, 17, 22, 15, 39, 296583)}
res = DeepDiff(d1, d2, truncate_datetime='minute')
assert res == {}
res = DeepDiff(d1, d2, truncate_datetime='second')
expected = datetime.datetime(2020, 5, 17, 22, 15, 39, tzinfo=datetime.timezone.utc)
assert res['values_changed']["root['a']"]['new_value'] == expected
d1 = {'a': datetime.time(22, 15, 34, 913070)}
d2 = {'a': datetime.time(22, 15, 39, 296583)}
res = DeepDiff(d1, d2, truncate_datetime='minute')
assert res == {}
res = DeepDiff(d1, d2, truncate_datetime='second')
assert res['values_changed']["root['a']"]['new_value'] == 80139
def test_bool_str(self):
t1 = {'key1': True}
t2 = {'key1': 'Yes'}
diff = DeepDiff(t1, t2, ignore_type_in_groups=[(bool, str)],
ignore_numeric_type_changes=True)
expected = {'values_changed':
{"root['key1']":
{'new_value': 'Yes', 'old_value': True}
}
}
assert diff == expected
def write_migration(schema, counter, path, previous, current):
filename = "%s_%04i.json" % (schema, counter)
migration = DeepDiff(previous, current, verbose_level=2).json
if migration == "{}":
hfoslog('Nothing changed - no new migration data.', lvl=warn)
return
print('Writing migration: ', os.path.join(path, filename))
pprint(migration)
with open(os.path.join(path, filename), 'w') as f:
f.write(migration)
def test_dataobject_generation(source):
### DataObjects cannot be created itself
with pytest.raises(NotImplementedError):
DataObject("nosource")
obj = MinimalObject(source)
assert hasattr(obj, "matrix")
assert hasattr(obj, "string")
assert hasattr(obj, "some_meta")
assert not hasattr(obj, "bogus")
assert (obj.matrix == source).all()
assert obj.string is "TestMeta"
assert obj.some_meta is "nothing"
assert (obj.source == source).all()
assert not DeepDiff(obj.meta, {
"string": "TestMeta",
"some_meta": "nothing"
},
ignore_order=True)
assert not DeepDiff(obj.data, {"matrix": np.array(source)})
def write_migration(schema, counter, path, previous, current):
"""Write out complete migration data"""
filename = "%s_%04i.json" % (schema, counter)
migration = DeepDiff(previous, current,
verbose_level=2).to_json_pickle()
if migration == "{}":
log("Nothing changed - no new migration data.", lvl=warn)
return
log("Writing migration: ", os.path.join(path, filename))
log(migration, pretty=True)
with open(os.path.join(path, filename), "w") as f:
f.write(migration)
def test_cache_purge_level_max(self):
diff = DeepDiff([1], [2], cache_purge_level=1)
assert len(diff.__dict__.keys()) > 10
diff2 = DeepDiff([1], [2], cache_purge_level=2)
assert not diff2.__dict__
expected = {'values_changed': {'root[0]': {'new_value': 2, 'old_value': 1}}}
assert expected == diff2
diff2 = DeepDiff([1], [2], cache_purge_level=2, view='tree')
assert not diff2.__dict__
assert list(diff2.keys()) == ['values_changed']
def process_analysis(self):
logger.debug("Analyzing rule.yml file %s", self.filepath)
logger.debug("Rule name: %s", self.rule_name)
if self.is_added():
msg = "Rule %s added." % self.rule_name
self.diff_struct.add_changed_product_by_rule(self.rule_name,
msg=msg)
self.diff_struct.add_changed_rule(self.rule_name, msg=msg)
return self.diff_struct
elif self.is_removed():
msg = "Rule %s was deleted" % self.rule_name
self.diff_struct.add_rule_log(self.rule_name, msg)
return self.diff_struct
before_start = re.search(r"^\s*template:", self.content_before,
re.MULTILINE)
after_start = re.search(r"^\s*template:", self.content_after,
re.MULTILINE)
# No template section found, no tests selected
if type(before_start) is type(after_start) and before_start is None:
return self.diff_struct
# Template section either added or removed
elif before_start is None or after_start is None:
msg = "Template section has been removed/added in %s" % self.rule_name
self.diff_struct.add_changed_product_by_rule(self.rule_name,
msg=msg)
self.diff_struct.add_changed_rule(self.rule_name, msg=msg)
return self.diff_struct
# Both sections with template section, do diff
content_before = self.content_before[before_start.start():]
content_after = self.content_after[after_start.start():]
diff = DeepDiff(content_before, content_after)
if diff:
msg = "Template section has been changed in %s" % self.rule_name
self.diff_struct.add_changed_product_by_rule(self.rule_name,
msg=msg)
self.diff_struct.add_changed_rule(self.rule_name, msg=msg)
return self.diff_struct
def make_migrations(schema=None):
"""Create migration data for a specified schema"""
entrypoints = {}
old = {}
def _apply_migrations(migrations, new_model):
"""Apply migration data to compile an up to date model"""
def get_path(raw_path):
"""Get local path of schema definition"""
log("RAW PATH:", raw_path, type(raw_path))
path = []
for item in raw_path.split("["):
log(item)
item = item.rstrip("]")
item = item.replace('"', "")
item = item.replace("'", "")
try:
item = int(item)
except ValueError:
pass
path.append(item)
path.remove("root")
log("PATH:", path)
return path
def apply_entry(changetype, change, result):
"""Upgrade with a single migration"""
def apply_removes(removes, result):
"""Delete removed fields"""
for remove in removes:
path = get_path(remove)
amount = dpath.util.delete(result, path)
if amount != 1:
log("Not exactly one removed!", path, remove, lvl=warn)
return result
def apply_additions(additions, result):
"""Add newly added fields"""
for addition in additions:
path = get_path(addition)
entry = additions[addition]
log("Adding:", entry, "at", path)
dpath.util.new(result, path, entry)
return result
if changetype == "type_changes":
log("Creating new object")
result = change["root"]["new_value"]
return result
if changetype == "dictionary_item_added":
log("Adding items")
result = apply_additions(change, result)
elif changetype == "dictionary_item_removed":
log("Removing items")
result = apply_removes(change, result)
elif changetype == "values_changed":
log("Changing items' types")
for item in change:
path = get_path(item)
log(
"Changing",
path,
"from",
change[item]["old_value"],
" to",
change[item]["new_value"],
)
if dpath.util.get(result,
path) != change[item]["old_value"]:
log("Value change did not work!", lvl=warn)
amount = dpath.util.set(result, path,
change[item]["new_value"])
if amount != 1:
log("Not exactly one changed!", path, item, lvl=warn)
return result
def get_renames(migrations):
"""Check migrations for renamed fields"""
log("Checking for rename operations:")
# pprint(migrations)
added = removed = None
for entry in migrations:
added = entry.get("dictionary_item_added", None)
removed = entry.get("dictionary_item_removed", None)
renames = []
if added and removed:
for addition in added:
path = get_path(addition)
for removal in removed:
removed_path = get_path(removal)
if path[:-1] == removed_path[:-1]:
log("Possible rename detected:", removal, "->",
addition)
renames.append((removed_path, path))
return renames
result = {}
for no, migration in enumerate(migrations):
log("Migrating", no)
log("Migration:", migration, lvl=debug)
renamed = get_renames(migrations)
for entry in migration:
result = apply_entry(entry, migration[entry], result)
pprint(result)
return result
def write_migration(schema, counter, path, previous, current):
"""Write out complete migration data"""
filename = "%s_%04i.json" % (schema, counter)
migration = DeepDiff(previous, current,
verbose_level=2).to_json_pickle()
if migration == "{}":
log("Nothing changed - no new migration data.", lvl=warn)
return
log("Writing migration: ", os.path.join(path, filename))
log(migration, pretty=True)
with open(os.path.join(path, filename), "w") as f:
f.write(migration)
for schema_entrypoint in iter_entry_points(group="isomer.schemata",
name=None):
try:
log("Schemata found: ", schema_entrypoint.name, lvl=debug)
if schema is not None and schema_entrypoint.name != schema:
continue
entrypoints[schema_entrypoint.name] = schema_entrypoint
pprint(schema_entrypoint.dist.location)
schema_top = schema_entrypoint.dist.location
schema_migrations = schema_entrypoint.module_name.replace(
"schemata", "migrations").replace(".", "/")
path = os.path.join(schema_top, schema_migrations)
new_model = schema_entrypoint.load()["schema"]
migrations = []
try:
for file in sorted(os.listdir(path)):
if not file.endswith(".json"):
continue
fullpath = os.path.join(path, file)
log("Importing migration", fullpath)
with open(fullpath, "r") as f:
migration = DeepDiff.from_json_pickle(f.read())
migrations.append(migration)
log("Successfully imported")
if len(migrations) == 0:
raise ImportError
pprint(migrations)
model = _apply_migrations(migrations, new_model)
write_migration(schema,
len(migrations) + 1, path, model, new_model)
except ImportError as e:
log("No previous migrations for", schema, e, type(e), exc=True)
if len(migrations) == 0:
write_migration(schema, 1, path, None, new_model)
except (ImportError, DistributionNotFound) as e:
log(
"Problematic schema: ",
e,
type(e),
schema_entrypoint.name,
exc=True,
lvl=warn,
)
log("Found schemata: ", sorted(entrypoints.keys()), lvl=debug)
log("Entrypoints:", entrypoints, pretty=True, lvl=debug)
def make_single_migration(old, new):
pass
def make_migrations(schema=None):
entrypoints = {}
old = {}
def apply_migrations(migrations, new_model):
def get_path(raw_path):
print("RAW PATH:", raw_path, type(raw_path))
path = []
for item in raw_path.split("["):
print(item)
item = item.rstrip("]")
item = item.replace('"', '')
item = item.replace("'", '')
try:
item = int(item)
except ValueError:
pass
path.append(item)
path.remove('root')
print("PATH:", path)
return path
def apply_entry(changetype, change, result):
def apply_removes(removes, result):
for remove in removes:
path = get_path(remove)
amount = dpath.util.delete(result, path)
assert amount == 1
return result
def apply_additions(additions, result):
for addition in additions:
path = get_path(addition)
entry = additions[addition]
hfoslog('Adding:', entry, 'at', path)
dpath.util.new(result, path, entry)
return result
if changetype == 'type_changes':
hfoslog('Creating new object')
result = change['root']['new_value']
return result
if changetype == 'dictionary_item_added':
hfoslog('Adding items')
result = apply_additions(change, result)
elif changetype == 'dictionary_item_removed':
hfoslog('Removing items')
result = apply_removes(change, result)
elif changetype == 'values_changed':
hfoslog("Changing items' types")
for item in change:
path = get_path(item)
hfoslog('Changing', path, 'from',
change[item]['old_value'], ' to',
change[item]['new_value'])
assert dpath.util.get(result, path) == change[item][
'old_value']
amount = dpath.util.set(result, path, change[item][
'new_value'])
assert amount == 1
return result
def get_renames(migrations):
hfoslog('Checking for rename operations:')
pprint(migrations)
for entry in migrations:
added = entry.get('dictionary_item_added', None)
removed = entry.get('dictionary_item_removed', None)
renames = []
if added and removed:
for addition in added:
path = get_path(addition)
for removal in removed:
removed_path = get_path(removal)
if path[:-1] == removed_path[:-1]:
hfoslog('Possible rename detected:', removal, '->',
addition)
renames.append((removed_path, path))
return renames
result = {}
for no, migration in enumerate(migrations):
hfoslog('Migrating', no)
hfoslog('Migration:', migration, lvl=debug)
renamed = get_renames(migrations)
for entry in migration:
result = apply_entry(entry, migration[entry], result)
pprint(result)
return result
def write_migration(schema, counter, path, previous, current):
filename = "%s_%04i.json" % (schema, counter)
migration = DeepDiff(previous, current, verbose_level=2).json
if migration == "{}":
hfoslog('Nothing changed - no new migration data.', lvl=warn)
return
print('Writing migration: ', os.path.join(path, filename))
pprint(migration)
with open(os.path.join(path, filename), 'w') as f:
f.write(migration)
for schema_entrypoint in iter_entry_points(group='hfos.schemata',
name=None):
try:
hfoslog("Schemata found: ", schema_entrypoint.name, lvl=debug,
emitter='DB')
if schema is not None and schema_entrypoint.name != schema:
continue
entrypoints[schema_entrypoint.name] = schema_entrypoint
pprint(schema_entrypoint.dist.location)
schema_top = schema_entrypoint.dist.location
schema_migrations = schema_entrypoint.module_name.replace(
'schemata', 'migrations').replace('.', '/')
path = os.path.join(schema_top, schema_migrations)
new_model = schema_entrypoint.load()['schema']
migrations = []
try:
for file in sorted(os.listdir(path)):
if not file.endswith('.json'):
continue
fullpath = os.path.join(path, file)
hfoslog('Importing migration', fullpath)
with open(fullpath, 'r') as f:
migration = DeepDiff.from_json(f.read())
migrations.append(migration)
hfoslog('Successfully imported')
if len(migrations) == 0:
raise ImportError
pprint(migrations)
model = apply_migrations(migrations, new_model)
write_migration(schema, len(migrations) + 1, path, model,
new_model)
except ImportError as e:
hfoslog('No previous migrations for', schema, e,
type(e), exc=True)
if len(migrations) == 0:
write_migration(schema, 1, path, None, new_model)
except (ImportError, DistributionNotFound) as e:
hfoslog("Problematic schema: ", e, type(e),
schema_entrypoint.name, exc=True, lvl=warn,
emitter='SCHEMATA')
hfoslog("Found schemata: ", sorted(entrypoints.keys()), lvl=debug,
emitter='SCHEMATA')
pprint(entrypoints)
def make_single_migration(old, new):
pass
def test_invalid_cache_purge_level(self):
with pytest.raises(ValueError) as excinfo:
DeepDiff(1, 2, cache_purge_level=5)
assert str(excinfo.value) == PURGE_LEVEL_RANGE_MSG
def test_invalid_verbose_level(self):
with pytest.raises(ValueError) as excinfo:
DeepDiff(1, 2, verbose_level=5)
assert str(excinfo.value) == VERBOSE_LEVEL_RANGE_MSG
def test_invalid_view(self):
t1 = [1]
t2 = [2]
with pytest.raises(ValueError) as excinfo:
DeepDiff(t1, t2, view='blah')
assert str(excinfo.value) == INVALID_VIEW_MSG.format('blah')
def test_path_cache(self):
diff = DeepDiff([1], [2], cache_purge_level=2, view='tree')
path1 = diff['values_changed'][0].path()
path2 = diff['values_changed'][0].path()
assert 'root[0]' == path1 == path2
def make_migrations(schema=None):
entrypoints = {}
old = {}
def apply_migrations(migrations, new_model):
def get_path(raw_path):
print("RAW PATH:", raw_path, type(raw_path))
path = []
for item in raw_path.split("["):
print(item)
item = item.rstrip("]")
item = item.replace('"', '')
item = item.replace("'", '')
try:
item = int(item)
except ValueError:
pass
path.append(item)
path.remove('root')
print("PATH:", path)
return path
def apply_entry(changetype, change, result):
def apply_removes(removes, result):
for remove in removes:
path = get_path(remove)
amount = dpath.util.delete(result, path)
assert amount == 1
return result
def apply_additions(additions, result):
for addition in additions:
path = get_path(addition)
entry = additions[addition]
hfoslog('Adding:', entry, 'at', path)
dpath.util.new(result, path, entry)
return result
if changetype == 'type_changes':
hfoslog('Creating new object')
result = change['root']['new_value']
return result
if changetype == 'dictionary_item_added':
hfoslog('Adding items')
result = apply_additions(change, result)
elif changetype == 'dictionary_item_removed':
hfoslog('Removing items')
result = apply_removes(change, result)
elif changetype == 'values_changed':
hfoslog("Changing items' types")
for item in change:
path = get_path(item)
hfoslog('Changing', path, 'from',
change[item]['old_value'], ' to',
change[item]['new_value'])
assert dpath.util.get(result,
path) == change[item]['old_value']
amount = dpath.util.set(result, path,
change[item]['new_value'])
assert amount == 1
return result
def get_renames(migrations):
hfoslog('Checking for rename operations:')
pprint(migrations)
for entry in migrations:
added = entry.get('dictionary_item_added', None)
removed = entry.get('dictionary_item_removed', None)
renames = []
if added and removed:
for addition in added:
path = get_path(addition)
for removal in removed:
removed_path = get_path(removal)
if path[:-1] == removed_path[:-1]:
hfoslog('Possible rename detected:', removal, '->',
addition)
renames.append((removed_path, path))
return renames
result = {}
for no, migration in enumerate(migrations):
hfoslog('Migrating', no)
hfoslog('Migration:', migration, lvl=debug)
renamed = get_renames(migrations)
for entry in migration:
result = apply_entry(entry, migration[entry], result)
pprint(result)
return result
def write_migration(schema, counter, path, previous, current):
filename = "%s_%04i.json" % (schema, counter)
migration = DeepDiff(previous, current, verbose_level=2).json
if migration == "{}":
hfoslog('Nothing changed - no new migration data.', lvl=warn)
return
print('Writing migration: ', os.path.join(path, filename))
pprint(migration)
with open(os.path.join(path, filename), 'w') as f:
f.write(migration)
for schema_entrypoint in iter_entry_points(group='hfos.schemata',
name=None):
try:
hfoslog("Schemata found: ",
schema_entrypoint.name,
lvl=debug,
emitter='DB')
if schema is not None and schema_entrypoint.name != schema:
continue
entrypoints[schema_entrypoint.name] = schema_entrypoint
pprint(schema_entrypoint.dist.location)
schema_top = schema_entrypoint.dist.location
schema_migrations = schema_entrypoint.module_name.replace(
'schemata', 'migrations').replace('.', '/')
path = os.path.join(schema_top, schema_migrations)
new_model = schema_entrypoint.load()['schema']
migrations = []
try:
for file in sorted(os.listdir(path)):
if not file.endswith('.json'):
continue
fullpath = os.path.join(path, file)
hfoslog('Importing migration', fullpath)
with open(fullpath, 'r') as f:
migration = DeepDiff.from_json(f.read())
migrations.append(migration)
hfoslog('Successfully imported')
if len(migrations) == 0:
raise ImportError
pprint(migrations)
model = apply_migrations(migrations, new_model)
write_migration(schema,
len(migrations) + 1, path, model, new_model)
except ImportError as e:
hfoslog('No previous migrations for',
schema,
e,
type(e),
exc=True)
if len(migrations) == 0:
write_migration(schema, 1, path, None, new_model)
except (ImportError, DistributionNotFound) as e:
hfoslog("Problematic schema: ",
e,
type(e),
schema_entrypoint.name,
exc=True,
lvl=warn,
emitter='SCHEMATA')
hfoslog("Found schemata: ",
sorted(entrypoints.keys()),
lvl=debug,
emitter='SCHEMATA')
pprint(entrypoints)
def make_single_migration(old, new):
pass
def test_get_distance_cache_key(self):
result = DeepDiff._get_distance_cache_key(added_hash=5,
removed_hash=20)
assert b'0x14--0x5dc' == result