def merge(self, diff_other, diff_local, baseline, upwards):
# Merge is possible if it does not matter
# in which order we apply the two deltas
build_up_ok = True
try:
one_way = dictdiffer.patch(diff_other, baseline)
one_way = dictdiffer.patch(diff_local, one_way, in_place=True)
other_way = dictdiffer.patch(diff_local, baseline)
other_way = dictdiffer.patch(diff_other, other_way, in_place=True)
except Exception as e:
build_up_ok = False
if build_up_ok and one_way == other_way:
#return one_way
return diff_other
else:
#print('merge failed')
if not upwards:
#return dictdiffer.patch(diff_other, baseline)
#print('PROBLEM')
return list(dictdiffer.diff(self.content,
baseline)) + diff_other
else:
#return self.content
return []
def move_to_commited(log_action, *args, **kwargs):
sl = data.SL()
item = next((i for i in sl if i.log_action == log_action), None)
if item:
resource_obj = resource.load(item.resource)
commited = CommitedResource.get_or_create(item.resource)
updated = resource_obj.db_obj.updated
if item.action == CHANGES.remove.name:
resource_obj.delete()
commited.state = resource.RESOURCE_STATE.removed.name
else:
resource_obj.set_operational()
commited.state = resource.RESOURCE_STATE.operational.name
commited.base_path = item.base_path
updated = resource_obj.db_obj.updated
# required to update `updated` field
resource_obj.db_obj.save()
commited.inputs = patch(item.diff, commited.inputs)
# TODO fix TagsWrp to return list
# commited.tags = resource_obj.tags
sorted_connections = sorted(commited.connections)
commited.connections = patch(item.connections_diff, sorted_connections)
commited.save()
item.log = 'history'
item.state = 'success'
item.updated = updated
item.save()
def handle_conflict(self, to_save, key, local_doc, conflict_callback):
last_good = self.last_known_good.get(key, {})
remote_doc = self.load_specific(key)
last_rev = remote_doc.pop('_rev')
with do_not_track(to_save):
local_doc.pop('_rev', None)
remote_diff = list(dictdiffer.diff(last_good, remote_doc))
local_diff = list(dictdiffer.diff(last_good, local_doc))
one_way = dictdiffer.patch(remote_diff, last_good)
one_way = dictdiffer.patch(local_diff, one_way)
other_way = dictdiffer.patch(local_diff, last_good)
other_way = dictdiffer.patch(remote_diff, other_way)
eprint(one_way, other_way)
if one_way == other_way:
eprint('resolved')
one_way['_rev'] = last_rev
self.db.save(one_way)
with do_not_track(to_save):
to_save[key] = one_way
else:
eprint('no resolve')
remote_doc['_rev'] = last_rev
to_save[key] = remote_doc
def test_add_list(self):
first = {'a': [1]}
second = {'a': [1, 2]}
assert second == patch([('add', 'a', [(1, 2)])], first)
first = {'a': {'b': [1]}}
second = {'a': {'b': [1, 2]}}
assert second == patch([('add', 'a.b', [(1, 2)])], first)
def test_changes(self):
first = {"a": "b"}
second = {"a": "c"}
assert second == patch([("change", "a", ("b", "c"))], first)
first = {"a": {"b": {"c": "d"}}}
second = {"a": {"b": {"c": "e"}}}
assert second == patch([("change", "a.b.c", ("d", "e"))], first)
def test_add_value(self):
original = {'a': 1, 'b': 2}
modified = {'a': 1, 'b': 2, 'c': 3}
delta = list(diff(original, modified))
new = patch(delta, original)
self.assertEqual(new, modified)
new = patch(delta, modified)
self.assertEqual(new, modified)
def test_push(self):
first = {'a': [1]}
second = {'a': [1, 2]}
assert second == patch([('push', 'a', [2])], first)
first = {'a': {'b': [1]}}
second = {'a': {'b': [1, 2]}}
assert second == patch([('push', 'a.b', [2])], first)
def test_addition(self):
first = {}
second = {'a': 'b'}
assert second == patch([('add', '', [('a', 'b')])], first)
first = {'a': {'b': 'c'}}
second = {'a': {'b': 'c', 'd': 'e'}}
assert second == patch([('add', 'a', [('d', 'e')])], first)
def test_addition(self):
first = {}
second = {"a": "b"}
assert second == patch([("add", "", [("a", "b")])], first)
first = {"a": {"b": "c"}}
second = {"a": {"b": "c", "d": "e"}}
assert second == patch([("add", "a", [("d", "e")])], first)
def test_changes(self):
first = {'a': 'b'}
second = {'a': 'c'}
assert second == patch([('change', 'a', ('b', 'c'))], first)
first = {'a': {'b': {'c': 'd'}}}
second = {'a': {'b': {'c': 'e'}}}
assert second == patch([('change', 'a.b.c', ('d', 'e'))], first)
def test_remove(self):
first = {'a': {'b': 'c'}}
second = {'a': {}}
assert second == patch([('remove', 'a', [('b', 'c')])], first)
first = {'a': 'b'}
second = {}
assert second == patch([('remove', '', [('a', 'b')])], first)
def test_remove(self):
first = {"a": {"b": "c"}}
second = {"a": {}}
assert second == patch([("remove", "a", [("b", "c")])], first)
first = {"a": "b"}
second = {}
assert second == patch([("remove", "", [("a", "b")])], first)
def test_add_list(self):
first = {"a": [1]}
second = {"a": [1, 2]}
assert second == patch([("add", "a", [(1, 2)])], first)
first = {"a": {"b": [1]}}
second = {"a": {"b": [1, 2]}}
assert second == patch([("add", "a.b", [(1, 2)])], first)
def test_no_changes(self):
original = {'a': 1, 'b': 2}
modified = {'a': 1, 'b': 2}
delta = list(diff(original, modified))
new = patch(delta, original)
self.assertEqual(new, modified)
new = patch(delta, modified)
self.assertEqual(new, modified)
def test_remove_value(self):
original = {'a': 1, 'b': 2, 'c': 3}
modified = {'a': 1, 'b': 2}
delta = list(diff(original, modified))
new = patch(delta, original)
self.assertEqual(new, modified)
with self.assertRaises(KeyError) as cm:
patch(delta, modified)
self.assertEqual(cm.exception.args[0], 'c')
def test_addition(self):
first = {}
second = {'a': 'b'}
assert second == patch(
[('add', '', [('a', 'b')])], first)
first = {'a': {'b': 'c'}}
second = {'a': {'b': 'c', 'd': 'e'}}
assert second == patch(
[('add', 'a', [('d', 'e')])], first)
def redo(self):
if self.active == 0:
return self.active
self.active -= 1
delta = self[self.active]
with self.handler.root:
self.handler.track = False
dictdiffer.patch(delta, self.handler.root, in_place=True)
self.handler.track = True
return self.active
def test_changes(self):
first = {'a': 'b'}
second = {'a': 'c'}
assert second == patch(
[('change', 'a', ('b', 'c'))], first)
first = {'a': {'b': {'c': 'd'}}}
second = {'a': {'b': {'c': 'e'}}}
assert second == patch(
[('change', 'a.b.c', ('d', 'e'))], first)
def test_push(self):
first = {'a': [1]}
second = {'a': [1, 2]}
assert second == patch(
[('push', 'a', [2])], first)
first = {'a': {'b': [1]}}
second = {'a': {'b': [1, 2]}}
assert second == patch(
[('push', 'a.b', [2])], first)
def test_add_list(self):
first = {'a': [1]}
second = {'a': [1, 2]}
assert second == patch(
[('add', 'a', [(1, 2)])], first)
first = {'a': {'b': [1]}}
second = {'a': {'b': [1, 2]}}
assert second == patch(
[('add', 'a.b', [(1, 2)])], first)
def test_remove(self):
first = {'a': {'b': 'c'}}
second = {'a': {}}
assert second == patch(
[('remove', 'a', [('b', 'c')])], first)
first = {'a': 'b'}
second = {}
assert second == patch(
[('remove', '', [('a', 'b')])], first)
def test_in_place_patch_and_revert(self):
first = {'a': 1}
second = {'a': 2}
changes = list(diff(first, second))
patched_copy = patch(changes, first)
assert first != patched_copy
reverted_in_place = revert(changes, patched_copy, in_place=True)
assert first == reverted_in_place
assert patched_copy == reverted_in_place
patched_in_place = patch(changes, first, in_place=True)
assert first == patched_in_place
def test_in_place_patch_and_revert(self):
first = {'a': 1}
second = {'a': 2}
changes = list(diff(first, second))
patched_copy = patch(changes, first)
assert first != patched_copy
reverted_in_place = revert(changes, patched_copy, in_place=True)
assert first == reverted_in_place
assert patched_copy == reverted_in_place
patched_in_place = patch(changes, first, in_place=True)
assert first == patched_in_place
def test_change_list(self):
first = {"a": ["b"]}
second = {"a": ["c"]}
assert second == patch([("change", "a.0", ("b", "c"))], first)
first = {"a": {"b": {"c": ["d"]}}}
second = {"a": {"b": {"c": ["e"]}}}
assert second == patch([("change", "a.b.c.0", ("d", "e"))], first)
first = {"a": {"b": {"c": [{"d": "e"}]}}}
second = {"a": {"b": {"c": [{"d": "f"}]}}}
assert second == patch([("change", "a.b.c.0.d", ("e", "f"))], first)
def test_change_list(self):
first = {'a': ['b']}
second = {'a': ['c']}
assert second == patch([('change', 'a.0', ('b', 'c'))], first)
first = {'a': {'b': {'c': ['d']}}}
second = {'a': {'b': {'c': ['e']}}}
assert second == patch([('change', 'a.b.c.0', ('d', 'e'))], first)
first = {'a': {'b': {'c': [{'d': 'e'}]}}}
second = {'a': {'b': {'c': [{'d': 'f'}]}}}
assert second == patch([('change', 'a.b.c.0.d', ('e', 'f'))], first)
def test_change_list(self):
first = {'a': ['b']}
second = {'a': ['c']}
assert second == patch(
[('change', 'a.0', ('b', 'c'))], first)
first = {'a': {'b': {'c': ['d']}}}
second = {'a': {'b': {'c': ['e']}}}
assert second == patch(
[('change', 'a.b.c.0', ('d', 'e'))], first)
first = {'a': {'b': {'c': [{'d': 'e'}]}}}
second = {'a': {'b': {'c': [{'d': 'f'}]}}}
assert second == patch(
[('change', 'a.b.c.0.d', ('e', 'f'))], first)
def main():
base_path = os.environ.get('BASE')
local_path = os.environ.get('LOCAL')
remote_path = os.environ.get('REMOTE')
merged_path = os.environ.get('MERGED')
print('BASE: %s' % (base_path,))
print('LOCAL: %s' % (local_path,))
print('REMOTE: %s' % (remote_path,))
print('MERGED: %s' % (merged_path,))
print()
# Ensure paths have been provided
for path in (base_path, local_path, remote_path, merged_path):
if path is None:
print('Missing path for base, local, remote or merged')
sys.exit(1)
return
# Ensure source paths exist
for path in (base_path, local_path, remote_path):
if not os.path.exists(path):
print('File doesn\'t exist at: %s' % (path,))
sys.exit(1)
return
# Parse items
base, _ = read(base_path)
local, unix = read(local_path)
remote, _ = read(remote_path)
# Find changes between base and remote
print('Finding change(s)...')
changes = list(resolve(base, local, remote))
# Apply changes to local
print('Applying %d change(s)...' % (len(changes),))
patch(changes, local, in_place=True)
# Write result to file
print('Writing result (unix: %r)...' % (unix,))
write(
merged_path, local,
unix=unix
)
def _compute_merge_results(refenv, ancestorHEAD, masterHEAD, devHEAD):
'''Compute the diff of a 3-way merge and patch historical contents to get new state
.. warning::
This method is not robust, and will require a require in the future.
Parameters
----------
refenv : lmdb.Environment
db where the commit records are stored
ancestorHEAD : str
commit hash of the common ancestor of dev and master merge branch
masterHEAD : str
commit hash of the merge master branch head
devHEAD : str
commit hash of the merge dev branch head
Returns
-------
dict
nested dict specifying datasets and metadata record specs of the new
merge commit.
'''
a_contents = commiting.get_commit_ref_contents(refenv, ancestorHEAD)
m_contents = commiting.get_commit_ref_contents(refenv, masterHEAD)
d_contents = commiting.get_commit_ref_contents(refenv, devHEAD)
ancestorToDevDiff = dictdiffer.diff(a_contents, d_contents)
patchedMergeContents = dictdiffer.patch(ancestorToDevDiff, m_contents)
return patchedMergeContents
def send_update(self, receiver_id, upwards):
#global debugging
#self.to_file('start send_update')
#with self.lock:
state = self.get_state_for(receiver_id)
#(baseline,
#edits
#) = self.get_values_for(receiver_id)
if debugging:
print(self.conduit.node_id[:8], '->', receiver_id[:8])
#1
#previous_version = int(edits[-1][0].split('-')[0])
add_to_baseline = Sync.collapse_edits(state.edits)
previous_value = dictdiffer.patch(add_to_baseline, state.baseline)
with self.content:
latest_edit = list(dictdiffer.diff(previous_value, self.content))
latest_checksum = Sync.generate_checksum(self.content)
#edits.append((str(previous_version+1)+'-'+str(latest_checksum), latest_edit))
if len(latest_edit) > 0:
state.edits.append((latest_checksum, latest_edit))
if debugging:
print('edits out', state.edits)
#2
message = {
'upwards': upwards,
'edits': copy.deepcopy(state.edits),
}
self.conduit.send_to(receiver_id, message)
def main(argv):
parser = OptionParser(
usage="usage clitool.py %prog -i file -o file.config...",
description="""Make cli file to cmdconfig file , read example file
"""
)
parser.add_option("-i", "--input", dest="input_file", help="Input cli file", metavar="FILE")
parser.add_option("-o", "--output", dest="output_file", help="Output file")
(options, args) = parser.parse_args(argv)
cli_line = load_file(options.input_file)
root = cli_line[0].split()[0]
out_file = clean_file(root, options.output_file)
conf = Shconfig.config(name="", conffile=out_file)
print conf
if conf is None:
print "output file is not correct format"
return
d = conf.dict
main_dict = {root: {}}
for cli in cli_line:
cmd = cli.split()
res = list_to_deep_dict(cmd)
result = diff(main_dict, res, nodel=True)
patched = patch(result, main_dict, set_remove=False)
main_dict = patched
print json.dumps(main_dict, indent=2)
cof = conf.dict
print cof
cof[root] = main_dict[root]
print cof
print conf.save()
def main():
first = {
"title": "hello",
"fork_count": 20,
"stargazers": ["/users/20", "/users/30"],
"settings": {
"assignees": [100, 101, 201],
}
}
second = {
"title": "hellooo",
"fork_count": 20,
"stargazers": ["/users/20", "/users/30", "/users/40"],
"settings": {
"assignees": [100, 101, 202],
}
}
# Calculate the diff
result = diff(first, second) # Return generator (yield)
print(json.dumps(list(result)))
# Apply the diff like a patch
patched = patch(result, first)
print(json.dumps(patched))
print(json.dumps(second))
def _merge_dirs(self, ancestor_info, our_info, their_info):
from operator import itemgetter
from dictdiffer import patch
ancestor = self._to_dict(ancestor_info)
our = self._to_dict(our_info)
their = self._to_dict(their_info)
our_diff = self._diff(ancestor, our)
if not our_diff:
return self._from_dict(their)
their_diff = self._diff(ancestor, their)
if not their_diff:
return self._from_dict(our)
# make sure there are no conflicting files
self._diff(our, their, allow_removed=True)
merged = patch(our_diff + their_diff, ancestor, in_place=True)
# Sorting the list by path to ensure reproducibility
return sorted(self._from_dict(merged),
key=itemgetter(self.tree.PARAM_RELPATH))
def build_document_action_chain(action_cls: Type[BaseDocumentAction],
left_schema: Schema,
right_schema: Schema,
document_types: Iterable[str]) -> Iterable[BaseAction]:
"""
Walk through schema changes, and produce chain of Action objects
of given type which could handle schema changes from left to right
:param action_cls: Action type to consider
:param left_schema:
:param right_schema:
:param document_types: list of document types to inspect
:return: iterable of suitable Action objects
"""
for document_type in document_types:
action_obj = action_cls.build_object(document_type, left_schema, right_schema)
if action_obj is not None:
try:
left_schema = patch(action_obj.to_schema_patch(left_schema), left_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action_obj!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
yield action_obj
def test_continue_run_multiple_conflicts_per_patch(self):
lca = {'foo': [{'x': 1}, {'y': 2}]}
first = {'foo': [{'x': 1}, {'y': 2}, {'z': 4}]}
second = {'bar': 'baz'}
expected = {
'f': {
'foo': [{
'x': 1
}, {
'y': 2
}, {
'z': 4
}],
'bar': 'baz'
},
's': {
'bar': 'baz'
}
}
for resolution, expected_value in expected.items():
m = Merger(lca, first, second, {})
try:
m.run()
except UnresolvedConflictsException as e:
m.continue_run([resolution for _ in e.content])
self.assertEqual(patch(m.unified_patches, lca), expected_value)
def makemigrations(self):
"""
Compare current mongoengine documents state and the last db
state and make a migration file if needed
"""
log.debug('Loading migration files...')
graph = self.build_graph()
log.debug('Loading schema from database...')
db_schema = self.load_db_schema()
# Obtain schema changes which migrations would make (including
# unapplied ones)
# If mongoengine models schema was changed regarding db schema
# then try to guess which actions would reflect such changes
for migration in graph.walk_down(graph.initial, unapplied_only=False):
for action_object in migration.get_actions():
try:
db_schema = patch(action_object.to_schema_patch(db_schema),
db_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action_object!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
log.debug('Collecting schema from mongoengine documents...')
models_schema = collect_models_schema()
if db_schema == models_schema:
log.info('No changes detected')
return
log.debug('Building actions chain...')
actions_chain = build_actions_chain(db_schema, models_schema)
import_expressions = {'from mongoengine_migrate.actions import *'}
for action in actions_chain:
# If `regex` is set in action, then we probably need 're'
if isinstance(action.parameters.get('regex'), re.Pattern):
import_expressions.add('import re')
break
log.debug('Writing migrations file...')
env = Environment()
env.filters['symbol_wrap'] = symbol_wrap
tpl_ctx = {
'graph': graph,
'actions_chain': actions_chain,
'policy_enum': MigrationPolicy,
'import_expressions': import_expressions
}
tpl_path = Path(__file__).parent / 'migration_template.tpl'
tpl = env.from_string(tpl_path.read_text())
migration_source = tpl.render(tpl_ctx)
seq_number = str(len(graph.migrations)).zfill(4)
name = f'{seq_number}_auto_{datetime.now().strftime("%Y%m%d_%H%M")}.py'
migration_file = Path(self.migration_dir) / name
migration_file.write_text(migration_source)
log.info('Migration file "%s" was created', migration_file)
def patch_dict(original_dict: dict, patch_dictionary: dict) -> dict:
"""Patches a dict with another. Patching means that any path defines in the
patch is either added (if it does not exist), or replaces the existing value (if
it exists). Nothing is removed from the original dict, only added/replaced.
Parameters
----------
original_dict : dict
Base dictionary which will get paths added/changed
patch_dictionary: dict
Dictionary which will be overlaid on top of original_dict
Examples
--------
>>> patch_dict({"highKey":{"lowkey1":1, "lowkey2":2}}, {"highKey":{"lowkey1":10}})
{'highKey': {'lowkey1': 10, 'lowkey2': 2}}
>>> patch_dict({"highKey":{"lowkey1":1, "lowkey2":2}}, {"highKey":{"lowkey3":3}})
{'highKey': {'lowkey1': 1, 'lowkey2': 2, 'lowkey3': 3}}
>>> patch_dict({"highKey":{"lowkey1":1, "lowkey2":2}}, {"highKey2":4})
{'highKey': {'lowkey1': 1, 'lowkey2': 2}, 'highKey2': 4}
Returns
-------
dict
A new dictionary which is the result of overlaying `patch_dictionary` on top of
`original_dict`
"""
diff = dictdiffer.diff(original_dict, patch_dictionary)
adds_and_mods = [(f, d, s) for (f, d, s) in diff if f != "remove"]
return dictdiffer.patch(adds_and_mods, original_dict)
def build_actions_chain(left_schema: Schema,
right_schema: Schema) -> Iterable[BaseAction]:
"""
Build full Action objects chain which suitable for such schema
change.
:param left_schema: current schema
:param right_schema: schema collected from mongoengine models
:return: iterable of Action objects
"""
action_chain = []
# Actions registry sorted by priority
registry = list(sorted(actions_registry.values(),
key=lambda x: x.priority))
left_schema = copy(left_schema)
document_types = get_all_document_types(left_schema, right_schema)
for action_cls in registry:
if issubclass(action_cls, BaseDocumentAction):
new_actions = list(
build_document_action_chain(action_cls, left_schema,
right_schema, document_types))
elif issubclass(action_cls, BaseFieldAction):
new_actions = list(
build_field_action_chain(action_cls, left_schema, right_schema,
document_types))
elif issubclass(action_cls, BaseIndexAction):
new_actions = list(
build_index_action_chain(action_cls, left_schema, right_schema,
document_types))
else:
continue
for action in new_actions:
log.debug('> %s', action)
try:
left_schema = patch(action.to_schema_patch(left_schema),
left_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
action_chain.extend(new_actions)
document_types = get_all_document_types(left_schema, right_schema)
if right_schema != left_schema:
log.error(
'Schema is still not reached the target state after applying all actions. '
'Changes left to make (diff): %s',
list(diff(left_schema, right_schema)))
raise ActionError(
'Could not reach target schema state after applying whole Action chain. '
'This could be a problem in some Action which does not process schema '
'properly or produces wrong schema diff. This is a programming error'
)
return action_chain
def test_revert(self):
first = {'a': [1, 2]}
second = {'a': []}
diffed = diff(first, second)
patched = patch(diffed, first)
assert patched == second
diffed = diff(first, second)
reverted = revert(diffed, second)
assert reverted == first
def test_revert(self):
first = {'a': 'b'}
second = {'a': 'c'}
diffed = diff(first, second)
patched = patch(diffed, first)
assert patched == second
diffed = diff(first, second)
reverted = revert(diffed, second)
assert reverted == first
def test_revert(self):
first = {"a": [1, 2]}
second = {"a": []}
diffed = diff(first, second)
patched = patch(diffed, first)
assert patched == second
diffed = diff(first, second)
reverted = revert(diffed, second)
assert reverted == first
def commit_log_item(item):
resource_obj = resource.load(item.resource)
commited = CommitedResource.get_or_create(item.resource)
if item.action == CHANGES.remove.name:
resource_obj.delete()
commited.state = resource.RESOURCE_STATE.removed.name
else:
resource_obj.set_operational()
commited.state = resource.RESOURCE_STATE.operational.name
commited.base_path = item.base_path
resource_obj.db_obj.save_lazy()
commited.inputs = patch(item.diff, commited.inputs)
# TODO fix TagsWrp to return list
# commited.tags = resource_obj.tags
sorted_connections = sorted(commited.connections)
commited.connections = patch(item.connections_diff, sorted_connections)
commited.save_lazy()
item.to_history().save_lazy()
item.delete()
def merge_changes(deposition, dest, a, b):
"""Find changes between two dicts and apply them to a destination dict.
This method is useful when A is a subset of the destination dictionary.
"""
# Generate patch
patch = dictdiffer.diff(a, b)
# Apply patch (returns a deep copy of dest with patch applied)
return dictdiffer.patch(patch, dest)
def move_to_commited(log_action, *args, **kwargs):
sl = data.SL()
item = next((i for i in sl if i.log_action == log_action), None)
sl.pop(item.uid)
if item:
commited = data.CD()
staged_data = patch(item.diff, commited.get(item.res, {}))
cl = data.CL()
item.state = data.STATES.success
cl.append(item)
commited[item.res] = staged_data
def revert(diff_result, destination):
"""
A helper function that calles swap function to revert
patched dictionary object.
>>> first = {'a': 'b'}
>>> second = {'a': 'c'}
>>> revert(diff(first, second), second)
{'a': 'b'}
"""
return patch(swap(diff_result), destination)
def _merge_dicts(self):
self._backup_lists()
non_list_merger = Merger(self.root, self.head, self.update, {})
try:
non_list_merger.run()
except UnresolvedConflictsException as e:
self._solve_dict_conflicts(non_list_merger, e.content)
self._restore_lists()
self.merged_root = patch(
dedupe_list(non_list_merger.unified_patches),
self.root
)
def move_to_commited(log_action, *args, **kwargs):
sl = data.SL()
item = next((i for i in sl if i.log_action == log_action), None)
if item:
sl.pop(item.uid)
resource_obj = resource.load(item.res)
commited = orm.DBCommitedState.get_or_create(item.res)
if item.action == CHANGES.remove.name:
resource_obj.delete()
commited.state = resource.RESOURCE_STATE.removed.name
else:
resource_obj.set_operational()
commited.state = resource.RESOURCE_STATE.operational.name
commited.inputs = patch(item.diff, commited.inputs)
commited.tags = resource_obj.tags
sorted_connections = sorted(commited.connections)
commited.connections = patch(item.signals_diff, sorted_connections)
commited.base_path = item.base_path
commited.save()
cl = data.CL()
item.state = data.STATES.success
cl.append(item)
def merge_with_published(self):
"""Merge changes with latest published version."""
pid, first = self.fetch_published()
lca = first.revisions[self['_deposit']['pid']['revision_id']]
# ignore _deposit and $schema field
args = [lca.dumps(), first.dumps(), self.dumps()]
for arg in args:
del arg['$schema'], arg['_deposit']
args.append({})
m = Merger(*args)
try:
m.run()
except UnresolvedConflictsException:
raise MergeConflict()
return patch(m.unified_patches, lca)
def test_continue_run_multiple_conflicts_per_patch(self):
lca = {'foo': [{'x': 1}, {'y': 2}]}
first = {'foo': [{'x': 1}, {'y': 2}, {'z': 4}]}
second = {'bar': 'baz'}
expected = {
'f': {'foo': [{'x': 1}, {'y': 2}, {'z': 4}],
'bar': 'baz'},
's': {'bar': 'baz'}}
for resolution, expected_value in expected.items():
m = Merger(lca, first, second, {})
try:
m.run()
except UnresolvedConflictsException as e:
m.continue_run([resolution for _ in e.content])
self.assertEqual(patch(m.unified_patches, lca),
expected_value)
def test_dict_int_key(self):
first = {0: 0}
second = {0: 'a'}
first_patch = [('change', [0], (0, 'a'))]
assert second == patch(first_patch, first)
def test_pull(self):
first = {'a': [1, 2, 3]}
second = {'a': [1, 2]}
assert second == patch(
[('pull', 'a', [3])], first)
def test_remove_list(self):
first = {'a': [1, 2, 3]}
second = {'a': [1, ]}
assert second == patch(
[('remove', 'a', [(2, 3), (1, 2), ]), ], first)
def _update(obj, eng):
import dictdiffer
from lxml import objectify, etree
from invenio.base.globals import cfg
from invenio_workflows.utils import convert_marcxml_to_bibfield
from invenio_records.api import Record
from inspire.utils.robotupload import make_robotupload_marcxml
try:
recid = obj.extra_data["recid"]
except KeyError:
obj.log.error("Cannot locate record ID")
return
callback_url = os.path.join(cfg["CFG_SITE_URL"],
"callback/workflows/continue")
search_url = "%s?p=recid:%s&of=xm" % (cfg["WORKFLOWS_MATCH_REMOTE_SERVER_URL"], recid)
prod_data = objectify.parse(search_url)
# remove controlfields
root = prod_data.getroot()
record = root['record']
while True:
try:
record.remove(record['controlfield'])
except AttributeError:
break
prod_data = etree.tostring(record)
prod_data = convert_marcxml_to_bibfield(prod_data, model=["hep"])
new_data = dict(obj.data.dumps(clean=True))
prod_data = dict(prod_data.dumps(clean=True))
updated_keys = []
diff = dictdiffer.diff(prod_data, new_data)
for diff_type, new_key, content in diff:
if diff_type == 'add':
if new_key:
if isinstance(new_key, list):
# ['subject_terms', 0]
updated_keys.append(new_key[0])
else:
# 'subject_terms'
updated_keys.append(new_key)
else:
# content must be list of new adds
for key in content:
updated_keys.append(key)
updates = dictdiffer.patch(diff, new_data)
for key in updates.keys():
if key not in updated_keys:
del updates[key]
if updates:
updates['recid'] = recid
marcxml = Record(updates).legacy_export_as_marc()
result = make_robotupload_marcxml(
url=url,
marcxml=marcxml,
callback_url=callback_url,
mode='correct',
nonce=obj.id
)
if "[INFO]" not in result.text:
if "cannot use the service" in result.text:
# IP not in the list
obj.log.error("Your IP is not in "
"CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS "
"on host")
obj.log.error(result.text)
from invenio_workflows.errors import WorkflowError
txt = "Error while submitting robotupload: {0}".format(result.text)
raise WorkflowError(txt, eng.uuid, obj.id)
else:
obj.log.info("Robotupload sent!")
obj.log.info(result.text)
eng.halt("Waiting for robotupload: {0}".format(result.text))
obj.log.info("end of upload")
else:
obj.log.info("No updates to do.")
def test_remove_set(self):
first = {'a': set([1, 2, 3])}
second = {'a': set([1])}
assert second == patch(
[('remove', 'a', [(0, set([2, 3]))])], first)
def test_add_set(self):
first = {'a': set([1])}
second = {'a': set([1, 2])}
assert second == patch(
[('add', 'a', [(0, set([2]))])], first)
def test_dict_combined_key_type(self):
first = {0: {'1': {2: 3}}}
second = {0: {'1': {2: '3'}}}
first_patch = [('change', [0, '1', 2], (3, '3'))]
assert second == patch(first_patch, first)
assert first_patch[0] == list(diff(first, second))[0]
def test_verify_patch_creates_expected(staged, diff_for_update, commited):
expected = patch(diff_for_update, commited)
assert expected == staged
