def test_contains(self):
p1 = JsonPointer("/a/b/c")
p2 = JsonPointer("/a/b")
p3 = JsonPointer("/b/c")
self.assertTrue(p1.contains(p2))
self.assertFalse(p1.contains(p3))
def apply(self, obj):
try:
from_ptr = JsonPointer(self.operation['from'])
except KeyError as ex:
raise InvalidJsonPatch(
"The operation does not contain a 'from' member")
subobj, part = from_ptr.to_last(obj)
try:
value = subobj[part]
except (KeyError, IndexError) as ex:
raise JsonPatchConflict(str(ex))
if isinstance(subobj, MutableMapping) and \
self.pointer.contains(from_ptr):
raise JsonPatchConflict('Cannot move values into its own children')
obj = RemoveOperation({
'op': 'remove',
'path': self.operation['from']
}).apply(obj)
obj = AddOperation({
'op': 'add',
'path': self.location,
'value': value
}).apply(obj)
return obj
class ComparisonTests(unittest.TestCase):
def setUp(self):
self.ptr1 = JsonPointer("/a/b/c")
self.ptr2 = JsonPointer("/a/b")
self.ptr3 = JsonPointer("/b/c")
def test_eq_hash(self):
p1 = JsonPointer("/something/1/b")
p2 = JsonPointer("/something/1/b")
p3 = JsonPointer("/something/1.0/b")
self.assertEqual(p1, p2)
self.assertNotEqual(p1, p3)
self.assertNotEqual(p2, p3)
self.assertEqual(hash(p1), hash(p2))
self.assertNotEqual(hash(p1), hash(p3))
self.assertNotEqual(hash(p2), hash(p3))
# a pointer compares not-equal to objects of other types
self.assertFalse(p1 == "/something/1/b")
def test_contains(self):
self.assertTrue(self.ptr1.contains(self.ptr2))
self.assertTrue(self.ptr1.contains(self.ptr1))
self.assertFalse(self.ptr1.contains(self.ptr3))
def test_contains_magic(self):
self.assertTrue(self.ptr2 in self.ptr1)
self.assertTrue(self.ptr1 in self.ptr1)
self.assertFalse(self.ptr3 in self.ptr1)
def createOffsetMeta(offset, bookkeeping):
'''
sets up a location to track rule and step ids for a given scope offset
'''
pointer = JsonPointer(offset)
view = bookkeeping
for x in pointer.parts:
view = view.setdefault(x,{})
pointer.resolve(bookkeeping).setdefault('_meta',{'stages': [], 'steps': []})
def __init__(self, workflowobj, offset=''):
self.wflow = workflowobj
self.dag = workflowobj.dag
self.rules = workflowobj.rules
self.applied_rules = workflowobj.applied_rules
self.offset = offset
self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage)
self.bookkeeper = JsonPointer(self.offset).resolve(workflowobj.bookkeeping)
self.values = JsonPointer(self.offset).resolve(workflowobj.values)
def add(self, name, value, *ignored_args, **ignored_kwargs):
""" Adds a new JSON Pointer expression to the store.
"""
# Make sure it's valid, no exception in 'resolve' means the expression was valid.
pointer = JsonPointer(value)
pointer.resolve({}, None)
with self.update_lock:
self.data[name] = pointer
def apply(self, obj):
from_ptr = JsonPointer(self.operation['from'])
subobj, part = from_ptr.to_last(obj)
value = copy.deepcopy(subobj[part])
obj = AddOperation({
'op': 'add',
'path': self.location,
'value': value
}).apply(obj)
return obj
def compare_dicts(path, src, dst):
for key in src:
if key not in dst:
ptr = JsonPointer.from_parts(path + [key])
yield {"op": "remove", "path": ptr.path}
continue
current = path + [key]
for operation in compare_values(current, src[key], dst[key]):
yield operation
for key in dst:
if key not in src:
ptr = JsonPointer.from_parts(path + [key])
yield {"op": "add", "path": ptr.path, "value": dst[key]}
def resolve(self, doc, default=jsonpointer._nothing):
if self.isHash:
if len(self.parts) == 1:
refdata = doc
else:
p = JsonPointer('/' + '/'.join(self.parts[:-1]))
refdata = p.resolve(doc)
if isinstance(refdata, list):
return int(self.parts[-1])
else:
return self.parts[-1]
else:
return super(RelJsonPointer, self).resolve(doc, default)
def addRule(self, rule, offset='', identifier = None):
'''
add a DAG extension rule, possibly with a scope offset
'''
thisoffset = JsonPointer(offset)
if offset != '':
createIndexData(thisoffset.path, self.steps, self.values)
createOffsetMeta(thisoffset.path, self.bookkeeper)
offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier = identifier)
self.rules += [offsetstage]
thisoffset.resolve(self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier]
return offsetstage.identifier
def compare_dicts(path, src, dst):
for key in src:
if key not in dst:
ptr = JsonPointer.from_parts(path + [key])
yield {'op': 'remove', 'path': ptr.path}
continue
current = path + [key]
for operation in compare_values(current, src[key], dst[key]):
yield operation
for key in dst:
if key not in src:
ptr = JsonPointer.from_parts(path + [key])
yield {'op': 'add',
'path': ptr.path,
'value': dst[key]}
def apply(self, obj):
from_ptr = JsonPointer(self.operation['from'])
subobj, part = from_ptr.to_last(obj)
try:
value = copy.deepcopy(subobj[part])
except (KeyError, IndexError) as ex:
raise JsonPatchConflict(str(ex))
obj = AddOperation({
'op': 'add',
'path': self.location,
'value': value
}).apply(obj)
return obj
def apply(self, obj):
try:
from_ptr = JsonPointer(self.operation["from"])
except KeyError as ex:
raise InvalidJsonPatch("The operation does not contain a 'from' member")
subobj, part = from_ptr.to_last(obj)
try:
value = copy.deepcopy(subobj[part])
except (KeyError, IndexError) as ex:
raise JsonPatchConflict(str(ex))
obj = AddOperation({"op": "add", "path": self.location, "value": value}).apply(obj)
return obj
def _compare_right(path, dst, right, shift):
"""Yields JSON patch ``add`` operations for elements that are only
exists in the `dst` list"""
start, end = right
if end == -1:
end = len(dst)
for idx in range(start, end):
ptr = JsonPointer.from_parts(path + [str(idx)])
yield ({"op": "add", "path": ptr.path, "value": dst[idx]}, shift + 1)
shift += 1
def apply(self, obj):
from_ptr = JsonPointer(self.operation['from'])
subobj, part = from_ptr.to_last(obj)
value = subobj[part]
if self.pointer.contains(from_ptr):
raise JsonPatchException('Cannot move values into its own children')
obj = RemoveOperation({
'op': 'remove',
'path': self.operation['from']
}).apply(obj)
obj = AddOperation({
'op': 'add',
'path': self.location,
'value': value
}).apply(obj)
return obj
def compare_values(path, value, other):
if value == other:
return
if isinstance(value, dict) and isinstance(other, dict):
for operation in compare_dicts(path, value, other):
yield operation
elif isinstance(value, list) and isinstance(other, list):
for operation in compare_lists(path, value, other):
yield operation
else:
ptr = JsonPointer.from_parts(path)
yield {'op': 'replace', 'path': ptr.path, 'value': other}
def compare_values(path, value, other):
if value == other:
return
if isinstance(value, MutableMapping) and isinstance(other, MutableMapping):
for operation in compare_dicts(path, value, other):
yield operation
elif isinstance(value, MutableSequence) and isinstance(other, MutableSequence):
for operation in compare_lists(path, value, other):
yield operation
else:
ptr = JsonPointer.from_parts(path)
yield {"op": "replace", "path": ptr.path, "value": other}
def _makeoffset(self, offset):
'''
prepare a full offset based on this views' offset and a relative offset
:param offset: the relative offset
'''
thisoffset = JsonPointer(offset)
if self.offset:
fulloffset = JsonPointer.from_parts(JsonPointer(self.offset).parts + thisoffset.parts).path
else:
fulloffset = thisoffset.path
return fulloffset
def apply(self, obj):
from_ptr = JsonPointer(self.operation['from'])
subobj, part = from_ptr.to_last(obj)
try:
value = subobj[part]
except (KeyError, IndexError) as ex:
raise JsonPatchConflict(str(ex))
if isinstance(subobj, dict) and self.pointer.contains(from_ptr):
raise JsonPatchException('Cannot move values into its own children')
obj = RemoveOperation({
'op': 'remove',
'path': self.operation['from']
}).apply(obj)
obj = AddOperation({
'op': 'add',
'path': self.location,
'value': value
}).apply(obj)
return obj
def compare_values(path, value, other):
if value == other:
return
if isinstance(value, MutableMapping) and \
isinstance(other, MutableMapping):
for operation in compare_dicts(path, value, other):
yield operation
elif isinstance(value, MutableSequence) and \
isinstance(other, MutableSequence):
for operation in compare_lists(path, value, other):
yield operation
else:
ptr = JsonPointer.from_parts(path)
yield {'op': 'replace', 'path': ptr.path, 'value': other}
def apply(self, obj):
try:
from_ptr = JsonPointer(self.operation["from"])
except KeyError as ex:
raise InvalidJsonPatch("The operation does not contain a 'from' member")
subobj, part = from_ptr.to_last(obj)
try:
value = subobj[part]
except (KeyError, IndexError) as ex:
raise JsonPatchConflict(str(ex))
# If source and target are equal, this is a no-op
if self.pointer == from_ptr:
return obj
if isinstance(subobj, MutableMapping) and self.pointer.contains(from_ptr):
raise JsonPatchConflict("Cannot move values into its own children")
obj = RemoveOperation({"op": "remove", "path": self.operation["from"]}).apply(obj)
obj = AddOperation({"op": "add", "path": self.location, "value": value}).apply(obj)
return obj
def addWorkflow(self, rules, stage=None):
'''
add a (sub-)workflow (i.e. list of stages) to the overall workflow
'''
offset = ''
if stage is not None:
#make sure storage for the 'authoring' stage is present and
#register the workflow as part of that 'author'
#needed e.g. for predicate handlers trying to determing if
#the author stage is done
nextindex = len(self.steps.get(stage,[]))
offset = JsonPointer.from_parts([stage, nextindex]).path
self.steps.setdefault(stage,[]).append({})
self.values.setdefault(stage,[]).append({})
for rule in rules:
self.addRule(rule, offset)
def _compare_left(path, src, left, shift):
"""Yields JSON patch ``remove`` operations for elements that are only
exists in the `src` list."""
start, end = left
if end == -1:
end = len(src)
# we need to `remove` elements from list tail to not deal with index shift
for idx in reversed(range(start + shift, end + shift)):
ptr = JsonPointer.from_parts(path + [str(idx)])
yield (
{'op': 'remove',
# yes, there should be any value field, but we'll use it
# to apply `move` optimization a bit later and will remove
# it in _optimize function.
'value': src[idx - shift],
'path': ptr.path,
},
shift - 1
)
shift -= 1
def test_round_trip(self):
paths = [
"",
"/foo",
"/foo/0",
"/",
"/a~1b",
"/c%d",
"/e^f",
"/g|h",
"/i\\j",
"/k\"l",
"/ ",
"/m~0n",
]
for path in paths:
ptr = JsonPointer(path)
self.assertEqual(path, ptr.path)
parts = ptr.parts
new_ptr = JsonPointer.from_parts(parts)
self.assertEqual(ptr, new_ptr)
def from_key(self, value):
from_ptr = JsonPointer(self.operation['from'])
from_ptr.parts[-1] = str(value)
self.operation['from'] = from_ptr.path
def from_path(self):
from_ptr = JsonPointer(self.operation['from'])
return '/'.join(from_ptr.parts[:-1])
def from_key(self):
from_ptr = JsonPointer(self.operation['from'])
try:
return int(from_ptr.parts[-1])
except TypeError:
return from_ptr.parts[-1]
def setUp(self):
self.ptr1 = JsonPointer("/a/b/c")
self.ptr2 = JsonPointer("/a/b")
self.ptr3 = JsonPointer("/b/c")
def from_key(self, value):
from_ptr = JsonPointer(self.operation['from'])
from_ptr.parts[-1] = str(value)
self.operation['from'] = from_ptr.path
class WorkflowView(object):
'''
Provides a 'view' of the overall workflow object that corresponds
to a particular level of nesting. That is, this presents the scope
within which extension rules operate (i.e. add steps, reference
other steps, etc)
'''
def __init__(self, workflowobj, offset=''):
self.wflow = workflowobj
self.dag = workflowobj.dag
self.rules = workflowobj.rules
self.applied_rules = workflowobj.applied_rules
self.offset = offset
self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage)
self.bookkeeper = JsonPointer(self.offset).resolve(workflowobj.bookkeeping)
self.values = JsonPointer(self.offset).resolve(workflowobj.values)
def view(self, offset):
'''
return a view with an additional offset to this view's offset
'''
return self.wflow.view(self._makeoffset(offset))
def query(self, query, collection):
'''
:return
'''
matches = jsonpath_rw.parse(query).find(collection)
return matches
def getSteps(self, query):
'''
returns steps related to the JSONPath query.
if a query points to a stage (say 'stagename'):
will return all toplevel steps.. no recursion into subworkflows
if a query points to steps (e.g. 'stagename[*]')
will return a steps directly
'''
nodeids = []
matches = self.query(query, self.steps)
for match in matches:
value = match.value
#step endpoint case
if isinstance(value,dict) and '_nodeid' in value:
nodeids.append(value['_nodeid'])
#stage endpoint case
elif isinstance(value,list):
for item in value:
if '_nodeid' in item:
nodeids.append(item['_nodeid'])
result = [self.dag.getNode(nodeid) for nodeid in nodeids]
return result
def _makeoffset(self, offset):
'''
prepare a full offset based on this views' offset and a relative offset
:param offset: the relative offset
'''
thisoffset = JsonPointer(offset)
if self.offset:
fulloffset = JsonPointer.from_parts(JsonPointer(self.offset).parts + thisoffset.parts).path
else:
fulloffset = thisoffset.path
return fulloffset
def getRule(self, name=None, offset='', identifier=None):
'''retrieve a rule by offset or name or identifier'''
fulloffset = self._makeoffset(offset)
for x in self.rules + self.applied_rules:
if x.identifier == identifier or (x.offset == fulloffset and x.rule.name == name):
return x
return None
def init(self, initdata, init_provider = None, used_inputs = None, name='init', discover = False, relative = True):
'''
initialize this scope by adding an initialization stage.
:param inidata: initialization JSON data
'''
spec = init_stage_spec(
initdata, discover, used_inputs or [], name, relative = relative)
self.addRule(JsonStage(spec, init_provider), self.offset)
def addRule(self, rule, offset='', identifier = None):
'''
add a DAG extension rule, possibly with a scope offset
'''
thisoffset = JsonPointer(offset)
if offset != '':
createIndexData(thisoffset.path, self.steps, self.values)
createOffsetMeta(thisoffset.path, self.bookkeeper)
offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier = identifier)
self.rules += [offsetstage]
thisoffset.resolve(self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier]
return offsetstage.identifier
def addValue(self, key, value):
v = self.values.setdefault('_values', {})
if key in v:
raise RuntimeError('cannot overwrite value')
v[key] = value
def getValue(self, key):
return self.values.setdefault('_values', {}).get(key)
def addStep(self, task, stage, depends_on=None):
'''
adds a node to the DAG connecting it to the passed depending nodes
while tracking that it was added by the specified stage
:param task: the task object for the step
:param stage: the stage name
:param depends_on: dependencies of this step
'''
self.steps.setdefault(stage,[])
node = YadageNode(task.metadata['name'], task, identifier=get_obj_id(task))
node.task.metadata['wflow_node_id'] = node.identifier
node.task.metadata['wflow_offset'] = self.offset
node.task.metadata['wflow_stage'] = stage
node.task.metadata['wflow_stage_node_idx'] = len(self.steps[stage])
node.task.metadata['wflow_hints'] = {'is_purepub': task.pubOnlyTask()}
self.dag.addNode(node, depends_on=depends_on)
self.steps[stage].append({'_nodeid': node.identifier})
self.bookkeeper['_meta']['steps'] += [node.identifier]
log.info('added %s',node)
return node
def addWorkflow(self, rules, stage=None):
'''
add a (sub-)workflow (i.e. list of stages) to the overall workflow
'''
offset = ''
if stage is not None:
#make sure storage for the 'authoring' stage is present and
#register the workflow as part of that 'author'
#needed e.g. for predicate handlers trying to determing if
#the author stage is done
nextindex = len(self.steps.get(stage,[]))
offset = JsonPointer.from_parts([stage, nextindex]).path
self.steps.setdefault(stage,[]).append({})
self.values.setdefault(stage,[]).append({})
for rule in rules:
self.addRule(rule, offset)
def createIndexData(offset, stepindex, valueindex):
pointer = JsonPointer(offset)
pointer.resolve(stepindex)['_offset'] = offset
pointer.set(valueindex, {})
def __init__(self, operation):
self.location = operation['path']
self.pointer = JsonPointer(self.location)
self.operation = operation
def test_empty_path(self):
doc = {'a': [1, 2, 3]}
ptr = JsonPointer('')
last, nxt = ptr.to_last(doc)
self.assertEqual(doc, last)
self.assertTrue(nxt is None)
def test_path(self):
doc = {'a': [{'b': 1, 'c': 2}, 5]}
ptr = JsonPointer('/a/0/b')
last, nxt = ptr.to_last(doc)
self.assertEqual(last, {'b': 1, 'c': 2})
self.assertEqual(nxt, 'b')
