def test_contains(self): p1 = JsonPointer("/a/b/c") p2 = JsonPointer("/a/b") p3 = JsonPointer("/b/c") self.assertTrue(p1.contains(p2)) self.assertFalse(p1.contains(p3))
def apply(self, obj): try: from_ptr = JsonPointer(self.operation['from']) except KeyError as ex: raise InvalidJsonPatch( "The operation does not contain a 'from' member") subobj, part = from_ptr.to_last(obj) try: value = subobj[part] except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) if isinstance(subobj, MutableMapping) and \ self.pointer.contains(from_ptr): raise JsonPatchConflict('Cannot move values into its own children') obj = RemoveOperation({ 'op': 'remove', 'path': self.operation['from'] }).apply(obj) obj = AddOperation({ 'op': 'add', 'path': self.location, 'value': value }).apply(obj) return obj
class ComparisonTests(unittest.TestCase): def setUp(self): self.ptr1 = JsonPointer("/a/b/c") self.ptr2 = JsonPointer("/a/b") self.ptr3 = JsonPointer("/b/c") def test_eq_hash(self): p1 = JsonPointer("/something/1/b") p2 = JsonPointer("/something/1/b") p3 = JsonPointer("/something/1.0/b") self.assertEqual(p1, p2) self.assertNotEqual(p1, p3) self.assertNotEqual(p2, p3) self.assertEqual(hash(p1), hash(p2)) self.assertNotEqual(hash(p1), hash(p3)) self.assertNotEqual(hash(p2), hash(p3)) # a pointer compares not-equal to objects of other types self.assertFalse(p1 == "/something/1/b") def test_contains(self): self.assertTrue(self.ptr1.contains(self.ptr2)) self.assertTrue(self.ptr1.contains(self.ptr1)) self.assertFalse(self.ptr1.contains(self.ptr3)) def test_contains_magic(self): self.assertTrue(self.ptr2 in self.ptr1) self.assertTrue(self.ptr1 in self.ptr1) self.assertFalse(self.ptr3 in self.ptr1)
def createOffsetMeta(offset, bookkeeping): ''' sets up a location to track rule and step ids for a given scope offset ''' pointer = JsonPointer(offset) view = bookkeeping for x in pointer.parts: view = view.setdefault(x,{}) pointer.resolve(bookkeeping).setdefault('_meta',{'stages': [], 'steps': []})
def __init__(self, workflowobj, offset=''): self.wflow = workflowobj self.dag = workflowobj.dag self.rules = workflowobj.rules self.applied_rules = workflowobj.applied_rules self.offset = offset self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage) self.bookkeeper = JsonPointer(self.offset).resolve(workflowobj.bookkeeping) self.values = JsonPointer(self.offset).resolve(workflowobj.values)
def add(self, name, value, *ignored_args, **ignored_kwargs): """ Adds a new JSON Pointer expression to the store. """ # Make sure it's valid, no exception in 'resolve' means the expression was valid. pointer = JsonPointer(value) pointer.resolve({}, None) with self.update_lock: self.data[name] = pointer
def apply(self, obj): from_ptr = JsonPointer(self.operation['from']) subobj, part = from_ptr.to_last(obj) value = copy.deepcopy(subobj[part]) obj = AddOperation({ 'op': 'add', 'path': self.location, 'value': value }).apply(obj) return obj
def compare_dicts(path, src, dst): for key in src: if key not in dst: ptr = JsonPointer.from_parts(path + [key]) yield {"op": "remove", "path": ptr.path} continue current = path + [key] for operation in compare_values(current, src[key], dst[key]): yield operation for key in dst: if key not in src: ptr = JsonPointer.from_parts(path + [key]) yield {"op": "add", "path": ptr.path, "value": dst[key]}
def resolve(self, doc, default=jsonpointer._nothing): if self.isHash: if len(self.parts) == 1: refdata = doc else: p = JsonPointer('/' + '/'.join(self.parts[:-1])) refdata = p.resolve(doc) if isinstance(refdata, list): return int(self.parts[-1]) else: return self.parts[-1] else: return super(RelJsonPointer, self).resolve(doc, default)
def addRule(self, rule, offset='', identifier = None): ''' add a DAG extension rule, possibly with a scope offset ''' thisoffset = JsonPointer(offset) if offset != '': createIndexData(thisoffset.path, self.steps, self.values) createOffsetMeta(thisoffset.path, self.bookkeeper) offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier = identifier) self.rules += [offsetstage] thisoffset.resolve(self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier] return offsetstage.identifier
def compare_dicts(path, src, dst): for key in src: if key not in dst: ptr = JsonPointer.from_parts(path + [key]) yield {'op': 'remove', 'path': ptr.path} continue current = path + [key] for operation in compare_values(current, src[key], dst[key]): yield operation for key in dst: if key not in src: ptr = JsonPointer.from_parts(path + [key]) yield {'op': 'add', 'path': ptr.path, 'value': dst[key]}
def apply(self, obj): from_ptr = JsonPointer(self.operation['from']) subobj, part = from_ptr.to_last(obj) try: value = copy.deepcopy(subobj[part]) except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) obj = AddOperation({ 'op': 'add', 'path': self.location, 'value': value }).apply(obj) return obj
def apply(self, obj): try: from_ptr = JsonPointer(self.operation["from"]) except KeyError as ex: raise InvalidJsonPatch("The operation does not contain a 'from' member") subobj, part = from_ptr.to_last(obj) try: value = copy.deepcopy(subobj[part]) except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) obj = AddOperation({"op": "add", "path": self.location, "value": value}).apply(obj) return obj
def _compare_right(path, dst, right, shift): """Yields JSON patch ``add`` operations for elements that are only exists in the `dst` list""" start, end = right if end == -1: end = len(dst) for idx in range(start, end): ptr = JsonPointer.from_parts(path + [str(idx)]) yield ({"op": "add", "path": ptr.path, "value": dst[idx]}, shift + 1) shift += 1
def apply(self, obj): from_ptr = JsonPointer(self.operation['from']) subobj, part = from_ptr.to_last(obj) value = subobj[part] if self.pointer.contains(from_ptr): raise JsonPatchException('Cannot move values into its own children') obj = RemoveOperation({ 'op': 'remove', 'path': self.operation['from'] }).apply(obj) obj = AddOperation({ 'op': 'add', 'path': self.location, 'value': value }).apply(obj) return obj
def compare_values(path, value, other): if value == other: return if isinstance(value, dict) and isinstance(other, dict): for operation in compare_dicts(path, value, other): yield operation elif isinstance(value, list) and isinstance(other, list): for operation in compare_lists(path, value, other): yield operation else: ptr = JsonPointer.from_parts(path) yield {'op': 'replace', 'path': ptr.path, 'value': other}
def compare_values(path, value, other): if value == other: return if isinstance(value, MutableMapping) and isinstance(other, MutableMapping): for operation in compare_dicts(path, value, other): yield operation elif isinstance(value, MutableSequence) and isinstance(other, MutableSequence): for operation in compare_lists(path, value, other): yield operation else: ptr = JsonPointer.from_parts(path) yield {"op": "replace", "path": ptr.path, "value": other}
def _makeoffset(self, offset): ''' prepare a full offset based on this views' offset and a relative offset :param offset: the relative offset ''' thisoffset = JsonPointer(offset) if self.offset: fulloffset = JsonPointer.from_parts(JsonPointer(self.offset).parts + thisoffset.parts).path else: fulloffset = thisoffset.path return fulloffset
def apply(self, obj): from_ptr = JsonPointer(self.operation['from']) subobj, part = from_ptr.to_last(obj) try: value = subobj[part] except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) if isinstance(subobj, dict) and self.pointer.contains(from_ptr): raise JsonPatchException('Cannot move values into its own children') obj = RemoveOperation({ 'op': 'remove', 'path': self.operation['from'] }).apply(obj) obj = AddOperation({ 'op': 'add', 'path': self.location, 'value': value }).apply(obj) return obj
def compare_values(path, value, other): if value == other: return if isinstance(value, MutableMapping) and \ isinstance(other, MutableMapping): for operation in compare_dicts(path, value, other): yield operation elif isinstance(value, MutableSequence) and \ isinstance(other, MutableSequence): for operation in compare_lists(path, value, other): yield operation else: ptr = JsonPointer.from_parts(path) yield {'op': 'replace', 'path': ptr.path, 'value': other}
def apply(self, obj): try: from_ptr = JsonPointer(self.operation["from"]) except KeyError as ex: raise InvalidJsonPatch("The operation does not contain a 'from' member") subobj, part = from_ptr.to_last(obj) try: value = subobj[part] except (KeyError, IndexError) as ex: raise JsonPatchConflict(str(ex)) # If source and target are equal, this is a no-op if self.pointer == from_ptr: return obj if isinstance(subobj, MutableMapping) and self.pointer.contains(from_ptr): raise JsonPatchConflict("Cannot move values into its own children") obj = RemoveOperation({"op": "remove", "path": self.operation["from"]}).apply(obj) obj = AddOperation({"op": "add", "path": self.location, "value": value}).apply(obj) return obj
def addWorkflow(self, rules, stage=None): ''' add a (sub-)workflow (i.e. list of stages) to the overall workflow ''' offset = '' if stage is not None: #make sure storage for the 'authoring' stage is present and #register the workflow as part of that 'author' #needed e.g. for predicate handlers trying to determing if #the author stage is done nextindex = len(self.steps.get(stage,[])) offset = JsonPointer.from_parts([stage, nextindex]).path self.steps.setdefault(stage,[]).append({}) self.values.setdefault(stage,[]).append({}) for rule in rules: self.addRule(rule, offset)
def _compare_left(path, src, left, shift): """Yields JSON patch ``remove`` operations for elements that are only exists in the `src` list.""" start, end = left if end == -1: end = len(src) # we need to `remove` elements from list tail to not deal with index shift for idx in reversed(range(start + shift, end + shift)): ptr = JsonPointer.from_parts(path + [str(idx)]) yield ( {'op': 'remove', # yes, there should be any value field, but we'll use it # to apply `move` optimization a bit later and will remove # it in _optimize function. 'value': src[idx - shift], 'path': ptr.path, }, shift - 1 ) shift -= 1
def test_round_trip(self): paths = [ "", "/foo", "/foo/0", "/", "/a~1b", "/c%d", "/e^f", "/g|h", "/i\\j", "/k\"l", "/ ", "/m~0n", ] for path in paths: ptr = JsonPointer(path) self.assertEqual(path, ptr.path) parts = ptr.parts new_ptr = JsonPointer.from_parts(parts) self.assertEqual(ptr, new_ptr)
def from_key(self, value): from_ptr = JsonPointer(self.operation['from']) from_ptr.parts[-1] = str(value) self.operation['from'] = from_ptr.path
def from_path(self): from_ptr = JsonPointer(self.operation['from']) return '/'.join(from_ptr.parts[:-1])
def from_key(self): from_ptr = JsonPointer(self.operation['from']) try: return int(from_ptr.parts[-1]) except TypeError: return from_ptr.parts[-1]
def setUp(self): self.ptr1 = JsonPointer("/a/b/c") self.ptr2 = JsonPointer("/a/b") self.ptr3 = JsonPointer("/b/c")
class WorkflowView(object): ''' Provides a 'view' of the overall workflow object that corresponds to a particular level of nesting. That is, this presents the scope within which extension rules operate (i.e. add steps, reference other steps, etc) ''' def __init__(self, workflowobj, offset=''): self.wflow = workflowobj self.dag = workflowobj.dag self.rules = workflowobj.rules self.applied_rules = workflowobj.applied_rules self.offset = offset self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage) self.bookkeeper = JsonPointer(self.offset).resolve(workflowobj.bookkeeping) self.values = JsonPointer(self.offset).resolve(workflowobj.values) def view(self, offset): ''' return a view with an additional offset to this view's offset ''' return self.wflow.view(self._makeoffset(offset)) def query(self, query, collection): ''' :return ''' matches = jsonpath_rw.parse(query).find(collection) return matches def getSteps(self, query): ''' returns steps related to the JSONPath query. if a query points to a stage (say 'stagename'): will return all toplevel steps.. no recursion into subworkflows if a query points to steps (e.g. 'stagename[*]') will return a steps directly ''' nodeids = [] matches = self.query(query, self.steps) for match in matches: value = match.value #step endpoint case if isinstance(value,dict) and '_nodeid' in value: nodeids.append(value['_nodeid']) #stage endpoint case elif isinstance(value,list): for item in value: if '_nodeid' in item: nodeids.append(item['_nodeid']) result = [self.dag.getNode(nodeid) for nodeid in nodeids] return result def _makeoffset(self, offset): ''' prepare a full offset based on this views' offset and a relative offset :param offset: the relative offset ''' thisoffset = JsonPointer(offset) if self.offset: fulloffset = JsonPointer.from_parts(JsonPointer(self.offset).parts + thisoffset.parts).path else: fulloffset = thisoffset.path return fulloffset def getRule(self, name=None, offset='', identifier=None): '''retrieve a rule by offset or name or identifier''' fulloffset = self._makeoffset(offset) for x in self.rules + self.applied_rules: if x.identifier == identifier or (x.offset == fulloffset and x.rule.name == name): return x return None def init(self, initdata, init_provider = None, used_inputs = None, name='init', discover = False, relative = True): ''' initialize this scope by adding an initialization stage. :param inidata: initialization JSON data ''' spec = init_stage_spec( initdata, discover, used_inputs or [], name, relative = relative) self.addRule(JsonStage(spec, init_provider), self.offset) def addRule(self, rule, offset='', identifier = None): ''' add a DAG extension rule, possibly with a scope offset ''' thisoffset = JsonPointer(offset) if offset != '': createIndexData(thisoffset.path, self.steps, self.values) createOffsetMeta(thisoffset.path, self.bookkeeper) offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier = identifier) self.rules += [offsetstage] thisoffset.resolve(self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier] return offsetstage.identifier def addValue(self, key, value): v = self.values.setdefault('_values', {}) if key in v: raise RuntimeError('cannot overwrite value') v[key] = value def getValue(self, key): return self.values.setdefault('_values', {}).get(key) def addStep(self, task, stage, depends_on=None): ''' adds a node to the DAG connecting it to the passed depending nodes while tracking that it was added by the specified stage :param task: the task object for the step :param stage: the stage name :param depends_on: dependencies of this step ''' self.steps.setdefault(stage,[]) node = YadageNode(task.metadata['name'], task, identifier=get_obj_id(task)) node.task.metadata['wflow_node_id'] = node.identifier node.task.metadata['wflow_offset'] = self.offset node.task.metadata['wflow_stage'] = stage node.task.metadata['wflow_stage_node_idx'] = len(self.steps[stage]) node.task.metadata['wflow_hints'] = {'is_purepub': task.pubOnlyTask()} self.dag.addNode(node, depends_on=depends_on) self.steps[stage].append({'_nodeid': node.identifier}) self.bookkeeper['_meta']['steps'] += [node.identifier] log.info('added %s',node) return node def addWorkflow(self, rules, stage=None): ''' add a (sub-)workflow (i.e. list of stages) to the overall workflow ''' offset = '' if stage is not None: #make sure storage for the 'authoring' stage is present and #register the workflow as part of that 'author' #needed e.g. for predicate handlers trying to determing if #the author stage is done nextindex = len(self.steps.get(stage,[])) offset = JsonPointer.from_parts([stage, nextindex]).path self.steps.setdefault(stage,[]).append({}) self.values.setdefault(stage,[]).append({}) for rule in rules: self.addRule(rule, offset)
def createIndexData(offset, stepindex, valueindex): pointer = JsonPointer(offset) pointer.resolve(stepindex)['_offset'] = offset pointer.set(valueindex, {})
def __init__(self, operation): self.location = operation['path'] self.pointer = JsonPointer(self.location) self.operation = operation
def test_empty_path(self): doc = {'a': [1, 2, 3]} ptr = JsonPointer('') last, nxt = ptr.to_last(doc) self.assertEqual(doc, last) self.assertTrue(nxt is None)
def test_path(self): doc = {'a': [{'b': 1, 'c': 2}, 5]} ptr = JsonPointer('/a/0/b') last, nxt = ptr.to_last(doc) self.assertEqual(last, {'b': 1, 'c': 2}) self.assertEqual(nxt, 'b')