class WorkflowView(object): ''' Provides a 'view' of the overall workflow object that corresponds to a particular level of nesting. That is, this presents the scope within which extension rules operate (i.e. add steps, reference other steps, etc) ''' def __init__(self, workflowobj, offset=''): self.wflow = workflowobj self.dag = workflowobj.dag self.rules = workflowobj.rules self.applied_rules = workflowobj.applied_rules self.offset = offset self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage) self.bookkeeper = JsonPointer(self.offset).resolve(workflowobj.bookkeeping) self.values = JsonPointer(self.offset).resolve(workflowobj.values) def view(self, offset): ''' return a view with an additional offset to this view's offset ''' return self.wflow.view(self._makeoffset(offset)) def query(self, query, collection): ''' :return ''' matches = jsonpath_rw.parse(query).find(collection) return matches def getSteps(self, query): ''' returns steps related to the JSONPath query. if a query points to a stage (say 'stagename'): will return all toplevel steps.. no recursion into subworkflows if a query points to steps (e.g. 'stagename[*]') will return a steps directly ''' nodeids = [] matches = self.query(query, self.steps) for match in matches: value = match.value #step endpoint case if isinstance(value,dict) and '_nodeid' in value: nodeids.append(value['_nodeid']) #stage endpoint case elif isinstance(value,list): for item in value: if '_nodeid' in item: nodeids.append(item['_nodeid']) result = [self.dag.getNode(nodeid) for nodeid in nodeids] return result def _makeoffset(self, offset): ''' prepare a full offset based on this views' offset and a relative offset :param offset: the relative offset ''' thisoffset = JsonPointer(offset) if self.offset: fulloffset = JsonPointer.from_parts(JsonPointer(self.offset).parts + thisoffset.parts).path else: fulloffset = thisoffset.path return fulloffset def getRule(self, name=None, offset='', identifier=None): '''retrieve a rule by offset or name or identifier''' fulloffset = self._makeoffset(offset) for x in self.rules + self.applied_rules: if x.identifier == identifier or (x.offset == fulloffset and x.rule.name == name): return x return None def init(self, initdata, init_provider = None, used_inputs = None, name='init', discover = False, relative = True): ''' initialize this scope by adding an initialization stage. :param inidata: initialization JSON data ''' spec = init_stage_spec( initdata, discover, used_inputs or [], name, relative = relative) self.addRule(JsonStage(spec, init_provider), self.offset) def addRule(self, rule, offset='', identifier = None): ''' add a DAG extension rule, possibly with a scope offset ''' thisoffset = JsonPointer(offset) if offset != '': createIndexData(thisoffset.path, self.steps, self.values) createOffsetMeta(thisoffset.path, self.bookkeeper) offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier = identifier) self.rules += [offsetstage] thisoffset.resolve(self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier] return offsetstage.identifier def addValue(self, key, value): v = self.values.setdefault('_values', {}) if key in v: raise RuntimeError('cannot overwrite value') v[key] = value def getValue(self, key): return self.values.setdefault('_values', {}).get(key) def addStep(self, task, stage, depends_on=None): ''' adds a node to the DAG connecting it to the passed depending nodes while tracking that it was added by the specified stage :param task: the task object for the step :param stage: the stage name :param depends_on: dependencies of this step ''' self.steps.setdefault(stage,[]) node = YadageNode(task.metadata['name'], task, identifier=get_obj_id(task)) node.task.metadata['wflow_node_id'] = node.identifier node.task.metadata['wflow_offset'] = self.offset node.task.metadata['wflow_stage'] = stage node.task.metadata['wflow_stage_node_idx'] = len(self.steps[stage]) node.task.metadata['wflow_hints'] = {'is_purepub': task.pubOnlyTask()} self.dag.addNode(node, depends_on=depends_on) self.steps[stage].append({'_nodeid': node.identifier}) self.bookkeeper['_meta']['steps'] += [node.identifier] log.info('added %s',node) return node def addWorkflow(self, rules, stage=None): ''' add a (sub-)workflow (i.e. list of stages) to the overall workflow ''' offset = '' if stage is not None: #make sure storage for the 'authoring' stage is present and #register the workflow as part of that 'author' #needed e.g. for predicate handlers trying to determing if #the author stage is done nextindex = len(self.steps.get(stage,[])) offset = JsonPointer.from_parts([stage, nextindex]).path self.steps.setdefault(stage,[]).append({}) self.values.setdefault(stage,[]).append({}) for rule in rules: self.addRule(rule, offset)
class WorkflowView(object): ''' Provides a 'view' of the overall workflow object that corresponds to a particular level of nesting. That is, this presents the scope within which extension rules operate (i.e. add steps, reference other steps, etc) ''' def __init__(self, workflowobj, offset=''): self.wflow = workflowobj self.dag = workflowobj.dag self.rules = workflowobj.rules self.applied_rules = workflowobj.applied_rules self.offset = offset self.steps = JsonPointer(self.offset).resolve(workflowobj.stepsbystage) self.bookkeeper = JsonPointer(self.offset).resolve( workflowobj.bookkeeping) self.values = JsonPointer(self.offset).resolve(workflowobj.values) def view(self, offset): ''' return a view with an additional offset to this view's offset ''' return self.wflow.view(self._makeoffset(offset)) def query(self, query, collection): ''' :return ''' matches = jsonpath_rw.parse(query).find(collection) return matches def getSteps(self, query): ''' returns steps related to the JSONPath query. if a query points to a stage (say 'stagename'): will return all toplevel steps.. no recursion into subworkflows if a query points to steps (e.g. 'stagename[*]') will return a steps directly ''' nodeids = [] matches = self.query(query, self.steps) for match in matches: value = match.value #step endpoint case if isinstance(value, dict) and '_nodeid' in value: nodeids.append(value['_nodeid']) #stage endpoint case elif isinstance(value, list): for item in value: if '_nodeid' in item: nodeids.append(item['_nodeid']) result = [self.dag.getNode(nodeid) for nodeid in nodeids] return result def _makeoffset(self, offset): ''' prepare a full offset based on this views' offset and a relative offset :param offset: the relative offset ''' thisoffset = JsonPointer(offset) if self.offset: fulloffset = JsonPointer.from_parts( JsonPointer(self.offset).parts + thisoffset.parts).path else: fulloffset = thisoffset.path return fulloffset def getRule(self, name=None, offset='', identifier=None): '''retrieve a rule by offset or name or identifier''' fulloffset = self._makeoffset(offset) for x in self.rules + self.applied_rules: if x.identifier == identifier or (x.offset == fulloffset and x.rule.name == name): return x return None def init(self, initdata, init_provider=None, used_inputs=None, name='init', discover=False, relative=True): ''' initialize this scope by adding an initialization stage. :param inidata: initialization JSON data ''' spec = init_stage_spec(initdata, discover, used_inputs or [], name, relative=relative) self.addRule(JsonStage(spec, init_provider), self.offset) def addRule(self, rule, offset='', identifier=None): ''' add a DAG extension rule, possibly with a scope offset ''' thisoffset = JsonPointer(offset) if offset != '': createIndexData(thisoffset.path, self.steps, self.values) createOffsetMeta(thisoffset.path, self.bookkeeper) offsetstage = OffsetStage(rule, self._makeoffset(offset), identifier=identifier) self.rules += [offsetstage] thisoffset.resolve( self.bookkeeper)['_meta']['stages'] += [offsetstage.identifier] return offsetstage.identifier def addValue(self, key, value): v = self.values.setdefault('_values', {}) if key in v: raise RuntimeError('cannot overwrite value') v[key] = value def getValue(self, key): return self.values.setdefault('_values', {}).get(key) def addStep(self, task, stage, depends_on=None): ''' adds a node to the DAG connecting it to the passed depending nodes while tracking that it was added by the specified stage :param task: the task object for the step :param stage: the stage name :param depends_on: dependencies of this step ''' self.steps.setdefault(stage, []) node = YadageNode(task.metadata['name'], task, identifier=get_obj_id(task)) node.task.metadata['wflow_node_id'] = node.identifier node.task.metadata['wflow_offset'] = self.offset node.task.metadata['wflow_stage'] = stage node.task.metadata['wflow_stage_node_idx'] = len(self.steps[stage]) node.task.metadata['wflow_hints'] = {'is_purepub': task.pubOnlyTask()} self.dag.addNode(node, depends_on=depends_on) self.steps[stage].append({'_nodeid': node.identifier}) self.bookkeeper['_meta']['steps'] += [node.identifier] log.info('added %s', node) return node def addWorkflow(self, rules, stage=None): ''' add a (sub-)workflow (i.e. list of stages) to the overall workflow ''' offset = '' if stage is not None: #make sure storage for the 'authoring' stage is present and #register the workflow as part of that 'author' #needed e.g. for predicate handlers trying to determing if #the author stage is done nextindex = len(self.steps.get(stage, [])) offset = JsonPointer.from_parts([stage, nextindex]).path self.steps.setdefault(stage, []).append({}) self.values.setdefault(stage, []).append({}) for rule in rules: self.addRule(rule, offset)