def get_wf_steps(self):
     wf_steps = self.steps
     steps_dict = CommentedMap()
     for step in wf_steps:
         individual_step_dict = step.to_dict_with_id_key()
         steps_dict.update(individual_step_dict)
     return steps_dict
 def get_wf_inputs(self):
     wf_inputs = self.inputs
     inputs_dict = CommentedMap()
     for input in wf_inputs:
         single_input_dict = input.to_dict_with_id_key()
         inputs_dict.update(single_input_dict)
     return inputs_dict
 def get_wf_outputs(self):
     wf_outputs = self.outputs
     outputs_dict = CommentedMap()
     for output in wf_outputs:
         individual_output_dict = output.to_dict_with_id_key(
         )  # output is instance of WorkflowOutputParameter
         outputs_dict.update(individual_output_dict)
     return outputs_dict
Ejemplo n.º 4
0
    def create(self, args, tccConfigObject):
        result = self.check(args)
        if not result:
            sys.exit()
        for attribute in self.mandatoryAttributes:
            setattr(self,attribute,getattr(args,attribute))
            if self.mandatoryAttributes[attribute] == 'ref':
                refElementCategory = element_lookup[attribute]
                refElement = tcc.get(refElementCategory,getattr(args, attribute))
                back_refs = refElement.back_refs
                back_refs.append(args.name)
                for element in tccConfigObject[refElementCategory]:
                    if element['name'] == refElement.name:
                        element['back_refs'] = back_refs
        if not tccConfigObject:
            tccConfigObject = CommentedMap([(self.elementCategory, None)])
        elif self.elementCategory not in tccConfigObject:
            tccConfigObject.update(CommentedMap([(self.elementCategory, None)]))
        if not tccConfigObject[self.elementCategory]:
            tccConfigObject[self.elementCategory] = []
        attributes = copy.deepcopy(self.mandatoryAttributes)
        for element in self.mandatoryAttributes:
            attributes[element] = getattr(self, element)
        if hasattr(self,'back_refs'):
            attributes['back_refs'] = self.back_refs
        del self.mandatoryAttributes

        self.mgmtNetmask = tccConfigObject['Network']['mgmt']['netmask']
        self.mgmtGateway = tccConfigObject['Network']['mgmt']['gateway']
        self.mgmtDns = tccConfigObject['Network']['mgmt']['dns']
        self.vxlanNetmask = tccConfigObject['Network']['vxlan']['netmask']
        createFunction = getattr(self, self.createMethod)
        result = createFunction()
        if 'Error' in result:
            print result['Error']
            return False
        if args.type == 'service':
            terminalDict = { attributes['terminal']:self.Id }
            attributes['terminal'] = []
            attributes['terminal'].append(terminalDict)
        else:
            if hasattr(self,'Id'):
                attributes['Id'] = self.Id
        tccConfigObject[self.elementCategory].append(attributes)
        self.updateYaml(tccConfigObject)
        return self
Ejemplo n.º 5
0
    def create(self, args, tccConfigObject):
        result = self.check(args)
        if not result:
            sys.exit()
        for attribute in self.mandatoryAttributes:
            setattr(self, attribute, getattr(args, attribute))
            if self.mandatoryAttributes[attribute] == "ref":
                refElementCategory = element_lookup[attribute]
                refElement = tcc.get(refElementCategory, getattr(args, attribute))
                back_refs = refElement.back_refs
                back_refs.append(args.name)
                for element in tccConfigObject[refElementCategory]:
                    if element["name"] == refElement.name:
                        element["back_refs"] = back_refs
        if not tccConfigObject:
            tccConfigObject = CommentedMap([(self.elementCategory, None)])
        elif self.elementCategory not in tccConfigObject:
            tccConfigObject.update(CommentedMap([(self.elementCategory, None)]))
        if not tccConfigObject[self.elementCategory]:
            tccConfigObject[self.elementCategory] = []
        attributes = copy.deepcopy(self.mandatoryAttributes)
        for element in self.mandatoryAttributes:
            attributes[element] = getattr(self, element)
        if hasattr(self, "back_refs"):
            attributes["back_refs"] = self.back_refs
        del self.mandatoryAttributes

        self.mgmtNetmask = tccConfigObject["Network"]["mgmt"]["netmask"]
        self.mgmtGateway = tccConfigObject["Network"]["mgmt"]["gateway"]
        self.mgmtDns = tccConfigObject["Network"]["mgmt"]["dns"]
        self.vxlanNetmask = tccConfigObject["Network"]["vxlan"]["netmask"]
        createFunction = getattr(self, self.createMethod)
        result = createFunction()
        if "Error" in result:
            print result["Error"]
            return False
        if args.type == "service":
            terminalDict = {attributes["terminal"]: self.Id}
            attributes["terminal"] = []
            attributes["terminal"].append(terminalDict)
        else:
            if hasattr(self, "Id"):
                attributes["Id"] = self.Id
        tccConfigObject[self.elementCategory].append(attributes)
        self.updateYaml(tccConfigObject)
        return self
Ejemplo n.º 6
0
 def _commit_head(self, tag: AnyStr = None, comment: AnyStr = None):
     """
     apply the modifications on curr_data to the underling opened version
     and create a new tag
     """
     commented_map = CommentedMap()
     commented_map.update(self._curr_data or self.data)
     if tag:
         self._raw[tag] = commented_map
         self._raw['__current'] = tag
     else:
         new_tag = self._make_tag()
         self._raw.insert(2, new_tag, commented_map, comment=comment)
         self._raw['__current'] = new_tag
     self._curr_version = None
     self._curr_data = None
     return self
Ejemplo n.º 7
0
def save_task(task):
    """
    Convert dictionary suitable for serializing as yaml
      or creating a Changeset.

    .. code-block:: YAML

      changeId:
      target:
      implementation:
      inputs:
      changes:
      dependencies:
      messages:
      outputs:
      result:  # an object or "skipped"
    """
    output = CommentedMap()
    output["changeId"] = task.changeId
    if task.previousId:
        output["previousId"] = task.previousId
    if task.target:
        output["target"] = task.target.key
    save_status(task, output)
    output["implementation"] = save_config_spec(task.configSpec)
    if task._resolved_inputs:  # only serialize resolved inputs
        output["inputs"] = serialize_value(task._resolved_inputs)
    changes = save_resource_changes(task._resourceChanges)
    if changes:
        output["changes"] = changes
    if task.messages:
        output["messages"] = task.messages
    dependencies = [save_dependency(val) for val in task.dependencies]
    if dependencies:
        output["dependencies"] = dependencies
    if task.result:
        if task.result.outputs:
            output["outputs"] = save_result(task.result.outputs)
        if task.result.result:
            output["result"] = save_result(task.result.result)
    else:
        output["result"] = "skipped"
    output.update(task.configurator.save_digest(task))
    output["summary"] = task.summary()
    return output
Ejemplo n.º 8
0
def test_section_yaml_obj():
    commented_map = CommentedMap(
        {"section1": {
            "subsection1": {
                "subsection2": "item"
            }
        }})
    sec = Section("MAIN_SECTION", parent_yaml=commented_map)
    assert sec.yaml_obj == commented_map.update({"MAIN_SECTION": None})
Ejemplo n.º 9
0
    def create(self, args, tccConfigObject):
        result = self.check(args)
        if not result:
            sys.exit()
        for attribute in self.mandatoryAttributes:
            setattr(self,attribute,getattr(args,attribute))
            if self.mandatoryAttributes[attribute] == 'ref':
                refElementCategory = element_lookup[attribute]
                refElement = tcc.get(refElementCategory,getattr(args, attribute))
                back_refs = refElement.back_refs
                back_refs.append(args.name)
                for element in tccConfigObject[refElementCategory]:
                    if element['name'] == refElement.name:
                        element['back_refs'] = back_refs
        if not tccConfigObject:
            tccConfigObject = CommentedMap([(self.elementCategory, None)])
        elif self.elementCategory not in tccConfigObject:
            tccConfigObject.update(CommentedMap([(self.elementCategory, None)]))
        if not tccConfigObject[self.elementCategory]:
            tccConfigObject[self.elementCategory] = []
        attributes = copy.deepcopy(self.mandatoryAttributes)
        for element in self.mandatoryAttributes:
            attributes[element] = getattr(self, element)
        if hasattr(self,'back_refs'):
            attributes['back_refs'] = self.back_refs
        del self.mandatoryAttributes

        self.mgmtNetmask = tccConfigObject['Network']['mgmt']['netmask']
        self.mgmtGateway = tccConfigObject['Network']['mgmt']['gateway']
        self.mgmtDns = tccConfigObject['Network']['mgmt']['dns']
        self.vxlanNetmask = tccConfigObject['Network']['vxlan']['netmask']
        createFunction = getattr(self, self.createMethod)
        result = createFunction()

        if 'Error' in result:
            print result['Error']
            return False
        if hasattr(self,'Id'):
            attributes['Id'] = self.Id
        tccConfigObject[self.elementCategory].append(attributes)
        self.updateYaml(tccConfigObject)
        return self
Ejemplo n.º 10
0
def render(container_names, version=3):
    struct = {}
    networks = []

    for cname in container_names:
        cfile, networks = _generate(cname)
        struct.update(cfile)

    with io.StringIO() as buffer:
        if version == 1:
            y.dump(CommentedMap(struct), buffer)
        elif version == 3:
            d = CommentedMap({'version': '3', 'services': struct})

            if networks:
                d.update({'networks': networks})

            y.dump(d, buffer)
        else:
            raise NotImplementedError(
                'Not implemented for version {}'.format(version))

        return buffer.getvalue()
Ejemplo n.º 11
0
def workflow_clean(document):  # type: (MutableMapping[Text, Any]) -> None
    """Transform draft-3 style Workflows to more idiomatic v1.0"""
    input_output_clean(document)
    hints_and_requirements_clean(document)
    outputs = document['outputs']
    for output_id in outputs:
        outputs[output_id]["outputSource"] = \
            outputs[output_id].pop("source").lstrip('#').replace(".", "/")
    new_steps = CommentedMap()
    for step in document["steps"]:
        new_step = CommentedMap()
        new_step.update(step)
        step = new_step
        step_id = step.pop("id")
        step_id_len = len(step_id)+1
        step["out"] = []
        for outp in step["outputs"]:
            clean_outp_id = outp["id"]
            if clean_outp_id.startswith(step_id):
                clean_outp_id = clean_outp_id[step_id_len:]
            step["out"].append(clean_outp_id)
        del step["outputs"]
        ins = CommentedMap()
        for inp in step["inputs"]:
            ident = inp["id"]
            if ident.startswith(step_id):
                ident = ident[step_id_len:]
            if 'source' in inp:
                inp["source"] = inp["source"].lstrip('#').replace(".", "/")
            del inp["id"]
            if len(inp) > 1:
                ins[ident] = inp
            elif len(inp) == 1:
                if "source" in inp:
                    ins[ident] = inp.popitem()[1]
                else:
                    ins[ident] = inp
            else:
                ins[ident] = {}
        step["in"] = ins
        del step["inputs"]
        if "scatter" in step:
            if isinstance(step["scatter"], (str, Text)) == 1:
                source = step["scatter"]
                if source.startswith(step_id):
                    source = source[step_id_len:]
                step["scatter"] = source
            elif isinstance(step["scatter"], list) and len(step["scatter"]) > 1:
                step["scatter"] = []
                for source in step["scatter"]:
                    if source.startswith(step_id):
                        source = source[step_id_len:]
                    step["scatter"].append(source)
            else:
                source = step["scatter"][0]
                if source.startswith(step_id):
                    source = source[step_id_len:]
                step["scatter"] = source
        if "description" in step:
            step["doc"] = step.pop("description")
        new_steps[step_id.lstrip('#')] = step
    document["steps"] = new_steps
Ejemplo n.º 12
0
    def __init__(self,
                 toolpath_object,      # type: Dict[Text, Any]
                 pos,                  # type: int
                 loadingContext,       # type: LoadingContext
                 parentworkflowProv=None  # type: Optional[CreateProvProfile]
                ):  # type: (...) -> None
        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step" + Text(pos)

        loadingContext = loadingContext.copy()

        loadingContext.requirements = (getdefault(loadingContext.requirements, []) +
                                  toolpath_object.get("requirements", []) +
                                  get_overrides(getdefault(loadingContext.overrides_list, []),
                                                self.id).get("requirements", []))
        loadingContext.hints = getdefault(loadingContext.hints, []) + toolpath_object.get("hints", [])

        try:
            if isinstance(toolpath_object["run"], dict):
                self.embedded_tool = loadingContext.construct_tool_object(toolpath_object["run"], loadingContext)
            else:
                self.embedded_tool = load_tool(
                    toolpath_object["run"], loadingContext)
        except validate.ValidationException as vexc:
            if loadingContext.debug:
                _logger.exception("Validation exception")
            raise WorkflowException(
                u"Tool definition %s failed validation:\n%s" %
                (toolpath_object["run"], validate.indent(str(vexc))))

        validation_errors = []
        self.tool = toolpath_object = copy.deepcopy(toolpath_object)
        bound = set()
        for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
            toolpath_object[toolfield] = []
            for index, step_entry in enumerate(toolpath_object[stepfield]):
                if isinstance(step_entry, string_types):
                    param = CommentedMap()  # type: CommentedMap
                    inputid = step_entry
                else:
                    param = CommentedMap(six.iteritems(step_entry))
                    inputid = step_entry["id"]

                shortinputid = shortname(inputid)
                found = False
                for tool_entry in self.embedded_tool.tool[toolfield]:
                    frag = shortname(tool_entry["id"])
                    if frag == shortinputid:
                        #if the case that the step has a default for a parameter,
                        #we do not want the default of the tool to override it
                        step_default = None
                        if "default" in param and "default" in tool_entry:
                            step_default = param["default"]
                        param.update(tool_entry)
                        param["_tool_entry"] = tool_entry
                        if step_default is not None:
                            param["default"] = step_default
                        found = True
                        bound.add(frag)
                        break
                if not found:
                    if stepfield == "in":
                        param["type"] = "Any"
                        param["not_connected"] = True
                    else:
                        validation_errors.append(
                            SourceLine(self.tool["out"], index).makeError(
                                "Workflow step output '%s' does not correspond to"
                                % shortname(step_entry))
                            + "\n" + SourceLine(self.embedded_tool.tool, "outputs").makeError(
                                "  tool output (expected '%s')" % (
                                    "', '".join(
                                        [shortname(tool_entry["id"]) for tool_entry in
                                         self.embedded_tool.tool[toolfield]]))))
                param["id"] = inputid
                param.lc.line = toolpath_object[stepfield].lc.data[index][0]
                param.lc.col = toolpath_object[stepfield].lc.data[index][1]
                param.lc.filename = toolpath_object[stepfield].lc.filename
                toolpath_object[toolfield].append(param)

        missing = []
        for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
            if shortname(tool_entry["id"]) not in bound:
                if "null" not in tool_entry["type"] and "default" not in tool_entry:
                    missing.append(shortname(tool_entry["id"]))

        if missing:
            validation_errors.append(SourceLine(self.tool, "in").makeError(
                "Step is missing required parameter%s '%s'" %
                ("s" if len(missing) > 1 else "", "', '".join(missing))))

        if validation_errors:
            raise validate.ValidationException("\n".join(validation_errors))

        super(WorkflowStep, self).__init__(toolpath_object, loadingContext)

        if self.embedded_tool.tool["class"] == "Workflow":
            (feature, _) = self.get_requirement("SubworkflowFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains embedded workflow but "
                    "SubworkflowFeatureRequirement not in requirements")

        if "scatter" in self.tool:
            (feature, _) = self.get_requirement("ScatterFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains scatter but ScatterFeatureRequirement "
                    "not in requirements")

            inputparms = copy.deepcopy(self.tool["inputs"])
            outputparms = copy.deepcopy(self.tool["outputs"])
            scatter = aslist(self.tool["scatter"])

            method = self.tool.get("scatterMethod")
            if method is None and len(scatter) != 1:
                raise validate.ValidationException(
                    "Must specify scatterMethod when scattering over multiple inputs")

            inp_map = {i["id"]: i for i in inputparms}
            for inp in scatter:
                if inp not in inp_map:
                    raise validate.ValidationException(
                        SourceLine(self.tool, "scatter").makeError(
                            "Scatter parameter '%s' does not correspond to "
                            "an input parameter of this step, expecting '%s'"
                            % (shortname(inp), "', '".join(
                                shortname(k) for k in inp_map.keys()))))

                inp_map[inp]["type"] = {"type": "array", "items": inp_map[inp]["type"]}

            if self.tool.get("scatterMethod") == "nested_crossproduct":
                nesting = len(scatter)
            else:
                nesting = 1

            for index in range(0, nesting):
                for oparam in outputparms:
                    oparam["type"] = {"type": "array", "items": oparam["type"]}
            self.tool["inputs"] = inputparms
            self.tool["outputs"] = outputparms
        self.prov_obj = None  # type: Optional[CreateProvProfile]
        if loadingContext.research_obj:
            self.prov_obj = parentworkflowProv
            if self.embedded_tool.tool["class"] == "Workflow":
                self.parent_wf = self.embedded_tool.parent_wf
            else:
                self.parent_wf = self.prov_obj
Ejemplo n.º 13
0
    def __init__(self, toolpath_object, pos, **kwargs):
        # type: (Dict[Text, Any], int, **Any) -> None
        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step" + Text(pos)

        kwargs["requirements"] = kwargs.get("requirements", []) + toolpath_object.get("requirements", [])
        kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get("hints", [])

        try:
            if isinstance(toolpath_object["run"], dict):
                self.embedded_tool = kwargs.get("makeTool")(toolpath_object["run"], **kwargs)
            else:
                self.embedded_tool = load_tool(
                    toolpath_object["run"], kwargs.get("makeTool"), kwargs,
                    enable_dev=kwargs.get("enable_dev"),
                    strict=kwargs.get("strict"),
                    fetcher_constructor=kwargs.get("fetcher_constructor"))
        except validate.ValidationException as v:
            raise WorkflowException(
                u"Tool definition %s failed validation:\n%s" %
                (toolpath_object["run"], validate.indent(str(v))))

        validation_errors = []
        self.tool = toolpath_object = copy.deepcopy(toolpath_object)
        bound = set()
        for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
            toolpath_object[toolfield] = []
            for n, step_entry in enumerate(toolpath_object[stepfield]):
                if isinstance(step_entry, (str, unicode)):
                    param = CommentedMap()  # type: CommentedMap
                    inputid = step_entry
                else:
                    param = CommentedMap(step_entry.iteritems())
                    inputid = step_entry["id"]

                shortinputid = shortname(inputid)
                found = False
                for tool_entry in self.embedded_tool.tool[toolfield]:
                    frag = shortname(tool_entry["id"])
                    if frag == shortinputid:
                        param.update(tool_entry)  # type: ignore
                        found = True
                        bound.add(frag)
                        break
                if not found:
                    if stepfield == "in":
                        param["type"] = "Any"
                    else:
                        validation_errors.append(
                            SourceLine(self.tool["out"], n).makeError(
                                "Workflow step output '%s' does not correspond to" % shortname(step_entry))
                            + "\n" + SourceLine(self.embedded_tool.tool, "outputs").makeError(
                                "  tool output (expected '%s')" % (
                                    "', '".join(
                                        [shortname(tool_entry["id"]) for tool_entry in
                                         self.embedded_tool.tool[toolfield]]))))
                param["id"] = inputid
                param.lc.line = toolpath_object[stepfield].lc.data[n][0]
                param.lc.col = toolpath_object[stepfield].lc.data[n][1]
                param.lc.filename = toolpath_object[stepfield].lc.filename
                toolpath_object[toolfield].append(param)

        missing = []
        for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
            if shortname(tool_entry["id"]) not in bound:
                if "null" not in tool_entry["type"] and "default" not in tool_entry:
                    missing.append(shortname(tool_entry["id"]))

        if missing:
            validation_errors.append(SourceLine(self.tool, "in").makeError(
                "Step is missing required parameter%s '%s'" % ("s" if len(missing) > 1 else "", "', '".join(missing))))

        if validation_errors:
            raise validate.ValidationException("\n".join(validation_errors))

        super(WorkflowStep, self).__init__(toolpath_object, **kwargs)

        if self.embedded_tool.tool["class"] == "Workflow":
            (feature, _) = self.get_requirement("SubworkflowFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements")

        if "scatter" in self.tool:
            (feature, _) = self.get_requirement("ScatterFeatureRequirement")
            if not feature:
                raise WorkflowException("Workflow contains scatter but ScatterFeatureRequirement not in requirements")

            inputparms = copy.deepcopy(self.tool["inputs"])
            outputparms = copy.deepcopy(self.tool["outputs"])
            scatter = aslist(self.tool["scatter"])

            method = self.tool.get("scatterMethod")
            if method is None and len(scatter) != 1:
                raise validate.ValidationException("Must specify scatterMethod when scattering over multiple inputs")

            inp_map = {i["id"]: i for i in inputparms}
            for s in scatter:
                if s not in inp_map:
                    raise validate.ValidationException(
                        SourceLine(self.tool, "scatter").makeError(u"Scatter parameter '%s' does not correspond to an input parameter of this "
                                                                   u"step, expecting '%s'" % (shortname(s), "', '".join(shortname(k) for k in inp_map.keys()))))

                inp_map[s]["type"] = {"type": "array", "items": inp_map[s]["type"]}

            if self.tool.get("scatterMethod") == "nested_crossproduct":
                nesting = len(scatter)
            else:
                nesting = 1

            for r in xrange(0, nesting):
                for op in outputparms:
                    op["type"] = {"type": "array", "items": op["type"]}
            self.tool["inputs"] = inputparms
            self.tool["outputs"] = outputparms
Ejemplo n.º 14
0
    def __init__(
        self,
        toolpath_object: CommentedMap,
        pos: int,
        loadingContext: LoadingContext,
        parentworkflowProv: Optional[ProvenanceProfile] = None,
    ) -> None:
        """Initialize this WorkflowStep."""
        debug = loadingContext.debug
        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step" + str(pos)

        loadingContext = loadingContext.copy()

        loadingContext.requirements = copy.deepcopy(
            getdefault(loadingContext.requirements, []))
        assert loadingContext.requirements is not None  # nosec
        loadingContext.requirements.extend(
            toolpath_object.get("requirements", []))
        loadingContext.requirements.extend(
            cast(
                List[CWLObjectType],
                get_overrides(getdefault(loadingContext.overrides_list, []),
                              self.id).get("requirements", []),
            ))

        hints = copy.deepcopy(getdefault(loadingContext.hints, []))
        hints.extend(toolpath_object.get("hints", []))
        loadingContext.hints = hints

        try:
            if isinstance(toolpath_object["run"], CommentedMap):
                self.embedded_tool = loadingContext.construct_tool_object(
                    toolpath_object["run"], loadingContext)  # type: Process
            else:
                loadingContext.metadata = {}
                self.embedded_tool = load_tool(toolpath_object["run"],
                                               loadingContext)
        except ValidationException as vexc:
            if loadingContext.debug:
                _logger.exception("Validation exception")
            raise WorkflowException(
                "Tool definition %s failed validation:\n%s" %
                (toolpath_object["run"], indent(str(vexc)))) from vexc

        validation_errors = []
        self.tool = toolpath_object = copy.deepcopy(toolpath_object)
        bound = set()

        if self.embedded_tool.get_requirement("SchemaDefRequirement")[0]:
            if "requirements" not in toolpath_object:
                toolpath_object["requirements"] = []
            toolpath_object["requirements"].append(
                self.embedded_tool.get_requirement("SchemaDefRequirement")[0])

        for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
            toolpath_object[toolfield] = []
            for index, step_entry in enumerate(toolpath_object[stepfield]):
                if isinstance(step_entry, str):
                    param = CommentedMap()  # type: CommentedMap
                    inputid = step_entry
                else:
                    param = CommentedMap(step_entry.items())
                    inputid = step_entry["id"]

                shortinputid = shortname(inputid)
                found = False
                for tool_entry in self.embedded_tool.tool[toolfield]:
                    frag = shortname(tool_entry["id"])
                    if frag == shortinputid:
                        # if the case that the step has a default for a parameter,
                        # we do not want the default of the tool to override it
                        step_default = None
                        if "default" in param and "default" in tool_entry:
                            step_default = param["default"]
                        param.update(tool_entry)
                        param["_tool_entry"] = tool_entry
                        if step_default is not None:
                            param["default"] = step_default
                        found = True
                        bound.add(frag)
                        break
                if not found:
                    if stepfield == "in":
                        param["type"] = "Any"
                        param["used_by_step"] = used_by_step(
                            self.tool, shortinputid)
                        param["not_connected"] = True
                    else:
                        if isinstance(step_entry, Mapping):
                            step_entry_name = step_entry["id"]
                        else:
                            step_entry_name = step_entry
                        validation_errors.append(
                            SourceLine(self.tool["out"],
                                       index,
                                       include_traceback=debug).
                            makeError(
                                "Workflow step output '%s' does not correspond to"
                                % shortname(step_entry_name)) + "\n" +
                            SourceLine(
                                self.embedded_tool.tool,
                                "outputs",
                                include_traceback=debug,
                            ).makeError("  tool output (expected '%s')" %
                                        ("', '".join([
                                            shortname(tool_entry["id"])
                                            for tool_entry in
                                            self.embedded_tool.tool["outputs"]
                                        ]))))
                param["id"] = inputid
                param.lc.line = toolpath_object[stepfield].lc.data[index][0]
                param.lc.col = toolpath_object[stepfield].lc.data[index][1]
                param.lc.filename = toolpath_object[stepfield].lc.filename
                toolpath_object[toolfield].append(param)

        missing_values = []
        for _, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
            if shortname(tool_entry["id"]) not in bound:
                if "null" not in tool_entry[
                        "type"] and "default" not in tool_entry:
                    missing_values.append(shortname(tool_entry["id"]))

        if missing_values:
            validation_errors.append(
                SourceLine(self.tool, "in", include_traceback=debug).makeError(
                    "Step is missing required parameter%s '%s'" % (
                        "s" if len(missing_values) > 1 else "",
                        "', '".join(missing_values),
                    )))

        if validation_errors:
            raise ValidationException("\n".join(validation_errors))

        super().__init__(toolpath_object, loadingContext)

        if self.embedded_tool.tool["class"] == "Workflow":
            (feature,
             _) = self.get_requirement("SubworkflowFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains embedded workflow but "
                    "SubworkflowFeatureRequirement not in requirements")

        if "scatter" in self.tool:
            (feature, _) = self.get_requirement("ScatterFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains scatter but ScatterFeatureRequirement "
                    "not in requirements")

            inputparms = copy.deepcopy(self.tool["inputs"])
            outputparms = copy.deepcopy(self.tool["outputs"])
            scatter = aslist(self.tool["scatter"])

            method = self.tool.get("scatterMethod")
            if method is None and len(scatter) != 1:
                raise ValidationException(
                    "Must specify scatterMethod when scattering over multiple inputs"
                )

            inp_map = {i["id"]: i for i in inputparms}
            for inp in scatter:
                if inp not in inp_map:
                    SourceLine(
                        self.tool, "scatter", ValidationException,
                        debug).makeError(
                            "Scatter parameter '%s' does not correspond to "
                            "an input parameter of this step, expecting '%s'" %
                            (
                                shortname(inp),
                                "', '".join(
                                    shortname(k) for k in inp_map.keys()),
                            ))

                inp_map[inp]["type"] = {
                    "type": "array",
                    "items": inp_map[inp]["type"]
                }

            if self.tool.get("scatterMethod") == "nested_crossproduct":
                nesting = len(scatter)
            else:
                nesting = 1

            for _ in range(0, nesting):
                for oparam in outputparms:
                    oparam["type"] = {"type": "array", "items": oparam["type"]}
            self.tool["inputs"] = inputparms
            self.tool["outputs"] = outputparms
        self.prov_obj = None  # type: Optional[ProvenanceProfile]
        if loadingContext.research_obj is not None:
            self.prov_obj = parentworkflowProv
            if self.embedded_tool.tool["class"] == "Workflow":
                self.parent_wf = self.embedded_tool.parent_wf
            else:
                self.parent_wf = self.prov_obj
Ejemplo n.º 15
0
 def dump(self):
     map_object = CommentedMap()
     map_object['id'] = getattr(self, 'id')
     map_object.update(self.io_object.dump())
     return map_object
Ejemplo n.º 16
0
def _to_doc(d: DatasetDoc, with_formatting: bool):
    if with_formatting:
        doc = CommentedMap()
        doc.yaml_set_comment_before_after_key("$schema", before="Dataset")
    else:
        doc = {}

    doc["$schema"] = ODC_DATASET_SCHEMA_URL
    doc.update(
        attr.asdict(
            d,
            recurse=True,
            dict_factory=CommentedMap if with_formatting else dict,
            # Exclude fields that are the default.
            filter=lambda attr, value: "doc_exclude" not in attr.metadata and
            value != attr.default
            # Exclude any fields set to None. The distinction should never matter in our docs.
            and value is not None,
            retain_collection_types=False,
        ))

    # Sort properties for readability.
    # PyCharm '19 misunderstands the type of a `sorted(dict.items())`
    # noinspection PyTypeChecker
    doc["properties"] = CommentedMap(
        sorted(doc["properties"].items(), key=_stac_key_order))

    if d.geometry is not None:
        doc["geometry"] = shapely.geometry.mapping(d.geometry)
    doc["id"] = str(d.id)

    if with_formatting:
        if "geometry" in doc:
            # Set some numeric fields to be compact yaml format.
            _use_compact_format(doc["geometry"], "coordinates")
        if "grids" in doc:
            for grid in doc["grids"].values():
                _use_compact_format(grid, "shape", "transform")

        # Add user-readable names for measurements as a comment if present.
        if d.measurements:
            for band_name, band_doc in d.measurements.items():
                if band_doc.alias and band_name.lower(
                ) != band_doc.alias.lower():
                    doc["measurements"].yaml_add_eol_comment(
                        band_doc.alias, band_name)

        _add_space_before(
            doc,
            "label" if "label" in doc else "id",
            "crs",
            "properties",
            "measurements",
            "accessories",
            "lineage",
        )

        p: CommentedMap = doc["properties"]
        p.yaml_add_eol_comment("# Ground sample distance (m)", "eo:gsd")

    return doc
Ejemplo n.º 17
0
class MVYaml(object):
    protected_keys = (
        '__current',
        '__type',
    )

    def __init__(self, base64=False):
        self._b64 = base64
        self._raw = CommentedMap()
        self._yaml = YAML()
        self._curr_version = None
        self._curr_data = None
        self._create()

    def _create(self):
        tag = self._make_tag()
        self._raw[tag] = CommentedMap()
        self._raw.insert(0, '__current', tag, 'current version')
        self._raw.insert(1, '__type', None, 'base64 if value are base64')
        self._commit(tag=tag, comment='Initial version')

    def import_yaml(self, file: AnyStr = None, stream: AnyStr = None):
        data = None
        if file:
            with open(file, 'r') as fp:
                data = fp.read()
        imported_data = self._yaml.load(data or stream)
        self.override(imported_data)
        return self

    def load(self, file_handler: AnyStr = None, stream_data: AnyStr = None):
        data = None
        if file_handler:
            with open(file_handler, 'r') as fp:
                data = fp.read()
        self._raw = self._yaml.load(data or stream_data)
        if self.protected_keys not in self._raw.keys():
            raise MVYamlFileException(
                f'Not a valid mvyaml file. Perhaps is a yaml you want to import with '
                f'import_yaml()?')
        return self

    def write(self,
              file_handler: IO = None,
              comment: AnyStr = None) -> [AnyStr, None]:
        if not self._raw:
            return
        if self._has_changes():
            self._commit(comment=comment)
        output = file_handler or StringIO()
        self._yaml.dump(self._raw, output)
        return output.getvalue() if not file_handler else None

    @property
    def versions(self):
        if not self._raw:
            return []
        return [k for k in self._raw.keys() if k not in self.protected_keys]

    @property
    def current(self):
        return self._raw['__current']

    @property
    def data(self):
        if not self._curr_data:
            self._curr_data = deepcopy(self._raw[self._curr_version
                                                 or self.current])
        return self._curr_data

    def with_version(self, version: str = '__current'):
        if version not in self.versions:
            raise MVYamlVersionNotFoundException(
                f'version {version} not found')
        self._curr_version = version
        self._curr_data = None
        return self

    @staticmethod
    def _make_tag() -> str:
        d = datetime.utcnow().isoformat()
        return d

    def override(self, data: [Iterable]):
        self._curr_data = CommentedMap()
        self._curr_data.update(data)
        self._commit(comment='Overridden')
        return self

    def _commit(self, *args, **kwargs):
        return self._commit_head(*args, **kwargs)

    def _commit_head(self, tag: AnyStr = None, comment: AnyStr = None):
        """
        apply the modifications on curr_data to the underling opened version
        and create a new tag
        """
        commented_map = CommentedMap()
        commented_map.update(self._curr_data or self.data)
        if tag:
            self._raw[tag] = commented_map
            self._raw['__current'] = tag
        else:
            new_tag = self._make_tag()
            self._raw.insert(2, new_tag, commented_map, comment=comment)
            self._raw['__current'] = new_tag
        self._curr_version = None
        self._curr_data = None
        return self

    def _commit_tail(self, tag: AnyStr = None, comment: AnyStr = None):
        """
        apply the modifications on curr_data to the underling opened version
        and create a new tag
        """
        commented_map = CommentedMap()
        commented_map.update(self._curr_data or self.data)
        if tag:
            self._raw[tag] = commented_map
            self._raw['__current'] = tag
        else:
            new_tag = self._make_tag()
            self._raw.insert(len(self._raw.keys()),
                             new_tag,
                             commented_map,
                             comment=comment)
            self._raw['__current'] = new_tag
        self._curr_version = None
        self._curr_data = None
        return self

    def _has_changes(self):
        orig = self._raw[self._curr_version or self.current]
        current = self._curr_data or self.data
        try:
            assert_equal(orig, current)
        except AssertionError:
            return True
        return False

    @property
    def changes(self) -> AnyStr:
        if not self._has_changes():
            return ''
        yaml_orig = as_yaml(self._raw[self._curr_version or self.current])
        yaml_curr = as_yaml(self._curr_data)
        differ = Differ()
        result = list(
            differ.compare(yaml_orig.splitlines(), yaml_curr.splitlines()))
        return '\n'.join(result)

    def set_current(self, version_label: AnyStr):
        if version_label not in self.versions:
            raise MVYamlVersionNotFoundException(
                f'request version [{version_label}] not found')
        self._raw['__current'] = version_label
        self.with_version(version_label)
        return self
Ejemplo n.º 18
0
    def __init__(self,
                 toolpath_object,      # type: Dict[Text, Any]
                 pos,                  # type: int
                 loadingContext,       # type: LoadingContext
                 parentworkflowProv=None  # type: Optional[CreateProvProfile]
                ):  # type: (...) -> None
        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step" + Text(pos)

        loadingContext = loadingContext.copy()

        loadingContext.requirements = (getdefault(loadingContext.requirements, []) +
                                  toolpath_object.get("requirements", []) +
                                  get_overrides(getdefault(loadingContext.overrides_list, []),
                                                self.id).get("requirements", []))
        loadingContext.hints = getdefault(loadingContext.hints, []) + toolpath_object.get("hints", [])

        try:
            if isinstance(toolpath_object["run"], dict):
                self.embedded_tool = loadingContext.construct_tool_object(toolpath_object["run"], loadingContext)
            else:
                self.embedded_tool = load_tool(
                    toolpath_object["run"], loadingContext)
        except validate.ValidationException as vexc:
            if loadingContext.debug:
                _logger.exception("Validation exception")
            raise WorkflowException(
                u"Tool definition %s failed validation:\n%s" %
                (toolpath_object["run"], validate.indent(str(vexc))))

        validation_errors = []
        self.tool = toolpath_object = copy.deepcopy(toolpath_object)
        bound = set()
        for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
            toolpath_object[toolfield] = []
            for index, step_entry in enumerate(toolpath_object[stepfield]):
                if isinstance(step_entry, string_types):
                    param = CommentedMap()  # type: CommentedMap
                    inputid = step_entry
                else:
                    param = CommentedMap(six.iteritems(step_entry))
                    inputid = step_entry["id"]

                shortinputid = shortname(inputid)
                found = False
                for tool_entry in self.embedded_tool.tool[toolfield]:
                    frag = shortname(tool_entry["id"])
                    if frag == shortinputid:
                        #if the case that the step has a default for a parameter,
                        #we do not want the default of the tool to override it
                        step_default = None
                        if "default" in param and "default" in tool_entry:
                            step_default = param["default"]
                        param.update(tool_entry)
                        param["_tool_entry"] = tool_entry
                        if step_default is not None:
                            param["default"] = step_default
                        found = True
                        bound.add(frag)
                        break
                if not found:
                    if stepfield == "in":
                        param["type"] = "Any"
                        param["not_connected"] = True
                    else:
                        validation_errors.append(
                            SourceLine(self.tool["out"], index).makeError(
                                "Workflow step output '%s' does not correspond to"
                                % shortname(step_entry))
                            + "\n" + SourceLine(self.embedded_tool.tool, "outputs").makeError(
                                "  tool output (expected '%s')" % (
                                    "', '".join(
                                        [shortname(tool_entry["id"]) for tool_entry in
                                         self.embedded_tool.tool[toolfield]]))))
                param["id"] = inputid
                param.lc.line = toolpath_object[stepfield].lc.data[index][0]
                param.lc.col = toolpath_object[stepfield].lc.data[index][1]
                param.lc.filename = toolpath_object[stepfield].lc.filename
                toolpath_object[toolfield].append(param)

        missing = []
        for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
            if shortname(tool_entry["id"]) not in bound:
                if "null" not in tool_entry["type"] and "default" not in tool_entry:
                    missing.append(shortname(tool_entry["id"]))

        if missing:
            validation_errors.append(SourceLine(self.tool, "in").makeError(
                "Step is missing required parameter%s '%s'" %
                ("s" if len(missing) > 1 else "", "', '".join(missing))))

        if validation_errors:
            raise validate.ValidationException("\n".join(validation_errors))

        super(WorkflowStep, self).__init__(toolpath_object, loadingContext)

        if self.embedded_tool.tool["class"] == "Workflow":
            (feature, _) = self.get_requirement("SubworkflowFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains embedded workflow but "
                    "SubworkflowFeatureRequirement not in requirements")

        if "scatter" in self.tool:
            (feature, _) = self.get_requirement("ScatterFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains scatter but ScatterFeatureRequirement "
                    "not in requirements")

            inputparms = copy.deepcopy(self.tool["inputs"])
            outputparms = copy.deepcopy(self.tool["outputs"])
            scatter = aslist(self.tool["scatter"])

            method = self.tool.get("scatterMethod")
            if method is None and len(scatter) != 1:
                raise validate.ValidationException(
                    "Must specify scatterMethod when scattering over multiple inputs")

            inp_map = {i["id"]: i for i in inputparms}
            for inp in scatter:
                if inp not in inp_map:
                    raise validate.ValidationException(
                        SourceLine(self.tool, "scatter").makeError(
                            "Scatter parameter '%s' does not correspond to "
                            "an input parameter of this step, expecting '%s'"
                            % (shortname(inp), "', '".join(
                                shortname(k) for k in inp_map.keys()))))

                inp_map[inp]["type"] = {"type": "array", "items": inp_map[inp]["type"]}

            if self.tool.get("scatterMethod") == "nested_crossproduct":
                nesting = len(scatter)
            else:
                nesting = 1

            for index in range(0, nesting):
                for oparam in outputparms:
                    oparam["type"] = {"type": "array", "items": oparam["type"]}
            self.tool["inputs"] = inputparms
            self.tool["outputs"] = outputparms
        self.prov_obj = None  # type: Optional[CreateProvProfile]
        if loadingContext.research_obj:
            self.prov_obj = parentworkflowProv
            if self.embedded_tool.tool["class"] == "Workflow":
                self.parent_wf = self.embedded_tool.parent_wf
            else:
                self.parent_wf = self.prov_obj
Ejemplo n.º 19
0
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin):
    """
    Class for YAML-based (roundtrip) configurations
    """

    def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]):
        self.data = CommentedMap()

        super().__init__(owner, manager, path, *args, **kwargs)

    def load(self):
        with open(self.path, "r") as fh:
            self.data = yaml.round_trip_load(fh, version=(1, 2))

    def reload(self):
        self.unload()
        self.load()

    def unload(self):
        self.data.clear()

    def save(self):
        if not self.mutable:
            raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!")

        with open(self.path, "w") as fh:
            yaml.round_trip_dump(self.data, fh)

    # region: CommentedMap functions

    def insert(self, pos, key, value, *, comment=None):
        """
        Insert a `key: value` pair at the given position, attaching a comment if provided

        Wrapper for `CommentedMap.insert()`
        """

        return self.data.insert(pos, key, value, comment)

    def add_eol_comment(self, comment, *, key=NoComment, column=30):
        """
        Add an end-of-line comment for a key at a particular column (30 by default)

        Wrapper for `CommentedMap.yaml_add_eol_comment()`
        """

        # Setting the column to None as the API actually defaults to will raise an exception, so we have to
        # specify one unfortunately

        return self.data.yaml_add_eol_comment(comment, key=key, column=column)

    def set_comment_before_key(self, key, comment, *, indent=0):
        """
        Set a comment before a given key

        Wrapper for `CommentedMap.yaml_set_comment_before_after_key()`
        """

        return self.data.yaml_set_comment_before_after_key(
            key, before=comment, indent=indent, after=None, after_indent=None
        )

    def set_start_comment(self, comment, indent=0):
        """
        Set the starting comment

        Wrapper for `CommentedMap.yaml_set_start_comment()`
        """

        return self.data.yaml_set_start_comment(comment, indent=indent)

    # endregion

    # region: Dict functions

    def clear(self):
        return self.data.clear()

    def copy(self):
        return self.data.copy()

    def get(self, key, default=None):
        return self.data.get(key, default)

    def items(self):
        return self.data.items()

    def keys(self):
        return self.data.keys()

    def pop(self, key, default=None):
        return self.data.pop(key, default)

    def popitem(self):
        return self.data.popitem()

    def setdefault(self, key, default=None):
        if key not in self.data:
            self.data[key] = default
            return default

        return self.data[key]

    def update(self, other):
        return self.data.update(other)

    def values(self):
        return self.data.values()

    # endregion

    # Item access functions

    def __contains__(self, key):
        """
        Wrapper for `dict.__contains__()`
        """

        return self.data.__contains__(key)

    def __delitem__(self, key):
        """
        Wrapper for `dict.__delitem__()`
        """

        del self.data[key]

    def __getitem__(self, key):
        """
        Wrapper for `dict.__getitem__()`
        """

        return self.data.__getitem__(key)

    def __iter__(self):
        """
        Wrapper for `dict.__iter__()`
        """

        return self.data.__iter__()

    def __len__(self):
        """
        Wrapper for `dict.__len__()`
        """

        return self.data.__len__()

    def __setitem__(self, key, value):
        """
        Wrapper for `dict.__getitem__()`
        """

        return self.data.__setitem__(key, value)
Ejemplo n.º 20
0
def main(docname):

    with open(docname, "r") as fi:
        lines = fi.readlines()
    context = {}
    rest_lines = []
    for line in lines:
        # print(line)
        if "{%" in line:
            set_expr = re.search("{%(.*)%}", line)
            set_expr = set_expr.group(1)
            set_expr = set_expr.replace("set", "", 1).strip()
            exec(set_expr, globals(), context)
        else:
            rest_lines.append(line)

    yaml = YAML(typ="rt")
    yaml.preserve_quotes = True
    yaml.default_flow_style = False
    yaml.indent(sequence=4, offset=2)
    yaml.width = 1000
    yaml.Representer = MyRepresenter
    yaml.Loader = ruamel.yaml.RoundTripLoader

    result_yaml = CommentedMap()
    result_yaml["context"] = context

    def has_selector(s):
        return s.strip().endswith("]")

    quoted_lines = []
    for line in rest_lines:
        if has_selector(line):
            selector_start = line.rfind("[")
            selector_end = line.rfind("]")
            selector_content = line[selector_start + 1 : selector_end]

            if line.strip().startswith("-"):
                line = (
                    line[: line.find("-") + 1]
                    + f" sel({selector_content}): "
                    + line[
                        line.find("-") + 1 : min(line.rfind("#"), line.rfind("["))
                    ].strip()
                    + "\n"
                )
        quoted_lines.append(line)
    rest_lines = quoted_lines

    def check_if_quoted(s):
        s = s.strip()
        return s.startswith('"') or s.startswith("'")

    quoted_lines = []
    for line in rest_lines:
        if "{{" in line:
            # make sure that jinja stuff is quoted
            if line.find(":") != -1:
                idx = line.find(":")
            elif line.strip().startswith("-"):
                idx = line.find("-")
            rest = line[idx + 1 :]

            if not check_if_quoted(rest):
                if "'" in rest:
                    rest = rest.replace("'", '"')

                line = line[: idx + 1] + f" '{rest.strip()}'\n"
        quoted_lines.append(line)
    rest_lines = quoted_lines

    skips, wo_skip_lines = [], []
    for line in rest_lines:
        if line.strip().startswith("skip"):
            parts = line.split(":")
            rhs = parts[1].strip()
            if rhs.startswith("true"):
                selector_start = line.rfind("[")
                selector_end = line.rfind("]")
                selector_content = line[selector_start + 1 : selector_end]
                skips.append(selector_content)
            else:
                print("ATTENTION skip: false not handled!")
        else:
            wo_skip_lines.append(line)

    rest_lines = wo_skip_lines
    result_yaml.update(
        ruamel.yaml.load("".join(rest_lines), ruamel.yaml.RoundTripLoader)
    )

    if len(skips) != 0:
        result_yaml["build"]["skip"] = skips

    if result_yaml.get("outputs"):
        for o in result_yaml["outputs"]:
            name = o["name"]
            package = {"name": name}
            del o["name"]
            if o.get("version"):
                package["version"] = o["version"]
                del o["version"]

            build = {}
            if o.get("script"):
                build["script"] = o["script"]
                del o["script"]

            o["package"] = package
            o["build"] = build

        for d in result_yaml["outputs"]:
            print(order_output_dict(d))
        result_yaml["outputs"] = [order_output_dict(d) for d in result_yaml["outputs"]]

    from io import StringIO

    output = StringIO()
    yaml.dump(result_yaml, output)

    # Hacky way to insert an empty line after the context-key-object
    context_output = StringIO()
    yaml.dump(context, context_output)
    context_output = context_output.getvalue()
    context_output_len = len(context_output.split("\n"))

    final_result = output.getvalue()
    final_result_lines = final_result.split("\n")
    final_result_lines.insert(context_output_len, "")

    print("\n".join(final_result_lines))
Ejemplo n.º 21
0
    def __init__(self, toolpath_object, pos, **kwargs):
        # type: (Dict[Text, Any], int, **Any) -> None
        if "id" in toolpath_object:
            self.id = toolpath_object["id"]
        else:
            self.id = "#step" + Text(pos)

        kwargs["requirements"] = kwargs.get(
            "requirements", []) + toolpath_object.get("requirements", [])
        kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get(
            "hints", [])

        try:
            if isinstance(toolpath_object["run"], dict):
                self.embedded_tool = kwargs.get("makeTool")(
                    toolpath_object["run"], **kwargs)
            else:
                self.embedded_tool = load_tool(
                    toolpath_object["run"],
                    kwargs.get("makeTool"),
                    kwargs,
                    enable_dev=kwargs.get("enable_dev"),
                    strict=kwargs.get("strict"),
                    fetcher_constructor=kwargs.get("fetcher_constructor"))
        except validate.ValidationException as v:
            raise WorkflowException(
                u"Tool definition %s failed validation:\n%s" %
                (toolpath_object["run"], validate.indent(str(v))))

        validation_errors = []
        self.tool = toolpath_object = copy.deepcopy(toolpath_object)
        bound = set()
        for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
            toolpath_object[toolfield] = []
            for n, step_entry in enumerate(toolpath_object[stepfield]):
                if isinstance(step_entry, (str, unicode)):
                    param = CommentedMap()  # type: CommentedMap
                    inputid = step_entry
                else:
                    param = CommentedMap(step_entry.iteritems())
                    inputid = step_entry["id"]

                shortinputid = shortname(inputid)
                found = False
                for tool_entry in self.embedded_tool.tool[toolfield]:
                    frag = shortname(tool_entry["id"])
                    if frag == shortinputid:
                        param.update(tool_entry)  # type: ignore
                        found = True
                        bound.add(frag)
                        break
                if not found:
                    if stepfield == "in":
                        param["type"] = "Any"
                    else:
                        validation_errors.append(
                            SourceLine(self.tool["out"], n).makeError(
                                "Workflow step output '%s' does not correspond to"
                                % shortname(step_entry)) + "\n" +
                            SourceLine(self.embedded_tool.tool, "outputs").
                            makeError("  tool output (expected '%s')" %
                                      ("', '".join([
                                          shortname(tool_entry["id"])
                                          for tool_entry in
                                          self.embedded_tool.tool[toolfield]
                                      ]))))
                param["id"] = inputid
                param.lc.line = toolpath_object[stepfield].lc.data[n][0]
                param.lc.col = toolpath_object[stepfield].lc.data[n][1]
                param.lc.filename = toolpath_object[stepfield].lc.filename
                toolpath_object[toolfield].append(param)

        missing = []
        for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
            if shortname(tool_entry["id"]) not in bound:
                if "null" not in tool_entry[
                        "type"] and "default" not in tool_entry:
                    missing.append(shortname(tool_entry["id"]))

        if missing:
            validation_errors.append(
                SourceLine(self.tool, "in").makeError(
                    "Step is missing required parameter%s '%s'" %
                    ("s" if len(missing) > 1 else "", "', '".join(missing))))

        if validation_errors:
            raise validate.ValidationException("\n".join(validation_errors))

        super(WorkflowStep, self).__init__(toolpath_object, **kwargs)

        if self.embedded_tool.tool["class"] == "Workflow":
            (feature,
             _) = self.get_requirement("SubworkflowFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements"
                )

        if "scatter" in self.tool:
            (feature, _) = self.get_requirement("ScatterFeatureRequirement")
            if not feature:
                raise WorkflowException(
                    "Workflow contains scatter but ScatterFeatureRequirement not in requirements"
                )

            inputparms = copy.deepcopy(self.tool["inputs"])
            outputparms = copy.deepcopy(self.tool["outputs"])
            scatter = aslist(self.tool["scatter"])

            method = self.tool.get("scatterMethod")
            if method is None and len(scatter) != 1:
                raise validate.ValidationException(
                    "Must specify scatterMethod when scattering over multiple inputs"
                )

            inp_map = {i["id"]: i for i in inputparms}
            for s in scatter:
                if s not in inp_map:
                    raise validate.ValidationException(
                        SourceLine(self.tool, "scatter").makeError(
                            u"Scatter parameter '%s' does not correspond to an input parameter of this "
                            u"step, expecting '%s'" %
                            (shortname(s), "', '".join(
                                shortname(k) for k in inp_map.keys()))))

                inp_map[s]["type"] = {
                    "type": "array",
                    "items": inp_map[s]["type"]
                }

            if self.tool.get("scatterMethod") == "nested_crossproduct":
                nesting = len(scatter)
            else:
                nesting = 1

            for r in xrange(0, nesting):
                for op in outputparms:
                    op["type"] = {"type": "array", "items": op["type"]}
            self.tool["inputs"] = inputparms
            self.tool["outputs"] = outputparms
Ejemplo n.º 22
0
class RootManifest:
    """
    Manifest: defaults to the "vault.yml" file in
    the current directory.
    """
    def __init__(self, path: str = None, load: bool = True):
        self.path = os.path.abspath(path) if path else None
        self._backing = CommentedMap()

        if load:
            self._load()

        self._changes = deepcopy(self._backing)

    def _load(self):
        if os.path.exists(self.path):
            with open(self.path, mode="r") as f:
                self._backing.update(yaml.safe_load(f))

    def set_header(self, header: str) -> None:
        self._header = header

    def create_secrets_backend_section(self) -> None:
        if "secrets_backends" not in self._changes:
            self._changes["secrets_backends"] = {}
            self._changes.yaml_set_comment_before_after_key(
                "secrets_backends",
                before=
                "Secrets backends. Each key is the mount to a secrets engine.")

    def add_secrets_backend(self, name: str, manifest: ManifestItem) -> None:
        converted = manifest.convert()
        if not converted:
            return

        name = name.strip("/")
        new_dict = self._changes["secrets_backends"].get(name, {})
        new_dict.update(converted)
        self._changes["secrets_backends"][name] = new_dict

    def delete_secrets_backend(self, name: str) -> None:
        name = name.strip("/")

        if "secrets_backends" in self._changes and name in self._changes[
                "secrets_backends"]:
            del self._changes["secrets_backends"][name]

    def list_secrets_backend_names(self) -> List[str]:
        return [
            name.strip("/")
            for name in self._backing.get("secrets_backends", {})
        ]

    def create_auth_method_section(self) -> None:
        if "auth_methods" not in self._changes:
            self._changes["auth_methods"] = {}
            self._changes.yaml_set_comment_before_after_key(
                "auth_methods",
                before=
                "Authentication methods. Each key is the name of the auth method."
            )

    def add_auth_method(self, name: str, manifest: ManifestItem) -> None:
        converted = manifest.convert()
        if not converted:
            return

        if "auth_methods" not in self._changes:
            self._changes["auth_methods"] = {}

        name = name.strip("/")
        new_dict = self._changes["auth_methods"].get(name, {})
        new_dict.update(converted)
        self._changes["auth_methods"][name] = new_dict

    def yaml(self) -> str:
        output = ""
        if self._header:
            for line in self._header.split("\n"):
                output += f"# {line}\n"
            output += "\n"

        output += yaml.round_trip_dump(self._changes)
        return output

    def save(self) -> None:
        with open(self.path, "w") as f:
            f.write(self.yaml())