def _check_old_yaml_format(self):
if self.flow_config.steps is None:
if "tasks" in self.flow_config.config:
raise FlowConfigError(
'Old flow syntax detected. Please change from "tasks" to "steps" in the flow definition.'
)
else:
raise FlowConfigError("No steps found in the flow definition")
def _get_steps_ordered(self):
if self.flow_config.steps is None:
if self.flow_config.tasks:
raise FlowConfigError(
'Old flow syntax detected. Please change from "tasks" to "steps" in the flow definition'
)
else:
raise FlowConfigError("No steps found in the flow definition")
if not self.nested:
self._check_infinite_flows(self.flow_config.steps)
steps = []
for step_num, config in list(self.flow_config.steps.items()):
if "flow" in config and "task" in config:
raise FlowConfigError(
'"flow" and "task" in same config item: {}'.format(config))
if ("flow" in config and config["flow"] == "None") or (
"task" in config and config["task"] == "None"):
# allows skipping flows/tasks using YAML overrides
continue
parsed_step_num = LooseVersion(str(step_num))
if "flow" in config: # nested flow
flow_config = self.project_config.get_flow(config["flow"])
steps.append((
parsed_step_num,
{
"step_config": config,
"flow_config": flow_config
},
))
elif "task" in config:
task_config = self.project_config.get_task(config["task"])
steps.append((
parsed_step_num,
{
"step_config": config,
"task_config": task_config
},
))
steps.sort() # sort by step number
return steps
def _visit_step(
self,
number,
step_config,
project_config,
visited_steps=None,
parent_options=None,
parent_ui_options=None,
from_flow=None,
):
"""
for each step (as defined in the flow YAML), _visit_step is called with only
the first two parameters. this takes care of validating the step, collating the
option overrides, and if it is a task, creating a StepSpec for it.
If it is a flow, we recursively call _visit_step with the rest of the parameters of context.
:param number: StepVersion representation of the current step number
:param step_config: the current step's config (dict from YAML)
:param visited_steps: used when called recursively for nested steps, becomes the return value
:param parent_options: used when called recursively for nested steps, options from parent flow
:param parent_ui_options: used when called recursively for nested steps, UI options from parent flow
:param from_flow: used when called recursively for nested steps, name of parent flow
:return: List[StepSpec] a list of all resolved steps including/under the one passed in
"""
number = StepVersion(str(number))
if visited_steps is None:
visited_steps = []
if parent_options is None:
parent_options = {}
if parent_ui_options is None:
parent_ui_options = {}
# Step Validation
# - A step is either a task OR a flow.
if all(k in step_config for k in ("flow", "task")):
raise FlowConfigError(
f"Step {number} is configured as both a flow AND a task. \n\t{step_config}."
)
# Skips
# - either in YAML (with the None string)
# - or by providing a skip list to the FlowRunner at initialization.
if (("flow" in step_config and step_config["flow"] == "None")
or ("task" in step_config and step_config["task"] == "None") or
("task" in step_config and step_config["task"] in self.skip)):
visited_steps.append(
StepSpec(
step_num=number,
task_name=step_config.get("task", step_config.get("flow")),
task_config=step_config.get("options", {}),
task_class=None,
project_config=project_config,
from_flow=from_flow,
skip=
True, # someday we could use different vals for why skipped
))
return visited_steps
if "task" in step_config:
name = step_config["task"]
# get the base task_config from the project config, as a dict for easier manipulation.
# will raise if the task doesn't exist / is invalid
task_config = project_config.get_task(name)
task_config_dict = copy.deepcopy(task_config.config)
if "options" not in task_config_dict:
task_config_dict["options"] = {}
# merge the options together, from task_config all the way down through parent_options
step_overrides = copy.deepcopy(parent_options.get(name, {}))
step_overrides.update(step_config.get("options", {}))
task_config_dict["options"].update(step_overrides)
# merge UI options from task config and parent flow
if "ui_options" not in task_config_dict:
task_config_dict["ui_options"] = {}
step_ui_overrides = copy.deepcopy(parent_ui_options.get(name, {}))
step_ui_overrides.update(step_config.get("ui_options", {}))
task_config_dict["ui_options"].update(step_ui_overrides)
# merge checks from task config and flow step
if "checks" not in task_config_dict:
task_config_dict["checks"] = []
task_config_dict["checks"].extend(step_config.get("checks", []))
# merge runtime options
if name in self.runtime_options:
task_config_dict["options"].update(self.runtime_options[name])
# get implementation class. raise/fail if it doesn't exist, because why continue
try:
task_class = import_global(task_config_dict["class_path"])
except (ImportError, AttributeError):
raise FlowConfigError(f"Task named {name} has bad classpath")
visited_steps.append(
StepSpec(
step_num=number,
task_name=name,
task_config=task_config_dict,
task_class=task_class,
project_config=task_config.project_config,
allow_failure=step_config.get("ignore_failure", False),
from_flow=from_flow,
when=step_config.get("when"),
))
return visited_steps
if "flow" in step_config:
name = step_config["flow"]
if from_flow:
path = ".".join([from_flow, name])
else:
path = name
step_options = step_config.get("options", {})
step_ui_options = step_config.get("ui_options", {})
flow_config = project_config.get_flow(name)
for sub_number, sub_stepconf in flow_config.steps.items():
# append the flow number to the child number, since its a LooseVersion.
# e.g. if we're in step 2.3 which references a flow with steps 1-5, it
# simply ends up as five steps: 2.3.1, 2.3.2, 2.3.3, 2.3.4, 2.3.5
# TODO: how does this work with nested flowveride? what does defining step 2.3.2 later do?
num = f"{number}/{sub_number}"
self._visit_step(
number=num,
step_config=sub_stepconf,
project_config=flow_config.project_config,
visited_steps=visited_steps,
parent_options=step_options,
parent_ui_options=step_ui_options,
from_flow=path,
)
return visited_steps