def change_self_to_inputs(cls, v):
refs = parse_double_quotes_vars(v)
for ref in refs:
v = v.replace(
ref, ref.replace('self.', 'inputs.').replace('_', '-')
)
return v
def _clean_command(command: str) -> str:
"""A helper function to reformat python command to Queenbee function commands."""
refs = parse_double_quotes_vars(command)
for ref in refs:
command = command.replace(
ref,
ref.replace('self.', 'inputs.').replace('_', '-'))
return command
def _try(self, inputs: Dict[str, Any], folder: str = None) -> str:
"""Try running a function locally for testing.
This method does not return the output values. See the folder to validate the
outputs.
Args:
inputs: A dictionary that maps input names to values
(e.g. {'input_one': 5, ...}).
folder: An optional folder to run the function. A temporary folder
will be created if this folder is not provided.
Returns:
str -- path to run_folder.
"""
func = self.queenbee
# check all the required inputs are provided
for inp in func.inputs:
name = inp.name.replace('-', '_')
if inp.required:
assert name in inputs, f'Required input "{name}" is missing from inputs.'
continue
# see if default value should be used
if name not in inputs:
inputs[name] = inp.default
dst = folder or tempfile.TemporaryDirectory().name
command = ' '.join(func.command.split())
refs = parse_double_quotes_vars(command)
command = command.replace('{{', '{').replace('}}', '}')
for ref in refs:
assert ref.startswith('inputs.'), \
'All referenced values must start with {{inputs followed with' \
f' variable name. Invalid referenced value: {ref}'
var = ref.replace('inputs.', '').replace('-', '_')
command = command.replace('{%s}' % ref, str(inputs[var]))
for art in func.artifact_inputs:
print(f"copying input artifact: {art.name}...")
name = art.name.replace('-', '_')
_copy_artifacts(inputs[name], os.path.join(dst, art.path))
cur_dir = os.getcwd()
os.chdir(dst)
print(f'command: {command}')
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
env=os.environ)
stdout, stderr = p.communicate()
if p.returncode != 0 and stderr != b'':
raise RuntimeError(stderr.decode('utf-8'))
if stderr.decode('utf-8'):
print(stderr.decode('utf-8'))
if stdout.decode('utf-8'):
print(stdout.decode('utf-8'))
# change back to initial directory
os.chdir(cur_dir)
return dst
def to_queenbee(method, returns: List[Dict],
dag_inputs: Dict[int, str], dag_package: str):
"""Convert a task method to a Queenbee DAGTask."""
name = method.__name__.replace('_', '-')
tt = method.__task_template__
if tt.__decorator__ == 'dag' and tt._package == dag_package:
# template is another dag in the same plugin (AKA package)
template = f'{camel_to_snake(tt.__class__.__name__)}'
elif tt.__decorator__ == 'dag':
# template is a recipe entry point - use recipe name as the package name
template = tt._package["name"]
else:
# it is a function from a plugin
template = \
f'{tt._package["name"]}/{camel_to_snake(tt.__class__.__name__)}'
task_needs = [
need.__name__.replace('_', '-')
for need in method.__task_needs__
]
# TODO: validate arguments against needs to ensure all the task names are
# included in needs.
task_arguments = _get_task_arguments(method, dag_inputs, sub_paths)
task_loop = _get_task_loop(method.__task_loop__, dag_inputs)
task_returns = []
for out in returns or []:
from_ = out.get('from', None)
from_ = from_['value'] if from_ else from_
to_ = out.get('to', None)
description = out.get('description', None)
if from_.is_artifact:
assert to_ is not None, 'Missing \'to\' key for {from_.name}. ' \
'All file and folder returns must provide a target path using ' \
'the `to` key.'
if to_:
# find and replace referenced values
refs = parse_double_quotes_vars(to_)
for ref in refs:
to_ = to_.replace(
ref,
ref.replace('self.',
'inputs.').replace('_', '-'))
return_ = TaskPathReturn(name=from_.name,
description=description,
path=to_)
else:
# returning a parameter
return_ = TaskReturn(name=from_.name,
description=description)
task_returns.append(return_)
annotations = method.__task_annotations__
sub_folder = method.__task_subfolder__
return DAGTask(name=name,
template=template,
needs=task_needs,
arguments=task_arguments,
returns=task_returns,
loop=task_loop,
sub_folder=sub_folder,
annotations=annotations)