def do_translate(args):
jc = JanisConfiguration.initial_configuration(args.config)
container_override = parse_container_override_format(
args.container_override)
hints = {
k[5:]: v
for k, v in vars(args).items()
if k.startswith("hint_") and v is not None
}
inputs = args.inputs or []
# the args.extra_inputs parameter are inputs that we MUST match
# we'll need to parse them manually and then pass them to fromjanis as requiring a match
# required_inputs = parse_additional_arguments(args.extra_inputs)
translate(
config=jc,
tool=args.workflow,
translation=args.translation,
name=args.name,
output_dir=args.output_dir,
force=args.no_cache,
allow_empty_container=args.allow_empty_container,
container_override=container_override,
skip_digest_lookup=args.skip_digest_lookup,
skip_digest_cache=args.skip_digest_cache,
inputs=inputs,
recipes=args.recipe,
hints=hints,
)
def do_inputs(args):
if args.config or args.recipes:
JanisConfiguration.initial_configuration(args.config)
quality_type = None
if args.user:
quality_type = [InputQualityType.user]
elif args.static:
quality_type = [
InputQualityType.static, InputQualityType.configuration
]
hints = {
k[5:]: v
for k, v in vars(args).items()
if k.startswith("hint_") and v is not None
}
outd = generate_inputs(
args.workflow,
all=args.all,
name=args.name,
force=args.no_cache,
additional_inputs=args.inputs,
with_resources=args.resources,
quality_type=quality_type,
recipes=args.recipes,
hints=hints,
)
if args.json:
outs = json.dumps(outd,
sort_keys=True,
indent=4,
separators=(",", ": "))
else:
outs = ruamel.yaml.dump(outd, default_flow_style=False)
if args.output:
with open(args.output, "w+") as out:
out.write(str(outs))
else:
print(outs, file=sys.stdout)
def from_path_with_wid(path, wid, readonly=False):
"""
:param wid: Workflow ID
:param path: Path to workflow
:return: TaskManager after resuming (might include a wait)
"""
# get everything and pass to constructor
# database path
path = WorkflowManager.get_task_path_for(path)
if not os.path.exists(path):
raise FileNotFoundError(f"Couldn't find path '{path}'")
db = WorkflowDbManager.get_workflow_metadatadb(path, wid, readonly=readonly)
if not wid:
wid = db.wid # .get_meta_info(InfoKeys.taskId)
if not wid:
raise Exception(f"Couldn't find workflow with id '{wid}'")
envid = db.environment # .get_meta_info(InfoKeys.environment)
eng = db.engine
fs = db.filescheme
env = Environment(envid, eng, fs)
try:
JanisConfiguration._managed = db.configuration
except Exception as e:
Logger.critical(
"The JanisConfiguration could not be loaded from the DB, this might be due to an older version, we'll load your current config instead. Error: "
+ str(e)
)
JanisConfiguration.initial_configuration(None)
db.close()
tm = WorkflowManager(outdir=path, wid=wid, environment=env, readonly=readonly)
return tm
def config(self):
if self._config is None:
from janis_assistant.management.configuration import JanisConfiguration
if not self._raw_config:
pass
elif isinstance(self._raw_config, JanisConfiguration):
self._config = self._raw_config
elif isinstance(self._raw_config, str):
self._config = JanisConfiguration.initial_configuration(
path=self._raw_config
)
else:
raise ValueError(
f"Unrecognised type for janis configuration {self._raw_config} (type: {type(self._raw_config)})"
)
return self._config
def do_runtest(args):
config = None
if args.config:
config = JanisConfiguration.initial_configuration(path=args.config)
runner_path = test_runner.__file__
cli_args = sys.argv[2:]
run_test_commands = ["python", runner_path] + cli_args
if config:
commands = config.template.template.prepare_run_test_command(
run_test_commands)
else:
commands = run_test_commands
joined_command = "' '".join(commands)
Logger.info(f"Deploying test with command: '{joined_command}'")
subprocess.run(commands)
def run_with_outputs(
tool: Union[j.CommandTool, j.Workflow],
inputs: Dict[str, any],
output_dir: str,
config: JanisConfiguration = None,
engine: Optional[str] = None,
workflow_reference: Optional[str] = None,
):
"""
Run and WAIT for a Janis workflow to complete. This helper method runs a workflow,
and returns a dictionary of output values to their output tag. This method MAY throw,
so ensure it's try-catch wrapped.
:param tool: An INSTANTIATED tool definition. Seek this from the 'get_janis_workflow_from_searchname' earlier
:param inputs: A dictionary of pure input values, not file paths.
:param output_dir: Where to run the execution
:param config: Optional config, else choose the default at $HOME/.janis/janis.conf
:param workflow_reference: A reference to the workflow being run, this gets used to write a run.sh file
:return: A dictionary of output values by the output tag
"""
job = prepare_job(
tool=tool,
output_dir=output_dir,
required_inputs=inputs,
jc=config or JanisConfiguration.initial_configuration(None),
# params to be automatically evaluated
execution_dir=None,
inputs={},
allow_empty_container=False,
check_files=True,
container_override={},
skip_digest_cache=False,
skip_digest_lookup=False,
batchrun_reqs=None,
validation_reqs=None,
engine=engine,
hints={},
keep_intermediate_files=False,
max_cores=None,
max_memory=None,
max_duration=None,
no_store=True,
recipes=[],
run_in_background=None,
run_in_foreground=None,
strict_inputs=False,
watch=False,
workflow_reference=workflow_reference,
# don't do extra preprocessing steps
run_prepare_processing=False,
localise_all_files=True,
)
wm = run_from_jobfile(tool, jobfile=job, wait=True)
if not wm:
Logger.critical(f"An error occurred when running workflow {tool.id()}")
return None
if not wm.database:
Logger.critical(
f"An error occurred when getting the outputs for workflow {tool.id()}"
)
return None
status = wm.database.get_uncached_status()
if status != TaskStatus.COMPLETED:
error = ""
if wm.database.submission_metadata and wm.database.submission_metadata.metadata:
error = wm.database.submission_metadata.metadata.error or ""
Logger.critical(
f"The workflow {tool.id()} ended with status {status}, and hence won't return outputs. Error: {error}"
)
return None
outs = wm.database.outputsDB.get()
return {
o.id_: o.value or o.new_path
for o in outs if o.value or o.new_path
}
def do_run(args):
jc = JanisConfiguration.initial_configuration(args.config)
validation_reqs, batchrun_reqs = None, None
if args.validation_fields:
Logger.info("Will prepare validation")
validation_reqs = ValidationRequirements(
truthVCF=args.validation_truth_vcf,
reference=args.validation_reference,
fields=args.validation_fields,
intervals=args.validation_intervals,
)
if args.batchrun:
Logger.info("Will prepare batch run")
batchrun_reqs = BatchRunRequirements(fields=args.batchrun_fields,
groupby=args.batchrun_groupby)
hints = {
k[5:]: v
for k, v in vars(args).items()
if k.startswith("hint_") and v is not None
}
# the args.extra_inputs parameter are inputs that we MUST match
# we'll need to parse them manually and then pass them to fromjanis as requiring a match
required_inputs = parse_additional_arguments(args.extra_inputs)
inputs = args.inputs or []
# we'll manually suck "inputs" out of the extra parms, otherwise it's actually really
# annoying if you forget to put the inputs before the workflow positional argument.
# TBH, we could automatically do this for all params, but it's a little trickier
if "inputs" in required_inputs:
ins = required_inputs.pop("inputs")
inputs.extend(ins if isinstance(ins, list) else [ins])
if "i" in required_inputs:
ins = required_inputs.pop("i")
inputs.extend(ins if isinstance(ins, list) else [ins])
keep_intermediate_files = args.keep_intermediate_files is True
db_config = jc.cromwell.get_database_config_helper()
if args.mysql:
db_config.should_manage_mysql = True
if args.no_database:
db_config.skip_database = True
if args.development:
# no change for using mysql, as a database is the default
keep_intermediate_files = True
JanisConfiguration.manager().cromwell.call_caching_enabled = True
wid = fromjanis(
args.workflow,
name=args.name,
validation_reqs=validation_reqs,
batchrun_reqs=batchrun_reqs,
engine=args.engine,
filescheme=args.filescheme,
hints=hints,
output_dir=args.output_dir,
inputs=inputs,
required_inputs=required_inputs,
filescheme_ssh_binding=args.filescheme_ssh_binding,
cromwell_url=args.cromwell_url,
watch=args.progress,
max_cores=args.max_cores,
max_mem=args.max_memory,
force=args.no_cache,
recipes=args.recipe,
keep_intermediate_files=keep_intermediate_files,
run_in_background=(args.background is True),
run_in_foreground=(args.foreground is True),
dbconfig=db_config,
only_toolbox=args.toolbox,
no_store=args.no_store,
allow_empty_container=args.allow_empty_container,
check_files=not args.skip_file_check,
container_override=parse_container_override_format(
args.container_override),
)
Logger.info("Exiting")
raise SystemExit
def prepare_from_args(
args, run_prepare_processing: bool) -> Tuple[PreparedJob, Tool]:
jc = JanisConfiguration.initial_configuration(path=args.config)
# the args.extra_inputs parameter are inputs that we MUST match
# we'll need to parse them manually and then pass them to fromjanis as requiring a match
required_inputs = parse_additional_arguments(args.extra_inputs)
inputs = args.inputs or []
# we'll manually suck "inputs" out of the extra parms, otherwise it's actually really
# annoying if you forget to put the inputs before the workflow positional argument.
# TBH, we could automatically do this for all params, but it's a little trickier
if "inputs" in required_inputs:
ins = required_inputs.pop("inputs")
inputs.extend(ins if isinstance(ins, list) else [ins])
if "i" in required_inputs:
ins = required_inputs.pop("i")
inputs.extend(ins if isinstance(ins, list) else [ins])
validation, batchrun = None, None
if args.validation_fields:
Logger.info("Will prepare validation")
validation = ValidationRequirements(
truthVCF=args.validation_truth_vcf,
reference=args.validation_reference,
fields=args.validation_fields,
intervals=args.validation_intervals,
)
if args.batchrun:
Logger.info("Will prepare batch run")
batchrun = BatchRunRequirements(fields=args.batchrun_fields,
groupby=args.batchrun_groupby)
db_type: Optional[DatabaseTypeToUse] = None
if args.no_database:
db_type = DatabaseTypeToUse.none
elif args.mysql:
db_type = DatabaseTypeToUse.managed
try:
source_hints = args.source_hint
except:
source_hints = []
wf, wf_reference = resolve_tool(
tool=args.workflow,
name=args.name,
from_toolshed=True,
only_toolbox=args.toolbox,
force=args.no_cache,
)
job = prepare_job(
run_prepare_processing=run_prepare_processing,
tool=wf,
workflow_reference=wf_reference,
jc=jc,
engine=args.engine,
output_dir=args.output_dir,
execution_dir=args.execution_dir,
max_cores=args.max_cores,
max_memory=args.max_memory,
max_duration=args.max_duration,
run_in_foreground=args.foreground is True,
run_in_background=args.background is True,
check_files=not args.skip_file_check,
container_override=parse_container_override_format(
args.container_override),
skip_digest_lookup=args.skip_digest_lookup,
skip_digest_cache=args.skip_digest_cache,
hints={
k[5:]: v
for k, v in vars(args).items()
if k.startswith("hint_") and v is not None
},
validation_reqs=validation,
batchrun_reqs=batchrun,
# inputs
inputs=inputs,
required_inputs=required_inputs,
watch=args.progress,
recipes=args.recipe,
keep_intermediate_files=args.keep_intermediate_files is True,
no_store=args.no_store,
allow_empty_container=args.allow_empty_container,
strict_inputs=args.strict_inputs,
db_type=db_type,
source_hints=source_hints,
post_run_script=args.post_run_script,
localise_all_files=args.localise_all_files,
)
return job, wf