def stop_workers(args):
"""
CLI command for stopping all workers.
:param `args`: parsed CLI arguments
"""
print(banner_small)
router.stop_workers(args.task_server, args.queues, args.workers)
def stop_workers(args):
"""
CLI command for stopping all workers.
:param `args`: parsed CLI arguments
"""
print(banner_small)
worker_names = []
if args.spec:
spec_path = verify_filepath(args.spec)
spec = MerlinSpec.load_specification(spec_path)
worker_names = spec.get_worker_names()
for worker_name in worker_names:
if "$" in worker_name:
LOG.warning(
f"Worker '{worker_name}' is unexpanded. Target provenance spec instead?"
)
router.stop_workers(args.task_server, worker_names, args.queues, args.workers)
def shutdown_workers(self, shutdown_queues):
"""
This task issues a call to shutdown workers.
It wraps the stop_celery_workers call as a task.
It is acknolwedged right away, so that it will not be requeued when
executed by a worker.
:param: shutdown_queues: The specific queues to shutdown (list)
"""
if shutdown_queues is not None:
LOG.warning(f"Shutting down workers in queues {shutdown_queues}!")
else:
LOG.warning(f"Shutting down workers in all queues!")
return stop_workers("celery", None, shutdown_queues, None)
def merlin_step(self, *args, **kwargs):
"""
Executes a Merlin Step
:param args: The arguments, one of which should be an instance of Step
:param kwargs: The optional keyword arguments that describe adapter_config and
the next step in the chain, if there is one.
Example kwargs dict:
{"adapter_config": {'type':'local'},
"next_in_chain": <Step object> } # merlin_step will be added to the current chord
# with next_in_chain as an argument
"""
step = None
LOG.debug(f"args is {len(args)} long")
for a in args:
if isinstance(a, Step):
step = a
else:
LOG.debug(f"discard argument {a}")
config = kwargs.pop("adapter_config", {"type": "local"})
next_in_chain = kwargs.pop("next_in_chain", None)
if step:
self.max_retries = step.max_retries
step_name = step.name()
step_dir = step.get_workspace()
LOG.debug(
f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'")
finished_filename = os.path.join(step_dir, "MERLIN_FINISHED")
# if we've already finished this task, skip it
if os.path.exists(finished_filename):
LOG.info(f"Skipping step '{step_name}' in '{step_dir}'.")
result = ReturnCode.OK
else:
result = step.execute(config)
if result == ReturnCode.OK:
LOG.info(
f"Step '{step_name}' in '{step_dir}' finished successfully.")
# touch a file indicating we're done with this step
open(finished_filename, "a").close()
elif result == ReturnCode.DRY_OK:
LOG.info(f"Dry-ran step '{step_name}' in '{step_dir}'.")
elif result == ReturnCode.RESTART:
LOG.info(f"** Restarting step '{step_name}' in '{step_dir}'.")
step.restart = True
raise RestartException
elif result == ReturnCode.RETRY:
LOG.warning(f"** Retrying step '{step_name}' in '{step_dir}'.")
step.restart = False
raise RetryException
elif result == ReturnCode.SOFT_FAIL:
LOG.warning(
f"*** Step '{step_name}' in '{step_dir}' soft failed. Continuing with workflow."
)
elif result == ReturnCode.HARD_FAIL:
LOG.error(
f"*** Step '{step_name}' in '{step_dir}' hard failed. Quitting workflow."
)
# TODO purge queues? function requires maestro_spec
# router.purge_tasks("celery", ?, force=True)
# stop workers TODO make this more discriminatory, stopping only the relevant workers
stop_workers("celery", None, None)
raise HardFailException
else:
LOG.warning(
f"**** Step '{step_name}' in '{step_dir}' had unhandled exit code {result}. Continuing with workflow."
)
# queue off the next task in a chain while adding it to the current chord so that the chordfinisher actually
# waits for the next task in the chain
if next_in_chain is not None:
if self.request.is_eager:
LOG.debug(f"calling next_in_chain {signature(next_in_chain)}")
next_in_chain.delay()
else:
LOG.debug(f"adding {next_in_chain} to chord")
self.add_to_chord(next_in_chain, lazy=False)
return result
LOG.error("Failed to find step!")
return None