def _update_step(category, steps):
current_step_count = Step.select().where(Step.category == category).count()
for step in steps[current_step_count:]:
import_class(step["module"]).register(step.get("config", {}))
Step.create(
category=category,
module=step["module"],
config=json.dumps(step.get("config", None)),
)
def bootstrap_steps(config):
Step.insert_many(
[
{
"category": key,
"module": step["module"],
"config": json.dumps(step.get("config", None)),
}
for key, steps in config["steps"].items()
for step in steps
]
).execute()
def bootstrap_steps(config):
count = Step.select().count()
new_steps = config["steps"]["run"][count:]
if len(new_steps) > 0:
query = Step.insert_many(
[
{
"category": "run",
"module": step["module"],
"config": json.dumps(step.get("config", None)),
}
for step in new_steps
]
)
query.execute()
def get_pending_run_ids(cls):
last_step = (Step.select(fn.MAX(
Step.id)).where(Step.category == Step.RUN).scalar())
Run.update(status=Run.PENDING).where((Run.status < Run.DONE) | (
Run.last_step_id != last_step)).execute()
pending_runs = Run.select(Run.id).where(Run.status == Run.PENDING)
return [r.id for r in pending_runs]
def get_run(cls, run_id):
run = Run.get_by_id(run_id)
if run is None:
return None
run.status = Run.SUBMITTED
run.save()
last_step = run.last_step_id or 0
runsteps = Step.select().where((Step.category == Step.RUN)
& (Step.id > last_step))
limits = cls.get_limits()
parameters = {p.key: p.value for p in run.parameter_group.parameters}
run_dict = dict(
id=run.id,
task=run.task_id,
tool=run.tool_id,
directory=run.directory,
parameters=parameters,
steps=list(runsteps.dicts()),
limits=limits,
)
return run_dict
def get_next_pending_run(cls):
try:
run = Run.select().where(Run.status == Run.PENDING).limit(1).get()
except Run.DoesNotExist:
return None
run.status = Run.SUBMITTED
run.save()
last_step = run.last_step_id or 0
runsteps = Step.select().where((Step.category == Step.RUN)
& (Step.id > last_step))
limits = cls.get_limits()
parameters = {p.key: p.value for p in run.parameter_group.parameters}
run_dict = dict(
id=run.id,
task=run.task_id,
tool=run.tool.module,
parameters=parameters,
steps=list(runsteps.dicts()),
limits=limits,
)
return run_dict
def run(self):
steps = Step.select().where(Step.category == Step.ANALYSIS)
context = dict(output_dir=self.output_dir, db_path=self.db_path)
for step in steps:
logger.debug(f"Running {step.module}")
module = import_class(step.module)
config = json.loads(step.config)
module.execute(context, config)
def handle_event(cls, event_type, payload, **kwargs):
reply = kwargs.pop("reply")
address = kwargs.pop("address")
if event_type == BOOTSTRAP:
update(**payload)
elif event_type == REQUEST_PENDING:
run_ids = cls.get_pending_run_ids()
reply.send_multipart([address, encode_message(run_ids)])
elif event_type == WORKER_JOIN:
run = cls.get_run(payload)
reply.send_multipart([address, encode_message(run)])
elif event_type == RUN_INTERRUPT:
Run.update(status=Run.PENDING).where(Run.id == payload).execute()
elif event_type == RUN_START:
run_id = payload.pop("run_id")
Run.update(status=Run.RUNNING,
**payload).where(Run.id == run_id).execute()
elif event_type == RUN_STEP:
step = Step.get(module=payload["step"])
Run.update(last_step=step).where(
Run.id == payload["run_id"]).execute()
elif event_type == RUN_FINISH:
Run.update(status=Run.DONE).where(Run.id == payload).execute()