def _update_step(category, steps): current_step_count = Step.select().where(Step.category == category).count() for step in steps[current_step_count:]: import_class(step["module"]).register(step.get("config", {})) Step.create( category=category, module=step["module"], config=json.dumps(step.get("config", None)), )
def bootstrap_steps(config): Step.insert_many( [ { "category": key, "module": step["module"], "config": json.dumps(step.get("config", None)), } for key, steps in config["steps"].items() for step in steps ] ).execute()
def bootstrap_steps(config): count = Step.select().count() new_steps = config["steps"]["run"][count:] if len(new_steps) > 0: query = Step.insert_many( [ { "category": "run", "module": step["module"], "config": json.dumps(step.get("config", None)), } for step in new_steps ] ) query.execute()
def get_pending_run_ids(cls): last_step = (Step.select(fn.MAX( Step.id)).where(Step.category == Step.RUN).scalar()) Run.update(status=Run.PENDING).where((Run.status < Run.DONE) | ( Run.last_step_id != last_step)).execute() pending_runs = Run.select(Run.id).where(Run.status == Run.PENDING) return [r.id for r in pending_runs]
def get_run(cls, run_id): run = Run.get_by_id(run_id) if run is None: return None run.status = Run.SUBMITTED run.save() last_step = run.last_step_id or 0 runsteps = Step.select().where((Step.category == Step.RUN) & (Step.id > last_step)) limits = cls.get_limits() parameters = {p.key: p.value for p in run.parameter_group.parameters} run_dict = dict( id=run.id, task=run.task_id, tool=run.tool_id, directory=run.directory, parameters=parameters, steps=list(runsteps.dicts()), limits=limits, ) return run_dict
def get_next_pending_run(cls): try: run = Run.select().where(Run.status == Run.PENDING).limit(1).get() except Run.DoesNotExist: return None run.status = Run.SUBMITTED run.save() last_step = run.last_step_id or 0 runsteps = Step.select().where((Step.category == Step.RUN) & (Step.id > last_step)) limits = cls.get_limits() parameters = {p.key: p.value for p in run.parameter_group.parameters} run_dict = dict( id=run.id, task=run.task_id, tool=run.tool.module, parameters=parameters, steps=list(runsteps.dicts()), limits=limits, ) return run_dict
def run(self): steps = Step.select().where(Step.category == Step.ANALYSIS) context = dict(output_dir=self.output_dir, db_path=self.db_path) for step in steps: logger.debug(f"Running {step.module}") module = import_class(step.module) config = json.loads(step.config) module.execute(context, config)
def handle_event(cls, event_type, payload, **kwargs): reply = kwargs.pop("reply") address = kwargs.pop("address") if event_type == BOOTSTRAP: update(**payload) elif event_type == REQUEST_PENDING: run_ids = cls.get_pending_run_ids() reply.send_multipart([address, encode_message(run_ids)]) elif event_type == WORKER_JOIN: run = cls.get_run(payload) reply.send_multipart([address, encode_message(run)]) elif event_type == RUN_INTERRUPT: Run.update(status=Run.PENDING).where(Run.id == payload).execute() elif event_type == RUN_START: run_id = payload.pop("run_id") Run.update(status=Run.RUNNING, **payload).where(Run.id == run_id).execute() elif event_type == RUN_STEP: step = Step.get(module=payload["step"]) Run.update(last_step=step).where( Run.id == payload["run_id"]).execute() elif event_type == RUN_FINISH: Run.update(status=Run.DONE).where(Run.id == payload).execute()