def run(self):
atexit.register(self.killed, self.run_id)
context = zmq.Context()
self.socket = context.socket(zmq.DEALER)
self.socket.connect(self.server_address)
send_event(self.socket, WORKER_JOIN, self.run_id)
run = decode_message(self.socket.recv())
tool = import_class(run["tool"])
context = {}
context["socket"] = self.socket
context["tool"] = tool
context["run"] = run
logger.info(f"Processing task: {run['directory']}")
directory = Path(run["directory"])
directory.mkdir(parents=True, exist_ok=True)
payload = dict(tool_version=tool.version(), run_id=self.run_id)
send_event(self.socket, RUN_START, payload)
for runstep in run["steps"]:
logger.debug(f"Running step {runstep['module']}")
step = import_class(runstep["module"])
config = json.loads(runstep["config"])
step.execute(context, config)
payload = {"run_id": self.run_id, "step": runstep["module"]}
send_event(self.socket, RUN_STEP, payload)
send_event(self.socket, RUN_FINISH, self.run_id)
atexit.unregister(self.killed)
send_event(self.socket, WORKER_LEAVE, self.run_id)
def _update_step(category, steps):
current_step_count = Step.select().where(Step.category == category).count()
for step in steps[current_step_count:]:
import_class(step["module"]).register(step.get("config", {}))
Step.create(
category=category,
module=step["module"],
config=json.dumps(step.get("config", None)),
)
def bootstrap_tools(config):
logger.info("Setting up tools...")
tools = {}
for tool_name, tool in config["tools"].items():
tool_module = import_class(tool["module"])
if not tool_module.is_ready():
tool_module.setup()
version = import_class(tool["module"]).version()
tools[tool_name] = dict(
module=tool["module"], version=version, parameters=tool.get("parameters")
)
return tools
def run(self):
steps = self.config["steps"]["analysis"]
context = dict(output_dir=self.output_dir, db_path=self.db_path)
for step in steps:
logger.debug(f"Running {step['module']}")
module = import_class(step["module"])
module.execute(context, step["config"])
def run(self):
steps = Step.select().where(Step.category == Step.ANALYSIS)
context = dict(output_dir=self.output_dir, db_path=self.db_path)
for step in steps:
logger.debug(f"Running {step.module}")
module = import_class(step.module)
config = json.loads(step.config)
module.execute(context, config)
def get_dataframe(cls, config):
joins = config.get("joins", [])
query = Run.select()
for model_class in joins:
model = import_class(model_class)
query = query.join_from(
Run, model).select_extend(*model._meta.fields.values())
sql, params = query.sql()
return pd.read_sql_query(sql, db, params=params)
def bootstrap_observers(config, observe_args):
count = Observer.select().count()
new_observers = config["observers"][count:]
if len(new_observers) > 0:
query = Observer.insert_many(
[
{
"module": observer["module"],
"config": json.dumps(observer.get("config", None)),
}
for observer in new_observers
]
)
query.execute()
for observer in new_observers:
observer_class = import_class(observer["module"])
gevent.spawn(observer_class.observe, *observe_args)
def register_steps(config):
logger.info("Registering steps...")
for step in itertools.chain.from_iterable(config["steps"].values()):
import_class(step["module"]).register(step.get("config", {}))
