def release_annos_on_session_timeout(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) c_imgs, c_2dannos = release_annos_by_timeout(dbm, lostconfig.session_timeout) dbm.close_session() return c_imgs, c_2dannos
def delete_pipe(pipe_id): # logger = get_task_logger(__name__) # logger.info("DELETED BY CELERY {}".format(pipe_id)) lostconfig = LOSTConfig() dbm = DBMan(lostconfig) pipeline.delete(dbm, pipe_id) dbm.close_session()
def exec_pipe(): lostconfig = get_args() dbm = DBMan(lostconfig) pipe_list = dbm.get_pipes_to_process() # For each task in this project for p in pipe_list: pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig) pipe_man.process_pipeline() dbm.close_session()
def celery_exec_script(pipe_element_id): try: # Collect context information for celery task logger = get_task_logger(__name__) lostconfig = LOSTConfig() dbm = DBMan(lostconfig) pipe_e = dbm.get_pipe_element(pipe_e_id=pipe_element_id) worker = CurrentWorker(dbm, lostconfig) if not worker.enough_resources(pipe_e.script): logger.warning( 'Not enough resources! Rejected {} (PipeElement ID {})'.format( pipe_e.script.path, pipe_e.idx)) return pipe_e.state = state.PipeElement.IN_PROGRESS dbm.save_obj(pipe_e) file_man = FileMan(lostconfig) pipe = pipe_e.pipe cmd = gen_run_cmd("pudb3", pipe_e, lostconfig) debug_script_path = file_man.get_instance_path(pipe_e) debug_script_path = os.path.join(debug_script_path, 'debug.sh') with open(debug_script_path, 'w') as sfile: sfile.write(cmd) cmd = gen_run_cmd("python3", pipe_e, lostconfig) start_script_path = file_man.get_instance_path(pipe_e) start_script_path = os.path.join(start_script_path, 'start.sh') with open(start_script_path, 'w') as sfile: sfile.write(cmd) p = subprocess.Popen('bash {}'.format(start_script_path), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) logger.info("{} ({}): Started script\n{}".format( pipe.name, pipe.idx, cmd)) worker.add_script(pipe_e, pipe_e.script) out, err = p.communicate() worker.remove_script(pipe_e, pipe_e.script) if p.returncode != 0: raise Exception(err.decode('utf-8')) logger.info('{} ({}): Executed script successful: {}'.format( pipe.name, pipe.idx, pipe_e.script.path)) dbm.close_session() except: pipe = pipe_e.pipe logger.info('{} ({}): Exception occurred in script: {}'.format( pipe.name, pipe.idx, pipe_e.script.path)) msg = traceback.format_exc() logger.error(msg) script_api.report_script_err(pipe_e, pipe, dbm, msg) dbm.close_session()
def process_pipes(log_name, client): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) pipe_list = dbm.get_pipes_to_process() # For each task in this project for p in pipe_list: pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig, client=client, logger_name=log_name) pipe_man.process_pipeline() dbm.close_session()
def send_life_sign(): logger = get_task_logger(__name__) lostconfig = LOSTConfig() dbm = DBMan(lostconfig) worker = dbm.get_worker(lostconfig.worker_name) if worker is None: register_worker(dbm, lostconfig) logger.info('Registered worker: {}'.format(lostconfig.worker_name)) else: worker.timestamp = datetime.utcnow() dbm.add(worker) dbm.commit() logger.info('Sent lifesign: {}'.format(worker.worker_name)) dbm.close_session()
def init_worker_on_startup(): lostconfig = LOSTConfig() dbm = DBMan(lostconfig) worker = dbm.get_worker(lostconfig.worker_name) if worker is None: register_worker(dbm, lostconfig) print('Registered worker: {}'.format(lostconfig.worker_name)) else: worker.timestamp = datetime.utcnow() worker.resources = '[]' worker.in_progress = '{}' dbm.add(worker) dbm.commit() print('Reset worker on startup: {}'.format(worker.worker_name)) dbm.close_session()
def exec_pipe(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) pipe_list = dbm.get_pipes_to_process() if lostconfig.worker_management != 'dynamic': client = Client('{}:{}'.format(lostconfig.scheduler_ip, lostconfig.scheduler_port)) else: client = None # For each task in this project for p in pipe_list: pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig, client=client) pipe_man.process_pipeline() dbm.close_session()
def remove_empty_annos(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) c_2dannos = remove_empty_annos_by_timeout(dbm, lostconfig.session_timeout) dbm.close_session() return c_2dannos
def release_annos(): lostconfig = get_args() dbm = DBMan(lostconfig) __release_project_annos(dbm) dbm.close_session()