def get_args(): aparser = argparse.ArgumentParser(description='Execute a job.') aparser.add_argument('--logfile', nargs='?', action='store', help='Path logfile. ["pipe_cron.log"]') aparser.add_argument('--debug', nargs='?', action='store', help='true, if exec_pipe should start in debug mode. [false]') args = aparser.parse_args() if args.logfile is None: raise Exception('A logfile argument is required!') else: logfile = args.logfile if args.debug is None: debug = False else: debug = args.debug.lower() == 'true' if debug: logging.basicConfig(filename=logfile, filemode='a',level=logging.DEBUG, format='%(asctime)s %(message)s') else: logging.basicConfig(filename=logfile, filemode='a',level=logging.INFO, format='%(asctime)s %(message)s') return config.LOSTConfig()
def release_annos_on_session_timeout(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) c_imgs, c_2dannos = release_annos_by_timeout(dbm, lostconfig.session_timeout) dbm.close_session() return c_imgs, c_2dannos
def main(): parser = argparse.ArgumentParser(description='Run LOST cronjobs') parser.add_argument('--debug', action='store_true', help='start cronjobs just once for debugging') args = parser.parse_args() lostconfig = config.LOSTConfig() fm = AppFileMan(lostconfig) log_name = 'cron_jobs' logger = get_file_logger(log_name, fm.get_app_log_path('cron_jobs.log')) logger.info('Starting cron jobs!') if args.debug: t = threading.Thread(target=worker_lifesign_loop, args=(log_name, ), daemon=True) t.start() client = Client('{}:{}'.format(lostconfig.scheduler_ip, lostconfig.scheduler_port)) process_pipes(log_name, client) else: jobs = [ process_pipes_loop, worker_lifesign_loop, release_annos_loop, remove_empty_annos_loop ] if lostconfig.worker_management == 'dynamic': jobs.append(dask_session.release_client_by_timeout_loop) jobs += lostconfig.extra_cron_jobs threads = [] for j in jobs: t = threading.Thread(target=j, args=(log_name, ), daemon=True) t.start() threads.append(t) [t.join() for t in threads]
def main(args): lostconfig = config.LOSTConfig() dbm = access.DBMan(lostconfig) if args.csv_file is not None: df = pd.read_csv(args.csv_file) name = df[df['parent_leaf_id'].isnull()]['name'].values[0] elif args.name is not None: name = args.name else: logging.error( "Either a *csv_file* or a *name* for a label tree needs to be provided!" ) return root_leaf = next( filter(lambda x: x.name == name, dbm.get_all_label_trees()), None) if root_leaf is None: logging.warning('LabelTree not present in database! {}'.format( args.csv_file)) else: try: LabelTree(dbm, root_leaf=root_leaf, logger=logging).delete_tree() logging.info('Deleted tree with name: {}'.format(name)) except: logging.error( 'Can not delete label tree. One of the labels is used by a pipeline!' ) dbm.close_session()
def test_create_delete_tree(self): assert True return None dbm = DBMan(config.LOSTConfig()) tree = LabelTree(dbm) root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID) assert root_leaf.name == ROOT_NAME assert root_leaf.external_id == ROOT_EXTERNAL_ID assert tree.tree[root_leaf.idx] == root_leaf tree.delete_tree()
def process_pipes_loop(log_name): lostconfig = config.LOSTConfig() if lostconfig.worker_management != 'dynamic': client = Client('{}:{}'.format(lostconfig.scheduler_ip, lostconfig.scheduler_port)) else: client = None run_loop(process_pipes, lostconfig.pipe_schedule, log_name=log_name, client=client)
def full_img_anno(): dbm = DBMan(config.LOSTConfig()) test_user = testils.get_user(dbm) tree = testils.get_voc_label_tree(dbm) label_vec = tree.get_child_vec(tree.root.idx) twod_anno = model.TwoDAnno(data=json.dumps({ 'x': REF_BBOX[0], 'y': REF_BBOX[1], 'w': REF_BBOX[2], 'h': REF_BBOX[3] }), dtype=dtype.TwoDAnno.BBOX) lbl = model.Label(label_leaf_id=label_vec[0]) dbm.add(lbl) twod_anno.labels.append(lbl) twod_anno.annotator = test_user twod_anno2 = model.TwoDAnno(data=json.dumps({ 'x': REF_POINT[0], 'y': REF_POINT[1] }), dtype=dtype.TwoDAnno.POINT) lbl = model.Label(label_leaf_id=label_vec[1]) dbm.add(lbl) twod_anno2.labels.append(lbl) twod_anno2.annotator = test_user line = model.TwoDAnno() lbl = model.Label(label_leaf_id=label_vec[4]) line.labels.append(lbl) dbm.add(lbl) line.annotator = test_user line.line = REF_LINE img_anno = model.ImageAnno(img_path='path/to/img1.jpg') fs = add_local_fs(dbm, 'local_fs_for_full_img_anno') img_anno.fs_id = fs.idx lbl = model.Label(label_leaf_id=label_vec[3]) dbm.add(lbl) img_anno.labels.append(lbl) dbm.commit() img_anno.twod_annos.append(twod_anno) img_anno.twod_annos.append(twod_anno2) img_anno.twod_annos.append(line) # dbm.add(img_anno.labels) dbm.add(img_anno) # dbm.add(twod_anno) dbm.commit() yield img_anno # dbm.delete(twod_anno.labels) dbm.delete(twod_anno) # dbm.delete(twod_anno2.labels) dbm.delete(twod_anno2) # dbm.delete(img_anno.labels) dbm.delete(img_anno) dbm.commit() delete_local_fs(dbm, fs)
def empty_img_anno(): dbm = DBMan(config.LOSTConfig()) img_anno = model.ImageAnno(img_path='path/to/img1.jpg') fs = add_local_fs(dbm, 'local_fs_for_empty_img_anno') img_anno.fs_id = fs.idx dbm.add(img_anno) dbm.commit() yield img_anno dbm.delete(img_anno) dbm.commit() delete_local_fs(dbm, fs)
def local_fs(): dbm = DBMan(config.LOSTConfig()) test_user = testils.get_user(dbm) fs = model.FileSystem(group_id=test_user.groups[0].idx, connection=json.dumps(dict()), root_path='', fs_type='file', timestamp=datetime.datetime.now(), name='test_request_annos_fs') dbm.add(fs) dbm.commit() yield fs
def tree_plus_childs(): dbm = DBMan(config.LOSTConfig()) tree = LabelTree(dbm) root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID) horse = tree.create_child(tree.root.idx, CHILD_HORSE_NAME, external_id=CHILD_HORSE_EXTERNAL_ID) cow = tree.create_child(tree.root.idx, CHILD_COW_NAME, external_id=CHILD_COW_EXTERNAL_ID) yield tree # type: lost.logic.label.LabelTree tree.delete_tree()
def process_pipes(log_name, client): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) pipe_list = dbm.get_pipes_to_process() # For each task in this project for p in pipe_list: pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig, client=client, logger_name=log_name) pipe_man.process_pipeline() dbm.close_session()
def main(): lostconfig = config.LOSTConfig() # project_root = join(lostconfig.project_path, "data") # if not os.path.exists(project_root): # os.makedirs(project_root) fman = file_man.FileMan(lostconfig) fman.create_project_folders() # Create Tables dbm = access.DBMan(lostconfig) dbm.create_database() create_first_user(dbm) create_lost_filesystem_entry(dbm, lostconfig) create_project_config(dbm) dbm.close_session()
def simple_bbox_anno(): dbm = DBMan(config.LOSTConfig()) twod_anno = model.TwoDAnno(data=json.dumps({ 'x': REF_BBOX[0], 'y': REF_BBOX[1], 'w': REF_BBOX[2], 'h': REF_BBOX[3] }), dtype=dtype.TwoDAnno.BBOX) dbm.add(twod_anno) dbm.commit() yield twod_anno dbm.delete(twod_anno) dbm.commit()
def exec_pipe(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) pipe_list = dbm.get_pipes_to_process() if lostconfig.worker_management != 'dynamic': client = Client('{}:{}'.format(lostconfig.scheduler_ip, lostconfig.scheduler_port)) else: client = None # For each task in this project for p in pipe_list: pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig, client=client) pipe_man.process_pipeline() dbm.close_session()
def full_bbox_anno(): dbm = DBMan(config.LOSTConfig()) test_user = testils.get_user(dbm) tree = testils.get_voc_label_tree(dbm) label_leaf_id = tree.get_child_vec(tree.root.idx)[0] twod_anno = model.TwoDAnno(data=json.dumps({ 'x': REF_BBOX[0], 'y': REF_BBOX[1], 'w': REF_BBOX[2], 'h': REF_BBOX[3] }), dtype=dtype.TwoDAnno.BBOX) lbl = model.Label(label_leaf_id=label_leaf_id) twod_anno.labels.append(lbl) twod_anno.annotator = test_user dbm.add(twod_anno) dbm.commit() yield twod_anno dbm.delete(twod_anno) dbm.commit()
def test_import_df(self, tree_plus_childs): assert True return None df = tree_plus_childs.to_df() df2 = df.copy() root_idx = df2[df2['parent_leaf_id'].isnull()].index.values[0] no_root_idx = df2[~df2['parent_leaf_id'].isnull()].index df2.loc[root_idx, 'name'] = 'second tree' df2.loc[root_idx, 'idx'] = 0 df2.loc[no_root_idx, 'parent_leaf_id'] = 0 dbm = DBMan(config.LOSTConfig()) tree2 = LabelTree(dbm) root_leaf = tree2.import_df(df2) if root_leaf is None: raise Exception( 'A label tree with name "{}" already exists. Clean your Test Database!' .format(df2.loc[root_idx, 'name'])) for key, val in tree2.tree.items(): print(val.to_df()[['idx', 'name', 'external_id', 'parent_leaf_id']]) for ll in tree2.root.label_leaves: assert ll.name == CHILD_COW_NAME or ll.name == CHILD_HORSE_NAME tree2.delete_tree()
def tree(): dbm = DBMan(config.LOSTConfig()) tree = testils.get_voc_label_tree(dbm) yield tree
def script_element(): dbm = DBMan(config.LOSTConfig()) pe_s, pe_a, pipe = testils.get_script_pipeline_fragment(dbm) yield pe_s testils.delete_script_pipeline_fragment(dbm, pipe)
def tree(): dbm = DBMan(config.LOSTConfig()) tree = LabelTree(dbm) root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID) yield tree tree.delete_tree()
import os from datetime import date, datetime, timedelta from lost.logic.pipeline import exec_utils import logging from time import sleep # from lost.lost_session import lost_session from lost.logic.file_man import AppFileMan, FileMan import lostconfig config = lostconfig.LOSTConfig() def _read_fs_img(fs, img_path): fm = FileMan(fs_db=fs) img = fm.load_img(img_path) return img def _user_key(user): # return user.idx return getattr(user, config.dask_user_key) def release_client_by_timeout_loop(log_name): logger = logging.getLogger(log_name) logger.info('Run release_client_by_timeout_loop') while True: ds_man.shutdown_timed_out_clients(logger) sleep(config.session_timeout / 2)
def __init__(self): self.session = dict() self.lostconfig = lostconfig.LOSTConfig()
def worker_lifesign_loop(log_name): lostconfig = config.LOSTConfig() run_loop(worker_lifesign, lostconfig.worker_beat, log_name=log_name)
def remove_empty_annos(): lostconfig = config.LOSTConfig() dbm = DBMan(lostconfig) c_2dannos = remove_empty_annos_by_timeout(dbm, lostconfig.session_timeout) dbm.close_session() return c_2dannos
def remove_empty_annos_loop(log_name): lostconfig = config.LOSTConfig() run_loop(remove_empty_annos, lostconfig.session_timeout * 60, log_name=log_name)