def get_args():
aparser = argparse.ArgumentParser(description='Execute a job.')
aparser.add_argument('--logfile', nargs='?', action='store',
help='Path logfile. ["pipe_cron.log"]')
aparser.add_argument('--debug', nargs='?', action='store',
help='true, if exec_pipe should start in debug mode. [false]')
args = aparser.parse_args()
if args.logfile is None:
raise Exception('A logfile argument is required!')
else:
logfile = args.logfile
if args.debug is None:
debug = False
else:
debug = args.debug.lower() == 'true'
if debug:
logging.basicConfig(filename=logfile, filemode='a',level=logging.DEBUG,
format='%(asctime)s %(message)s')
else:
logging.basicConfig(filename=logfile, filemode='a',level=logging.INFO,
format='%(asctime)s %(message)s')
return config.LOSTConfig()
def release_annos_on_session_timeout():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
c_imgs, c_2dannos = release_annos_by_timeout(dbm,
lostconfig.session_timeout)
dbm.close_session()
return c_imgs, c_2dannos
def main():
parser = argparse.ArgumentParser(description='Run LOST cronjobs')
parser.add_argument('--debug',
action='store_true',
help='start cronjobs just once for debugging')
args = parser.parse_args()
lostconfig = config.LOSTConfig()
fm = AppFileMan(lostconfig)
log_name = 'cron_jobs'
logger = get_file_logger(log_name, fm.get_app_log_path('cron_jobs.log'))
logger.info('Starting cron jobs!')
if args.debug:
t = threading.Thread(target=worker_lifesign_loop,
args=(log_name, ),
daemon=True)
t.start()
client = Client('{}:{}'.format(lostconfig.scheduler_ip,
lostconfig.scheduler_port))
process_pipes(log_name, client)
else:
jobs = [
process_pipes_loop, worker_lifesign_loop, release_annos_loop,
remove_empty_annos_loop
]
if lostconfig.worker_management == 'dynamic':
jobs.append(dask_session.release_client_by_timeout_loop)
jobs += lostconfig.extra_cron_jobs
threads = []
for j in jobs:
t = threading.Thread(target=j, args=(log_name, ), daemon=True)
t.start()
threads.append(t)
[t.join() for t in threads]
def main(args):
lostconfig = config.LOSTConfig()
dbm = access.DBMan(lostconfig)
if args.csv_file is not None:
df = pd.read_csv(args.csv_file)
name = df[df['parent_leaf_id'].isnull()]['name'].values[0]
elif args.name is not None:
name = args.name
else:
logging.error(
"Either a *csv_file* or a *name* for a label tree needs to be provided!"
)
return
root_leaf = next(
filter(lambda x: x.name == name, dbm.get_all_label_trees()), None)
if root_leaf is None:
logging.warning('LabelTree not present in database! {}'.format(
args.csv_file))
else:
try:
LabelTree(dbm, root_leaf=root_leaf, logger=logging).delete_tree()
logging.info('Deleted tree with name: {}'.format(name))
except:
logging.error(
'Can not delete label tree. One of the labels is used by a pipeline!'
)
dbm.close_session()
def test_create_delete_tree(self):
assert True
return None
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
assert root_leaf.name == ROOT_NAME
assert root_leaf.external_id == ROOT_EXTERNAL_ID
assert tree.tree[root_leaf.idx] == root_leaf
tree.delete_tree()
def process_pipes_loop(log_name):
lostconfig = config.LOSTConfig()
if lostconfig.worker_management != 'dynamic':
client = Client('{}:{}'.format(lostconfig.scheduler_ip,
lostconfig.scheduler_port))
else:
client = None
run_loop(process_pipes,
lostconfig.pipe_schedule,
log_name=log_name,
client=client)
def full_img_anno():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
tree = testils.get_voc_label_tree(dbm)
label_vec = tree.get_child_vec(tree.root.idx)
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
lbl = model.Label(label_leaf_id=label_vec[0])
dbm.add(lbl)
twod_anno.labels.append(lbl)
twod_anno.annotator = test_user
twod_anno2 = model.TwoDAnno(data=json.dumps({
'x': REF_POINT[0],
'y': REF_POINT[1]
}),
dtype=dtype.TwoDAnno.POINT)
lbl = model.Label(label_leaf_id=label_vec[1])
dbm.add(lbl)
twod_anno2.labels.append(lbl)
twod_anno2.annotator = test_user
line = model.TwoDAnno()
lbl = model.Label(label_leaf_id=label_vec[4])
line.labels.append(lbl)
dbm.add(lbl)
line.annotator = test_user
line.line = REF_LINE
img_anno = model.ImageAnno(img_path='path/to/img1.jpg')
fs = add_local_fs(dbm, 'local_fs_for_full_img_anno')
img_anno.fs_id = fs.idx
lbl = model.Label(label_leaf_id=label_vec[3])
dbm.add(lbl)
img_anno.labels.append(lbl)
dbm.commit()
img_anno.twod_annos.append(twod_anno)
img_anno.twod_annos.append(twod_anno2)
img_anno.twod_annos.append(line)
# dbm.add(img_anno.labels)
dbm.add(img_anno)
# dbm.add(twod_anno)
dbm.commit()
yield img_anno
# dbm.delete(twod_anno.labels)
dbm.delete(twod_anno)
# dbm.delete(twod_anno2.labels)
dbm.delete(twod_anno2)
# dbm.delete(img_anno.labels)
dbm.delete(img_anno)
dbm.commit()
delete_local_fs(dbm, fs)
def empty_img_anno():
dbm = DBMan(config.LOSTConfig())
img_anno = model.ImageAnno(img_path='path/to/img1.jpg')
fs = add_local_fs(dbm, 'local_fs_for_empty_img_anno')
img_anno.fs_id = fs.idx
dbm.add(img_anno)
dbm.commit()
yield img_anno
dbm.delete(img_anno)
dbm.commit()
delete_local_fs(dbm, fs)
def local_fs():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
fs = model.FileSystem(group_id=test_user.groups[0].idx,
connection=json.dumps(dict()),
root_path='',
fs_type='file',
timestamp=datetime.datetime.now(),
name='test_request_annos_fs')
dbm.add(fs)
dbm.commit()
yield fs
def tree_plus_childs():
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
horse = tree.create_child(tree.root.idx,
CHILD_HORSE_NAME,
external_id=CHILD_HORSE_EXTERNAL_ID)
cow = tree.create_child(tree.root.idx,
CHILD_COW_NAME,
external_id=CHILD_COW_EXTERNAL_ID)
yield tree # type: lost.logic.label.LabelTree
tree.delete_tree()
def process_pipes(log_name, client):
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
pipe_list = dbm.get_pipes_to_process()
# For each task in this project
for p in pipe_list:
pipe_man = cron.PipeEngine(dbm=dbm,
pipe=p,
lostconfig=lostconfig,
client=client,
logger_name=log_name)
pipe_man.process_pipeline()
dbm.close_session()
def main():
lostconfig = config.LOSTConfig()
# project_root = join(lostconfig.project_path, "data")
# if not os.path.exists(project_root):
# os.makedirs(project_root)
fman = file_man.FileMan(lostconfig)
fman.create_project_folders()
# Create Tables
dbm = access.DBMan(lostconfig)
dbm.create_database()
create_first_user(dbm)
create_lost_filesystem_entry(dbm, lostconfig)
create_project_config(dbm)
dbm.close_session()
def simple_bbox_anno():
dbm = DBMan(config.LOSTConfig())
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
dbm.add(twod_anno)
dbm.commit()
yield twod_anno
dbm.delete(twod_anno)
dbm.commit()
def exec_pipe():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
pipe_list = dbm.get_pipes_to_process()
if lostconfig.worker_management != 'dynamic':
client = Client('{}:{}'.format(lostconfig.scheduler_ip,
lostconfig.scheduler_port))
else:
client = None
# For each task in this project
for p in pipe_list:
pipe_man = cron.PipeEngine(dbm=dbm,
pipe=p,
lostconfig=lostconfig,
client=client)
pipe_man.process_pipeline()
dbm.close_session()
def full_bbox_anno():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
tree = testils.get_voc_label_tree(dbm)
label_leaf_id = tree.get_child_vec(tree.root.idx)[0]
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
lbl = model.Label(label_leaf_id=label_leaf_id)
twod_anno.labels.append(lbl)
twod_anno.annotator = test_user
dbm.add(twod_anno)
dbm.commit()
yield twod_anno
dbm.delete(twod_anno)
dbm.commit()
def test_import_df(self, tree_plus_childs):
assert True
return None
df = tree_plus_childs.to_df()
df2 = df.copy()
root_idx = df2[df2['parent_leaf_id'].isnull()].index.values[0]
no_root_idx = df2[~df2['parent_leaf_id'].isnull()].index
df2.loc[root_idx, 'name'] = 'second tree'
df2.loc[root_idx, 'idx'] = 0
df2.loc[no_root_idx, 'parent_leaf_id'] = 0
dbm = DBMan(config.LOSTConfig())
tree2 = LabelTree(dbm)
root_leaf = tree2.import_df(df2)
if root_leaf is None:
raise Exception(
'A label tree with name "{}" already exists. Clean your Test Database!'
.format(df2.loc[root_idx, 'name']))
for key, val in tree2.tree.items():
print(val.to_df()[['idx', 'name', 'external_id',
'parent_leaf_id']])
for ll in tree2.root.label_leaves:
assert ll.name == CHILD_COW_NAME or ll.name == CHILD_HORSE_NAME
tree2.delete_tree()
def tree():
dbm = DBMan(config.LOSTConfig())
tree = testils.get_voc_label_tree(dbm)
yield tree
def script_element():
dbm = DBMan(config.LOSTConfig())
pe_s, pe_a, pipe = testils.get_script_pipeline_fragment(dbm)
yield pe_s
testils.delete_script_pipeline_fragment(dbm, pipe)
def tree():
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
yield tree
tree.delete_tree()
import os
from datetime import date, datetime, timedelta
from lost.logic.pipeline import exec_utils
import logging
from time import sleep
# from lost.lost_session import lost_session
from lost.logic.file_man import AppFileMan, FileMan
import lostconfig
config = lostconfig.LOSTConfig()
def _read_fs_img(fs, img_path):
fm = FileMan(fs_db=fs)
img = fm.load_img(img_path)
return img
def _user_key(user):
# return user.idx
return getattr(user, config.dask_user_key)
def release_client_by_timeout_loop(log_name):
logger = logging.getLogger(log_name)
logger.info('Run release_client_by_timeout_loop')
while True:
ds_man.shutdown_timed_out_clients(logger)
sleep(config.session_timeout / 2)
def __init__(self):
self.session = dict()
self.lostconfig = lostconfig.LOSTConfig()
def worker_lifesign_loop(log_name):
lostconfig = config.LOSTConfig()
run_loop(worker_lifesign, lostconfig.worker_beat, log_name=log_name)
def remove_empty_annos():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
c_2dannos = remove_empty_annos_by_timeout(dbm, lostconfig.session_timeout)
dbm.close_session()
return c_2dannos
def remove_empty_annos_loop(log_name):
lostconfig = config.LOSTConfig()
run_loop(remove_empty_annos,
lostconfig.session_timeout * 60,
log_name=log_name)
