def delete_pipe(pipe_id):
# logger = get_task_logger(__name__)
# logger.info("DELETED BY CELERY {}".format(pipe_id))
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipeline.delete(dbm, pipe_id)
dbm.close_session()
def script_element():
dbm = DBMan(config.LOSTConfig())
pe_s, pe_a, pipe = testils.get_script_pipeline_fragment(dbm)
yield pe_s
testils.delete_script_pipeline_fragment(
dbm, pipe
)
def release_annos_on_session_timeout():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
c_imgs, c_2dannos = release_annos_by_timeout(dbm,
lostconfig.session_timeout)
dbm.close_session()
return c_imgs, c_2dannos
def test_create_delete_tree(self):
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
assert root_leaf.name == ROOT_NAME
assert root_leaf.external_id == ROOT_EXTERNAL_ID
assert tree.tree[root_leaf.idx] == root_leaf
tree.delete_tree()
def exec_pipe():
lostconfig = get_args()
dbm = DBMan(lostconfig)
pipe_list = dbm.get_pipes_to_process()
# For each task in this project
for p in pipe_list:
pipe_man = cron.PipeEngine(dbm=dbm, pipe=p, lostconfig=lostconfig)
pipe_man.process_pipeline()
dbm.close_session()
def full_img_anno():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
tree = testils.get_voc_label_tree(dbm)
label_vec = tree.get_child_vec(tree.root.idx)
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
lbl = model.Label(label_leaf_id=label_vec[0])
dbm.add(lbl)
twod_anno.labels.append(lbl)
twod_anno.annotator = test_user
twod_anno2 = model.TwoDAnno(data=json.dumps({
'x': REF_POINT[0],
'y': REF_POINT[1]
}),
dtype=dtype.TwoDAnno.POINT)
lbl = model.Label(label_leaf_id=label_vec[1])
dbm.add(lbl)
twod_anno2.labels.append(lbl)
twod_anno2.annotator = test_user
line = model.TwoDAnno()
lbl = model.Label(label_leaf_id=label_vec[4])
line.labels.append(lbl)
dbm.add(lbl)
line.annotator = test_user
line.line = REF_LINE
img_anno = model.ImageAnno(img_path='path/to/img1.jpg')
fs = add_local_fs(dbm, 'local_fs_for_full_img_anno')
img_anno.fs_id = fs.idx
lbl = model.Label(label_leaf_id=label_vec[3])
dbm.add(lbl)
img_anno.labels.append(lbl)
dbm.commit()
img_anno.twod_annos.append(twod_anno)
img_anno.twod_annos.append(twod_anno2)
img_anno.twod_annos.append(line)
# dbm.add(img_anno.labels)
dbm.add(img_anno)
# dbm.add(twod_anno)
dbm.commit()
yield img_anno
# dbm.delete(twod_anno.labels)
dbm.delete(twod_anno)
# dbm.delete(twod_anno2.labels)
dbm.delete(twod_anno2)
# dbm.delete(img_anno.labels)
dbm.delete(img_anno)
dbm.commit()
delete_local_fs(dbm, fs)
def empty_img_anno():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
tree = testils.get_voc_label_tree(dbm)
label_leaf_id = tree.get_child_vec(tree.root.idx)[0]
img_anno = model.ImageAnno(img_path='path/to/img1.jpg')
dbm.add(img_anno)
dbm.commit()
yield img_anno
dbm.delete(img_anno)
dbm.commit()
def empty_img_anno():
dbm = DBMan(config.LOSTConfig())
img_anno = model.ImageAnno(img_path='path/to/img1.jpg')
fs = add_local_fs(dbm, 'local_fs_for_empty_img_anno')
img_anno.fs_id = fs.idx
dbm.add(img_anno)
dbm.commit()
yield img_anno
dbm.delete(img_anno)
dbm.commit()
delete_local_fs(dbm, fs)
def tree_plus_childs():
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
horse = tree.create_child(tree.root.idx,
CHILD_HORSE_NAME,
external_id=CHILD_HORSE_EXTERNAL_ID)
cow = tree.create_child(tree.root.idx,
CHILD_COW_NAME,
external_id=CHILD_COW_EXTERNAL_ID)
yield tree # type: lost.logic.label.LabelTree
tree.delete_tree()
def local_fs():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
fs = model.FileSystem(group_id=test_user.groups[0].idx,
connection=json.dumps(dict()),
root_path='',
fs_type='file',
timestamp=datetime.datetime.now(),
name='test_request_annos_fs')
dbm.add(fs)
dbm.commit()
yield fs
def celery_exec_script(pipe_element_id):
try:
# Collect context information for celery task
logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipe_e = dbm.get_pipe_element(pipe_e_id=pipe_element_id)
worker = CurrentWorker(dbm, lostconfig)
if not worker.enough_resources(pipe_e.script):
logger.warning(
'Not enough resources! Rejected {} (PipeElement ID {})'.format(
pipe_e.script.path, pipe_e.idx))
return
pipe_e.state = state.PipeElement.IN_PROGRESS
dbm.save_obj(pipe_e)
file_man = FileMan(lostconfig)
pipe = pipe_e.pipe
cmd = gen_run_cmd("pudb3", pipe_e, lostconfig)
debug_script_path = file_man.get_instance_path(pipe_e)
debug_script_path = os.path.join(debug_script_path, 'debug.sh')
with open(debug_script_path, 'w') as sfile:
sfile.write(cmd)
cmd = gen_run_cmd("python3", pipe_e, lostconfig)
start_script_path = file_man.get_instance_path(pipe_e)
start_script_path = os.path.join(start_script_path, 'start.sh')
with open(start_script_path, 'w') as sfile:
sfile.write(cmd)
p = subprocess.Popen('bash {}'.format(start_script_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
logger.info("{} ({}): Started script\n{}".format(
pipe.name, pipe.idx, cmd))
worker.add_script(pipe_e, pipe_e.script)
out, err = p.communicate()
worker.remove_script(pipe_e, pipe_e.script)
if p.returncode != 0:
raise Exception(err.decode('utf-8'))
logger.info('{} ({}): Executed script successful: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
dbm.close_session()
except:
pipe = pipe_e.pipe
logger.info('{} ({}): Exception occurred in script: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
msg = traceback.format_exc()
logger.error(msg)
script_api.report_script_err(pipe_e, pipe, dbm, msg)
dbm.close_session()
def process_pipes(log_name, client):
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
pipe_list = dbm.get_pipes_to_process()
# For each task in this project
for p in pipe_list:
pipe_man = cron.PipeEngine(dbm=dbm,
pipe=p,
lostconfig=lostconfig,
client=client,
logger_name=log_name)
pipe_man.process_pipeline()
dbm.close_session()
def send_life_sign():
logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
logger.info('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
dbm.add(worker)
dbm.commit()
logger.info('Sent lifesign: {}'.format(worker.worker_name))
dbm.close_session()
def simple_bbox_anno():
dbm = DBMan(config.LOSTConfig())
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
dbm.add(twod_anno)
dbm.commit()
yield twod_anno
dbm.delete(twod_anno)
dbm.commit()
def init_worker_on_startup():
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
print('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
worker.resources = '[]'
worker.in_progress = '{}'
dbm.add(worker)
dbm.commit()
print('Reset worker on startup: {}'.format(worker.worker_name))
dbm.close_session()
def exec_pipe():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
pipe_list = dbm.get_pipes_to_process()
if lostconfig.worker_management != 'dynamic':
client = Client('{}:{}'.format(lostconfig.scheduler_ip,
lostconfig.scheduler_port))
else:
client = None
# For each task in this project
for p in pipe_list:
pipe_man = cron.PipeEngine(dbm=dbm,
pipe=p,
lostconfig=lostconfig,
client=client)
pipe_man.process_pipeline()
dbm.close_session()
def full_bbox_anno():
dbm = DBMan(config.LOSTConfig())
test_user = testils.get_user(dbm)
tree = testils.get_voc_label_tree(dbm)
label_leaf_id = tree.get_child_vec(tree.root.idx)[0]
twod_anno = model.TwoDAnno(data=json.dumps({
'x': REF_BBOX[0],
'y': REF_BBOX[1],
'w': REF_BBOX[2],
'h': REF_BBOX[3]
}),
dtype=dtype.TwoDAnno.BBOX)
lbl = model.Label(label_leaf_id=label_leaf_id)
twod_anno.labels.append(lbl)
twod_anno.annotator = test_user
dbm.add(twod_anno)
dbm.commit()
yield twod_anno
dbm.delete(twod_anno)
dbm.commit()
def test_import_df(self, tree_plus_childs):
df = tree_plus_childs.to_df()
df2 = df.copy()
root_idx = df2[df2['parent_leaf_id'].isnull()].index.values[0]
no_root_idx = df2[~df2['parent_leaf_id'].isnull()].index
df2.loc[root_idx, 'name'] = 'second tree'
df2.loc[root_idx, 'idx'] = 0
df2.loc[no_root_idx, 'parent_leaf_id'] = 0
dbm = DBMan(config.LOSTConfig())
tree2 = LabelTree(dbm)
root_leaf = tree2.import_df(df2)
if root_leaf is None:
raise Exception(
'A label tree with name "{}" already exists. Clean your Test Database!'
.format(df2.loc[root_idx, 'name']))
for key, val in tree2.tree.items():
print(val.to_df()[['idx', 'name', 'external_id',
'parent_leaf_id']])
for ll in tree2.root.label_leaves:
assert ll.name == CHILD_COW_NAME or ll.name == CHILD_HORSE_NAME
tree2.delete_tree()
def remove_empty_annos():
lostconfig = config.LOSTConfig()
dbm = DBMan(lostconfig)
c_2dannos = remove_empty_annos_by_timeout(dbm, lostconfig.session_timeout)
dbm.close_session()
return c_2dannos
def tree():
dbm = DBMan(config.LOSTConfig())
tree = testils.get_voc_label_tree(dbm)
yield tree
def tree():
dbm = DBMan(config.LOSTConfig())
tree = LabelTree(dbm)
root_leaf = tree.create_root(ROOT_NAME, external_id=ROOT_EXTERNAL_ID)
yield tree
tree.delete_tree()
def release_annos():
lostconfig = get_args()
dbm = DBMan(lostconfig)
__release_project_annos(dbm)
dbm.close_session()