def __init__(self, pe_id=None):
if pe_id is None:
parser = argparse.ArgumentParser(
description='A user defined script.')
parser.add_argument('--idx',
nargs='?',
action='store',
help='Id of related pipeline element.')
args = parser.parse_args()
lostconfig = LOSTConfig()
dbm = access.DBMan(lostconfig)
db_fs = dbm.get_fs(name='lost_data')
self.file_man = FileMan(fs_db=db_fs)
self._dbm = dbm #type: lost.db.access.DBMan
if pe_id is None:
pe = dbm.get_pipe_element(int(args.idx))
else:
pe = dbm.get_pipe_element(pe_id)
super().__init__(pe, dbm)
logfile_path = self.file_man.get_pipe_log_path(self._pipe.idx)
self._log_stream = self.file_man.fs.open(logfile_path, 'a')
self._logger = log.get_stream_logger(os.path.basename(pe.script.path),
self._log_stream)
if self.pipe_info.logfile_path is None or not self.pipe_info.logfile_path:
self.pipe_info.logfile_path = self.get_rel_path(logfile_path)
self._inp = inout.Input(self)
self._outp = inout.ScriptOutput(self)
self.rejected_execution = False
# If pe_id is None we have a normal script
# If pe_id is not None a JupyterNotebook uses this script
if pe_id is None:
return self._run()
def __init__(self, pe_id=None):
if pe_id is None:
parser = argparse.ArgumentParser(description='A user defined script.')
parser.add_argument('--idx', nargs='?', action='store',
help='Id of related pipeline element.')
args = parser.parse_args()
lostconfig = LOSTConfig()
self.file_man = FileMan(lostconfig)
dbm = access.DBMan(lostconfig)
self._dbm = dbm #type: lost.db.access.DBMan
if pe_id is None:
pe = dbm.get_pipe_element(int(args.idx))
else:
pe = dbm.get_pipe_element(pe_id)
super().__init__(pe, dbm)
logfile_path = self.file_man.get_pipe_log_path(self._pipe.idx)
self._logger = log.get_file_logger(os.path.basename(pe.script.path),
logfile_path)
if self.pipe_info.logfile_path is None or not self.pipe_info.logfile_path:
self.pipe_info.logfile_path = self.get_rel_path(logfile_path)
self._inp = inout.Input(self)
self._outp = inout.ScriptOutput(self)
self.rejected_execution = False
# If pe_id is None we have a normal script
# If pe_id is not None a JupyterNotebook uses this script
if pe_id is None:
try:
self.main()
self.i_am_done()
self._dbm.close_session()
except:
err_msg = str(datetime.datetime.now()) + '\n'
err_msg += traceback.format_exc()
self.report_err(err_msg)
self._dbm.close_session()
def __init__(self, pipe_template_dir, dbm, forTest=False):
'''Load json file.
Args:
pipe_template_dir: Path to pipeline directory
'''
self.forTest = forTest
self.dbm = dbm
self.file_man = FileMan(self.dbm.lostconfig)
if pipe_template_dir.endswith('/'):
pipe_template_dir = pipe_template_dir[:-1]
self.src_pipe_template_path = pipe_template_dir
self.dst_pipe_template_path = os.path.join(
self.file_man.pipe_path,
os.path.basename(self.src_pipe_template_path))
self.json_files = glob(os.path.join(pipe_template_dir, '*.json'))
self.pipes = []
self.namespace = os.path.basename(
self.src_pipe_template_path).strip('/')
for json_path in self.json_files:
with open(json_path) as jfile:
pipe = json.load(jfile)
pipe['namespace'] = self.namespace
pipe['name'] = self._namespaced_name(
os.path.splitext(os.path.basename(json_path))[0])
self.pipes.append(pipe)
# Set name to name of the script file
for pe in pipe['elements']:
if 'script' in pe:
pe['script']['name'] = self._namespaced_name(
pe['script']['path'])
self.checker = PipeDefChecker(logging)
def __init__(self, pe, dbm):
'''Represents a file or folder in the filesystem.
Args:
pe (object): :class:`lost.db.model.PipeElement`
dbm (object): Database Management object.
'''
super().__init__(pe, dbm)
self.file_man = FileMan(fs_db=pe.datasource.fs)
def post(self):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.ANNOTATOR):
dbm.close_session()
return "You need to be {} in order to perform this request.".format(
roles.ANNOTATOR), 401
else:
#TODO: Check if user is permitted to load this image
data = json.loads(request.data)
#flask.current_app.logger.info('mia -> getimage. Received data: {}'.format(data))
if data['type'] == 'imageBased':
db_img = dbm.get_image_anno(data['id'])
fm = FileMan(fs_db=db_img.fs)
img = load_img(db_img, fm, user)
elif data['type'] == 'annoBased':
db_anno = dbm.get_two_d_anno(two_d_anno_id=data['id'])
db_img = dbm.get_image_anno(db_anno.img_anno_id)
fm = FileMan(fs_db=db_img.fs)
# image = fm.load_img(db_img.img_path)
image = load_img(db_img, fm, user)
# get annotation_task config
draw_anno = False
context = 0.0
try:
draw_anno = data['drawAnno']
except:
pass
try:
context = float(data['addContext'])
except:
pass
df = db_img.to_df()
df = df[df['anno_uid'] == db_anno.idx]
# ds = lds.LOSTDataset(df, filesystem=fm.fs)
# ds.df.img_path = ds.df.abs_path
# ds.transform_bbox_style('x1y1x2y2', inplace=True)
# ds.to_abs(inplace=True)
# ds.df.anno_lbl = ""
# image = lds.vis_sample(image, ds.df, radius=10)
crops, _ = anno_helper.crop_boxes(df['anno_data'].values,
df['anno_dtype'].values,
image,
context=context,
draw_annotations=draw_anno)
# img = image
img = crops[0]
else:
raise Exception('Unknown mia image type')
_, data = cv2.imencode('.jpg', img)
data64 = base64.b64encode(data.tobytes())
dbm.close_session()
return u'data:img/jpg;base64,' + data64.decode('utf-8')
def __init__(self, dbm, pipe, lostconfig):
'''
:type dbm: lost.db.access.DBMan
:type pipe: lost.db.model.Pipe
'''
super().__init__(dbm=dbm, pipe=pipe)
self.lostconfig = lostconfig #type: lost.logic.config.LOSTConfig
self.file_man = FileMan(self.lostconfig)
# self.logger = lost.logic.log.get_file_logger(
# 'Executor: {}'.format(self.lostconfig.env),
# self.file_man.app_log_path)
self.logger = get_task_logger(__name__)
def celery_exec_script(pipe_element_id):
try:
# Collect context information for celery task
logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipe_e = dbm.get_pipe_element(pipe_e_id=pipe_element_id)
worker = CurrentWorker(dbm, lostconfig)
if not worker.enough_resources(pipe_e.script):
logger.warning(
'Not enough resources! Rejected {} (PipeElement ID {})'.format(
pipe_e.script.path, pipe_e.idx))
return
pipe_e.state = state.PipeElement.IN_PROGRESS
dbm.save_obj(pipe_e)
file_man = FileMan(lostconfig)
pipe = pipe_e.pipe
cmd = gen_run_cmd("pudb3", pipe_e, lostconfig)
debug_script_path = file_man.get_instance_path(pipe_e)
debug_script_path = os.path.join(debug_script_path, 'debug.sh')
with open(debug_script_path, 'w') as sfile:
sfile.write(cmd)
cmd = gen_run_cmd("python3", pipe_e, lostconfig)
start_script_path = file_man.get_instance_path(pipe_e)
start_script_path = os.path.join(start_script_path, 'start.sh')
with open(start_script_path, 'w') as sfile:
sfile.write(cmd)
p = subprocess.Popen('bash {}'.format(start_script_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
logger.info("{} ({}): Started script\n{}".format(
pipe.name, pipe.idx, cmd))
worker.add_script(pipe_e, pipe_e.script)
out, err = p.communicate()
worker.remove_script(pipe_e, pipe_e.script)
if p.returncode != 0:
raise Exception(err.decode('utf-8'))
logger.info('{} ({}): Executed script successful: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
dbm.close_session()
except:
pipe = pipe_e.pipe
logger.info('{} ({}): Exception occurred in script: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
msg = traceback.format_exc()
logger.error(msg)
script_api.report_script_err(pipe_e, pipe, dbm, msg)
dbm.close_session()
def __init__(self,
db_man,
annos,
user_id,
anno_task_id,
proposedLabel=True):
self.mia_json = dict()
self.db_man = db_man
self.annos = annos
self.user_id = user_id
self.anno_task_id = anno_task_id
self.file_man = FileMan(self.db_man.lostconfig)
self.proposedLabel = proposedLabel
def post(self):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.ANNOTATOR):
dbm.close_session()
return "You need to be {} in order to perform this request.".format(
roles.ANNOTATOR), 401
else:
data = json.loads(request.data)
img = dbm.get_image_anno(data['imageId'])
flask.current_app.logger.info('img.img_path: {}'.format(
img.img_path))
flask.current_app.logger.info('img.fs.name: {}'.format(
img.fs.name))
# fs_db = dbm.get_fs(img.fs_id)
fs = FileMan(fs_db=img.fs)
#img = PIL.Image.open('/home/lost/data/media/10_voc2012/2007_008547.jpg')
# img = PIL.Image.open(img_path)
if data['clahe']['active']:
img = fs.load_img(img.img_path, color_type='gray')
else:
img = fs.load_img(img.img_path, color_type='color')
flask.current_app.logger.info(
'Triggered filter. Received data: {}'.format(data))
# img_io = BytesIO()
# img.save(img_io, 'PNG')
# img_io.seek(0)
# return send_file(img_io, mimetype='image/png')
if data['rotate']['active']:
if data['rotate']['angle'] == 90:
img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
elif data['rotate']['angle'] == -90:
img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)
elif data['rotate']['angle'] == 180:
img = cv2.rotate(img, cv2.ROTATE_180)
if data['clahe']['active']:
clahe = cv2.createCLAHE(data['clahe']['clipLimit'])
img = clahe.apply(img)
# img = img.rotate(data['rotate']['angle'], expand=True)
# img = ImageOps.autocontrast(img)
# data = BytesIO()
# img.save(data, "PNG")
_, data = cv2.imencode('.jpg', img)
data64 = base64.b64encode(data.tobytes())
dbm.close_session()
return u'data:img/jpg;base64,' + data64.decode('utf-8')
def post(self):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.ANNOTATOR):
dbm.close_session()
return "You need to be {} in order to perform this request.".format(
roles.ANNOTATOR), 401
else:
#TODO: Check if user is permitted to load this image
data = json.loads(request.data)
#flask.current_app.logger.info('mia -> getimage. Received data: {}'.format(data))
if data['type'] == 'imageBased':
db_img = dbm.get_image_anno(data['id'])
fm = FileMan(fs_db=db_img.fs)
img = load_img(db_img, fm, user)
elif data['type'] == 'annoBased':
db_anno = dbm.get_two_d_anno(two_d_anno_id=data['id'])
db_img = dbm.get_image_anno(db_anno.img_anno_id)
fm = FileMan(fs_db=db_img.fs)
# image = fm.load_img(db_img.img_path)
image = load_img(db_img, fm, user)
# get annotation_task config
config = mia.get_config(dbm, user.idx)
draw_anno = False
context = None
try:
draw_anno = config['drawAnno']
except:
pass
try:
context = float(config['addContext'])
except:
pass
crops, _ = anno_helper.crop_boxes(
[db_anno.to_vec('anno.data')],
[db_anno.to_vec('anno.dtype')],
image,
context=context,
draw_annotations=draw_anno)
img = crops[0]
else:
raise Exception('Unknown mia image type')
_, data = cv2.imencode('.jpg', img)
data64 = base64.b64encode(data.tobytes())
dbm.close_session()
return u'data:img/jpg;base64,' + data64.decode('utf-8')
def update_version_log():
fm = FileMan(LOSTConfig())
path = fm.get_version_log_path()
if not os.path.exists(path):
print('Patchsystem: Created version log file: {}'.format(path))
versions = []
versions.append(lost.__version__)
with open(path, 'w') as json_file:
json.dump(versions, json_file)
else:
with open(path) as json_file:
versions = json.load(json_file)
if versions[-1] == lost.__version__:
print('Patchsystem: No version change!')
else:
print('Patchsystem: We maybe need to patch!')
class Datasource(Element):
def __init__(self, pe, dbm):
'''Represents a file or folder in the filesystem.
Args:
pe (object): :class:`lost.db.model.PipeElement`
dbm (object): Database Management object.
'''
super().__init__(pe, dbm)
self.file_man = FileMan(fs_db=pe.datasource.fs)
@property
def path(self):
'''str: Relative path to file or folder'''
return self.file_man.get_abs_path(self.pe.datasource.selected_path)
def get_fs(self):
'''Get filesystem for this datasource'''
# ds = self.pe.datasource
# fs_args = ast.literal_eval(ds.fs.connection)
# fs = fsspec.filesystem(ds.fs.fs_type, **fs_args)
# fs.lost_fs = ds.fs
return self.file_man.fs
def get_fm(self):
'''Get FileMan object'''
return self.file_man
def get(self, path):
print(path)
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.ANNOTATOR):
dbm.close_session()
return "You are not authorized.", 401
else:
# raise Exception('data/logs/ -> Not Implemented!')
fm = FileMan(LOST_CONFIG)
with fm.fs.open(fm.get_abs_path(path), 'rb') as f:
resp = make_response(f.read())
resp.headers[
"Content-Disposition"] = "attachment; filename=log.csv"
resp.headers["Content-Type"] = "text/csv"
return resp
def get_filesystem(self, name=None):
'''Get default lost filesystem or a specific filesystem by name.
Returns:
fsspec.spec.AbstractFileSystem: See https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem
'''
if name is None:
return self.file_man.fs
#TODO: Check if pipeline user is permitted to load fs
fs_db = self._dbm.get_fs(name=name)
fm = FileMan(fs_db=fs_db)
return fm.fs
def post(self):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.DESIGNER):
dbm.close_session()
return "You need to be {} in order to perform this request.".format(
roles.DESIGNER), 401
else:
data = json.loads(request.data)
fs_db = dbm.get_fs(name=data['fs']['name'])
fm = FileMan(fs_db=fs_db)
commonprefix = os.path.commonprefix(
[data['path'], fs_db.root_path])
if commonprefix != fs_db.root_path:
path = fs_db.root_path
else:
path = data['path']
res = fm.ls(path, detail=True)
dbm.close_session()
return chonkyfy(res, path, fm)
def get_template(db_man, template_id, user):
''' read out one template
Args:
db_man:
template_id: id of the template
Returns: JSON with all nescessary template info
'''
#TODO: implement the following access methods
template = db_man.get_pipe_template(template_id)
if template is None:
error_msg = "PipeTemplate with ID '" + str(
template_id) + "' does not exist."
try:
raise PipeTemplateNotFoundError(error_msg)
finally:
return error_msg
file_man = FileMan(db_man.lostconfig)
available_raw_files = file_man.get_media_rel_path_tree()
available_groups = db_man.get_groups()
default_group = db_man.get_group_by_name(user.user_name)
available_label_trees = db_man.get_all_label_trees(
group_id=default_group.idx, add_global=True)
available_scripts = db_man.get_all_scripts()
available_fs = list(db_man.get_public_fs())
for user_group in db_man.get_user_groups_by_user_id(user.idx):
if user_group.group.is_user_default:
group_id = user_group.group.idx
available_fs += list(db_man.get_fs(group_id=group_id))
try:
template_serialize = TemplateSerialize(db_man, template,
available_raw_files,
available_label_trees,
available_groups,
available_scripts, available_fs)
except TypeError:
return "No JSON found in PipeTemplate."
template_serialize.add_available_info()
return template_serialize.template_json
def post(self):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
# raise Exception('lostsession: {}'.format(dask_session.ds_man.session))
if not user.has_role(roles.ANNOTATOR):
dbm.close_session()
return "You need to be {} in order to perform this request.".format(
roles.ANNOTATOR), 401
else:
#TODO: Check if user is permitted to load this image
#TODO: Read img from stream -> cv2.imdecode()
data = json.loads(request.data)
flask.current_app.logger.info(
'sia -> getimage. Received data: {}'.format(data))
db_img = dbm.get_image_anno(data['imgId'])
if LOST_CONFIG.worker_management != 'dynamic':
fm = FileMan(fs_db=db_img.fs)
flask.current_app.logger.info(
'sia -> getimage. fs.name: {} fs.root_path: {}'.format(
db_img.fs.name, db_img.fs.root_path))
img = fm.load_img(db_img.img_path)
else:
img = dask_session.ds_man.read_fs_img(user, db_img.fs,
db_img.img_path)
flask.current_app.logger.info(
'dask_session read_fs_img type: {}'.format(type(img)))
flask.current_app.logger.info(
'dask_session read_fs_img shape: {}'.format(img.shape))
# img_path = fm.get_abs_path(db_img.img_path)
# #raise Exception('sia -> getimage: {}'.format(img_path))
# img = cv2.imread(img_path)
_, data = cv2.imencode('.jpg', img)
data64 = base64.b64encode(data.tobytes())
dbm.close_session()
return u'data:img/jpg;base64,' + data64.decode('utf-8')
def __init__(self, db_man, data, user_id):
"""
:type db_man: lost.db.access.DBMan
"""
self.timestamp = datetime.now()
self.db_man = db_man
self.user_id = user_id
self.at = get_sia_anno_task(db_man,
user_id) #type: lost.db.model.AnnoTask
self.sia_history_file = FileMan(
self.db_man.lostconfig).get_sia_history_path(self.at)
self.iteration = db_man.get_pipe_element(
pipe_e_id=self.at.pipe_element_id).iteration
self.image_anno = self.db_man.get_image_annotation(data['imgId'])
self.image_anno.timestamp = self.timestamp
if self.image_anno.anno_time is None:
self.image_anno.anno_time = 0.0
self.image_anno.anno_time += (
self.image_anno.timestamp -
self.image_anno.timestamp_lock).total_seconds()
self.b_boxes = list()
self.points = list()
self.lines = list()
self.polygons = list()
self.history_json = dict()
self.history_json['annotations'] = dict()
self.history_json['annotations']['new'] = list()
self.history_json['annotations']['unchanged'] = list()
self.history_json['annotations']['changed'] = list()
self.history_json['annotations']['deleted'] = list()
self._update_img_labels(data)
self.image_anno.is_junk = data['isJunk']
# store certain annotations
if 'bBoxes' in data['annotations']:
self.b_boxes = data['annotations']['bBoxes']
else:
self.b_boxes = None
if 'points' in data['annotations']:
self.points = data['annotations']['points']
else:
self.points = None
if 'lines' in data['annotations']:
self.lines = data['annotations']['lines']
else:
self.lines = None
if 'polygons' in data['annotations']:
self.polygons = data['annotations']['polygons']
else:
self.polygons = None
def get(self, deid):
dbm = access.DBMan(LOST_CONFIG)
identity = get_jwt_identity()
user = dbm.get_user_by_id(identity)
if not user.has_role(roles.DESIGNER):
dbm.close_session()
return "You are not authorized.", 401
else:
# data = json.loads(request.data)
de = dbm.get_data_export(deid)
fs_db = de.fs
fm = FileMan(fs_db=fs_db)
with fm.fs.open(de.file_path, 'rb') as f:
resp = make_response(f.read())
resp.headers[
"Content-Disposition"] = "attachment; filename=annos.parquet"
resp.headers["Content-Type"] = "blob"
return resp
def _read_fs_img(fs, img_path):
fm = FileMan(fs_db=fs)
img = fm.load_img(img_path)
return img
class Script(pe_base.Element):
'''Superclass for a user defined Script.
Custom scripts need to inherit from Script and implement the main method.
Attributes:
pe_id (int): Pipe element id. Assign the pe id of a pipline script
in order to emulate this script in a jupyter notebook for example.
'''
def __init__(self, pe_id=None):
if pe_id is None:
parser = argparse.ArgumentParser(
description='A user defined script.')
parser.add_argument('--idx',
nargs='?',
action='store',
help='Id of related pipeline element.')
args = parser.parse_args()
lostconfig = LOSTConfig()
dbm = access.DBMan(lostconfig)
db_fs = dbm.get_fs(name='lost_data')
self.file_man = FileMan(fs_db=db_fs)
self._dbm = dbm #type: lost.db.access.DBMan
if pe_id is None:
pe = dbm.get_pipe_element(int(args.idx))
else:
pe = dbm.get_pipe_element(pe_id)
super().__init__(pe, dbm)
logfile_path = self.file_man.get_pipe_log_path(self._pipe.idx)
self._log_stream = self.file_man.fs.open(logfile_path, 'a')
self._logger = log.get_stream_logger(os.path.basename(pe.script.path),
self._log_stream)
if self.pipe_info.logfile_path is None or not self.pipe_info.logfile_path:
self.pipe_info.logfile_path = self.get_rel_path(logfile_path)
self._inp = inout.Input(self)
self._outp = inout.ScriptOutput(self)
self.rejected_execution = False
# If pe_id is None we have a normal script
# If pe_id is not None a JupyterNotebook uses this script
if pe_id is None:
return self._run()
def _run(self, ret_success=False):
try:
self.main()
self.i_am_done()
success = 'PipeElementID: {}, Successfully executed script: {}'.format(
self._pipe_element.idx, self._pipe_element.script.name)
self._dbm.close_session()
if ret_success:
return success
except:
err_msg = str(datetime.datetime.now()) + '\n'
err_msg += traceback.format_exc()
self.report_err(err_msg)
self._dbm.close_session()
def __str__(self):
my_str = 'I am a Script.\nMy name is: {}\nPipeElementID: {}'.format(
self._pipe_element.script.name, self._pipe_element.idx)
return my_str
def main(self):
#raise NotImplementedError("You need to implement a main method to get your Script running.")
pass
@property
def logger(self):
''':class:`logging.Logger`: A standard python logger for this script.
It will log to the pipline log file.
'''
return self._logger
@property
def inp(self):
''':class:`lost.pyapi.inout.Input`
'''
return self._inp #type: inout.Input
@property
def outp(self):
''':class:`lost.pyapi.inout.ScriptOutput`
'''
return self._outp #type: inout.ScriptOutput
def get_rel_path(self, path):
'''Get relativ path for current project
Args:
path (str): A absolute path
Returns:
str : Relative path
'''
return self.file_man.get_rel_path(path)
def get_label_tree(self, name):
'''Get a LabelTree by name.
Args:
name (str): Name of the desired LabelTree.
Retruns:
:class:`lost.logic.label.LabelTree` or None:
If a label tree with the given name exists
it will be returned. Otherwise None
will be returned'''
group_id = self._pipe.group_id
root_list = self._dbm.get_all_label_trees(group_id, add_global=True)
root = next(filter(lambda x: x.name == name, root_list), None)
if root is None:
return None
else:
return LabelTree(self._dbm, root_leaf=root)
def create_label_tree(self, name, external_id=None):
'''Create a new LabelTree
Args:
name (str): Name of the tree / name of the root leaf.
external_id (str): An external id for the root leaf.
Returns:
:class:`lost.logic.label.LabelTree`:
The created LabelTree.
'''
tree = LabelTree(self._dbm)
tree.create_root(name, external_id=external_id)
return tree
def get_abs_path(self, path):
'''Get absolute path in current file system.
Args:
path (str): A relative path.
Returns:
str: Absolute path
'''
return self.file_man.get_abs_path(path)
def break_loop(self):
'''Break next loop in pipeline.
'''
loop_e = self._pipe_man.get_next_loop(self._pipe_element)
if loop_e is not None:
loop_e.loop.break_loop = True
self._dbm.add(loop_e)
def loop_is_broken(self):
'''Check if the current loop is broken'''
loop_e = self._pipe_man.get_next_loop(self._pipe_element)
if loop_e is not None:
return loop_e.loop.break_loop
else:
self.logger.warning(
'loop_is_broken method was used, but no loop seems to be in this pipeline!'
)
return False
def get_arg(self, arg_name):
'''Get argument value by name for this script.
Args:
arg_name (str): Name of the argument.
Returns:
Value of the given argument.
'''
if self._pipe_element.arguments:
args = json.loads(self._pipe_element.arguments)
# args = ast.literal_eval(self._pipe_element.arguments)
my_arg = args[arg_name]['value']
if my_arg in ['t', 'true', 'yes']:
return True
if my_arg in ['f', 'false', 'no']:
return False
if my_arg in ['-', '', '[]']:
return None
try:
return ast.literal_eval(my_arg)
except:
return my_arg
else:
return None
def get_filesystem(self, name=None):
'''Get default lost filesystem or a specific filesystem by name.
Returns:
fsspec.spec.AbstractFileSystem: See https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem
'''
if name is None:
return self.file_man.fs
#TODO: Check if pipeline user is permitted to load fs
fs_db = self._dbm.get_fs(name=name)
fm = FileMan(fs_db=fs_db)
return fm.fs
def get_path(self, file_name, context='instance', ptype='abs'):
'''Get path for the filename in a specific context in filesystem.
Args:
file_name (str): Name or relative path for a file.
context (str): Options: *instance*, *pipe*, *static*
ptype (str): Type of this path. Can be relative or absolute
Options: *abs*, *rel*
Returns:
str: Path to the file in the specified context.
'''
if context == 'instance':
path = os.path.join(self.instance_context, file_name)
elif context == 'pipe':
path = os.path.join(self.pipe_context, file_name)
elif context == 'static':
path = os.path.join(self.static_context, file_name)
else:
raise Exception('Unknown context: {}'.format(context))
if ptype == 'abs':
return path
elif ptype == 'rel':
return self.get_rel_path(path)
else:
raise Exception('Unknown argument ptype: {}'.format(ptype))
@property
def iteration(self):
'''int: Get the current iteration.
Number of times this script has been executed.
'''
return self._pipe_element.iteration
@property
def instance_context(self):
'''str: Get the path to store files that are only valid for this instance.
'''
abs_path = self.file_man.create_instance_path(self._pipe_element)
rel_path = self.file_man.make_path_relative(abs_path)
self._pipe_element.instance_context = rel_path
self._dbm.add(self._pipe_element)
return abs_path
@property
def pipe_context(self):
'''str: Root path to store files that should be visible for all elements
in the pipeline.
'''
return self.file_man.get_pipe_context_path(self._pipe_element)
@property
def static_context(self):
'''str: Get the static path.
Files that are stored at this path can be accessed by all instances of a
script.
'''
#TODO: Check how to handle different filesystem!
return os.path.join(self._lostconfig.app_path,
os.path.split(self._pipe_element.script.path)[0])
@property
def progress(self):
'''float: Get current progress that is displayed in the progress bar of this script.
Current progress in percent 0...100
'''
return self._pipe_element.progress
def update_progress(self, value):
'''Update the progress for this script.
Args:
value (float): Progress in percent 0...100
'''
self._pipe_element.progress = value
self._dbm.commit()
def reject_execution(self):
'''Reject execution of this script and set it to PENDING again.
Note:
This method is useful if you want to execute this script only
when some condition based on previous pipeline elements is
meet.
'''
self.rejected_execution = True
def get_alien_element(self, pe_id):
'''Get an pipeline element by id from somewhere in the LOST system.
It is an alien element since it is most likely not part of the
pipeline instance this script belongs to.
Args:
pe_id (int): PipeElementID of the alien element.
Returns:
* :class:`lost.pyapi.script.Script`
* :class:`lost.pyapi.pipe_elements.AnnoTask`
* :class:`lost.pyapi.pipe_elements.Datasource`
* :class:`lost.pyapi.pipe_elements.VisualOutput`
* :class:`lost.pyapi.pipe_elements.DataExport`
* :class:`lost.pyapi.pipe_elements.Loop`
'''
pe = self._dbm.get_pipe_element(pe_id)
if pe.dtype == dtype.PipeElement.SCRIPT:
return Script(pe_id=pe_id)
elif pe.dtype == dtype.PipeElement.ANNO_TASK:
return pipe_elements.AnnoTask(pe, self._dbm)
elif pe.dtype == dtype.PipeElement.DATASOURCE:
return pipe_elements.Datasource(pe, self._dbm)
elif pe.dtype == dtype.PipeElement.VISUALIZATION:
return pipe_elements.VisualOutput(pe, self._dbm)
elif pe.dtype == dtype.PipeElement.DATA_EXPORT:
return pipe_elements.DataExport(pe, self._dbm)
elif pe.dtype == dtype.PipeElement.LOOP:
return pipe_elements.Loop(pe, self._dbm)
else:
raise Exception('Unknown pipe element type!')
def i_am_done(self):
if self.rejected_execution:
self._pipe_element.state = state.PipeElement.PENDING
self._dbm.add(self._pipe)
self._dbm.add(self._pipe_element)
self._dbm.commit()
return
#Save all changes to database
if self._pipe_element.is_debug_mode == False:
self._pipe_element.state = state.PipeElement.FINISHED
self._pipe_element.progress = 100.0
self._pipe.state = state.Pipe.IN_PROGRESS
self._dbm.add(self._pipe)
self._dbm.add(self._pipe_element)
self._dbm.commit()
else:
answer = input("Have you finished debugging? [y/n]: ")
if answer[0].lower() == 'y':
self._pipe_element.state = state.PipeElement.FINISHED
self._pipe_element.progress = 100.0
self._pipe.state = state.Pipe.IN_PROGRESS
self._dbm.add(self._pipe)
self._dbm.add(self._pipe_element)
else:
self.outp.clean_up()
self._pipe_man.pipe.state = state.Pipe.IN_PROGRESS
self._dbm.commit()
self._log_stream.close()
def report_err(self, msg):
'''Report an error for this user script to portal
Args:
msg: The error message that should be reported.
Note:
You can call this method multiple times if you like. All messages
will be concatenated an sent to the portal.
'''
self.logger.error(msg)
report_script_err(self._pipe_element, self._pipe, self._dbm, msg)
def __init__(self, dbm, pipe):
self.pipe = pipe #type: lost.db.model.Pipe
self.dbm = dbm #type: lost.db.access.DBMan
self.fm = FileMan(self.dbm.lostconfig)
#!/usr/bin/env python3
import argparse
import lostconfig as config
from lost.logic.file_man import FileMan
import logging
logging.basicConfig(level=logging.INFO, format='(%(levelname)s): %(message)s')
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Copy example data')
parser.add_argument('src',
nargs='?',
action='store',
help='Source dir with example data to copy')
args = parser.parse_args()
lostconfig = config.LOSTConfig()
fm = FileMan(lostconfig)
fm.fs.put(args.src, fm.media_root_path, recursive=True)
logging.info('Copyed images from {} to {}'.format(args.src,
fm.media_root_path))
class PipeImporter(object):
def __init__(self, pipe_template_dir, dbm, forTest=False):
'''Load json file.
Args:
pipe_template_dir: Path to pipeline directory
'''
self.forTest = forTest
self.dbm = dbm
self.file_man = FileMan(self.dbm.lostconfig)
if pipe_template_dir.endswith('/'):
pipe_template_dir = pipe_template_dir[:-1]
self.src_pipe_template_path = pipe_template_dir
self.dst_pipe_template_path = os.path.join(
self.file_man.pipe_path,
os.path.basename(self.src_pipe_template_path))
self.json_files = glob(os.path.join(pipe_template_dir, '*.json'))
self.pipes = []
self.namespace = os.path.basename(
self.src_pipe_template_path).strip('/')
for json_path in self.json_files:
with open(json_path) as jfile:
pipe = json.load(jfile)
pipe['namespace'] = self.namespace
pipe['name'] = self._namespaced_name(
os.path.splitext(os.path.basename(json_path))[0])
self.pipes.append(pipe)
# Set name to name of the script file
for pe in pipe['elements']:
if 'script' in pe:
pe['script']['name'] = self._namespaced_name(
pe['script']['path'])
self.checker = PipeDefChecker(logging)
def _namespaced_name(self, name):
return '{}.{}'.format(self.namespace, name)
def update_pipe_project(self):
if os.path.exists(self.dst_pipe_template_path):
logging.info('\n\n++++++++++++++++++++++\n\n')
for pipe in self.pipes:
if not self.checker.check(pipe):
logging.error('Pipeline was not updated!')
return False
for pipe in self.pipes:
self.update_pipe(pipe)
dir_util.copy_tree(self.src_pipe_template_path,
self.dst_pipe_template_path)
logging.info(
"Copyed pipeline template dir from %s to %s" %
(self.src_pipe_template_path, self.dst_pipe_template_path))
else:
logging.warning(('Cannot update. No such pipe project: *{}*. '
'Maybe you want to import a pipeline instead '
'of updating it.').format(self.namespace))
def update_pipe(self, pipe):
for db_pipe in self.dbm.get_all_pipeline_templates():
db_json = json.loads(db_pipe.json_template)
# update pipeline if already present in db
if db_json['name'].lower() == pipe['name'].lower():
# Do everything relative from pipeline definition file path.
oldwd = os.getcwd()
os.chdir(self.src_pipe_template_path)
logging.info('Updated pipeline: {}'.format(db_json['name']))
for pe_j in pipe['elements']:
if 'script' in pe_j:
element_j = pe_j['script']
script = parse_script(element_j)
db_script = self.dbm.get_script(
name=self._get_script_name(script))
script_arguments = get_default_script_arguments(
script.path)
script_envs = get_default_script_envs(script.path)
script_resources = get_default_script_resources(
script.path)
if 'arguments' in element_j:
for arg in element_j['arguments']:
if arg not in script_arguments:
logging.error(
"Invalid argument >> {} << in pipeline definition json"
.format(arg))
valid_args = ""
for v_arg in script_arguments:
valid_args += ">> {} <<\n".format(
v_arg)
logging.error(
"Valid arguments are: \n{}".format(
valid_args[:-1]))
raise Exception(
'Invalid arguments. Start Cleanup')
if db_script is None:
self.dbm.add(script)
self.dbm.commit()
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
script.path = self.file_man.make_path_relative(
script_out_path)
script.arguments = json.dumps(script_arguments)
script.envs = json.dumps(script_envs)
script.resources = json.dumps(script_resources)
self.dbm.save_obj(script)
logging.info("Added script to database")
else:
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
db_script.path = self.file_man.make_path_relative(
script_out_path)
db_script.arguments = json.dumps(script_arguments)
db_script.envs = json.dumps(script_envs)
db_script.description = script.description
db_script.resources = json.dumps(script_resources)
self.dbm.save_obj(db_script)
logging.info('Updated script: {}'.format(
db_script.name))
# self._fix_sia_config(pe_j)
db_pipe.json_template = json.dumps(pipe)
self.dbm.save_obj(db_pipe)
os.chdir(oldwd) # Change dir back to old working directory.
return True
# import pipe if not already present in database
self.import_pipe(pipe)
def _get_script_name(self, script):
return self._namespaced_name(os.path.basename(script.path))
def start_import(self):
logging.info('\n\n++++++++++++++++++++++ \n\n')
logging.info('Start pipe project import for: {}'.format(
self.src_pipe_template_path))
for pipe in self.pipes:
if not self.checker.check(pipe):
logging.error(
'Wrong pipeline definition! Did not import pipe project!')
return False
if os.path.exists(self.dst_pipe_template_path):
logging.warning('Cannot import pipeline!')
logging.warning('Pipe Template Dir already exist: {}'.format(
self.dst_pipe_template_path))
return
dir_util.copy_tree(self.src_pipe_template_path,
self.dst_pipe_template_path)
logging.info(
"Copyed pipeline template dir from %s to %s" %
(self.src_pipe_template_path, self.dst_pipe_template_path))
for pipe in self.pipes:
self.import_pipe(pipe)
def import_pipe(self, pipe):
try:
logging.info('\n---\n')
# Do everything relative from pipeline definition file path.
oldwd = os.getcwd()
os.chdir(self.src_pipe_template_path)
for db_pipe in self.dbm.get_all_pipeline_templates():
db_json = json.loads(db_pipe.json_template)
if db_json['name'].lower() == pipe['name'].lower():
logging.warning("PipeTemplate in database.")
logging.warning("Name of this template is: %s" %
(pipe['name'], ))
logging.warning("Will not import PipeTemplate.")
return db_pipe.idx
for pe_j in pipe['elements']:
if 'script' in pe_j:
element_j = pe_j['script']
script = parse_script(element_j)
db_script = self.dbm.get_script(
name=self._get_script_name(script))
script_arguments = get_default_script_arguments(
script.path)
script_envs = get_default_script_envs(script.path)
script_resources = get_default_script_resources(
script.path)
if 'arguments' in element_j:
for arg in element_j['arguments']:
if arg not in script_arguments:
logging.error(
"Invalid argument >> {} << in pipeline definition json"
.format(arg))
valid_args = ""
for v_arg in script_arguments:
valid_args += ">> {} <<\n".format(v_arg)
logging.error(
"Valid arguments are: \n{}".format(
valid_args[:-1]))
raise Exception(
'Invalid arguments. Start Cleanup')
if db_script is None:
self.dbm.add(script)
self.dbm.commit()
script_out_path = os.path.join(
self.dst_pipe_template_path, script.path)
script.path = self.file_man.make_path_relative(
script_out_path)
script.arguments = json.dumps(script_arguments)
script.envs = json.dumps(script_envs)
script.resources = json.dumps(script_resources)
self.dbm.save_obj(script)
logging.info("Added script to database\n")
else:
logging.warning(
"Script is already present in database.\n")
logging.warning((str(db_script.idx), db_script.name,
db_script.path))
# self._fix_sia_config(pe_j)
pipe_temp = model.PipeTemplate(json_template=json.dumps(pipe),
timestamp=datetime.now())
self.dbm.save_obj(pipe_temp)
logging.info("Added Pipeline: *** %s ***" % (pipe['name'], ))
os.chdir(oldwd) # Change dir back to old working directory.
return pipe_temp.idx
except Exception as e:
logging.error(e, exc_info=True)
if not self.forTest:
self.remove_pipe_project()
logging.error('Cleanup successful. Removed buggy pipeline.')
def remove_pipe_project(self):
'''Remove an imported pipeline project from lost system.
Note:
Pipeline folder in LOST filesystem and all related db
entrys will be deleted.
'''
clean_filesystem = True
for pipe in self.pipes:
if not self.remove_pipeline(pipe):
clean_filesystem = False
if clean_filesystem:
shutil.rmtree(self.dst_pipe_template_path)
logging.info(
'Removed pipeline project from lost filesystem {}'.format(
self.dst_pipe_template_path))
logging.info(
'Whole pipeline project {} was successfull removed'.format(
self.namespace))
else:
logging.info('''Pipeline project {} was not completely removed
since some pipes are still in use'''.format(self.namespace))
def remove_pipeline(self, pipe):
'''Remove all related db entrys of a pipeline from lost database.
'''
#TODO: Remove script
for db_pipe in self.dbm.get_all_pipeline_templates():
db_json = json.loads(db_pipe.json_template)
if db_json['name'].lower() == pipe['name'].lower():
t = self.dbm.get_pipe(pipe_template_id=db_pipe.idx)
if t is None:
for pe_j in db_json['elements']:
if 'script' in pe_j:
script_man.remove_script(
self.dbm,
os.path.join(
self.namespace,
os.path.basename(pe_j['script']['path'])))
self.dbm.delete(db_pipe)
else:
logging.warning(
"Cannot remove pipeline. It is already in use by task with ID: %s"
% (t.idx, ))
return False
self.dbm.commit()
logging.info("Removed pipeline successfull: {}".format(
pipe['name']))
return True
return True
class PipeInstance(object):
'''Model a Pipeline instance within LOST.
'''
def __init__(self, dbm, pipe):
self.pipe = pipe #type: lost.db.model.Pipe
self.dbm = dbm #type: lost.db.access.DBMan
self.fm = FileMan(self.dbm.lostconfig)
def _delete_result_links(self, pe):
for result_link in self.dbm.get_resultlinks_pe_n(pe_n_id=pe.idx):
self.dbm.delete(result_link)
self.dbm.commit()
def _delete_results(self, pe):
'''Delete results of pe and all related data.
Args:
pe (:class:`lost.db.model.PipeElement`): PipeElement where
results should be deleted for.
Note:
Only scripts contain their own Results
for all other elements results are just
just looped through from Scripts.
'''
pe = pe #type: lost.db.model.PipeElement
#Only scripts contain their own Results
#all other elements just loop through results from Scripts.
if pe.dtype == dtype.PipeElement.SCRIPT:
for res in pe.result_out:
#Delete annotations
for img_anno in res.img_annos:
for twod_anno in img_anno.twod_annos:
self.dbm.delete(twod_anno.label)
self.dbm.delete(twod_anno)
self.dbm.delete(img_anno)
#Delete visual outputs
for vis_out in res.visual_outputs:
self.dbm.delete(vis_out)
#Delete data exports
for de in res.data_exports:
self.dbm.delete(de)
self.dbm.delete(res)
self.dbm.commit()
def delete_pipeline(self):
'''Delete this pipeline instance from LOST'''
for pe in self.pipe.pe_list:
self._delete_result_links(pe)
for pe in self.pipe.pe_list:
self._delete_results(pe)
for pe in self.pipe.pe_list:
self._delete_pe(pe)
self.fm.rm_pipe_log_path(self.pipe)
self.fm.rm_pipe_context_path(self.pipe)
self.dbm.delete(self.pipe)
self.dbm.commit()
def _delete_pe(self, pe):
'''Delete a PipeElement of this pipeline
Args:
pe (:class:`lost.db.model.PipeElement`): The pipeline element
to delete.
'''
pe = pe #type: lost.db.model.PipeElement
if pe.dtype == dtype.PipeElement.SCRIPT:
self.dbm.delete(pe.script)
self.fm.rm_instance_path(pe)
elif pe.dtype == dtype.PipeElement.ANNO_TASK:
for req_leaf in pe.anno_task.req_label_leaves:
self.dbm.delete(req_leaf)
self.dbm.delete(pe.anno_task)
self.dbm.commit()
elif pe.dtype == dtype.PipeElement.DATASOURCE:
self.dbm.delete(pe.datasource)
elif pe.dtype == dtype.PipeElement.LOOP:
self.dbm.delete(pe.loop)
elif pe.dtype == dtype.PipeElement.DATA_EXPORT:
pass
elif pe.dtype == dtype.PipeElement.VISUALIZATION:
pass
else:
raise Exception('Unknown dtype for PipeElement!')
self.dbm.delete(pe)
self.dbm.commit()
class TwoDSerialize(object):
def __init__(self,
db_man,
annos,
user_id,
anno_task_id,
proposedLabel=True):
self.mia_json = dict()
self.db_man = db_man
self.annos = annos
self.user_id = user_id
self.anno_task_id = anno_task_id
self.file_man = FileMan(self.db_man.lostconfig)
self.proposedLabel = proposedLabel
def serialize(self):
directory = self.file_man.get_mia_crop_path(self.anno_task_id)
self.mia_json['images'] = list()
self.mia_json['proposedLabel'] = None
if self.proposedLabel:
self.mia_json['proposedLabel'] = get_proposed_label(
self.db_man, self.annos[0], self.user_id)
for anno in self.annos:
image_json = dict()
image_json['id'] = anno.idx
# get image_anno of two_d anno
image_anno = self.db_man.get_image_annotation(
img_anno_id=anno.img_anno_id)
cropped_image_path = os.path.join(directory, str(
anno.idx)) + '.png'
relative_cropped_image_path = self.file_man.mia_crop_rel_path + \
str(self.anno_task_id) + "/" + str(anno.idx) + ".png"
if os.path.exists(cropped_image_path):
image_json['path'] = relative_cropped_image_path
self.mia_json['images'].append(image_json)
continue
else:
# crop two_d_anno out of image_anno
config = get_config(self.db_man, self.user_id)
draw_anno = False
context = None
try:
draw_anno = config['drawAnno']
except:
pass
try:
context = float(config['addContext'])
except:
pass
self._crop_twod_anno(image_anno, anno, draw_anno, context,
cropped_image_path)
image_json['path'] = relative_cropped_image_path
self.mia_json['images'].append(image_json)
def _crop_twod_anno(self, img_anno, twod_anno, draw_anno, context,
out_path):
'''Helper method to crop a bounding box for a TwoDAnnotation and store it on disc
Args:
img_anno (:class:`model.ImageAnno`): The ImageAnno where the twod_anno belongs to.
twod_anno (:class:`model.TwoDAnno`): The 2D-anno to crop.
draw_anno (bool): Indicates wether the annotation should be painted inside the crop.
context (float): Value that indicates how much context should
be cropped around the 2D annotation.
out_path (str): Path to store the cropped image.
'''
image = skimage.io.imread(self.file_man.get_abs_path(
img_anno.img_path))
crops, _ = anno_helper.crop_boxes([twod_anno.to_vec('anno.data')],
[twod_anno.to_vec('anno.dtype')],
image,
context=context,
draw_annotations=draw_anno)
cropped_image = crops[0]
skimage.io.imsave(out_path, cropped_image)
class PipeEngine(pipe_model.PipeEngine):
def __init__(self, dbm, pipe, lostconfig):
'''
:type dbm: lost.db.access.DBMan
:type pipe: lost.db.model.Pipe
'''
super().__init__(dbm=dbm, pipe=pipe)
self.lostconfig = lostconfig #type: lost.logic.config.LOSTConfig
self.file_man = FileMan(self.lostconfig)
# self.logger = lost.logic.log.get_file_logger(
# 'Executor: {}'.format(self.lostconfig.env),
# self.file_man.app_log_path)
self.logger = get_task_logger(__name__)
def process_annotask(self, pipe_e):
anno_task = self.dbm.get_anno_task(pipe_element_id=pipe_e.idx)
if anno_task.state == state.AnnoTask.IN_PROGRESS or \
anno_task.state == state.AnnoTask.PAUSED:
if not at_man.has_annotation(self.dbm, anno_task.idx):
at_man.set_finished(self.dbm, anno_task.idx)
self.logger.warning('No Annotations have been requested for AnnoTask {}'\
.format(anno_task.idx))
self.logger.warning("%d: AnnoTask has been finished (ID: %d, Name: %s)"\
%(self.pipe.idx, anno_task.idx, anno_task.name))
# if pipe_e.anno_task.dtype == dtype.AnnoTask.MIA:
# if anno_task.progress is None:
# anno_task.progress = 0.0
# if anno_task.progress >= 100.0:
# anno_task.state = state.AnnoTask.FINISHED
# self.dbm.add(anno_task)
# pipe_e.state = state.PipeElement.FINISHED
# self.dbm.add(pipe_e)
# self.dbm.commit()
# print("%d: AnnoTask has been finished (ID: %d, Name: %s)"\
# %(self.pipe.idx, anno_task.idx, anno_task.name))
# else:
# return
# state = finished will be set in annotation tool
if anno_task.state == state.AnnoTask.PENDING:
anno_task.state = state.AnnoTask.IN_PROGRESS
self.dbm.save_obj(anno_task)
self.logger.info("%d: AnnoTask IN_PROGRESS (ID: %d, Name: %s)"\
%(self.pipe.idx, anno_task.idx, anno_task.name))
def __gen_run_cmd(self, program, pipe_e):
# script = self.dbm.get_script(pipe_e.script_id)
script_path = os.path.join(self.lostconfig.project_path,
pipe_e.script.path)
cmd = self.lostconfig.py3_init + " && "
cmd += program + " " + script_path + " --idx " + str(pipe_e.idx)
return cmd
def make_debug_session(self, pipe_e):
debug_path = self.file_man.create_debug_path(pipe_element=pipe_e)
debug_file_path = os.path.join(debug_path, 'debug.sh')
# init = self.lostconfig.py3_init + '\n'
cmd = self.__gen_run_cmd('pudb3', pipe_e)
# script_content = init + cmd
script_content = cmd
with open(debug_file_path, 'w') as dfile:
dfile.write(script_content)
script_path = os.path.join(self.lostconfig.project_path,
pipe_e.script.path)
dsession_str = "For DEBUG start: bash " + debug_file_path
dsession_str += "<br>If you want to EDIT go to: " + script_path
pipe_e.debug_session = dsession_str
self.dbm.save_obj(pipe_e)
self.logger.info('Created debug script: {}'.format(debug_file_path))
self.logger.info(pipe_e.debug_session)
def __release_loop_iteration(self, pipe_e):
pipe_e.loop.iteration += 1
self.logger.info('{}: Run loop with id {} in iteration {}'.format(
self.pipe.idx, pipe_e.loop.idx, pipe_e.loop.iteration))
loop_pes = self.get_loop_pes(pipe_e.loop.pe_jump, pipe_e)
for pe in loop_pes:
pe.iteration += 1
pe.state = state.PipeElement.PENDING
if pe.dtype == dtype.PipeElement.ANNO_TASK:
pe.anno_task.state = state.AnnoTask.PENDING
elif pe.dtype == dtype.PipeElement.SCRIPT:
pe.progress = 0.0
elif pe.dtype == dtype.PipeElement.LOOP:
# Check for loop in loop case; Set iteration of all inner loops
# to zero.
if pe is not pipe_e:
pe.loop.iteration = 0
self.set_to_visit(pe)
self.dbm.add(pe)
def process_loop(self, pipe_e):
if pipe_e.loop.break_loop:
pipe_e.state = state.PipeElement.FINISHED
self.dbm.add(pipe_e)
self.logger.info('{}: Break loop with id {}'.format(
self.pipe.idx, pipe_e.loop.idx))
return
if pipe_e.loop.max_iteration is not None:
if pipe_e.loop.iteration is None:
pipe_e.loop.iteration = 0
if pipe_e.loop.iteration < pipe_e.loop.max_iteration - 1:
self.__release_loop_iteration(pipe_e)
else:
pipe_e.state = state.PipeElement.FINISHED
self.logger.info('{}: Loop ({}) terminated. Max iterations = {}'\
.format(self.pipe.idx, pipe_e.loop.idx,
pipe_e.loop.max_iteration))
else:
self.__release_loop_iteration(pipe_e)
self.dbm.add(pipe_e)
def select_env_for_script(self, pipe_e):
'''Select an environment where the script should be executed'''
w_man = WorkerMan(self.dbm, self.lostconfig)
if pipe_e.script.envs is not None:
script_envs = json.loads(pipe_e.script.envs)
if len(script_envs) == 0:
return 'celery'
else:
script_envs = list()
return 'celery' # Return default queue
worker_envs = w_man.get_worker_envs()
for script_env in script_envs:
if script_env in worker_envs:
return script_env
self.logger.warning('No suitable env to execute script: {}'.format(
pipe_e.script.path))
return None
def process_pipe_element(self):
pipe_e = self.get_next_element()
while (pipe_e is not None):
# if pipe_e is None:
# return
if pipe_e.dtype == dtype.PipeElement.SCRIPT:
if pipe_e.state != state.PipeElement.SCRIPT_ERROR:
# if pipe_e.is_debug_mode:
# pipe_e.state = state.PipeElement.IN_PROGRESS
# self.dbm.save_obj(pipe_e)
# self.make_debug_session(pipe_e)
# else:
if pipe_e.state == state.PipeElement.PENDING:
env = self.select_env_for_script(pipe_e)
if env is None:
return
celery_exec_script.apply_async(args=[pipe_e.idx],
queue=env)
elif pipe_e.dtype == dtype.PipeElement.ANNO_TASK:
if pipe_e.state == state.PipeElement.PENDING:
update_anno_task(self.dbm, pipe_e.anno_task.idx)
try:
email.send_annotask_available(self.dbm,
pipe_e.anno_task)
except:
msg = "Could not send Email. \n"
msg += traceback.format_exc()
self.logger.error(msg)
pipe_e.state = state.PipeElement.IN_PROGRESS
self.dbm.save_obj(pipe_e)
self.process_annotask(pipe_e)
elif pipe_e.dtype == dtype.PipeElement.DATASOURCE:
pipe_e.state = state.PipeElement.FINISHED
self.dbm.save_obj(pipe_e)
elif pipe_e.dtype == dtype.PipeElement.VISUALIZATION:
pipe_e.state = state.PipeElement.FINISHED
self.dbm.save_obj(pipe_e)
elif pipe_e.dtype == dtype.PipeElement.DATA_EXPORT:
pipe_e.state = state.PipeElement.FINISHED
self.dbm.save_obj(pipe_e)
elif pipe_e.dtype == dtype.PipeElement.LOOP:
self.process_loop(pipe_e)
self.dbm.commit()
pipe_e = self.get_next_element()
def process_pipeline(self):
try:
p = self.pipe
# print('Process pipe: {}'.format(self.pipe.name))
if p.is_locked is None:
p.is_locked = False
if not p.is_locked:
p.is_locked = True
self.dbm.save_obj(p)
else:
return
if p.state == state.Pipe.PENDING:
p.state = state.Pipe.IN_PROGRESS
self.dbm.save_obj(p)
self.process_pipe_element()
elif p.state == state.Pipe.IN_PROGRESS:
self.process_pipe_element()
elif p.state == state.Pipe.FINISHED:
return
elif p.state == state.Pipe.ERROR:
self.__report_error(p)
else:
raise Exception("Unknown PipeState!")
p.is_locked = False
self.dbm.save_obj(p)
except:
p.is_locked = False
self.dbm.save_obj(p)
raise
def get_next_element(self):
pe_wait = None
for candidate in self.get_to_visit():
if candidate is None:
if self.pipe_finished():
self.pipe.state = state.Pipe.FINISHED
self.pipe.timestamp_finished = datetime.now()
self.dbm.save_obj(self.pipe)
self.logger.info("%d: Task is finished (Name: %s)" %
(self.pipe.idx, self.pipe.name))
try:
email.send_pipeline_finished(self.pipe)
except:
msg = "Could not send Email. \n"
msg += traceback.format_exc()
self.logger.error(msg)
return None
else:
continue
else:
pe = self.check_candiate(candidate)
if pe is None:
continue
#If there is a loop under candidates, it should be executed as
#last possible element. Since a loop will set all other elements
#within the loop to pending when processed. So if the last element
#before the loop has subsequent elements. These elements would never
#be executed since the loop would set the last element in the loop
#to pending.
elif pe.dtype == dtype.PipeElement.LOOP:
pe_wait = pe
continue
else:
self.set_visited(pe)
return pe
return pe_wait
def pipe_finished(self):
for pe in self.get_final_pes():
if pe.state != state.PipeElement.FINISHED:
return False
return True
def check_candiate(self, candidate):
# If all prev elements are finished return candidate
for pe_prev in self.get_prev_pes(candidate):
if pe_prev is not None:
if pe_prev.state != state.PipeElement.FINISHED:
return None
# if pe_prev.state == state.PipeElement.FINISHED:
# if candidate.state == state.PipeElement.PENDING:
# return candidate
# elif candidate.dtype == dtype.PipeElement.ANNOTATION_TASK and\
# candidate.state == state.PipeElement.IN_PROGRESS:
# return candidate
else:
# if pe_prev is None and candidate.state == PENDING
if candidate.state == state.PipeElement.PENDING:
return candidate
return candidate
def __report_error(self, pipe):
for pipe_element in self.dbm.get_script_errors(pipe.idx):
# Send mail to inform user about script error.
try:
email.send_script_error(pipe, pipe_element)
pipe_element.error_reported = True
self.dbm.add(pipe_element)
self.dbm.commit()
except:
pipe_element.error_reported = True
pipe_element.error_msg += traceback.format_exc()
self.dbm.add(pipe_element)
self.dbm.commit()