def delete_pipe(pipe_id):
# logger = get_task_logger(__name__)
# logger.info("DELETED BY CELERY {}".format(pipe_id))
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipeline.delete(dbm, pipe_id)
dbm.close_session()
def __init__(self, pe_id=None):
if pe_id is None:
parser = argparse.ArgumentParser(
description='A user defined script.')
parser.add_argument('--idx',
nargs='?',
action='store',
help='Id of related pipeline element.')
args = parser.parse_args()
lostconfig = LOSTConfig()
dbm = access.DBMan(lostconfig)
db_fs = dbm.get_fs(name='lost_data')
self.file_man = FileMan(fs_db=db_fs)
self._dbm = dbm #type: lost.db.access.DBMan
if pe_id is None:
pe = dbm.get_pipe_element(int(args.idx))
else:
pe = dbm.get_pipe_element(pe_id)
super().__init__(pe, dbm)
logfile_path = self.file_man.get_pipe_log_path(self._pipe.idx)
self._log_stream = self.file_man.fs.open(logfile_path, 'a')
self._logger = log.get_stream_logger(os.path.basename(pe.script.path),
self._log_stream)
if self.pipe_info.logfile_path is None or not self.pipe_info.logfile_path:
self.pipe_info.logfile_path = self.get_rel_path(logfile_path)
self._inp = inout.Input(self)
self._outp = inout.ScriptOutput(self)
self.rejected_execution = False
# If pe_id is None we have a normal script
# If pe_id is not None a JupyterNotebook uses this script
if pe_id is None:
return self._run()
def __patch_user_api_token(self):
lost_config = LOSTConfig()
db = MySQLdb.connect(host=lost_config.lost_db_ip,
port=int(lost_config.lost_db_port),
user=lost_config.lost_db_user,
passwd=lost_config.lost_db_pwd,
db=lost_config.lost_db_name)
cur = db.cursor()
cur.execute("ALTER TABLE user ADD COLUMN api_token varchar(4096)")
db.close()
def send_life_sign():
# logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
# logger.info('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
dbm.add(worker)
dbm.commit()
#logger.info('Sent lifesign: {}'.format(worker.worker_name))
dbm.close_session()
def exec_script_in_subprocess(pipe_element_id):
try:
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipe_e = dbm.get_pipe_element(pipe_e_id=pipe_element_id)
logger = logging
if lostconfig.worker_management == 'static':
worker = CurrentWorker(dbm, lostconfig)
if not worker.enough_resources(pipe_e.script):
# logger.warning('Not enough resources! Rejected {} (PipeElement ID {})'.format(pipe_e.script.path, pipe_e.idx))
raise Exception('Not enough resources')
pipe_e.state = state.PipeElement.IN_PROGRESS
dbm.save_obj(pipe_e)
file_man = AppFileMan(lostconfig)
pipe = pipe_e.pipe
cmd = gen_run_cmd("pudb3", pipe_e, lostconfig)
debug_script_path = file_man.get_debug_path(pipe_e)
debug_script_path = os.path.join(debug_script_path, 'debug.sh')
with open(debug_script_path, 'w') as sfile:
sfile.write(cmd)
cmd = gen_run_cmd("python3", pipe_e, lostconfig)
# file_man.create_debug_path(pipe_e)
start_script_path = file_man.get_debug_path(pipe_e)
start_script_path = os.path.join(start_script_path, 'start.sh')
with open(start_script_path, 'w') as sfile:
sfile.write(cmd)
p = subprocess.Popen('bash {}'.format(start_script_path), stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
logger.info("{} ({}): Started script\n{}".format(pipe.name, pipe.idx, cmd))
if lostconfig.worker_management == 'static':
worker.add_script(pipe_e, pipe_e.script)
out, err = p.communicate()
if lostconfig.worker_management == 'static':
worker.remove_script(pipe_e, pipe_e.script)
if p.returncode != 0:
raise Exception(err.decode('utf-8'))
logger.info('{} ({}): Executed script successful: {}'.format(pipe.name,
pipe.idx, pipe_e.script.path))
dbm.close_session()
except:
pipe = pipe_e.pipe
logger.info('{} ({}): Exception occurred in script: {}'.format(pipe.name,
pipe.idx, pipe_e.script.path))
msg = traceback.format_exc()
logger.error(msg)
script_api.report_script_err(pipe_e, pipe, dbm, msg)
dbm.close_session()
def init_worker_on_startup():
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
print('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
worker.resources = '[]'
worker.in_progress = '{}'
dbm.add(worker)
dbm.commit()
print('Reset worker on startup: {}'.format(worker.worker_name))
dbm.close_session()
def update_version_log():
fm = AppFileMan(LOSTConfig())
path = fm.get_version_log_path()
if not os.path.exists(path):
print('Patchsystem: Created version log file: {}'.format(path))
versions = []
versions.append(lost.__version__)
with open(path, 'w') as json_file:
json.dump(versions, json_file)
else:
with open(path) as json_file:
versions = json.load(json_file)
print("Versions: ", versions)
if versions[-1] == lost.__version__:
print('Patchsystem: No version change!')
else:
print('Patchsystem: We maybe need to patch!')
dbp = DBPatcher()
dbp.patch()
versions.append(lost.__version__)
with open(path, 'w') as json_file:
json.dump(versions, json_file)
from lostconfig import LOSTConfig FLASK_THREADED = True RESTPLUS_SWAGGER_EXPANSIONS = 'list' RESTPLUS_VAL = True RESTPLUS_MASK_SWAGGER = False LOST_CONFIG = LOSTConfig() FLASK_DEBUG = LOST_CONFIG.debug # Flask settings SECRET_KEY = 'Test' # Flask-Mail SMTP server settings MAIL_SERVER = LOST_CONFIG.mail_server #'smtp.gmail.com' MAIL_PORT = LOST_CONFIG.mail_port #465 MAIL_USE_SSL = LOST_CONFIG.mail_use_ssl #True MAIL_USE_TLS = LOST_CONFIG.mail_use_tls #False MAIL_USERNAME = LOST_CONFIG.mail_username #'*****@*****.**' MAIL_PASSWORD = LOST_CONFIG.mail_password #'password' MAIL_DEFAULT_SENDER = LOST_CONFIG.mail_default_sender #'"MyApp" <*****@*****.**>' # Flask-User settings USER_APP_NAME = "LOST" # Shown in and email templates and page footers USER_ENABLE_EMAIL = True # Enable email authentication USER_ENABLE_USERNAME = False # Disable username authentication USER_EMAIL_SENDER_NAME = USER_APP_NAME USER_EMAIL_SENDER_EMAIL = "*****@*****.**"
def __init__(self):
self.chipper = Fernet(LOSTConfig().secret_key.encode())