def delete_pipe(pipe_id):
logger = get_task_logger(__name__)
logger.info("DELETED BY CELERY {}".format(pipe_id))
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipeline.delete(dbm, pipe_id)
dbm.close_session()
def __init__(self, pe_id=None):
if pe_id is None:
parser = argparse.ArgumentParser(description='A user defined script.')
parser.add_argument('--idx', nargs='?', action='store',
help='Id of related pipeline element.')
args = parser.parse_args()
lostconfig = LOSTConfig()
self.file_man = FileMan(lostconfig)
dbm = access.DBMan(lostconfig)
self._dbm = dbm #type: lost.db.access.DBMan
if pe_id is None:
pe = dbm.get_pipe_element(int(args.idx))
else:
pe = dbm.get_pipe_element(pe_id)
super().__init__(pe, dbm)
logfile_path = self.file_man.get_pipe_log_path(self._pipe.idx)
self._logger = log.get_file_logger(os.path.basename(pe.script.path),
logfile_path)
if self.pipe_info.logfile_path is None or not self.pipe_info.logfile_path:
self.pipe_info.logfile_path = self.get_rel_path(logfile_path)
self._inp = inout.Input(self)
self._outp = inout.ScriptOutput(self)
self.rejected_execution = False
# If pe_id is None we have a normal script
# If pe_id is not None a JupyterNotebook uses this script
if pe_id is None:
try:
self.main()
self.i_am_done()
self._dbm.close_session()
except:
err_msg = str(datetime.datetime.now()) + '\n'
err_msg += traceback.format_exc()
self.report_err(err_msg)
self._dbm.close_session()
def __patch_user_api_token(self):
lost_config = LOSTConfig()
db = MySQLdb.connect(host=lost_config.lost_db_ip,
port=int(lost_config.lost_db_port),
user=lost_config.lost_db_user,
passwd=lost_config.lost_db_pwd,
db=lost_config.lost_db_name)
cur = db.cursor()
cur.execute("ALTER TABLE user ADD COLUMN api_token varchar(4096)")
db.close()
def celery_exec_script(pipe_element_id):
try:
# Collect context information for celery task
logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
pipe_e = dbm.get_pipe_element(pipe_e_id=pipe_element_id)
worker = CurrentWorker(dbm, lostconfig)
if not worker.enough_resources(pipe_e.script):
logger.warning(
'Not enough resources! Rejected {} (PipeElement ID {})'.format(
pipe_e.script.path, pipe_e.idx))
return
pipe_e.state = state.PipeElement.IN_PROGRESS
dbm.save_obj(pipe_e)
file_man = FileMan(lostconfig)
pipe = pipe_e.pipe
cmd = gen_run_cmd("pudb3", pipe_e, lostconfig)
debug_script_path = file_man.get_instance_path(pipe_e)
debug_script_path = os.path.join(debug_script_path, 'debug.sh')
with open(debug_script_path, 'w') as sfile:
sfile.write(cmd)
cmd = gen_run_cmd("python3", pipe_e, lostconfig)
start_script_path = file_man.get_instance_path(pipe_e)
start_script_path = os.path.join(start_script_path, 'start.sh')
with open(start_script_path, 'w') as sfile:
sfile.write(cmd)
p = subprocess.Popen('bash {}'.format(start_script_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
logger.info("{} ({}): Started script\n{}".format(
pipe.name, pipe.idx, cmd))
worker.add_script(pipe_e, pipe_e.script)
out, err = p.communicate()
worker.remove_script(pipe_e, pipe_e.script)
if p.returncode != 0:
raise Exception(err.decode('utf-8'))
logger.info('{} ({}): Executed script successful: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
dbm.close_session()
except:
pipe = pipe_e.pipe
logger.info('{} ({}): Exception occurred in script: {}'.format(
pipe.name, pipe.idx, pipe_e.script.path))
msg = traceback.format_exc()
logger.error(msg)
script_api.report_script_err(pipe_e, pipe, dbm, msg)
dbm.close_session()
def send_life_sign():
logger = get_task_logger(__name__)
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
logger.info('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
dbm.add(worker)
dbm.commit()
logger.info('Sent lifesign: {}'.format(worker.worker_name))
dbm.close_session()
def init_worker_on_startup():
lostconfig = LOSTConfig()
dbm = DBMan(lostconfig)
worker = dbm.get_worker(lostconfig.worker_name)
if worker is None:
register_worker(dbm, lostconfig)
print('Registered worker: {}'.format(lostconfig.worker_name))
else:
worker.timestamp = datetime.utcnow()
worker.resources = '[]'
worker.in_progress = '{}'
dbm.add(worker)
dbm.commit()
print('Reset worker on startup: {}'.format(worker.worker_name))
dbm.close_session()
def update_version_log():
fm = FileMan(LOSTConfig())
path = fm.get_version_log_path()
if not os.path.exists(path):
print('Patchsystem: Created version log file: {}'.format(path))
versions = []
versions.append(lost.__version__)
with open(path, 'w') as json_file:
json.dump(versions, json_file)
else:
with open(path) as json_file:
versions = json.load(json_file)
if versions[-1] == lost.__version__:
print('Patchsystem: No version change!')
else:
print('Patchsystem: We maybe need to patch!')
import os from lost.logic.config import LOSTConfig FLASK_THREADED = True RESTPLUS_SWAGGER_EXPANSIONS = 'list' RESTPLUS_VAL = True RESTPLUS_MASK_SWAGGER = False LOST_CONFIG = LOSTConfig() FLASK_DEBUG = LOST_CONFIG.debug # Flask settings SECRET_KEY = 'Test' # Flask-Mail SMTP server settings MAIL_SERVER = LOST_CONFIG.mail_server #'smtp.gmail.com' MAIL_PORT = LOST_CONFIG.mail_port #465 MAIL_USE_SSL = LOST_CONFIG.mail_use_ssl #True MAIL_USE_TLS = LOST_CONFIG.mail_use_tls #False MAIL_USERNAME = LOST_CONFIG.mail_username #'*****@*****.**' MAIL_PASSWORD = LOST_CONFIG.mail_password #'password' MAIL_DEFAULT_SENDER = LOST_CONFIG.mail_default_sender #'"MyApp" <*****@*****.**>' # Flask-User settings USER_APP_NAME = "LOST" # Shown in and email templates and page footers USER_ENABLE_EMAIL = True # Enable email authentication USER_ENABLE_USERNAME = False # Disable username authentication USER_EMAIL_SENDER_NAME = USER_APP_NAME USER_EMAIL_SENDER_EMAIL = "*****@*****.**"