def session(): if ROOT_FOLDER: shutil.rmtree(ROOT_FOLDER) reload(mlcomp) migrate() res = Session.create_session() yield res
def _dag(config: str, debug: bool = False, control_reqs=True): migrate() config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) type_name = config_parsed['info'].get('type', 'standard') if type_name == DagType.Standard.name.lower(): return dag_standard(session=_session, config=config_parsed, debug=debug, config_text=config_text, config_path=config, control_reqs=control_reqs) return dag_pipe(session=_session, config=config_parsed, config_text=config_text)
def start(daemon: bool, debug: bool, workers: int, log_level: str): """ Start both server and worker on the same machine. It starts: redis-server, site, worker_supervisor, workers """ migrate() # creating supervisord config supervisor_command = 'mlcomp-worker worker-supervisor' worker_command = 'mlcomp-worker worker' server_command = 'mlcomp-server start-site' if debug: supervisor_command = 'python mlcomp/worker/__main__.py ' \ 'worker-supervisor' worker_command = 'python mlcomp/worker/__main__.py worker' server_command = 'python mlcomp/server/__main__.py start-site' folder = os.path.dirname(os.path.dirname(__file__)) redis_path = os.path.join(folder, 'bin/redis-server') daemon_text = 'false' if daemon else 'true' text = [ '[supervisord]', f'nodaemon={daemon_text}', '', '[program:supervisor]', f'command={supervisor_command}', 'autostart=true', 'autorestart=true', '', '[program:redis]', f'command={redis_path} --port {REDIS_PORT}' f' --requirepass {REDIS_PASSWORD}', 'autostart=true', 'autorestart=true', '', '[program:server]', f'command={server_command}', 'autostart=true', 'autorestart=true', '' ] for p in range(workers): text.append(f'[program:worker{p}]') text.append(f'command={worker_command} {p}') text.append('autostart=true') text.append('autorestart=true') text.append('') conf = os.path.join(CONFIG_FOLDER, 'supervisord.conf') with open(conf, 'w') as f: f.writelines('\n'.join(text)) os.system(f'supervisord ' f'-c {conf} -e {log_level}')
def _dag(config: str, debug: bool = False, control_reqs=True, params: Tuple[str] = ()): migrate() config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) params = dict_from_list_str(params) config_parsed = merge_dicts_smart(config_parsed, params) config_text = yaml_dump(config_parsed) type_name = config_parsed['info'].get('type', 'standard') if type_name == DagType.Standard.name.lower(): return dag_standard(session=_session, config=config_parsed, debug=debug, config_text=config_text, config_path=config, control_reqs=control_reqs) return dag_pipe(session=_session, config=config_parsed, config_text=config_text)
def start_site(): """ Start only site """ migrate() _start_server()