def session():
if ROOT_FOLDER:
shutil.rmtree(ROOT_FOLDER)
reload(mlcomp)
migrate()
res = Session.create_session()
yield res
def _dag(config: str, debug: bool = False, control_reqs=True):
migrate()
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
return dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs)
return dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
def start(daemon: bool, debug: bool, workers: int, log_level: str):
"""
Start both server and worker on the same machine.
It starts: redis-server, site, worker_supervisor, workers
"""
migrate()
# creating supervisord config
supervisor_command = 'mlcomp-worker worker-supervisor'
worker_command = 'mlcomp-worker worker'
server_command = 'mlcomp-server start-site'
if debug:
supervisor_command = 'python mlcomp/worker/__main__.py ' \
'worker-supervisor'
worker_command = 'python mlcomp/worker/__main__.py worker'
server_command = 'python mlcomp/server/__main__.py start-site'
folder = os.path.dirname(os.path.dirname(__file__))
redis_path = os.path.join(folder, 'bin/redis-server')
daemon_text = 'false' if daemon else 'true'
text = [
'[supervisord]', f'nodaemon={daemon_text}', '', '[program:supervisor]',
f'command={supervisor_command}', 'autostart=true', 'autorestart=true',
'', '[program:redis]', f'command={redis_path} --port {REDIS_PORT}'
f' --requirepass {REDIS_PASSWORD}', 'autostart=true',
'autorestart=true', '', '[program:server]',
f'command={server_command}', 'autostart=true', 'autorestart=true', ''
]
for p in range(workers):
text.append(f'[program:worker{p}]')
text.append(f'command={worker_command} {p}')
text.append('autostart=true')
text.append('autorestart=true')
text.append('')
conf = os.path.join(CONFIG_FOLDER, 'supervisord.conf')
with open(conf, 'w') as f:
f.writelines('\n'.join(text))
os.system(f'supervisord ' f'-c {conf} -e {log_level}')
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
migrate()
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
return dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs)
return dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
def start_site():
"""
Start only site
"""
migrate()
_start_server()