def _from_config(cls, executor: dict, config: Config,
additional_info: dict):
args = Args()
for k, v in executor['args'].items():
v = str(v)
if v in ['False', 'True']:
v = v == 'True'
elif v.isnumeric():
v = int(v)
setattr(args, k, v)
assert 'report_config' in additional_info, 'layout was not filled'
report_config = additional_info['report_config']
grid_cell = additional_info.get('grid_cell')
report = ReportLayoutInfo(report_config)
if len(args.configs) == 0:
args.configs = [args.config]
grid_config = {}
if grid_cell is not None:
grid_config = grid_cells(executor['grid'])[grid_cell][0]
distr_info = additional_info.get('distr_info', {})
resume = additional_info.get('resume')
params = executor.get('params', {})
return cls(args=args,
report=report,
grid_config=grid_config,
distr_info=distr_info,
resume=resume,
trace=executor.get('trace'),
params=params)
def _from_config(
cls, executor: dict, config: Config, additional_info: dict
):
grid_cell = additional_info.get('grid_cell')
grid_config = {}
if grid_cell is not None:
grid_config = grid_cells(executor['grid'])[grid_cell][0]
return cls(**executor, **grid_config)
def _from_config(cls, executor: dict, config: Config,
additional_info: dict):
grid_cell = additional_info.get('grid_cell')
grid_config = {}
if grid_cell is not None:
grid_config = grid_cells(executor['grid'])[grid_cell][0]
for k, v in grid_config.items():
executor['command'] = executor['command'].replace(f'${k}', str(v))
return cls(**executor)
def create_tasks(self):
created = OrderedDict()
executors = self.config['executors']
while len(created) < len(executors):
for k, v in executors.items():
valid = True
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
if d == k:
raise Exception(f'Executor {k} depends ot itself')
if d not in executors:
raise Exception(f'Executor {k} depend on {d} '
f'which does not exist')
valid = valid and d in created
if valid:
names = []
infos = []
if 'grid' in v:
grid = v['grid']
cells = grid_cells(grid)
for i, (cell, cell_name) in enumerate(cells):
name = f'{k} {cell_name}'
names.append(name)
infos.append({'grid_cell': i})
else:
names.append(k)
infos.append({})
ids = []
for name, info in zip(names, infos):
id = self.create_task(k, v, name=name, info=info)
ids.append(id)
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
for dd in created[d]:
self.provider.add_dependency(id, dd)
created[k] = ids
self.created = created
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
logger = create_logger(_session, name='_dag')
logger.info('started', ComponentType.Client)
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
logger.info('config parsed', ComponentType.Client)
try:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
config_parsed['info']['name'] += f'_{commit.decode("utf-8")[:6]}'
except Exception:
logger.info('commit not parsed')
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
cells = grid_cells(
config_parsed['grid']) if 'grid' in config_parsed else [None]
dags = []
for cell in cells:
dag = dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs,
logger=logger,
component=ComponentType.Client,
grid_cell=cell)
dags.append(dag)
return dags
return [
dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
]
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
logger = create_logger(_session, name='_dag')
logger.info('started', ComponentType.Client)
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
logger.info('config parsed', ComponentType.Client)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
cells = grid_cells(
config_parsed['grid']) if 'grid' in config_parsed else [None]
dags = []
for cell in cells:
dag = dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs,
logger=logger,
component=ComponentType.Client,
grid_cell=cell)
dags.append(dag)
return dags
return [
dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
]
def create_tasks(self):
self.log_info('create_tasks')
created = OrderedDict()
executors = self.config['executors']
tasks = []
dependencies = []
reports = []
while len(created) < len(executors):
for k, v in executors.items():
valid = True
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
if d == k:
raise Exception(f'Executor {k} depends on itself')
if d not in executors:
raise Exception(f'Executor {k} depend on {d} '
f'which does not exist')
valid = valid and d in created
if valid:
names = []
infos = []
if 'grid' in v:
grid = v['grid']
cells = grid_cells(grid)
for i, (cell, cell_name) in enumerate(cells):
names.append(cell_name)
infos.append({'grid_cell': i})
else:
names.append(v.get('name', k))
infos.append({})
k_tasks = []
for name, info in zip(names, infos):
task, report = self.create_task(k,
v,
name=name,
info=info)
tasks.append(task)
k_tasks.append(task)
reports.append(report)
if 'depends' in v:
depends = v['depends']
if not isinstance(depends, list):
depends = [depends]
for d in depends:
for dd in created[d]:
dependencies.append((task, dd))
created[k] = k_tasks
not_empty_reports = [r for r in reports if r is not None]
if len(not_empty_reports) > 0:
self.provider.bulk_save_objects(not_empty_reports,
return_defaults=True)
for report, task in zip(reports, tasks):
if report is not None:
task.report = report.id
self.provider.bulk_save_objects(tasks, return_defaults=True)
if len(not_empty_reports) > 0:
report_tasks = []
for report, task in zip(reports, tasks):
if report is not None:
report_tasks.append(
ReportTasks(report=report.id, task=task.id))
self.report_tasks_provider.bulk_save_objects(report_tasks)
dependencies = [
TaskDependence(task_id=task.id, depend_id=dd.id)
for task, dd in dependencies
]
self.provider.bulk_save_objects(dependencies)
for k, v in created.items():
created[k] = [vv.id for vv in v]
self.created = created