def _dag(config: str, debug: bool = False, control_reqs=True,
params: Tuple[str] = ()):
logger = create_logger(_session, name='_dag')
logger.info('started', ComponentType.Client)
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
logger.info('config parsed', ComponentType.Client)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
return dag_standard(
session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs,
logger=logger,
component=ComponentType.Client
)
return dag_pipe(
session=_session, config=config_parsed, config_text=config_text
)
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
logger = create_logger(_session, name='_dag')
logger.info('started', ComponentType.Client)
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
logger.info('config parsed', ComponentType.Client)
try:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
config_parsed['info']['name'] += f'_{commit.decode("utf-8")[:6]}'
except Exception:
logger.info('commit not parsed')
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
cells = grid_cells(
config_parsed['grid']) if 'grid' in config_parsed else [None]
dags = []
for cell in cells:
dag = dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs,
logger=logger,
component=ComponentType.Client,
grid_cell=cell)
dags.append(dag)
return dags
return [
dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
]
def _dag(config: str, debug: bool = False, control_reqs=True):
migrate()
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
return dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs)
return dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)
def _dag(config: str,
debug: bool = False,
control_reqs=True,
params: Tuple[str] = ()):
migrate()
config_text = open(config, 'r').read()
config_parsed = yaml_load(config_text)
params = dict_from_list_str(params)
config_parsed = merge_dicts_smart(config_parsed, params)
config_text = yaml_dump(config_parsed)
type_name = config_parsed['info'].get('type', 'standard')
if type_name == DagType.Standard.name.lower():
return dag_standard(session=_session,
config=config_parsed,
debug=debug,
config_text=config_text,
config_path=config,
control_reqs=control_reqs)
return dag_pipe(session=_session,
config=config_parsed,
config_text=config_text)