def _dag(config: str, debug: bool = False, control_reqs=True, params: Tuple[str] = ()): logger = create_logger(_session, name='_dag') logger.info('started', ComponentType.Client) config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) params = dict_from_list_str(params) config_parsed = merge_dicts_smart(config_parsed, params) config_text = yaml_dump(config_parsed) logger.info('config parsed', ComponentType.Client) type_name = config_parsed['info'].get('type', 'standard') if type_name == DagType.Standard.name.lower(): return dag_standard( session=_session, config=config_parsed, debug=debug, config_text=config_text, config_path=config, control_reqs=control_reqs, logger=logger, component=ComponentType.Client ) return dag_pipe( session=_session, config=config_parsed, config_text=config_text )
def _dag(config: str, debug: bool = False, control_reqs=True, params: Tuple[str] = ()): logger = create_logger(_session, name='_dag') logger.info('started', ComponentType.Client) config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) params = dict_from_list_str(params) config_parsed = merge_dicts_smart(config_parsed, params) config_text = yaml_dump(config_parsed) logger.info('config parsed', ComponentType.Client) try: commit = subprocess.check_output(["git", "rev-parse", "HEAD"]).strip() config_parsed['info']['name'] += f'_{commit.decode("utf-8")[:6]}' except Exception: logger.info('commit not parsed') type_name = config_parsed['info'].get('type', 'standard') if type_name == DagType.Standard.name.lower(): cells = grid_cells( config_parsed['grid']) if 'grid' in config_parsed else [None] dags = [] for cell in cells: dag = dag_standard(session=_session, config=config_parsed, debug=debug, config_text=config_text, config_path=config, control_reqs=control_reqs, logger=logger, component=ComponentType.Client, grid_cell=cell) dags.append(dag) return dags return [ dag_pipe(session=_session, config=config_parsed, config_text=config_text) ]
def _dag(config: str, debug: bool = False, control_reqs=True, params: Tuple[str] = ()): migrate() config_text = open(config, 'r').read() config_parsed = yaml_load(config_text) params = dict_from_list_str(params) config_parsed = merge_dicts_smart(config_parsed, params) config_text = yaml_dump(config_parsed) type_name = config_parsed['info'].get('type', 'standard') if type_name == DagType.Standard.name.lower(): return dag_standard(session=_session, config=config_parsed, debug=debug, config_text=config_text, config_path=config, control_reqs=control_reqs) return dag_pipe(session=_session, config=config_parsed, config_text=config_text)