def _init_context_uris(self): """ Generate all context URIs for this workflow run. Context URIs are generated based on contexts given in _parsed_job_work_uri, and the "final" context for steps given in the _parsed_job_output_uri. Args: None. Returns: On failure: Raises WorkflowDAGException. """ self._context_uris['inputs'] = {} self._context_uris['steps'] = {'final': {}} self._parsed_context_uris['inputs'] = {} self._parsed_context_uris['steps'] = {'final': {}} # init all data contexts for context in { Contexts.get_data_scheme_of_exec_context(con) for con in self._exec_contexts } | self._data_contexts: self._context_uris['inputs'][context] = {} self._parsed_context_uris['inputs'][context] = {} for node_name in self._topo_sort: node = self._graph.nodes[node_name] if node['type'] == 'input': if node['source_context'] == context: # use original input URI parsed_uri = URIParser.parse( self._workflow['inputs'][node['name']]['value']) if not parsed_uri: msg = 'invalid input uri: {}'.format( self._workflow['inputs'][ node['name']]['value']) raise WorkflowDAGException(msg) self._context_uris['inputs'][context][node['name']]\ = parsed_uri['chopped_uri'] self._parsed_context_uris['inputs'][context]\ [node['name']] = parsed_uri else: # skip if _parsed_job_work_uri is not defined for this context # this implies that there is no execution defined for that context, # so no need to setup the data staging location at the work_uri if context not in self._parsed_job_work_uri: continue # switch context of input URI new_base_uri = '{}/_input-{}'.format( self._parsed_job_work_uri[context]['chopped_uri'], slugify(node['name'], regex_pattern=r'[^-a-z0-9_]+')) # create new base URI if not DataManager.mkdir( uri=new_base_uri, recursive=True, **{context: self._context_options[context]}): msg = 'cannot create new base uri for input: {}'\ .format(new_base_uri) Log.an().error(msg) raise WorkflowDAGException(msg) # switch input URI base switched_uri = URIParser.switch_context( self._workflow['inputs'][node['name']]['value'], new_base_uri) if not switched_uri: msg = ( 'cannot switch input uri context to ' 'new base URI: {}->{}' ).format( self._workflow['inputs'][node['name']]\ ['value'], new_base_uri ) Log.an().error(msg) raise WorkflowDAGException(msg) self._context_uris['inputs'][context][node['name']]\ = switched_uri['chopped_uri'] self._parsed_context_uris['inputs'][context]\ [node['name']] = switched_uri for context in { Contexts.get_data_scheme_of_exec_context(con) for con in self._exec_contexts }: self._context_uris['steps'][context] = {} self._parsed_context_uris['steps'][context] = {} for node_name in self._topo_sort: node = self._graph.nodes[node_name] if node['type'] == 'step': self._context_uris['steps'][context][node['name']]\ = '{}/{}'.format( self._parsed_job_work_uri[context]['chopped_uri'], slugify(node['name'], regex_pattern=r'[^-a-z0-9_]+') ) self._parsed_context_uris['steps'][context][node['name']]\ = URIParser.parse( self._context_uris['steps'][context][node['name']] ) # init final contexts for steps for node_name in self._topo_sort: node = self._graph.nodes[node_name] if node['type'] == 'step': self._context_uris['steps']['final'][node['name']]\ = '{}/{}'.format( self._parsed_job_output_uri['chopped_uri'], slugify(node['name'], regex_pattern=r'[^-a-z0-9_]+') ) self._parsed_context_uris['steps']['final'][node['name']]\ = URIParser.parse( self._context_uris['steps']['final'][node['name']] )
def step_impl(context, new_base_uri): for uri in context.uris: new_uri = URIParser.switch_context(uri, new_base_uri) assert new_uri context.uris[uri] = new_uri
def _init_context_uris(self): """ Generate all context URIs for this workflow run. Context URIs are generated based on contexts given in _parsed_job_work_uri, and the "final" context for steps given in the _parsed_job_output_uri. Args: None. Returns: On failure: Raises WorkflowDAGException. """ self._context_uris['inputs'] = {} self._context_uris['steps'] = {'final': {}} self._parsed_context_uris['inputs'] = {} self._parsed_context_uris['steps'] = {'final': {}} # init contexts in parsed_job_work_uri for inputs and steps for context in self._parsed_job_work_uri: self._context_uris['inputs'][context] = {} self._context_uris['steps'][context] = {} self._parsed_context_uris['inputs'][context] = {} self._parsed_context_uris['steps'][context] = {} for node_name in self._topo_sort: node = self._graph.nodes[node_name] if node['type'] == 'input': if node['source_context'] == context: # use original input URI parsed_uri = URIParser.parse( self._workflow['inputs'][node['name']]['value']) if not parsed_uri: msg = 'invalid input uri: {}'.format( self._workflow['inputs'][ node['name']]['value']) raise WorkflowDAGException(msg) self._context_uris['inputs'][context][node['name']]\ = parsed_uri['chopped_uri'] self._parsed_context_uris['inputs'][context]\ [node['name']] = parsed_uri else: # switch context of input URI new_base_uri = '{}/_input-{}'.format( self._parsed_job_work_uri[context]['chopped_uri'], slugify(node['name'])) # create new base URI if not DataManager.mkdir( uri=new_base_uri, recursive=True, **{context: self._context_options[context]}): msg = 'cannot create new base uri for input: {}'\ .format(new_base_uri) Log.an().error(msg) raise WorkflowDAGException(msg) # switch input URI base switched_uri = URIParser.switch_context( self._workflow['inputs'][node['name']]['value'], new_base_uri) if not switched_uri: msg = ( 'cannot switch input uri context to ' 'new base URI: {}->{}' ).format( self._workflow['inputs'][node['name']]\ ['value'], new_base_uri ) Log.an().error(msg) raise WorkflowDAGException(msg) self._context_uris['inputs'][context][node['name']]\ = switched_uri['chopped_uri'] self._parsed_context_uris['inputs'][context]\ [node['name']] = switched_uri else: # node['type'] == 'step' self._context_uris['steps'][context][node['name']]\ = '{}/{}'.format( self._parsed_job_work_uri[context]['chopped_uri'], slugify(node['name']) ) self._parsed_context_uris['steps'][context][node['name']]\ = URIParser.parse( self._context_uris['steps'][context][node['name']] ) # init final contexts for steps for node_name in self._topo_sort: node = self._graph.nodes[node_name] if node['type'] == 'step': self._context_uris['steps']['final'][node['name']]\ = '{}/{}'.format( self._parsed_job_output_uri['chopped_uri'], slugify(node['name']) ) self._parsed_context_uris['steps']['final'][node['name']]\ = URIParser.parse( self._context_uris['steps']['final'][node['name']] )