def _init_context_uris(self):
"""
Generate all context URIs for this workflow run.
Context URIs are generated based on contexts given in
_parsed_job_work_uri, and the "final" context for steps given in the
_parsed_job_output_uri.
Args:
None.
Returns:
On failure: Raises WorkflowDAGException.
"""
self._context_uris['inputs'] = {}
self._context_uris['steps'] = {'final': {}}
self._parsed_context_uris['inputs'] = {}
self._parsed_context_uris['steps'] = {'final': {}}
# init all data contexts
for context in {
Contexts.get_data_scheme_of_exec_context(con)
for con in self._exec_contexts
} | self._data_contexts:
self._context_uris['inputs'][context] = {}
self._parsed_context_uris['inputs'][context] = {}
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'input':
if node['source_context'] == context:
# use original input URI
parsed_uri = URIParser.parse(
self._workflow['inputs'][node['name']]['value'])
if not parsed_uri:
msg = 'invalid input uri: {}'.format(
self._workflow['inputs'][
node['name']]['value'])
raise WorkflowDAGException(msg)
self._context_uris['inputs'][context][node['name']]\
= parsed_uri['chopped_uri']
self._parsed_context_uris['inputs'][context]\
[node['name']] = parsed_uri
else:
# skip if _parsed_job_work_uri is not defined for this context
# this implies that there is no execution defined for that context,
# so no need to setup the data staging location at the work_uri
if context not in self._parsed_job_work_uri:
continue
# switch context of input URI
new_base_uri = '{}/_input-{}'.format(
self._parsed_job_work_uri[context]['chopped_uri'],
slugify(node['name'],
regex_pattern=r'[^-a-z0-9_]+'))
# create new base URI
if not DataManager.mkdir(
uri=new_base_uri,
recursive=True,
**{context: self._context_options[context]}):
msg = 'cannot create new base uri for input: {}'\
.format(new_base_uri)
Log.an().error(msg)
raise WorkflowDAGException(msg)
# switch input URI base
switched_uri = URIParser.switch_context(
self._workflow['inputs'][node['name']]['value'],
new_base_uri)
if not switched_uri:
msg = (
'cannot switch input uri context to '
'new base URI: {}->{}'
).format(
self._workflow['inputs'][node['name']]\
['value'],
new_base_uri
)
Log.an().error(msg)
raise WorkflowDAGException(msg)
self._context_uris['inputs'][context][node['name']]\
= switched_uri['chopped_uri']
self._parsed_context_uris['inputs'][context]\
[node['name']] = switched_uri
for context in {
Contexts.get_data_scheme_of_exec_context(con)
for con in self._exec_contexts
}:
self._context_uris['steps'][context] = {}
self._parsed_context_uris['steps'][context] = {}
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'step':
self._context_uris['steps'][context][node['name']]\
= '{}/{}'.format(
self._parsed_job_work_uri[context]['chopped_uri'],
slugify(node['name'], regex_pattern=r'[^-a-z0-9_]+')
)
self._parsed_context_uris['steps'][context][node['name']]\
= URIParser.parse(
self._context_uris['steps'][context][node['name']]
)
# init final contexts for steps
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'step':
self._context_uris['steps']['final'][node['name']]\
= '{}/{}'.format(
self._parsed_job_output_uri['chopped_uri'],
slugify(node['name'], regex_pattern=r'[^-a-z0-9_]+')
)
self._parsed_context_uris['steps']['final'][node['name']]\
= URIParser.parse(
self._context_uris['steps']['final'][node['name']]
)
def step_impl(context, new_base_uri):
for uri in context.uris:
new_uri = URIParser.switch_context(uri, new_base_uri)
assert new_uri
context.uris[uri] = new_uri
def _init_context_uris(self):
"""
Generate all context URIs for this workflow run.
Context URIs are generated based on contexts given in
_parsed_job_work_uri, and the "final" context for steps given in the
_parsed_job_output_uri.
Args:
None.
Returns:
On failure: Raises WorkflowDAGException.
"""
self._context_uris['inputs'] = {}
self._context_uris['steps'] = {'final': {}}
self._parsed_context_uris['inputs'] = {}
self._parsed_context_uris['steps'] = {'final': {}}
# init contexts in parsed_job_work_uri for inputs and steps
for context in self._parsed_job_work_uri:
self._context_uris['inputs'][context] = {}
self._context_uris['steps'][context] = {}
self._parsed_context_uris['inputs'][context] = {}
self._parsed_context_uris['steps'][context] = {}
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'input':
if node['source_context'] == context:
# use original input URI
parsed_uri = URIParser.parse(
self._workflow['inputs'][node['name']]['value'])
if not parsed_uri:
msg = 'invalid input uri: {}'.format(
self._workflow['inputs'][
node['name']]['value'])
raise WorkflowDAGException(msg)
self._context_uris['inputs'][context][node['name']]\
= parsed_uri['chopped_uri']
self._parsed_context_uris['inputs'][context]\
[node['name']] = parsed_uri
else:
# switch context of input URI
new_base_uri = '{}/_input-{}'.format(
self._parsed_job_work_uri[context]['chopped_uri'],
slugify(node['name']))
# create new base URI
if not DataManager.mkdir(
uri=new_base_uri,
recursive=True,
**{context: self._context_options[context]}):
msg = 'cannot create new base uri for input: {}'\
.format(new_base_uri)
Log.an().error(msg)
raise WorkflowDAGException(msg)
# switch input URI base
switched_uri = URIParser.switch_context(
self._workflow['inputs'][node['name']]['value'],
new_base_uri)
if not switched_uri:
msg = (
'cannot switch input uri context to '
'new base URI: {}->{}'
).format(
self._workflow['inputs'][node['name']]\
['value'],
new_base_uri
)
Log.an().error(msg)
raise WorkflowDAGException(msg)
self._context_uris['inputs'][context][node['name']]\
= switched_uri['chopped_uri']
self._parsed_context_uris['inputs'][context]\
[node['name']] = switched_uri
else: # node['type'] == 'step'
self._context_uris['steps'][context][node['name']]\
= '{}/{}'.format(
self._parsed_job_work_uri[context]['chopped_uri'],
slugify(node['name'])
)
self._parsed_context_uris['steps'][context][node['name']]\
= URIParser.parse(
self._context_uris['steps'][context][node['name']]
)
# init final contexts for steps
for node_name in self._topo_sort:
node = self._graph.nodes[node_name]
if node['type'] == 'step':
self._context_uris['steps']['final'][node['name']]\
= '{}/{}'.format(
self._parsed_job_output_uri['chopped_uri'],
slugify(node['name'])
)
self._parsed_context_uris['steps']['final'][node['name']]\
= URIParser.parse(
self._context_uris['steps']['final'][node['name']]
)