def gpu_reqs(self, value):
if not isinstance(value, dict):
raise ree.TypeError(expected_type=dict, actual_type=type(value))
expected_keys = set([
'processes', 'threads_per_process', 'process_type', 'thread_type'
])
if set(value.keys()) < expected_keys:
raise ree.MissingError(obj='gpu_reqs',
missing_attribute=expected_keys -
set(value.keys()))
if not isinstance(value.get('processes'), (type(None), int)):
raise ree.TypeError(expected_type=dict,
entity='processes',
actual_type=type(value.get('processes')))
if value.get('process_type') not in [None, 'MPI', '']:
raise ree.ValueError(expected_value='None or MPI',
obj='gpu_reqs',
actual_value=value.get('process_type'),
attribute='process_type')
if not isinstance(value.get('threads_per_process'), (type(None), int)):
raise ree.TypeError(expected_type=int,
entity='threads_per_process',
actual_type=type(
value.get('threads_per_process')))
if value.get('thread_type') not in [None, 'OpenMP', 'CUDA', '']:
raise ree.ValueError(expected_value='None or OpenMP or CUDA',
actual_value=value.get('thread_type'),
obj='gpu_reqs',
attribute='thread_type')
self._gpu_reqs['processes'] = value.get('processes', 1)
self._gpu_reqs['process_type'] = value.get('process_type')
self._gpu_reqs['threads_per_process'] = value.get(
'threads_per_process', 1)
self._gpu_reqs['thread_type'] = value.get('thread_type')
def name(self, value):
if not isinstance(value, basestring):
raise ree.TypeError(expected_type=basestring,
actual_type=type(value))
if ',' in value:
raise ree.ValueError(
obj=self._uid,
attribute='name',
actual_value=value,
expected_value="Using ',' in an object's name will "
"corrupt the profiling and internal mapping tables")
self._name = value
def resolve_arguments(args, placeholders):
resolved_args = list()
for entry in args:
# If entry starts with $, it has a placeholder
# and needs to be resolved based after a lookup in
# the placeholders
if not isinstance(entry, str) or \
not entry.startswith('$'):
resolved_args.append(entry)
continue
placeholder = entry.split('/')[0]
if placeholder == "$SHARED":
entry = entry.replace(placeholder, '$RP_PILOT_STAGING')
elif placeholder.startswith('$Pipeline'):
elems = placeholder.split('_')
if len(elems) != 6:
expected = '$Pipeline_{pipeline.uid}_' \
'Stage_{stage.uid}_' \
'Task_{task.uid} or $SHARED'
raise ree.ValueError(obj='placeholder',
attribute='length',
expected_value=expected,
actual_value=elems)
pname = elems[1]
sname = elems[3]
tname = elems[5]
try:
entry = entry.replace(
placeholder, placeholders[pname][sname][tname]['path'])
except Exception:
logger.warning(
'Argument parsing failed. Task %s of Stage %s '
'in Pipeline %s does not exist', tname, sname, pname)
resolved_args.append(entry)
return resolved_args
def _validate(self):
"""
Purpose: Validate that the state of the task is 'DESCRIBED' and that an
executable has been specified for the task.
"""
if self._state is not states.INITIAL:
raise ree.ValueError(obj=self._uid,
attribute='state',
expected_value=states.INITIAL,
actual_value=self._state)
if not self._executable:
raise ree.MissingError(obj=self._uid,
missing_attribute='executable')
def state(self, value):
if not isinstance(value, basestring):
raise ree.TypeError(expected_type=basestring,
actual_type=type(value))
if value not in states._task_state_values:
raise ree.ValueError(
obj=self._uid,
attribute='state',
expected_value=states._task_state_values.keys(),
actual_value=value)
self._state = value
self._state_history.append(value)
def resolve_placeholders(path, placeholders):
"""
**Purpose**: Substitute placeholders in staging attributes of a Task with
actual paths to the corresponding tasks.
:arguments:
:path: string describing the staging paths, possibly
containing a placeholder
:placeholders: dictionary holding the values for placeholders
"""
try:
if isinstance(path, str):
path = str(path)
if not isinstance(path, str):
raise ree.TypeError(expected_type=str,
actual_type=type(path))
if '$' not in path:
return path
# Extract placeholder from path
if len(path.split('>')) == 1:
placeholder = path.split('/')[0]
else:
if path.split('>')[0].strip().startswith('$'):
placeholder = path.split('>')[0].strip().split('/')[0]
else:
placeholder = path.split('>')[1].strip().split('/')[0]
# SHARED
if placeholder == "$SHARED":
return path.replace(placeholder, 'pilot://')
# Expected placeholder format:
# $Pipeline_{pipeline.uid}_Stage_{stage.uid}_Task_{task.uid}
elems = placeholder.split('/')[0].split('_')
if not len(elems) == 6:
expected = '$Pipeline_(pipeline_name)_' \
'Stage_(stage_name)_' \
'Task_(task_name) or $SHARED',
raise ree.ValueError(obj='placeholder', attribute='task',
expected_value=expected, actual_value=elems)
pname = elems[1]
sname = elems[3]
tname = elems[5]
resolved = None
if pname in placeholders:
if sname in placeholders[pname]:
if tname in placeholders[pname][sname]:
resolved = path.replace(placeholder,
placeholders[pname][sname][tname]['path'])
else:
logger.warning('%s not assigned to any task in Stage %s Pipeline %s' %
(tname, sname, pname))
else:
logger.warning('%s not assigned to any Stage in Pipeline %s' % (
sname, pname))
else:
logger.warning('%s not assigned to any Pipeline' % (pname))
if not resolved:
logger.warning('No placeholder could be found for task name %s \
stage name %s and pipeline name %s. Please be sure to \
use object names and not uids in your references,i.e, \
$Pipeline_(pipeline_name)_Stage_(stage_name)_Task_(task_name)')
expected = '$Pipeline_(pipeline_name)_' \
'Stage_(stage_name)_' \
'Task_(task_name) or $SHARED'
raise ree.ValueError(obj='placeholder', attribute='task',
expected_value=expected, actual_value=elems)
return resolved
except Exception as ex:
logger.exception('Failed to resolve placeholder %s, error: %s' %
(path, ex))
raise