def create(unit_manager_obj, unit_description, local_state):
""" PRIVATE: Create a new compute unit.
"""
# create and return pilot object
computeunit = ComputeUnit()
# Make a copy of the UD to work on without side-effects.
ud_copy = copy.deepcopy(unit_description)
# sanity check on description
if (not 'executable' in unit_description or \
not unit_description['executable'] ) and \
(not 'kernel' in unit_description or \
not unit_description['kernel'] ) :
raise PilotException ("ComputeUnitDescription needs an executable or application kernel")
# If staging directives exist, try to expand them
if ud_copy.input_staging:
ud_copy.input_staging = expand_staging_directive(ud_copy.input_staging, logger)
if ud_copy.output_staging:
ud_copy.output_staging = expand_staging_directive(ud_copy.output_staging, logger)
computeunit._description = ud_copy
computeunit._manager = unit_manager_obj
computeunit._worker = unit_manager_obj._worker
computeunit._uid = ru.generate_id('unit.%(counter)06d', ru.ID_CUSTOM)
computeunit._name = unit_description['name']
computeunit._local_state = local_state
return computeunit
def create(unit_manager_obj, unit_description, local_state):
""" PRIVATE: Create a new compute unit.
"""
# create and return pilot object
computeunit = ComputeUnit()
# Make a copy of the UD to work on without side-effects.
ud_copy = copy.deepcopy(unit_description)
# sanity check on description
if (not 'executable' in unit_description or \
not unit_description['executable'] ) and \
(not 'kernel' in unit_description or \
not unit_description['kernel'] ) :
raise PilotException(
"ComputeUnitDescription needs an executable or application kernel"
)
# If staging directives exist, try to expand them
if ud_copy.input_staging:
ud_copy.input_staging = expand_staging_directive(
ud_copy.input_staging, logger)
if ud_copy.output_staging:
ud_copy.output_staging = expand_staging_directive(
ud_copy.output_staging, logger)
computeunit._description = ud_copy
computeunit._manager = unit_manager_obj
computeunit._worker = unit_manager_obj._worker
computeunit._uid = ru.generate_id('unit.%(counter)06d', ru.ID_CUSTOM)
computeunit._name = unit_description['name']
computeunit._local_state = local_state
return computeunit
def stage_in(self, directives):
"""Stages the content of the staging directive into the pilot's
staging area"""
# Wait until we can assume the pilot directory to be created
if self.state == NEW:
self.wait(state=[PENDING_LAUNCH, LAUNCHING, PENDING_ACTIVE, ACTIVE])
elif self.state in [DONE, FAILED, CANCELED]:
raise Exception("Pilot already finished, no need to stage anymore!")
# Iterate over all directives
for directive in expand_staging_directive(directives, logger):
# TODO: respect flags in directive
src_url = saga.Url(directive['source'])
action = directive['action']
# Convert the target url into a SAGA Url object
tgt_url = saga.Url(directive['target'])
# Create a pointer to the directory object that we will use
tgt_dir_url = tgt_url
if tgt_url.path.endswith('/'):
# If the original target was a directory (ends with /),
# we assume that the user wants the same filename as the source.
tgt_filename = os.path.basename(src_url.path)
else:
# Otherwise, extract the filename and update the directory
tgt_filename = os.path.basename(tgt_dir_url.path)
tgt_dir_url.path = os.path.dirname(tgt_dir_url.path)
# Handle special 'staging' scheme
if tgt_dir_url.scheme == 'staging':
# We expect a staging:///relative/path/file.txt URI,
# as hostname would have unclear semantics currently.
if tgt_dir_url.host:
raise Exception("hostname not supported with staging:// scheme")
# Remove the leading slash to get a relative path from the staging area
target = os.path.relpath(tgt_dir_url.path, '/')
# Now base the target directory relative of the sandbox and staging prefix
tgt_dir_url = saga.Url(os.path.join(self.sandbox, STAGING_AREA, target))
# Define and open the staging directory for the pilot
# We use the target dir construct here, so that we can create
# the directory if it does not yet exist.
target_dir = saga.filesystem.Directory(tgt_dir_url, flags=saga.filesystem.CREATE_PARENTS)
if action == LINK:
# TODO: Does this make sense?
#log_message = 'Linking %s to %s' % (source, abs_target)
#os.symlink(source, abs_target)
logger.error("action 'LINK' not supported on pilot level staging")
raise ValueError("action 'LINK' not supported on pilot level staging")
elif action == COPY:
# TODO: Does this make sense?
#log_message = 'Copying %s to %s' % (source, abs_target)
#shutil.copyfile(source, abs_target)
logger.error("action 'COPY' not supported on pilot level staging")
raise ValueError("action 'COPY' not supported on pilot level staging")
elif action == MOVE:
# TODO: Does this make sense?
#log_message = 'Moving %s to %s' % (source, abs_target)
#shutil.move(source, abs_target)
logger.error("action 'MOVE' not supported on pilot level staging")
raise ValueError("action 'MOVE' not supported on pilot level staging")
elif action == TRANSFER:
log_message = 'Transferring %s to %s' % (src_url, os.path.join(str(tgt_dir_url), tgt_filename))
logger.info(log_message)
# Transfer the source file to the target staging area
target_dir.copy(src_url, tgt_filename)
else:
raise Exception('Action %s not supported' % action)
