def create(unit_manager_obj, unit_description, local_state): """ PRIVATE: Create a new compute unit. """ # create and return pilot object computeunit = ComputeUnit() # Make a copy of the UD to work on without side-effects. ud_copy = copy.deepcopy(unit_description) # sanity check on description if (not 'executable' in unit_description or \ not unit_description['executable'] ) and \ (not 'kernel' in unit_description or \ not unit_description['kernel'] ) : raise PilotException ("ComputeUnitDescription needs an executable or application kernel") # If staging directives exist, try to expand them if ud_copy.input_staging: ud_copy.input_staging = expand_staging_directive(ud_copy.input_staging, logger) if ud_copy.output_staging: ud_copy.output_staging = expand_staging_directive(ud_copy.output_staging, logger) computeunit._description = ud_copy computeunit._manager = unit_manager_obj computeunit._worker = unit_manager_obj._worker computeunit._uid = ru.generate_id('unit.%(counter)06d', ru.ID_CUSTOM) computeunit._name = unit_description['name'] computeunit._local_state = local_state return computeunit
def create(unit_manager_obj, unit_description, local_state): """ PRIVATE: Create a new compute unit. """ # create and return pilot object computeunit = ComputeUnit() # Make a copy of the UD to work on without side-effects. ud_copy = copy.deepcopy(unit_description) # sanity check on description if (not 'executable' in unit_description or \ not unit_description['executable'] ) and \ (not 'kernel' in unit_description or \ not unit_description['kernel'] ) : raise PilotException( "ComputeUnitDescription needs an executable or application kernel" ) # If staging directives exist, try to expand them if ud_copy.input_staging: ud_copy.input_staging = expand_staging_directive( ud_copy.input_staging, logger) if ud_copy.output_staging: ud_copy.output_staging = expand_staging_directive( ud_copy.output_staging, logger) computeunit._description = ud_copy computeunit._manager = unit_manager_obj computeunit._worker = unit_manager_obj._worker computeunit._uid = ru.generate_id('unit.%(counter)06d', ru.ID_CUSTOM) computeunit._name = unit_description['name'] computeunit._local_state = local_state return computeunit
def stage_in(self, directives): """Stages the content of the staging directive into the pilot's staging area""" # Wait until we can assume the pilot directory to be created if self.state == NEW: self.wait(state=[PENDING_LAUNCH, LAUNCHING, PENDING_ACTIVE, ACTIVE]) elif self.state in [DONE, FAILED, CANCELED]: raise Exception("Pilot already finished, no need to stage anymore!") # Iterate over all directives for directive in expand_staging_directive(directives, logger): # TODO: respect flags in directive src_url = saga.Url(directive['source']) action = directive['action'] # Convert the target url into a SAGA Url object tgt_url = saga.Url(directive['target']) # Create a pointer to the directory object that we will use tgt_dir_url = tgt_url if tgt_url.path.endswith('/'): # If the original target was a directory (ends with /), # we assume that the user wants the same filename as the source. tgt_filename = os.path.basename(src_url.path) else: # Otherwise, extract the filename and update the directory tgt_filename = os.path.basename(tgt_dir_url.path) tgt_dir_url.path = os.path.dirname(tgt_dir_url.path) # Handle special 'staging' scheme if tgt_dir_url.scheme == 'staging': # We expect a staging:///relative/path/file.txt URI, # as hostname would have unclear semantics currently. if tgt_dir_url.host: raise Exception("hostname not supported with staging:// scheme") # Remove the leading slash to get a relative path from the staging area target = os.path.relpath(tgt_dir_url.path, '/') # Now base the target directory relative of the sandbox and staging prefix tgt_dir_url = saga.Url(os.path.join(self.sandbox, STAGING_AREA, target)) # Define and open the staging directory for the pilot # We use the target dir construct here, so that we can create # the directory if it does not yet exist. target_dir = saga.filesystem.Directory(tgt_dir_url, flags=saga.filesystem.CREATE_PARENTS) if action == LINK: # TODO: Does this make sense? #log_message = 'Linking %s to %s' % (source, abs_target) #os.symlink(source, abs_target) logger.error("action 'LINK' not supported on pilot level staging") raise ValueError("action 'LINK' not supported on pilot level staging") elif action == COPY: # TODO: Does this make sense? #log_message = 'Copying %s to %s' % (source, abs_target) #shutil.copyfile(source, abs_target) logger.error("action 'COPY' not supported on pilot level staging") raise ValueError("action 'COPY' not supported on pilot level staging") elif action == MOVE: # TODO: Does this make sense? #log_message = 'Moving %s to %s' % (source, abs_target) #shutil.move(source, abs_target) logger.error("action 'MOVE' not supported on pilot level staging") raise ValueError("action 'MOVE' not supported on pilot level staging") elif action == TRANSFER: log_message = 'Transferring %s to %s' % (src_url, os.path.join(str(tgt_dir_url), tgt_filename)) logger.info(log_message) # Transfer the source file to the target staging area target_dir.copy(src_url, tgt_filename) else: raise Exception('Action %s not supported' % action)