def submit_data_unit(self, data_unit_description): """ creates a data unit object and binds it to a physical resource (a pilotdata) """ du = DataUnit(pilot_data=None, data_unit_description=data_unit_description) self.data_units[du.id]=du self.du_queue.put(du) # queue currently not persisted CoordinationAdaptor.update_cds(self.url, self) return du
def __stage_out_data_units(self, output_data=[], workingdirectory=None): """ stage out data to a specified data unit pilot data """ logger.debug("Stage out output files") """ Parsing output data field of job description: { ... "output_data": [ { output_data_unit.get_url(): ["stdout.txt", "stderr.txt"] } ] } """ try: for data_unit_dict in output_data: logger.debug("Process: " + str(data_unit_dict)) for du_url in data_unit_dict.keys( ): # go through all dicts (each representing 1 PD) #pd_url = self.__get_pd_url(du_url) #pilot_data = PilotData(pd_url=pd_url) #du = pilot_data.get_du(du_url) du = DataUnit(du_url=du_url) file_list = data_unit_dict[du_url] logger.debug("Add files: " + str(file_list)) all_files = [] for output_file in file_list: expanded_files = [output_file] if output_file.find("*") >= 0 or output_file.find( "?") >= 0: expanded_files = self.__expand_file_pattern( output_file, workingdirectory) logger.debug("Expanded files: " + str(expanded_files)) for f in expanded_files: all_files.append(os.path.join(workingdirectory, f)) du.add_files(all_files) for f in all_files: os.remove(f) except: logger.error("Stage out of files failed.") self.__print_traceback()
def __stage_out_data_units(self, output_data=[], workingdirectory=None): """ stage out data to a specified data unit pilot data """ logger.debug("Stage out output files") """ Parsing output data field of job description: { ... "output_data": [ { output_data_unit.get_url(): ["stdout.txt", "stderr.txt"] } ] } """ try: for data_unit_dict in output_data: logger.debug("Process: " + str(data_unit_dict)) for du_url in data_unit_dict.keys(): # go through all dicts (each representing 1 PD) #pd_url = self.__get_pd_url(du_url) #pilot_data = PilotData(pd_url=pd_url) #du = pilot_data.get_du(du_url) du = DataUnit(du_url=du_url) file_list = data_unit_dict[du_url] logger.debug("Add files: " + str(file_list)) all_files=[] for output_file in file_list: expanded_files = [output_file] if output_file.find("*")>=0 or output_file.find("?")>=0: expanded_files = self.__expand_file_pattern(output_file, workingdirectory) logger.debug("Expanded files: " + str(expanded_files)) for f in expanded_files: all_files.append(os.path.join(workingdirectory, f)) du.add_files(all_files) for f in all_files: os.remove(f) except: logger.error("Stage out of files failed.") self.__print_traceback()
def __stage_in_data_units(self, input_data=[], target_directory="."): """ stage in data units specified in input_data field """ try: logger.debug("Stage in input files to: %s" % target_directory) for i in input_data: du = DataUnit(du_url=i) logger.debug("Restored DU... call get state()") logger.debug("DU State: " + du.get_state()) du.wait() logger.debug("Reconnected to DU. Exporting it now...") du.export(target_directory) except: logger.error("Stage-in of files failed.") self.__print_traceback()
def __stage_in_data_units(self, input_data=[], target_directory="."): """ stage in data units specified in input_data field """ logger.debug("Stage in input files to: %s" % target_directory) for i in input_data: #pd_url = self.__get_pd_url(i) #du_id = self.__get_du_id(i) #pd = PilotData(pd_url=pd_url) #du = pd.get_du(du_id) #du.export(target_directory) du = DataUnit(du_url=i) du.wait() du.export(target_directory)
def __stage_in_data_units(self, input_data=[], target_directory="."): """ stage in data units specified in input_data field """ try: logger.debug("Stage in input files to: %s"%target_directory) for i in input_data: du = DataUnit(du_url=i) logger.debug("Restored DU... call get state()") logger.debug("DU State: " + du.get_state()) du.wait() logger.debug("Reconnected to DU. Exporting it now...") du.export(target_directory) except: logger.error("Stage-in of files failed.") self.__print_traceback()