def checkCompleted(self, job): """Check if this unit is complete""" if job.status == "completed": if job.outputdata and job.outputdata._impl._name == "DQ2OutputDataset" and job.backend.__class__.__name__ != "Jedi": # make sure all datasets are complete if job.backend.requirements.enableMerge: # get container list cont_list = self.getContainerList() for mj in job.backend.mergejobs: if mj.status != "finished": # merge jobs failed - reset the unit for the moment logger.error("Merge jobs failed. Resetting unit...") self._getParent().resetUnit(self.getID()) return False for cont in cont_list: dq2_list = dq2.listFilesInDataset(cont) for guid in dq2_list[0].keys(): if dq2_list[0][guid]['lfn'].find("merge") == -1: logger.warning("Merged files not transferred to out DS by Panda yet. Waiting...") return False return True else: return False
def checkCompleted(self, job): """Check if this unit is complete""" if job.status == "completed": if job.outputdata and job.outputdata._impl._name == "DQ2OutputDataset" and job.backend.__class__.__name__ != "Jedi": # make sure all datasets are complete if job.backend.requirements.enableMerge: # get container list cont_list = self.getContainerList() for mj in job.backend.mergejobs: if mj.status != "finished": # merge jobs failed - reset the unit for the moment logger.error("Merge jobs failed. Resetting unit...") self._getParent().resetUnit(self.getID()) return False for cont in cont_list: dq2_list = dq2.listFilesInDataset(cont) for guid in dq2_list[0].keys(): if dq2_list[0][guid]['lfn'].find("merge") == -1: logger.warning("Merged files not transferred to out DS by Panda yet. Waiting...") return False return True else: return False
def copyOutput(self): """Copy the output data to local storage""" job = GPI.jobs(self.active_job_ids[0]) if self.copy_output._name != "TaskLocalCopy" or job.outputdata._impl._name != "DQ2OutputDataset": logger.error("Cannot transfer from DS type '%s' to '%s'. Please contact plugin developer." % (job.outputdata._name, self.copy_output._name)) return False # get list of output files self._acquireDownloadLock() dq2_list = [] if len(self.output_file_list) == 0: for ds in self.getOutputDatasetList(): dq2_list = dq2.listFilesInDataset(ds) # merge job DSs leave empty non-merged DSs around if job.backend.__class__.__name__ == "Panda" and job.backend.requirements.enableMerge and not ds.endswith("merge") and len(dq2_list) == 0: continue for guid in dq2_list[0].keys(): self.output_file_list[ dq2_list[0][guid]['lfn'] ] = ds # check which ones still need downloading to_download = {} for f in self.output_file_list.keys(): # check for REs if self.copy_output.isValid(f) and not self.copy_output.isDownloaded(f): to_download[ f ] = self.output_file_list[f] # store download location in case it's changed while downloading download_loc = self.copy_output.local_location self._releaseDownloadLock() # is everything downloaded? if len(to_download.keys()) == 0: return True # nope, so pick the requested number and off we go thread_array = [] for fname in to_download.keys()[:self._getParent().num_dq2_threads]: dsname = to_download[fname] exe = 'dq2-get -L ROAMING -a -d -H %s -f %s %s' % (download_loc, fname, dsname) logger.info("Downloading '%s' to %s..." % (fname, download_loc)) thread = Download.download_dq2(exe) thread.start() thread_array.append(thread) for t in thread_array: t.join() self._acquireDownloadLock() # check for valid download - SHOULD REALLY BE A HASH CHECK for fname in to_download.keys()[:self._getParent().num_dq2_threads]: full_path = os.path.join(self.copy_output.local_location, fname) if not os.path.exists(full_path): logger.error("Error downloading '%s'. File doesn't exist after download." % full_path) elif os.path.getsize( full_path ) < 4: logger.error("Error downloading '%s'. File size smaller than 4 bytes (%d)" % (full_path, os.path.getsize( full_path ) )) else: self.copy_output.files.append(fname) logger.info("File '%s' downloaded successfully" % full_path) self._releaseDownloadLock() return False
def copyOutput(self): """Copy the output data to local storage""" job = GPI.jobs(self.active_job_ids[0]) if self.copy_output._name != "TaskLocalCopy" or job.outputdata._impl._name != "DQ2OutputDataset": logger.error("Cannot transfer from DS type '%s' to '%s'. Please contact plugin developer." % (job.outputdata._name, self.copy_output._name)) return False # get list of output files self._acquireDownloadLock() dq2_list = [] if len(self.output_file_list) == 0: for ds in self.getOutputDatasetList(): dq2_list = dq2.listFilesInDataset(ds) # merge job DSs leave empty non-merged DSs around if job.backend.__class__.__name__ == "Panda" and job.backend.requirements.enableMerge and not ds.endswith("merge") and len(dq2_list) == 0: continue for guid in dq2_list[0].keys(): self.output_file_list[ dq2_list[0][guid]['lfn'] ] = ds # check which ones still need downloading to_download = {} for f in self.output_file_list.keys(): # check for REs if self.copy_output.isValid(f) and not self.copy_output.isDownloaded(f): to_download[ f ] = self.output_file_list[f] # store download location in case it's changed while downloading download_loc = self.copy_output.local_location self._releaseDownloadLock() # is everything downloaded? if len(to_download.keys()) == 0: return True # nope, so pick the requested number and off we go thread_array = [] for fname in to_download.keys()[:self._getParent().num_dq2_threads]: dsname = to_download[fname] exe = 'dq2-get -L ROAMING -a -d -H %s -f %s %s' % (download_loc, fname, dsname) logger.info("Downloading '%s' to %s..." % (fname, download_loc)) thread = Download.download_dq2(exe) thread.start() thread_array.append(thread) for t in thread_array: t.join() self._acquireDownloadLock() # check for valid download - SHOULD REALLY BE A HASH CHECK for fname in to_download.keys()[:self._getParent().num_dq2_threads]: full_path = os.path.join(self.copy_output.local_location, fname) if not os.path.exists(full_path): logger.error("Error downloading '%s'. File doesn't exist after download." % full_path) elif os.path.getsize( full_path ) < 4: logger.error("Error downloading '%s'. File size smaller than 4 bytes (%d)" % (full_path, os.path.getsize( full_path ) )) else: self.copy_output.files.append(fname) logger.info("File '%s' downloaded successfully" % full_path) self._releaseDownloadLock() return False