def replicate(self, destSE=''): '''Replicate all LFNs to destSE. For a list of valid SE\'s, type ds.replicate().''' if not destSE: from GangaDirac.Lib.Files.DiracFile import DiracFile DiracFile().replicate('') return if not self.hasLFNs(): raise GangaException('Cannot replicate dataset w/ no LFNs.') retry_files = [] for f in self.files: if not isDiracFile(f): continue try: result = f.replicate( destSE=destSE ) except Exception as err: msg = 'Replication error for file %s (will retry in a bit).' % f.lfn logger.warning(msg) logger.warning("Error: %s" % str(err)) retry_files.append(f) for f in retry_files: try: result = f.replicate( destSE=destSE ) except Exception as err: msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn logger.warning(msg) logger.warning(str(err))
def replicate(self, destSE='', srcSE='', locCache=''): '''Replicate all LFNs to destSE. For a list of valid SE\'s, type ds.replicate().''' if not destSE: from Ganga.GPI import DiracFile DiracFile().replicate('') return if not self.hasLFNs(): raise GangaException('Cannot replicate dataset w/ no LFNs.') retry_files = [] for f in self.files: if not isDiracFile(GPIProxyObjectFactory(f)): continue try: result = f.replicate(destSE, srcSE, locCache) except: msg = 'Replication error for file %s (will retry in a bit).'\ % f.lfn logger.warning(msg) retry_files.append(f) for f in retry_files: try: result = f.replicate(destSE, srcSE, locCache) except: msg = '2nd replication attempt failed for file %s.' \ ' (will not retry)' % f.lfn logger.warning(msg) logger.warning(str(result))
def getLFNs(self): 'Returns a list of all LFNs (by name) stored in the dataset.' lfns = [] if not self: return lfns for f in self.files: if isDiracFile(f): subfiles = f.getSubFiles() if len(subfiles) == 0: lfns.append(f.lfn) else: for file in subfiles: lfns.append(file.lfn) #logger.debug( "Returning LFNS:\n%s" % str(lfns) ) logger.debug("Returning #%s LFNS" % str(len(lfns))) return lfns
def optionsString(self, file=None): 'Returns the Gaudi-style options string for the dataset (if a filename' \ ' is given, the file is created and output is written there).' if not self or len(self) == 0: return '' snew = '' if self.persistency == 'ROOT': snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == 'POOL': snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == None: snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' else: logger.warning( "Unknown LHCbDataset persistency technology... reverting to None") snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=[' sdatasetsnew = '' sdatasetsold = '' dtype_str_default = getConfig('LHCb')['datatype_string_default'] dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns'] for f in self.files: dtype_str = dtype_str_default for this_str in dtype_str_patterns: matched = False for pat in dtype_str_patterns[this_str]: if fnmatch.fnmatch(f.namePattern, pat): dtype_str = this_str matched = True break if matched: break sdatasetsnew += '\n ' sdatasetsold += '\n ' if isDiracFile(f): sdatasetsnew += """ \"LFN:%s\",""" % f.lfn sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str) else: sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str) if sdatasetsold.endswith(","): if self.persistency == 'ROOT': sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)""" else: sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)""" sdatasetsold = sdatasetsold[:-1] sdatasetsold += """\n ]""" if(file): f = open(file, 'w') if self.persistency == 'ROOT': f.write(snew) f.write(sdatasetsnew) else: f.write(snew) f.write(sdatasetsnew) f.write(sold) f.write(sdatasetsold) f.close() else: if self.persistency == 'ROOT': return snew + sdatasetsnew else: return snew + sdatasetsnew + sold + sdatasetsold
def hasPFNs(self): 'Returns True is the dataset has PFNs and False otherwise.' for f in self.files: if not isDiracFile(f): return True return False
def hasPFNs(self): 'Returns True is the dataset has PFNs and False otherwise.' for f in self.files: if not isDiracFile(GPIProxyObjectFactory(f)): return True return False