def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def replicate(self, destSE='', srcSE='', locCache=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from Ganga.GPI import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(GPIProxyObjectFactory(f)):
continue
try:
result = f.replicate(destSE, srcSE, locCache)
except:
msg = 'Replication error for file %s (will retry in a bit).'\
% f.lfn
logger.warning(msg)
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate(destSE, srcSE, locCache)
except:
msg = '2nd replication attempt failed for file %s.' \
' (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(result))
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None")
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str)
else:
sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern
sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if(file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(GPIProxyObjectFactory(f)):
return True
return False
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None")
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str)
else:
sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern
sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if(file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
