class TestGangaList(GangaGPITestCase):
def __init__(self):
self.ganga_list = None
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
def _makeRandomString(self):
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
def _makeRandomTFile(self):
name = self._makeRandomString()
subdir = self._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(
self.plain1), 'Somthings wrong with construction 1'
assert len(getProxyAttr(self.proxied2, '_list')) == len(
self.plain2), 'Somthings wrong with construction 2'
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
assert not missing_methods, \
'Not all of lists methods are implemented: %s' % str(
missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'
def testNq(self):
assert self.proxied1 != None
assert not self.proxied1 != self.proxied1, 'Lists are the same'
assert self.proxied1 != self.proxied2, 'Lists are different'
assert self.plain1 != self.proxied2, 'Lists are different'
def testNonZero(self):
"""@ExpectedFailure"""
assert not GangaList(
), 'An empty GangaList should be false, just like a list'
def testAdd(self):
"""Test __add__"""
assert len(self.plain1 + self.plain2) == len(self.proxied1 +
self.proxied2)
assert isProxy(self.proxied1 + self.proxied2)
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
assert len(self.plain1 + self.plain2) == len(self.proxied1 +
self.plain2)
assert len(self.plain2 + self.plain1) == len(self.plain2 +
self.proxied1)
assert isProxy(self.proxied1 + self.plain2)
assert len(self.plain2 + self.plain1) == len(self.plain2 +
self.proxied1)
assert len(self.plain1 + self.plain2) == len(self.plain1 +
self.proxied2)
assert isinstance(self.plain1 + self.proxied2, list)
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
assert not isProxy(self.plain1[0]), 'Element access must not proxies'
assert isProxy(
(self.plain1 +
self.proxied2)[-1]), 'File objects should remain proxies'
assert not isProxy(
(self.plain1 +
self.proxied2)[0]), 'Objects in plain lists should be left alone'
assert (self.plain1 + self.proxied2
)[-1] == self.proxied2[-1], 'File objects should be equal'
assert (self.plain1 + self.proxied2
)[-1] is self.proxied2[-1], 'File objects should be identical'
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
assert plist == self.proxied1
for p in plist:
assert isProxy(p)
assert p in self.proxied1, 'Proxied list should contain each proxied object'
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
assert isProxy(p)
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
assert (self.plain1 >= self.plain2) == (
self.proxied1 >=
self.proxied2), 'The lists should have the same ge'
assert (self.plain2 >= self.plain1) == (
self.proxied2 >=
self.proxied1), 'The lists should have the same ge'
assert (self.proxied1 >= self.proxied2) != (
self.proxied2 >= self.proxied1), 'The gt should invert correctly'
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
assert isProxy(self.proxied1[i])
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
assert self.plain1[s[0]:s[1]] == self.proxied1[
s[0]:s[1]], 'Slices %s should be the same' % str(s)
t = self.plain1[:]
assert t is not self.plain1, 'Slice should be a copy.'
assert self.plain1[:] is not t
t = self.proxied1[:]
assert t is not self.proxied1, 'Slice should be a copy.'
assert self.proxied1[:] is not t
def testGT(self):
"""Test __gt__"""
assert (self.plain1 > self.plain2) == (
self.proxied1 > self.proxied2), 'The lists should have the same gt'
assert (self.plain2 > self.plain1) == (
self.proxied2 > self.proxied1), 'The lists should have the same gt'
assert (self.proxied1 > self.proxied2) != (
self.proxied2 > self.proxied1), 'The gt should invert correctly'
def testIAdd(self):
"""Test __iadd__"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.proxied2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.plain2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
assert isinstance(self.plain1, list), 'Must be list instance'
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
assert self.plain2 == self.proxied2, 'Addition should be the same'
assert isProxy(self.proxied2), 'Proxy must be added'
assert isinstance(self.plain2, list), 'Must be list instance'
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
assert s1 == s2, 'Addition should be the same'
assert not isProxy(s1), 'Proxy Not Needed'
assert isProxy(s2), 'Proxy Needed'
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
assert self.plain1[4] is t
t = self.proxied1[4]
assert self.proxied1[4] is t
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
assert self.plain1 == self.proxied1, 'Multiplication should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
assert s1 == s2, 'Addition should be the same'
def testLE(self):
"""Test __le__"""
assert (self.plain1 <= self.plain2) == (
self.proxied1 <=
self.proxied2), 'The lists should have the same le'
assert (self.plain2 <= self.plain1) == (
self.proxied2 <=
self.proxied1), 'The lists should have the same le'
assert (self.proxied1 <= self.proxied2) != (
self.proxied2 <= self.proxied1), 'The le should invert correctly'
def testLen(self):
"""Tests __len__"""
assert len(self.plain1) == len(
self.proxied1), 'Lengths should be the same'
def testLT(self):
"""Test __lt__"""
assert (self.plain1 < self.plain2) == (
self.proxied1 < self.proxied2), 'The lists should have the same lt'
assert (self.plain2 < self.plain1) == (
self.proxied2 < self.proxied1), 'The lists should have the same lt'
assert (self.proxied1 < self.proxied2) != (
self.proxied2 < self.proxied1), 'The lt should invert correctly'
def testMul(self):
"""Test __mul__"""
assert len(self.plain1 * 7) == len(self.proxied1 * 7)
assert isProxy(self.proxied1 * 9)
for p in self.proxied1:
assert isProxy(p)
def testNE(self):
"""Test __ne__"""
assert self.plain1 != self.plain2
assert self.proxied1 != self.proxied2, 'Lists should be different'
assert self.plain1[0:5] != self.plain1[2:7]
assert self.proxied1[0:5] != self.proxied1[
2:7], 'Lists should be different'
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
assert len(t1) == len(t2), 'Multiplication should be the same'
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
from TFile import TFile as tF
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
assert i is self.proxied1[count]
assert isProxy(i)
assert isType(i, tF)
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
assert self.plain1[test_index] is t
self.proxied1[test_index] = t
assert self.proxied1[test_index] is t
assert isProxy(self.proxied1[test_index])
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
assert self.plain1[3:7] == self.plain2[
3:7], 'The lists should be equal'
assert self.plain1[3:7] is not self.plain2[
3:7], 'The lists should be copies'
self.plain1[4] = self.plain2[9]
assert self.plain1[4] != self.plain2[4]
tmp = self.plain1[2:9]
assert self.plain1[2:9] is not tmp
self.proxied1[3:7] = self.proxied2[3:7]
assert self.proxied1[3:7] == self.proxied2[
3:7], 'The lists should be equal'
assert self.proxied1[3:7] is not self.proxied2[
3:7], 'The lists should be copies'
self.proxied1[4] = self.proxied2[9]
assert self.proxied1[4] != self.proxied2[4]
tmp = self.proxied1[2:9]
assert self.proxied1[2:9] is not tmp
def testAppend(self):
t = addProxy(TFile(name='foo'))
self.plain1.append(t)
assert self.plain1[-1] == t
assert self.plain1.pop() == t
self.proxied1.append(t)
assert self.proxied1[-1] == t
assert self.proxied1[-1] is t, 'Identity Test'
assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
assert self.proxied1.pop() == t
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
assert self.plain1 == self.proxied1, 'Lists should be the same'
t2 = self.proxied1[4:7]
assert isProxy(t2)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
assert isType(t2, glist)
self.plain1.extend(t2)
self.proxied1.extend(t2)
assert self.plain1 == self.proxied1, 'Lists should be the same'
def testIndex(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
assert self.proxied1.index(t) == 8
def testInsert(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
t = self.proxied1[6]
r = self.proxied1.pop(6)
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 2
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testRemove(self):
t = addProxy(TFile(name='bar'))
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testIter(self):
from TFile import TFile as tF
count = 0
for f in self.proxied1:
count += 1
assert isProxy(f)
assert isType(f, tF)
assert count == len(self.proxied1), 'Must visit every member'
def testCmp(self):
assert cmp(self.proxied1,
self.proxied2) == cmp(self.plain1, self.plain2)
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
class LHCbDataset(GangaDataset):
'''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).
Example Usage:
ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
len(ds) # 2 (number of files)
ds.getReplicas() # returns replicas for *all* files in the data set
ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
ds.getCatalog() # returns XML catalog slice
ds.optionsString() # returns Gaudi-sytle options
[...etc...]
'''
schema = {}
docstr = 'List of PhysicalFile and DiracFile objects'
schema['files'] = GangaFileItem(defvalue=[], typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'], sequence=1, doc=docstr)
docstr = 'Ancestor depth to be queried from the Bookkeeping'
schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
docstr = 'Use contents of file rather than generating catalog.'
schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
docstr = 'Specify the dataset persistency technology'
schema['persistency'] = SimpleItem(
defvalue=None, typelist=['str', 'type(None)'], doc=docstr)
schema['treat_as_inputfiles'] = SimpleItem(defvalue=False, doc="Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")
_schema = Schema(Version(3, 0), schema)
_category = 'datasets'
_name = "LHCbDataset"
_exportmethods = ['getReplicas', '__len__', '__getitem__', 'replicate',
'hasLFNs', 'append', 'extend', 'getCatalog', 'optionsString',
'getLFNs', 'getFileNames', 'getFullFileNames',
'difference', 'isSubset', 'isSuperset', 'intersection',
'symmetricDifference', 'union', 'bkMetadata',
'isEmpty', 'hasPFNs', 'getPFNs'] # ,'pop']
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(this_file))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
raise GangaException("Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
#this_file = self.files[i]
# print type(this_file)
# return this_file
# return this_file
# return this_file
if type(i) == type(slice(0)):
ds = LHCbDataset(files=self.files[i])
ds.depth = self.depth
#ds.XMLCatalogueSlice = self.XMLCatalogueSlice
return ds
else:
return self.files[i]
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error. Could not get replicas.')
return result['Successful']
def hasLFNs(self):
'Returns True is the dataset has LFNs and False otherwise.'
for f in self.files:
if isDiracFile(f):
return True
return False
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError('object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def removeFile(self, input_file):
try:
self.files.remove(input_file)
except:
raise GangaException('Dataset has no file named %s' % input_file.namePattern)
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def getPFNs(self):
'Returns a list of all PFNs (by name) stored in the dataset.'
pfns = []
if not self:
return pfns
for f in self.files:
if isPFN(f):
pfns.append(f.namePattern)
return pfns
def getFullFileNames(self):
'Returns all file names w/ PFN or LFN prepended.'
names = []
from GangaDirac.Lib.Files.DiracFile import DiracFile
for f in self.files:
if isType(f, DiracFile):
names.append('LFN:%s' % f.lfn)
else:
try:
names.append('PFN:%s' % f.namePattern)
except:
logger.warning("Cannot determine filename for: %s " % f)
raise GangaException("Cannot Get File Name")
return names
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None")
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str)
else:
sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern
sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if(file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def _checkOtherFiles(self, other ):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def difference(self, other):
'''Returns a new data set w/ files in this that are not in other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def isSubset(self, other):
'''Is every file in this data set in other?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issubset(other_files)
def isSuperset(self, other):
'''Is every file in other in this data set?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issuperset(other_files)
def symmetricDifference(self, other):
'''Returns a new data set w/ files in either this or other but not
both.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).symmetric_difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def intersection(self, other):
'''Returns a new data set w/ files common to this and other.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).intersection(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def union(self, other):
'''Returns a new data set w/ files from this and other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).union(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info("Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info...")
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica. Replica rm error.')
return b
class TestGangaList(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestGangaList, self).__init__(*args, **kwargs)
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
self.ganga_list = GangaList()
@staticmethod
def _makeRandomString():
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
@staticmethod
def _makeRandomTFile():
name = TestGangaList._makeRandomString()
subdir = TestGangaList._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')),
len(self.plain1),
"Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')),
len(self.plain2),
"Something's wrong with construction")
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
self.assertFalse(
missing_methods,
'Not all of lists methods are implemented: %s' % missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
self.assertEqual(
self.proxied1, self.plain1,
'Proxied and non-proxied versions should be the same')
def testNq(self):
self.assertIsNotNone(self.proxied1, None)
self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
self.assertNotEqual(self.proxied1, self.proxied2,
'Lists are different')
self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')
def testNonZero(self):
"""@ExpectedFailure"""
self.assertFalse(
GangaList(),
'An empty GangaList should be false, just like a list')
def testAdd(self):
"""Test __add__"""
self.assertEqual(self.plain1 + self.plain2,
self.proxied1 + self.proxied2)
self.assertTrue(isProxy(self.proxied1 + self.proxied2))
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
self.assertEqual((self.plain1 + self.plain2),
(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1),
(self.plain2 + self.proxied1))
self.assertTrue(isProxy(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1),
(self.plain2 + self.proxied1))
self.assertEqual((self.plain1 + self.plain2),
(self.plain1 + self.proxied2))
self.assertTrue(isinstance(self.plain1 + self.proxied2, list))
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
self.assertTrue(isProxy(self.proxied2[-1]),
'Element access must get proxies')
self.assertFalse(isProxy(self.plain1[0]),
'Element access must not proxies')
self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]),
'File objects should remain proxies')
self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]),
'Objects in plain lists should be left alone')
self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
'File objects should be equal')
self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
'File objects should be identical')
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
self.assertEqual(plist, self.proxied1)
for p in plist:
self.assertTrue(isProxy(p))
self.assertIn(p, self.proxied1,
'Proxied list should contain each proxied object')
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
self.assertTrue(isProxy(p))
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
self.assertEqual((self.plain1 >= self.plain2),
(self.proxied1 >= self.proxied2),
'The lists should have the same ge')
self.assertEqual((self.plain2 >= self.plain1),
(self.proxied2 >= self.proxied1),
'The lists should have the same ge')
self.assertNotEqual((self.proxied1 >= self.proxied2),
(self.proxied2 >= self.proxied1),
'The gt should invert correctly')
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
self.assertTrue(isProxy(self.proxied1[i]))
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]],
'Slices {0} should be the same'.format(s))
t = self.plain1[:]
self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
self.assertIsNot(self.plain1[:], t)
t = self.proxied1[:]
self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
self.assertIsNot(self.proxied1[:], t)
def testGT(self):
"""Test __gt__"""
self.assertEqual((self.plain1 > self.plain2),
(self.proxied1 > self.proxied2),
'The lists should have the same gt')
self.assertEqual((self.plain2 > self.plain1),
(self.proxied2 > self.proxied1),
'The lists should have the same gt')
self.assertNotEqual((self.proxied1 > self.proxied2),
(self.proxied2 > self.proxied1),
'The gt should invert correctly')
def testIAdd(self):
"""Test __iadd__"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.proxied2
self.assertEqual(self.plain1, self.proxied1,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.plain2
self.assertEqual(self.plain1, self.proxied1,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
self.assertEqual(self.plain2, self.proxied2,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
self.assertEqual(s1, s2, 'Addition should be the same')
self.assertFalse(isProxy(s1), 'Proxy Not Needed')
self.assertTrue(isProxy(s2), 'Proxy Needed')
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
self.assertIs(self.plain1[4], t)
t = self.proxied1[4]
self.assertIs(self.proxied1[4], t)
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
self.assertEqual(self.plain1, self.proxied1,
'Multiplication should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
self.assertEqual(s1, s2, 'Addition should be the same')
def testLE(self):
"""Test __le__"""
self.assertEqual(self.plain1 <= self.plain2,
self.proxied1 <= self.proxied2,
'The lists should have the same le')
self.assertEqual(self.plain2 <= self.plain1,
self.proxied2 <= self.proxied1,
'The lists should have the same le')
self.assertNotEqual(self.proxied1 <= self.proxied2,
self.proxied2 <= self.proxied1,
'The le should invert correctly')
def testLen(self):
"""Tests __len__"""
self.assertEqual(len(self.plain1), len(self.proxied1),
'Lengths should be the same')
def testLT(self):
"""Test __lt__"""
self.assertEqual(self.plain1 < self.plain2,
self.proxied1 < self.proxied2,
'The lists should have the same lt')
self.assertEqual(self.plain2 < self.plain1,
self.proxied2 < self.proxied1,
'The lists should have the same lt')
self.assertNotEqual(self.proxied1 < self.proxied2,
self.proxied2 < self.proxied1,
'The lt should invert correctly')
def testMul(self):
"""Test __mul__"""
self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
self.assertTrue(isProxy(self.proxied1 * 9))
for p in self.proxied1:
self.assertTrue(isProxy(p))
def testNE(self):
"""Test __ne__"""
self.assertNotEqual(self.plain1, self.plain2)
self.assertNotEqual(self.proxied1, self.proxied2,
'Lists should be different')
self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7],
'Lists should be different')
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
self.assertEqual(t1, t2, 'Multiplication should be the same')
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
self.assertIs(i, self.proxied1[count])
self.assertTrue(isProxy(i))
self.assertTrue(isType(i, TFile))
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
self.assertIs(self.plain1[test_index], t)
self.proxied1[test_index] = t
self.assertIs(self.proxied1[test_index], t)
self.assertTrue(isProxy(self.proxied1[test_index]))
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
self.assertEqual(self.plain1[3:7], self.plain2[3:7],
'The lists should be equal')
self.assertIsNot(self.plain1[3:7], self.plain2[3:7],
'The lists should be copies')
self.plain1[4] = self.plain2[9]
self.assertNotEqual(self.plain1[4], self.plain2[4])
tmp = self.plain1[2:9]
self.assertIsNot(self.plain1[2:9], tmp)
self.proxied1[3:7] = self.proxied2[3:7]
self.assertEqual(self.proxied1[3:7], self.proxied2[3:7],
'The lists should be equal')
self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7],
'The lists should be copies')
self.proxied1[4] = self.proxied2[9]
self.assertNotEqual(self.proxied1[4], self.proxied2[4])
tmp = self.proxied1[2:9]
self.assertIsNot(self.proxied1[2:9], tmp)
def testAppend(self):
t = TFile(name='foo')
self.plain1.append(t)
self.assertEqual(self.plain1[-1], t)
self.assertEqual(self.plain1.pop(), t)
self.proxied1.append(t)
self.assertEqual(self.proxied1[-1], t)
self.assertIs(self.proxied1[-1], t, 'Identity Test')
self.assertTrue(isProxy(self.proxied1[-1]),
'Make sure we get back a proxy')
self.assertEqual(self.proxied1.pop(), t)
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
self.assertEqual(self.plain1, self.proxied1,
'Lists should be the same')
t2 = self.proxied1[4:7]
self.assertTrue(isProxy(t2))
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
self.assertTrue(isType(t2, glist))
self.plain1.extend(t2)
self.proxied1.extend(t2)
self.assertEqual(self.plain1, self.proxied1,
'Lists should be the same')
def testIndex(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
self.assertEqual(self.proxied1.index(t), 8)
def testInsert(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
t = self.proxied1[6]
r = self.proxied1.pop(6)
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 2)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testRemove(self):
t = TFile(name='bar')
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testIter(self):
count = 0
for f in self.proxied1:
count += 1
self.assertTrue(isProxy(f))
self.assertTrue(isType(f, TFile))
self.assertEqual(count, len(self.proxied1), 'Must visit every member')
def testCmp(self):
self.assertEqual(cmp(self.proxied1, self.proxied2),
cmp(self.plain1, self.plain2))
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
class TestGangaList(GangaGPITestCase):
def __init__(self):
self.ganga_list = None
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
def _makeRandomString(self):
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
def _makeRandomTFile(self):
name = self._makeRandomString()
subdir = self._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(
self.plain1), 'Somthings wrong with construction'
assert len(getProxyAttr(self.proxied2, '_list')) == len(
self.plain2), 'Somthings wrong with construction'
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
assert not missing_methods, \
'Not all of lists methods are implemented: %s' % str(
missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'
def testNq(self):
assert self.proxied1 != None
assert not self.proxied1 != self.proxied1, 'Lists are the same'
assert self.proxied1 != self.proxied2, 'Lists are different'
assert self.plain1 != self.proxied2, 'Lists are different'
def testNonZero(self):
"""@ExpectedFailure"""
assert not GangaList(
), 'An empty GangaList should be false, just like a list'
def testAdd(self):
"""Test __add__"""
assert (self.plain1 + self.plain2) == (self.proxied1 + self.proxied2)
assert isProxy(self.proxied1 + self.proxied2)
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
assert (self.plain1 + self.plain2) == (self.proxied1 + self.plain2)
assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
assert isProxy(self.proxied1 + self.plain2)
assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
assert (self.plain1 + self.plain2) == (self.plain1 + self.proxied2)
assert isinstance(self.plain1 + self.proxied2, list)
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
assert not isProxy(self.plain1[0]), 'Element access must not proxies'
assert isProxy(
(self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies'
assert not isProxy(
(self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone'
assert (
self.plain1 + self.proxied2)[-1] == self.proxied2[-1], 'File objects should be equal'
assert (
self.plain1 + self.proxied2)[-1] is self.proxied2[-1], 'File objects should be identical'
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
assert plist == self.proxied1
for p in plist:
assert isProxy(p)
assert p in self.proxied1, 'Proxied list should contain each proxied object'
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
assert isProxy(p)
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
assert (self.plain1 >= self.plain2) == (
self.proxied1 >= self.proxied2), 'The lists should have the same ge'
assert (self.plain2 >= self.plain1) == (
self.proxied2 >= self.proxied1), 'The lists should have the same ge'
assert (self.proxied1 >= self.proxied2) != (
self.proxied2 >= self.proxied1), 'The gt should invert correctly'
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
assert isProxy(self.proxied1[i])
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
assert self.plain1[s[0]:s[1]] == self.proxied1[
s[0]:s[1]], 'Slices %s should be the same' % str(s)
t = self.plain1[:]
assert t is not self.plain1, 'Slice should be a copy.'
assert self.plain1[:] is not t
t = self.proxied1[:]
assert t is not self.proxied1, 'Slice should be a copy.'
assert self.proxied1[:] is not t
def testGT(self):
"""Test __gt__"""
assert (self.plain1 > self.plain2) == (
self.proxied1 > self.proxied2), 'The lists should have the same gt'
assert (self.plain2 > self.plain1) == (
self.proxied2 > self.proxied1), 'The lists should have the same gt'
assert (self.proxied1 > self.proxied2) != (
self.proxied2 > self.proxied1), 'The gt should invert correctly'
def testIAdd(self):
"""Test __iadd__"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.proxied2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.plain2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
assert isinstance(self.plain1, list), 'Must be list instance'
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
assert self.plain2 == self.proxied2, 'Addition should be the same'
assert isProxy(self.proxied2), 'Proxy must be added'
assert isinstance(self.plain2, list), 'Must be list instance'
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
assert s1 == s2, 'Addition should be the same'
assert not isProxy(s1), 'Proxy Not Needed'
assert isProxy(s2), 'Proxy Needed'
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
assert self.plain1[4] is t
t = self.proxied1[4]
assert self.proxied1[4] is t
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
assert self.plain1 == self.proxied1, 'Multiplication should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
assert s1 == s2, 'Addition should be the same'
def testLE(self):
"""Test __le__"""
assert (self.plain1 <= self.plain2) == (
self.proxied1 <= self.proxied2), 'The lists should have the same le'
assert (self.plain2 <= self.plain1) == (
self.proxied2 <= self.proxied1), 'The lists should have the same le'
assert (self.proxied1 <= self.proxied2) != (
self.proxied2 <= self.proxied1), 'The le should invert correctly'
def testLen(self):
"""Tests __len__"""
assert len(self.plain1) == len(
self.proxied1), 'Lengths should be the same'
def testLT(self):
"""Test __lt__"""
assert (self.plain1 < self.plain2) == (
self.proxied1 < self.proxied2), 'The lists should have the same lt'
assert (self.plain2 < self.plain1) == (
self.proxied2 < self.proxied1), 'The lists should have the same lt'
assert (self.proxied1 < self.proxied2) != (
self.proxied2 < self.proxied1), 'The lt should invert correctly'
def testMul(self):
"""Test __mul__"""
assert (self.plain1 * 7) == (self.proxied1 * 7)
assert isProxy(self.proxied1 * 9)
for p in self.proxied1:
assert isProxy(p)
def testNE(self):
"""Test __ne__"""
assert self.plain1 != self.plain2
assert self.proxied1 != self.proxied2, 'Lists should be different'
assert self.plain1[0:5] != self.plain1[2:7]
assert self.proxied1[0:5] != self.proxied1[
2:7], 'Lists should be different'
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
assert t1 == t2, 'Multiplication should be the same'
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
from TFile import TFile as tF
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
assert i is self.proxied1[count]
assert isProxy(i)
assert isType(i, tF)
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
assert self.plain1[test_index] is t
self.proxied1[test_index] = t
assert self.proxied1[test_index] is t
assert isProxy(self.proxied1[test_index])
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
assert self.plain1[3:7] == self.plain2[
3:7], 'The lists should be equal'
assert self.plain1[3:7] is not self.plain2[
3:7], 'The lists should be copies'
self.plain1[4] = self.plain2[9]
assert self.plain1[4] != self.plain2[4]
tmp = self.plain1[2:9]
assert self.plain1[2:9] is not tmp
self.proxied1[3:7] = self.proxied2[3:7]
assert self.proxied1[3:7] == self.proxied2[
3:7], 'The lists should be equal'
assert self.proxied1[3:7] is not self.proxied2[
3:7], 'The lists should be copies'
self.proxied1[4] = self.proxied2[9]
assert self.proxied1[4] != self.proxied2[4]
tmp = self.proxied1[2:9]
assert self.proxied1[2:9] is not tmp
def testAppend(self):
t = addProxy(TFile(name='foo'))
self.plain1.append(t)
assert self.plain1[-1] == t
assert self.plain1.pop() == t
self.proxied1.append(t)
assert self.proxied1[-1] == t
assert self.proxied1[-1] is t, 'Identity Test'
assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
assert self.proxied1.pop() == t
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
assert self.plain1 == self.proxied1, 'Lists should be the same'
t2 = self.proxied1[4:7]
assert isProxy(t2)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
assert isType(t2, glist)
self.plain1.extend(t2)
self.proxied1.extend(t2)
assert self.plain1 == self.proxied1, 'Lists should be the same'
def testIndex(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
assert self.proxied1.index(t) == 8
def testInsert(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
t = self.proxied1[6]
r = self.proxied1.pop(6)
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 2
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testRemove(self):
t = addProxy(TFile(name='bar'))
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testIter(self):
from TFile import TFile as tF
count = 0
for f in self.proxied1:
count += 1
assert isProxy(f)
assert isType(f, tF)
assert count == len(self.proxied1), 'Must visit every member'
def testCmp(self):
assert cmp(self.proxied1, self.proxied2) == cmp(
self.plain1, self.plain2)
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
class TestGangaList(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestGangaList, self).__init__(*args, **kwargs)
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
self.ganga_list = GangaList()
@staticmethod
def _makeRandomString():
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
@staticmethod
def _makeRandomTFile():
name = TestGangaList._makeRandomString()
subdir = TestGangaList._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
self.assertFalse(missing_methods, 'Not all of lists methods are implemented: %s' % missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
self.assertEqual(self.proxied1, self.plain1, 'Proxied and non-proxied versions should be the same')
def testNq(self):
self.assertIsNotNone(self.proxied1, None)
self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
self.assertNotEqual(self.proxied1, self.proxied2, 'Lists are different')
self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')
def testNonZero(self):
"""@ExpectedFailure"""
self.assertFalse(GangaList(), 'An empty GangaList should be false, just like a list')
def testAdd(self):
"""Test __add__"""
self.assertEqual(self.plain1 + self.plain2, self.proxied1 + self.proxied2)
self.assertTrue(isProxy(self.proxied1 + self.proxied2))
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
self.assertEqual((self.plain1 + self.plain2), (self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
self.assertTrue(isProxy(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
self.assertEqual((self.plain1 + self.plain2), (self.plain1 + self.proxied2))
self.assertTrue(isinstance(self.plain1 + self.proxied2, list))
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
self.assertTrue(isProxy(self.proxied2[-1]), 'Element access must get proxies')
self.assertFalse(isProxy(self.plain1[0]), 'Element access must not proxies')
self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies')
self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone')
self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be equal')
self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be identical')
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
self.assertEqual(plist, self.proxied1)
for p in plist:
self.assertTrue(isProxy(p))
self.assertIn(p, self.proxied1, 'Proxied list should contain each proxied object')
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
self.assertTrue(isProxy(p))
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
self.assertEqual((self.plain1 >= self.plain2), (self.proxied1 >= self.proxied2), 'The lists should have the same ge')
self.assertEqual((self.plain2 >= self.plain1), (self.proxied2 >= self.proxied1), 'The lists should have the same ge')
self.assertNotEqual((self.proxied1 >= self.proxied2), (self.proxied2 >= self.proxied1), 'The gt should invert correctly')
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
self.assertTrue(isProxy(self.proxied1[i]))
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]], 'Slices {0} should be the same'.format(s))
t = self.plain1[:]
self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
self.assertIsNot(self.plain1[:], t)
t = self.proxied1[:]
self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
self.assertIsNot(self.proxied1[:], t)
def testGT(self):
"""Test __gt__"""
self.assertEqual((self.plain1 > self.plain2), (self.proxied1 > self.proxied2), 'The lists should have the same gt')
self.assertEqual((self.plain2 > self.plain1), (self.proxied2 > self.proxied1), 'The lists should have the same gt')
self.assertNotEqual((self.proxied1 > self.proxied2), (self.proxied2 > self.proxied1), 'The gt should invert correctly')
def testIAdd(self):
"""Test __iadd__"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.proxied2
self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.plain2
self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
self.assertEqual(self.plain2, self.proxied2, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
self.assertEqual(s1, s2, 'Addition should be the same')
self.assertFalse(isProxy(s1), 'Proxy Not Needed')
self.assertTrue(isProxy(s2), 'Proxy Needed')
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
self.assertIs(self.plain1[4], t)
t = self.proxied1[4]
self.assertIs(self.proxied1[4], t)
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
self.assertEqual(self.plain1, self.proxied1, 'Multiplication should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
self.assertEqual(s1, s2, 'Addition should be the same')
def testLE(self):
"""Test __le__"""
self.assertEqual(self.plain1 <= self.plain2, self.proxied1 <= self.proxied2, 'The lists should have the same le')
self.assertEqual(self.plain2 <= self.plain1, self.proxied2 <= self.proxied1, 'The lists should have the same le')
self.assertNotEqual(self.proxied1 <= self.proxied2, self.proxied2 <= self.proxied1, 'The le should invert correctly')
def testLen(self):
"""Tests __len__"""
self.assertEqual(len(self.plain1), len(self.proxied1), 'Lengths should be the same')
def testLT(self):
"""Test __lt__"""
self.assertEqual(self.plain1 < self.plain2, self.proxied1 < self.proxied2, 'The lists should have the same lt')
self.assertEqual(self.plain2 < self.plain1, self.proxied2 < self.proxied1, 'The lists should have the same lt')
self.assertNotEqual(self.proxied1 < self.proxied2, self.proxied2 < self.proxied1, 'The lt should invert correctly')
def testMul(self):
"""Test __mul__"""
self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
self.assertTrue(isProxy(self.proxied1 * 9))
for p in self.proxied1:
self.assertTrue(isProxy(p))
def testNE(self):
"""Test __ne__"""
self.assertNotEqual(self.plain1, self.plain2)
self.assertNotEqual(self.proxied1, self.proxied2, 'Lists should be different')
self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7], 'Lists should be different')
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
self.assertEqual(t1, t2, 'Multiplication should be the same')
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
self.assertIs(i, self.proxied1[count])
self.assertTrue(isProxy(i))
self.assertTrue(isType(i, TFile))
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
self.assertIs(self.plain1[test_index], t)
self.proxied1[test_index] = t
self.assertIs(self.proxied1[test_index], t)
self.assertTrue(isProxy(self.proxied1[test_index]))
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
self.assertEqual(self.plain1[3:7], self.plain2[3:7], 'The lists should be equal')
self.assertIsNot(self.plain1[3:7], self.plain2[3:7], 'The lists should be copies')
self.plain1[4] = self.plain2[9]
self.assertNotEqual(self.plain1[4], self.plain2[4])
tmp = self.plain1[2:9]
self.assertIsNot(self.plain1[2:9], tmp)
self.proxied1[3:7] = self.proxied2[3:7]
self.assertEqual(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be equal')
self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be copies')
self.proxied1[4] = self.proxied2[9]
self.assertNotEqual(self.proxied1[4], self.proxied2[4])
tmp = self.proxied1[2:9]
self.assertIsNot(self.proxied1[2:9], tmp)
def testAppend(self):
t = TFile(name='foo')
self.plain1.append(t)
self.assertEqual(self.plain1[-1], t)
self.assertEqual(self.plain1.pop(), t)
self.proxied1.append(t)
self.assertEqual(self.proxied1[-1], t)
self.assertIs(self.proxied1[-1], t, 'Identity Test')
self.assertTrue(isProxy(self.proxied1[-1]), 'Make sure we get back a proxy')
self.assertEqual(self.proxied1.pop(), t)
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')
t2 = self.proxied1[4:7]
self.assertTrue(isProxy(t2))
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
self.assertTrue(isType(t2, glist))
self.plain1.extend(t2)
self.proxied1.extend(t2)
self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')
def testIndex(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
self.assertEqual(self.proxied1.index(t), 8)
def testInsert(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
t = self.proxied1[6]
r = self.proxied1.pop(6)
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 2)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testRemove(self):
t = TFile(name='bar')
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testIter(self):
count = 0
for f in self.proxied1:
count += 1
self.assertTrue(isProxy(f))
self.assertTrue(isType(f, TFile))
self.assertEqual(count, len(self.proxied1), 'Must visit every member')
def testCmp(self):
self.assertEqual(cmp(self.proxied1, self.proxied2), cmp(self.plain1, self.plain2))
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
class LHCbDataset(GangaDataset):
'''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).
Example Usage:
ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
len(ds) # 2 (number of files)
ds.getReplicas() # returns replicas for *all* files in the data set
ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
ds.getCatalog() # returns XML catalog slice
ds.optionsString() # returns Gaudi-sytle options
[...etc...]
'''
schema = {}
docstr = 'List of PhysicalFile and DiracFile objects'
schema['files'] = GangaFileItem(
defvalue=[],
typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'],
sequence=1,
doc=docstr)
docstr = 'Ancestor depth to be queried from the Bookkeeping'
schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
docstr = 'Use contents of file rather than generating catalog.'
schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
docstr = 'Specify the dataset persistency technology'
schema['persistency'] = SimpleItem(defvalue=None,
typelist=['str', 'type(None)'],
doc=docstr)
schema['treat_as_inputfiles'] = SimpleItem(
defvalue=False,
doc=
"Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")
_schema = Schema(Version(3, 0), schema)
_category = 'datasets'
_name = "LHCbDataset"
_exportmethods = [
'getReplicas', '__len__', '__getitem__', 'replicate', 'hasLFNs',
'append', 'extend', 'getCatalog', 'optionsString', 'getLFNs',
'getFileNames', 'getFullFileNames', 'difference', 'isSubset',
'isSuperset', 'intersection', 'symmetricDifference', 'union',
'bkMetadata', 'isEmpty', 'hasPFNs', 'getPFNs'
] # ,'pop']
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(files))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(
this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(files, None),
False)
else:
raise GangaException(
"Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
#this_file = self.files[i]
# print type(this_file)
# return this_file
# return this_file
# return this_file
if type(i) == type(slice(0)):
ds = LHCbDataset(files=self.files[i])
ds.depth = self.depth
#ds.XMLCatalogueSlice = self.XMLCatalogueSlice
return ds
else:
return self.files[i]
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error. Could not get replicas.')
return result['Successful']
def hasLFNs(self):
'Returns True is the dataset has LFNs and False otherwise.'
for f in self.files:
if isDiracFile(f):
return True
return False
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError(
'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now'
% (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(
files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(
string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def removeFile(self, input_file):
try:
self.files.remove(input_file)
except:
raise GangaException('Dataset has no file named %s' %
input_file.namePattern)
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def getPFNs(self):
'Returns a list of all PFNs (by name) stored in the dataset.'
pfns = []
if not self:
return pfns
for f in self.files:
if isPFN(f):
pfns.append(f.namePattern)
return pfns
def getFullFileNames(self):
'Returns all file names w/ PFN or LFN prepended.'
names = []
from GangaDirac.Lib.Files.DiracFile import DiracFile
for f in self.files:
if isType(f, DiracFile):
names.append('LFN:%s' % f.lfn)
else:
try:
names.append('PFN:%s' % f.namePattern)
except:
logger.warning("Cannot determine filename for: %s " % f)
raise GangaException("Cannot Get File Name")
return names
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None"
)
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn,
dtype_str)
else:
sdatasetsnew += """ \"%s\",""" % f.accessURL()[0]
sdatasetsold += """ \"DATAFILE='%s' %s\",""" % (
f.accessURL()[0], dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if (file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def _checkOtherFiles(self, other):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def difference(self, other):
'''Returns a new data set w/ files in this that are not in other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def isSubset(self, other):
'''Is every file in this data set in other?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issubset(other_files)
def isSuperset(self, other):
'''Is every file in other in this data set?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issuperset(other_files)
def symmetricDifference(self, other):
'''Returns a new data set w/ files in either this or other but not
both.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).symmetric_difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def intersection(self, other):
'''Returns a new data set w/ files common to this and other.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).intersection(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def union(self, other):
'''Returns a new data set w/ files from this and other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).union(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info(
"Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info..."
)
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica. Replica rm error.')
return b
