def expandWildCards(filelist): """ """ l = GangaList() l.extend(iexpandWildCards(filelist)) return l
def getDiracFiles(): import os from GangaDirac.Lib.Files.DiracFile import DiracFile from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns' logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...') execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None) g = GangaList() with open(filename[1:], 'r') as lfnlist: lfnlist.seek(0) g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines())) return addProxy(g)
def getDiracFiles(): import os from GangaDirac.Lib.Files.DiracFile import DiracFile from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns' logger.info( 'Creating list, this can take a while if you have a large number of SE files, please wait...' ) execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None) g = GangaList() with open(filename[1:], 'r') as lfnlist: lfnlist.seek(0) g.extend( (DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines())) return addProxy(g)
class TestGangaList(GangaGPITestCase): def __init__(self): self.ganga_list = None self.plain1 = [] self.plain2 = [] self.proxied1 = [] self.proxied2 = [] def _makeRandomString(self): str_len = random.randint(3, 10) s = '' for _ in range(str_len): s += random.choice(string.ascii_letters) return s def _makeRandomTFile(self): name = self._makeRandomString() subdir = self._makeRandomString() return TFile(name=name, subdir=subdir) def setUp(self): self.ganga_list = GangaList() self.plain1 = [self._makeRandomTFile() for _ in range(15)] self.plain2 = [self._makeRandomTFile() for _ in range(10)] self.proxied1 = GangaList() self.proxied1.extend(self.plain1[:]) self.proxied2 = GangaList() self.proxied2.extend(self.plain2[:]) assert len(getProxyAttr(self.proxied1, '_list')) == len( self.plain1), 'Somthings wrong with construction 1' assert len(getProxyAttr(self.proxied2, '_list')) == len( self.plain2), 'Somthings wrong with construction 2' def testAllListMethodsExported(self): """Tests that all methods on list are exposed by GangaList""" def getmethods(_obj): # get all the method names for an object return dir(_obj) list_methods = getmethods([]) gangalist_methods = getmethods(self.ganga_list) missing_methods = [] for m in list_methods: if not m in gangalist_methods: missing_methods.append(m) if missing_methods: logger.info(missing_methods) assert not missing_methods, \ 'Not all of lists methods are implemented: %s' % str( missing_methods) def testEq(self): """Tests the equality op that the rest of the tests rely on.""" assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same' def testNq(self): assert self.proxied1 != None assert not self.proxied1 != self.proxied1, 'Lists are the same' assert self.proxied1 != self.proxied2, 'Lists are different' assert self.plain1 != self.proxied2, 'Lists are different' def testNonZero(self): """@ExpectedFailure""" assert not GangaList( ), 'An empty GangaList should be false, just like a list' def testAdd(self): """Test __add__""" assert len(self.plain1 + self.plain2) == len(self.proxied1 + self.proxied2) assert isProxy(self.proxied1 + self.proxied2) def testAddMixed(self): """Test __add__ with mixed lists and GangaLists""" assert len(self.plain1 + self.plain2) == len(self.proxied1 + self.plain2) assert len(self.plain2 + self.plain1) == len(self.plain2 + self.proxied1) assert isProxy(self.proxied1 + self.plain2) assert len(self.plain2 + self.plain1) == len(self.plain2 + self.proxied1) assert len(self.plain1 + self.plain2) == len(self.plain1 + self.proxied2) assert isinstance(self.plain1 + self.proxied2, list) def testAddMixed2(self): self.plain1 = range(10) self.plain2 = range(10) assert isProxy(self.proxied2[-1]), 'Element access must get proxies' assert not isProxy(self.plain1[0]), 'Element access must not proxies' assert isProxy( (self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies' assert not isProxy( (self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone' assert (self.plain1 + self.proxied2 )[-1] == self.proxied2[-1], 'File objects should be equal' assert (self.plain1 + self.proxied2 )[-1] is self.proxied2[-1], 'File objects should be identical' def testAddStr(self): """Makes sure that only lists can be added.""" try: [] + '' assert False, 'Line above should throw a TypeError' except TypeError: pass try: self.proxied1 + '' assert False, 'Line above should throw a TypeError' except TypeError: pass def testContains(self): """Tests __contains__""" plist = [addProxy(x) for x in self.plain1] assert plist == self.proxied1 for p in plist: assert isProxy(p) assert p in self.proxied1, 'Proxied list should contain each proxied object' def testDelItem(self): """Test __delitem__""" for p in [addProxy(x) for x in self.plain1[:]]: assert isProxy(p) del self.proxied1[self.proxied1.index(p)] def testGE(self): """Test __ge__""" assert (self.plain1 >= self.plain2) == ( self.proxied1 >= self.proxied2), 'The lists should have the same ge' assert (self.plain2 >= self.plain1) == ( self.proxied2 >= self.proxied1), 'The lists should have the same ge' assert (self.proxied1 >= self.proxied2) != ( self.proxied2 >= self.proxied1), 'The gt should invert correctly' def testGetItem(self): """Test __getitem__""" for i in range(len(self.proxied1)): assert isProxy(self.proxied1[i]) def testGetSlice(self): """Test __getslice__""" slices = [(0, 0), (0, len(self.plain1))] for s in slices: assert self.plain1[s[0]:s[1]] == self.proxied1[ s[0]:s[1]], 'Slices %s should be the same' % str(s) t = self.plain1[:] assert t is not self.plain1, 'Slice should be a copy.' assert self.plain1[:] is not t t = self.proxied1[:] assert t is not self.proxied1, 'Slice should be a copy.' assert self.proxied1[:] is not t def testGT(self): """Test __gt__""" assert (self.plain1 > self.plain2) == ( self.proxied1 > self.proxied2), 'The lists should have the same gt' assert (self.plain2 > self.plain1) == ( self.proxied2 > self.proxied1), 'The lists should have the same gt' assert (self.proxied1 > self.proxied2) != ( self.proxied2 > self.proxied1), 'The gt should invert correctly' def testIAdd(self): """Test __iadd__""" assert isProxy(self.proxied1), 'Must be Proxy' assert isProxy(self.proxied2), 'Must be Proxy' self.plain1 += self.plain2 self.proxied1 += self.proxied2 assert self.plain1 == self.proxied1, 'Addition should be the same' assert isProxy(self.proxied1), 'Proxy must be added' def testIAddMixed(self): """Test __iadd__ where we mix lists and GangaLists""" assert isProxy(self.proxied1), 'Must be Proxy' assert isProxy(self.proxied2), 'Must be Proxy' self.plain1 += self.plain2 self.proxied1 += self.plain2 assert self.plain1 == self.proxied1, 'Addition should be the same' assert isProxy(self.proxied1), 'Proxy must be added' assert isinstance(self.plain1, list), 'Must be list instance' self.plain2 += self.proxied1 self.proxied2 += self.proxied1 assert self.plain2 == self.proxied2, 'Addition should be the same' assert isProxy(self.proxied2), 'Proxy must be added' assert isinstance(self.plain2, list), 'Must be list instance' def testIAddSlice(self): """Test __iadd__ on slices""" s1 = self.plain1[3:7] s1 += self.plain2[2:5] s2 = self.proxied1[3:7] s2 += self.proxied2[2:5] assert s1 == s2, 'Addition should be the same' assert not isProxy(s1), 'Proxy Not Needed' assert isProxy(s2), 'Proxy Needed' def testIdentity(self): """Tests we obey list like identity relations.""" t = self.plain1[4] assert self.plain1[4] is t t = self.proxied1[4] assert self.proxied1[4] is t def testIMul(self): """Test __imul__""" self.plain1 *= 5 self.proxied1 *= 5 assert self.plain1 == self.proxied1, 'Multiplication should be the same' assert isProxy(self.proxied1), 'Proxy must be added' def testIMulSlice(self): """Test __imul__ on slices""" s1 = self.plain1[3:7] s1 *= 6 s2 = self.proxied1[3:7] s2 *= 6 assert s1 == s2, 'Addition should be the same' def testLE(self): """Test __le__""" assert (self.plain1 <= self.plain2) == ( self.proxied1 <= self.proxied2), 'The lists should have the same le' assert (self.plain2 <= self.plain1) == ( self.proxied2 <= self.proxied1), 'The lists should have the same le' assert (self.proxied1 <= self.proxied2) != ( self.proxied2 <= self.proxied1), 'The le should invert correctly' def testLen(self): """Tests __len__""" assert len(self.plain1) == len( self.proxied1), 'Lengths should be the same' def testLT(self): """Test __lt__""" assert (self.plain1 < self.plain2) == ( self.proxied1 < self.proxied2), 'The lists should have the same lt' assert (self.plain2 < self.plain1) == ( self.proxied2 < self.proxied1), 'The lists should have the same lt' assert (self.proxied1 < self.proxied2) != ( self.proxied2 < self.proxied1), 'The lt should invert correctly' def testMul(self): """Test __mul__""" assert len(self.plain1 * 7) == len(self.proxied1 * 7) assert isProxy(self.proxied1 * 9) for p in self.proxied1: assert isProxy(p) def testNE(self): """Test __ne__""" assert self.plain1 != self.plain2 assert self.proxied1 != self.proxied2, 'Lists should be different' assert self.plain1[0:5] != self.plain1[2:7] assert self.proxied1[0:5] != self.proxied1[ 2:7], 'Lists should be different' def testRMul(self): """Test __rmul__""" t1 = 5 * self.plain1 t2 = 5 * self.proxied1 assert len(t1) == len(t2), 'Multiplication should be the same' def testReversed(self): """Test the __reversed__ feature (new in python 2.4).""" from TFile import TFile as tF count = len(self.proxied1) - 1 for i in self.proxied1.__reversed__(): assert i is self.proxied1[count] assert isProxy(i) assert isType(i, tF) count -= 1 def testSetItem(self): """Test __setitem__""" t = TFile(name='foo', subdir='cheese') test_index = 7 self.plain1[test_index] = t assert self.plain1[test_index] is t self.proxied1[test_index] = t assert self.proxied1[test_index] is t assert isProxy(self.proxied1[test_index]) def testSetSliceProxyList(self): self.plain1[3:7] = self.plain2[3:7] assert self.plain1[3:7] == self.plain2[ 3:7], 'The lists should be equal' assert self.plain1[3:7] is not self.plain2[ 3:7], 'The lists should be copies' self.plain1[4] = self.plain2[9] assert self.plain1[4] != self.plain2[4] tmp = self.plain1[2:9] assert self.plain1[2:9] is not tmp self.proxied1[3:7] = self.proxied2[3:7] assert self.proxied1[3:7] == self.proxied2[ 3:7], 'The lists should be equal' assert self.proxied1[3:7] is not self.proxied2[ 3:7], 'The lists should be copies' self.proxied1[4] = self.proxied2[9] assert self.proxied1[4] != self.proxied2[4] tmp = self.proxied1[2:9] assert self.proxied1[2:9] is not tmp def testAppend(self): t = addProxy(TFile(name='foo')) self.plain1.append(t) assert self.plain1[-1] == t assert self.plain1.pop() == t self.proxied1.append(t) assert self.proxied1[-1] == t assert self.proxied1[-1] is t, 'Identity Test' assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy' assert self.proxied1.pop() == t def testExtend(self): t1 = [self._makeRandomTFile() for _ in xrange(10)] self.plain1.extend(t1) self.proxied1.extend(t1) assert self.plain1 == self.proxied1, 'Lists should be the same' t2 = self.proxied1[4:7] assert isProxy(t2) from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist assert isType(t2, glist) self.plain1.extend(t2) self.proxied1.extend(t2) assert self.plain1 == self.proxied1, 'Lists should be the same' def testIndex(self): t = addProxy(TFile(name='foo')) self.proxied1.insert(8, t) assert self.proxied1[8] == t assert self.proxied1.index(t) == 8 def testInsert(self): t = addProxy(TFile(name='foo')) self.proxied1.insert(8, t) assert self.proxied1[8] == t def testPop(self): list_len = len(self.proxied1) t = self.proxied1[-1] r = self.proxied1.pop() assert t == r assert t is r assert len(self.proxied1) == list_len - 1 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl t = self.proxied1[6] r = self.proxied1.pop(6) assert t == r assert t is r assert len(self.proxied1) == list_len - 2 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl def testRemove(self): t = addProxy(TFile(name='bar')) self.proxied1.insert(7, t) list_len = len(self.proxied1) self.proxied1.remove(t) assert len(self.proxied1) == list_len - 1 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl def testIter(self): from TFile import TFile as tF count = 0 for f in self.proxied1: count += 1 assert isProxy(f) assert isType(f, tF) assert count == len(self.proxied1), 'Must visit every member' def testCmp(self): assert cmp(self.proxied1, self.proxied2) == cmp(self.plain1, self.plain2) def testHash(self): try: hash(int(self.proxied1)) assert False, 'Lists are not hashable' except TypeError: pass
class LHCbDataset(GangaDataset): '''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs). Example Usage: ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"]) ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage len(ds) # 2 (number of files) ds.getReplicas() # returns replicas for *all* files in the data set ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE ds.getCatalog() # returns XML catalog slice ds.optionsString() # returns Gaudi-sytle options [...etc...] ''' schema = {} docstr = 'List of PhysicalFile and DiracFile objects' schema['files'] = GangaFileItem(defvalue=[], typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'], sequence=1, doc=docstr) docstr = 'Ancestor depth to be queried from the Bookkeeping' schema['depth'] = SimpleItem(defvalue=0, doc=docstr) docstr = 'Use contents of file rather than generating catalog.' schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr) docstr = 'Specify the dataset persistency technology' schema['persistency'] = SimpleItem( defvalue=None, typelist=['str', 'type(None)'], doc=docstr) schema['treat_as_inputfiles'] = SimpleItem(defvalue=False, doc="Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN") _schema = Schema(Version(3, 0), schema) _category = 'datasets' _name = "LHCbDataset" _exportmethods = ['getReplicas', '__len__', '__getitem__', 'replicate', 'hasLFNs', 'append', 'extend', 'getCatalog', 'optionsString', 'getLFNs', 'getFileNames', 'getFullFileNames', 'difference', 'isSubset', 'isSuperset', 'intersection', 'symmetricDifference', 'union', 'bkMetadata', 'isEmpty', 'hasPFNs', 'getPFNs'] # ,'pop'] def __init__(self, files=None, persistency=None, depth=0, fromRef=False): super(LHCbDataset, self).__init__() if files is None: files = [] self.files = GangaList() process_files = True if fromRef: self.files._list.extend(files) process_files = False elif isinstance(files, GangaList): def isFileTest(_file): return isinstance(_file, IGangaFile) areFiles = all([isFileTest(f) for f in files._list]) if areFiles: self.files._list.extend(files._list) process_files = False elif isinstance(files, LHCbDataset): self.files._list.extend(files.files._list) process_files = False if process_files: if isType(files, LHCbDataset): for this_file in files: self.files.append(deepcopy(this_file)) elif isType(files, IGangaFile): self.files.append(deepcopy(this_file)) elif isType(files, (list, tuple, GangaList)): new_list = [] for this_file in files: if type(this_file) is str: new_file = string_datafile_shortcut_lhcb(this_file, None) elif isType(this_file, IGangaFile): new_file = stripProxy(this_file) else: new_file = strToDataFile(this_file) new_list.append(new_file) self.files.extend(new_list) elif type(files) is str: self.files.append(string_datafile_shortcut_lhcb(this_file, None), False) else: raise GangaException("Unknown object passed to LHCbDataset constructor!") self.files._setParent(self) logger.debug("Processed inputs, assigning files") # Feel free to turn this on again for debugging but it's potentially quite expensive #logger.debug( "Creating dataset with:\n%s" % self.files ) logger.debug("Assigned files") self.persistency = persistency self.depth = depth logger.debug("Dataset Created") def __getitem__(self, i): '''Proivdes scripting (e.g. ds[2] returns the 3rd file) ''' #this_file = self.files[i] # print type(this_file) # return this_file # return this_file # return this_file if type(i) == type(slice(0)): ds = LHCbDataset(files=self.files[i]) ds.depth = self.depth #ds.XMLCatalogueSlice = self.XMLCatalogueSlice return ds else: return self.files[i] def getReplicas(self): 'Returns the replicas for all files in the dataset.' lfns = self.getLFNs() cmd = 'getReplicas(%s)' % str(lfns) result = get_result(cmd, 'LFC query error. Could not get replicas.') return result['Successful'] def hasLFNs(self): 'Returns True is the dataset has LFNs and False otherwise.' for f in self.files: if isDiracFile(f): return True return False def hasPFNs(self): 'Returns True is the dataset has PFNs and False otherwise.' for f in self.files: if not isDiracFile(f): return True return False def replicate(self, destSE=''): '''Replicate all LFNs to destSE. For a list of valid SE\'s, type ds.replicate().''' if not destSE: from GangaDirac.Lib.Files.DiracFile import DiracFile DiracFile().replicate('') return if not self.hasLFNs(): raise GangaException('Cannot replicate dataset w/ no LFNs.') retry_files = [] for f in self.files: if not isDiracFile(f): continue try: result = f.replicate( destSE=destSE ) except Exception as err: msg = 'Replication error for file %s (will retry in a bit).' % f.lfn logger.warning(msg) logger.warning("Error: %s" % str(err)) retry_files.append(f) for f in retry_files: try: result = f.replicate( destSE=destSE ) except Exception as err: msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn logger.warning(msg) logger.warning(str(err)) def extend(self, files, unique=False): '''Extend the dataset. If unique, then only add files which are not already in the dataset.''' from Ganga.GPIDev.Base import ReadOnlyObjectError if self._parent is not None and self._parent._readonly(): raise ReadOnlyObjectError('object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self))) _external_files = [] if type(files) is str or isType(files, IGangaFile): _external_files = [files] elif type(files) in [list, tuple]: _external_files = files elif isType(files, LHCbDataset): _external_files = files.files else: if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'): _external_files = [files] # just in case they extend w/ self _to_remove = [] for this_file in _external_files: if hasattr(this_file, 'subfiles'): if len(this_file.subfiles) > 0: _external_files = makeGangaListByRef(this_file.subfiles) _to_remove.append(this_file) if type(this_file) is str: _external_files.append(string_datafile_shortcut_lhcb(this_file, None)) _to_remove.append(this_file) for _this_file in _to_remove: _external_files.pop(_external_files.index(_this_file)) for this_f in _external_files: _file = getDataFile(this_f) if _file is None: _file = this_f myName = _file.namePattern from GangaDirac.Lib.Files.DiracFile import DiracFile if isType(_file, DiracFile): myName = _file.lfn if unique and myName in self.getFileNames(): continue self.files.append(stripProxy(_file)) def removeFile(self, input_file): try: self.files.remove(input_file) except: raise GangaException('Dataset has no file named %s' % input_file.namePattern) def getLFNs(self): 'Returns a list of all LFNs (by name) stored in the dataset.' lfns = [] if not self: return lfns for f in self.files: if isDiracFile(f): subfiles = f.getSubFiles() if len(subfiles) == 0: lfns.append(f.lfn) else: for file in subfiles: lfns.append(file.lfn) #logger.debug( "Returning LFNS:\n%s" % str(lfns) ) logger.debug("Returning #%s LFNS" % str(len(lfns))) return lfns def getPFNs(self): 'Returns a list of all PFNs (by name) stored in the dataset.' pfns = [] if not self: return pfns for f in self.files: if isPFN(f): pfns.append(f.namePattern) return pfns def getFullFileNames(self): 'Returns all file names w/ PFN or LFN prepended.' names = [] from GangaDirac.Lib.Files.DiracFile import DiracFile for f in self.files: if isType(f, DiracFile): names.append('LFN:%s' % f.lfn) else: try: names.append('PFN:%s' % f.namePattern) except: logger.warning("Cannot determine filename for: %s " % f) raise GangaException("Cannot Get File Name") return names def getCatalog(self, site=''): '''Generates an XML catalog from the dataset (returns the XML string). Note: site defaults to config.LHCb.LocalSite Note: If the XMLCatalogueSlice attribute is set, then it returns what is written there.''' if hasattr(self.XMLCatalogueSlice, 'name'): if self.XMLCatalogueSlice.name: f = open(self.XMLCatalogueSlice.name) xml_catalog = f.read() f.close() return xml_catalog if not site: site = getConfig('LHCb')['LocalSite'] lfns = self.getLFNs() depth = self.depth tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml') cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \ % (str(lfns), depth, site, tmp_xml.name) result = get_result(cmd, 'LFN->PFN error. XML catalog error.') xml_catalog = tmp_xml.read() tmp_xml.close() return xml_catalog def optionsString(self, file=None): 'Returns the Gaudi-style options string for the dataset (if a filename' \ ' is given, the file is created and output is written there).' if not self or len(self) == 0: return '' snew = '' if self.persistency == 'ROOT': snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == 'POOL': snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == None: snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' else: logger.warning( "Unknown LHCbDataset persistency technology... reverting to None") snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=[' sdatasetsnew = '' sdatasetsold = '' dtype_str_default = getConfig('LHCb')['datatype_string_default'] dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns'] for f in self.files: dtype_str = dtype_str_default for this_str in dtype_str_patterns: matched = False for pat in dtype_str_patterns[this_str]: if fnmatch.fnmatch(f.namePattern, pat): dtype_str = this_str matched = True break if matched: break sdatasetsnew += '\n ' sdatasetsold += '\n ' if isDiracFile(f): sdatasetsnew += """ \"LFN:%s\",""" % f.lfn sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str) else: sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str) if sdatasetsold.endswith(","): if self.persistency == 'ROOT': sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)""" else: sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)""" sdatasetsold = sdatasetsold[:-1] sdatasetsold += """\n ]""" if(file): f = open(file, 'w') if self.persistency == 'ROOT': f.write(snew) f.write(sdatasetsnew) else: f.write(snew) f.write(sdatasetsnew) f.write(sold) f.write(sdatasetsold) f.close() else: if self.persistency == 'ROOT': return snew + sdatasetsnew else: return snew + sdatasetsnew + sold + sdatasetsold def _checkOtherFiles(self, other ): if isType(other, GangaList) or isType(other, []): other_files = LHCbDataset(other).getFullFileNames() elif isType(other, LHCbDataset): other_files = other.getFullFileNames() else: raise GangaException("Unknown type for difference") return other_files def difference(self, other): '''Returns a new data set w/ files in this that are not in other.''' other_files = self._checkOtherFiles(other) files = set(self.getFullFileNames()).difference(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def isSubset(self, other): '''Is every file in this data set in other?''' other_files = self._checkOtherFiles(other) return set(self.getFileNames()).issubset(other_files) def isSuperset(self, other): '''Is every file in other in this data set?''' other_files = self._checkOtherFiles(other) return set(self.getFileNames()).issuperset(other_files) def symmetricDifference(self, other): '''Returns a new data set w/ files in either this or other but not both.''' other_files = other._checkOtherFiles(other) files = set(self.getFullFileNames()).symmetric_difference(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def intersection(self, other): '''Returns a new data set w/ files common to this and other.''' other_files = other._checkOtherFiles(other) files = set(self.getFullFileNames()).intersection(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def union(self, other): '''Returns a new data set w/ files from this and other.''' other_files = self._checkOtherFiles(other) files = set(self.getFullFileNames()).union(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def bkMetadata(self): 'Returns the bookkeeping metadata for all LFNs. ' logger.info("Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info...") cmd = 'bkMetaData(%s)' % self.getLFNs() b = get_result(cmd, 'Error removing replica. Replica rm error.') return b
class TestGangaList(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGangaList, self).__init__(*args, **kwargs) self.plain1 = [] self.plain2 = [] self.proxied1 = [] self.proxied2 = [] self.ganga_list = GangaList() @staticmethod def _makeRandomString(): str_len = random.randint(3, 10) s = '' for _ in range(str_len): s += random.choice(string.ascii_letters) return s @staticmethod def _makeRandomTFile(): name = TestGangaList._makeRandomString() subdir = TestGangaList._makeRandomString() return TFile(name=name, subdir=subdir) def setUp(self): super(TestGangaList, self).setUp() self.plain1 = [self._makeRandomTFile() for _ in range(15)] self.plain2 = [self._makeRandomTFile() for _ in range(10)] self.proxied1 = GangaList() self.proxied1.extend(self.plain1[:]) self.proxied2 = GangaList() self.proxied2.extend(self.plain2[:]) t = TFile() real_t = stripProxy(t) new_proxy_t = addProxy(real_t) #hopefully_t = stripProxy(new_proxy_t) #assert real_t is hopefully_t assert t is new_proxy_t self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction") self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction") def testAllListMethodsExported(self): """Tests that all methods on list are exposed by GangaList""" def getmethods(_obj): # get all the method names for an object return dir(_obj) list_methods = getmethods([]) gangalist_methods = getmethods(self.ganga_list) missing_methods = [] for m in list_methods: if not m in gangalist_methods: missing_methods.append(m) if missing_methods: logger.info(missing_methods) self.assertFalse( missing_methods, 'Not all of lists methods are implemented: %s' % missing_methods) def testEq(self): """Tests the equality op that the rest of the tests rely on.""" self.assertEqual( self.proxied1, self.plain1, 'Proxied and non-proxied versions should be the same') def testNq(self): self.assertIsNotNone(self.proxied1, None) self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same') self.assertNotEqual(self.proxied1, self.proxied2, 'Lists are different') self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different') def testNonZero(self): """@ExpectedFailure""" self.assertFalse( GangaList(), 'An empty GangaList should be false, just like a list') def testAdd(self): """Test __add__""" self.assertEqual(self.plain1 + self.plain2, self.proxied1 + self.proxied2) self.assertTrue(isProxy(self.proxied1 + self.proxied2)) def testAddMixed(self): """Test __add__ with mixed lists and GangaLists""" self.assertEqual((self.plain1 + self.plain2), (self.proxied1 + self.plain2)) self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1)) self.assertTrue(isProxy(self.proxied1 + self.plain2)) self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1)) self.assertEqual((self.plain1 + self.plain2), (self.plain1 + self.proxied2)) self.assertTrue(isinstance(self.plain1 + self.proxied2, list)) def testAddMixed2(self): self.plain1 = range(10) self.plain2 = range(10) self.assertTrue(isProxy(self.proxied2[-1]), 'Element access must get proxies') self.assertFalse(isProxy(self.plain1[0]), 'Element access must not proxies') self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies') self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone') self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be equal') self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be identical') def testAddStr(self): """Makes sure that only lists can be added.""" try: [] + '' assert False, 'Line above should throw a TypeError' except TypeError: pass try: self.proxied1 + '' assert False, 'Line above should throw a TypeError' except TypeError: pass def testContains(self): """Tests __contains__""" plist = [addProxy(x) for x in self.plain1] self.assertEqual(plist, self.proxied1) for p in plist: self.assertTrue(isProxy(p)) self.assertIn(p, self.proxied1, 'Proxied list should contain each proxied object') def testDelItem(self): """Test __delitem__""" for p in [addProxy(x) for x in self.plain1[:]]: self.assertTrue(isProxy(p)) del self.proxied1[self.proxied1.index(p)] def testGE(self): """Test __ge__""" self.assertEqual((self.plain1 >= self.plain2), (self.proxied1 >= self.proxied2), 'The lists should have the same ge') self.assertEqual((self.plain2 >= self.plain1), (self.proxied2 >= self.proxied1), 'The lists should have the same ge') self.assertNotEqual((self.proxied1 >= self.proxied2), (self.proxied2 >= self.proxied1), 'The gt should invert correctly') def testGetItem(self): """Test __getitem__""" for i in range(len(self.proxied1)): self.assertTrue(isProxy(self.proxied1[i])) def testGetSlice(self): """Test __getslice__""" slices = [(0, 0), (0, len(self.plain1))] for s in slices: self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]], 'Slices {0} should be the same'.format(s)) t = self.plain1[:] self.assertIsNot(t, self.plain1, 'Slice should be a copy.') self.assertIsNot(self.plain1[:], t) t = self.proxied1[:] self.assertIsNot(t, self.proxied1, 'Slice should be a copy.') self.assertIsNot(self.proxied1[:], t) def testGT(self): """Test __gt__""" self.assertEqual((self.plain1 > self.plain2), (self.proxied1 > self.proxied2), 'The lists should have the same gt') self.assertEqual((self.plain2 > self.plain1), (self.proxied2 > self.proxied1), 'The lists should have the same gt') self.assertNotEqual((self.proxied1 > self.proxied2), (self.proxied2 > self.proxied1), 'The gt should invert correctly') def testIAdd(self): """Test __iadd__""" self.assertTrue(isProxy(self.proxied1), 'Must be Proxy') self.assertTrue(isProxy(self.proxied2), 'Must be Proxy') self.plain1 += self.plain2 self.proxied1 += self.proxied2 self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') def testIAddMixed(self): """Test __iadd__ where we mix lists and GangaLists""" self.assertTrue(isProxy(self.proxied1), 'Must be Proxy') self.assertTrue(isProxy(self.proxied2), 'Must be Proxy') self.plain1 += self.plain2 self.proxied1 += self.plain2 self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') self.assertTrue(isinstance(self.plain1, list), 'Must be list instance') self.plain2 += self.proxied1 self.proxied2 += self.proxied1 self.assertEqual(self.plain2, self.proxied2, 'Addition should be the same') self.assertTrue(isProxy(self.proxied2), 'Proxy must be added') self.assertTrue(isinstance(self.plain2, list), 'Must be list instance') def testIAddSlice(self): """Test __iadd__ on slices""" s1 = self.plain1[3:7] s1 += self.plain2[2:5] s2 = self.proxied1[3:7] s2 += self.proxied2[2:5] self.assertEqual(s1, s2, 'Addition should be the same') self.assertFalse(isProxy(s1), 'Proxy Not Needed') self.assertTrue(isProxy(s2), 'Proxy Needed') def testIdentity(self): """Tests we obey list like identity relations.""" t = self.plain1[4] self.assertIs(self.plain1[4], t) t = self.proxied1[4] self.assertIs(self.proxied1[4], t) def testIMul(self): """Test __imul__""" self.plain1 *= 5 self.proxied1 *= 5 self.assertEqual(self.plain1, self.proxied1, 'Multiplication should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') def testIMulSlice(self): """Test __imul__ on slices""" s1 = self.plain1[3:7] s1 *= 6 s2 = self.proxied1[3:7] s2 *= 6 self.assertEqual(s1, s2, 'Addition should be the same') def testLE(self): """Test __le__""" self.assertEqual(self.plain1 <= self.plain2, self.proxied1 <= self.proxied2, 'The lists should have the same le') self.assertEqual(self.plain2 <= self.plain1, self.proxied2 <= self.proxied1, 'The lists should have the same le') self.assertNotEqual(self.proxied1 <= self.proxied2, self.proxied2 <= self.proxied1, 'The le should invert correctly') def testLen(self): """Tests __len__""" self.assertEqual(len(self.plain1), len(self.proxied1), 'Lengths should be the same') def testLT(self): """Test __lt__""" self.assertEqual(self.plain1 < self.plain2, self.proxied1 < self.proxied2, 'The lists should have the same lt') self.assertEqual(self.plain2 < self.plain1, self.proxied2 < self.proxied1, 'The lists should have the same lt') self.assertNotEqual(self.proxied1 < self.proxied2, self.proxied2 < self.proxied1, 'The lt should invert correctly') def testMul(self): """Test __mul__""" self.assertEqual((self.plain1 * 7), (self.proxied1 * 7)) self.assertTrue(isProxy(self.proxied1 * 9)) for p in self.proxied1: self.assertTrue(isProxy(p)) def testNE(self): """Test __ne__""" self.assertNotEqual(self.plain1, self.plain2) self.assertNotEqual(self.proxied1, self.proxied2, 'Lists should be different') self.assertNotEqual(self.plain1[0:5], self.plain1[2:7]) self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7], 'Lists should be different') def testRMul(self): """Test __rmul__""" t1 = 5 * self.plain1 t2 = 5 * self.proxied1 self.assertEqual(t1, t2, 'Multiplication should be the same') def testReversed(self): """Test the __reversed__ feature (new in python 2.4).""" count = len(self.proxied1) - 1 for i in self.proxied1.__reversed__(): self.assertIs(i, self.proxied1[count]) self.assertTrue(isProxy(i)) self.assertTrue(isType(i, TFile)) count -= 1 def testSetItem(self): """Test __setitem__""" t = TFile(name='foo', subdir='cheese') test_index = 7 self.plain1[test_index] = t self.assertIs(self.plain1[test_index], t) self.proxied1[test_index] = t self.assertIs(self.proxied1[test_index], t) self.assertTrue(isProxy(self.proxied1[test_index])) def testSetSliceProxyList(self): self.plain1[3:7] = self.plain2[3:7] self.assertEqual(self.plain1[3:7], self.plain2[3:7], 'The lists should be equal') self.assertIsNot(self.plain1[3:7], self.plain2[3:7], 'The lists should be copies') self.plain1[4] = self.plain2[9] self.assertNotEqual(self.plain1[4], self.plain2[4]) tmp = self.plain1[2:9] self.assertIsNot(self.plain1[2:9], tmp) self.proxied1[3:7] = self.proxied2[3:7] self.assertEqual(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be equal') self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be copies') self.proxied1[4] = self.proxied2[9] self.assertNotEqual(self.proxied1[4], self.proxied2[4]) tmp = self.proxied1[2:9] self.assertIsNot(self.proxied1[2:9], tmp) def testAppend(self): t = TFile(name='foo') self.plain1.append(t) self.assertEqual(self.plain1[-1], t) self.assertEqual(self.plain1.pop(), t) self.proxied1.append(t) self.assertEqual(self.proxied1[-1], t) self.assertIs(self.proxied1[-1], t, 'Identity Test') self.assertTrue(isProxy(self.proxied1[-1]), 'Make sure we get back a proxy') self.assertEqual(self.proxied1.pop(), t) def testExtend(self): t1 = [self._makeRandomTFile() for _ in xrange(10)] self.plain1.extend(t1) self.proxied1.extend(t1) self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same') t2 = self.proxied1[4:7] self.assertTrue(isProxy(t2)) from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist self.assertTrue(isType(t2, glist)) self.plain1.extend(t2) self.proxied1.extend(t2) self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same') def testIndex(self): t = TFile(name='foo') self.proxied1.insert(8, t) self.assertEqual(self.proxied1[8], t) self.assertEqual(self.proxied1.index(t), 8) def testInsert(self): t = TFile(name='foo') self.proxied1.insert(8, t) self.assertEqual(self.proxied1[8], t) def testPop(self): list_len = len(self.proxied1) t = self.proxied1[-1] r = self.proxied1.pop() self.assertEqual(t, r) self.assertIs(t, r) self.assertEqual(len(self.proxied1), list_len - 1) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) t = self.proxied1[6] r = self.proxied1.pop(6) self.assertEqual(t, r) self.assertIs(t, r) self.assertEqual(len(self.proxied1), list_len - 2) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) def testRemove(self): t = TFile(name='bar') self.proxied1.insert(7, t) list_len = len(self.proxied1) self.proxied1.remove(t) self.assertEqual(len(self.proxied1), list_len - 1) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) def testIter(self): count = 0 for f in self.proxied1: count += 1 self.assertTrue(isProxy(f)) self.assertTrue(isType(f, TFile)) self.assertEqual(count, len(self.proxied1), 'Must visit every member') def testCmp(self): self.assertEqual(cmp(self.proxied1, self.proxied2), cmp(self.plain1, self.plain2)) def testHash(self): try: hash(int(self.proxied1)) assert False, 'Lists are not hashable' except TypeError: pass
class TestGangaList(GangaGPITestCase): def __init__(self): self.ganga_list = None self.plain1 = [] self.plain2 = [] self.proxied1 = [] self.proxied2 = [] def _makeRandomString(self): str_len = random.randint(3, 10) s = '' for _ in range(str_len): s += random.choice(string.ascii_letters) return s def _makeRandomTFile(self): name = self._makeRandomString() subdir = self._makeRandomString() return TFile(name=name, subdir=subdir) def setUp(self): self.ganga_list = GangaList() self.plain1 = [self._makeRandomTFile() for _ in range(15)] self.plain2 = [self._makeRandomTFile() for _ in range(10)] self.proxied1 = GangaList() self.proxied1.extend(self.plain1[:]) self.proxied2 = GangaList() self.proxied2.extend(self.plain2[:]) assert len(getProxyAttr(self.proxied1, '_list')) == len( self.plain1), 'Somthings wrong with construction' assert len(getProxyAttr(self.proxied2, '_list')) == len( self.plain2), 'Somthings wrong with construction' def testAllListMethodsExported(self): """Tests that all methods on list are exposed by GangaList""" def getmethods(_obj): # get all the method names for an object return dir(_obj) list_methods = getmethods([]) gangalist_methods = getmethods(self.ganga_list) missing_methods = [] for m in list_methods: if not m in gangalist_methods: missing_methods.append(m) if missing_methods: logger.info(missing_methods) assert not missing_methods, \ 'Not all of lists methods are implemented: %s' % str( missing_methods) def testEq(self): """Tests the equality op that the rest of the tests rely on.""" assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same' def testNq(self): assert self.proxied1 != None assert not self.proxied1 != self.proxied1, 'Lists are the same' assert self.proxied1 != self.proxied2, 'Lists are different' assert self.plain1 != self.proxied2, 'Lists are different' def testNonZero(self): """@ExpectedFailure""" assert not GangaList( ), 'An empty GangaList should be false, just like a list' def testAdd(self): """Test __add__""" assert (self.plain1 + self.plain2) == (self.proxied1 + self.proxied2) assert isProxy(self.proxied1 + self.proxied2) def testAddMixed(self): """Test __add__ with mixed lists and GangaLists""" assert (self.plain1 + self.plain2) == (self.proxied1 + self.plain2) assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1) assert isProxy(self.proxied1 + self.plain2) assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1) assert (self.plain1 + self.plain2) == (self.plain1 + self.proxied2) assert isinstance(self.plain1 + self.proxied2, list) def testAddMixed2(self): self.plain1 = range(10) self.plain2 = range(10) assert isProxy(self.proxied2[-1]), 'Element access must get proxies' assert not isProxy(self.plain1[0]), 'Element access must not proxies' assert isProxy( (self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies' assert not isProxy( (self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone' assert ( self.plain1 + self.proxied2)[-1] == self.proxied2[-1], 'File objects should be equal' assert ( self.plain1 + self.proxied2)[-1] is self.proxied2[-1], 'File objects should be identical' def testAddStr(self): """Makes sure that only lists can be added.""" try: [] + '' assert False, 'Line above should throw a TypeError' except TypeError: pass try: self.proxied1 + '' assert False, 'Line above should throw a TypeError' except TypeError: pass def testContains(self): """Tests __contains__""" plist = [addProxy(x) for x in self.plain1] assert plist == self.proxied1 for p in plist: assert isProxy(p) assert p in self.proxied1, 'Proxied list should contain each proxied object' def testDelItem(self): """Test __delitem__""" for p in [addProxy(x) for x in self.plain1[:]]: assert isProxy(p) del self.proxied1[self.proxied1.index(p)] def testGE(self): """Test __ge__""" assert (self.plain1 >= self.plain2) == ( self.proxied1 >= self.proxied2), 'The lists should have the same ge' assert (self.plain2 >= self.plain1) == ( self.proxied2 >= self.proxied1), 'The lists should have the same ge' assert (self.proxied1 >= self.proxied2) != ( self.proxied2 >= self.proxied1), 'The gt should invert correctly' def testGetItem(self): """Test __getitem__""" for i in range(len(self.proxied1)): assert isProxy(self.proxied1[i]) def testGetSlice(self): """Test __getslice__""" slices = [(0, 0), (0, len(self.plain1))] for s in slices: assert self.plain1[s[0]:s[1]] == self.proxied1[ s[0]:s[1]], 'Slices %s should be the same' % str(s) t = self.plain1[:] assert t is not self.plain1, 'Slice should be a copy.' assert self.plain1[:] is not t t = self.proxied1[:] assert t is not self.proxied1, 'Slice should be a copy.' assert self.proxied1[:] is not t def testGT(self): """Test __gt__""" assert (self.plain1 > self.plain2) == ( self.proxied1 > self.proxied2), 'The lists should have the same gt' assert (self.plain2 > self.plain1) == ( self.proxied2 > self.proxied1), 'The lists should have the same gt' assert (self.proxied1 > self.proxied2) != ( self.proxied2 > self.proxied1), 'The gt should invert correctly' def testIAdd(self): """Test __iadd__""" assert isProxy(self.proxied1), 'Must be Proxy' assert isProxy(self.proxied2), 'Must be Proxy' self.plain1 += self.plain2 self.proxied1 += self.proxied2 assert self.plain1 == self.proxied1, 'Addition should be the same' assert isProxy(self.proxied1), 'Proxy must be added' def testIAddMixed(self): """Test __iadd__ where we mix lists and GangaLists""" assert isProxy(self.proxied1), 'Must be Proxy' assert isProxy(self.proxied2), 'Must be Proxy' self.plain1 += self.plain2 self.proxied1 += self.plain2 assert self.plain1 == self.proxied1, 'Addition should be the same' assert isProxy(self.proxied1), 'Proxy must be added' assert isinstance(self.plain1, list), 'Must be list instance' self.plain2 += self.proxied1 self.proxied2 += self.proxied1 assert self.plain2 == self.proxied2, 'Addition should be the same' assert isProxy(self.proxied2), 'Proxy must be added' assert isinstance(self.plain2, list), 'Must be list instance' def testIAddSlice(self): """Test __iadd__ on slices""" s1 = self.plain1[3:7] s1 += self.plain2[2:5] s2 = self.proxied1[3:7] s2 += self.proxied2[2:5] assert s1 == s2, 'Addition should be the same' assert not isProxy(s1), 'Proxy Not Needed' assert isProxy(s2), 'Proxy Needed' def testIdentity(self): """Tests we obey list like identity relations.""" t = self.plain1[4] assert self.plain1[4] is t t = self.proxied1[4] assert self.proxied1[4] is t def testIMul(self): """Test __imul__""" self.plain1 *= 5 self.proxied1 *= 5 assert self.plain1 == self.proxied1, 'Multiplication should be the same' assert isProxy(self.proxied1), 'Proxy must be added' def testIMulSlice(self): """Test __imul__ on slices""" s1 = self.plain1[3:7] s1 *= 6 s2 = self.proxied1[3:7] s2 *= 6 assert s1 == s2, 'Addition should be the same' def testLE(self): """Test __le__""" assert (self.plain1 <= self.plain2) == ( self.proxied1 <= self.proxied2), 'The lists should have the same le' assert (self.plain2 <= self.plain1) == ( self.proxied2 <= self.proxied1), 'The lists should have the same le' assert (self.proxied1 <= self.proxied2) != ( self.proxied2 <= self.proxied1), 'The le should invert correctly' def testLen(self): """Tests __len__""" assert len(self.plain1) == len( self.proxied1), 'Lengths should be the same' def testLT(self): """Test __lt__""" assert (self.plain1 < self.plain2) == ( self.proxied1 < self.proxied2), 'The lists should have the same lt' assert (self.plain2 < self.plain1) == ( self.proxied2 < self.proxied1), 'The lists should have the same lt' assert (self.proxied1 < self.proxied2) != ( self.proxied2 < self.proxied1), 'The lt should invert correctly' def testMul(self): """Test __mul__""" assert (self.plain1 * 7) == (self.proxied1 * 7) assert isProxy(self.proxied1 * 9) for p in self.proxied1: assert isProxy(p) def testNE(self): """Test __ne__""" assert self.plain1 != self.plain2 assert self.proxied1 != self.proxied2, 'Lists should be different' assert self.plain1[0:5] != self.plain1[2:7] assert self.proxied1[0:5] != self.proxied1[ 2:7], 'Lists should be different' def testRMul(self): """Test __rmul__""" t1 = 5 * self.plain1 t2 = 5 * self.proxied1 assert t1 == t2, 'Multiplication should be the same' def testReversed(self): """Test the __reversed__ feature (new in python 2.4).""" from TFile import TFile as tF count = len(self.proxied1) - 1 for i in self.proxied1.__reversed__(): assert i is self.proxied1[count] assert isProxy(i) assert isType(i, tF) count -= 1 def testSetItem(self): """Test __setitem__""" t = TFile(name='foo', subdir='cheese') test_index = 7 self.plain1[test_index] = t assert self.plain1[test_index] is t self.proxied1[test_index] = t assert self.proxied1[test_index] is t assert isProxy(self.proxied1[test_index]) def testSetSliceProxyList(self): self.plain1[3:7] = self.plain2[3:7] assert self.plain1[3:7] == self.plain2[ 3:7], 'The lists should be equal' assert self.plain1[3:7] is not self.plain2[ 3:7], 'The lists should be copies' self.plain1[4] = self.plain2[9] assert self.plain1[4] != self.plain2[4] tmp = self.plain1[2:9] assert self.plain1[2:9] is not tmp self.proxied1[3:7] = self.proxied2[3:7] assert self.proxied1[3:7] == self.proxied2[ 3:7], 'The lists should be equal' assert self.proxied1[3:7] is not self.proxied2[ 3:7], 'The lists should be copies' self.proxied1[4] = self.proxied2[9] assert self.proxied1[4] != self.proxied2[4] tmp = self.proxied1[2:9] assert self.proxied1[2:9] is not tmp def testAppend(self): t = addProxy(TFile(name='foo')) self.plain1.append(t) assert self.plain1[-1] == t assert self.plain1.pop() == t self.proxied1.append(t) assert self.proxied1[-1] == t assert self.proxied1[-1] is t, 'Identity Test' assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy' assert self.proxied1.pop() == t def testExtend(self): t1 = [self._makeRandomTFile() for _ in xrange(10)] self.plain1.extend(t1) self.proxied1.extend(t1) assert self.plain1 == self.proxied1, 'Lists should be the same' t2 = self.proxied1[4:7] assert isProxy(t2) from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist assert isType(t2, glist) self.plain1.extend(t2) self.proxied1.extend(t2) assert self.plain1 == self.proxied1, 'Lists should be the same' def testIndex(self): t = addProxy(TFile(name='foo')) self.proxied1.insert(8, t) assert self.proxied1[8] == t assert self.proxied1.index(t) == 8 def testInsert(self): t = addProxy(TFile(name='foo')) self.proxied1.insert(8, t) assert self.proxied1[8] == t def testPop(self): list_len = len(self.proxied1) t = self.proxied1[-1] r = self.proxied1.pop() assert t == r assert t is r assert len(self.proxied1) == list_len - 1 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl t = self.proxied1[6] r = self.proxied1.pop(6) assert t == r assert t is r assert len(self.proxied1) == list_len - 2 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl def testRemove(self): t = addProxy(TFile(name='bar')) self.proxied1.insert(7, t) list_len = len(self.proxied1) self.proxied1.remove(t) assert len(self.proxied1) == list_len - 1 assert t not in self.proxied1 assert t._impl not in self.proxied1._impl def testIter(self): from TFile import TFile as tF count = 0 for f in self.proxied1: count += 1 assert isProxy(f) assert isType(f, tF) assert count == len(self.proxied1), 'Must visit every member' def testCmp(self): assert cmp(self.proxied1, self.proxied2) == cmp( self.plain1, self.plain2) def testHash(self): try: hash(int(self.proxied1)) assert False, 'Lists are not hashable' except TypeError: pass
class TestGangaList(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGangaList, self).__init__(*args, **kwargs) self.plain1 = [] self.plain2 = [] self.proxied1 = [] self.proxied2 = [] self.ganga_list = GangaList() @staticmethod def _makeRandomString(): str_len = random.randint(3, 10) s = '' for _ in range(str_len): s += random.choice(string.ascii_letters) return s @staticmethod def _makeRandomTFile(): name = TestGangaList._makeRandomString() subdir = TestGangaList._makeRandomString() return TFile(name=name, subdir=subdir) def setUp(self): super(TestGangaList, self).setUp() self.plain1 = [self._makeRandomTFile() for _ in range(15)] self.plain2 = [self._makeRandomTFile() for _ in range(10)] self.proxied1 = GangaList() self.proxied1.extend(self.plain1[:]) self.proxied2 = GangaList() self.proxied2.extend(self.plain2[:]) t = TFile() real_t = stripProxy(t) new_proxy_t = addProxy(real_t) #hopefully_t = stripProxy(new_proxy_t) #assert real_t is hopefully_t assert t is new_proxy_t self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction") self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction") def testAllListMethodsExported(self): """Tests that all methods on list are exposed by GangaList""" def getmethods(_obj): # get all the method names for an object return dir(_obj) list_methods = getmethods([]) gangalist_methods = getmethods(self.ganga_list) missing_methods = [] for m in list_methods: if not m in gangalist_methods: missing_methods.append(m) if missing_methods: logger.info(missing_methods) self.assertFalse(missing_methods, 'Not all of lists methods are implemented: %s' % missing_methods) def testEq(self): """Tests the equality op that the rest of the tests rely on.""" self.assertEqual(self.proxied1, self.plain1, 'Proxied and non-proxied versions should be the same') def testNq(self): self.assertIsNotNone(self.proxied1, None) self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same') self.assertNotEqual(self.proxied1, self.proxied2, 'Lists are different') self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different') def testNonZero(self): """@ExpectedFailure""" self.assertFalse(GangaList(), 'An empty GangaList should be false, just like a list') def testAdd(self): """Test __add__""" self.assertEqual(self.plain1 + self.plain2, self.proxied1 + self.proxied2) self.assertTrue(isProxy(self.proxied1 + self.proxied2)) def testAddMixed(self): """Test __add__ with mixed lists and GangaLists""" self.assertEqual((self.plain1 + self.plain2), (self.proxied1 + self.plain2)) self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1)) self.assertTrue(isProxy(self.proxied1 + self.plain2)) self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1)) self.assertEqual((self.plain1 + self.plain2), (self.plain1 + self.proxied2)) self.assertTrue(isinstance(self.plain1 + self.proxied2, list)) def testAddMixed2(self): self.plain1 = range(10) self.plain2 = range(10) self.assertTrue(isProxy(self.proxied2[-1]), 'Element access must get proxies') self.assertFalse(isProxy(self.plain1[0]), 'Element access must not proxies') self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies') self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone') self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be equal') self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be identical') def testAddStr(self): """Makes sure that only lists can be added.""" try: [] + '' assert False, 'Line above should throw a TypeError' except TypeError: pass try: self.proxied1 + '' assert False, 'Line above should throw a TypeError' except TypeError: pass def testContains(self): """Tests __contains__""" plist = [addProxy(x) for x in self.plain1] self.assertEqual(plist, self.proxied1) for p in plist: self.assertTrue(isProxy(p)) self.assertIn(p, self.proxied1, 'Proxied list should contain each proxied object') def testDelItem(self): """Test __delitem__""" for p in [addProxy(x) for x in self.plain1[:]]: self.assertTrue(isProxy(p)) del self.proxied1[self.proxied1.index(p)] def testGE(self): """Test __ge__""" self.assertEqual((self.plain1 >= self.plain2), (self.proxied1 >= self.proxied2), 'The lists should have the same ge') self.assertEqual((self.plain2 >= self.plain1), (self.proxied2 >= self.proxied1), 'The lists should have the same ge') self.assertNotEqual((self.proxied1 >= self.proxied2), (self.proxied2 >= self.proxied1), 'The gt should invert correctly') def testGetItem(self): """Test __getitem__""" for i in range(len(self.proxied1)): self.assertTrue(isProxy(self.proxied1[i])) def testGetSlice(self): """Test __getslice__""" slices = [(0, 0), (0, len(self.plain1))] for s in slices: self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]], 'Slices {0} should be the same'.format(s)) t = self.plain1[:] self.assertIsNot(t, self.plain1, 'Slice should be a copy.') self.assertIsNot(self.plain1[:], t) t = self.proxied1[:] self.assertIsNot(t, self.proxied1, 'Slice should be a copy.') self.assertIsNot(self.proxied1[:], t) def testGT(self): """Test __gt__""" self.assertEqual((self.plain1 > self.plain2), (self.proxied1 > self.proxied2), 'The lists should have the same gt') self.assertEqual((self.plain2 > self.plain1), (self.proxied2 > self.proxied1), 'The lists should have the same gt') self.assertNotEqual((self.proxied1 > self.proxied2), (self.proxied2 > self.proxied1), 'The gt should invert correctly') def testIAdd(self): """Test __iadd__""" self.assertTrue(isProxy(self.proxied1), 'Must be Proxy') self.assertTrue(isProxy(self.proxied2), 'Must be Proxy') self.plain1 += self.plain2 self.proxied1 += self.proxied2 self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') def testIAddMixed(self): """Test __iadd__ where we mix lists and GangaLists""" self.assertTrue(isProxy(self.proxied1), 'Must be Proxy') self.assertTrue(isProxy(self.proxied2), 'Must be Proxy') self.plain1 += self.plain2 self.proxied1 += self.plain2 self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') self.assertTrue(isinstance(self.plain1, list), 'Must be list instance') self.plain2 += self.proxied1 self.proxied2 += self.proxied1 self.assertEqual(self.plain2, self.proxied2, 'Addition should be the same') self.assertTrue(isProxy(self.proxied2), 'Proxy must be added') self.assertTrue(isinstance(self.plain2, list), 'Must be list instance') def testIAddSlice(self): """Test __iadd__ on slices""" s1 = self.plain1[3:7] s1 += self.plain2[2:5] s2 = self.proxied1[3:7] s2 += self.proxied2[2:5] self.assertEqual(s1, s2, 'Addition should be the same') self.assertFalse(isProxy(s1), 'Proxy Not Needed') self.assertTrue(isProxy(s2), 'Proxy Needed') def testIdentity(self): """Tests we obey list like identity relations.""" t = self.plain1[4] self.assertIs(self.plain1[4], t) t = self.proxied1[4] self.assertIs(self.proxied1[4], t) def testIMul(self): """Test __imul__""" self.plain1 *= 5 self.proxied1 *= 5 self.assertEqual(self.plain1, self.proxied1, 'Multiplication should be the same') self.assertTrue(isProxy(self.proxied1), 'Proxy must be added') def testIMulSlice(self): """Test __imul__ on slices""" s1 = self.plain1[3:7] s1 *= 6 s2 = self.proxied1[3:7] s2 *= 6 self.assertEqual(s1, s2, 'Addition should be the same') def testLE(self): """Test __le__""" self.assertEqual(self.plain1 <= self.plain2, self.proxied1 <= self.proxied2, 'The lists should have the same le') self.assertEqual(self.plain2 <= self.plain1, self.proxied2 <= self.proxied1, 'The lists should have the same le') self.assertNotEqual(self.proxied1 <= self.proxied2, self.proxied2 <= self.proxied1, 'The le should invert correctly') def testLen(self): """Tests __len__""" self.assertEqual(len(self.plain1), len(self.proxied1), 'Lengths should be the same') def testLT(self): """Test __lt__""" self.assertEqual(self.plain1 < self.plain2, self.proxied1 < self.proxied2, 'The lists should have the same lt') self.assertEqual(self.plain2 < self.plain1, self.proxied2 < self.proxied1, 'The lists should have the same lt') self.assertNotEqual(self.proxied1 < self.proxied2, self.proxied2 < self.proxied1, 'The lt should invert correctly') def testMul(self): """Test __mul__""" self.assertEqual((self.plain1 * 7), (self.proxied1 * 7)) self.assertTrue(isProxy(self.proxied1 * 9)) for p in self.proxied1: self.assertTrue(isProxy(p)) def testNE(self): """Test __ne__""" self.assertNotEqual(self.plain1, self.plain2) self.assertNotEqual(self.proxied1, self.proxied2, 'Lists should be different') self.assertNotEqual(self.plain1[0:5], self.plain1[2:7]) self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7], 'Lists should be different') def testRMul(self): """Test __rmul__""" t1 = 5 * self.plain1 t2 = 5 * self.proxied1 self.assertEqual(t1, t2, 'Multiplication should be the same') def testReversed(self): """Test the __reversed__ feature (new in python 2.4).""" count = len(self.proxied1) - 1 for i in self.proxied1.__reversed__(): self.assertIs(i, self.proxied1[count]) self.assertTrue(isProxy(i)) self.assertTrue(isType(i, TFile)) count -= 1 def testSetItem(self): """Test __setitem__""" t = TFile(name='foo', subdir='cheese') test_index = 7 self.plain1[test_index] = t self.assertIs(self.plain1[test_index], t) self.proxied1[test_index] = t self.assertIs(self.proxied1[test_index], t) self.assertTrue(isProxy(self.proxied1[test_index])) def testSetSliceProxyList(self): self.plain1[3:7] = self.plain2[3:7] self.assertEqual(self.plain1[3:7], self.plain2[3:7], 'The lists should be equal') self.assertIsNot(self.plain1[3:7], self.plain2[3:7], 'The lists should be copies') self.plain1[4] = self.plain2[9] self.assertNotEqual(self.plain1[4], self.plain2[4]) tmp = self.plain1[2:9] self.assertIsNot(self.plain1[2:9], tmp) self.proxied1[3:7] = self.proxied2[3:7] self.assertEqual(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be equal') self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be copies') self.proxied1[4] = self.proxied2[9] self.assertNotEqual(self.proxied1[4], self.proxied2[4]) tmp = self.proxied1[2:9] self.assertIsNot(self.proxied1[2:9], tmp) def testAppend(self): t = TFile(name='foo') self.plain1.append(t) self.assertEqual(self.plain1[-1], t) self.assertEqual(self.plain1.pop(), t) self.proxied1.append(t) self.assertEqual(self.proxied1[-1], t) self.assertIs(self.proxied1[-1], t, 'Identity Test') self.assertTrue(isProxy(self.proxied1[-1]), 'Make sure we get back a proxy') self.assertEqual(self.proxied1.pop(), t) def testExtend(self): t1 = [self._makeRandomTFile() for _ in xrange(10)] self.plain1.extend(t1) self.proxied1.extend(t1) self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same') t2 = self.proxied1[4:7] self.assertTrue(isProxy(t2)) from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist self.assertTrue(isType(t2, glist)) self.plain1.extend(t2) self.proxied1.extend(t2) self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same') def testIndex(self): t = TFile(name='foo') self.proxied1.insert(8, t) self.assertEqual(self.proxied1[8], t) self.assertEqual(self.proxied1.index(t), 8) def testInsert(self): t = TFile(name='foo') self.proxied1.insert(8, t) self.assertEqual(self.proxied1[8], t) def testPop(self): list_len = len(self.proxied1) t = self.proxied1[-1] r = self.proxied1.pop() self.assertEqual(t, r) self.assertIs(t, r) self.assertEqual(len(self.proxied1), list_len - 1) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) t = self.proxied1[6] r = self.proxied1.pop(6) self.assertEqual(t, r) self.assertIs(t, r) self.assertEqual(len(self.proxied1), list_len - 2) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) def testRemove(self): t = TFile(name='bar') self.proxied1.insert(7, t) list_len = len(self.proxied1) self.proxied1.remove(t) self.assertEqual(len(self.proxied1), list_len - 1) self.assertNotIn(t, self.proxied1) self.assertNotIn(t._impl, self.proxied1._impl) def testIter(self): count = 0 for f in self.proxied1: count += 1 self.assertTrue(isProxy(f)) self.assertTrue(isType(f, TFile)) self.assertEqual(count, len(self.proxied1), 'Must visit every member') def testCmp(self): self.assertEqual(cmp(self.proxied1, self.proxied2), cmp(self.plain1, self.plain2)) def testHash(self): try: hash(int(self.proxied1)) assert False, 'Lists are not hashable' except TypeError: pass
class LHCbDataset(GangaDataset): '''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs). Example Usage: ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"]) ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage len(ds) # 2 (number of files) ds.getReplicas() # returns replicas for *all* files in the data set ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE ds.getCatalog() # returns XML catalog slice ds.optionsString() # returns Gaudi-sytle options [...etc...] ''' schema = {} docstr = 'List of PhysicalFile and DiracFile objects' schema['files'] = GangaFileItem( defvalue=[], typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'], sequence=1, doc=docstr) docstr = 'Ancestor depth to be queried from the Bookkeeping' schema['depth'] = SimpleItem(defvalue=0, doc=docstr) docstr = 'Use contents of file rather than generating catalog.' schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr) docstr = 'Specify the dataset persistency technology' schema['persistency'] = SimpleItem(defvalue=None, typelist=['str', 'type(None)'], doc=docstr) schema['treat_as_inputfiles'] = SimpleItem( defvalue=False, doc= "Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN") _schema = Schema(Version(3, 0), schema) _category = 'datasets' _name = "LHCbDataset" _exportmethods = [ 'getReplicas', '__len__', '__getitem__', 'replicate', 'hasLFNs', 'append', 'extend', 'getCatalog', 'optionsString', 'getLFNs', 'getFileNames', 'getFullFileNames', 'difference', 'isSubset', 'isSuperset', 'intersection', 'symmetricDifference', 'union', 'bkMetadata', 'isEmpty', 'hasPFNs', 'getPFNs' ] # ,'pop'] def __init__(self, files=None, persistency=None, depth=0, fromRef=False): super(LHCbDataset, self).__init__() if files is None: files = [] self.files = GangaList() process_files = True if fromRef: self.files._list.extend(files) process_files = False elif isinstance(files, GangaList): def isFileTest(_file): return isinstance(_file, IGangaFile) areFiles = all([isFileTest(f) for f in files._list]) if areFiles: self.files._list.extend(files._list) process_files = False elif isinstance(files, LHCbDataset): self.files._list.extend(files.files._list) process_files = False if process_files: if isType(files, LHCbDataset): for this_file in files: self.files.append(deepcopy(this_file)) elif isType(files, IGangaFile): self.files.append(deepcopy(files)) elif isType(files, (list, tuple, GangaList)): new_list = [] for this_file in files: if type(this_file) is str: new_file = string_datafile_shortcut_lhcb( this_file, None) elif isType(this_file, IGangaFile): new_file = stripProxy(this_file) else: new_file = strToDataFile(this_file) new_list.append(new_file) self.files.extend(new_list) elif type(files) is str: self.files.append(string_datafile_shortcut_lhcb(files, None), False) else: raise GangaException( "Unknown object passed to LHCbDataset constructor!") self.files._setParent(self) logger.debug("Processed inputs, assigning files") # Feel free to turn this on again for debugging but it's potentially quite expensive #logger.debug( "Creating dataset with:\n%s" % self.files ) logger.debug("Assigned files") self.persistency = persistency self.depth = depth logger.debug("Dataset Created") def __getitem__(self, i): '''Proivdes scripting (e.g. ds[2] returns the 3rd file) ''' #this_file = self.files[i] # print type(this_file) # return this_file # return this_file # return this_file if type(i) == type(slice(0)): ds = LHCbDataset(files=self.files[i]) ds.depth = self.depth #ds.XMLCatalogueSlice = self.XMLCatalogueSlice return ds else: return self.files[i] def getReplicas(self): 'Returns the replicas for all files in the dataset.' lfns = self.getLFNs() cmd = 'getReplicas(%s)' % str(lfns) result = get_result(cmd, 'LFC query error. Could not get replicas.') return result['Successful'] def hasLFNs(self): 'Returns True is the dataset has LFNs and False otherwise.' for f in self.files: if isDiracFile(f): return True return False def hasPFNs(self): 'Returns True is the dataset has PFNs and False otherwise.' for f in self.files: if not isDiracFile(f): return True return False def replicate(self, destSE=''): '''Replicate all LFNs to destSE. For a list of valid SE\'s, type ds.replicate().''' if not destSE: from GangaDirac.Lib.Files.DiracFile import DiracFile DiracFile().replicate('') return if not self.hasLFNs(): raise GangaException('Cannot replicate dataset w/ no LFNs.') retry_files = [] for f in self.files: if not isDiracFile(f): continue try: result = f.replicate(destSE=destSE) except Exception as err: msg = 'Replication error for file %s (will retry in a bit).' % f.lfn logger.warning(msg) logger.warning("Error: %s" % str(err)) retry_files.append(f) for f in retry_files: try: result = f.replicate(destSE=destSE) except Exception as err: msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn logger.warning(msg) logger.warning(str(err)) def extend(self, files, unique=False): '''Extend the dataset. If unique, then only add files which are not already in the dataset.''' from Ganga.GPIDev.Base import ReadOnlyObjectError if self._parent is not None and self._parent._readonly(): raise ReadOnlyObjectError( 'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self))) _external_files = [] if type(files) is str or isType(files, IGangaFile): _external_files = [files] elif type(files) in [list, tuple]: _external_files = files elif isType(files, LHCbDataset): _external_files = files.files else: if not hasattr(files, "__getitem__") or not hasattr( files, '__iter__'): _external_files = [files] # just in case they extend w/ self _to_remove = [] for this_file in _external_files: if hasattr(this_file, 'subfiles'): if len(this_file.subfiles) > 0: _external_files = makeGangaListByRef(this_file.subfiles) _to_remove.append(this_file) if type(this_file) is str: _external_files.append( string_datafile_shortcut_lhcb(this_file, None)) _to_remove.append(this_file) for _this_file in _to_remove: _external_files.pop(_external_files.index(_this_file)) for this_f in _external_files: _file = getDataFile(this_f) if _file is None: _file = this_f myName = _file.namePattern from GangaDirac.Lib.Files.DiracFile import DiracFile if isType(_file, DiracFile): myName = _file.lfn if unique and myName in self.getFileNames(): continue self.files.append(stripProxy(_file)) def removeFile(self, input_file): try: self.files.remove(input_file) except: raise GangaException('Dataset has no file named %s' % input_file.namePattern) def getLFNs(self): 'Returns a list of all LFNs (by name) stored in the dataset.' lfns = [] if not self: return lfns for f in self.files: if isDiracFile(f): subfiles = f.getSubFiles() if len(subfiles) == 0: lfns.append(f.lfn) else: for file in subfiles: lfns.append(file.lfn) #logger.debug( "Returning LFNS:\n%s" % str(lfns) ) logger.debug("Returning #%s LFNS" % str(len(lfns))) return lfns def getPFNs(self): 'Returns a list of all PFNs (by name) stored in the dataset.' pfns = [] if not self: return pfns for f in self.files: if isPFN(f): pfns.append(f.namePattern) return pfns def getFullFileNames(self): 'Returns all file names w/ PFN or LFN prepended.' names = [] from GangaDirac.Lib.Files.DiracFile import DiracFile for f in self.files: if isType(f, DiracFile): names.append('LFN:%s' % f.lfn) else: try: names.append('PFN:%s' % f.namePattern) except: logger.warning("Cannot determine filename for: %s " % f) raise GangaException("Cannot Get File Name") return names def getCatalog(self, site=''): '''Generates an XML catalog from the dataset (returns the XML string). Note: site defaults to config.LHCb.LocalSite Note: If the XMLCatalogueSlice attribute is set, then it returns what is written there.''' if hasattr(self.XMLCatalogueSlice, 'name'): if self.XMLCatalogueSlice.name: f = open(self.XMLCatalogueSlice.name) xml_catalog = f.read() f.close() return xml_catalog if not site: site = getConfig('LHCb')['LocalSite'] lfns = self.getLFNs() depth = self.depth tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml') cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \ % (str(lfns), depth, site, tmp_xml.name) result = get_result(cmd, 'LFN->PFN error. XML catalog error.') xml_catalog = tmp_xml.read() tmp_xml.close() return xml_catalog def optionsString(self, file=None): 'Returns the Gaudi-style options string for the dataset (if a filename' \ ' is given, the file is created and output is written there).' if not self or len(self) == 0: return '' snew = '' if self.persistency == 'ROOT': snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == 'POOL': snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency elif self.persistency == None: snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' else: logger.warning( "Unknown LHCbDataset persistency technology... reverting to None" ) snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles([' sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=[' sdatasetsnew = '' sdatasetsold = '' dtype_str_default = getConfig('LHCb')['datatype_string_default'] dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns'] for f in self.files: dtype_str = dtype_str_default for this_str in dtype_str_patterns: matched = False for pat in dtype_str_patterns[this_str]: if fnmatch.fnmatch(f.namePattern, pat): dtype_str = this_str matched = True break if matched: break sdatasetsnew += '\n ' sdatasetsold += '\n ' if isDiracFile(f): sdatasetsnew += """ \"LFN:%s\",""" % f.lfn sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str) else: sdatasetsnew += """ \"%s\",""" % f.accessURL()[0] sdatasetsold += """ \"DATAFILE='%s' %s\",""" % ( f.accessURL()[0], dtype_str) if sdatasetsold.endswith(","): if self.persistency == 'ROOT': sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)""" else: sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)""" sdatasetsold = sdatasetsold[:-1] sdatasetsold += """\n ]""" if (file): f = open(file, 'w') if self.persistency == 'ROOT': f.write(snew) f.write(sdatasetsnew) else: f.write(snew) f.write(sdatasetsnew) f.write(sold) f.write(sdatasetsold) f.close() else: if self.persistency == 'ROOT': return snew + sdatasetsnew else: return snew + sdatasetsnew + sold + sdatasetsold def _checkOtherFiles(self, other): if isType(other, GangaList) or isType(other, []): other_files = LHCbDataset(other).getFullFileNames() elif isType(other, LHCbDataset): other_files = other.getFullFileNames() else: raise GangaException("Unknown type for difference") return other_files def difference(self, other): '''Returns a new data set w/ files in this that are not in other.''' other_files = self._checkOtherFiles(other) files = set(self.getFullFileNames()).difference(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def isSubset(self, other): '''Is every file in this data set in other?''' other_files = self._checkOtherFiles(other) return set(self.getFileNames()).issubset(other_files) def isSuperset(self, other): '''Is every file in other in this data set?''' other_files = self._checkOtherFiles(other) return set(self.getFileNames()).issuperset(other_files) def symmetricDifference(self, other): '''Returns a new data set w/ files in either this or other but not both.''' other_files = other._checkOtherFiles(other) files = set(self.getFullFileNames()).symmetric_difference(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def intersection(self, other): '''Returns a new data set w/ files common to this and other.''' other_files = other._checkOtherFiles(other) files = set(self.getFullFileNames()).intersection(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def union(self, other): '''Returns a new data set w/ files from this and other.''' other_files = self._checkOtherFiles(other) files = set(self.getFullFileNames()).union(other_files) data = LHCbDataset() data.extend(list(files)) data.depth = self.depth return data def bkMetadata(self): 'Returns the bookkeeping metadata for all LFNs. ' logger.info( "Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info..." ) cmd = 'bkMetaData(%s)' % self.getLFNs() b = get_result(cmd, 'Error removing replica. Replica rm error.') return b