def expandWildCards(filelist):
"""
"""
l = GangaList()
l.extend(iexpandWildCards(filelist))
return l
def expandWildCards(filelist):
"""
"""
l = GangaList()
l.extend(iexpandWildCards(filelist))
return l
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(files))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(
this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(files, None),
False)
else:
raise GangaException(
"Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def testPrintingGPIObjectList(self):
g = GangaList()
for _ in range(10):
g.append(self._makeRandomTFile())
g_string = str(g)
assert eval(g_string) == g, 'String should correctly eval'
def testPrintingGPIObjectList(self):
g = GangaList()
for _ in range(10):
g.append(self._makeRandomTFile())
g_string = str(g)
assert eval(g_string) == g, 'String should correctly eval'
def cloneVal(v):
GangaList = _getGangaList()
if isinstance(v, (list, tuple, GangaList)):
new_v = GangaList()
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
return new_v
else:
return self.__cloneVal(v, obj)
def testFullPrintingGPIObjectList(self):
g = GangaList()
for _ in range(10):
g.append(self._makeRandomTFile())
g_string = str(g)
import StringIO
sio = StringIO.StringIO()
full_print(g, sio)
assert g_string == str(sio.getvalue()).rstrip(), 'Orphaned lists should full_print'
def getDiracFiles():
import os
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...')
execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
g = GangaList()
with open(filename[1:], 'r') as lfnlist:
lfnlist.seek(0)
g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
return addProxy(g)
def testFullPrintingGPIObjectList(self):
g = GangaList()
for _ in range(10):
g.append(self._makeRandomTFile())
g_string = str(g)
import StringIO
sio = StringIO.StringIO()
full_print(g, sio)
assert g_string == str(
sio.getvalue()).rstrip(), 'Orphaned lists should full_print'
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(self.plain1), 'Somthings wrong with construction 1'
assert len(getProxyAttr(self.proxied2, '_list')) == len(self.plain2), 'Somthings wrong with construction 2'
def testPrintingPlainList(self):
g = GangaList()
l = []
print('"'+str(g)+'"')
print('"'+str(l)+'"')
print(l == g)
assert str(l) == str(g), 'Empty lists should print the same'
for i in xrange(100):
g.append(i)
l.append(i)
assert str(l) == str(g), 'Normal Python objects should print the same'
def testPrintingPlainList(self):
g = GangaList()
l = []
print('"' + str(g) + '"')
print('"' + str(l) + '"')
print(l == g)
assert str(l) == str(g), 'Empty lists should print the same'
for i in xrange(100):
g.append(i)
l.append(i)
assert str(l) == str(g), 'Normal Python objects should print the same'
def _stripAttribute(obj, v, name):
# just warn
# print '**** checking',v,v.__class__,
# isinstance(val,GPIProxyObject)
if isinstance(v, list):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
v_new = GangaList()
for elem in v:
v_new.append(elem)
v = v_new
if isinstance(v, GPIProxyObject) or hasattr(v, implRef):
v = stripProxy(v)
logger.debug('%s property: assigned a component object (%s used)' % (name, implRef))
return stripProxy(obj)._attribute_filter__set__(name, v)
def testCopy(self):
gl = GangaList()
numberOfFiles = 100
for _ in range(numberOfFiles):
# add something which is generally not allowed by GangaList
gl.append([self._makeRandomTFile()])
assert len(gl) == numberOfFiles, 'Right number of files must be made'
gl2 = copy.copy(gl)
assert len(gl2) == len(gl), 'lists must be equal'
assert gl2 is not gl, 'list must be copies'
def testCopy(self):
gl = GangaList()
numberOfFiles = 100
for _ in range(numberOfFiles):
# add something which is generally not allowed by GangaList
gl.append([self._makeRandomTFile()])
assert len(gl) == numberOfFiles, 'Right number of files must be made'
gl2 = copy.copy(gl)
assert gl2 == gl, 'lists must be equal'
assert gl2 is not gl, 'list must be copies'
def getDiracFiles():
import os
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns'
logger.info(
'Creating list, this can take a while if you have a large number of SE files, please wait...'
)
execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None)
g = GangaList()
with open(filename[1:], 'r') as lfnlist:
lfnlist.seek(0)
g.extend(
(DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines()))
return addProxy(g)
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(
self.plain1), 'Somthings wrong with construction 1'
assert len(getProxyAttr(self.proxied2, '_list')) == len(
self.plain2), 'Somthings wrong with construction 2'
def _stripAttribute(obj, v, name):
# just warn
# print '**** checking',v,v.__class__,
# isinstance(val,GPIProxyObject)
if isinstance(v, list):
from Ganga.GPI import GangaList
v_new = GangaList()
for elem in v:
v_new.append(elem)
v = v_new
if isinstance(v, GPIProxyObject) or hasattr(v, implRef):
v = stripProxy(v)
logger.debug('%s property: assigned a component object (%s used)' %
(name, implRef))
return stripProxy(obj)._attribute_filter__set__(name, v)
def testDeepCopy(self):
gl = GangaList()
numberOfFiles = 100
for _ in range(numberOfFiles):
# add something which is generally not allowed by GangaList
gl.append([self._makeRandomTFile()])
assert len(gl) == numberOfFiles, "Right number of files must be made"
gl2 = copy.deepcopy(gl)
assert gl2 == gl, "lists must be equal"
assert gl2 is not gl, "list must be copies"
assert gl[0] is not gl2[0], "the references must be copied"
def __cloneVal(self, v, obj):
item = obj._schema[getName(self)]
if v is None:
if item.hasProperty('category'):
assertion = item['optional'] and (item['category'] != 'internal')
else:
assertion = item['optional']
#assert(assertion)
if assertion is False:
logger.warning("Item: '%s'. of class type: '%s'. Has a Default value of 'None' but is NOT optional!!!" % (getName(self), type(obj)))
logger.warning("Please contact the developers and make sure this is updated!")
return None
elif isinstance(v, str):
return str(v)
elif isinstance(v, int):
return int(v)
elif isinstance(v, dict):
new_dict = {}
for key, item in new_dict.iteritems():
new_dict[key] = self.__cloneVal(v, obj)
return new_dict
else:
if not isinstance(v, Node) and isinstance(v, (list, tuple)):
try:
GangaList = _getGangaList()
new_v = GangaList()
except ImportError:
new_v = []
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
#return new_v
elif not isinstance(v, Node):
if inspect.isclass(v):
new_v = v()
else:
new_v = v
if not isinstance(new_v, Node):
logger.error("v: %s" % str(v))
raise GangaException("Error: found Object: %s of type: %s expected an object inheriting from Node!" % (str(v), str(type(v))))
else:
new_v = self.__copyNodeObject(new_v, obj)
else:
new_v = self.__copyNodeObject(v, obj)
return new_v
def testDeepCopy(self):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
gl = GangaList()
numberOfFiles = 100
for _ in range(numberOfFiles):
# add something which is generally not allowed by GangaList
gl.append([self._makeRandomTFile()])
assert len(gl) == numberOfFiles, 'Right number of files must be made'
gl2 = copy.deepcopy(gl)
assert len(gl2) == len(gl), 'lists must be equal'
assert gl2 is not gl, 'list must be copies'
assert gl[0] is not gl2[0], 'the references must not be copied'
def testDeepCopy(self):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
gl = GangaList()
numberOfFiles = 100
for _ in range(numberOfFiles):
# add something which is generally not allowed by GangaList
gl.append([self._makeRandomTFile()])
assert len(gl) == numberOfFiles, 'Right number of files must be made'
gl2 = copy.deepcopy(gl)
assert len(gl2) == len(gl), 'lists must be equal'
assert gl2 is not gl, 'list must be copies'
assert gl[0] is not gl2[0], 'the references must not be copied'
def __init__(self, *args, **kwargs):
super(TestGangaList, self).__init__(*args, **kwargs)
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
self.ganga_list = GangaList()
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(this_file))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
raise GangaException("Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
def __init__(self, files=None, persistency=None, depth=0):
super(LHCbDataset, self).__init__()
if files is None:
files = []
new_files = GangaList()
if isType(files, LHCbDataset):
for this_file in files:
new_files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
new_files.append(deepcopy(this_file))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(this_file, None)
elif isType(this_file, IGangaFile):
new_file = this_file
else:
new_file = strToDataFile(this_file)
new_list.append(stripProxy(new_file))
stripProxy(new_files)._list = new_list
elif type(files) is str:
new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
raise GangaException("Unknown object passed to LHCbDataset constructor!")
new_files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % files )
self.files = new_files
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def testNonZero(self):
"""@ExpectedFailure"""
assert not GangaList(
), 'An empty GangaList should be false, just like a list'
def put(self, lfn='', force=False, uploadSE="", replicate=False):
"""
Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
File will be uploaded to the first SE that the upload command succeeds for.
The file is uploaded to the SE described by the DiracFile.defaultSE attribute
Alternatively, the user can specify an uploadSE which contains an SE
which the file is to be uploaded to.
If the user wants to replicate this file(s) across all SE then they should state replicate = True.
Return value will be either the stdout from the dirac upload command if not
using the wildcard characters '*?[]' in the namePattern.
If the wildcard characters are used then the return value will be a list containing
newly created DiracFile objects which were the result of glob-ing the wildcards.
The objects in this list will have been uploaded or had their failureReason attribute populated if the
upload failed.
"""
if self.lfn != "" and force == False and lfn == '':
logger.warning(
"Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s"
% (self.namePattern, self.lfn))
decision = raw_input('y / [n]:')
while not (decision in ['y', 'n'] or decision == ''):
decision = raw_input('y / [n]:')
if decision == 'y':
pass
else:
return
if (lfn != '' and self.lfn != '') and force == False:
logger.warning(
"Warning you're attempting to put this DiracFile: %s" %
self.namePattern)
logger.warning("It currently has an LFN associated with it: %s" %
self.lfn)
logger.warning(
"Do you want to continue and attempt to upload to: %s" % lfn)
decision = raw_input('y / [n]:')
while not (decision in ['y', 'n', '']):
decision = raw_input('y / [n]:')
if decision == 'y':
pass
else:
return
if lfn != '':
self.lfn = lfn
# looks like will only need this for the interactive uploading of jobs.
# Also if any backend need dirac upload on client then when downloaded
# this will upload then delete the file.
if self.namePattern == "":
if self.lfn != '':
logger.warning(
"'Put'-ing a file with ONLY an existing LFN makes no sense!"
)
raise GangaException(
'Can\'t upload a file without a local file name.')
sourceDir = self.localDir
if self.localDir is None:
sourceDir = os.getcwd()
# attached to a job, use the joboutputdir
if self._parent != None and os.path.isdir(
self.getJobObject().outputdir):
sourceDir = self.getJobObject().outputdir
if not os.path.isdir(sourceDir):
raise GangaException(
'localDir attribute is not a valid dir, don\'t know from which dir to take the file'
)
if regex.search(self.namePattern) is not None:
if self.lfn != "":
logger.warning(
"Cannot specify a single lfn for a wildcard namePattern")
logger.warning("LFN will be generated automatically")
self.lfn = ""
selfConstructedLFN = False
import glob
if self.remoteDir == '' and self.lfn == '':
import datetime
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
self.lfn = os.path.join(configDirac['DiracLFNBase'],
'GangaFiles_%s' % this_date)
selfConstructedLFN = True
#if self.remoteDir == '' and self.lfn != '':
# self.remoteDir = configDirac['DiracLFNBase']
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
if uploadSE != "":
storage_elements = all_SE_list(uploadSE)
else:
storage_elements = all_SE_list(self.defaultSE)
outputFiles = GangaList()
for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
name = this_file
if not os.path.exists(name):
if not self.compressed:
raise GangaException(
'Cannot upload file. File "%s" must exist!' % name)
name += '.gz'
if not os.path.exists(name):
raise GangaException('File "%s" must exist!' % name)
else:
if self.compressed:
os.system('gzip -c %s > %s.gz' % (name, name))
name += '.gz'
if not os.path.exists(name):
raise GangaException('File "%s" must exist!' % name)
if lfn == "":
lfn = os.path.join(lfn_base, os.path.basename(name))
if selfConstructedLFN is True:
self.lfn = os.path.join(self.lfn, os.path.basename(name))
lfn = self.lfn
d = DiracFile()
d.namePattern = os.path.basename(name)
d.compressed = self.compressed
d.localDir = sourceDir
stderr = ''
stdout = ''
logger.info('Uploading file %s to %s as %s' %
(name, storage_elements[0], lfn))
stdout = execute('uploadFile("%s", "%s", %s)' %
(lfn, name, str([storage_elements[0]])))
if type(stdout) == str:
logger.warning("Couldn't upload file '%s': %s" %
(os.path.basename(name), stdout))
continue
if stdout.get('OK', False) and lfn in stdout.get(
'Value', {'Successful': {}})['Successful']:
# when doing the two step upload delete the temp file
if self.compressed or self._parent != None:
os.remove(name)
# need another eval as datetime needs to be included.
guid = stdout['Value']['Successful'][lfn].get('GUID', '')
if regex.search(self.namePattern) is not None:
d.lfn = lfn
d.remoteDir = os.path.dirname(lfn)
d.locations = stdout['Value']['Successful'][lfn].get(
'allDiracSE', '')
d.guid = guid
outputFiles.append(GPIProxyObjectFactory(d))
continue
else:
self.lfn = lfn
self.remoteDir = os.path.dirname(lfn)
self.locations = stdout['Value']['Successful'][lfn].get(
'allDiracSE', '')
self.guid = guid
# return ## WHY?
else:
failureReason = "Error in uploading file %s : %s" % (
os.path.basename(name), str(stdout))
logger.error(failureReason)
if regex.search(self.namePattern) is not None:
d.failureReason = failureReason
outputFiles.append(GPIProxyObjectFactory(d))
continue
self.failureReason = failureReason
return str(stdout)
if replicate == True:
if len(outputFiles) == 1 or len(outputFiles) == 0:
storage_elements.pop(0)
for se in storage_elements:
self.replicate(se)
else:
storage_elements.pop(0)
for this_file in outputFiles:
for se in storage_elements:
this_file.replicate(se)
if len(outputFiles) > 0:
return GPIProxyObjectFactory(outputFiles)
else:
outputFiles.append(self)
return GPIProxyObjectFactory(outputFiles)
class TestGangaList(GangaGPITestCase):
def __init__(self):
self.ganga_list = None
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
def _makeRandomString(self):
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
def _makeRandomTFile(self):
name = self._makeRandomString()
subdir = self._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(
self.plain1), 'Somthings wrong with construction 1'
assert len(getProxyAttr(self.proxied2, '_list')) == len(
self.plain2), 'Somthings wrong with construction 2'
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
assert not missing_methods, \
'Not all of lists methods are implemented: %s' % str(
missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'
def testNq(self):
assert self.proxied1 != None
assert not self.proxied1 != self.proxied1, 'Lists are the same'
assert self.proxied1 != self.proxied2, 'Lists are different'
assert self.plain1 != self.proxied2, 'Lists are different'
def testNonZero(self):
"""@ExpectedFailure"""
assert not GangaList(
), 'An empty GangaList should be false, just like a list'
def testAdd(self):
"""Test __add__"""
assert len(self.plain1 + self.plain2) == len(self.proxied1 +
self.proxied2)
assert isProxy(self.proxied1 + self.proxied2)
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
assert len(self.plain1 + self.plain2) == len(self.proxied1 +
self.plain2)
assert len(self.plain2 + self.plain1) == len(self.plain2 +
self.proxied1)
assert isProxy(self.proxied1 + self.plain2)
assert len(self.plain2 + self.plain1) == len(self.plain2 +
self.proxied1)
assert len(self.plain1 + self.plain2) == len(self.plain1 +
self.proxied2)
assert isinstance(self.plain1 + self.proxied2, list)
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
assert not isProxy(self.plain1[0]), 'Element access must not proxies'
assert isProxy(
(self.plain1 +
self.proxied2)[-1]), 'File objects should remain proxies'
assert not isProxy(
(self.plain1 +
self.proxied2)[0]), 'Objects in plain lists should be left alone'
assert (self.plain1 + self.proxied2
)[-1] == self.proxied2[-1], 'File objects should be equal'
assert (self.plain1 + self.proxied2
)[-1] is self.proxied2[-1], 'File objects should be identical'
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
assert plist == self.proxied1
for p in plist:
assert isProxy(p)
assert p in self.proxied1, 'Proxied list should contain each proxied object'
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
assert isProxy(p)
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
assert (self.plain1 >= self.plain2) == (
self.proxied1 >=
self.proxied2), 'The lists should have the same ge'
assert (self.plain2 >= self.plain1) == (
self.proxied2 >=
self.proxied1), 'The lists should have the same ge'
assert (self.proxied1 >= self.proxied2) != (
self.proxied2 >= self.proxied1), 'The gt should invert correctly'
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
assert isProxy(self.proxied1[i])
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
assert self.plain1[s[0]:s[1]] == self.proxied1[
s[0]:s[1]], 'Slices %s should be the same' % str(s)
t = self.plain1[:]
assert t is not self.plain1, 'Slice should be a copy.'
assert self.plain1[:] is not t
t = self.proxied1[:]
assert t is not self.proxied1, 'Slice should be a copy.'
assert self.proxied1[:] is not t
def testGT(self):
"""Test __gt__"""
assert (self.plain1 > self.plain2) == (
self.proxied1 > self.proxied2), 'The lists should have the same gt'
assert (self.plain2 > self.plain1) == (
self.proxied2 > self.proxied1), 'The lists should have the same gt'
assert (self.proxied1 > self.proxied2) != (
self.proxied2 > self.proxied1), 'The gt should invert correctly'
def testIAdd(self):
"""Test __iadd__"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.proxied2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.plain2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
assert isinstance(self.plain1, list), 'Must be list instance'
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
assert self.plain2 == self.proxied2, 'Addition should be the same'
assert isProxy(self.proxied2), 'Proxy must be added'
assert isinstance(self.plain2, list), 'Must be list instance'
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
assert s1 == s2, 'Addition should be the same'
assert not isProxy(s1), 'Proxy Not Needed'
assert isProxy(s2), 'Proxy Needed'
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
assert self.plain1[4] is t
t = self.proxied1[4]
assert self.proxied1[4] is t
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
assert self.plain1 == self.proxied1, 'Multiplication should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
assert s1 == s2, 'Addition should be the same'
def testLE(self):
"""Test __le__"""
assert (self.plain1 <= self.plain2) == (
self.proxied1 <=
self.proxied2), 'The lists should have the same le'
assert (self.plain2 <= self.plain1) == (
self.proxied2 <=
self.proxied1), 'The lists should have the same le'
assert (self.proxied1 <= self.proxied2) != (
self.proxied2 <= self.proxied1), 'The le should invert correctly'
def testLen(self):
"""Tests __len__"""
assert len(self.plain1) == len(
self.proxied1), 'Lengths should be the same'
def testLT(self):
"""Test __lt__"""
assert (self.plain1 < self.plain2) == (
self.proxied1 < self.proxied2), 'The lists should have the same lt'
assert (self.plain2 < self.plain1) == (
self.proxied2 < self.proxied1), 'The lists should have the same lt'
assert (self.proxied1 < self.proxied2) != (
self.proxied2 < self.proxied1), 'The lt should invert correctly'
def testMul(self):
"""Test __mul__"""
assert len(self.plain1 * 7) == len(self.proxied1 * 7)
assert isProxy(self.proxied1 * 9)
for p in self.proxied1:
assert isProxy(p)
def testNE(self):
"""Test __ne__"""
assert self.plain1 != self.plain2
assert self.proxied1 != self.proxied2, 'Lists should be different'
assert self.plain1[0:5] != self.plain1[2:7]
assert self.proxied1[0:5] != self.proxied1[
2:7], 'Lists should be different'
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
assert len(t1) == len(t2), 'Multiplication should be the same'
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
from TFile import TFile as tF
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
assert i is self.proxied1[count]
assert isProxy(i)
assert isType(i, tF)
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
assert self.plain1[test_index] is t
self.proxied1[test_index] = t
assert self.proxied1[test_index] is t
assert isProxy(self.proxied1[test_index])
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
assert self.plain1[3:7] == self.plain2[
3:7], 'The lists should be equal'
assert self.plain1[3:7] is not self.plain2[
3:7], 'The lists should be copies'
self.plain1[4] = self.plain2[9]
assert self.plain1[4] != self.plain2[4]
tmp = self.plain1[2:9]
assert self.plain1[2:9] is not tmp
self.proxied1[3:7] = self.proxied2[3:7]
assert self.proxied1[3:7] == self.proxied2[
3:7], 'The lists should be equal'
assert self.proxied1[3:7] is not self.proxied2[
3:7], 'The lists should be copies'
self.proxied1[4] = self.proxied2[9]
assert self.proxied1[4] != self.proxied2[4]
tmp = self.proxied1[2:9]
assert self.proxied1[2:9] is not tmp
def testAppend(self):
t = addProxy(TFile(name='foo'))
self.plain1.append(t)
assert self.plain1[-1] == t
assert self.plain1.pop() == t
self.proxied1.append(t)
assert self.proxied1[-1] == t
assert self.proxied1[-1] is t, 'Identity Test'
assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
assert self.proxied1.pop() == t
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
assert self.plain1 == self.proxied1, 'Lists should be the same'
t2 = self.proxied1[4:7]
assert isProxy(t2)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
assert isType(t2, glist)
self.plain1.extend(t2)
self.proxied1.extend(t2)
assert self.plain1 == self.proxied1, 'Lists should be the same'
def testIndex(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
assert self.proxied1.index(t) == 8
def testInsert(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
t = self.proxied1[6]
r = self.proxied1.pop(6)
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 2
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testRemove(self):
t = addProxy(TFile(name='bar'))
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testIter(self):
from TFile import TFile as tF
count = 0
for f in self.proxied1:
count += 1
assert isProxy(f)
assert isType(f, tF)
assert count == len(self.proxied1), 'Must visit every member'
def testCmp(self):
assert cmp(self.proxied1,
self.proxied2) == cmp(self.plain1, self.plain2)
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
def __set__(self, _obj, _val):
obj = stripProxy(_obj)
val = stripProxy(_val)
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList
cs = self._bind_method(obj, self._checkset_name)
if cs:
cs(val)
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(val)
# LOCKING
obj._getWriteAccess()
# self._check_getter()
item = stripProxy(obj._schema[self._name])
def cloneVal(v):
return self.__cloneVal(v, obj)
obj_reg = obj._getRegistry()
full_reg = obj_reg is not None and hasattr(obj_reg, 'dirty_flush_counter')
### THIS CODE HERE CHANGES THE DIRTY FLUSH COUNTER SUCH THAT GANGALIST OBJECTS ARE WRITTEN TO DISK ATOMICALLY
### THIS COMES WITH A MAJOR PERFORMANCE IMPROVEMENT IN SOME CODE THAT REALLY SHOULDN'T BE SLOW
### Maybe delete this if/when we aren't using XML repo in future as allowing a really dirty repo is probably bad m'kay
## NB Should really tie in the default here to defaults from Core
old_count = 10
if hasattr(val, '__len__') and full_reg:
old_count = obj_reg.dirty_flush_counter
val_len = 2*len(val) + 10
obj_reg.dirty_flush_counter = val_len
obj_len = 1
if len(val) > 0:
bare_val = stripProxy(val)
if hasattr(bare_val, '_schema'):
obj_len = len(stripProxy(bare_val._schema).datadict.keys())
obj_len = obj_len*2
val_len = val_len * obj_len
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
val = GangaList()
else:
if isType(item, Schema.ComponentItem):
val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
if isType(item, Schema.ComponentItem):
val = cloneVal(val)
if hasattr(val, '_setParent'):
val._setParent(obj)
### RESET DIRTY COUNT
if full_reg:
obj_reg.dirty_flush_counter = old_count
obj._data[self._name] = val
obj._setDirty()
class LHCbDataset(GangaDataset):
'''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).
Example Usage:
ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
len(ds) # 2 (number of files)
ds.getReplicas() # returns replicas for *all* files in the data set
ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
ds.getCatalog() # returns XML catalog slice
ds.optionsString() # returns Gaudi-sytle options
[...etc...]
'''
schema = {}
docstr = 'List of PhysicalFile and DiracFile objects'
schema['files'] = GangaFileItem(defvalue=[], typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'], sequence=1, doc=docstr)
docstr = 'Ancestor depth to be queried from the Bookkeeping'
schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
docstr = 'Use contents of file rather than generating catalog.'
schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
docstr = 'Specify the dataset persistency technology'
schema['persistency'] = SimpleItem(
defvalue=None, typelist=['str', 'type(None)'], doc=docstr)
schema['treat_as_inputfiles'] = SimpleItem(defvalue=False, doc="Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")
_schema = Schema(Version(3, 0), schema)
_category = 'datasets'
_name = "LHCbDataset"
_exportmethods = ['getReplicas', '__len__', '__getitem__', 'replicate',
'hasLFNs', 'append', 'extend', 'getCatalog', 'optionsString',
'getLFNs', 'getFileNames', 'getFullFileNames',
'difference', 'isSubset', 'isSuperset', 'intersection',
'symmetricDifference', 'union', 'bkMetadata',
'isEmpty', 'hasPFNs', 'getPFNs'] # ,'pop']
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(this_file))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
raise GangaException("Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
#this_file = self.files[i]
# print type(this_file)
# return this_file
# return this_file
# return this_file
if type(i) == type(slice(0)):
ds = LHCbDataset(files=self.files[i])
ds.depth = self.depth
#ds.XMLCatalogueSlice = self.XMLCatalogueSlice
return ds
else:
return self.files[i]
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error. Could not get replicas.')
return result['Successful']
def hasLFNs(self):
'Returns True is the dataset has LFNs and False otherwise.'
for f in self.files:
if isDiracFile(f):
return True
return False
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate( destSE=destSE )
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError('object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def removeFile(self, input_file):
try:
self.files.remove(input_file)
except:
raise GangaException('Dataset has no file named %s' % input_file.namePattern)
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def getPFNs(self):
'Returns a list of all PFNs (by name) stored in the dataset.'
pfns = []
if not self:
return pfns
for f in self.files:
if isPFN(f):
pfns.append(f.namePattern)
return pfns
def getFullFileNames(self):
'Returns all file names w/ PFN or LFN prepended.'
names = []
from GangaDirac.Lib.Files.DiracFile import DiracFile
for f in self.files:
if isType(f, DiracFile):
names.append('LFN:%s' % f.lfn)
else:
try:
names.append('PFN:%s' % f.namePattern)
except:
logger.warning("Cannot determine filename for: %s " % f)
raise GangaException("Cannot Get File Name")
return names
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None")
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn, dtype_str)
else:
sdatasetsnew += """ \"PFN:%s\",""" % f.namePattern
sdatasetsold += """ \"DATAFILE='PFN:%s' %s\",""" % (f.namePattern, dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if(file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def _checkOtherFiles(self, other ):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def difference(self, other):
'''Returns a new data set w/ files in this that are not in other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def isSubset(self, other):
'''Is every file in this data set in other?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issubset(other_files)
def isSuperset(self, other):
'''Is every file in other in this data set?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issuperset(other_files)
def symmetricDifference(self, other):
'''Returns a new data set w/ files in either this or other but not
both.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).symmetric_difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def intersection(self, other):
'''Returns a new data set w/ files common to this and other.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).intersection(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def union(self, other):
'''Returns a new data set w/ files from this and other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).union(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info("Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info...")
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica. Replica rm error.')
return b
class TestGangaList(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestGangaList, self).__init__(*args, **kwargs)
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
self.ganga_list = GangaList()
@staticmethod
def _makeRandomString():
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
@staticmethod
def _makeRandomTFile():
name = TestGangaList._makeRandomString()
subdir = TestGangaList._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')),
len(self.plain1),
"Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')),
len(self.plain2),
"Something's wrong with construction")
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
self.assertFalse(
missing_methods,
'Not all of lists methods are implemented: %s' % missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
self.assertEqual(
self.proxied1, self.plain1,
'Proxied and non-proxied versions should be the same')
def testNq(self):
self.assertIsNotNone(self.proxied1, None)
self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
self.assertNotEqual(self.proxied1, self.proxied2,
'Lists are different')
self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')
def testNonZero(self):
"""@ExpectedFailure"""
self.assertFalse(
GangaList(),
'An empty GangaList should be false, just like a list')
def testAdd(self):
"""Test __add__"""
self.assertEqual(self.plain1 + self.plain2,
self.proxied1 + self.proxied2)
self.assertTrue(isProxy(self.proxied1 + self.proxied2))
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
self.assertEqual((self.plain1 + self.plain2),
(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1),
(self.plain2 + self.proxied1))
self.assertTrue(isProxy(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1),
(self.plain2 + self.proxied1))
self.assertEqual((self.plain1 + self.plain2),
(self.plain1 + self.proxied2))
self.assertTrue(isinstance(self.plain1 + self.proxied2, list))
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
self.assertTrue(isProxy(self.proxied2[-1]),
'Element access must get proxies')
self.assertFalse(isProxy(self.plain1[0]),
'Element access must not proxies')
self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]),
'File objects should remain proxies')
self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]),
'Objects in plain lists should be left alone')
self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
'File objects should be equal')
self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1],
'File objects should be identical')
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
self.assertEqual(plist, self.proxied1)
for p in plist:
self.assertTrue(isProxy(p))
self.assertIn(p, self.proxied1,
'Proxied list should contain each proxied object')
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
self.assertTrue(isProxy(p))
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
self.assertEqual((self.plain1 >= self.plain2),
(self.proxied1 >= self.proxied2),
'The lists should have the same ge')
self.assertEqual((self.plain2 >= self.plain1),
(self.proxied2 >= self.proxied1),
'The lists should have the same ge')
self.assertNotEqual((self.proxied1 >= self.proxied2),
(self.proxied2 >= self.proxied1),
'The gt should invert correctly')
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
self.assertTrue(isProxy(self.proxied1[i]))
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]],
'Slices {0} should be the same'.format(s))
t = self.plain1[:]
self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
self.assertIsNot(self.plain1[:], t)
t = self.proxied1[:]
self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
self.assertIsNot(self.proxied1[:], t)
def testGT(self):
"""Test __gt__"""
self.assertEqual((self.plain1 > self.plain2),
(self.proxied1 > self.proxied2),
'The lists should have the same gt')
self.assertEqual((self.plain2 > self.plain1),
(self.proxied2 > self.proxied1),
'The lists should have the same gt')
self.assertNotEqual((self.proxied1 > self.proxied2),
(self.proxied2 > self.proxied1),
'The gt should invert correctly')
def testIAdd(self):
"""Test __iadd__"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.proxied2
self.assertEqual(self.plain1, self.proxied1,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.plain2
self.assertEqual(self.plain1, self.proxied1,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
self.assertEqual(self.plain2, self.proxied2,
'Addition should be the same')
self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
self.assertEqual(s1, s2, 'Addition should be the same')
self.assertFalse(isProxy(s1), 'Proxy Not Needed')
self.assertTrue(isProxy(s2), 'Proxy Needed')
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
self.assertIs(self.plain1[4], t)
t = self.proxied1[4]
self.assertIs(self.proxied1[4], t)
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
self.assertEqual(self.plain1, self.proxied1,
'Multiplication should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
self.assertEqual(s1, s2, 'Addition should be the same')
def testLE(self):
"""Test __le__"""
self.assertEqual(self.plain1 <= self.plain2,
self.proxied1 <= self.proxied2,
'The lists should have the same le')
self.assertEqual(self.plain2 <= self.plain1,
self.proxied2 <= self.proxied1,
'The lists should have the same le')
self.assertNotEqual(self.proxied1 <= self.proxied2,
self.proxied2 <= self.proxied1,
'The le should invert correctly')
def testLen(self):
"""Tests __len__"""
self.assertEqual(len(self.plain1), len(self.proxied1),
'Lengths should be the same')
def testLT(self):
"""Test __lt__"""
self.assertEqual(self.plain1 < self.plain2,
self.proxied1 < self.proxied2,
'The lists should have the same lt')
self.assertEqual(self.plain2 < self.plain1,
self.proxied2 < self.proxied1,
'The lists should have the same lt')
self.assertNotEqual(self.proxied1 < self.proxied2,
self.proxied2 < self.proxied1,
'The lt should invert correctly')
def testMul(self):
"""Test __mul__"""
self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
self.assertTrue(isProxy(self.proxied1 * 9))
for p in self.proxied1:
self.assertTrue(isProxy(p))
def testNE(self):
"""Test __ne__"""
self.assertNotEqual(self.plain1, self.plain2)
self.assertNotEqual(self.proxied1, self.proxied2,
'Lists should be different')
self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7],
'Lists should be different')
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
self.assertEqual(t1, t2, 'Multiplication should be the same')
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
self.assertIs(i, self.proxied1[count])
self.assertTrue(isProxy(i))
self.assertTrue(isType(i, TFile))
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
self.assertIs(self.plain1[test_index], t)
self.proxied1[test_index] = t
self.assertIs(self.proxied1[test_index], t)
self.assertTrue(isProxy(self.proxied1[test_index]))
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
self.assertEqual(self.plain1[3:7], self.plain2[3:7],
'The lists should be equal')
self.assertIsNot(self.plain1[3:7], self.plain2[3:7],
'The lists should be copies')
self.plain1[4] = self.plain2[9]
self.assertNotEqual(self.plain1[4], self.plain2[4])
tmp = self.plain1[2:9]
self.assertIsNot(self.plain1[2:9], tmp)
self.proxied1[3:7] = self.proxied2[3:7]
self.assertEqual(self.proxied1[3:7], self.proxied2[3:7],
'The lists should be equal')
self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7],
'The lists should be copies')
self.proxied1[4] = self.proxied2[9]
self.assertNotEqual(self.proxied1[4], self.proxied2[4])
tmp = self.proxied1[2:9]
self.assertIsNot(self.proxied1[2:9], tmp)
def testAppend(self):
t = TFile(name='foo')
self.plain1.append(t)
self.assertEqual(self.plain1[-1], t)
self.assertEqual(self.plain1.pop(), t)
self.proxied1.append(t)
self.assertEqual(self.proxied1[-1], t)
self.assertIs(self.proxied1[-1], t, 'Identity Test')
self.assertTrue(isProxy(self.proxied1[-1]),
'Make sure we get back a proxy')
self.assertEqual(self.proxied1.pop(), t)
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
self.assertEqual(self.plain1, self.proxied1,
'Lists should be the same')
t2 = self.proxied1[4:7]
self.assertTrue(isProxy(t2))
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
self.assertTrue(isType(t2, glist))
self.plain1.extend(t2)
self.proxied1.extend(t2)
self.assertEqual(self.plain1, self.proxied1,
'Lists should be the same')
def testIndex(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
self.assertEqual(self.proxied1.index(t), 8)
def testInsert(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
t = self.proxied1[6]
r = self.proxied1.pop(6)
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 2)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testRemove(self):
t = TFile(name='bar')
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testIter(self):
count = 0
for f in self.proxied1:
count += 1
self.assertTrue(isProxy(f))
self.assertTrue(isType(f, TFile))
self.assertEqual(count, len(self.proxied1), 'Must visit every member')
def testCmp(self):
self.assertEqual(cmp(self.proxied1, self.proxied2),
cmp(self.plain1, self.plain2))
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
def __atomic_set__(self, _obj, _val):
## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed
## _obj: parent class which 'owns' the attribute
## _val: value of the attribute which we're about to set
#if hasattr(_obj, getName(self)):
# if not isinstance(getattr(_obj, getName(self)), GangaObject):
# if type( getattr(_obj, getName(self)) ) == type(_val):
# object.__setattr__(_obj, getName(self), deepcopy(_val))
# return
#
# if not isinstance(_obj, GangaObject) and type(_obj) == type(_val):
# _obj = deepcopy(_val)
# return
obj = _obj
temp_val = _val
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if hasattr(obj, '_checkset_name'):
checkSet = self._bind_method(obj, self._checkset_name)
if checkSet is not None:
checkSet(temp_val)
if hasattr(obj, '_filter_name'):
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(temp_val)
else:
val = temp_val
else:
val = temp_val
# LOCKING
obj._getWriteAccess()
#self._check_getter()
item = obj._schema[getName(self)]
def cloneVal(v):
GangaList = _getGangaList()
if isinstance(v, (list, tuple, GangaList)):
new_v = GangaList()
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
return new_v
else:
return self.__cloneVal(v, obj)
## If the item has been defined as a sequence great, let's continue!
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
GangaList = _getGangaList()
new_val = GangaList()
else:
if isinstance(item, Schema.ComponentItem):
new_val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
new_val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
## Else we need to work out what we've got.
if isinstance(item, Schema.ComponentItem):
GangaList = _getGangaList()
if isinstance(val, (list, tuple, GangaList)):
## Can't have a GangaList inside a GangaList easily so lets not
if isinstance(_obj, GangaList):
newListObj = []
else:
newListObj = GangaList()
self.__createNewList(newListObj, val, cloneVal)
#for elem in val:
# newListObj.append(cloneVal(elem))
new_val = newListObj
else:
new_val = cloneVal(val)
else:
new_val = val
pass
#val = deepcopy(val)
if isinstance(new_val, Node):
new_val._setParent(obj)
obj.setNodeAttribute(getName(self), new_val)
obj._setDirty()
def __init__(self, files=[], persistency=None, depth=0):
new_files = GangaList()
if isType(files, LHCbDataset):
for this_file in files:
new_files.append(copy.deepcopy(this_file))
elif isType(files, IGangaFile):
new_files.append(copy.deepcopy(this_file))
elif type(files) == type([]):
for this_file in files:
if type(this_file) == type(''):
new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
elif isType(this_file, IGangaFile):
new_files.append(this_file, False)
else:
new_files.append(strToDataFile(this_file))
elif type(files) == type(''):
new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
from Ganga.Core.exceptions import GangaException
raise GangaException("Unknown object passed to LHCbDataset constructor!")
new_files._setParent(self)
super(LHCbDataset, self).__init__()
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % files )
self.files = new_files
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def put(self, lfn='', force=False, uploadSE="", replicate=False):
"""
Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
File will be uploaded to the first SE that the upload command succeeds for.
The file is uploaded to the SE described by the DiracFile.defaultSE attribute
Alternatively, the user can specify an uploadSE which contains an SE
which the file is to be uploaded to.
If the user wants to replicate this file(s) across all SE then they should state replicate = True.
Return value will be either the stdout from the dirac upload command if not
using the wildcard characters '*?[]' in the namePattern.
If the wildcard characters are used then the return value will be a list containing
newly created DiracFile objects which were the result of glob-ing the wildcards.
The objects in this list will have been uploaded or had their failureReason attribute populated if the
upload failed.
"""
if self.lfn != "" and force == False and lfn == '':
logger.warning(
"Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s"
% (self.namePattern, self.lfn))
decision = raw_input('y / [n]:')
while not (decision.lower() in ['y', 'n']
or decision.lower() == ''):
decision = raw_input('y / [n]:')
if decision.lower() == 'y':
pass
else:
return
if (lfn != '' and self.lfn != '') and force == False:
logger.warning(
"Warning you're attempting to put this DiracFile: %s" %
self.namePattern)
logger.warning("It currently has an LFN associated with it: %s" %
self.lfn)
logger.warning(
"Do you want to continue and attempt to upload to: %s" % lfn)
decision = raw_input('y / [n]:')
while not (decision.lower() in ['y', 'n', '']):
decision = raw_input('y / [n]:')
if decision.lower() == 'y':
pass
else:
return
if lfn and os.path.basename(lfn) != self.namePattern:
logger.warning(
"Changing namePattern from: '%s' to '%s' during put operation"
% (self.namePattern, os.path.basename(lfn)))
if lfn:
self.lfn = lfn
# looks like will only need this for the interactive uploading of jobs.
# Also if any backend need dirac upload on client then when downloaded
# this will upload then delete the file.
if self.namePattern == "":
if self.lfn != '':
logger.warning(
"'Put'-ing a file with ONLY an existing LFN makes no sense!"
)
raise GangaFileError(
'Can\'t upload a file without a local file name.')
sourceDir = self.localDir
if self.localDir is None:
sourceDir = os.getcwd()
# attached to a job, use the joboutputdir
if self._parent != None and os.path.isdir(
self.getJobObject().outputdir):
sourceDir = self.getJobObject().outputdir
if not os.path.isdir(sourceDir):
raise GangaFileError(
'localDir attribute is not a valid dir, don\'t know from which dir to take the file'
)
if regex.search(self.namePattern) is not None:
if self.lfn != "":
logger.warning(
"Cannot specify a single lfn for a wildcard namePattern")
logger.warning("LFN will be generated automatically")
self.lfn = ""
if not self.remoteDir:
try:
job = self.getJobObject()
lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('/'),
"OutputFiles")
except AssertionError:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
lfn_folder = os.path.join('GangaFiles_%s' % this_date)
lfn_base = os.path.join(
DiracFile.diracLFNBase(self.credential_requirements),
lfn_folder)
else:
lfn_base = os.path.join(
DiracFile.diracLFNBase(self.credential_requirements),
self.remoteDir)
if uploadSE == "":
if self.defaultSE != "":
storage_elements = [self.defaultSE]
else:
if configDirac['allDiracSE']:
storage_elements = [
random.choice(configDirac['allDiracSE'])
]
else:
raise GangaFileError(
"Can't upload a file without a valid defaultSE or storageSE, please provide one"
)
elif isinstance(uploadSE, list):
storage_elements = uploadSE
else:
storage_elements = [uploadSE]
outputFiles = GangaList()
for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
name = this_file
if not os.path.exists(name):
if not self.compressed:
raise GangaFileError(
'Cannot upload file. File "%s" must exist!' % name)
name += '.gz'
if not os.path.exists(name):
raise GangaFileError('File "%s" must exist!' % name)
else:
if self.compressed:
os.system('gzip -c %s > %s.gz' % (name, name))
name += '.gz'
if not os.path.exists(name):
raise GangaFileError('File "%s" must exist!' % name)
lfn = os.path.join(lfn_base, os.path.basename(this_file))
d = DiracFile()
d.namePattern = os.path.basename(name)
d.compressed = self.compressed
d.localDir = sourceDir
stderr = ''
stdout = ''
logger.info('Uploading file \'%s\' to \'%s\' as \'%s\'' %
(name, storage_elements[0], lfn))
logger.debug('execute: uploadFile("%s", "%s", %s)' %
(lfn, os.path.join(sourceDir,
name), str([storage_elements[0]])))
try:
stdout = execute('uploadFile("%s", "%s", %s)' %
(lfn, os.path.join(sourceDir, name),
str([storage_elements[0]])),
cred_req=self.credential_requirements)
except GangaDiracError as err:
logger.warning("Couldn't upload file '%s': \'%s\'" %
(os.path.basename(name), err))
failureReason = "Error in uploading file '%s' : '%s'" % (
os.path.basename(name), err)
if regex.search(self.namePattern) is not None:
d.failureReason = failureReason
outputFiles.append(d)
continue
self.failureReason += '\n' + failureReason
continue
stdout_temp = stdout.get('Successful')
if not stdout_temp:
msg = "Couldn't upload file '%s': \'%s\'" % (
os.path.basename(name), stdout)
logger.warning(msg)
if regex.search(self.namePattern) is not None:
d.failureReason = msg
outputFiles.append(d)
continue
self.failureReason = msg
continue
else:
lfn_out = stdout_temp[lfn]
# when doing the two step upload delete the temp file
if self.compressed or self._parent != None:
os.remove(name)
# need another eval as datetime needs to be included.
guid = lfn_out.get('GUID', '')
if regex.search(self.namePattern) is not None:
d.lfn = lfn
d.remoteDir = os.path.dirname(lfn)
d.locations = lfn_out.get('allDiracSE', '')
d.guid = guid
outputFiles.append(d)
continue
else:
self.lfn = lfn
self.remoteDir = os.path.dirname(lfn)
self.locations = lfn_out.get('allDiracSE', '')
self.guid = guid
if replicate == True:
if len(outputFiles) == 1 or len(outputFiles) == 0:
storage_elements.pop(0)
for se in storage_elements:
self.replicate(se)
else:
storage_elements.pop(0)
for this_file in outputFiles:
for se in storage_elements:
this_file.replicate(se)
if len(outputFiles) > 0:
return outputFiles
else:
outputFiles.append(self)
return outputFiles
def _create_subjob(self, job, dataset):
logger.debug("_create_subjob")
datatmp = []
logger.debug("dataset size: %s" % str(len(dataset)))
#logger.debug( "dataset: %s" % str(dataset) )
from GangaLHCb.Lib.LHCbDataset.LHCbDataset import LHCbDataset
if isinstance(dataset, LHCbDataset):
for i in dataset:
if isType(i, DiracFile):
datatmp.append(i)
else:
logger.error(
"Unkown file-type %s, cannot perform split with file %s"
% (type(i), str(i)))
from Ganga.Core.exceptions import GangaException
raise GangaException(
"Unkown file-type %s, cannot perform split with file %s"
% (type(i), str(i)))
elif type(dataset) == type([]) or isType(dataset, GangaList()):
for this_file in dataset:
if type(this_file) is str:
datatmp.append(allComponentFilters['gangafiles'](this_file,
None))
elif isType(this_file, IGangaFile):
datatmp.append(this_file)
else:
logger.error("Unexpected type: %s" % str(type(this_file)))
logger.error("Wanted object to inherit from type: %s: %s" %
(str(type(IGangaFile()))))
from Ganga.Core.exceptions import GangaException
x = GangaException("Unknown(unexpected) file object: %s" %
this_file)
raise x
elif type(dataset) is str:
datatmp.append(DiracFile(lfn=dataset))
else:
logger.error("Unkown dataset type, cannot perform split here")
from Ganga.Core.exceptions import GangaException
logger.error("Dataset found: " + str(dataset))
raise GangaException(
"Unkown dataset type, cannot perform split here")
logger.debug("Creating new Job in Splitter")
j = Job()
logger.debug("Copying From Job")
j.copyFrom(
stripProxy(job),
['splitter', 'subjobs', 'inputdata', 'inputsandbox', 'inputfiles'])
logger.debug("Unsetting Splitter")
j.splitter = None
#logger.debug("Unsetting Merger")
#j.merger = None
#j.inputsandbox = [] ## master added automatically
#j.inputfiles = []
logger.debug("Setting InputData")
j.inputdata = LHCbDataset(files=datatmp[:],
persistency=self.persistency,
depth=self.depth)
#j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
logger.debug("Returning new subjob")
return j
class TestGangaList(GangaGPITestCase):
def __init__(self):
self.ganga_list = None
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
def _makeRandomString(self):
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
def _makeRandomTFile(self):
name = self._makeRandomString()
subdir = self._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
self.ganga_list = GangaList()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
assert len(getProxyAttr(self.proxied1, '_list')) == len(
self.plain1), 'Somthings wrong with construction'
assert len(getProxyAttr(self.proxied2, '_list')) == len(
self.plain2), 'Somthings wrong with construction'
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
assert not missing_methods, \
'Not all of lists methods are implemented: %s' % str(
missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
assert self.proxied1 == self.plain1, 'Proxied and non-proxied versions should be the same'
def testNq(self):
assert self.proxied1 != None
assert not self.proxied1 != self.proxied1, 'Lists are the same'
assert self.proxied1 != self.proxied2, 'Lists are different'
assert self.plain1 != self.proxied2, 'Lists are different'
def testNonZero(self):
"""@ExpectedFailure"""
assert not GangaList(
), 'An empty GangaList should be false, just like a list'
def testAdd(self):
"""Test __add__"""
assert (self.plain1 + self.plain2) == (self.proxied1 + self.proxied2)
assert isProxy(self.proxied1 + self.proxied2)
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
assert (self.plain1 + self.plain2) == (self.proxied1 + self.plain2)
assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
assert isProxy(self.proxied1 + self.plain2)
assert (self.plain2 + self.plain1) == (self.plain2 + self.proxied1)
assert (self.plain1 + self.plain2) == (self.plain1 + self.proxied2)
assert isinstance(self.plain1 + self.proxied2, list)
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
assert isProxy(self.proxied2[-1]), 'Element access must get proxies'
assert not isProxy(self.plain1[0]), 'Element access must not proxies'
assert isProxy(
(self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies'
assert not isProxy(
(self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone'
assert (
self.plain1 + self.proxied2)[-1] == self.proxied2[-1], 'File objects should be equal'
assert (
self.plain1 + self.proxied2)[-1] is self.proxied2[-1], 'File objects should be identical'
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
assert plist == self.proxied1
for p in plist:
assert isProxy(p)
assert p in self.proxied1, 'Proxied list should contain each proxied object'
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
assert isProxy(p)
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
assert (self.plain1 >= self.plain2) == (
self.proxied1 >= self.proxied2), 'The lists should have the same ge'
assert (self.plain2 >= self.plain1) == (
self.proxied2 >= self.proxied1), 'The lists should have the same ge'
assert (self.proxied1 >= self.proxied2) != (
self.proxied2 >= self.proxied1), 'The gt should invert correctly'
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
assert isProxy(self.proxied1[i])
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
assert self.plain1[s[0]:s[1]] == self.proxied1[
s[0]:s[1]], 'Slices %s should be the same' % str(s)
t = self.plain1[:]
assert t is not self.plain1, 'Slice should be a copy.'
assert self.plain1[:] is not t
t = self.proxied1[:]
assert t is not self.proxied1, 'Slice should be a copy.'
assert self.proxied1[:] is not t
def testGT(self):
"""Test __gt__"""
assert (self.plain1 > self.plain2) == (
self.proxied1 > self.proxied2), 'The lists should have the same gt'
assert (self.plain2 > self.plain1) == (
self.proxied2 > self.proxied1), 'The lists should have the same gt'
assert (self.proxied1 > self.proxied2) != (
self.proxied2 > self.proxied1), 'The gt should invert correctly'
def testIAdd(self):
"""Test __iadd__"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.proxied2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
assert isProxy(self.proxied1), 'Must be Proxy'
assert isProxy(self.proxied2), 'Must be Proxy'
self.plain1 += self.plain2
self.proxied1 += self.plain2
assert self.plain1 == self.proxied1, 'Addition should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
assert isinstance(self.plain1, list), 'Must be list instance'
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
assert self.plain2 == self.proxied2, 'Addition should be the same'
assert isProxy(self.proxied2), 'Proxy must be added'
assert isinstance(self.plain2, list), 'Must be list instance'
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
assert s1 == s2, 'Addition should be the same'
assert not isProxy(s1), 'Proxy Not Needed'
assert isProxy(s2), 'Proxy Needed'
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
assert self.plain1[4] is t
t = self.proxied1[4]
assert self.proxied1[4] is t
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
assert self.plain1 == self.proxied1, 'Multiplication should be the same'
assert isProxy(self.proxied1), 'Proxy must be added'
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
assert s1 == s2, 'Addition should be the same'
def testLE(self):
"""Test __le__"""
assert (self.plain1 <= self.plain2) == (
self.proxied1 <= self.proxied2), 'The lists should have the same le'
assert (self.plain2 <= self.plain1) == (
self.proxied2 <= self.proxied1), 'The lists should have the same le'
assert (self.proxied1 <= self.proxied2) != (
self.proxied2 <= self.proxied1), 'The le should invert correctly'
def testLen(self):
"""Tests __len__"""
assert len(self.plain1) == len(
self.proxied1), 'Lengths should be the same'
def testLT(self):
"""Test __lt__"""
assert (self.plain1 < self.plain2) == (
self.proxied1 < self.proxied2), 'The lists should have the same lt'
assert (self.plain2 < self.plain1) == (
self.proxied2 < self.proxied1), 'The lists should have the same lt'
assert (self.proxied1 < self.proxied2) != (
self.proxied2 < self.proxied1), 'The lt should invert correctly'
def testMul(self):
"""Test __mul__"""
assert (self.plain1 * 7) == (self.proxied1 * 7)
assert isProxy(self.proxied1 * 9)
for p in self.proxied1:
assert isProxy(p)
def testNE(self):
"""Test __ne__"""
assert self.plain1 != self.plain2
assert self.proxied1 != self.proxied2, 'Lists should be different'
assert self.plain1[0:5] != self.plain1[2:7]
assert self.proxied1[0:5] != self.proxied1[
2:7], 'Lists should be different'
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
assert t1 == t2, 'Multiplication should be the same'
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
from TFile import TFile as tF
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
assert i is self.proxied1[count]
assert isProxy(i)
assert isType(i, tF)
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
assert self.plain1[test_index] is t
self.proxied1[test_index] = t
assert self.proxied1[test_index] is t
assert isProxy(self.proxied1[test_index])
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
assert self.plain1[3:7] == self.plain2[
3:7], 'The lists should be equal'
assert self.plain1[3:7] is not self.plain2[
3:7], 'The lists should be copies'
self.plain1[4] = self.plain2[9]
assert self.plain1[4] != self.plain2[4]
tmp = self.plain1[2:9]
assert self.plain1[2:9] is not tmp
self.proxied1[3:7] = self.proxied2[3:7]
assert self.proxied1[3:7] == self.proxied2[
3:7], 'The lists should be equal'
assert self.proxied1[3:7] is not self.proxied2[
3:7], 'The lists should be copies'
self.proxied1[4] = self.proxied2[9]
assert self.proxied1[4] != self.proxied2[4]
tmp = self.proxied1[2:9]
assert self.proxied1[2:9] is not tmp
def testAppend(self):
t = addProxy(TFile(name='foo'))
self.plain1.append(t)
assert self.plain1[-1] == t
assert self.plain1.pop() == t
self.proxied1.append(t)
assert self.proxied1[-1] == t
assert self.proxied1[-1] is t, 'Identity Test'
assert isProxy(self.proxied1[-1]), 'Make sure we get back a proxy'
assert self.proxied1.pop() == t
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
assert self.plain1 == self.proxied1, 'Lists should be the same'
t2 = self.proxied1[4:7]
assert isProxy(t2)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
assert isType(t2, glist)
self.plain1.extend(t2)
self.proxied1.extend(t2)
assert self.plain1 == self.proxied1, 'Lists should be the same'
def testIndex(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
assert self.proxied1.index(t) == 8
def testInsert(self):
t = addProxy(TFile(name='foo'))
self.proxied1.insert(8, t)
assert self.proxied1[8] == t
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
t = self.proxied1[6]
r = self.proxied1.pop(6)
assert t == r
assert t is r
assert len(self.proxied1) == list_len - 2
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testRemove(self):
t = addProxy(TFile(name='bar'))
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
assert len(self.proxied1) == list_len - 1
assert t not in self.proxied1
assert t._impl not in self.proxied1._impl
def testIter(self):
from TFile import TFile as tF
count = 0
for f in self.proxied1:
count += 1
assert isProxy(f)
assert isType(f, tF)
assert count == len(self.proxied1), 'Must visit every member'
def testCmp(self):
assert cmp(self.proxied1, self.proxied2) == cmp(
self.plain1, self.plain2)
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
def put(self, lfn='', force=False, uploadSE="", replicate=False):
"""
Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
File will be uploaded to the first SE that the upload command succeeds for.
The file is uploaded to the SE described by the DiracFile.defaultSE attribute
Alternatively, the user can specify an uploadSE which contains an SE
which the file is to be uploaded to.
If the user wants to replicate this file(s) across all SE then they should state replicate = True.
Return value will be either the stdout from the dirac upload command if not
using the wildcard characters '*?[]' in the namePattern.
If the wildcard characters are used then the return value will be a list containing
newly created DiracFile objects which were the result of glob-ing the wildcards.
The objects in this list will have been uploaded or had their failureReason attribute populated if the
upload failed.
"""
if self.lfn != "" and force == False and lfn == '':
logger.warning("Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s" % (self.namePattern, self.lfn))
decision = raw_input('y / [n]:')
while not (decision.lower() in ['y', 'n'] or decision.lower() == ''):
decision = raw_input('y / [n]:')
if decision.lower() == 'y':
pass
else:
return
if (lfn != '' and self.lfn != '') and force == False:
logger.warning("Warning you're attempting to put this DiracFile: %s" % self.namePattern)
logger.warning("It currently has an LFN associated with it: %s" % self.lfn)
logger.warning("Do you want to continue and attempt to upload to: %s" % lfn)
decision = raw_input('y / [n]:')
while not (decision.lower() in ['y', 'n', '']):
decision = raw_input('y / [n]:')
if decision.lower() == 'y':
pass
else:
return
if lfn and os.path.basename(lfn) != self.namePattern:
logger.warning("Changing namePattern from: '%s' to '%s' during put operation" % (self.namePattern, os.path.basename(lfn)))
if lfn:
self.lfn = lfn
# looks like will only need this for the interactive uploading of jobs.
# Also if any backend need dirac upload on client then when downloaded
# this will upload then delete the file.
if self.namePattern == "":
if self.lfn != '':
logger.warning("'Put'-ing a file with ONLY an existing LFN makes no sense!")
raise GangaFileError('Can\'t upload a file without a local file name.')
sourceDir = self.localDir
if self.localDir is None:
sourceDir = os.getcwd()
# attached to a job, use the joboutputdir
if self._parent != None and os.path.isdir(self.getJobObject().outputdir):
sourceDir = self.getJobObject().outputdir
if not os.path.isdir(sourceDir):
raise GangaFileError('localDir attribute is not a valid dir, don\'t know from which dir to take the file')
if regex.search(self.namePattern) is not None:
if self.lfn != "":
logger.warning("Cannot specify a single lfn for a wildcard namePattern")
logger.warning("LFN will be generated automatically")
self.lfn = ""
if not self.remoteDir:
try:
job = self.getJobObject()
lfn_folder = os.path.join("GangaJob_%s" % job.getFQID('/'), "OutputFiles")
except AssertionError:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
lfn_folder = os.path.join('GangaFiles_%s' % this_date)
lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), lfn_folder)
else:
lfn_base = os.path.join(DiracFile.diracLFNBase(self.credential_requirements), self.remoteDir)
if uploadSE == "":
if self.defaultSE != "":
storage_elements = [self.defaultSE]
else:
if configDirac['allDiracSE']:
storage_elements = [random.choice(configDirac['allDiracSE'])]
else:
raise GangaFileError("Can't upload a file without a valid defaultSE or storageSE, please provide one")
elif isinstance(uploadSE, list):
storage_elements = uploadSE
else:
storage_elements = [uploadSE]
outputFiles = GangaList()
for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
name = this_file
if not os.path.exists(name):
if not self.compressed:
raise GangaFileError('Cannot upload file. File "%s" must exist!' % name)
name += '.gz'
if not os.path.exists(name):
raise GangaFileError('File "%s" must exist!' % name)
else:
if self.compressed:
os.system('gzip -c %s > %s.gz' % (name, name))
name += '.gz'
if not os.path.exists(name):
raise GangaFileError('File "%s" must exist!' % name)
lfn = os.path.join(lfn_base, os.path.basename(this_file))
d = DiracFile()
d.namePattern = os.path.basename(name)
d.compressed = self.compressed
d.localDir = sourceDir
stderr = ''
stdout = ''
logger.info('Uploading file \'%s\' to \'%s\' as \'%s\'' % (name, storage_elements[0], lfn))
logger.debug('execute: uploadFile("%s", "%s", %s)' % (lfn, os.path.join(sourceDir, name), str([storage_elements[0]])))
try:
stdout = execute('uploadFile("%s", "%s", %s)' % (lfn, os.path.join(sourceDir, name), str([storage_elements[0]])), cred_req=self.credential_requirements)
except GangaDiracError as err:
logger.warning("Couldn't upload file '%s': \'%s\'" % (os.path.basename(name), err))
failureReason = "Error in uploading file '%s' : '%s'" % (os.path.basename(name), err)
if regex.search(self.namePattern) is not None:
d.failureReason = failureReason
outputFiles.append(d)
continue
self.failureReason += '\n' + failureReason
continue
stdout_temp = stdout.get('Successful')
if not stdout_temp:
msg = "Couldn't upload file '%s': \'%s\'" % (os.path.basename(name), stdout)
logger.warning(msg)
if regex.search(self.namePattern) is not None:
d.failureReason = msg
outputFiles.append(d)
continue
self.failureReason = msg
continue
else:
lfn_out = stdout_temp[lfn]
# when doing the two step upload delete the temp file
if self.compressed or self._parent != None:
os.remove(name)
# need another eval as datetime needs to be included.
guid = lfn_out.get('GUID', '')
if regex.search(self.namePattern) is not None:
d.lfn = lfn
d.remoteDir = os.path.dirname(lfn)
d.locations = lfn_out.get('allDiracSE', '')
d.guid = guid
outputFiles.append(d)
continue
else:
self.lfn = lfn
self.remoteDir = os.path.dirname(lfn)
self.locations = lfn_out.get('allDiracSE', '')
self.guid = guid
if replicate == True:
if len(outputFiles) == 1 or len(outputFiles) == 0:
storage_elements.pop(0)
for se in storage_elements:
self.replicate(se)
else:
storage_elements.pop(0)
for this_file in outputFiles:
for se in storage_elements:
this_file.replicate(se)
if len(outputFiles) > 0:
return outputFiles
else:
outputFiles.append(self)
return outputFiles
class LHCbDataset(GangaDataset):
'''Class for handling LHCb data sets (i.e. inputdata for LHCb jobs).
Example Usage:
ds = LHCbDataset(["lfn:/some/lfn.file","pfn:/some/pfn.file"])
ds[0] # DiracFile("/some/lfn.file") - see DiracFile docs for usage
ds[1] # PhysicalFile("/some/pfn.file")- see PhysicalFile docs for usage
len(ds) # 2 (number of files)
ds.getReplicas() # returns replicas for *all* files in the data set
ds.replicate("CERN-USER") # replicate *all* LFNs to "CERN-USER" SE
ds.getCatalog() # returns XML catalog slice
ds.optionsString() # returns Gaudi-sytle options
[...etc...]
'''
schema = {}
docstr = 'List of PhysicalFile and DiracFile objects'
schema['files'] = GangaFileItem(
defvalue=[],
typelist=['str', 'Ganga.GPIDev.Adapters.IGangaFile.IGangaFile'],
sequence=1,
doc=docstr)
docstr = 'Ancestor depth to be queried from the Bookkeeping'
schema['depth'] = SimpleItem(defvalue=0, doc=docstr)
docstr = 'Use contents of file rather than generating catalog.'
schema['XMLCatalogueSlice'] = GangaFileItem(defvalue=None, doc=docstr)
docstr = 'Specify the dataset persistency technology'
schema['persistency'] = SimpleItem(defvalue=None,
typelist=['str', 'type(None)'],
doc=docstr)
schema['treat_as_inputfiles'] = SimpleItem(
defvalue=False,
doc=
"Treat the inputdata as inputfiles, i.e. copy the inputdata to the WN")
_schema = Schema(Version(3, 0), schema)
_category = 'datasets'
_name = "LHCbDataset"
_exportmethods = [
'getReplicas', '__len__', '__getitem__', 'replicate', 'hasLFNs',
'append', 'extend', 'getCatalog', 'optionsString', 'getLFNs',
'getFileNames', 'getFullFileNames', 'difference', 'isSubset',
'isSuperset', 'intersection', 'symmetricDifference', 'union',
'bkMetadata', 'isEmpty', 'hasPFNs', 'getPFNs'
] # ,'pop']
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(files))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(
this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(files, None),
False)
else:
raise GangaException(
"Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def __getitem__(self, i):
'''Proivdes scripting (e.g. ds[2] returns the 3rd file) '''
#this_file = self.files[i]
# print type(this_file)
# return this_file
# return this_file
# return this_file
if type(i) == type(slice(0)):
ds = LHCbDataset(files=self.files[i])
ds.depth = self.depth
#ds.XMLCatalogueSlice = self.XMLCatalogueSlice
return ds
else:
return self.files[i]
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error. Could not get replicas.')
return result['Successful']
def hasLFNs(self):
'Returns True is the dataset has LFNs and False otherwise.'
for f in self.files:
if isDiracFile(f):
return True
return False
def hasPFNs(self):
'Returns True is the dataset has PFNs and False otherwise.'
for f in self.files:
if not isDiracFile(f):
return True
return False
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError(
'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now'
% (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(
files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(
string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def removeFile(self, input_file):
try:
self.files.remove(input_file)
except:
raise GangaException('Dataset has no file named %s' %
input_file.namePattern)
def getLFNs(self):
'Returns a list of all LFNs (by name) stored in the dataset.'
lfns = []
if not self:
return lfns
for f in self.files:
if isDiracFile(f):
subfiles = f.getSubFiles()
if len(subfiles) == 0:
lfns.append(f.lfn)
else:
for file in subfiles:
lfns.append(file.lfn)
#logger.debug( "Returning LFNS:\n%s" % str(lfns) )
logger.debug("Returning #%s LFNS" % str(len(lfns)))
return lfns
def getPFNs(self):
'Returns a list of all PFNs (by name) stored in the dataset.'
pfns = []
if not self:
return pfns
for f in self.files:
if isPFN(f):
pfns.append(f.namePattern)
return pfns
def getFullFileNames(self):
'Returns all file names w/ PFN or LFN prepended.'
names = []
from GangaDirac.Lib.Files.DiracFile import DiracFile
for f in self.files:
if isType(f, DiracFile):
names.append('LFN:%s' % f.lfn)
else:
try:
names.append('PFN:%s' % f.namePattern)
except:
logger.warning("Cannot determine filename for: %s " % f)
raise GangaException("Cannot Get File Name")
return names
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def optionsString(self, file=None):
'Returns the Gaudi-style options string for the dataset (if a filename' \
' is given, the file is created and output is written there).'
if not self or len(self) == 0:
return ''
snew = ''
if self.persistency == 'ROOT':
snew = '\n#new method\nfrom GaudiConf import IOExtension\nIOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == 'POOL':
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension(\"%s\").inputFiles([' % self.persistency
elif self.persistency == None:
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
else:
logger.warning(
"Unknown LHCbDataset persistency technology... reverting to None"
)
snew = '\ntry:\n #new method\n from GaudiConf import IOExtension\n IOExtension().inputFiles(['
sold = '\nexcept ImportError:\n #Use previous method\n from Gaudi.Configuration import EventSelector\n EventSelector().Input=['
sdatasetsnew = ''
sdatasetsold = ''
dtype_str_default = getConfig('LHCb')['datatype_string_default']
dtype_str_patterns = getConfig('LHCb')['datatype_string_patterns']
for f in self.files:
dtype_str = dtype_str_default
for this_str in dtype_str_patterns:
matched = False
for pat in dtype_str_patterns[this_str]:
if fnmatch.fnmatch(f.namePattern, pat):
dtype_str = this_str
matched = True
break
if matched:
break
sdatasetsnew += '\n '
sdatasetsold += '\n '
if isDiracFile(f):
sdatasetsnew += """ \"LFN:%s\",""" % f.lfn
sdatasetsold += """ \"DATAFILE='LFN:%s' %s\",""" % (f.lfn,
dtype_str)
else:
sdatasetsnew += """ \"%s\",""" % f.accessURL()[0]
sdatasetsold += """ \"DATAFILE='%s' %s\",""" % (
f.accessURL()[0], dtype_str)
if sdatasetsold.endswith(","):
if self.persistency == 'ROOT':
sdatasetsnew = sdatasetsnew[:-1] + """\n], clear=True)"""
else:
sdatasetsnew = sdatasetsnew[:-1] + """\n ], clear=True)"""
sdatasetsold = sdatasetsold[:-1]
sdatasetsold += """\n ]"""
if (file):
f = open(file, 'w')
if self.persistency == 'ROOT':
f.write(snew)
f.write(sdatasetsnew)
else:
f.write(snew)
f.write(sdatasetsnew)
f.write(sold)
f.write(sdatasetsold)
f.close()
else:
if self.persistency == 'ROOT':
return snew + sdatasetsnew
else:
return snew + sdatasetsnew + sold + sdatasetsold
def _checkOtherFiles(self, other):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def difference(self, other):
'''Returns a new data set w/ files in this that are not in other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def isSubset(self, other):
'''Is every file in this data set in other?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issubset(other_files)
def isSuperset(self, other):
'''Is every file in other in this data set?'''
other_files = self._checkOtherFiles(other)
return set(self.getFileNames()).issuperset(other_files)
def symmetricDifference(self, other):
'''Returns a new data set w/ files in either this or other but not
both.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).symmetric_difference(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def intersection(self, other):
'''Returns a new data set w/ files common to this and other.'''
other_files = other._checkOtherFiles(other)
files = set(self.getFullFileNames()).intersection(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def union(self, other):
'''Returns a new data set w/ files from this and other.'''
other_files = self._checkOtherFiles(other)
files = set(self.getFullFileNames()).union(other_files)
data = LHCbDataset()
data.extend(list(files))
data.depth = self.depth
return data
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info(
"Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info..."
)
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica. Replica rm error.')
return b
def put(self, lfn='', force=False, uploadSE="", replicate=False):
"""
Try to upload file sequentially to storage elements defined in configDirac['allDiracSE'].
File will be uploaded to the first SE that the upload command succeeds for.
The file is uploaded to the SE described by the DiracFile.defaultSE attribute
Alternatively, the user can specify an uploadSE which contains an SE
which the file is to be uploaded to.
If the user wants to replicate this file(s) across all SE then they should state replicate = True.
Return value will be either the stdout from the dirac upload command if not
using the wildcard characters '*?[]' in the namePattern.
If the wildcard characters are used then the return value will be a list containing
newly created DiracFile objects which were the result of glob-ing the wildcards.
The objects in this list will have been uploaded or had their failureReason attribute populated if the
upload failed.
"""
if self.lfn != "" and force == False and lfn == '':
logger.warning("Warning you're about to 'put' this DiracFile: %s on the grid as it already has an lfn: %s" % (self.namePattern, self.lfn))
decision = raw_input('y / [n]:')
while not (decision in ['y', 'n'] or decision == ''):
decision = raw_input('y / [n]:')
if decision == 'y':
pass
else:
return
if (lfn != '' and self.lfn != '') and force == False:
logger.warning("Warning you're attempting to put this DiracFile: %s" % self.namePattern)
logger.warning("It currently has an LFN associated with it: %s" % self.lfn)
logger.warning("Do you want to continue and attempt to upload to: %s" % lfn)
decision = raw_input('y / [n]:')
while not (decision in ['y', 'n', '']):
decision = raw_input('y / [n]:')
if decision == 'y':
pass
else:
return
if lfn != '':
self.lfn = lfn
# looks like will only need this for the interactive uploading of jobs.
# Also if any backend need dirac upload on client then when downloaded
# this will upload then delete the file.
if self.namePattern == "":
if self.lfn != '':
logger.warning("'Put'-ing a file with ONLY an existing LFN makes no sense!")
raise GangaException('Can\'t upload a file without a local file name.')
sourceDir = self.localDir
if self.localDir is None:
sourceDir = os.getcwd()
# attached to a job, use the joboutputdir
if self._parent != None and os.path.isdir(self.getJobObject().outputdir):
sourceDir = self.getJobObject().outputdir
if not os.path.isdir(sourceDir):
raise GangaException('localDir attribute is not a valid dir, don\'t know from which dir to take the file')
if regex.search(self.namePattern) is not None:
if self.lfn != "":
logger.warning("Cannot specify a single lfn for a wildcard namePattern")
logger.warning("LFN will be generated automatically")
self.lfn = ""
selfConstructedLFN = False
import glob
if self.remoteDir == '' and self.lfn == '':
import datetime
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
self.lfn = os.path.join(configDirac['DiracLFNBase'], 'GangaFiles_%s' % this_date)
selfConstructedLFN = True
#if self.remoteDir == '' and self.lfn != '':
# self.remoteDir = configDirac['DiracLFNBase']
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
if uploadSE != "":
storage_elements = all_SE_list(uploadSE)
else:
storage_elements = all_SE_list(self.defaultSE)
outputFiles = GangaList()
for this_file in glob.glob(os.path.join(sourceDir, self.namePattern)):
name = this_file
if not os.path.exists(name):
if not self.compressed:
raise GangaException('Cannot upload file. File "%s" must exist!' % name)
name += '.gz'
if not os.path.exists(name):
raise GangaException('File "%s" must exist!' % name)
else:
if self.compressed:
os.system('gzip -c %s > %s.gz' % (name, name))
name += '.gz'
if not os.path.exists(name):
raise GangaException('File "%s" must exist!' % name)
if lfn == "":
lfn = os.path.join(lfn_base, os.path.basename(name))
if selfConstructedLFN is True:
self.lfn = os.path.join(self.lfn, os.path.basename(name))
lfn = self.lfn
d = DiracFile()
d.namePattern = os.path.basename(name)
d.compressed = self.compressed
d.localDir = sourceDir
stderr = ''
stdout = ''
logger.info('Uploading file %s to %s as %s' % (name, storage_elements[0], lfn))
stdout = execute('uploadFile("%s", "%s", %s)' % (lfn, name, str([storage_elements[0]])))
if type(stdout) == str:
logger.warning("Couldn't upload file '%s': %s" % (os.path.basename(name), stdout))
continue
if stdout.get('OK', False) and lfn in stdout.get('Value', {'Successful': {}})['Successful']:
# when doing the two step upload delete the temp file
if self.compressed or self._parent != None:
os.remove(name)
# need another eval as datetime needs to be included.
guid = stdout['Value']['Successful'][lfn].get('GUID', '')
if regex.search(self.namePattern) is not None:
d.lfn = lfn
d.remoteDir = os.path.dirname(lfn)
d.locations = stdout['Value']['Successful'][lfn].get('allDiracSE', '')
d.guid = guid
outputFiles.append(GPIProxyObjectFactory(d))
continue
else:
self.lfn = lfn
self.remoteDir = os.path.dirname(lfn)
self.locations = stdout['Value']['Successful'][lfn].get('allDiracSE', '')
self.guid = guid
# return ## WHY?
else:
failureReason = "Error in uploading file %s : %s" % (os.path.basename(name), str(stdout))
logger.error(failureReason)
if regex.search(self.namePattern) is not None:
d.failureReason = failureReason
outputFiles.append(GPIProxyObjectFactory(d))
continue
self.failureReason = failureReason
return str(stdout)
if replicate == True:
if len(outputFiles) == 1 or len(outputFiles) == 0:
storage_elements.pop(0)
for se in storage_elements:
self.replicate(se)
else:
storage_elements.pop(0)
for this_file in outputFiles:
for se in storage_elements:
this_file.replicate(se)
if len(outputFiles) > 0:
return GPIProxyObjectFactory(outputFiles)
else:
outputFiles.append(self)
return GPIProxyObjectFactory(outputFiles)
class TestGangaList(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestGangaList, self).__init__(*args, **kwargs)
self.plain1 = []
self.plain2 = []
self.proxied1 = []
self.proxied2 = []
self.ganga_list = GangaList()
@staticmethod
def _makeRandomString():
str_len = random.randint(3, 10)
s = ''
for _ in range(str_len):
s += random.choice(string.ascii_letters)
return s
@staticmethod
def _makeRandomTFile():
name = TestGangaList._makeRandomString()
subdir = TestGangaList._makeRandomString()
return TFile(name=name, subdir=subdir)
def setUp(self):
super(TestGangaList, self).setUp()
self.plain1 = [self._makeRandomTFile() for _ in range(15)]
self.plain2 = [self._makeRandomTFile() for _ in range(10)]
self.proxied1 = GangaList()
self.proxied1.extend(self.plain1[:])
self.proxied2 = GangaList()
self.proxied2.extend(self.plain2[:])
t = TFile()
real_t = stripProxy(t)
new_proxy_t = addProxy(real_t)
#hopefully_t = stripProxy(new_proxy_t)
#assert real_t is hopefully_t
assert t is new_proxy_t
self.assertEqual(len(getProxyAttr(self.proxied1, '_list')), len(self.plain1), "Something's wrong with construction")
self.assertEqual(len(getProxyAttr(self.proxied2, '_list')), len(self.plain2), "Something's wrong with construction")
def testAllListMethodsExported(self):
"""Tests that all methods on list are exposed by GangaList"""
def getmethods(_obj):
# get all the method names for an object
return dir(_obj)
list_methods = getmethods([])
gangalist_methods = getmethods(self.ganga_list)
missing_methods = []
for m in list_methods:
if not m in gangalist_methods:
missing_methods.append(m)
if missing_methods:
logger.info(missing_methods)
self.assertFalse(missing_methods, 'Not all of lists methods are implemented: %s' % missing_methods)
def testEq(self):
"""Tests the equality op that the rest of the tests rely on."""
self.assertEqual(self.proxied1, self.plain1, 'Proxied and non-proxied versions should be the same')
def testNq(self):
self.assertIsNotNone(self.proxied1, None)
self.assertFalse(self.proxied1 != self.proxied1, 'Lists are the same')
self.assertNotEqual(self.proxied1, self.proxied2, 'Lists are different')
self.assertNotEqual(self.plain1, self.proxied2, 'Lists are different')
def testNonZero(self):
"""@ExpectedFailure"""
self.assertFalse(GangaList(), 'An empty GangaList should be false, just like a list')
def testAdd(self):
"""Test __add__"""
self.assertEqual(self.plain1 + self.plain2, self.proxied1 + self.proxied2)
self.assertTrue(isProxy(self.proxied1 + self.proxied2))
def testAddMixed(self):
"""Test __add__ with mixed lists and GangaLists"""
self.assertEqual((self.plain1 + self.plain2), (self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
self.assertTrue(isProxy(self.proxied1 + self.plain2))
self.assertEqual((self.plain2 + self.plain1), (self.plain2 + self.proxied1))
self.assertEqual((self.plain1 + self.plain2), (self.plain1 + self.proxied2))
self.assertTrue(isinstance(self.plain1 + self.proxied2, list))
def testAddMixed2(self):
self.plain1 = range(10)
self.plain2 = range(10)
self.assertTrue(isProxy(self.proxied2[-1]), 'Element access must get proxies')
self.assertFalse(isProxy(self.plain1[0]), 'Element access must not proxies')
self.assertTrue(isProxy((self.plain1 + self.proxied2)[-1]), 'File objects should remain proxies')
self.assertFalse(isProxy((self.plain1 + self.proxied2)[0]), 'Objects in plain lists should be left alone')
self.assertEqual((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be equal')
self.assertIs((self.plain1 + self.proxied2)[-1], self.proxied2[-1], 'File objects should be identical')
def testAddStr(self):
"""Makes sure that only lists can be added."""
try:
[] + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
try:
self.proxied1 + ''
assert False, 'Line above should throw a TypeError'
except TypeError:
pass
def testContains(self):
"""Tests __contains__"""
plist = [addProxy(x) for x in self.plain1]
self.assertEqual(plist, self.proxied1)
for p in plist:
self.assertTrue(isProxy(p))
self.assertIn(p, self.proxied1, 'Proxied list should contain each proxied object')
def testDelItem(self):
"""Test __delitem__"""
for p in [addProxy(x) for x in self.plain1[:]]:
self.assertTrue(isProxy(p))
del self.proxied1[self.proxied1.index(p)]
def testGE(self):
"""Test __ge__"""
self.assertEqual((self.plain1 >= self.plain2), (self.proxied1 >= self.proxied2), 'The lists should have the same ge')
self.assertEqual((self.plain2 >= self.plain1), (self.proxied2 >= self.proxied1), 'The lists should have the same ge')
self.assertNotEqual((self.proxied1 >= self.proxied2), (self.proxied2 >= self.proxied1), 'The gt should invert correctly')
def testGetItem(self):
"""Test __getitem__"""
for i in range(len(self.proxied1)):
self.assertTrue(isProxy(self.proxied1[i]))
def testGetSlice(self):
"""Test __getslice__"""
slices = [(0, 0), (0, len(self.plain1))]
for s in slices:
self.assertEqual(self.plain1[s[0]:s[1]], self.proxied1[s[0]:s[1]], 'Slices {0} should be the same'.format(s))
t = self.plain1[:]
self.assertIsNot(t, self.plain1, 'Slice should be a copy.')
self.assertIsNot(self.plain1[:], t)
t = self.proxied1[:]
self.assertIsNot(t, self.proxied1, 'Slice should be a copy.')
self.assertIsNot(self.proxied1[:], t)
def testGT(self):
"""Test __gt__"""
self.assertEqual((self.plain1 > self.plain2), (self.proxied1 > self.proxied2), 'The lists should have the same gt')
self.assertEqual((self.plain2 > self.plain1), (self.proxied2 > self.proxied1), 'The lists should have the same gt')
self.assertNotEqual((self.proxied1 > self.proxied2), (self.proxied2 > self.proxied1), 'The gt should invert correctly')
def testIAdd(self):
"""Test __iadd__"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.proxied2
self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIAddMixed(self):
"""Test __iadd__ where we mix lists and GangaLists"""
self.assertTrue(isProxy(self.proxied1), 'Must be Proxy')
self.assertTrue(isProxy(self.proxied2), 'Must be Proxy')
self.plain1 += self.plain2
self.proxied1 += self.plain2
self.assertEqual(self.plain1, self.proxied1, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
self.assertTrue(isinstance(self.plain1, list), 'Must be list instance')
self.plain2 += self.proxied1
self.proxied2 += self.proxied1
self.assertEqual(self.plain2, self.proxied2, 'Addition should be the same')
self.assertTrue(isProxy(self.proxied2), 'Proxy must be added')
self.assertTrue(isinstance(self.plain2, list), 'Must be list instance')
def testIAddSlice(self):
"""Test __iadd__ on slices"""
s1 = self.plain1[3:7]
s1 += self.plain2[2:5]
s2 = self.proxied1[3:7]
s2 += self.proxied2[2:5]
self.assertEqual(s1, s2, 'Addition should be the same')
self.assertFalse(isProxy(s1), 'Proxy Not Needed')
self.assertTrue(isProxy(s2), 'Proxy Needed')
def testIdentity(self):
"""Tests we obey list like identity relations."""
t = self.plain1[4]
self.assertIs(self.plain1[4], t)
t = self.proxied1[4]
self.assertIs(self.proxied1[4], t)
def testIMul(self):
"""Test __imul__"""
self.plain1 *= 5
self.proxied1 *= 5
self.assertEqual(self.plain1, self.proxied1, 'Multiplication should be the same')
self.assertTrue(isProxy(self.proxied1), 'Proxy must be added')
def testIMulSlice(self):
"""Test __imul__ on slices"""
s1 = self.plain1[3:7]
s1 *= 6
s2 = self.proxied1[3:7]
s2 *= 6
self.assertEqual(s1, s2, 'Addition should be the same')
def testLE(self):
"""Test __le__"""
self.assertEqual(self.plain1 <= self.plain2, self.proxied1 <= self.proxied2, 'The lists should have the same le')
self.assertEqual(self.plain2 <= self.plain1, self.proxied2 <= self.proxied1, 'The lists should have the same le')
self.assertNotEqual(self.proxied1 <= self.proxied2, self.proxied2 <= self.proxied1, 'The le should invert correctly')
def testLen(self):
"""Tests __len__"""
self.assertEqual(len(self.plain1), len(self.proxied1), 'Lengths should be the same')
def testLT(self):
"""Test __lt__"""
self.assertEqual(self.plain1 < self.plain2, self.proxied1 < self.proxied2, 'The lists should have the same lt')
self.assertEqual(self.plain2 < self.plain1, self.proxied2 < self.proxied1, 'The lists should have the same lt')
self.assertNotEqual(self.proxied1 < self.proxied2, self.proxied2 < self.proxied1, 'The lt should invert correctly')
def testMul(self):
"""Test __mul__"""
self.assertEqual((self.plain1 * 7), (self.proxied1 * 7))
self.assertTrue(isProxy(self.proxied1 * 9))
for p in self.proxied1:
self.assertTrue(isProxy(p))
def testNE(self):
"""Test __ne__"""
self.assertNotEqual(self.plain1, self.plain2)
self.assertNotEqual(self.proxied1, self.proxied2, 'Lists should be different')
self.assertNotEqual(self.plain1[0:5], self.plain1[2:7])
self.assertNotEqual(self.proxied1[0:5], self.proxied1[2:7], 'Lists should be different')
def testRMul(self):
"""Test __rmul__"""
t1 = 5 * self.plain1
t2 = 5 * self.proxied1
self.assertEqual(t1, t2, 'Multiplication should be the same')
def testReversed(self):
"""Test the __reversed__ feature (new in python 2.4)."""
count = len(self.proxied1) - 1
for i in self.proxied1.__reversed__():
self.assertIs(i, self.proxied1[count])
self.assertTrue(isProxy(i))
self.assertTrue(isType(i, TFile))
count -= 1
def testSetItem(self):
"""Test __setitem__"""
t = TFile(name='foo', subdir='cheese')
test_index = 7
self.plain1[test_index] = t
self.assertIs(self.plain1[test_index], t)
self.proxied1[test_index] = t
self.assertIs(self.proxied1[test_index], t)
self.assertTrue(isProxy(self.proxied1[test_index]))
def testSetSliceProxyList(self):
self.plain1[3:7] = self.plain2[3:7]
self.assertEqual(self.plain1[3:7], self.plain2[3:7], 'The lists should be equal')
self.assertIsNot(self.plain1[3:7], self.plain2[3:7], 'The lists should be copies')
self.plain1[4] = self.plain2[9]
self.assertNotEqual(self.plain1[4], self.plain2[4])
tmp = self.plain1[2:9]
self.assertIsNot(self.plain1[2:9], tmp)
self.proxied1[3:7] = self.proxied2[3:7]
self.assertEqual(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be equal')
self.assertIsNot(self.proxied1[3:7], self.proxied2[3:7], 'The lists should be copies')
self.proxied1[4] = self.proxied2[9]
self.assertNotEqual(self.proxied1[4], self.proxied2[4])
tmp = self.proxied1[2:9]
self.assertIsNot(self.proxied1[2:9], tmp)
def testAppend(self):
t = TFile(name='foo')
self.plain1.append(t)
self.assertEqual(self.plain1[-1], t)
self.assertEqual(self.plain1.pop(), t)
self.proxied1.append(t)
self.assertEqual(self.proxied1[-1], t)
self.assertIs(self.proxied1[-1], t, 'Identity Test')
self.assertTrue(isProxy(self.proxied1[-1]), 'Make sure we get back a proxy')
self.assertEqual(self.proxied1.pop(), t)
def testExtend(self):
t1 = [self._makeRandomTFile() for _ in xrange(10)]
self.plain1.extend(t1)
self.proxied1.extend(t1)
self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')
t2 = self.proxied1[4:7]
self.assertTrue(isProxy(t2))
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as glist
self.assertTrue(isType(t2, glist))
self.plain1.extend(t2)
self.proxied1.extend(t2)
self.assertEqual(self.plain1, self.proxied1, 'Lists should be the same')
def testIndex(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
self.assertEqual(self.proxied1.index(t), 8)
def testInsert(self):
t = TFile(name='foo')
self.proxied1.insert(8, t)
self.assertEqual(self.proxied1[8], t)
def testPop(self):
list_len = len(self.proxied1)
t = self.proxied1[-1]
r = self.proxied1.pop()
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
t = self.proxied1[6]
r = self.proxied1.pop(6)
self.assertEqual(t, r)
self.assertIs(t, r)
self.assertEqual(len(self.proxied1), list_len - 2)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testRemove(self):
t = TFile(name='bar')
self.proxied1.insert(7, t)
list_len = len(self.proxied1)
self.proxied1.remove(t)
self.assertEqual(len(self.proxied1), list_len - 1)
self.assertNotIn(t, self.proxied1)
self.assertNotIn(t._impl, self.proxied1._impl)
def testIter(self):
count = 0
for f in self.proxied1:
count += 1
self.assertTrue(isProxy(f))
self.assertTrue(isType(f, TFile))
self.assertEqual(count, len(self.proxied1), 'Must visit every member')
def testCmp(self):
self.assertEqual(cmp(self.proxied1, self.proxied2), cmp(self.plain1, self.plain2))
def testHash(self):
try:
hash(int(self.proxied1))
assert False, 'Lists are not hashable'
except TypeError:
pass
