def test_e_MultipleFiles(self):
"""Test that the wildcards work"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
_ext2 = '.txt'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
file_3 = generate_unique_temp_file(_ext2)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
TestMassStorageWN._managed_files.append(file_3)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2), LocalFile(file_3)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{sjid}/{fname}'),
MassStorageFile(namePattern='*'+_ext2)]
j.submit()
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
MassStorageFile = self.fileClass
from Ganga.GPI import jobs, Job, LocalFile
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.standardFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.standardFormat
def test_c_SplitJob(self):
"""Test submitting subjobs"""
MassStorageFile = self.fileClass
from Ganga.GPI import Job, LocalFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.extendedFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.extendedFormat
def test_c_testCopyJob(self):
""" Test copying a completed job with a wildcard in the outputfiles """
from Ganga.GPI import jobs
j = jobs[-1]
j2 = j.copy()
assert len(j2.outputfiles) == 1
MassStorageFile = self.fileClass
assert j2.outputfiles == [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
assert len(j2.inputfiles) == 2
self.cleanUp()
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args=[[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + _ext)]
j.submit()
def test_a_test_put(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._temp_files.append(file_1)
self._temp_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
self._managed_files.append(msf_1)
self._managed_files.append(msf_2)
msf_1.put()
msf_2.put()
for file_ in [msf for msf in (msf_1, msf_2)]:
assert os.path.isfile(
os.path.join(self.outputFilePath, file_.namePattern))
file_.localDir = ''
assert file_.localDir == ''
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def string_file_shortcut(v, item):
if isinstance(v, str):
# use proxy class to enable all user conversions on the value itself
# but return the implementation object (not proxy)
key = findOutputFileTypeByFileName(v)
if key is not None:
if key == 'MassStorageFile':
from .MassStorageFile import MassStorageFile
return stripProxy(MassStorageFile._proxyClass(v))
elif key == 'LCGSEFile':
from .LCGSEFile import LCGSEFile
return stripProxy(LCGSEFile._proxyClass(v))
elif key == 'DiracFile':
try:
from GangaDirac.Lib.Files.DiracFile import DiracFile
return stripProxy(DiracFile._proxyClass(v))
except:
Ganga.Utility.logging.log_unknown_exception()
pass
return stripProxy(LocalFile._proxyClass(v))
return None
def string_file_shortcut(v, item):
if isinstance(v, str):
# use proxy class to enable all user conversions on the value itself
# but return the implementation object (not proxy)
key = findOutputFileTypeByFileName(v)
if key is not None:
if key == 'MassStorageFile':
from .MassStorageFile import MassStorageFile
return stripProxy(MassStorageFile._proxyClass(v))
elif key == 'LCGSEFile':
from .LCGSEFile import LCGSEFile
return stripProxy(LCGSEFile._proxyClass(v))
elif key == 'DiracFile':
try:
from GangaDirac.Lib.Files.DiracFile import DiracFile
return stripProxy(DiracFile._proxyClass(v))
except:
Ganga.Utility.logging.log_unknown_exception()
pass
return stripProxy(LocalFile._proxyClass(v))
return None
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.customOutputFormat)]
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args = [[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*'+_ext)]
j.submit()
def test_a_test_put(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._temp_files.append(file_1)
self._temp_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
self._managed_files.append(msf_1)
self._managed_files.append(msf_2)
msf_1.put()
msf_2.put()
for file_ in [msf for msf in (msf_1, msf_2)]:
assert os.path.isfile(os.path.join(self.outputFilePath, file_.namePattern))
file_.localDir = ''
assert file_.localDir == ''