def test_e_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
_ext2 = '.txt'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
file_3 = generate_unique_temp_file(_ext2)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
TestMassStorageWN._managed_files.append(file_3)
j = Job()
j.inputfiles = [
LocalFile(file_1),
LocalFile(file_2),
LocalFile(file_3)
]
j.splitter = ArgSplitter(
args=[[_] for _ in range(0, TestMassStorageWN.sj_len)])
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{sjid}/{fname}'),
MassStorageFile(namePattern='*' + _ext2)
]
j.submit()
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
def test_b_SetupJobs(self):
from Ganga.GPI import Job, jobs, Executable, ArgSplitter
for i in range(global_num_jobs):
print('creating job', end=' ')
j = Job(splitter=ArgSplitter(args=[[0], [1], [2]]))
print(j.id)
print('job len:', len(jobs))
assert len(jobs) == global_num_jobs
def testArgSplitter(self):
from Ganga.GPI import Job, ArgSplitter
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.splitter = ArgSplitter(args=[['1'], ['2'], ['3']])
j.submit()
self.assertTrue(sleep_until_completed(j, 60), 'Timeout on completing job')
self.assertEqual(len(j.subjobs), 3)
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs, ArgSplitter
j = Job(splitter=ArgSplitter(args=testArgs))
assert len(jobs) == 1
from Ganga.GPIDev.Base.Proxy import stripProxy
stripProxy(j)._getRegistry().flush_all()
stripProxy(j)._setDirty()
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs, ArgSplitter
j=Job()
assert len(jobs) == 1
j.splitter = ArgSplitter()
j.splitter.args = getNestedList()
assert j.splitter.args == getNestedList()
def testSubjobsSubmit(self):
from Ganga.GPI import Job, Executable, TestSubmitter, ArgSplitter
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isType
j = Job(application=Executable(), backend=TestSubmitter(time=1))
j.splitter = ArgSplitter(args=[['A'], ['B'], ['C']])
j.submit()
assert run_until_completed(j), 'Job must complete'
assert len(j.subjobs) == 3, 'splitting must occur'
for jj in j.subjobs:
assert not isType(jj.master, gangaList)
def test_a_jobSubmit(self):
"""here for testing a submit"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import LocalFile, Job, ArgSplitter
file_1 = generate_unique_temp_file(TestLocalFileClient._ext)
file_2 = generate_unique_temp_file(TestLocalFileClient._ext)
TestLocalFileClient._managed_files.append(file_1)
TestLocalFileClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(
args=[[_] for _ in range(TestLocalFileClient.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + TestLocalFileClient._ext)]
j.submit()
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs, ArgSplitter
j = Job()
self.assertEqual(len(jobs),
1) # Don't really gain anything from assertEqual...
j.splitter = ArgSplitter(args=[[i] for i in range(global_subjob_num)])
j.submit()
self.assertEqual(len(j.subjobs), global_subjob_num)
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j, 60)
def test_c_SelectSJTests(self):
""" Is is a bird is it a plane... no it's a test for selecting subjobs now"""
from Ganga.GPI import jobs, Job, ArgSplitter
j = Job(splitter=ArgSplitter(args=[[_] for _ in job_names]))
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
assert sleep_until_completed(j, 60)
mySlice = jobs(j.id).subjobs.select(status="completed")
assert len(mySlice) == len(job_names)
mySlice2 = jobs(j.id).subjobs.select(id=2)
assert len(mySlice2) == 1
assert mySlice2[2].id == 2
def test_c_SplitJob(self):
"""Test submitting subjobs"""
MassStorageFile = self.fileClass
from Ganga.GPI import Job, LocalFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.extendedFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.extendedFormat
def test_a_Savannah32342(self):
"""Basic splitting test"""
from Ganga.GPI import Job, ArgSplitter, jobs
j = Job()
j.splitter = ArgSplitter(args=[['A'], ['B']])
j.submit()
assert len(j.subjobs) == 2, 'Splitting must have occured'
for jj in j.subjobs:
assert jj._impl._getParent(), 'Parent must be set'
# make sure we have out job in the repository
job_seen = False
for jj in jobs:
if j is jj:
job_seen = True
break
assert job_seen, 'Job must be in the repository'
def split_prime_job(j,millions,number=None):
""" Create a splitter for job j to search a given number of millions of prime numbers.
A number to factorize may be specified else the one defined before is taken. This is useful if you make a copy of a job.
Example: split_prime_job(j,5,123456) - search first 5 millions of prime numbers and find factors of 123456
"""
if not number:
number = j.application.args[0]
from Ganga.GPI import File, TUTDIR, ArgSplitter, Executable
j.application.exe=File(TUTDIR+'/Lib/primes/prime_factor.py')
# make sure that we crab the data files!
j.outputsandbox = ['*.dat']
j.application.args = [str(number)]
j.splitter = ArgSplitter()
args = []
for m in range(1,millions+1):
args.append([str(number),'http://cern.ch/diane/download/primes/primes%d.zip'%m])
j.splitter.args = args
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args=[[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + _ext)]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPI import Job, jobs, ArgSplitter
j = Job()
orig_sj_proxy = j.subjobs
j.splitter = ArgSplitter()
j.splitter.args = [[0], [1]]
i = 0
for sj in stripProxy(j.splitter).split(stripProxy(j)):
sj.id = i
stripProxy(j).subjobs.append(sj)
i = i + 1
assert len(jobs) == 1
assert len(j.subjobs) == 2
sj_proxy = j.subjobs
assert sj_proxy is j.subjobs
assert orig_sj_proxy is sj_proxy
for sj in j.subjobs:
assert isinstance(sj, Job)
global global_AutoStartReg
global_AutoStartReg = False
stripProxy(sj)._getRegistry().flush_all()
for sj in j.subjobs:
stripProxy(sj)._setDirty()
stripProxy(sj)._getRegistry().flush_all()
for sj in j.subjobs:
stripProxy(sj)._setDirty()
def test_d_testXMLContent(self):
# Check content of XML is as expected
from Ganga.Core.GangaRepository.VStreamer import to_file, from_file
from Ganga.GPI import jobs, Job, ArgSplitter
from Ganga.GPIDev.Base.Proxy import stripProxy
from tempfile import NamedTemporaryFile
j = jobs(0)
assert path.isfile(getXMLFile(j))
with open(getXMLFile(j)) as handler:
tmpobj, errs = from_file(handler)
assert tmpobj.splitter
assert tmpobj.splitter.args == getNestedList()
ignore_subs = ''
with NamedTemporaryFile(delete=False) as new_temp_file:
to_file(stripProxy(j), new_temp_file, ignore_subs)
new_temp_file.flush()
with NamedTemporaryFile(delete=False) as new_temp_file2:
j2 = Job()
j2.splitter = ArgSplitter()
j2.splitter.args = getNestedList()
to_file(stripProxy(j2), new_temp_file2, ignore_subs)
new_temp_file2.flush()
assert open(handler.name).read() == open(new_temp_file.name).read()
assert open(handler.name) != open(new_temp_file2.name).read()
unlink(new_temp_file.name)
unlink(new_temp_file2.name)
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.customOutputFormat)]
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat