if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print res['Message']
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:" + lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov)
res = j.append(ma)
if not res['OK']:
print res['Message']
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print res['Message']
exit()
j.submit(d)
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
ov.setNbSigEvtsPerJob(1)
res = j.append(ov)
if not res["OK"]:
print res["Message"]
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res["OK"]:
print res["Message"]
dexit(1)
# print appplication's attributes.
ma.listAttributes()
j.setName("test")
j.setOutputSandbox("*.log")
res = dirac.checkparams(j)
if not res["OK"]:
print res["Message"]
dexit(1)
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print(res['Message'])
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:"+lfn)
ma.setNbEvts(10)
ma.setEnergy(500.)
ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov)
ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov)
res =j.append(ma)
if not res['OK']:
print(res['Message'])
exit()
j.setCPUTime(86400)
j.setOutputData("myrec_overlay_%s.slcio"%ov,"some/path")
j.setName("SomeName")
j.setJobGroup("SomeGroup")
res = d.checkparams(j)
if not res['OK']:
print(res['Message'])
exit()
j.submit(d)
ov.setNbSigEvtsPerJob(1)
ov.setBkgEvtType("gghad")
res = j.append(ov)
if not res['OK']:
print res['Message']
dexit(1)
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering_overlay.xml")
ma.setGearFile("clic_ild_cdr.gear")
ma.getInputFromApp(mo)
ma.setDebug(True)
res = j.append(ma)
if not res['OK']:
print res['Message']
dexit(1)
#print appplication's attributes.
ma.listAttributes()
j.setName("test")
j.setOutputSandbox("*.log")
res = dirac.checkparams(j)
if not res['OK']:
print res['Message']
dexit(1)
class DiracILCTestCase( unittest.TestCase ):
""" Base class for the DiracILC test cases
"""
def setUp( self ):
"""set up the objects"""
ops_mock = Mock()
mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
self.dilc = DiracILC()
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['SE']
if 'Minimum' in args[0]:
return 1
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = Mock()
ops_mock.getValue.side_effect = setOptions
self.dilc.ops = ops_mock
def tearDown( self ):
self.module_patcher.stop()
def test_getprocesslist( self ):
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='some_gconf_testval')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( 'some_gconf_testval' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
def test_getprocesslist_nopath( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( '' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
ops_mock.getValue.assert_called_once_with( '/ProcessList/Location', '' )
def test_presubmissionchecks_notoktosubmit( self ):
job_mock = Mock()
job_mock.oktosubmit = False
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'you should use job.submit(dirac)', self )
def test_presubmissionchecks_checkfails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytest_check_failed'))):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'mytest_check_failed', self )
def test_presubmissionchecks_askuser_fails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
job_mock._askUser.return_value = S_ERROR( 'user says no' )
self.dilc.checked = False
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'user says no', self )
def test_checkparams( self ):
job_mock = Mock()
job_mock.errorDict = {}
with patch('%s.DiracILC.preSubmissionChecks' % MODULE_NAME, new=Mock(return_value=S_OK('mytest'))) as check_mock:
assertDiracSucceedsWith( self.dilc.checkparams( job_mock ), 'mytest', self )
check_mock.assert_called_once_with( job_mock, mode = '' )
def test_checkparams_fails( self ):
job_mock = Mock()
job_mock.errorDict = { 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }
assertDiracFailsWith_equals( self.dilc.checkparams( job_mock ),
{ 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }, self )
def test_giveprocesslist( self ):
self.dilc.processList = '13985u185r9135r'
assertEqualsImproved( self.dilc.giveProcessList(), '13985u185r9135r', self )
def test_giveprocesslist_empty( self ):
self.dilc.processList = ''
assertEqualsImproved( self.dilc.giveProcessList(), '', self )
def test_giveprocesslist_false( self ):
self.dilc.processList = False
assertEqualsImproved( self.dilc.giveProcessList(), False, self )
def test_giveprocesslist_none( self ):
self.dilc.processList = None
assertEqualsImproved( self.dilc.giveProcessList(), None, self )
def test_retrievelfns_norepo( self ):
self.dilc.jobRepo = None
assertDiracSucceeds( self.dilc.retrieveRepositoryOutputDataLFNs(), self )
def test_retrievelfns( self ):
repo_mock = Mock()
ret_dict = { '1' : { 'State' : 'Done', 'UserOutputData' : '1389' }, '2' : {},
'3' : { 'State' : 'secret_teststate' }, '4' : { 'State' : 'invalid_state' },
'5' : { 'State' : 'Done', 'UserOutputData' : 0 }, '6' : { 'ignore_me' : True },
'7' : { 'State' : 'secret_teststate', 'UserOutputData' : 0 },
'148' : { 'State' : 'Done', 1 : False, True : 941, 'values_' : 'keys' } }
repo_mock.readRepository.return_value = S_OK( ret_dict )
self.dilc.jobRepo = repo_mock
with patch('%s.DiracILC.parameters' % MODULE_NAME, new=Mock(side_effect=[S_OK({'UploadedOutputData':'/my/test/lfn1'}),S_ERROR(),S_OK({}),S_OK({'some_entries':'some_values',1:True,'UploadedOutputData':'/more_lfns/append/testlfn.log'})])) as param_mock:
assertEqualsImproved( self.dilc.retrieveRepositoryOutputDataLFNs( [ 'Done', 'secret_teststate' ] ),
[ '/my/test/lfn1', '/more_lfns/append/testlfn.log' ], self )
assertMockCalls( param_mock, [ 3, 5, 7, 148 ], self )
def test_docheck_checksandbox_fails( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', 'other_value' ]
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_sandbox'))) as check_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'test_err_sandbox', self )
check_mock.assert_called_once_with( job_mock )
def test_docheck_too_many_lists( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', [ 'im_a_sandbox_file.stdhep', [ 'evil_list', 'deletethis'] ] ]
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'too many lists of lists in the input sandbox', self )
def test_docheck_checkapps_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(side_effect=[S_OK()] * 3 + [S_ERROR('checkapp_failed_testme')])) as checkapp_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'checkapp_failed_testme', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
def test_docheck_checkoutputpath_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_docheck_checkconsistency_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
data_mock = Mock()
data_mock.getValue.return_value = 'data1389518'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock,
'UserOutputPath' : path_mock, 'UserOutputData' : data_mock }
job_mock = Mock()
job_mock.addToOutputSandbox = 'job_sandbox13895'
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkpath_mock, \
patch('%s.DiracILC._checkdataconsistency' % MODULE_NAME, new=Mock(return_value=S_ERROR('consistency_testerr'))) as checkconsistency_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'consistency_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
checkconsistency_mock.assert_called_once_with( 'data1389518', 'job_sandbox13895' )
def test_checkapp( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
assertDiracFailsWith( self.dilc._checkapp( 'test_platform_341', 'testapp', 'v13.2' ),
'could not find the specified software testapp_v13.2 for test_platform_341, check in CS',
self )
assertMockCalls( ops_mock.getValue, [
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/TarBall', '' ),
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/CVMFSPath', '' ) ], self )
def test_checkoutputpath_invalidchar_1( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( 'http://www.mysitedoesnotexist3h3.abc/some/file.txt' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_2( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/./some/file.log' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_3( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/../dir2/somefile.txt' ),
'invalid path', self )
def test_checkoutputpath_trailing_slash( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/myfile.txt/ ' ),
'invalid path', self )
def test_checkdataconsistency_outputdata_sandbox_equal( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency( 'same_item;something_else',
[ 'distinct_item1', 'same_item' ] ),
'output data and sandbox should not contain the same thing', self )
def test_checkdataconsistency_wildcardchar( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency(
'/test/dir/file.txt;/mydir/something/*;/file/dir/log.log',
[ '/input/sandbox.pdf', '/other/sb/file.stdhep' ] ),
'wildcard character in outputdata definition', self )
def test_checkinputsb_getreplicas_notok( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'could not get replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_getreplicas_fails( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} }
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = {'Failed': [], 'Successful': {'/one/replica': {'SE': 'surl'}}}
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds( self.dilc.checkInputSandboxLFNs( job_mock ), self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_notInputSB(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value = None
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_notInputSB_Value(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = ''
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noLFNs(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep'
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Bad-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith(self.dilc.checkInputSandboxLFNs(job_mock), 'Not enough replicas', self)
replica_mock.assert_called_once_with(['/some/file.txt'])
def test_checkinputsb_goodRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Awesome-Disk-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
replica_mock.assert_called_once_with(['/some/file.txt'])