def test_updateproclist_and_getters( self ):
self.prol.cfg.createNewSection( 'Processes/myTestProcDeleteMe' )
self.prol.cfg.setOption( 'Processes/myTestProcDeleteMe/someRandomOption', True )
dict_1 = { 'CrossSection' : 'someCross' }
dict_1.update( STD_PROC_DICT )
dict_2 = { 'CrossSection' : 'some_other_val' }
dict_2.update( STD_PROC_DICT )
process_dict = {}
process_dict[ 'MytestProcess' ] = dict_1
process_dict[ 'myTestProcDeleteMe' ] = dict_2
result = self.prol.updateProcessList( process_dict )
assertDiracSucceeds( result, self )
conf = self.prol.cfg
self.assertFalse( conf.existsKey( 'Processes/myTestProcDeleteMe/someRandomOption' ) )
options = [ 'Processes/MytestProcess/CrossSection', 'Processes/myTestProcDeleteMe/CrossSection' ]
assertEqualsImproved( ( map( conf.getOption, options ) ), ( [ 'someCross', 'some_other_val' ] ), self )
assertEqualsImproved( ( self.prol.getCSPath( 'myTestProcDeleteMe' ),
self.prol.getInFile( 'myTestProcDeleteMe' ),
self.prol.existsProcess( 'myTestProcDeleteMe' ),
self.prol.existsProcess( '' ), self.prol.existsProcess( 'invalidProcess' ),
self.prol.existsProcess( 'myTestProcDeleteMeToo' ) ),
( '/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True), S_OK(False),
S_OK(False) ), self )
assertListContentEquals( self.prol.getProcesses(), [ 'myTestProcDeleteMe', 'MytestProcess' ], self )
all_processes_dict = self.prol.getProcessesDict()
assertEqualsImproved( len(all_processes_dict), 2, self )
assertEqualsImproved( ('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess' in all_processes_dict),
( True, True ), self )
self.prol.printProcesses()
def test_fromwhizarddotin(self):
self.whop.paramdict = {}
self.whop.whizardxml = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float"> </beam_intensity> <beam_density type="0/1/2/3"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string"> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray"> </myfloatarr> <lasttestvalue type="T/F"> </lasttestvalue> </lasttestkey> </whizardxml>'
)
expected_tree = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float" value="84.2"> </beam_intensity> <beam_density type="0/1/2/3" value="0"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string" value="TmyFstring"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string" value="teststring_dontloseme."> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray" value="943 0.1 01.2 9024.4"> </myfloatarr> <lasttestvalue type="T/F" value="F"> </lasttestvalue> </lasttestkey> </whizardxml>'
)
expected_tree.find(
'beam_input_1/beam_intensity').attrib['value'] = 84.2
expected_tree.find('beam_input_1/beam_density').attrib['value'] = 0
file_contents = [[
' !initial Comment, ignore this line ',
'/ Other comment. ',
' ', 'ignoreThisLineTOO',
'&beam_input', 'beam_intensity=84.2', 'beam_density=0',
'&myotherkey', '&testkey', 'testcurkey="teststring_dontloseme."',
'&beam_input ', ' FcoolTtestkey = "TmyFstring" ',
'&lasttestkey', 'myfloatarr= 943 0.1 01.2 9024.4',
'lasttestvalue=F'
]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo:
mo.side_effect = (h for h in handles)
result = self.whop.fromWhizardDotIn('filename.txt')
assertDiracSucceeds(result, self)
assertEqualsXmlTree(result['Value'], expected_tree, self)
def test_generatefailover_optionalfails(self):
container_mock = Mock()
container_mock.__len__.return_value = 1
container_mock.toJSON.return_value = S_OK('myjsonfilecontent_testme')
container_mock.getDigest.return_value = S_ERROR('bla')
reqval_mock = Mock()
reqval_mock.validate.return_value = S_OK('ok')
reqval_mock.getJSONData.return_value = True
request_mock = Mock()
request_mock.getJSONData.return_value = True
report_mock = Mock()
report_mock.generateForwardDISET.return_value = S_ERROR(
'test_faildiset')
report_mock.getJSONData.return_value = True
self.moba.jobReport = report_mock
accreport_mock = Mock()
accreport_mock.commit.return_value = S_OK('')
self.moba.workflowStatus['OK'] = False
self.moba.workflow_commons['AccountingReport'] = accreport_mock
self.moba.workflow_commons[
'ProductionOutputData'] = 'mylfn1;testmelfn2;'
with patch('%s.open' % MODULE_NAME, mock_open()) as mo, \
patch('%s.RequestValidator' % MODULE_NAME, new=Mock(return_value=reqval_mock)), \
patch('%s.ModuleBase._getRequestContainer' % MODULE_NAME, new=Mock(return_value=container_mock)), \
patch('%s.ModuleBase.setApplicationStatus' % MODULE_NAME) as appstat_mock, \
patch('%s.ModuleBase._cleanUp' % MODULE_NAME) as cleanup_mock:
result = self.moba.generateFailoverFile()
assertDiracSucceeds(result, self)
mo.assert_any_call('0_0_request.json', 'w')
mo = mo()
mo.write.assert_called_once_with('myjsonfilecontent_testme')
appstat_mock.assert_called_once_with('Creating Removal Requests')
cleanup_mock.assert_called_once_with(['mylfn1', 'testmelfn2', ''])
def test_transfer_register_and_failover_files( self ):
self.ujf.failoverSEs = [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ]
self.ujf.userFileCatalog = 'blaCatalogTestme'
transfer_mock = Mock()
transfer_mock.transferAndRegisterFileFailover.return_value = S_OK( {} )
filesUploaded = []
with patch('%s.random.shuffle' % MODULE_NAME) as shuffle_mock:
result = self.ujf.transferRegisterAndFailoverFiles( transfer_mock, {
'filename1test' : { 'metadatadict1' : True,
'resolvedSE' : ( 'etc', 'privateSite4' ),
'localpath' : '/my/local/path/Testme.txt',
'lfn' : 'LFN:/ilc/mytest/LFN.txt', 'filedict' : 89546 },
'my_other_testfile' : {
'metadatadict1' : True, 'resolvedSE' : ( 'testSite1', 'privateSite4' ),
'localpath' : '/my/local/path/Testme2.txt', 'lfn' : 'LFN:/ilc/othertest/new_lfn.lfn',
'filedict' : 475 } }, filesUploaded )
assertDiracSucceeds( result, self )
assertEqualsImproved( result['Value'], dict(cleanUp=False), self )
assertMockCalls( shuffle_mock, [ [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ],
[ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ] ],
self )
assertEqualsImproved( transfer_mock.transferAndRegisterFileFailover.mock_calls,
[ call( 'my_other_testfile', '/my/local/path/Testme2.txt',
'LFN:/ilc/othertest/new_lfn.lfn', 'testSite1',
[ 'testSite2', 'myTestSite', 'privateSite4', 'etc' ],
fileCatalog='blaCatalogTestme', fileMetaDict=475 ),
call( 'filename1test', '/my/local/path/Testme.txt', 'LFN:/ilc/mytest/LFN.txt',
'etc', [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4' ],
fileCatalog='blaCatalogTestme', fileMetaDict=89546 ) ], self )
self.assertIn( 'LFN:/ilc/mytest/LFN.txt', filesUploaded )
self.assertIn( 'LFN:/ilc/othertest/new_lfn.lfn', filesUploaded )
def test_checkconsistency_othercase(self):
self.slic.detectorModel = 'someModel'
self.slic.steeringFile = 'LFN:/my/consistent/steeringfile.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds(self.slic._checkConsistency(), self)
def test_transferreg_transfer_fails( self ):
self.ujf.failoverSEs = [ 'testSite1', 'testSite2', 'myTestSite',
'privateSite4', 'etc' ]
self.ujf.userFileCatalog = 'blaCatalogTestme'
transfer_mock = Mock()
transfer_mock.transferAndRegisterFileFailover.return_value = S_ERROR( 'some_transfer_test_err' )
filesUploaded = []
with patch('%s.random.shuffle' % MODULE_NAME) as shuffle_mock:
result = self.ujf.transferRegisterAndFailoverFiles( transfer_mock, {
'filename1test' : { 'metadatadict1' : True,
'resolvedSE' : ( 'nonexistant', ) ,
'localpath' : '/my/local/path/Testme.txt',
'lfn' : 'LFN:/ilc/mytest/LFN.txt', 'filedict' : 89546
} }, filesUploaded )
assertDiracSucceeds( result, self )
assertEqualsImproved( result['Value'], dict(cleanUp=True), self )
shuffle_mock.assert_called_once_with( [ 'testSite1', 'testSite2',
'myTestSite', 'privateSite4',
'etc' ] )
transfer_mock.transferAndRegisterFileFailover.assert_called_once_with(
'filename1test', '/my/local/path/Testme.txt', 'LFN:/ilc/mytest/LFN.txt',
'nonexistant', [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4',
'etc' ], fileCatalog='blaCatalogTestme',
fileMetaDict=89546 )
assertEqualsImproved( filesUploaded, [], self )
def test_checkconsistency_othercase(self):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = ''
self.lcs.trackingStrategy = 'lfn:/myTestStrat'
self.lcs.detectorModel = 'correctDetector.zip'
self.lcs._jobtype = 'notUser'
self.lcs._listofoutput = []
app1 = Mock()
app1.appame = 'myTestApp'
app2 = Mock()
app2.appname = 'marlin'
self.lcs._inputapp = [app1, app2]
self.lcs.outputFile = ''
self.lcs.willBeCut = False
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.lcs._checkConsistency('myTestJob'), self)
expected_output_list = [{
"outputFile": "@{outputREC}",
"outputPath": "@{outputPathREC}",
"outputDataSE": "@{OutputSE}"
}, {
"outputFile": "@{outputDST}",
"outputPath": "@{outputPathDST}",
"outputDataSE": "@{OutputSE}"
}]
assertEqualsImproved(self.lcs._listofoutput, expected_output_list,
self)
prodparam_dict = self.lcs.prodparameters
assertEqualsImproved((prodparam_dict['detectorType'],
prodparam_dict['lcsim_steeringfile'],
prodparam_dict['lcsim_trackingstrategy']),
('SID', '', 'lfn:/myTestStrat'), self)
def test_checkconsistency(self):
self.gsf.setFiles({
'file': '/invalid/dir/mycoolfile.txt',
'something': True
})
assertDiracSucceeds(self.gsf._checkConsistency(), self)
assertEqualsImproved(self.gsf.outputFile, 'mycoolfile.txt', self)
def test_checkconsistency_othercase( self ):
import inspect
self.lcs.version = 'v2.4'
self.lcs.steeringFile = ''
self.lcs.trackingStrategy = 'lfn:/myTestStrat'
self.lcs.detectorModel = 'correctDetector.zip'
self.lcs._jobtype = 'notUser'
self.lcs._listofoutput = []
app1 = Mock()
app1.appame = 'myTestApp'
app2 = Mock()
app2.appname = 'marlin'
self.lcs._inputapp = [ app1, app2 ]
self.lcs.outputFile = ''
self.lcs.willBeCut = False
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())):
assertDiracSucceeds( self.lcs._checkConsistency( 'myTestJob' ), self )
expected_output_list = [ { "outputFile" : "@{outputREC}", "outputPath" : "@{outputPathREC}",
"outputDataSE" : "@{OutputSE}" }, { "outputFile" : "@{outputDST}",
"outputPath" : "@{outputPathDST}",
"outputDataSE" : "@{OutputSE}" } ]
assertEqualsImproved( self.lcs._listofoutput, expected_output_list, self )
prodparam_dict = self.lcs.prodparameters
assertEqualsImproved( ( prodparam_dict['detectorType'], prodparam_dict['lcsim_steeringfile'],
prodparam_dict['lcsim_trackingstrategy'] ), ( 'SID' , '', 'lfn:/myTestStrat' ),
self )
def test_sliced_limited_add_all( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
util_mock = Mock()
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK(
[ ( { 'Status' : 'Processed' }, 4 ), ( { 'Status' : 'junk' }, 6 ), ( { 'Status' : 'Ignore_me' }, 8 ),
( { 'Status' : 'Assigned' }, 6 ) ] )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'SlicedLimited', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = 'Processed'
self.tfp.params[ 'MaxNumberOfTasks' ] = 16
self.tfp.params[ 'TransformationID' ] = 78456
self.tfp.util = util_mock
self.tfp.setInputData( {
'/my/file/here_abc' : [ 'myTestSE' ],
'/my/file/other_file.txt' : [ 'secretSE', 'StorageSE', 'TestSE' ],
'/my/file/newfile.pdf' : [ 'TestSE' ],
'/my/file/a' : [], '/dir/somefile': [ '' ] } )
result = self.tfp.run()
assertDiracSucceeds( result, self )
expected = [ ( '', [ '/my/file/here_abc' ] ), ( '', [ '/my/file/other_file.txt' ] ),
( '', [ '/my/file/newfile.pdf' ] ), ( '', [ '/my/file/a' ] ), ( '', [ '/dir/somefile' ] ) ]
assertListContentEquals( result[ 'Value' ], expected, self )
trans_mock.getCounters.assert_called_once_with( 'TransformationFiles', [ 'Status' ],
{ 'TransformationID' : 78456 } )
def test_checkconsistency(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = None
self.mar._jobtype = 'notUser'
self.mar.outputFile = None
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(Marlin), 'checkXMLValidity', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, None, self)
def test_setoutputsandbox( self ):
wf_mock = Mock()
self.ujo.workflow = wf_mock
with patch('%s.UserJob._addParameter' % MODULE_NAME, new=Mock()) as addparam_mock:
assertDiracSucceeds( self.ujo.setOutputSandbox( '/my/dir/myfile.txt' ), self )
addparam_mock.assert_called_once_with( wf_mock, 'OutputSandbox', 'JDL',
'/my/dir/myfile.txt', 'Output sandbox file' )
def test_split_atomicsubmission(self):
"""Test splitting atomic."""
self.ujo._splittingOption = None
info_message = "Job splitting successful"
with patch('%s.LOG' % MIXIN_MODULE, new=self.log_mock):
assertDiracSucceeds(self.ujo._split(), self)
self.log_mock.notice.assert_called_with(info_message)
def test_fccanalysis(self):
"""create test for fccanalysis"""
print("fccanalysis test")
jobs = self.myTests.createFccAnalysisTest()
assertDiracSucceeds( jobs, self )
thisJob = jobs['Value']
res = self.myTests.runJobLocally(thisJob, "FccAnalysis")
def test_checkinputsb( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = {'Failed': [], 'Successful': {'/one/replica': {'SE': 'surl'}}}
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds( self.dilc.checkInputSandboxLFNs( job_mock ), self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_upload_copy_works( self ):
from ILCDIRAC.Core.Utilities.FileUtils import upload
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=True)) as exists_mock, \
patch('%s.shutil.copy' % MODULE_NAME, new=Mock()) as copy_mock:
assertDiracSucceeds( upload( 'http://www.cern.ch/lcd-data/mypath', 'appTarTest' ), self )
exists_mock.assert_called_once_with( 'appTarTest' )
copy_mock.assert_called_once_with( 'appTarTest', '/afs/cern.ch/eng/clic/data/software/appTarTest' )
def test_checkconsistency_othercase(self):
self.mar.version = '13'
self.mar.steeringFile = '/mysteer/file.stdhep'
self.mar.gearFile = 'myGearOutput.mock'
self.mar._jobtype = 'notUser'
self.mar.outputFile = 'myoutput.test'
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds(self.mar._checkConsistency(), self)
self.assertNotIn(
{
'outputFile': '@{outputREC}',
'outputPath': '@{outputPathREC}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
self.assertNotIn(
{
'outputFile': '@{outputDST}',
'outputPath': '@{outputPathDST}',
'outputDataSE': '@{OutputSE}'
}, self.mar._listofoutput)
for keyword in [
'detectorType', 'marlin_gearfile', 'marlin_steeringfile'
]:
self.assertIn(keyword, self.mar.prodparameters)
assertEqualsImproved(self.mar.gearFile, 'myGearOutput.mock', self)
def test_checkconsistency_othercase( self ):
self.slic.detectorModel = 'someModel'
self.slic.steeringFile = 'LFN:/my/consistent/steeringfile.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds( self.slic._checkConsistency(), self )
def test_generatefailover_optionalfails( self ):
container_mock = Mock()
container_mock.__len__.return_value = 1
container_mock.toJSON.return_value = S_OK( 'myjsonfilecontent_testme' )
container_mock.getDigest.return_value = S_ERROR( 'bla' )
reqval_mock = Mock()
reqval_mock.validate.return_value = S_OK('ok')
reqval_mock.getJSONData.return_value = True
request_mock = Mock()
request_mock.getJSONData.return_value = True
report_mock = Mock()
report_mock.generateForwardDISET.return_value = S_ERROR( 'test_faildiset' )
report_mock.getJSONData.return_value = True
self.moba.jobReport = report_mock
accreport_mock = Mock()
accreport_mock.commit.return_value = S_OK( '' )
self.moba.workflowStatus['OK'] = False
self.moba.workflow_commons['AccountingReport'] = accreport_mock
self.moba.workflow_commons['ProductionOutputData'] = 'mylfn1;testmelfn2;'
with patch('%s.open' % MODULE_NAME, mock_open()) as mo, \
patch('%s.RequestValidator' % MODULE_NAME, new=Mock(return_value=reqval_mock)), \
patch('%s.ModuleBase._getRequestContainer' % MODULE_NAME, new=Mock(return_value=container_mock)), \
patch('%s.ModuleBase.setApplicationStatus' % MODULE_NAME) as appstat_mock, \
patch('%s.ModuleBase._cleanUp' % MODULE_NAME) as cleanup_mock:
result = self.moba.generateFailoverFile()
assertDiracSucceeds( result, self )
mo.assert_any_call( '0_0_request.json', 'w' )
mo = mo()
mo.write.assert_called_once_with( 'myjsonfilecontent_testme' )
appstat_mock.assert_called_once_with( 'Creating Removal Requests' )
cleanup_mock.assert_called_once_with( [ 'mylfn1', 'testmelfn2', '' ] )
def test_sliced_limited_add_up_to_maximum( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
util_mock = Mock()
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK(
[ ( { 'Status' : 'Processed' }, 4 ), ( { 'Status' : 'junk' }, 6 ), ( { 'Status' : 'Ignore_me' }, 8 ),
( { 'Status' : 'Assigned' }, 6 ) ] )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'SlicedLimited', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = 'Processed'
self.tfp.params[ 'MaxNumberOfTasks' ] = 12
self.tfp.params[ 'TransformationID' ] = 78456
self.tfp.util = util_mock
self.tfp.setInputData( {
'/my/file/here_abc' : [ 'myTestSE' ],
'/my/file/other_file.txt' : [ 'secretSE', 'StorageSE', 'TestSE' ],
'/my/file/newfile.pdf' : [ 'TestSE' ],
'/my/file/a' : [], '/dir/somefile': [ '' ] } )
result = self.tfp.run()
assertDiracSucceeds( result, self )
to_check = result[ 'Value' ]
print(result)
assertEqualsImproved(
( len( to_check ), to_check[0][0], to_check[1][0], len( to_check[0][1] ), len( to_check[1][1] ) ),
( 2, '', '', 1, 1 ), self ) # Checks that two tasks are added, that the two tasks have the form ('', [ a ])
self.assertNotEquals( to_check[0][1][0], to_check[1][1][0] ) # Checks that the same lfn isnt added twice
# Checks that the LFN is one of those expected (since keys() iteration is random for dictionaries)
expected = [ '/my/file/here_abc', '/my/file/other_file.txt', '/my/file/newfile.pdf',
'/my/file/a', '/dir/somefile' ]
assertInImproved( to_check[0][1][0], expected, self )
assertInImproved( to_check[1][1][0], expected, self )
trans_mock.getCounters.assert_called_once_with( 'TransformationFiles', [ 'Status' ],
{ 'TransformationID' : 78456 } )
def test_resolvelinkedstepparams_nothing_happens( self ): instance_mock = Mock() self.shs._inputappstep = None self.shs._jobsteps = None self.shs._linkedidx = [ 'abc' ] assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self ) self.assertFalse( instance_mock.setLink.called )
def test_applicationspecificinputs_nolist( self ):
self.marAna.InputFile = [ 'dontleavemealone.txt' ]
self.marAna.workflow_commons[ 'ParametricInputSandbox' ] = 'myEntryies;another_one.slcio'
assertDiracSucceeds( self.marAna.applicationSpecificInputs(), self )
assertEqualsImproved( ( self.marAna.InputFile, self.marAna.InputData ),
( [ 'dontleavemealone.txt', 'myEntryies',
'another_one.slcio' ], [] ), self )
def test_execute_ignoremissing_works( self ):
module_mock = Mock()
module_mock.execute.side_effect = [ { 'OK' : True, 'Failed' : [ 'myReplicaSite1', 'other_failure' ],
'Successful' : { 'WorkingSite1' : True, 'OtherGoodSite' : True } },
{ 'OK' : True, 'Failed' : [],
'Successful' : { 'other_site' : True, 'TestSite1' : True } } ]
self.mfac_mock.ModuleFactory().getModule.return_value = S_OK( module_mock )
from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution
self.idr = InputDataResolution( { 'Job' : { 'InputDataPolicy' : [ '/myTestPolicy/some/path',
'other_policy/path' ] },
'Configuration': { 'SiteName' :'myTestSitename' },
'IgnoreMissing' : True } )
res = self.idr.execute()
assertDiracSucceeds( res, self )
assertEqualsImproved( res, { 'OK' : True, 'Value' : None, 'Failed' : [],
'Successful' : { 'WorkingSite1' : True, 'OtherGoodSite' : True,
'other_site' : True, 'TestSite1' : True } }, self )
assertMockCalls( self.mfac_mock.ModuleFactory().getModule, [
( '/myTestPolicy/some/path', { 'Job' : { 'InputDataPolicy' : [ '/myTestPolicy/some/path',
'other_policy/path' ] },
'Configuration' : { 'SiteName' : 'myTestSitename' },
'IgnoreMissing' : True } ),
( 'other_policy/path', { 'Job': { 'InputDataPolicy' : [ '/myTestPolicy/some/path', 'other_policy/path' ] },
'Configuration' : { 'SiteName' : 'myTestSitename' }, 'IgnoreMissing' : True } ) ],
self )
def test_fccsw(self):
"""create test for fccsw"""
print("fccsw test")
jobs = self.myTests.createFccSwTest()
assertDiracSucceeds( jobs, self )
thisJob = jobs['Value']
res = self.myTests.runJobLocally(thisJob, "FccSw")
assertDiracSucceeds( res, self )
def test_checkconsistency_basic(self):
self.lcs.energy = 2489
self.lcs.numberOfEvents = 8245
self.lcs.version = 'v1.2'
self.lcs.steeringFile = ''
self.lcs.detectorModel = ''
self.lcs._jobtype = 'User'
assertDiracSucceeds(self.lcs._checkConsistency(), self)
def test_execute_nologs( self ):
with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \
patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)), \
patch('%s.os.makedirs' % MODULE_NAME, new=Mock(side_effect=OSError('os_mkdir_failed_testerr_populate'))), \
patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()):
assertDiracSucceeds( self.ulf.execute(), self )
self.log_mock.error.assert_called_once_with('Completely failed to populate temporary log file directory.', '')
def test_checkfields( self ):
self.whop.whizardxml = fromstring(
'<whizard> <process_input> <mytestelement type="floatarray"> </mytestelement> <test_bool type="T/F"> </test_bool> <test_bool2 type="T/F"> </test_bool2> <test_integer type="integer"> </test_integer> </process_input> <test_superelem> <test_float type="float"> </test_float> <test_float2 type="float"> </test_float2> <test_string type="string"> </test_string> <test_ignoreme type="invalid_type"></test_ignoreme> </test_superelem> </whizard>' )
assertDiracSucceeds( self.whop.checkFields(
{ 'process_input' : { 'mytestelement' : '842 021.2 123 98724 81', 'test_bool' : 'T', 'test_bool2' : 'F',
'test_integer' : 8492 },
'test_superelem' : { 'test_float' : 824.2, 'test_float2' : 98421, 'test_string' : 'oijrsg' }
} ), self )
def test_checkconsistency_basic( self ): self.lcs.energy = 2489 self.lcs.numberOfEvents = 8245 self.lcs.version = 'v1.2' self.lcs.steeringFile = '' self.lcs.detectorModel = '' self.lcs._jobtype = 'User' assertDiracSucceeds( self.lcs._checkConsistency(), self )
def test_resolvelinkedparams( self ):
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = { 'abc' : False }
self.tom._linkedidx = 3
self.tom._jobsteps = [ None, None, None, input_mock ]
assertDiracSucceeds( self.tom._resolveLinkedStepParameters( step_mock ), self )
step_mock.setLink.assert_called_once_with( 'InputFile', { 'abc' : False }, 'OutputFile' )
def test_fullcopy( self ):
from ILCDIRAC.Core.Utilities.FileUtils import fullCopy
with patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=['/my/src/directory/file_1_globbed.log','/my/src/directory/other__file.stdhep', 'lastfile_in_list.txt'])), \
patch('%s.dir_util.create_tree' % MODULE_NAME, new=Mock()), \
patch('%s.os.path.isfile' % MODULE_NAME, new=Mock(side_effect=[True,False,True])), \
patch('%s.shutil.copy2' % MODULE_NAME, new=Mock()), \
patch('%s.shutil.copytree' % MODULE_NAME, new=Mock()):
assertDiracSucceeds( fullCopy( '/my/src/directory/', '/my/destination/dir', './myfile123.txt ' ), self )
def test_root3(self):
"""create test for root 3"""
print("test root3")
jobs = self.myTests.createRootMacroTest()
assertDiracSucceeds( jobs, self )
thisJob = jobs['Value']
res = self.myTests.runJobLocally(thisJob, "Root")
assertDiracSucceeds( res, self )
def test_checkconsistency_download_succeeds(self):
import inspect
self.slic.steeringFile = '/my/remote/path.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_OK('ok'))):
assertDiracSucceeds(self.slic._checkConsistency(), self)
def test_checkworkflow_success(self):
self.pgs._inputapp = [
'some_dependency', 'other_dependencies', 'many_more'
]
self.pgs._jobapps = [
'ignore_me', 'many_more', 'some_dependency', 'other_dependencies'
]
assertDiracSucceeds(self.pgs._checkWorkflowConsistency(), self)
def test_marlin2(self):
"""create test for marlin"""
print("marlin test2")
jobs = self.myTests.createMarlinTest( True )
assertDiracSucceeds( jobs, self )
thisJob = jobs['Value']
res = self.myTests.runJobLocally(thisJob, "Marlin")
assertDiracSucceeds( res, self )
def test_mokka(self):
"""create test for mokka"""
print("mokka test")
jobs = self.myTests.createMokkaTest()
assertDiracSucceeds( jobs, self )
thisJob = jobs['Value']
res = self.myTests.runJobLocally(thisJob, "Mokka")
assertDiracSucceeds( res, self )
def test_towhizarddotin( self ):
self.whop.whizardxml = fromstring(
'<whizard> <process_input> <mytestelement type="floatarray" value="sqrts"> </mytestelement> <test_bool type="T/F" value="000"> </test_bool> <test_bool2 type="T/F" value="0.0.0"> </test_bool2> <test_integer type="integer" value="0..0..0"> </test_integer> </process_input> <lalabeam_input1238> <test_float type="float" value="dontchangeanything"> </test_float> <test_string type="string" value="some_teststring"> </test_string> <test_noval></test_noval> </lalabeam_input1238> </whizard>' )
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock:
result = self.whop.toWhizardDotIn( 'mytestfile.xml' )
assertDiracSucceeds( result, self )
open_mock.assert_any_call( 'mytestfile.xml', 'w' )
assertMockCalls( open_mock().write, [ '&process_input\n test_bool = 0 0 0\n test_bool2 = 0.0 0.0\n test_integer = \n 1 20000\n 10 20000\n 1 20000\n/\n&beam_input\n test_float = dontchangeanything\n test_string = "some_teststring"\n test_noval = None\n/', '\n'], self )
def test_root(self): """create test for root 1""" print "test root" jobs = self.myTests.createRootScriptTest() assertDiracSucceeds( jobs, self ) thisJob = jobs['Value'] res = self.myTests.runJobLocally(thisJob, "Root") assertDiracSucceeds( res, self )
def test_resolvelinkedstepparams_nothing_happens(self):
instance_mock = Mock()
self.sco._inputappstep = None
self.sco._jobsteps = None
self.sco._linkedidx = ['abc']
assertDiracSucceeds(
self.sco._resolveLinkedStepParameters(instance_mock), self)
self.assertFalse(instance_mock.setLink.called)
def test_resolvestepparams( self ):
step_mock = Mock()
inputstep_mock = Mock()
self.slic._linkedidx = 2
self.slic._jobsteps = [ None, None, inputstep_mock ]
assertDiracSucceeds( self.slic._resolveLinkedStepParameters( step_mock ), self )
assertDiracSucceeds( self.slic._checkWorkflowConsistency(), self )
def test_checkconsistency_download_succeeds( self ):
import inspect
self.slic.steeringFile = '/my/remote/path.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_OK('ok'))):
assertDiracSucceeds( self.slic._checkConsistency(), self )
def test_resolvelinkedparams( self ):
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = { 'abc' : False }
self.pgs._linkedidx = 3
self.pgs._jobsteps = [ None, None, None, input_mock ]
assertDiracSucceeds( self.pgs._resolveLinkedStepParameters( step_mock ), self )
step_mock.setLink.assert_called_once_with( 'InputFile', { 'abc' : False }, 'OutputFile' )
def test_jobdone_nojobs(self):
con_mock = Mock()
with patch('%s.OverlayDB._query' % MODULE_NAME,
new=Mock(return_value=S_OK([[1]]))) as query_mock:
assertDiracSucceeds(self.odb.jobDone('my_TestSite1', con_mock),
self)
query_mock.assert_called_once_with(
"SELECT NumberOfJobs FROM OverlayData WHERE Site='my_TestSite1';",
con_mock)
def test_Whizard2_runIt_failure_LogFile_ignore(self):
"""Whizard2.runit failure with applicationLog but ignore..........................................."""
self.whiz.platform = "Windows"
self.whiz.applicationLog = self.logFileName
self.whiz.ignoreapperrors = True
## side effect for Script, userlibs, log, logAfter
with patch("os.path.exists", new=Mock(side_effect=[False, False, False, False] ) ):
res = self.whiz.runIt()
assertDiracSucceeds( res, self )
def test_execute_getcandidatefiles_fails( self ):
self.ujf.workflow_commons = { 'TotalSteps' : '42', 'Owner' : 'something',
'VO' : 'something_else' }
self.ujf.userOutputData = [ 'mylist' ]
with patch('%s.UserJobFinalization.getCandidateFiles' % MODULE_NAME, new=Mock(return_value=S_ERROR('get_candidate_files_test_err'))), \
patch('%s.UserJobFinalization.setApplicationStatus' % MODULE_NAME) as set_app_status_mock:
result = self.ujf.execute()
assertDiracSucceeds( result, self )
set_app_status_mock.assert_called_once_with( 'get_candidate_files_test_err' )
def test_applicationspecificinputs_nolist(self):
self.marAna.InputFile = ['dontleavemealone.txt']
self.marAna.workflow_commons[
'ParametricInputSandbox'] = 'myEntryies;another_one.slcio'
assertDiracSucceeds(self.marAna.applicationSpecificInputs(), self)
assertEqualsImproved(
(self.marAna.InputFile, self.marAna.InputData),
(['dontleavemealone.txt', 'myEntryies', 'another_one.slcio'], []),
self)
def test_applicationspecificinputs(self):
self.marAna.InputFile = ['firstEntry.txt']
self.marAna.workflow_commons[ 'ParametricInputSandbox' ] = \
[ 'addthis.slcio', 'anotherFile_butignoreme', 'lastfile.slcio' ]
assertDiracSucceeds(self.marAna.applicationSpecificInputs(), self)
assertEqualsImproved((self.marAna.InputFile, self.marAna.InputData), ([
'firstEntry.txt', 'addthis.slcio', 'anotherFile_butignoreme',
'lastfile.slcio'
], []), self)
def test_resolvestepparams(self):
step_mock = Mock()
inputstep_mock = Mock()
self.slic._linkedidx = 2
self.slic._jobsteps = [None, None, inputstep_mock]
assertDiracSucceeds(self.slic._resolveLinkedStepParameters(step_mock),
self)
assertDiracSucceeds(self.slic._checkWorkflowConsistency(), self)
def test_checkconsistency_userjob( self ): job_mock = Mock() job_mock.datatype = 'testDatatype' self.shs._job = job_mock self.shs._jobtype = 'User' self.shs.OutputFile = None assertDiracSucceeds( self.shs._checkConsistency(), self ) self.assertFalse( self.shs.outputFile ) assertEqualsImproved( self.shs.datatype, 'testDatatype', self )
def test_resolvelinkedstepparams( self ): instance_mock = Mock() step_mock = Mock() step_mock.getType.return_value = 'abc' self.shs._inputappstep = None self.shs._jobsteps = [ '', '', step_mock ] self.shs._linkedidx = 2 assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self ) instance_mock.setLink.assert_called_once_with( 'InputFile', 'abc', 'OutputFile' )
def test_checkconsistency(self):
self.sco._jobtype = 'notUser'
self.sco.OutputFile = None
assertDiracSucceeds(self.sco._checkConsistency(), self)
assertInImproved(
{
'outputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.sco._listofoutput, self)
def test_checkconsistency(self):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = False
self.slic._jobtype = 'notUser'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(return_value=S_OK('success!'))):
assertDiracSucceeds(self.slic._checkConsistency('myTestJob'), self)
def test_checkconsistency_list(self):
self.gsf.setFiles([{
'file': '/invalid/dir/mycoolfile.txt',
'something': True
}, {
'file': '/some/other/dir/other_file.123.stdhep'
}])
assertDiracSucceeds(self.gsf._checkConsistency(), self)
assertEqualsImproved(self.gsf.outputFile,
'mycoolfile.txt;other_file.123.stdhep', self)
def test_checkconsistency_othercase(self):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = True
self.slic._jobtype = 'User'
with patch('os.path.exists', new=Mock(side_effect=[ True ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(side_effect=IOError('dont call me'))):
assertDiracSucceeds(self.slic._checkConsistency('myTestJob'), self)
def test_prodjobmodules_outputpath(self):
module_mock = Mock()
self.fcc.outputPath = 'aef'
assertDiracSucceeds(self.fcc._prodjobmodules(module_mock), self)
self.assertIn(
{
'OutputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.fcc._listofoutput)
def test_Whizard2_runIt_failure_LogFile_ignore(self):
"""Whizard2.runit failure with applicationLog but ignore..........................................."""
self.whiz.platform = "Windows"
self.whiz.applicationLog = self.logFileName
self.whiz.ignoreapperrors = True
## side effect for Script, userlibs, log, logAfter
with patch("os.path.exists",
new=Mock(side_effect=[False, False, False, False])):
res = self.whiz.runIt()
assertDiracSucceeds(res, self)
def test_execute_nologs( self ):
log_mock = Mock()
self.ulf.log = log_mock
with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \
patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)), \
patch('%s.os.makedirs' % MODULE_NAME, new=Mock(side_effect=OSError('os_mkdir_failed_testerr_populate'))), \
patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()):
assertDiracSucceeds( self.ulf.execute(), self )
log_mock.error.assert_called_once_with( 'Completely failed to populate temporary log file directory.', '' )
def test_Whizard2_runIt_success(self):
"""Whizard2.runit ................................................................................."""
self.whiz.platform = 'Windows'
self.whiz.applicationLog = self.logFileName
## side effect for Script, userlibs, log, logAfter
with patch("os.path.exists",
new=Mock(side_effect=[False, False, False, True])):
res = self.whiz.runIt()
print res
assertDiracSucceeds(res, self)
def test_applicationspecificinputs_useworkflow(self):
""" Get the OutputFile from the workflow_commons dictionary, don't write to file """
self.shc.workflow_commons['IS_PROD'] = True
self.shc.workflow_commons[
'ProductionOutputData'] = ';/some/folder/with/afile.txt; GENfile2;/mydir/folder/file3_Gen_experiment1.stdhep;ignoreme me too; last_file\n'
self.shc.OutputFile = '1'
with patch('%s.open' % MODULE_NAME, new=Mock()) as open_mock:
assertDiracSucceeds(self.shc.applicationSpecificInputs(), self)
assertEqualsImproved((self.shc.OutputFile, self.shc.SteeringFile),
('file3_Gen_experiment1.stdhep', ''), self)
self.assertFalse(open_mock.called)
def test_applicationspecificinputs_sourcesempty(self):
""" Impossible to get the outputfile, use existing one """
self.shc.workflow_commons['IS_PROD'] = True
self.shc.workflow_commons[
'ProductionOutputData'] = ';/some/folder/with/afile.txt; GENfile2;ignoreme me too; last_file\n'
self.shc.OutputFile = 'testFile_dontchangeme'
with patch('%s.open' % MODULE_NAME, new=Mock()) as open_mock:
assertDiracSucceeds(self.shc.applicationSpecificInputs(), self)
assertEqualsImproved((self.shc.OutputFile, self.shc.SteeringFile),
('testFile_dontchangeme', ''), self)
self.assertFalse(open_mock.called)
def test_resolvelinkedstepparams(self):
instance_mock = Mock()
step_mock = Mock()
step_mock.getType.return_value = 'abc'
self.sco._inputappstep = None
self.sco._jobsteps = ['', '', step_mock]
self.sco._linkedidx = 2
assertDiracSucceeds(
self.sco._resolveLinkedStepParameters(instance_mock), self)
instance_mock.setLink.assert_called_once_with('InputFile', 'abc',
'OutputFile')