def test_checkconsistency_nouserjob_fails3( self ): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV \ndecay_proc' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith( self.whiz._checkConsistency( Mock() ), 'No model set in sin file, please set "model=..."', self )
def test_checkconsistency_nouserjob_fails6(self): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV\n model=SM\n model=Susy' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith(self.whiz._checkConsistency(Mock()), '"decay_proc" not found', self)
def test_setoutputsandbox_dictpassed( self ):
wf_mock = Mock()
self.ujo.workflow = wf_mock
with patch('%s.UserJob._addParameter' % MODULE_NAME, new=Mock()) as addparam_mock:
assertDiracFailsWith( self.ujo.setOutputSandbox( { 'mydict' : True } ),
'Expected file string or list of files for output sandbox contents', self )
self.assertFalse( addparam_mock.called )
def test_applicationSpecificInputs_nonbsigevts( self ):
self.over.NbSigEvtsPerJob = 0
self.over.nbsigeventsperfile = 0
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=2)):
result = self.over.applicationSpecificInputs()
assertDiracFailsWith( result, 'could not determine the number of signal events per input file',
self )
def test_checkfinalconsistency_ops_fails(self):
ops_mock = Mock()
ops_mock.getSections.return_value = S_ERROR('some_ops_error')
self.olin._ops = ops_mock
self.olin.energy = 198
assertDiracFailsWith(self.olin._checkFinalConsistency(),
'could not resolve the cs path', self)
def test_checkinputsb_getreplicas_fails( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} }
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_execute_nopolicies(self):
module_mock = Mock()
module_mock.execute.side_effect = [{
'OK': True,
'Successful': {
'some_site': True,
'OtherGoodSite': True
}
}]
self.mfac_mock.ModuleFactory().getModule.return_value = S_OK(
module_mock)
from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution
self.idr = InputDataResolution({
'Configuration': {
'SiteName': 'myTestSitename'
},
'IgnoreMissing': True
})
ops_mock = Mock()
ops_mock.getOptionsDict.return_value = {
'Value': {
'wrong_sitename':
' module_path1,other_modpath , lastmodule',
'NotDefault': 'my/_path/module/ ,'
}
}
self.idr.ops = ops_mock
assertDiracFailsWith(self.idr.execute(),
'could not access any requested input data', self)
self.assertFalse(self.mfac_mock.ModuleFactory().getModule.called)
def test_checkconsistency_no_backgroundevts(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 0
assertDiracFailsWith(
self.olin._checkConsistency(),
'background events per bunch crossing is not defined', self)
def test_checkconsistency_nosteeringfile(self):
import inspect
self.slic.steeringFile = 'mysteer.file'
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_ERROR('failed downloading, testerr'))):
assertDiracFailsWith(self.slic._checkConsistency(),
'failed downloading, testerr', self)
def test_runit_log_failed_withapp(self):
self.exists_dict[self.fccAna.applicationLog] = False
self.fccAna.ignoreapperrors = False
with patch('os.makedirs'), \
patch('os.path.exists') as mock_exists, \
patch("%s.shellCall" % MODULE_NAME, new=Mock(return_value={'OK' : True, 'Value' : ["", "stdout", "stderr"]})) as mock_shellcall:
mock_exists.side_effect = self.replace_exists
error_message = '%s did not produce the expected log %s' % (
self.fccAna.applicationName, self.fccAna.applicationLog)
assertDiracFailsWith(self.fccAna.runIt(), error_message, self)
self.log_mock.error.assert_any_call(error_message)
self.log_mock.info.assert_any_call(
"Environment : Environment script look up successfull")
self.log_mock.debug.assert_any_call(
"Application code : Creation of the bash script successfull")
self.log_mock.debug.assert_any_call(
"Application : Application execution and log file creation...")
self.log_mock.debug.assert_any_call(
"Application : Application execution successfull")
mock_shellcall.assert_called_once_with(
0,
self.fccAna.applicationScript,
callbackFunction=self.fccAna.redirectLogOutput,
bufferLimit=20971520)
def test_runmarlin_nosteeringfile(self):
self.marAna.STEP_NUMBER = '932'
self.marAna.applicationVersion = 'V1T'
self.marAna.debug = False
self.marAna.extraCLIarguments = 'testMyArgs'
exists_dict = {
'./lib/lddlib': False,
'Marlin_V1T_Run_932.sh': False,
'inputxmlMy.test': False,
'./lib/': False,
}
def replace_exists(path):
return exists_dict[path]
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \
patch('%s.os.remove' % MODULE_NAME) as remove_mock, \
patch('%s.shellCall' % MODULE_NAME) as shell_mock, \
patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.MarlinAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock:
result = self.marAna.runMarlin('inputxmlMy.test', 'envscript.path',
'marlin_dll')
assertDiracFailsWith(result, 'steeringfile is missing', self)
open_mock.assert_called_once_with('Marlin_V1T_Run_932.sh', 'w')
open_mock().close.assert_called_once_with()
self.assertFalse(remove_mock.called)
self.assertFalse(shell_mock.called)
self.assertFalse(appstat_mock.called)
self.assertFalse(chmod_mock.called)
def test_runit_marlindllfails(self):
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = S_OK()
self.marAna.workflowStatus = S_OK()
result = self.marAna.runIt()
assertDiracFailsWith(result, 'wrong with software installation', self)
def test_runit_xmlgenerationfails(self):
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = S_OK()
self.marAna.workflowStatus = S_OK()
result = self.marAna.runIt()
assertDiracFailsWith(result, 'prepxml', self)
def test_checkconsistency_noefficiency( self ):
self.shc.steeringFile = 'abc.xml'
self.shc.inlineCuts = 'cuts.pdf'
self.shc.maxNumberOfEvents = 14
self.shc.selectionEfficiency = None
assertDiracFailsWith( self.shc._checkConsistency(),
'need to know the selection efficiency of your cuts', self )
def test_execute_input_missing_fail(self):
module_mock = Mock()
module_mock.execute.return_value = {
'OK': True,
'Failed': ['myReplicaSite1', 'other_failure'],
'Successful': {
'WorkingSite': True
}
}
self.mfac_mock.ModuleFactory().getModule.return_value = S_OK(
module_mock)
from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution
self.idr = InputDataResolution({
'Job': {
'InputDataPolicy': ['/myTestPolicy/some/path']
},
'Configuration': {
'SiteName': 'myTestSitename'
}
})
assertDiracFailsWith(self.idr.execute(),
'failed to access all of requested input data',
self)
self.mfac_mock.ModuleFactory().getModule.assert_called_once_with(
'/myTestPolicy/some/path', {
'Job': {
'InputDataPolicy': ['/myTestPolicy/some/path']
},
'Configuration': {
'SiteName': 'myTestSitename'
}
})
def test_checkconsistency_existsfails(self):
self.dds.version = '134'
self.dds.steeringFile = 'mysteer.file'
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(DDSim), 'Exists', new=Mock(return_value=S_ERROR('testerr_exists_mock'))):
assertDiracFailsWith(self.dds._checkConsistency(Mock()),
'testerr_exists_mock', self)
def test_runit_detXMLFails( self ):
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = { 'OK' : True }
self.marAna.detectorModel = "notCamelot.xml"
result = self.marAna.runIt()
assertDiracFailsWith( result, "zis iz not camelot", self )
def test_checkconsistency_no_bgevttype(self):
self.olin.pathToOverlayFiles = ''
self.olin.bxToOverlay = True
self.olin.numberOfGGToHadronInteractions = 1345
self.olin.backgroundEventType = ''
assertDiracFailsWith(self.olin._checkConsistency(),
'event type is not defined', self)
def test_runit_getEnvScriptFails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'env script', self )
def test_docheck_checkoutputpath_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_runit_no_steeringfile( self ):
exists_dict = { 'PandoraSettings.xml' : False, 'testGear.input' : False,
'/steeringdir/testGear.input' : True,
'/steeringdir/PandoraSettings.xml' : True, '' : True,
'lib': True,
}
def replace_exists( path ):
return exists_dict[path]
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = S_OK()
self.marAna.workflowStatus = S_OK()
self.marAna.detectorModel = None
self.marAna.inputGEAR = '/my/secret/path/testGear.input'
self.marAna.SteeringFile = ''
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('envscript_path.test'))), \
patch('%s.MarlinAnalysis._getInputFiles' % MODULE_NAME, new=Mock(return_value=S_OK(['testslcioList']))), \
patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/steeringdir'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \
patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('shutil_test_fail'))) as copy_mock, \
patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/curdir')):
result = self.marAna.runIt()
assertDiracFailsWith( result, 'could not find steering file', self )
self.log_mock.warn.assert_called_once_with('Could not copy PandoraSettings.xml, exception: shutil_test_fail')
self.log_mock.error.assert_called_once_with("Steering file not defined, shouldn't happen!")
copy_mock.assert_called_once_with( '/steeringdir/PandoraSettings.xml',
'/test/curdir/PandoraSettings.xml' )
def test_getdestinationse_no_assoc_alias(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
option_dict = {
'/Resources/Countries/country/AssignedTo':
S_OK('MyTestCountry.Assigned'),
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs':
S_OK('something')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
[],
'/Resources/StorageElementGroups/myOutputSE':
['some_elements', 'ignore_me'],
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE':
[],
'/Resources/Countries/country/AssociatedSEs/myOutputSE':
''
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getOption.side_effect = lambda path: option_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country', 'Local')
assertDiracFailsWith(result, 'failed to resolve se', self)
def test_runit_xmlgenerationfails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'prepxml', self )
def test_execute_resolve_paths_fails( self ):
self.shs.platform = 'TestPlatV1'
self.shs.InputFile = 'something'
with patch('%s.StdHepSplit.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())), \
patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_ERROR())) as resolve_mock:
assertDiracFailsWith( self.shs.execute(), 'missing stdhep file', self )
resolve_mock.assert_called_once_with( 'something' )
def test_runit_marlindllfails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'wrong with software installation', self )
def test_applicationSpecificInputs_allowedBkgFails_2( self ):
self.over.pathToOverlayFiles = 'some_path.txt'
self.over.detectormodel = ''
self.over.detector = 'supergooddetectorv2'
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=-1)):
result = self.over.applicationSpecificInputs()
assertDiracFailsWith( result, 'no background to overlay', self )
def test_runmarlin_nosteeringfile( self ):
self.marAna.STEP_NUMBER = '932'
self.marAna.applicationVersion = 'V1T'
self.marAna.debug = False
self.marAna.extraCLIarguments = 'testMyArgs'
exists_dict = { './lib/lddlib' : False, 'Marlin_V1T_Run_932.sh' : False,
'inputxmlMy.test' : False,
'./lib/': False,
}
def replace_exists( path ):
return exists_dict[path]
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \
patch('%s.os.remove' % MODULE_NAME) as remove_mock, \
patch('%s.shellCall' % MODULE_NAME) as shell_mock, \
patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.MarlinAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock:
result = self.marAna.runMarlin( 'inputxmlMy.test', 'envscript.path',
'marlin_dll' )
assertDiracFailsWith( result, 'steeringfile is missing', self )
open_mock.assert_called_once_with( 'Marlin_V1T_Run_932.sh', 'w' )
open_mock().close.assert_called_once_with()
self.assertFalse( remove_mock.called )
self.assertFalse( shell_mock.called )
self.assertFalse( appstat_mock.called )
self.assertFalse( chmod_mock.called )
def test_checkconsistency_nouserjob_fails5( self ): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV\n model=SM\n model=Susy \ndecay_proc' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith( self.whiz._checkConsistency( Mock() ), 'Multiple instances of "model=..." detected, only one can be processed', self )
def test_checkconsistency_nouserjob_fails(self):
self.whiz.version = '2.3.1'
self.whiz.whizard2SinFile = 'mymodel.sin'
self.whiz.numberOfEvents = 100
self.whiz._jobtype = 'notUser'
assertDiracFailsWith(self.whiz._checkConsistency(Mock()),
'evttype not set, please set event type!', self)
def test_runit_no_steeringdir(self):
exists_dict = {
'PandoraSettings.xml': True,
'testGear.input': False,
'': True
}
def replace_exists(path):
return exists_dict[path]
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = S_OK()
self.marAna.workflowStatus = S_OK()
self.marAna.detectorModel = None
self.marAna.inputGEAR = '/my/secret/path/testGear.input'
self.marAna.SteeringFile = ''
log_mock = Mock()
self.marAna.log = log_mock
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('envscript_path.test'))), \
patch('%s.MarlinAnalysis._getInputFiles' % MODULE_NAME, new=Mock(return_value=S_OK(['testslcioList']))), \
patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytesterr'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \
patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('shutil_test_fail'))) as copy_mock, \
patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/curdir')):
result = self.marAna.runIt()
assertDiracFailsWith(result, 'could not find steering file', self)
log_mock.warn.assert_called_once_with(
'Could not find the steering file directory', 'mytesterr')
log_mock.error.assert_called_once_with(
"Steering file not defined, shouldn't happen!")
self.assertFalse(copy_mock.called)
def test_execute_metadata_not_found(self):
self.upod.prodOutputLFNs = ['/ilc/prod/ilc/sid/example_file']
with patch.object(self.upod, 'getCandidateFiles', new=Mock(return_value=S_OK({}))), \
patch.object(self.upod, 'getFileMetadata', new=Mock(return_value=S_ERROR( 'no_meta_test_data_err' ))):
assertDiracFailsWith(self.upod.execute(), 'no_meta_test_data_err',
self)
assertEqualsImproved(self.upod.experiment, 'ILC_SID', self)
def test_applicationSpecificInputs_nonbevts(self):
self.over.InputData = ['abc']
with patch('%s.Operations.getValue' % MODULE_NAME,
new=Mock(return_value=2)):
result = self.over.applicationSpecificInputs()
assertDiracFailsWith(
result, 'number of events in the signal file is missing', self)
def test_getenvscript_pandorafrontend_missing(self):
exists_dict = {
'PandoraFrontend': False,
'/my/dir/test/me/Executable/PandoraFrontend': False
}
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \
patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \
patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \
patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \
patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock:
result = self.spa.getEnvScript('mytestsysconfig', 'SLIC Pandora',
'V2')
assertDiracFailsWith(result, 'missing pandorafrontend binary',
self)
getsoft_mock.assert_called_once_with('mytestsysconfig',
'SLIC Pandora', 'V2')
removelib_mock.assert_called_once_with('/my/dir/test/me/LDLibs')
getlib_mock.assert_called_once_with('mytestsysconfig',
'SLIC Pandora', 'V2')
getpath_mock.assert_called_once_with('mytestsysconfig',
'SLIC Pandora', 'V2')
self.assertFalse(open_mock.called)
self.assertFalse(chmod_mock.called)
assertMockCalls(exists_mock, [
'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend'
], self)
def test_generatefailover_commit_fails(self):
container_mock = Mock()
container_mock.__len__.return_value = 1
container_mock.toJSON.return_value = S_OK('myjsonfilecontent_testme')
container_mock.getDigest = S_ERROR('bla')
reqval_mock = Mock()
reqval_mock.validate.return_value = S_OK('ok')
reqval_mock.getJSONData.return_value = True
request_mock = Mock()
request_mock.getJSONData.return_value = True
report_mock = Mock()
report_mock.generateForwardDISET.return_value = S_ERROR(
'test_faildiset')
report_mock.getJSONData.return_value = True
self.moba.jobReport = report_mock
accreport_mock = Mock()
accreport_mock.commit.return_value = S_ERROR('test_commiterr')
self.moba.workflowStatus['OK'] = False
self.moba.workflow_commons['AccountingReport'] = accreport_mock
self.moba.workflow_commons[
'ProductionOutputData'] = 'mylfn1;testmelfn2;'
with patch('%s.RequestValidator' % MODULE_NAME, new=Mock(return_value=reqval_mock)), \
patch('%s.ModuleBase._getRequestContainer' % MODULE_NAME, new=Mock(return_value=container_mock)):
result = self.moba.generateFailoverFile()
assertDiracFailsWith(result, 'test_commiterr', self)
def test_runit_getEnvScriptFails(self):
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = S_OK()
self.marAna.workflowStatus = S_OK()
result = self.marAna.runIt()
assertDiracFailsWith(result, 'env script', self)
def test_constructoutputlfns_proxy_missing( self ):
self.ujf.jobID = 8234
self.ujf.workflow_commons = {}
with patch('%s.getProxyInfo' % MODULE_NAME, new=Mock(return_value=S_ERROR('proxy_err_testme'))):
result = self.ujf.constructOutputLFNs()
assertDiracFailsWith( result, 'could not obtain owner from proxy', self )
assertEqualsImproved( self.ujf.jobID, 8234, self )
def test_execute_treatsteering_fails( self ):
self.moba.step_commons['SteeringFileVers'] = 'mySteerTestVers'
self.moba.platform = 'TRestPlatformMine'
with patch('%s.getSteeringFileDir' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_filedir_steer'))) as steerdir_mock:
assertDiracFailsWith( self.moba.execute(),
'failed to locate steering files mysteertestvers', self )
steerdir_mock.assert_called_once_with( 'TRestPlatformMine', 'mySteerTestVers' )
def test_addparams_fails(self):
with patch.object(
self.gsf,
'_addBaseParameters',
new=Mock(return_value=S_ERROR('my_patched_testerr'))):
assertDiracFailsWith(self.gsf._addParametersToStep(None),
'failed to set base parameters', self)
def test_checkworkflow_app_missing(self):
self.pgs._inputapp = [
'some_depdency', 'unavailable_dependency_fail_on_this'
]
self.pgs._jobapps = ['myjobapp_1', 'some_dependency']
assertDiracFailsWith(self.pgs._checkWorkflowConsistency(),
'job order not correct', self)
def test_runit_missing_numbers(self):
exists_dict = {
'/my/steer/dir/SteerFile.testme': False,
'SteerFile.testme': False,
'myAppTestLog.log': True
}
open_mock = Mock()
open_mock.readlines.side_effect = [
'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n'
]
open_mock.__enter__.return_value = ['someline\n', 'otherline']
self.shc.applicationLog = 'myAppTestLog.log'
self.shc.OutputFile = 'something'
self.shc.SteeringFile = '/my/dir/SteerFile.testme'
(self.shc.platform, self.shc.applicationName,
self.shc.applicationVersion) = ('testPlatform', 'AppTestName', 'vT')
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 'status', )))), \
patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()), \
patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/my/steer/dir'))):
assertDiracFailsWith(self.shc.runIt(),
'apptestname exited with status 1', self)
remove_mock.assert_called_once_with('myAppTestLog.log')
def test_getcandidatefiles_missinglocally(self):
exists_dict = {
'dir/testfile_allworks.stdhep': True,
'testfile_notlocal.txt': False
}
log_mock = Mock()
self.moba.log = log_mock
self.moba.ignoreapperrors = False
mytest_outputlist = [{
'outputFile': 'dir/testfile_allworks.stdhep',
'outputDataSE': 'testSE_dip4_allgood',
'outputPath': '/test/clic/ilc/mytestfile.txt'
}, {
'outputFile': 'failhere',
'outputDataSE': ''
}, {
'outputFile':
'testfile_notlocal.txt',
'outputDataSE':
'no_se',
'outputPath':
'/test/clic/ilc/otherdir/newfile.txt'
}]
mylfns = [
'testfile_allworks.stdhep', 'ignorethis', 'testfile_notlocal.txt'
]
with patch('%s.os.path.exists' % MODULE_NAME,
new=Mock(side_effect=lambda path: exists_dict[path])):
result = self.moba.getCandidateFiles(mytest_outputlist, mylfns,
'dummy_file_mask')
assertDiracFailsWith(result, 'output data not found', self)
def test_limited_groupbyreplicas_fails(self):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK, S_ERROR
dataman_mock = Mock()
util_mock = Mock()
util_mock.groupByReplicas.return_value = S_ERROR('group_test_err')
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK([({
'Status': 'Processed'
}, 3), ({
'Status': 'junk'
}, 6), ({
'Status': 'Ignore_me'
}, 8), ({
'Status': 'Assigned'
}, 1)])
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('Limited', dataman_mock, trans_mock)
self.tfp.params['Status'] = 'Processed'
self.tfp.params['MaxNumberOfTasks'] = 12
self.tfp.params['TransformationID'] = 78456
self.tfp.util = util_mock
self.tfp.setInputData([('/my/file/here_abc', ['myTestSE']),
('/my/file/other_file.txt',
['secretSE', 'StorageSE', 'TestSE']),
('/my/file/newfile.pdf', ['TestSE']),
('/my/file/a', []), ('/dir/somefile', [''])])
assertDiracFailsWith(self.tfp.run(), 'group_test_err', self)
def test_checkconsistency_nofccswpath(self):
self.fcc.fccSwPath = None
error_message = (
"FCCSW specific consistency : Error in parsing FCCSW application :\n"
"You have to provide a valid path of the FCCSW installation")
assertDiracFailsWith(self.fcc._checkConsistency(), error_message, self)
self.log_mock.error.assert_called_once_with(error_message)
def test_runit_getenvscript_fails(self):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = '/my/test/script.sh'
with patch('%s.getEnvironmentScript' % MODULE_NAME,
new=Mock(return_value=S_ERROR('test_scripterr'))):
assertDiracFailsWith(self.rea.runIt(), 'test_scripterr', self)
def test_runit_getswfolder_fails( self ):
self.shc.OutputFile = 'something'
( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion ) = ( 'testPlatform',
'AppTestName', 'vT' )
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_ERROR( 'getsw_test' ))) as getfolder_mock:
assertDiracFailsWith( self.shc.runIt(), 'getsw_test', self )
getfolder_mock.assert_called_once_with( 'testPlatform', 'AppTestName', 'vT' )
def test_runit_missinglog(self):
exists_dict = {'SteerFile.testme': True, 'myAppTestLog.log': False}
open_mock = Mock()
open_mock.readlines.side_effect = [
'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n'
]
self.shc.scriptName = 'my_test_script.sh'
self.shc.OutputFile = 'something'
self.shc.applicationLog = 'myAppTestLog.log'
self.shc.SteeringFile = '/my/dir/SteerFile.testme'
(self.shc.platform, self.shc.applicationName,
self.shc.applicationVersion) = ('testPlatform', 'AppTestName', 'vT')
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 'status', )))) as shell_mock, \
patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()) as prep_mock:
assertDiracFailsWith(
self.shc.runIt(),
'apptestname did not produce the expected log', self)
self.assertFalse(remove_mock.called)
shell_mock.assert_called_once_with(
0,
'sh -c "./my_test_script.sh"',
callbackFunction=self.shc.redirectLogOutput,
bufferLimit=20971520)
chmod_mock.assert_called_once_with('my_test_script.sh', 0755)
prep_mock.assert_called_once_with('SoftDir')
def test_execute_resolveinput_fails( self ):
self.moba.isProdJob = True
self.moba.InputData = 'myInputData.testme'
with patch('%s.getNumberOfEvents' % MODULE_NAME, new=Mock(return_value=S_ERROR(''))) as getevts_mock:
assertDiracFailsWith( self.moba.execute(),
'failed to get numberofevents from filecatalog', self )
getevts_mock.assert_called_once_with( 'myInputData.testme' )
def test_checkconsistency_unknownmodel(self):
self.whiz.setFullParameterDict({
'process_input': {
'process_id': 'process_1 some_test_process more_stuff',
'sqrts': 0
},
'simulation_input': {
'n_events': 0
}
})
self.whiz.setEvtType('some_test_evttype other_types ignore_us')
self.whiz.setGlobalEvtType('myGlobalTestEvent')
self.whiz.energy = 341
self.whiz.model = 'myObscureModel'
self.whiz.version = '248'
self.whiz.numberOfEvents = 1235
exists_dict = {
'process_1': True,
'some_test_process': True,
'more_stuff': True
}
plist_mock = Mock()
plist_mock.existsProcess.side_effect = lambda process: S_OK(
exists_dict[process])
plist_mock.getCSPath.return_value = 'some_dir/whizard248.tgz'
self.whiz._generatormodels.models = {}
self.whiz._processlist = plist_mock
self.whiz.setGeneratorLevelCuts({'some_test_process': ['testlist']})
assertDiracFailsWith(self.whiz._checkConsistency(),
'unknown model myobscuremodel', self)
def test_checkconsistency_process_version_incompatible(self):
self.whiz.setFullParameterDict({
'process_input': {
'process_id': 'process_1 some_test_process more_stuff',
'sqrts': 0
},
'simulation_input': {
'n_events': 0
}
})
self.whiz.setEvtType('some_test_evttype other_types ignore_us')
self.whiz.setGlobalEvtType('myGlobalTestEvent')
self.whiz.energy = 341
self.whiz.version = 'v248'
self.whiz.numberOfEvents = 1235
exists_dict = {'process_1': True, 'some_test_process': True}
plist_mock = Mock()
plist_mock.existsProcess.side_effect = lambda process: S_OK(
exists_dict[process])
plist_mock.getCSPath.side_effect = [
'some_dir/whizardv248.tgz', 'whizard418.tar.gz'
]
self.whiz._processlist = plist_mock
self.whiz.setGeneratorLevelCuts({'some_test_process': ['testlist']})
assertDiracFailsWith(self.whiz._checkConsistency(),
'not available in the same whizard version', self)
def test_getcandidatefiles_filenametoolong( self ):
mytest_outputlist = [ {
'outputFile' : 'eruighnegjmneroiljger89igujmnerjhvreikvnmer9fig8erjg89iuerjhguie5hgieu7hg893j4tf4iufnugfyrhbgyukbfruwjfhwiefjhuiewfjwenfiuewnfieuwhuifrweuijfiuwerjhuiwer', 'outputDataSE' : 'testSE_dip4_allgood', 'outputPath' : '/test/clic/ilc/mytestfile.txt'
}, { 'outputFile' : 'failhere', 'outputDataSE' : '' } ]
mylfns = [ 'eruighnegjmneroiljger89igujmnerjhvreikvnmer9fig8erjg89iuerjhguie5hgieu7hg893j4tf4iufnugfyrhbgyukbfruwjfhwiefjhuiewfjwenfiuewnfieuwhuifrweuijfiuwerjhuiwer',
'ignorethis' ]
assertDiracFailsWith( self.moba.getCandidateFiles(
mytest_outputlist, mylfns, 'dummy_file_mask'), 'filename too long', self )
def test_setfilestatus_fails( self ):
report_mock = Mock()
report_mock.setFileStatus.return_value = S_ERROR('test_setfile_staterr')
with patch('%s.FileReport' % MODULE_NAME, new=Mock(return_value=report_mock)):
assertDiracFailsWith( self.moba.setFileStatus( 'production', 'lfn', 'status' ),
'test_setfile_staterr', self )
assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self )
self.log_mock.warn.assert_called_once_with('test_setfile_staterr')
def test_setfilestatus_fails_useexistingfilereport( self ):
report_mock = Mock()
report_mock.setFileStatus.return_value = S_ERROR('test_setfilestat_err')
self.moba.workflow_commons['FileReport'] = report_mock
assertDiracFailsWith( self.moba.setFileStatus( 'production', 'lfn', 'status' ),
'test_setfilestat_err', self )
assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self )
self.log_mock.warn.assert_called_once_with('test_setfilestat_err')
def test_sendstoredstatinfo_setting_fails( self ):
self.moba.jobID = 24986
report_mock = Mock()
report_mock.sendStoredStatusInfo.return_value = S_ERROR('failed setting appstat_testme')
self.moba.workflow_commons['JobReport'] = report_mock
assertDiracFailsWith( self.moba.sendStoredStatusInfo(), 'failed setting appstat_testme', self )
report_mock.sendStoredStatusInfo.assert_called_once_with()
self.log_mock.error.assert_called_once_with('failed setting appstat_testme')
def test_runit_noscript( self ):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = ''
self.rea.workflowStatus['OK'] = True
self.rea.stepStatus['OK'] = True
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))):
assertDiracFailsWith( self.rea.runIt(), 'executable file not defined', self )
def test_checkparams_analyse_fails( self ):
app_mock = Mock()
app_mock.appname = 'myCoolTestApp'
app_mock._analyseJob.return_value = S_ERROR('analyse_test_Err')
dirac_mock = Mock()
dirac_mock.checkparams.return_value = S_OK('dirac_test_retval')
self.job.applicationlist = [ app_mock ]
assertDiracFailsWith( self.job.checkparams( dirac_mock ), 'analyse_test_err', self )
def test_runit_resolveIFPaths_fails( self ):
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))) as getsoft_mock, \
patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_ERROR('slcio_err_test'))), \
patch('%s.SLICPandoraAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock:
assertDiracFailsWith( self.spa.runIt(), 'missing slcio file', self )
getsoft_mock.assert_called_once_with( 'myTestPlatform', 'SLICPandora', '',
self.spa.getEnvScript )
appstat_mock.assert_called_once_with( 'SLICPandora: missing slcio file' )
def test_execute_resolveinput_fails( self ):
from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution
self.idr = InputDataResolution( { 'Configuration': { 'SiteName' :'myTestSitename' } } )
ops_mock = Mock()
ops_mock.getOptionsDict.return_value = None
self.idr.ops = ops_mock
assertDiracFailsWith( self.idr.execute(), 'Could not resolve InputDataPolicy from /InputDataPolicy', self )
ops_mock.getOptionsDict.assert_called_once_with( '/InputDataPolicy' )
def test_append_jobspecificenergy_wrong( self ):
self.job.energy = 2451
app_mock = Mock()
app_mock.energy = 214
app_mock._analyseJob.return_value = S_OK('')
app_mock._checkConsistency.return_value = S_OK('')
app_mock._checkFinalConsistency.side_effect = IOError('dont call me')
assertDiracFailsWith( self.job.append( app_mock ), 'failed job specific checks', self )
def test_execute_runmodule_fails( self ):
self.mfac_mock.ModuleFactory().getModule.return_value = S_ERROR( 'module_test_mockerr' )
from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution
self.idr = InputDataResolution( { 'Configuration' : {},
'Job' : { 'InputDataPolicy' : '/myTestPolicy/some/path' },
'IgnoreMissing' : False } )
with patch('%s.DIRAC.siteName' % MODULE_NAME, new=Mock(return_value='SiteNameTestdirac')):
assertDiracFailsWith( self.idr.execute(), 'module_test_mockerr', self )
def test_checkconsistency_energyzero( self ):
self.whiz.setFullParameterDict( { 'process_input' :
{ 'process_id' : '',
'sqrts' : 0 } } )
self.whiz.eventType = 'some_test_evttype'
self.whiz.energy = 0
assertDiracFailsWith( self.whiz._checkConsistency(), 'energy set to 0',
self )
