def test_checkconsistency_nouserjob_fails3( self ): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV \ndecay_proc' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith( self.whiz._checkConsistency( Mock() ), 'No model set in sin file, please set "model=..."', self )
def test_checkconsistency_nouserjob_fails6(self): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV\n model=SM\n model=Susy' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith(self.whiz._checkConsistency(Mock()), '"decay_proc" not found', self)
def test_setoutputsandbox_dictpassed( self ): wf_mock = Mock() self.ujo.workflow = wf_mock with patch('%s.UserJob._addParameter' % MODULE_NAME, new=Mock()) as addparam_mock: assertDiracFailsWith( self.ujo.setOutputSandbox( { 'mydict' : True } ), 'Expected file string or list of files for output sandbox contents', self ) self.assertFalse( addparam_mock.called )
def test_applicationSpecificInputs_nonbsigevts( self ): self.over.NbSigEvtsPerJob = 0 self.over.nbsigeventsperfile = 0 with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=2)): result = self.over.applicationSpecificInputs() assertDiracFailsWith( result, 'could not determine the number of signal events per input file', self )
def test_checkfinalconsistency_ops_fails(self): ops_mock = Mock() ops_mock.getSections.return_value = S_ERROR('some_ops_error') self.olin._ops = ops_mock self.olin.energy = 198 assertDiracFailsWith(self.olin._checkFinalConsistency(), 'could not resolve the cs path', self)
def test_checkinputsb_getreplicas_fails( self ): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf' ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} } with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock: assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self ) replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_execute_nopolicies(self): module_mock = Mock() module_mock.execute.side_effect = [{ 'OK': True, 'Successful': { 'some_site': True, 'OtherGoodSite': True } }] self.mfac_mock.ModuleFactory().getModule.return_value = S_OK( module_mock) from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution self.idr = InputDataResolution({ 'Configuration': { 'SiteName': 'myTestSitename' }, 'IgnoreMissing': True }) ops_mock = Mock() ops_mock.getOptionsDict.return_value = { 'Value': { 'wrong_sitename': ' module_path1,other_modpath , lastmodule', 'NotDefault': 'my/_path/module/ ,' } } self.idr.ops = ops_mock assertDiracFailsWith(self.idr.execute(), 'could not access any requested input data', self) self.assertFalse(self.mfac_mock.ModuleFactory().getModule.called)
def test_checkconsistency_no_backgroundevts(self): self.olin.pathToOverlayFiles = '' self.olin.bxToOverlay = True self.olin.numberOfGGToHadronInteractions = 0 assertDiracFailsWith( self.olin._checkConsistency(), 'background events per bunch crossing is not defined', self)
def test_checkconsistency_nosteeringfile(self): import inspect self.slic.steeringFile = 'mysteer.file' with patch('os.path.exists', new=Mock(return_value=False)), \ patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_ERROR('failed downloading, testerr'))): assertDiracFailsWith(self.slic._checkConsistency(), 'failed downloading, testerr', self)
def test_runit_log_failed_withapp(self): self.exists_dict[self.fccAna.applicationLog] = False self.fccAna.ignoreapperrors = False with patch('os.makedirs'), \ patch('os.path.exists') as mock_exists, \ patch("%s.shellCall" % MODULE_NAME, new=Mock(return_value={'OK' : True, 'Value' : ["", "stdout", "stderr"]})) as mock_shellcall: mock_exists.side_effect = self.replace_exists error_message = '%s did not produce the expected log %s' % ( self.fccAna.applicationName, self.fccAna.applicationLog) assertDiracFailsWith(self.fccAna.runIt(), error_message, self) self.log_mock.error.assert_any_call(error_message) self.log_mock.info.assert_any_call( "Environment : Environment script look up successfull") self.log_mock.debug.assert_any_call( "Application code : Creation of the bash script successfull") self.log_mock.debug.assert_any_call( "Application : Application execution and log file creation...") self.log_mock.debug.assert_any_call( "Application : Application execution successfull") mock_shellcall.assert_called_once_with( 0, self.fccAna.applicationScript, callbackFunction=self.fccAna.redirectLogOutput, bufferLimit=20971520)
def test_runmarlin_nosteeringfile(self): self.marAna.STEP_NUMBER = '932' self.marAna.applicationVersion = 'V1T' self.marAna.debug = False self.marAna.extraCLIarguments = 'testMyArgs' exists_dict = { './lib/lddlib': False, 'Marlin_V1T_Run_932.sh': False, 'inputxmlMy.test': False, './lib/': False, } def replace_exists(path): return exists_dict[path] with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \ patch('%s.os.remove' % MODULE_NAME) as remove_mock, \ patch('%s.shellCall' % MODULE_NAME) as shell_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.MarlinAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.marAna.runMarlin('inputxmlMy.test', 'envscript.path', 'marlin_dll') assertDiracFailsWith(result, 'steeringfile is missing', self) open_mock.assert_called_once_with('Marlin_V1T_Run_932.sh', 'w') open_mock().close.assert_called_once_with() self.assertFalse(remove_mock.called) self.assertFalse(shell_mock.called) self.assertFalse(appstat_mock.called) self.assertFalse(chmod_mock.called)
def test_runit_marlindllfails(self): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith(result, 'wrong with software installation', self)
def test_runit_xmlgenerationfails(self): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith(result, 'prepxml', self)
def test_checkconsistency_noefficiency( self ): self.shc.steeringFile = 'abc.xml' self.shc.inlineCuts = 'cuts.pdf' self.shc.maxNumberOfEvents = 14 self.shc.selectionEfficiency = None assertDiracFailsWith( self.shc._checkConsistency(), 'need to know the selection efficiency of your cuts', self )
def test_execute_input_missing_fail(self): module_mock = Mock() module_mock.execute.return_value = { 'OK': True, 'Failed': ['myReplicaSite1', 'other_failure'], 'Successful': { 'WorkingSite': True } } self.mfac_mock.ModuleFactory().getModule.return_value = S_OK( module_mock) from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution self.idr = InputDataResolution({ 'Job': { 'InputDataPolicy': ['/myTestPolicy/some/path'] }, 'Configuration': { 'SiteName': 'myTestSitename' } }) assertDiracFailsWith(self.idr.execute(), 'failed to access all of requested input data', self) self.mfac_mock.ModuleFactory().getModule.assert_called_once_with( '/myTestPolicy/some/path', { 'Job': { 'InputDataPolicy': ['/myTestPolicy/some/path'] }, 'Configuration': { 'SiteName': 'myTestSitename' } })
def test_checkconsistency_existsfails(self): self.dds.version = '134' self.dds.steeringFile = 'mysteer.file' with patch('os.path.exists', new=Mock(return_value=False)), \ patch.object(inspect.getmodule(DDSim), 'Exists', new=Mock(return_value=S_ERROR('testerr_exists_mock'))): assertDiracFailsWith(self.dds._checkConsistency(Mock()), 'testerr_exists_mock', self)
def test_runit_detXMLFails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = { 'OK' : True } self.marAna.detectorModel = "notCamelot.xml" result = self.marAna.runIt() assertDiracFailsWith( result, "zis iz not camelot", self )
def test_checkconsistency_no_bgevttype(self): self.olin.pathToOverlayFiles = '' self.olin.bxToOverlay = True self.olin.numberOfGGToHadronInteractions = 1345 self.olin.backgroundEventType = '' assertDiracFailsWith(self.olin._checkConsistency(), 'event type is not defined', self)
def test_runit_getEnvScriptFails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'env script', self )
def test_docheck_checkoutputpath_fails( self ): platform_mock = Mock() platform_mock.getValue.return_value = 'pf14081' apps_mock = Mock() apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0' path_mock = Mock() path_mock.getValue.return_value = 'path1948512895' param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock } job_mock = Mock() wf_mock = Mock() wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ] job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ] job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ] job_mock.workflow = wf_mock with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \ patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \ patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self ) checksb_mock.assert_called_once_with( job_mock ) job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value', 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ) job_mock._addParameter.assert_called_once_with( wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' ) assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ), ( 'pf14081', 'other_appv', '91.3' ), ( 'pf14081', 'more_dependencies', '1' ), ( 'pf14081', 'last_app', '0' ) ], self ) checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_runit_no_steeringfile( self ): exists_dict = { 'PandoraSettings.xml' : False, 'testGear.input' : False, '/steeringdir/testGear.input' : True, '/steeringdir/PandoraSettings.xml' : True, '' : True, 'lib': True, } def replace_exists( path ): return exists_dict[path] self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() self.marAna.detectorModel = None self.marAna.inputGEAR = '/my/secret/path/testGear.input' self.marAna.SteeringFile = '' with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('envscript_path.test'))), \ patch('%s.MarlinAnalysis._getInputFiles' % MODULE_NAME, new=Mock(return_value=S_OK(['testslcioList']))), \ patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/steeringdir'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \ patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('shutil_test_fail'))) as copy_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/curdir')): result = self.marAna.runIt() assertDiracFailsWith( result, 'could not find steering file', self ) self.log_mock.warn.assert_called_once_with('Could not copy PandoraSettings.xml, exception: shutil_test_fail') self.log_mock.error.assert_called_once_with("Steering file not defined, shouldn't happen!") copy_mock.assert_called_once_with( '/steeringdir/PandoraSettings.xml', '/test/curdir/PandoraSettings.xml' )
def test_getdestinationse_no_assoc_alias(self): options_dict = { '/Resources/StorageElements/myOutputSE': S_ERROR('bla') } option_dict = { '/Resources/Countries/country/AssignedTo': S_OK('MyTestCountry.Assigned'), '/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs': S_OK('something') } values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE': [], '/Resources/StorageElementGroups/myOutputSE': ['some_elements', 'ignore_me'], '/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE': [], '/Resources/Countries/country/AssociatedSEs/myOutputSE': '' } conf_mock = Mock() conf_mock.getOptions.side_effect = lambda path: options_dict[path] conf_mock.getOption.side_effect = lambda path: option_dict[path] conf_mock.getValue.side_effect = lambda path, _: values_dict[path] with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \ patch('%s.gConfig' % MODULE_NAME, conf_mock): result = getDestinationSEList('myOutputSE', 'pre.myTestSite.country', 'Local') assertDiracFailsWith(result, 'failed to resolve se', self)
def test_runit_xmlgenerationfails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'prepxml', self )
def test_execute_resolve_paths_fails( self ): self.shs.platform = 'TestPlatV1' self.shs.InputFile = 'something' with patch('%s.StdHepSplit.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_ERROR())) as resolve_mock: assertDiracFailsWith( self.shs.execute(), 'missing stdhep file', self ) resolve_mock.assert_called_once_with( 'something' )
def test_runit_marlindllfails( self ): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith( result, 'wrong with software installation', self )
def test_applicationSpecificInputs_allowedBkgFails_2( self ): self.over.pathToOverlayFiles = 'some_path.txt' self.over.detectormodel = '' self.over.detector = 'supergooddetectorv2' with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=-1)): result = self.over.applicationSpecificInputs() assertDiracFailsWith( result, 'no background to overlay', self )
def test_runmarlin_nosteeringfile( self ): self.marAna.STEP_NUMBER = '932' self.marAna.applicationVersion = 'V1T' self.marAna.debug = False self.marAna.extraCLIarguments = 'testMyArgs' exists_dict = { './lib/lddlib' : False, 'Marlin_V1T_Run_932.sh' : False, 'inputxmlMy.test' : False, './lib/': False, } def replace_exists( path ): return exists_dict[path] with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \ patch('%s.os.remove' % MODULE_NAME) as remove_mock, \ patch('%s.shellCall' % MODULE_NAME) as shell_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.MarlinAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.marAna.runMarlin( 'inputxmlMy.test', 'envscript.path', 'marlin_dll' ) assertDiracFailsWith( result, 'steeringfile is missing', self ) open_mock.assert_called_once_with( 'Marlin_V1T_Run_932.sh', 'w' ) open_mock().close.assert_called_once_with() self.assertFalse( remove_mock.called ) self.assertFalse( shell_mock.called ) self.assertFalse( appstat_mock.called ) self.assertFalse( chmod_mock.called )
def test_checkconsistency_nouserjob_fails5( self ): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'sqrts=350 GeV\n model=SM\n model=Susy \ndecay_proc' self.whiz.eventType = 'ee -> ff' self.whiz._jobtype = 'notUser' self.whiz.numberOfEvents = 100 assertDiracFailsWith( self.whiz._checkConsistency( Mock() ), 'Multiple instances of "model=..." detected, only one can be processed', self )
def test_checkconsistency_nouserjob_fails(self): self.whiz.version = '2.3.1' self.whiz.whizard2SinFile = 'mymodel.sin' self.whiz.numberOfEvents = 100 self.whiz._jobtype = 'notUser' assertDiracFailsWith(self.whiz._checkConsistency(Mock()), 'evttype not set, please set event type!', self)
def test_runit_no_steeringdir(self): exists_dict = { 'PandoraSettings.xml': True, 'testGear.input': False, '': True } def replace_exists(path): return exists_dict[path] self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() self.marAna.detectorModel = None self.marAna.inputGEAR = '/my/secret/path/testGear.input' self.marAna.SteeringFile = '' log_mock = Mock() self.marAna.log = log_mock with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('envscript_path.test'))), \ patch('%s.MarlinAnalysis._getInputFiles' % MODULE_NAME, new=Mock(return_value=S_OK(['testslcioList']))), \ patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytesterr'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)), \ patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('shutil_test_fail'))) as copy_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/curdir')): result = self.marAna.runIt() assertDiracFailsWith(result, 'could not find steering file', self) log_mock.warn.assert_called_once_with( 'Could not find the steering file directory', 'mytesterr') log_mock.error.assert_called_once_with( "Steering file not defined, shouldn't happen!") self.assertFalse(copy_mock.called)
def test_execute_metadata_not_found(self): self.upod.prodOutputLFNs = ['/ilc/prod/ilc/sid/example_file'] with patch.object(self.upod, 'getCandidateFiles', new=Mock(return_value=S_OK({}))), \ patch.object(self.upod, 'getFileMetadata', new=Mock(return_value=S_ERROR( 'no_meta_test_data_err' ))): assertDiracFailsWith(self.upod.execute(), 'no_meta_test_data_err', self) assertEqualsImproved(self.upod.experiment, 'ILC_SID', self)
def test_applicationSpecificInputs_nonbevts(self): self.over.InputData = ['abc'] with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=2)): result = self.over.applicationSpecificInputs() assertDiracFailsWith( result, 'number of events in the signal file is missing', self)
def test_getenvscript_pandorafrontend_missing(self): exists_dict = { 'PandoraFrontend': False, '/my/dir/test/me/Executable/PandoraFrontend': False } with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \ patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \ patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \ patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.spa.getEnvScript('mytestsysconfig', 'SLIC Pandora', 'V2') assertDiracFailsWith(result, 'missing pandorafrontend binary', self) getsoft_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') removelib_mock.assert_called_once_with('/my/dir/test/me/LDLibs') getlib_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') getpath_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') self.assertFalse(open_mock.called) self.assertFalse(chmod_mock.called) assertMockCalls(exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self)
def test_generatefailover_commit_fails(self): container_mock = Mock() container_mock.__len__.return_value = 1 container_mock.toJSON.return_value = S_OK('myjsonfilecontent_testme') container_mock.getDigest = S_ERROR('bla') reqval_mock = Mock() reqval_mock.validate.return_value = S_OK('ok') reqval_mock.getJSONData.return_value = True request_mock = Mock() request_mock.getJSONData.return_value = True report_mock = Mock() report_mock.generateForwardDISET.return_value = S_ERROR( 'test_faildiset') report_mock.getJSONData.return_value = True self.moba.jobReport = report_mock accreport_mock = Mock() accreport_mock.commit.return_value = S_ERROR('test_commiterr') self.moba.workflowStatus['OK'] = False self.moba.workflow_commons['AccountingReport'] = accreport_mock self.moba.workflow_commons[ 'ProductionOutputData'] = 'mylfn1;testmelfn2;' with patch('%s.RequestValidator' % MODULE_NAME, new=Mock(return_value=reqval_mock)), \ patch('%s.ModuleBase._getRequestContainer' % MODULE_NAME, new=Mock(return_value=container_mock)): result = self.moba.generateFailoverFile() assertDiracFailsWith(result, 'test_commiterr', self)
def test_runit_getEnvScriptFails(self): self.marAna.platform = "Testplatform123" self.marAna.applicationLog = "testlog123" self.marAna.stepStatus = S_OK() self.marAna.workflowStatus = S_OK() result = self.marAna.runIt() assertDiracFailsWith(result, 'env script', self)
def test_constructoutputlfns_proxy_missing( self ): self.ujf.jobID = 8234 self.ujf.workflow_commons = {} with patch('%s.getProxyInfo' % MODULE_NAME, new=Mock(return_value=S_ERROR('proxy_err_testme'))): result = self.ujf.constructOutputLFNs() assertDiracFailsWith( result, 'could not obtain owner from proxy', self ) assertEqualsImproved( self.ujf.jobID, 8234, self )
def test_execute_treatsteering_fails( self ): self.moba.step_commons['SteeringFileVers'] = 'mySteerTestVers' self.moba.platform = 'TRestPlatformMine' with patch('%s.getSteeringFileDir' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_filedir_steer'))) as steerdir_mock: assertDiracFailsWith( self.moba.execute(), 'failed to locate steering files mysteertestvers', self ) steerdir_mock.assert_called_once_with( 'TRestPlatformMine', 'mySteerTestVers' )
def test_addparams_fails(self): with patch.object( self.gsf, '_addBaseParameters', new=Mock(return_value=S_ERROR('my_patched_testerr'))): assertDiracFailsWith(self.gsf._addParametersToStep(None), 'failed to set base parameters', self)
def test_checkworkflow_app_missing(self): self.pgs._inputapp = [ 'some_depdency', 'unavailable_dependency_fail_on_this' ] self.pgs._jobapps = ['myjobapp_1', 'some_dependency'] assertDiracFailsWith(self.pgs._checkWorkflowConsistency(), 'job order not correct', self)
def test_runit_missing_numbers(self): exists_dict = { '/my/steer/dir/SteerFile.testme': False, 'SteerFile.testme': False, 'myAppTestLog.log': True } open_mock = Mock() open_mock.readlines.side_effect = [ 'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n' ] open_mock.__enter__.return_value = ['someline\n', 'otherline'] self.shc.applicationLog = 'myAppTestLog.log' self.shc.OutputFile = 'something' self.shc.SteeringFile = '/my/dir/SteerFile.testme' (self.shc.platform, self.shc.applicationName, self.shc.applicationVersion) = ('testPlatform', 'AppTestName', 'vT') with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 'status', )))), \ patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()), \ patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/my/steer/dir'))): assertDiracFailsWith(self.shc.runIt(), 'apptestname exited with status 1', self) remove_mock.assert_called_once_with('myAppTestLog.log')
def test_getcandidatefiles_missinglocally(self): exists_dict = { 'dir/testfile_allworks.stdhep': True, 'testfile_notlocal.txt': False } log_mock = Mock() self.moba.log = log_mock self.moba.ignoreapperrors = False mytest_outputlist = [{ 'outputFile': 'dir/testfile_allworks.stdhep', 'outputDataSE': 'testSE_dip4_allgood', 'outputPath': '/test/clic/ilc/mytestfile.txt' }, { 'outputFile': 'failhere', 'outputDataSE': '' }, { 'outputFile': 'testfile_notlocal.txt', 'outputDataSE': 'no_se', 'outputPath': '/test/clic/ilc/otherdir/newfile.txt' }] mylfns = [ 'testfile_allworks.stdhep', 'ignorethis', 'testfile_notlocal.txt' ] with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])): result = self.moba.getCandidateFiles(mytest_outputlist, mylfns, 'dummy_file_mask') assertDiracFailsWith(result, 'output data not found', self)
def test_limited_groupbyreplicas_fails(self): from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin from DIRAC import S_OK, S_ERROR dataman_mock = Mock() util_mock = Mock() util_mock.groupByReplicas.return_value = S_ERROR('group_test_err') trans_mock = Mock() trans_mock.getCounters.return_value = S_OK([({ 'Status': 'Processed' }, 3), ({ 'Status': 'junk' }, 6), ({ 'Status': 'Ignore_me' }, 8), ({ 'Status': 'Assigned' }, 1)]) util_mock.transClient = trans_mock self.tfp = TransformationPlugin('Limited', dataman_mock, trans_mock) self.tfp.params['Status'] = 'Processed' self.tfp.params['MaxNumberOfTasks'] = 12 self.tfp.params['TransformationID'] = 78456 self.tfp.util = util_mock self.tfp.setInputData([('/my/file/here_abc', ['myTestSE']), ('/my/file/other_file.txt', ['secretSE', 'StorageSE', 'TestSE']), ('/my/file/newfile.pdf', ['TestSE']), ('/my/file/a', []), ('/dir/somefile', [''])]) assertDiracFailsWith(self.tfp.run(), 'group_test_err', self)
def test_checkconsistency_nofccswpath(self): self.fcc.fccSwPath = None error_message = ( "FCCSW specific consistency : Error in parsing FCCSW application :\n" "You have to provide a valid path of the FCCSW installation") assertDiracFailsWith(self.fcc._checkConsistency(), error_message, self) self.log_mock.error.assert_called_once_with(error_message)
def test_runit_getenvscript_fails(self): self.rea.platform = 'myTestPlatform' self.rea.applicationLog = '/my/applog/test.log' self.rea.script = '/my/test/script.sh' with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_scripterr'))): assertDiracFailsWith(self.rea.runIt(), 'test_scripterr', self)
def test_runit_getswfolder_fails( self ): self.shc.OutputFile = 'something' ( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion ) = ( 'testPlatform', 'AppTestName', 'vT' ) with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_ERROR( 'getsw_test' ))) as getfolder_mock: assertDiracFailsWith( self.shc.runIt(), 'getsw_test', self ) getfolder_mock.assert_called_once_with( 'testPlatform', 'AppTestName', 'vT' )
def test_runit_missinglog(self): exists_dict = {'SteerFile.testme': True, 'myAppTestLog.log': False} open_mock = Mock() open_mock.readlines.side_effect = [ 'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n' ] self.shc.scriptName = 'my_test_script.sh' self.shc.OutputFile = 'something' self.shc.applicationLog = 'myAppTestLog.log' self.shc.SteeringFile = '/my/dir/SteerFile.testme' (self.shc.platform, self.shc.applicationName, self.shc.applicationVersion) = ('testPlatform', 'AppTestName', 'vT') with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.os.chmod' % MODULE_NAME, new=Mock()) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 'status', )))) as shell_mock, \ patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()) as prep_mock: assertDiracFailsWith( self.shc.runIt(), 'apptestname did not produce the expected log', self) self.assertFalse(remove_mock.called) shell_mock.assert_called_once_with( 0, 'sh -c "./my_test_script.sh"', callbackFunction=self.shc.redirectLogOutput, bufferLimit=20971520) chmod_mock.assert_called_once_with('my_test_script.sh', 0755) prep_mock.assert_called_once_with('SoftDir')
def test_execute_resolveinput_fails( self ): self.moba.isProdJob = True self.moba.InputData = 'myInputData.testme' with patch('%s.getNumberOfEvents' % MODULE_NAME, new=Mock(return_value=S_ERROR(''))) as getevts_mock: assertDiracFailsWith( self.moba.execute(), 'failed to get numberofevents from filecatalog', self ) getevts_mock.assert_called_once_with( 'myInputData.testme' )
def test_checkconsistency_unknownmodel(self): self.whiz.setFullParameterDict({ 'process_input': { 'process_id': 'process_1 some_test_process more_stuff', 'sqrts': 0 }, 'simulation_input': { 'n_events': 0 } }) self.whiz.setEvtType('some_test_evttype other_types ignore_us') self.whiz.setGlobalEvtType('myGlobalTestEvent') self.whiz.energy = 341 self.whiz.model = 'myObscureModel' self.whiz.version = '248' self.whiz.numberOfEvents = 1235 exists_dict = { 'process_1': True, 'some_test_process': True, 'more_stuff': True } plist_mock = Mock() plist_mock.existsProcess.side_effect = lambda process: S_OK( exists_dict[process]) plist_mock.getCSPath.return_value = 'some_dir/whizard248.tgz' self.whiz._generatormodels.models = {} self.whiz._processlist = plist_mock self.whiz.setGeneratorLevelCuts({'some_test_process': ['testlist']}) assertDiracFailsWith(self.whiz._checkConsistency(), 'unknown model myobscuremodel', self)
def test_checkconsistency_process_version_incompatible(self): self.whiz.setFullParameterDict({ 'process_input': { 'process_id': 'process_1 some_test_process more_stuff', 'sqrts': 0 }, 'simulation_input': { 'n_events': 0 } }) self.whiz.setEvtType('some_test_evttype other_types ignore_us') self.whiz.setGlobalEvtType('myGlobalTestEvent') self.whiz.energy = 341 self.whiz.version = 'v248' self.whiz.numberOfEvents = 1235 exists_dict = {'process_1': True, 'some_test_process': True} plist_mock = Mock() plist_mock.existsProcess.side_effect = lambda process: S_OK( exists_dict[process]) plist_mock.getCSPath.side_effect = [ 'some_dir/whizardv248.tgz', 'whizard418.tar.gz' ] self.whiz._processlist = plist_mock self.whiz.setGeneratorLevelCuts({'some_test_process': ['testlist']}) assertDiracFailsWith(self.whiz._checkConsistency(), 'not available in the same whizard version', self)
def test_getcandidatefiles_filenametoolong( self ): mytest_outputlist = [ { 'outputFile' : 'eruighnegjmneroiljger89igujmnerjhvreikvnmer9fig8erjg89iuerjhguie5hgieu7hg893j4tf4iufnugfyrhbgyukbfruwjfhwiefjhuiewfjwenfiuewnfieuwhuifrweuijfiuwerjhuiwer', 'outputDataSE' : 'testSE_dip4_allgood', 'outputPath' : '/test/clic/ilc/mytestfile.txt' }, { 'outputFile' : 'failhere', 'outputDataSE' : '' } ] mylfns = [ 'eruighnegjmneroiljger89igujmnerjhvreikvnmer9fig8erjg89iuerjhguie5hgieu7hg893j4tf4iufnugfyrhbgyukbfruwjfhwiefjhuiewfjwenfiuewnfieuwhuifrweuijfiuwerjhuiwer', 'ignorethis' ] assertDiracFailsWith( self.moba.getCandidateFiles( mytest_outputlist, mylfns, 'dummy_file_mask'), 'filename too long', self )
def test_setfilestatus_fails( self ): report_mock = Mock() report_mock.setFileStatus.return_value = S_ERROR('test_setfile_staterr') with patch('%s.FileReport' % MODULE_NAME, new=Mock(return_value=report_mock)): assertDiracFailsWith( self.moba.setFileStatus( 'production', 'lfn', 'status' ), 'test_setfile_staterr', self ) assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self ) self.log_mock.warn.assert_called_once_with('test_setfile_staterr')
def test_setfilestatus_fails_useexistingfilereport( self ): report_mock = Mock() report_mock.setFileStatus.return_value = S_ERROR('test_setfilestat_err') self.moba.workflow_commons['FileReport'] = report_mock assertDiracFailsWith( self.moba.setFileStatus( 'production', 'lfn', 'status' ), 'test_setfilestat_err', self ) assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self ) self.log_mock.warn.assert_called_once_with('test_setfilestat_err')
def test_sendstoredstatinfo_setting_fails( self ): self.moba.jobID = 24986 report_mock = Mock() report_mock.sendStoredStatusInfo.return_value = S_ERROR('failed setting appstat_testme') self.moba.workflow_commons['JobReport'] = report_mock assertDiracFailsWith( self.moba.sendStoredStatusInfo(), 'failed setting appstat_testme', self ) report_mock.sendStoredStatusInfo.assert_called_once_with() self.log_mock.error.assert_called_once_with('failed setting appstat_testme')
def test_runit_noscript( self ): self.rea.platform = 'myTestPlatform' self.rea.applicationLog = '/my/applog/test.log' self.rea.script = '' self.rea.workflowStatus['OK'] = True self.rea.stepStatus['OK'] = True with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))): assertDiracFailsWith( self.rea.runIt(), 'executable file not defined', self )
def test_checkparams_analyse_fails( self ): app_mock = Mock() app_mock.appname = 'myCoolTestApp' app_mock._analyseJob.return_value = S_ERROR('analyse_test_Err') dirac_mock = Mock() dirac_mock.checkparams.return_value = S_OK('dirac_test_retval') self.job.applicationlist = [ app_mock ] assertDiracFailsWith( self.job.checkparams( dirac_mock ), 'analyse_test_err', self )
def test_runit_resolveIFPaths_fails( self ): with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))) as getsoft_mock, \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_ERROR('slcio_err_test'))), \ patch('%s.SLICPandoraAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock: assertDiracFailsWith( self.spa.runIt(), 'missing slcio file', self ) getsoft_mock.assert_called_once_with( 'myTestPlatform', 'SLICPandora', '', self.spa.getEnvScript ) appstat_mock.assert_called_once_with( 'SLICPandora: missing slcio file' )
def test_execute_resolveinput_fails( self ): from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution self.idr = InputDataResolution( { 'Configuration': { 'SiteName' :'myTestSitename' } } ) ops_mock = Mock() ops_mock.getOptionsDict.return_value = None self.idr.ops = ops_mock assertDiracFailsWith( self.idr.execute(), 'Could not resolve InputDataPolicy from /InputDataPolicy', self ) ops_mock.getOptionsDict.assert_called_once_with( '/InputDataPolicy' )
def test_append_jobspecificenergy_wrong( self ): self.job.energy = 2451 app_mock = Mock() app_mock.energy = 214 app_mock._analyseJob.return_value = S_OK('') app_mock._checkConsistency.return_value = S_OK('') app_mock._checkFinalConsistency.side_effect = IOError('dont call me') assertDiracFailsWith( self.job.append( app_mock ), 'failed job specific checks', self )
def test_execute_runmodule_fails( self ): self.mfac_mock.ModuleFactory().getModule.return_value = S_ERROR( 'module_test_mockerr' ) from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution self.idr = InputDataResolution( { 'Configuration' : {}, 'Job' : { 'InputDataPolicy' : '/myTestPolicy/some/path' }, 'IgnoreMissing' : False } ) with patch('%s.DIRAC.siteName' % MODULE_NAME, new=Mock(return_value='SiteNameTestdirac')): assertDiracFailsWith( self.idr.execute(), 'module_test_mockerr', self )
def test_checkconsistency_energyzero( self ): self.whiz.setFullParameterDict( { 'process_input' : { 'process_id' : '', 'sqrts' : 0 } } ) self.whiz.eventType = 'some_test_evttype' self.whiz.energy = 0 assertDiracFailsWith( self.whiz._checkConsistency(), 'energy set to 0', self )