def test_transfer_register_and_failover_files( self ): self.ujf.failoverSEs = [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ] self.ujf.userFileCatalog = 'blaCatalogTestme' transfer_mock = Mock() transfer_mock.transferAndRegisterFileFailover.return_value = S_OK( {} ) filesUploaded = [] with patch('%s.random.shuffle' % MODULE_NAME) as shuffle_mock: result = self.ujf.transferRegisterAndFailoverFiles( transfer_mock, { 'filename1test' : { 'metadatadict1' : True, 'resolvedSE' : ( 'etc', 'privateSite4' ), 'localpath' : '/my/local/path/Testme.txt', 'lfn' : 'LFN:/ilc/mytest/LFN.txt', 'filedict' : 89546 }, 'my_other_testfile' : { 'metadatadict1' : True, 'resolvedSE' : ( 'testSite1', 'privateSite4' ), 'localpath' : '/my/local/path/Testme2.txt', 'lfn' : 'LFN:/ilc/othertest/new_lfn.lfn', 'filedict' : 475 } }, filesUploaded ) assertDiracSucceeds( result, self ) assertEqualsImproved( result['Value'], dict(cleanUp=False), self ) assertMockCalls( shuffle_mock, [ [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ], [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4', 'etc' ] ], self ) assertEqualsImproved( transfer_mock.transferAndRegisterFileFailover.mock_calls, [ call( 'my_other_testfile', '/my/local/path/Testme2.txt', 'LFN:/ilc/othertest/new_lfn.lfn', 'testSite1', [ 'testSite2', 'myTestSite', 'privateSite4', 'etc' ], fileCatalog='blaCatalogTestme', fileMetaDict=475 ), call( 'filename1test', '/my/local/path/Testme.txt', 'LFN:/ilc/mytest/LFN.txt', 'etc', [ 'testSite1', 'testSite2', 'myTestSite', 'privateSite4' ], fileCatalog='blaCatalogTestme', fileMetaDict=89546 ) ], self ) self.assertIn( 'LFN:/ilc/mytest/LFN.txt', filesUploaded ) self.assertIn( 'LFN:/ilc/othertest/new_lfn.lfn', filesUploaded )
def test_preparescript( self ): exists_dict = { 'TestApp_vT_Run_148.sh' : True, './lib' : True } ( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion, self.shc.STEP_NUMBER, self.shc.MaxNbEvts, self.shc.OutputFile, self.shc.SteeringFile ) = ( 'testPlatformV1', 'TestApp', 'vT', 148, 13, 'test_OF.ile', 'steer_test.file' ) file_contents = [[]] handles = FileUtil.getMultipleReadHandles(file_contents) with patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/my/testsoft/dir1/')) as getldlibs_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo: mo.side_effect = ( h for h in handles ) self.shc.prepareScript( 'test_software/dir' ) remove_mock.assert_called_once_with( 'TestApp_vT_Run_148.sh' ) getldlibs_mock.assert_called_once_with( 'testPlatformV1', 'TestApp', 'vT' ) assertMockCalls( exists_mock, [ './lib', 'TestApp_vT_Run_148.sh' ], self ) mo.assert_called_once_with( 'TestApp_vT_Run_148.sh', 'w' ) assertEqualsImproved( len( handles ), 1, self ) assertMockCalls( handles[0].write, [ '#!/bin/sh \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'declare -x PATH=test_software/dir:$PATH\n', 'declare -x LD_LIBRARY_PATH=./lib:test_software/dir/lib:/my/testsoft/dir1/\n', 'env | sort >> localEnv.log\n', 'echo =============================\n', "stdhepCut -m 13 -o test_OF.ile -c steer_test.file ../*.stdhep\n", 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self ) assert handles[0].close.called
def test_preparemarlindll_swaplibpositions(self): self.marAna.ProcessorListToExclude = ['mytestlibrary.so'] exists_dict = {'./lib/marlin_dll': True} def replace_exists(path): return exists_dict[path] file_contents = [[]] handles = FileUtil.getMultipleReadHandles(file_contents) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \ patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock: open_mock.side_effect = (h for h in handles) result = self.marAna.prepareMARLIN_DLL('some_path') open_mock.assert_called_once_with('temp.sh', 'w') assertMockCalls( handles[0].__enter__().write, ['#!/bin/bash\n', 'source some_path\necho $MARLIN_DLL'], self) assertEqualsImproved(len(handles), 1, self) chmod_mock.assert_called_once_with('temp.sh', 0755) shell_mock.assert_called_once_with(0, './temp.sh') remove_mock.assert_called_once_with('temp.sh') exists_mock.assert_called_once_with('./lib/marlin_dll') glob_mock.assert_called_once_with('./lib/marlin_dll/*.so') assertDiracSucceedsWith( result, 'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so', self)
def test_getters(self): from ILCDIRAC.Core.Utilities.WhizardOptions import WhizardOptions expected_pdict = { 'OK': True, 'Value': { 'integration_input': {}, 'simulation_input': {}, 'diagnostics_input': {}, 'process_input': {}, 'beam_input_1': {}, 'beam_input_2': {} } } pdict = self.whiz.getPDict() self.whiz.setEvtType('myevent_test') self.assertIsNotNone(pdict) self.whiz.setGlobalEvtType('test_myglobalevt') self.whiz.setLuminosity(138.312) self.whiz.setRandomSeed(9024) self.whiz.setParameterDict({ 'mytestval': True, 'more_entres': 'value', 'something': 9103 }) self.whiz.setGeneratorLevelCuts({ 'generator': False, 'cuts': 123, 'level': 'OK' }) assert not self.whiz.willBeCut self.whiz.willCut() assert not self.whiz.useGridFiles self.whiz.usingGridFiles() self.whiz.setJobIndex('mytestJobIndexS&P500') self.whiz.setModel('mytestSModel') self.whiz._wo = WhizardOptions(self.whiz.model) with patch('__builtin__.open', mock_open(), create=True) as mo: self.whiz.addedtojob = True self.whiz.dumpWhizardDotIn('someFile.in') self.whiz.addedtojob = False mo.assert_called_once_with('someFile.in', 'w') assertMockCalls(mo().write, [TestWhizard.EXPECTED_PRINTOUT, '\n'], self) assertEqualsImproved( (pdict, self.whiz.eventType, self.whiz.globalEventType, self.whiz.luminosity, self.whiz.randomSeed, self.whiz.parameterDict, self.whiz.generatorLevelCuts, self.whiz.willBeCut, self.whiz.useGridFiles, self.whiz.jobIndex, self.whiz.model), (expected_pdict, 'myevent_test', 'test_myglobalevt', 138.312, 9024, { 'mytestval': True, 'more_entres': 'value', 'something': 9103 }, { 'generator': False, 'cuts': 123, 'level': 'OK' }, True, True, 'mytestJobIndexS&P500', 'mytestSModel'), self) assert not self.whiz._errorDict
def test_docheck_checkoutputpath_fails( self ): platform_mock = Mock() platform_mock.getValue.return_value = 'pf14081' apps_mock = Mock() apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0' path_mock = Mock() path_mock.getValue.return_value = 'path1948512895' param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock } job_mock = Mock() wf_mock = Mock() wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ] job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ] job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ] job_mock.workflow = wf_mock with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \ patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \ patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self ) checksb_mock.assert_called_once_with( job_mock ) job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value', 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ) job_mock._addParameter.assert_called_once_with( wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' ) assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ), ( 'pf14081', 'other_appv', '91.3' ), ( 'pf14081', 'more_dependencies', '1' ), ( 'pf14081', 'last_app', '0' ) ], self ) checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_getenvscript_other_prefixpath( self ): def replace_abspath( path ): """ Mock implementation of os.path.abspath """ if path == 'SLICPandora.sh': return '/abs/test/path/SLICPandora.sh' else: sys.exit() exists_dict = { 'PandoraFrontend' : False, '/my/dir/test/me/Executable/PandoraFrontend' : True } with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \ patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \ patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \ patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock: with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.os.path.abspath' % MODULE_NAME, new=Mock(side_effect=replace_abspath)) as abspath_mock: result = self.spa.getEnvScript( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) chmod_mock.assert_called_once_with('SLICPandora.sh', 0o755) assertMockCalls( exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self ) abspath_mock.assert_called_once_with( 'SLICPandora.sh' ) assertDiracSucceedsWith_equals( result, '/abs/test/path/SLICPandora.sh', self ) getsoft_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) removelib_mock.assert_called_once_with( '/my/dir/test/me/LDLibs' ) getlib_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) getpath_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) open_mock.assert_any_call( 'SLICPandora.sh', 'w' ) open_mock = open_mock() assertMockCalls( open_mock.write, [ '#!/bin/sh \n', '############################################################\n', '# Dynamically generated script to get the SLICPandora env. #\n', '############################################################\n', "declare -x PATH=/new/test/path:$PATH\n", 'declare -x ROOTSYS=/my/dir/test/me/ROOT\n', 'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:/my/dir/test/me/LDLibs:/new/ldpath\n', 'declare -x PANDORASETTINGSDIR=/my/dir/test/me/Settings\n', "declare -x PATH=/my/dir/test/me/Executable:$PATH\n" ], self )
def test_getenvscript_pandorafrontend_missing(self): exists_dict = { 'PandoraFrontend': False, '/my/dir/test/me/Executable/PandoraFrontend': False } with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \ patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \ patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \ patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.spa.getEnvScript('mytestsysconfig', 'SLIC Pandora', 'V2') assertDiracFailsWith(result, 'missing pandorafrontend binary', self) getsoft_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') removelib_mock.assert_called_once_with('/my/dir/test/me/LDLibs') getlib_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') getpath_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') self.assertFalse(open_mock.called) self.assertFalse(chmod_mock.called) assertMockCalls(exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self)
def test_preparescript(self): exists_dict = {'TestApp_vT_Run_148.sh': True, './lib': True} (self.shc.platform, self.shc.applicationName, self.shc.applicationVersion, self.shc.STEP_NUMBER, self.shc.MaxNbEvts, self.shc.OutputFile, self.shc.SteeringFile) = ('testPlatformV1', 'TestApp', 'vT', 148, 13, 'test_OF.ile', 'steer_test.file') file_contents = [[]] handles = FileUtil.getMultipleReadHandles(file_contents) with patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/my/testsoft/dir1/')) as getldlibs_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo: mo.side_effect = (h for h in handles) self.shc.prepareScript('test_software/dir') remove_mock.assert_called_once_with('TestApp_vT_Run_148.sh') getldlibs_mock.assert_called_once_with('testPlatformV1', 'TestApp', 'vT') assertMockCalls(exists_mock, ['./lib', 'TestApp_vT_Run_148.sh'], self) mo.assert_called_once_with('TestApp_vT_Run_148.sh', 'w') assertEqualsImproved(len(handles), 1, self) assertMockCalls(handles[0].write, [ '#!/bin/sh \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'declare -x PATH=test_software/dir:$PATH\n', 'declare -x LD_LIBRARY_PATH=./lib:test_software/dir/lib:/my/testsoft/dir1/\n', 'env | sort >> localEnv.log\n', 'echo =============================\n', "stdhepCut -m 13 -o test_OF.ile -c steer_test.file ../*.stdhep\n", 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self) assert handles[0].close.called
def test_preparemarlindll_swaplibpositions( self ): self.marAna.ProcessorListToExclude = [ 'mytestlibrary.so' ] exists_dict = { './lib/marlin_dll' : True } def replace_exists( path ): return exists_dict[path] file_contents = [[]] handles = FileUtil.getMultipleReadHandles( file_contents ) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \ patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock: open_mock.side_effect = (h for h in handles) result = self.marAna.prepareMARLIN_DLL( 'some_path' ) open_mock.assert_called_once_with( 'temp.sh', 'w' ) assertMockCalls( handles[0].__enter__().write, ['#!/bin/bash\n', 'source some_path > /dev/null\necho $MARLIN_DLL'], self) assertEqualsImproved( len( handles ), 1, self ) chmod_mock.assert_called_once_with('temp.sh', 0o755) shell_mock.assert_called_once_with( 0, './temp.sh' ) remove_mock.assert_called_once_with( 'temp.sh' ) exists_mock.assert_called_once_with( './lib/marlin_dll' ) glob_mock.assert_called_once_with( './lib/marlin_dll/*.so' ) assertDiracSucceedsWith( result, 'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so', self )
def test_removesyslibs(self): walk_list = [('/my/usr/lib/', 'ignore', [ '', 'libc.so.2', 'myfile.txt', 'libc-2.5.so', 'libm.so.1', 'libpthread.so.123', 'ignore_me.log', 'libdl.so', 'ignore_me.so.1', 'libstdc++.so.1', 'ignore_me.so', 'libgcc_s.so.1' ]), ('/lib/dir/subdir/', 'ignoretoo', [ 'some_file.so', 'other_lib.so.1', 'libc.so', 'libc-2.5.so.1', 'libm.so', 'libpthread.so', 'libdl.so.124', 'libstdc++.so', 'libgcc_s.so.1.18' ])] remove_vals = [None, OSError('test_remove_failure')] + [None] * 12 with patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)) as walk_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=remove_vals)) as remove_mock: removeSystemLibraries('my_folder_tbr') walk_mock.assert_called_once_with('my_folder_tbr') expected_remove_files_1 = [ 'libc.so.2', 'libc-2.5.so', 'libm.so.1', 'libpthread.so.123', 'libdl.so', 'libstdc++.so.1', 'libgcc_s.so.1' ] expected_remove_files_2 = [ 'libc.so', 'libc-2.5.so.1', 'libm.so', 'libpthread.so', 'libdl.so.124', 'libstdc++.so', 'libgcc_s.so.1.18' ] assertMockCalls( remove_mock, ['/my/usr/lib/' + x for x in expected_remove_files_1] + ['/lib/dir/subdir/' + x for x in expected_remove_files_2], self)
def test_uploadproclist( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK( '/local/path/proc.list' ) mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock with patch('shutil.copy') as copy_mock, \ patch('subprocess.call') as proc_mock: self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' ) assertMockCalls( copy_mock, [ ( 'myTestProcess.list', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/' ), ( 'myTestProcess.list', '/local/path/proc.list' ) ], self ) proc_mock.assert_called_once_with( [ 'svn', 'ci', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list', "-m'Process list for whizard version v120'" ], shell=False ) DIRAC.gConfig = backup_conf module_patcher.stop()
def test_execute_ignoremissing_works( self ): module_mock = Mock() module_mock.execute.side_effect = [ { 'OK' : True, 'Failed' : [ 'myReplicaSite1', 'other_failure' ], 'Successful' : { 'WorkingSite1' : True, 'OtherGoodSite' : True } }, { 'OK' : True, 'Failed' : [], 'Successful' : { 'other_site' : True, 'TestSite1' : True } } ] self.mfac_mock.ModuleFactory().getModule.return_value = S_OK( module_mock ) from ILCDIRAC.Core.Utilities.InputDataResolution import InputDataResolution self.idr = InputDataResolution( { 'Job' : { 'InputDataPolicy' : [ '/myTestPolicy/some/path', 'other_policy/path' ] }, 'Configuration': { 'SiteName' :'myTestSitename' }, 'IgnoreMissing' : True } ) res = self.idr.execute() assertDiracSucceeds( res, self ) assertEqualsImproved( res, { 'OK' : True, 'Value' : None, 'Failed' : [], 'Successful' : { 'WorkingSite1' : True, 'OtherGoodSite' : True, 'other_site' : True, 'TestSite1' : True } }, self ) assertMockCalls( self.mfac_mock.ModuleFactory().getModule, [ ( '/myTestPolicy/some/path', { 'Job' : { 'InputDataPolicy' : [ '/myTestPolicy/some/path', 'other_policy/path' ] }, 'Configuration' : { 'SiteName' : 'myTestSitename' }, 'IgnoreMissing' : True } ), ( 'other_policy/path', { 'Job': { 'InputDataPolicy' : [ '/myTestPolicy/some/path', 'other_policy/path' ] }, 'Configuration' : { 'SiteName' : 'myTestSitename' }, 'IgnoreMissing' : True } ) ], self )
def test_getrootstuff(self): file_list = [ '/my_test/aim/lib/some_lib.so', '/my_test/aim/lib/my_important_lib.so.1', '/my_test/aim/lib/last_lib.so.123' ] dependency_list = [{'first_dependency.so', 'base_lib.so.1'}, {}, {'last_dependency.so.39'}] with patch('%s.commands.getstatusoutput' % MODULE_NAME, new=Mock(return_value=(0, ''))) as cmd_mock, \ patch('%s.getFiles' % MODULE_NAME, new=Mock(return_value=file_list)) as getfile_mock, \ patch('%s.getDependentLibraries' % MODULE_NAME, new=Mock(side_effect=dependency_list)) as getlibs_mock, \ patch('%s.copyLibraries' % MODULE_NAME, new=Mock()) as copy_mock: getRootStuff('test_rootsys', '/my_test/aim') cmd_mock.assert_called_once_with( "rsync --exclude '.svn' -av test_rootsys/lib test_rootsys/etc test_rootsys/bin test_rootsys/cint /my_test/aim" ) getfile_mock.assert_called_once_with('/my_test/aim/lib', '.so') assertMockCalls(getlibs_mock, [ '/my_test/aim/lib/some_lib.so', '/my_test/aim/lib/my_important_lib.so.1', '/my_test/aim/lib/last_lib.so.123' ], self) copy_mock.assert_called_once_with( { 'first_dependency.so', 'base_lib.so.1', 'last_dependency.so.39' }, '/my_test/aim/lib')
def test_towhizarddotin( self ): self.whop.whizardxml = fromstring( '<whizard> <process_input> <mytestelement type="floatarray" value="sqrts"> </mytestelement> <test_bool type="T/F" value="000"> </test_bool> <test_bool2 type="T/F" value="0.0.0"> </test_bool2> <test_integer type="integer" value="0..0..0"> </test_integer> </process_input> <lalabeam_input1238> <test_float type="float" value="dontchangeanything"> </test_float> <test_string type="string" value="some_teststring"> </test_string> <test_noval></test_noval> </lalabeam_input1238> </whizard>' ) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock: result = self.whop.toWhizardDotIn( 'mytestfile.xml' ) assertDiracSucceeds( result, self ) open_mock.assert_any_call( 'mytestfile.xml', 'w' ) assertMockCalls( open_mock().write, [ '&process_input\n test_bool = 0 0 0\n test_bool2 = 0.0 0.0\n test_integer = \n 1 20000\n 10 20000\n 1 20000\n/\n&beam_input\n test_float = dontchangeanything\n test_string = "some_teststring"\n test_noval = None\n/', '\n'], self )
def test_setoutputdata( self ): wf_mock = Mock() self.ujo.workflow = wf_mock with patch('%s.UserJob._addParameter' % MODULE_NAME, new=Mock()) as addparam_mock: assertDiracSucceeds( self.ujo.setOutputData( [ 'mylfn1', 'other_lfn', 'last___lfn' ], OutputPath = '//some/dir/somedir/output.xml' ), self ) assertMockCalls( addparam_mock, [ ( wf_mock, 'UserOutputData', 'JDL', 'mylfn1;other_lfn;last___lfn', 'List of output data files' ), ( wf_mock, 'UserOutputPath', 'JDL', 'some/dir/somedir/output.xml', 'User specified Output Path' ) ], self )
def test_checkapp( self ): ops_mock = Mock() ops_mock.getValue.return_value = '' self.dilc.ops = ops_mock assertDiracFailsWith( self.dilc._checkapp( 'test_platform_341', 'testapp', 'v13.2' ), 'could not find the specified software testapp_v13.2 for test_platform_341, check in CS', self ) assertMockCalls( ops_mock.getValue, [ ( '/AvailableTarBalls/test_platform_341/testapp/v13.2/TarBall', '' ), ( '/AvailableTarBalls/test_platform_341/testapp/v13.2/CVMFSPath', '' ) ], self )
def test_killrpath( self ): walk_list = [ ( '/testRoot', 'nodirs', [ 'file1.txt', 'deletethislib.so', '', 'ignorefile.s o' ] ), ( '/otherRoot/mydir/', '', [ 'lib1.so', 'subfolder/otherlib.so', 'library.dll' ] ) ] with patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)) as walk_mock, \ patch('%s.commands.getstatusoutput' % MODULE_NAME, new=Mock()) as cmd_mock: killRPath( 'myTest_folder' ) walk_mock.assert_called_once_with( 'myTest_folder', followlinks=True ) expected_files = [ '/testRoot/deletethislib.so', '/otherRoot/mydir/lib1.so', '/otherRoot/mydir/subfolder/otherlib.so' ] assertMockCalls( cmd_mock, [ 'chrpath -d ' + x + ' ' for x in expected_files ], self )
def test_getsoftwarefolder_notfound( self ): exists_dict = { 'myapp_executable' : False, '/mylocalarea/test/myapp_executable' : False, '/testshared/area/myapp_executable' : False } with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapp_executable')) as getval_mock, \ patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \ patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: result = getSoftwareFolder( 'a', 'b', 'c' ) assertMockCalls( exists_mock, [ '/mylocalarea/test/myapp_executable', '/testshared/area/myapp_executable', 'myapp_executable' ], self ) getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' ) assertDiracFailsWith( result, 'missing installation of myapp_executable', self )
def test_getsoftwarefolder_uselocal( self ): exists_dict = { 'myapparchivev2.test.tar.gz' : False, '/mylocalarea/test/myapparchivev2.test' : True } with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \ patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \ patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: result = getSoftwareFolder( 'a', 'b', 'c' ) assertMockCalls( exists_mock, [ '/mylocalarea/test/myapparchivev2.test', 'myapparchivev2.test.tar.gz' ], self ) getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' ) assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self )
def test_setstepparams(self): job_mock = Mock() self.gapp._job = job_mock self.gapp.dependencies = { 'mydependency': 'v1', 'other_library': 'v100testme' } self.gapp._setStepParametersValues(Mock()) assertMockCalls(job_mock._addSoftware, [('mydependency', 'v1'), ('other_library', 'v100testme')], self)
def test_remove_libc_remove_fails(self): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock: result = removeLibc('mytestpath') self.assertFalse(result) assertMockCalls(chdir_mock, ['current_dir', 'mytestpath'], self) listdir_mock.assert_called_once_with('mytestpath') remove_mock.assert_called_once_with('mytestpath/libc.so') assertMockCalls(getcwd_mock, [()] * 3, self)
def test_runit_no_detectormodel_found( self ): exists_dict = { 'testmodelDetector.xml' : True, '/secret/dir/testmodelDetector.xml' : False } self.spa.detectorxml = '/secret/dir/testmodelDetector.xml' self.spa.InputFile = 'testInput.file' with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))) as resolve_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: assertDiracFailsWith( self.spa.runIt(), 'detector model xml was not found, exiting', self ) assertMockCalls( exists_mock, [ 'testmodelDetector.xml', '/secret/dir/testmodelDetector.xml' ], self ) resolve_mock.assert_called_once_with( 'testInput.file' )
def test_remove_libc_remove_fails( self ): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock: result = removeLibc( 'mytestpath' ) self.assertFalse( result ) assertMockCalls( chdir_mock, [ 'current_dir', 'mytestpath' ], self ) listdir_mock.assert_called_once_with( 'mytestpath' ) remove_mock.assert_called_once_with( 'mytestpath/libc.so' ) assertMockCalls( getcwd_mock, [ () ] * 3, self )
def test_remove_libc_nothing_to_remove( self ): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock: result = removeLibc( 'mytestpath' ) self.assertTrue( result ) assertMockCalls( chdir_mock, [ 'mytestpath', 'current_dir' ], self ) listdir_mock.assert_called_once_with( 'mytestpath' ) self.assertFalse( remove_mock.called ) assertMockCalls( getcwd_mock, [ () ] * 2, self )
def test_remove_libc_nothing_to_remove(self): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock: result = removeLibc('mytestpath') self.assertTrue(result) assertMockCalls(chdir_mock, ['mytestpath', 'current_dir'], self) listdir_mock.assert_called_once_with('mytestpath') self.assertFalse(remove_mock.called) assertMockCalls(getcwd_mock, [()] * 2, self)
def test_runit( self ): exists_dict = { 'testmodelDetector_pandora.xml' : False, '/secret/dir/testmodelDetector_pandora.xml' : True, '/secret/dir/testmodelDetector.zip' : False, 'SLICPandora__Run_465.sh' : True, './lib' : False, 'applogFile.test' : True } self.spa.detectorxml = '/secret/dir/testmodelDetector_pandora.xml' self.spa.InputFile = 'testInput.file' self.spa.STEP_NUMBER = 465 ops_mock = Mock() ops_mock.getValue.return_value = [ 'detector_1_url_throw_error.pdf', 'detector2_unzip_fails.xml', 'working_detector_v4.xml' ] self.spa.ops = ops_mock mo = mock_open( read_data='some_log_data\nsuccessful finish :)') with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))), \ patch('%s.unzip_file_into_dir' % MODULE_NAME, new=Mock(side_effect=[ OSError('unzipping failed'), True ])) as unzip_mock, \ patch('%s.os.unlink' % MODULE_NAME) as unlink_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.open' % MODULE_NAME, mo ) as open_mock, \ patch('%s.urllib.urlretrieve' % MODULE_NAME, new=Mock(side_effect=[ IOError('my_test_ioerr'), True, True ])), \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([0, 'Disabled execution', '']))) as shell_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/curdir/test')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, patch('%s.os.remove' % MODULE_NAME) as remove_mock: assertDiracSucceeds( self.spa.runIt(), self ) assertMockCalls( exists_mock, [ 'testmodelDetector_pandora.xml', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector_pandora.xml', 'SLICPandora__Run_465.sh', './lib', 'applogFile.test', 'applogFile.test' ], self ) assertMockCalls( open_mock, [ '/secret/dir/testmodelDetector.zip', ( 'SLICPandora__Run_465.sh', 'w' ), ( 'applogFile.test', 'r' ) ], self, only_these_calls = False ) mo = mo() assertMockCalls( unzip_mock, [ ( mo, '/my/curdir/test' ), ( mo, '/my/curdir/test') ], self ) open_mock = open_mock() assertMockCalls( open_mock.write, [ '#!/bin/bash \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'source myenvscriptpathtestme\n', 'declare -x file=./Settings/\n', '\nif [ -e "${file}" ]\nthen\n declare -x PANDORASETTINGS=$file\nelse\n if [ -d "${PANDORASETTINGSDIR}" ]\n then\n cp $PANDORASETTINGSDIR/*.xml .\n declare -x PANDORASETTINGS=\n fi\nfi\nif [ ! -e "${PANDORASETTINGS}" ]\nthen\n echo "Missing PandoraSettings file"\n exit 1\nfi \n', 'echo =============================\n', 'echo PATH is \n', 'echo $PATH | tr ":" "\n" \n', 'echo ==============\n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is \n', 'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo ============================= \n', 'env | sort >> localEnv.log\n', 'PandoraFrontend -g /secret/dir/testmodelDetector_pandora.xml -c $PANDORASETTINGS -i ruinonslcio.test -o -r 0 \n', 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self ) assertMockCalls( remove_mock, [ 'SLICPandora__Run_465.sh', 'applogFile.test' ], self ) chmod_mock.assert_called_once_with('SLICPandora__Run_465.sh', 0o755) shell_mock.assert_called_once_with( 0, 'sh -c "./SLICPandora__Run_465.sh"', callbackFunction = self.spa.redirectLogOutput, bufferLimit = 20971520 ) unlink_mock.assert_called_once_with( '/secret/dir/testmodelDetector.zip' )
def test_resolvelinks_none_match( self ): walk_list = [ ( '/my/test/rootdir', 'ignored', [ 'testfile_1.txt', 'mylib_a.so', 'subfolder/otherlib_b.so.1' ] ), ( '/other/libfolder/', 'other_ignore', [ 'library.so', 'requirement.so' ] ), ( '', '', [] ) ] with patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/dir/cwd')) as getcwd_mock, \ patch('%s.os.chdir' % MODULE_NAME, new=Mock()) as chdir_mock, \ patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)), \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.os.symlink' % MODULE_NAME, new=Mock()) as slink_mock: resolveLinks( 'my_test_dir' ) getcwd_mock.assert_called_once_with() assertMockCalls( chdir_mock, [ 'my_test_dir', '/test/dir/cwd' ], self ) self.assertFalse( remove_mock.called ) self.assertFalse( slink_mock.called )
def test_canrun_addsite_fails(self): con_mock = Mock() with patch('%s.OverlayDB._query' % MODULE_NAME, new=Mock(return_value=S_ERROR())), \ patch('%s.OverlayDB._update' % MODULE_NAME, new=Mock(side_effect=[S_ERROR('update_test_err'), S_OK()])) as update_mock: assertDiracSucceedsWith_equals( self.odb.canRun('tenJobSite', con_mock), True, self) assertMockCalls(update_mock, [ ("INSERT INTO OverlayData (Site,NumberOfJobs) VALUES ('tenJobSite',1);", con_mock), ("UPDATE OverlayData SET NumberOfJobs=2 WHERE Site='tenJobSite';", con_mock) ], self)
def test_runit_copy_steeringfile_fails( self ): self.shc.OutputFile = 'something' self.shc.SteeringFile = '/my/dir/SteerFile.testme' ( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion ) = ( 'testPlatform', 'AppTestName', 'vT' ) with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \ patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/my/steer/dir'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[False,True])) as exists_mock, \ patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('copy_test_err'))) as copy_mock: assertDiracFailsWith( self.shc.runIt(), 'failed to access file steerfile.testme: copy_test_err', self ) assertMockCalls( exists_mock, [ 'SteerFile.testme', '/my/steer/dir/SteerFile.testme' ], self ) copy_mock.assert_called_once_with( '/my/steer/dir/SteerFile.testme', './SteerFile.testme' )
def test_towhizarddotin(self): self.whop.whizardxml = fromstring( '<whizard> <process_input> <mytestelement type="floatarray" value="sqrts"> </mytestelement> <test_bool type="T/F" value="000"> </test_bool> <test_bool2 type="T/F" value="0.0.0"> </test_bool2> <test_integer type="integer" value="0..0..0"> </test_integer> </process_input> <lalabeam_input1238> <test_float type="float" value="dontchangeanything"> </test_float> <test_string type="string" value="some_teststring"> </test_string> <test_noval></test_noval> </lalabeam_input1238> </whizard>' ) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock: result = self.whop.toWhizardDotIn('mytestfile.xml') assertDiracSucceeds(result, self) open_mock.assert_any_call('mytestfile.xml', 'w') assertMockCalls(open_mock().write, [ '&process_input\n test_bool = 0 0 0\n test_bool2 = 0.0 0.0\n test_integer = \n 1 20000\n 10 20000\n 1 20000\n/\n&beam_input\n test_float = dontchangeanything\n test_string = "some_teststring"\n test_noval = None\n/', '\n' ], self)
def test_runit_unzip_fails_nodetectorURL(self): exists_dict = { 'testmodelDetector_pandora.xml': False, '/secret/dir/testmodelDetector_pandora.xml': True, '/secret/dir/testmodelDetector.zip': True, 'SLICPandora__Run_465.sh': False, './lib': False, 'applogFile.test': True } self.spa.detectorxml = '/secret/dir/testmodelDetector_pandora.xml' self.spa.InputFile = 'testInput.file' self.spa.STEP_NUMBER = 465 ops_mock = Mock() ops_mock.getValue.return_value = [''] self.spa.ops = ops_mock mo = mock_open() def replace_exists(path): """ Mock exists implementation """ if path == '/secret/dir/testmodelDetector.zip': result = exists_dict[path] exists_dict[path] = False return result else: return exists_dict[path] with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))), \ patch('%s.unzip_file_into_dir' % MODULE_NAME, side_effect=OSError('unable to unzip_testme')) as unzip_mock, \ patch('%s.os.unlink' % MODULE_NAME) as unlink_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.open' % MODULE_NAME, mo ) as open_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK(['something']))) as shell_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/curdir/test')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, patch('%s.os.remove' % MODULE_NAME) as remove_mock: assertDiracFailsWith( self.spa.runIt(), 'could not find in cs the url for detector model', self) open_mock.assert_called_once_with( '/secret/dir/testmodelDetector.zip') assertMockCalls(exists_mock, [ 'testmodelDetector_pandora.xml', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector.zip' ], self) unzip_mock.assert_called_once_with(mo(), '/my/curdir/test') open_mock = open_mock() self.assertFalse(open_mock.write.called) self.assertFalse(remove_mock.called) self.assertFalse(chmod_mock.called) self.assertFalse(shell_mock.called) unlink_mock.assert_called_once_with( '/secret/dir/testmodelDetector.zip')
def test_retrievelfns( self ): repo_mock = Mock() ret_dict = { '1' : { 'State' : 'Done', 'UserOutputData' : '1389' }, '2' : {}, '3' : { 'State' : 'secret_teststate' }, '4' : { 'State' : 'invalid_state' }, '5' : { 'State' : 'Done', 'UserOutputData' : 0 }, '6' : { 'ignore_me' : True }, '7' : { 'State' : 'secret_teststate', 'UserOutputData' : 0 }, '148' : { 'State' : 'Done', 1 : False, True : 941, 'values_' : 'keys' } } repo_mock.readRepository.return_value = S_OK( ret_dict ) self.dilc.jobRepo = repo_mock with patch('%s.DiracILC.parameters' % MODULE_NAME, new=Mock(side_effect=[S_OK({'UploadedOutputData':'/my/test/lfn1'}),S_ERROR(),S_OK({}),S_OK({'some_entries':'some_values',1:True,'UploadedOutputData':'/more_lfns/append/testlfn.log'})])) as param_mock: assertEqualsImproved( self.dilc.retrieveRepositoryOutputDataLFNs( [ 'Done', 'secret_teststate' ] ), [ '/my/test/lfn1', '/more_lfns/append/testlfn.log' ], self ) assertMockCalls( param_mock, [ 3, 5, 7, 148 ], self )
def test_getcssection( self ): from DIRAC import S_OK cs_mock = Mock() cs_mock.setOption.return_value = S_OK() assertDiracSucceeds( insertCSSection( cs_mock, '/TestConfig/full/path', { 'newsubsection/Parameter1' : '194851', 'newsubsection' : { 'newsubsubsection/ComplexParameter' : True } } ), self ) assertMockCalls( cs_mock.setOption, [ ( '/TestConfig/full/path/newsubsection/Parameter1', '194851' ), ( '/TestConfig/full/path/newsubsection/newsubsubsection/ComplexParameter', True ) ], self ) assertMockCalls( cs_mock.createSection, [ '/TestConfig/full/path', '/TestConfig/full/path', '/TestConfig/full/path/newsubsection' ], self )
def test_checkconsistency( self ): import inspect self.lcs.version = 'v2.4' self.lcs.steeringFile = 'myTestSteerFile.txt' self.lcs.trackingStrategy = 'myTestStrat' self.lcs.detectorModel = 'myDetModel.wrongFormat' with patch('os.path.exists', new=Mock(side_effect=[ False, True, False ])) as exists_mock, \ patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())), \ patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_OK())) as xml_mock: assertDiracFailsWith( self.lcs._checkConsistency( 'myTestJob' ), 'you have to pass an existing .zip file', self ) assertMockCalls( exists_mock, [ 'myTestSteerFile.txt', 'myTestSteerFile.txt', 'myTestStrat' ], self ) xml_mock.assert_called_once_with( 'myTestSteerFile.txt' )
def test_getenvscript_other_prefixpath(self): def replace_abspath(path): """ Mock implementation of os.path.abspath """ if path == 'SLICPandora.sh': return '/abs/test/path/SLICPandora.sh' else: sys.exit() exists_dict = { 'PandoraFrontend': False, '/my/dir/test/me/Executable/PandoraFrontend': True } with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \ patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \ patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \ patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock: with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.os.path.abspath' % MODULE_NAME, new=Mock(side_effect=replace_abspath)) as abspath_mock: result = self.spa.getEnvScript('mytestsysconfig', 'SLIC Pandora', 'V2') chmod_mock.assert_called_once_with('SLICPandora.sh', 0755) assertMockCalls(exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self) abspath_mock.assert_called_once_with('SLICPandora.sh') assertDiracSucceedsWith_equals(result, '/abs/test/path/SLICPandora.sh', self) getsoft_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') removelib_mock.assert_called_once_with('/my/dir/test/me/LDLibs') getlib_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') getpath_mock.assert_called_once_with('mytestsysconfig', 'SLIC Pandora', 'V2') open_mock.assert_any_call('SLICPandora.sh', 'w') open_mock = open_mock() assertMockCalls(open_mock.write, [ '#!/bin/sh \n', '############################################################\n', '# Dynamically generated script to get the SLICPandora env. #\n', '############################################################\n', "declare -x PATH=/new/test/path:$PATH\n", 'declare -x ROOTSYS=/my/dir/test/me/ROOT\n', 'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:/my/dir/test/me/LDLibs:/new/ldpath\n', 'declare -x PANDORASETTINGSDIR=/my/dir/test/me/Settings\n', "declare -x PATH=/my/dir/test/me/Executable:$PATH\n" ], self)
def test_runit_complete(self): self.rea.platform = 'myTestPlatform' self.rea.applicationLog = '/my/applog/test.log' self.rea.script = '/my/test/my_testscript123.sh' self.rea.workflowStatus['OK'] = True self.rea.stepStatus['OK'] = True self.rea.applicationVersion = 'v123' self.rea.STEP_NUMBER = 13 self.rea.arguments = 'mytest args 123' self.rea.extraCLIarguments = 'extra test_args' with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[False,False,False,False,True])) as exists_mock, \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.open' % MODULE_NAME, mock_open(), create=True) as open_mock, \ patch('%s.os.chmod' % MODULE_NAME, new=Mock()) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK((0, 'my_statusmsg')))) as cmd_mock: assertDiracSucceedsWith(self.rea.runIt(), 'ROOT v123 Successful', self) assertMockCalls(exists_mock, [ 'Root_v123_Run_13.sh', './lib', 'my_testscript123.sh', '/my/applog/test.log', '/my/applog/test.log' ], self) self.assertFalse(remove_mock.called) open_mock.assert_called_once_with('Root_v123_Run_13.sh', 'w') open_mock = open_mock() open_mock.close.assert_called_once_with() assertMockCalls(open_mock.write, [ '#!/bin/sh \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'source /mytestenvscript\n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is\n', 'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo =============================\n', 'echo PATH is\n', 'echo $PATH | tr ":" "\n"\n', 'echo =============================\n', 'env | sort >> localEnv.log\n', 'echo =============================\n', 'my_testscript123.sh mytest args 123 extra test_args\n', 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self) chmod_mock.assert_called_once_with('Root_v123_Run_13.sh', 0755) cmd_mock.assert_called_once_with( 0, 'sh -c "./Root_v123_Run_13.sh"', callbackFunction=self.rea.redirectLogOutput, bufferLimit=20971520)
def test_runmarlin( self ): self.marAna.STEP_NUMBER = '935' self.marAna.applicationVersion = 'VT' self.marAna.debug = True self.marAna.applicationLog = 'testLog.mymarlin' self.marAna.extraCLIarguments = 'testMyArgs' exists_dict = { './lib/lddlib' : True, 'Marlin_VT_Run_935.sh' : True, './lib/marlin_dll' : True, 'myXML.test' : True, 'testLog.mymarlin' : False, './lib/': False, } def replace_exists( path ): return exists_dict[path] with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \ patch('%s.os.remove' % MODULE_NAME) as remove_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK('shell succeeded'))) as shell_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.MarlinAnalysis.setApplicationStatus' % MODULE_NAME) as appstat_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.marAna.runMarlin( 'myXML.test', 'TestEnvscript.path', 'marlin_dll.test.so' ) assertDiracSucceedsWith_equals( result, 'shell succeeded', self ) open_mock.assert_called_once_with( 'Marlin_VT_Run_935.sh', 'w' ) open_mock = open_mock() assertMockCalls( open_mock.write, [ '#!/bin/bash \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'source TestEnvscript.path\n', 'declare -x MARLIN_DLL=marlin_dll.test.so\n', 'declare -x LD_LIBRARY_PATH=./lib/lddlib:$LD_LIBRARY_PATH\n', 'declare -x PATH=$ROOTSYS/bin:$PATH\n', 'declare -x MARLIN_DEBUG=1\n', '\nif [ -e "${PANDORASETTINGS}" ]\nthen\n cp $PANDORASETTINGS .\nfi \n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is\n', 'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo =============================\n', 'echo PATH is\n', 'echo $PATH | tr ":" "\n"\n', 'echo =============================\n', 'echo MARLIN_DLL is\n', 'echo $MARLIN_DLL | tr ":" "\n"\n', 'echo =============================\n', 'echo ldd of executable is\n', 'ldd `which Marlin` \n', 'echo =============================\n', 'ldd ./lib/marlin_dll/*.so \n', 'ldd ./lib/lddlib/*.so \n', 'echo =============================\n', 'env | sort >> localEnv.log\n', 'Marlin -c myXML.test testMyArgs\n', 'Marlin myXML.test testMyArgs\n', 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self ) remove_mock.assert_called_once_with( 'Marlin_VT_Run_935.sh' ) assertMockCalls( exists_mock, [ 'Marlin_VT_Run_935.sh', './lib/lddlib', './lib/marlin_dll', './lib/lddlib', 'myXML.test', 'testLog.mymarlin', './lib/'], self ) shell_mock.assert_called_once_with( 0, 'sh -c "./Marlin_VT_Run_935.sh"', callbackFunction=self.marAna.redirectLogOutput, bufferLimit=20971520 ) chmod_mock.assert_called_once_with('Marlin_VT_Run_935.sh', 0o755) appstat_mock.assert_called_once_with( 'Marlin VT step 935')
def test_preparemarlindll_shellcall_fails( self ): file_contents = [[]] handles = FileUtil.getMultipleReadHandles( file_contents ) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))) as shell_mock: open_mock.side_effect = (h for h in handles) result = self.marAna.prepareMARLIN_DLL( 'some_path' ) open_mock.assert_called_once_with( 'temp.sh', 'w' ) assertMockCalls( handles[0].__enter__().write, ['#!/bin/bash\n', 'source some_path > /dev/null\necho $MARLIN_DLL'], self) assertEqualsImproved( len( handles ), 1, self ) chmod_mock.assert_called_once_with('temp.sh', 0o755) shell_mock.assert_called_once_with( 0, './temp.sh' ) assertDiracFailsWith( result, 'failed getting the marlin_dll', self )
def test_fullcopy_copy2_fails( self ): from ILCDIRAC.Core.Utilities.FileUtils import fullCopy with patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=['/my/src/directory/file_1_globbed.log','/my/src/directory/other__file.stdhep', 'lastfile_in_list.txt'])), \ patch('%s.dir_util.create_tree' % MODULE_NAME, new=Mock()) as createtree_mock, \ patch('%s.os.path.isfile' % MODULE_NAME, new=Mock(side_effect=[True,False])) as isfile_mock, \ patch('%s.shutil.copy2' % MODULE_NAME, new=Mock()) as copy_mock, \ patch('%s.shutil.copytree' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('tree_test_copy_error'))) as copytree_mock: assertDiracFailsWith( fullCopy( '/my/src/directory/', '/my/destination/dir', './myfile123.txt ' ), 'tree_test_copy_error', self ) assertMockCalls( createtree_mock, [ ( '/my/destination/dir', [ 'file_1_globbed.log' ] ), ( '/my/destination/dir', [ 'other__file.stdhep' ] ) ], self ) assertMockCalls( isfile_mock, [ '/my/src/directory/file_1_globbed.log', '/my/src/directory/other__file.stdhep' ], self ) copy_mock.assert_called_once_with( '/my/src/directory/file_1_globbed.log', '/my/destination/dir/file_1_globbed.log' ) copytree_mock.assert_called_once_with( '/my/src/directory/other__file.stdhep', '/my/destination/dir/other__file.stdhep' )
def test_runit_no_applog_created( self ): exists_dict = { 'testmodelDetector_pandora.xml' : False, '/secret/dir/testmodelDetector_pandora.xml' : True, '/secret/dir/testmodelDetector.zip' : True, 'SLICPandora__Run_465.sh' : True, './lib' : True, 'applogFile.test' : True } self.spa.detectorxml = '/secret/dir/testmodelDetector_pandora.xml' self.spa.InputFile = 'testInput.file' self.spa.STEP_NUMBER = 465 mo = mock_open() def replace_exists( path ): """ Mock exists implementation """ if path == 'applogFile.test': result = exists_dict[path] exists_dict[path] = False return result else: return exists_dict[path] with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))), \ patch('%s.unzip_file_into_dir' % MODULE_NAME) as unzip_mock, \ patch('%s.os.unlink' % MODULE_NAME) as unlink_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.open' % MODULE_NAME, mo ) as open_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK(['something']))) as shell_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/curdir/test')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, patch('%s.os.remove' % MODULE_NAME) as remove_mock: assertDiracFailsWith( self.spa.runIt(), 'SLICPandora did not produce the expected log', self ) assertMockCalls( exists_mock, [ 'testmodelDetector_pandora.xml', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector_pandora.xml', 'SLICPandora__Run_465.sh', './lib', 'applogFile.test', 'applogFile.test' ], self ) assertMockCalls( open_mock, [ ('/secret/dir/testmodelDetector.zip' ), ( 'SLICPandora__Run_465.sh', 'w') ], self, only_these_calls = False ) mo = mo() unzip_mock.assert_called_once_with( mo, '/my/curdir/test' ) open_mock = open_mock() assertMockCalls( open_mock.write, [ '#!/bin/bash \n', '#####################################################################\n', '# Dynamically generated script to run a production or analysis job. #\n', '#####################################################################\n', 'source myenvscriptpathtestme\n', 'declare -x file=./Settings/\n', '\nif [ -e "${file}" ]\nthen\n declare -x PANDORASETTINGS=$file\nelse\n if [ -d "${PANDORASETTINGSDIR}" ]\n then\n cp $PANDORASETTINGSDIR/*.xml .\n declare -x PANDORASETTINGS=\n fi\nfi\nif [ ! -e "${PANDORASETTINGS}" ]\nthen\n echo "Missing PandoraSettings file"\n exit 1\nfi \n', 'echo =============================\n', 'echo PATH is \n', 'echo $PATH | tr ":" "\n" \n', 'echo ==============\n', 'declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is \n', 'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo ============================= \n', 'env | sort >> localEnv.log\n', 'PandoraFrontend -g /secret/dir/testmodelDetector_pandora.xml -c $PANDORASETTINGS -i ruinonslcio.test -o -r 0 \n', 'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self ) assertMockCalls( remove_mock, [ 'SLICPandora__Run_465.sh', 'applogFile.test' ], self ) chmod_mock.assert_called_once_with('SLICPandora__Run_465.sh', 0o755) shell_mock.assert_called_once_with( 0, 'sh -c "./SLICPandora__Run_465.sh"', callbackFunction = self.spa.redirectLogOutput, bufferLimit = 20971520 ) self.assertFalse( unlink_mock.called )
def test_preparemarlindll_shellcall_fails(self): file_contents = [[]] handles = FileUtil.getMultipleReadHandles(file_contents) with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))) as shell_mock: open_mock.side_effect = (h for h in handles) result = self.marAna.prepareMARLIN_DLL('some_path') open_mock.assert_called_once_with('temp.sh', 'w') assertMockCalls( handles[0].__enter__().write, ['#!/bin/bash\n', 'source some_path\necho $MARLIN_DLL'], self) assertEqualsImproved(len(handles), 1, self) chmod_mock.assert_called_once_with('temp.sh', 0755) shell_mock.assert_called_once_with(0, './temp.sh') assertDiracFailsWith(result, 'failed getting the marlin_dll', self)
def test_getrootstuff( self ): file_list = [ '/my_test/aim/lib/some_lib.so', '/my_test/aim/lib/my_important_lib.so.1', '/my_test/aim/lib/last_lib.so.123' ] dependency_list = [ { 'first_dependency.so', 'base_lib.so.1' }, {}, { 'last_dependency.so.39' } ] with patch('%s.commands.getstatusoutput' % MODULE_NAME, new=Mock(return_value=(0, ''))) as cmd_mock, \ patch('%s.getFiles' % MODULE_NAME, new=Mock(return_value=file_list)) as getfile_mock, \ patch('%s.getDependentLibraries' % MODULE_NAME, new=Mock(side_effect=dependency_list)) as getlibs_mock, \ patch('%s.copyLibraries' % MODULE_NAME, new=Mock()) as copy_mock: getRootStuff( 'test_rootsys', '/my_test/aim' ) cmd_mock.assert_called_once_with( "rsync --exclude '.svn' -av test_rootsys/lib test_rootsys/etc test_rootsys/bin test_rootsys/cint /my_test/aim") getfile_mock.assert_called_once_with( '/my_test/aim/lib', '.so' ) assertMockCalls( getlibs_mock, [ '/my_test/aim/lib/some_lib.so', '/my_test/aim/lib/my_important_lib.so.1', '/my_test/aim/lib/last_lib.so.123' ], self ) copy_mock.assert_called_once_with( { 'first_dependency.so', 'base_lib.so.1', 'last_dependency.so.39' }, '/my_test/aim/lib' )
def test_checkconsistency(self): import inspect self.lcs.version = 'v2.4' self.lcs.steeringFile = 'myTestSteerFile.txt' self.lcs.trackingStrategy = 'myTestStrat' self.lcs.detectorModel = 'myDetModel.wrongFormat' with patch('os.path.exists', new=Mock(side_effect=[ False, True, False ])) as exists_mock, \ patch.object(inspect.getmodule(LCSIM), 'Exists', new=Mock(return_value=S_OK())), \ patch.object(inspect.getmodule(LCSIM), 'checkXMLValidity', new=Mock(return_value=S_OK())) as xml_mock: assertDiracFailsWith(self.lcs._checkConsistency('myTestJob'), 'you have to pass an existing .zip file', self) assertMockCalls( exists_mock, ['myTestSteerFile.txt', 'myTestSteerFile.txt', 'myTestStrat'], self) xml_mock.assert_called_once_with('myTestSteerFile.txt')
def test_runit_copy_steeringfile_fails(self): self.shc.OutputFile = 'something' self.shc.SteeringFile = '/my/dir/SteerFile.testme' (self.shc.platform, self.shc.applicationName, self.shc.applicationVersion) = ('testPlatform', 'AppTestName', 'vT') with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \ patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/my/steer/dir'))), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[False,True])) as exists_mock, \ patch('%s.shutil.copy' % MODULE_NAME, new=Mock(side_effect=EnvironmentError('copy_test_err'))) as copy_mock: assertDiracFailsWith( self.shc.runIt(), 'failed to access file steerfile.testme: copy_test_err', self) assertMockCalls( exists_mock, ['SteerFile.testme', '/my/steer/dir/SteerFile.testme'], self) copy_mock.assert_called_once_with('/my/steer/dir/SteerFile.testme', './SteerFile.testme')
def test_resolvelinks_none_match(self): walk_list = [ ('/my/test/rootdir', 'ignored', ['testfile_1.txt', 'mylib_a.so', 'subfolder/otherlib_b.so.1']), ('/other/libfolder/', 'other_ignore', ['library.so', 'requirement.so']), ('', '', []) ] with patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/test/dir/cwd')) as getcwd_mock, \ patch('%s.os.chdir' % MODULE_NAME, new=Mock()) as chdir_mock, \ patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)), \ patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \ patch('%s.os.symlink' % MODULE_NAME, new=Mock()) as slink_mock: resolveLinks('my_test_dir') getcwd_mock.assert_called_once_with() assertMockCalls(chdir_mock, ['my_test_dir', '/test/dir/cwd'], self) self.assertFalse(remove_mock.called) self.assertFalse(slink_mock.called)
def test_killrpath(self): walk_list = [('/testRoot', 'nodirs', ['file1.txt', 'deletethislib.so', '', 'ignorefile.s o']), ('/otherRoot/mydir/', '', ['lib1.so', 'subfolder/otherlib.so', 'library.dll'])] with patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)) as walk_mock, \ patch('%s.commands.getstatusoutput' % MODULE_NAME, new=Mock()) as cmd_mock: killRPath('myTest_folder') walk_mock.assert_called_once_with('myTest_folder', followlinks=True) expected_files = [ '/testRoot/deletethislib.so', '/otherRoot/mydir/lib1.so', '/otherRoot/mydir/subfolder/otherlib.so' ] assertMockCalls(cmd_mock, ['chrpath -d ' + x + ' ' for x in expected_files], self)
def test_getters( self ): from ILCDIRAC.Core.Utilities.WhizardOptions import WhizardOptions expected_pdict = { 'OK' : True, 'Value' : { 'integration_input' : {}, 'simulation_input' : {}, 'diagnostics_input' : {}, 'process_input' : {}, 'beam_input_1' : {}, 'beam_input_2' : {} } } pdict = self.whiz.getPDict() self.whiz.setEvtType( 'myevent_test' ) self.assertIsNotNone( pdict ) self.whiz.setGlobalEvtType( 'test_myglobalevt' ) self.whiz.setLuminosity( 138.312 ) self.whiz.setRandomSeed( 9024 ) self.whiz.setParameterDict( { 'mytestval' : True, 'more_entres' : 'value', 'something' : 9103 } ) self.whiz.setGeneratorLevelCuts( { 'generator' : False, 'cuts' : 123, 'level' : 'OK' } ) assert not self.whiz.willBeCut self.whiz.willCut() assert not self.whiz.useGridFiles self.whiz.usingGridFiles() self.whiz.setJobIndex( 'mytestJobIndexS&P500' ) self.whiz.setModel( 'mytestSModel' ) self.whiz._wo = WhizardOptions( self.whiz.model ) with patch('__builtin__.open', mock_open(), create=True) as mo: self.whiz.addedtojob = True self.whiz.dumpWhizardDotIn( 'someFile.in' ) self.whiz.addedtojob = False mo.assert_called_once_with( 'someFile.in', 'w' ) assertMockCalls( mo().write, [ TestWhizard.EXPECTED_PRINTOUT, '\n' ], self ) assertEqualsImproved( ( pdict, self.whiz.eventType, self.whiz.globalEventType, self.whiz.luminosity, self.whiz.randomSeed, self.whiz.parameterDict, self.whiz.generatorLevelCuts, self.whiz.willBeCut, self.whiz.useGridFiles, self.whiz.jobIndex, self.whiz.model ), ( expected_pdict, 'myevent_test', 'test_myglobalevt', 138.312, 9024, { 'mytestval' : True, 'more_entres' : 'value', 'something' : 9103 }, { 'generator' : False, 'cuts' : 123, 'level' : 'OK' }, True, True, 'mytestJobIndexS&P500', 'mytestSModel' ) , self ) assert not self.whiz._errorDict
def test_getsoftwarefolder_uselocal(self): exists_dict = { 'myapparchivev2.test.tar.gz': False, '/mylocalarea/test/myapparchivev2.test': True } with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \ patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \ patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: result = getSoftwareFolder('a', 'b', 'c') assertMockCalls(exists_mock, [ '/mylocalarea/test/myapparchivev2.test', 'myapparchivev2.test.tar.gz' ], self) getval_mock.assert_called_with('/AvailableTarBalls/a/b/c/TarBall', '') assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self)
def test_getenvscript_pandorafrontend_missing( self ): exists_dict = { 'PandoraFrontend' : False, '/my/dir/test/me/Executable/PandoraFrontend' : False } with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \ patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \ patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \ patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \ patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock: result = self.spa.getEnvScript( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) assertDiracFailsWith( result, 'missing pandorafrontend binary', self ) getsoft_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) removelib_mock.assert_called_once_with( '/my/dir/test/me/LDLibs' ) getlib_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) getpath_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' ) self.assertFalse( open_mock.called ) self.assertFalse( chmod_mock.called ) assertMockCalls( exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self )
def test_runit_no_detectormodel_found(self): exists_dict = { 'testmodelDetector.xml': True, '/secret/dir/testmodelDetector.xml': False } self.spa.detectorxml = '/secret/dir/testmodelDetector.xml' self.spa.InputFile = 'testInput.file' with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))) as resolve_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: assertDiracFailsWith(self.spa.runIt(), 'detector model xml was not found, exiting', self) assertMockCalls( exists_mock, ['testmodelDetector.xml', '/secret/dir/testmodelDetector.xml'], self) resolve_mock.assert_called_once_with('testInput.file')
def test_runit_unzip_fails_nodetectorURL( self ): exists_dict = { 'testmodelDetector_pandora.xml' : False, '/secret/dir/testmodelDetector_pandora.xml' : True, '/secret/dir/testmodelDetector.zip' : True, 'SLICPandora__Run_465.sh' : False, './lib' : False, 'applogFile.test' : True } self.spa.detectorxml = '/secret/dir/testmodelDetector_pandora.xml' self.spa.InputFile = 'testInput.file' self.spa.STEP_NUMBER = 465 ops_mock = Mock() ops_mock.getValue.return_value = [ '' ] self.spa.ops = ops_mock mo = mock_open() def replace_exists( path ): """ Mock exists implementation """ if path == '/secret/dir/testmodelDetector.zip': result = exists_dict[path] exists_dict[path] = False return result else: return exists_dict[path] with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \ patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))), \ patch('%s.unzip_file_into_dir' % MODULE_NAME, side_effect=OSError('unable to unzip_testme')) as unzip_mock, \ patch('%s.os.unlink' % MODULE_NAME) as unlink_mock, \ patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \ patch('%s.open' % MODULE_NAME, mo ) as open_mock, \ patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK(['something']))) as shell_mock, \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/curdir/test')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, patch('%s.os.remove' % MODULE_NAME) as remove_mock: assertDiracFailsWith( self.spa.runIt(), 'could not find in cs the url for detector model', self ) open_mock.assert_called_once_with( '/secret/dir/testmodelDetector.zip' ) assertMockCalls( exists_mock, [ 'testmodelDetector_pandora.xml', '/secret/dir/testmodelDetector.zip', '/secret/dir/testmodelDetector.zip' ], self ) unzip_mock.assert_called_once_with( mo(), '/my/curdir/test' ) open_mock = open_mock() self.assertFalse( open_mock.write.called ) self.assertFalse( remove_mock.called ) self.assertFalse( chmod_mock.called ) self.assertFalse( shell_mock.called ) unlink_mock.assert_called_once_with( '/secret/dir/testmodelDetector.zip' )
def test_setjobsatsites(self): con_mock = Mock() with patch('%s.OverlayDB._update' % MODULE_NAME, new=Mock(return_value=S_OK())) as update_mock: assertDiracSucceeds( self.odb.setJobsAtSites( { 'MyTestSite1': 1487, 'other_site': '138', 'large_testsite': 40913.2 }, con_mock), self) assertMockCalls(update_mock, [ ("UPDATE OverlayData SET NumberOfJobs=1487 WHERE Site='MyTestSite1';", con_mock), ("UPDATE OverlayData SET NumberOfJobs=138 WHERE Site='other_site';", con_mock), ("UPDATE OverlayData SET NumberOfJobs=40913 WHERE Site='large_testsite';", con_mock) ], self)
def test_execute_tarfails( self ): with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \ patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \ patch.object(self.ulf, '_populateLogDirectory', new=Mock(return_value=S_OK())), \ patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value= ['file1', 'testfile2' ])), \ patch('%s.os.path.islink' % MODULE_NAME, new=Mock(side_effect=[ True, False ])), \ patch('%s.os.chmod' % MODULE_NAME, new=Mock(side_effect=OSError('chmod_mock_testerr'))), \ patch('%s.os.path.realpath' % MODULE_NAME, new=Mock(return_value='./job/log/prodID/jobID')), \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/inval/cwd')), \ patch('%s.os.chdir' % MODULE_NAME), \ patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'mytarfile', 'otherfile' ])), \ patch('%s.tarfile.open' % MODULE_NAME), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)), \ patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()): assertDiracSucceeds( self.ulf.execute(), self ) assertMockCalls(self.log_mock.error, [ 'Could not set permissions of log files to 0755 with message:\nchmod_mock_testerr', ( 'Failed to create tar file from directory', './job/log/prodID/jobID File was not created' ), ( 'Problem changing shared area permissions', 'chmod_mock_testerr' ) ], self )
def test_getsoftwarefolder_notfound(self): exists_dict = { 'myapp_executable': False, '/mylocalarea/test/myapp_executable': False, '/testshared/area/myapp_executable': False } with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapp_executable')) as getval_mock, \ patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \ patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock: result = getSoftwareFolder('a', 'b', 'c') assertMockCalls(exists_mock, [ '/mylocalarea/test/myapp_executable', '/testshared/area/myapp_executable', 'myapp_executable' ], self) getval_mock.assert_called_with('/AvailableTarBalls/a/b/c/TarBall', '') assertDiracFailsWith(result, 'missing installation of myapp_executable', self)
def test_execute_tarfails( self ): log_mock = Mock() self.ulf.log = log_mock with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \ patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \ patch.object(self.ulf, '_populateLogDirectory', new=Mock(return_value=S_OK())), \ patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value= ['file1', 'testfile2' ])), \ patch('%s.os.path.islink' % MODULE_NAME, new=Mock(side_effect=[ True, False ])), \ patch('%s.os.chmod' % MODULE_NAME, new=Mock(side_effect=OSError('chmod_mock_testerr'))), \ patch('%s.os.path.realpath' % MODULE_NAME, new=Mock(return_value='./job/log/prodID/jobID')), \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/inval/cwd')), \ patch('%s.os.chdir' % MODULE_NAME), \ patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'mytarfile', 'otherfile' ])), \ patch('%s.tarfile.open' % MODULE_NAME), \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)), \ patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()): assertDiracSucceeds( self.ulf.execute(), self ) assertMockCalls( log_mock.error, [ 'Could not set permissions of log files to 0755 with message:\nchmod_mock_testerr', ( 'Failed to create tar file from directory', './job/log/prodID/jobID File was not created' ), ( 'Problem changing shared area permissions', 'chmod_mock_testerr' ) ], self )
def test_getcssection(self): from DIRAC import S_OK cs_mock = Mock() cs_mock.setOption.return_value = S_OK() assertDiracSucceeds( insertCSSection( cs_mock, '/TestConfig/full/path', { 'newsubsection/Parameter1': '194851', 'newsubsection': { 'newsubsubsection/ComplexParameter': True } }), self) assertMockCalls(cs_mock.setOption, [ ('/TestConfig/full/path/newsubsection/Parameter1', '194851'), ('/TestConfig/full/path/newsubsection/newsubsubsection/ComplexParameter', True) ], self) assertMockCalls(cs_mock.createSection, [ '/TestConfig/full/path', '/TestConfig/full/path', '/TestConfig/full/path/newsubsection' ], self)
def test_transferreg_some_files_fail( self ): self.ujf.failoverSEs = [ 'onlyAvailableSE' ] self.ujf.userFileCatalog = 'myCat' transfer_mock = Mock() def my_transfer_mock( fileName, localpath, lfn, targetSE, failoverSEs, fileCatalog = None, fileMetaDict = None ): """ Return S_ERROR for the appropriate file from the dictionary and S_OK for the rest. Necssary since dictionary iteration order is random """ return TRANSFER_AND_REGISTER_DICT[fileName] transfer_mock.transferAndRegisterFileFailover.side_effect = my_transfer_mock filesUploaded = [] with patch('%s.random.shuffle' % MODULE_NAME) as shuffle_mock: result = self.ujf.transferRegisterAndFailoverFiles( transfer_mock, { 'noMoreSEs' : { 'metadatadict1' : True, 'resolvedSE' : ( 'onlyAvailableSE', ), 'localpath' : '/doesntmatter', 'lfn' : 'doesntmattereither', 'filedict' : 32 }, 'fail_transfer' : { 'metadatadict1' : True, 'resolvedSE' : ( 'someOtherSite', ), 'localpath' : '/matterdoesnt', 'lfn' : 'neither', 'filedict' : 92 }, 'workingFile1' : { 'resolvedSE' : ( 'someOtherSite', ), 'localpath' : '/my/local/first/path', 'lfn' : 'LFN:/ilc/some/dir/file1.txt', 'filedict' : 8520 }, 'thisFileWorks.too' : { 'resolvedSE' : ( 'someOtherOtherSite', ), 'localpath' : '/dir/current/local.lfn', 'lfn' : 'LFN:/ilc/mydir/file2.ppt', 'filedict' : 98453 } }, filesUploaded ) assertDiracSucceeds( result, self ) assertEqualsImproved( result['Value'], dict(cleanUp=True), self ) assertMockCalls( shuffle_mock, [ [ 'onlyAvailableSE' ] ] * 4, self ) assertEqualsImproved( transfer_mock.transferAndRegisterFileFailover.mock_calls, [ call( 'workingFile1', '/my/local/first/path', 'LFN:/ilc/some/dir/file1.txt', 'someOtherSite', [ 'onlyAvailableSE' ], fileCatalog = 'myCat', fileMetaDict = 8520 ), call( 'fail_transfer', '/matterdoesnt', 'neither', 'someOtherSite', [ 'onlyAvailableSE' ], fileCatalog = 'myCat', fileMetaDict = 92 ), call( 'thisFileWorks.too', '/dir/current/local.lfn', 'LFN:/ilc/mydir/file2.ppt', 'someOtherOtherSite', [ 'onlyAvailableSE' ], fileCatalog = 'myCat', fileMetaDict = 98453 ) ], self ) assertEqualsImproved( filesUploaded, [ 'LFN:/ilc/some/dir/file1.txt', 'LFN:/ilc/mydir/file2.ppt' ], self )
def test_uploadproclist(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('/local/path/proc.list') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock with patch('shutil.copy') as copy_mock, \ patch('subprocess.call') as proc_mock: self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120') assertMockCalls( copy_mock, [('myTestProcess.list', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/'), ('myTestProcess.list', '/local/path/proc.list')], self) proc_mock.assert_called_once_with([ 'svn', 'ci', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list', "-m'Process list for whizard version v120'" ], shell=False) DIRAC.gConfig = backup_conf module_patcher.stop()
def test_execute_logupload_fails( self ): log_mock = Mock() self.ulf.log = log_mock self.ulf.failoverTest = False se_mock = Mock() se_mock.name = 'mySEMOCK' self.ulf.logSE = se_mock se_mock.putFile.return_value = S_OK( { 'Failed' : [ 'some_file_failed' ], 'Successful' : [] } ) request_mock = Mock() request_mock.RequestName = 'mymockreq' with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \ patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \ patch.object(self.ulf, '_populateLogDirectory', new=Mock(return_value=S_OK())), \ patch.object(self.ulf, '_setLogFilePermissions', new=Mock(return_value=S_OK())), \ patch.object(self.ulf, '_tarTheLogFiles', new=Mock(return_value=S_OK( { 'fileName' : 'some_name' } ))), \ patch.object(self.ulf, '_tryFailoverTransfer', new=Mock(return_value=S_OK( { 'Request' : request_mock, 'uploadedSE' : 'mock_se' } ))), \ patch.object(self.ulf, '_createLogUploadRequest', new=Mock(return_value=S_ERROR( 'upload_mock_err' ))) as uploadreq_mock, \ patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()): assertDiracSucceeds( self.ulf.execute(), self ) assertMockCalls( log_mock.error, [ ( 'Completely failed to upload log files to mySEMOCK, will attempt upload to failover SE', { 'Successful' : [], 'Failed' : [ 'some_file_failed' ] } ), ( 'Failed to create failover request', 'upload_mock_err' ) ], self ) uploadreq_mock.assert_called_once_with( 'mySEMOCK', '', 'mock_se' )
def test_execute_ignoreapperrors( self ): exists_dict = { 'list_of.txt' : True, 'something' : True } self.ujf.jobID = 512 self.ujf.ignoreapperrors = True request_mock = Mock() request_mock.RequestName = 'job_512_request.xml' request_mock.JobID = 512 request_mock.SourceComponent = "Job_512" self.ujf.workflow_commons = { 'TotalSteps' : '42', 'Request' : request_mock, 'JOB_ID' : '512', 'IS_PROD' : True, 'PRODUCTION_ID' : '98245', 'Platform' : 'myTestPlatform', 'Owner' : 'myTestOwner123RichGuy', 'VO' : 'myTestVirtualOrga', 'UserOutputSE' : 'myTestReceivingSE' } transfer_mock = Mock() transfer_mock.transferAndRegisterFile.return_value = S_OK( {} ) transfer_mock.transferAndRegisterFileFailover.return_value = S_OK() dataman_mock = Mock() dataman_mock.replicateAndRegister.return_value = S_OK() self.ujf.workflowStatus = S_OK() self.ujf.stepStatus = S_OK() self.ujf.userOutputData = [ 'list_of.txt', 'filenames.jar' ] self.ujf.userOutputSE = 'thisValueIsntUsed' self.ujf.userOutputPath = 'my/User/OPPath' with patch('%s.constructUserLFNs' % MODULE_NAME, new=Mock(return_value=S_OK(['/myTestVirtualOrga/testpre/m/myTestOwner123RichGuy/my/User/OPPath/list_of.txt', '/myTestVirtualOrga/testpre/m/myTestOwner123RichGuy/my/User/OPPath/filenames.jar']))) as constructlfn_mock, \ patch('%s.UserJobFinalization.getCandidateFiles' % MODULE_NAME, new=Mock(return_value={ 'OK' : False, 'Value' : {} })) as getcf_mock, \ patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('%s.open' % MODULE_NAME, mock_open(read_data='myfilecontent,makeniceadlerchecksum')), \ patch('%s.os.path.getsize' % MODULE_NAME, new=Mock(return_value=3048)), \ patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/mycurdirTestMe')), \ patch('%s.getDestinationSEList' % MODULE_NAME, new=Mock(return_value=S_OK(['CERN-DIP-4']))), \ patch('%s.FailoverTransfer' % MODULE_NAME, new=transfer_mock), \ patch('%s.DataManager' % MODULE_NAME, new=dataman_mock), \ patch('%s.time.sleep' % MODULE_NAME, new=Mock(return_value=True)), \ patch('%s.UserJobFinalization.getFileMetadata' % MODULE_NAME, new=Mock(return_value={ 'OK' : False, 'Value' : { 'workingFile1' : { 'resolvedSE' : ( 'someOtherSite', ), 'localpath' : '/my/local/first/path', 'lfn' : 'LFN:/ilc/some/dir/file1.txt', 'filedict' : 8520 }, 'thisFileWorks.too' : { 'resolvedSE' : ( 'someOtherOtherSite', ), 'localpath' : '/dir/current/local.lfn', 'lfn' : 'LFN:/ilc/mydir/file2.ppt', 'filedict' : 98453 } } })) as getfmd_mock: #TODO: does this (getcandidatefiles mock, getfilemetadata mock) make sense? S_ERROR usually will not have a value, so this would actually throw an error in real environment result = self.ujf.execute() assertDiracSucceeds( result, self ) getcf_mock.assert_called_once_with( [ { 'outputPath': 'TXT', 'outputFile': 'list_of.txt', 'outputDataSE': ['myTestReceivingSE'] }, {'outputPath': 'JAR', 'outputFile': 'filenames.jar', 'outputDataSE': ['myTestReceivingSE'] } ], [ '/myTestVirtualOrga/testpre/m/myTestOwner123RichGuy/my/User/OPPath/list_of.txt', '/myTestVirtualOrga/testpre/m/myTestOwner123RichGuy/my/User/OPPath/filenames.jar' ], '' ) constructlfn_mock.assert_called_once_with( 512, 'myTestVirtualOrga', 'myTestOwner123RichGuy', [ 'list_of.txt', 'filenames.jar' ], 'my/User/OPPath' ) getfmd_mock.assert_called_once_with( {} ) transfer_mock = transfer_mock() # Necessary for the assumptions transfer_mock.transferAndRegisterFile.assert_any_call( 'workingFile1', '/my/local/first/path', 'LFN:/ilc/some/dir/file1.txt', ['myTestReceivingSE', 'CERN-DIP-4'], fileCatalog=['FileCatalog'], fileMetaDict=8520 ) transfer_mock.transferAndRegisterFile.assert_called_with( 'thisFileWorks.too', '/dir/current/local.lfn', 'LFN:/ilc/mydir/file2.ppt', ['myTestReceivingSE', 'CERN-DIP-4'], fileCatalog=['FileCatalog'], fileMetaDict=98453 ) assertEqualsImproved( len(transfer_mock.transferAndRegisterFile.mock_calls), 14, self ) self.assertFalse( transfer_mock.transferAndRegisterFileFailover.called ) dataman_mock = dataman_mock() assertMockCalls( dataman_mock.replicateAndRegister, [ ( 'LFN:/ilc/mydir/file2.ppt', 'myTestReceivingSE' ), ( 'LFN:/ilc/some/dir/file1.txt', 'myTestReceivingSE' ) ], self, only_these_calls = False )