def test_runit(self):
with patch('os.makedirs'), \
patch('os.path.exists') as mock_exists, \
patch("%s.shellCall" % MODULE_NAME, new=Mock(return_value={'OK' : True, 'Value' : ["", "stdout", "stderr"]})) as mock_shellcall:
mock_exists.side_effect = self.replace_exists
assertDiracSucceedsWith(
self.fccAna.runIt(),
"Execution of the FCC application successfull", self)
self.log_mock.info.assert_any_call(
"Environment : Environment script look up successfull")
self.log_mock.debug.assert_any_call(
"Application code : Creation of the bash script successfull")
self.log_mock.debug.assert_any_call(
"Application : Application execution and log file creation...")
self.log_mock.debug.assert_any_call(
"Application : Application execution successfull")
self.log_mock.debug.assert_any_call(
"Application : Log file creation successfull")
self.log_mock.warn.assert_called_once_with(
"Application : no root files have been generated, was that intentional ?"
)
mock_shellcall.assert_called_once_with(
0,
self.fccAna.applicationScript,
callbackFunction=self.fccAna.redirectLogOutput,
bufferLimit=20971520)
def test_runit_log_failed_withoutapp(self):
self.fccAna.ignoreapperrors = True
self.exists_dict[self.fccAna.applicationLog] = False
with patch('os.makedirs'), \
patch('os.path.exists') as mock_exists, \
patch("%s.shellCall" % MODULE_NAME, new=Mock(return_value={'OK' : True, 'Value' : ["", "stdout", "stderr"]})) as mock_shellcall:
mock_exists.side_effect = self.replace_exists
assertDiracSucceedsWith(
self.fccAna.runIt(),
"Execution of the FCC application successfull", self)
self.log_mock.info.assert_any_call(
"Environment : Environment script look up successfull")
self.log_mock.debug.assert_any_call(
"Application code : Creation of the bash script successfull")
self.log_mock.debug.assert_any_call(
"Application : Application execution and log file creation...")
self.log_mock.debug.assert_any_call(
"Application : Application execution successfull")
self.log_mock.error.assert_called_once_with(
"Application : Log file creation failed")
mock_shellcall.assert_called_once_with(
0,
self.fccAna.applicationScript,
callbackFunction=self.fccAna.redirectLogOutput,
bufferLimit=20971520)
def test_runit_with_many_inputfiles(self):
get_input_from_app = [
"output1_%s.root" % self.fccAppIndex,
"output2_%s.root" % self.fccAppIndex
]
self.fccAna.step_commons['InputFile'] = get_input_from_app
input_file1 = os.path.realpath(get_input_from_app[0])
input_file2 = os.path.realpath(get_input_from_app[1])
self.exists_dict[input_file1] = True
self.exists_dict[input_file2] = True
with patch('os.makedirs'), \
patch('os.path.exists') as mock_exists :
mock_exists.side_effect = self.replace_exists
assertDiracSucceedsWith(
self.fccAna.runIt(),
"Execution of the FCC application successfull", self)
assertEqualsImproved(self.fccAna.InputFile,
[input_file1, input_file2], self)
assertEqualsImproved(self.fccAna.SteeringFile,
" ".join(self.fccAna.InputFile), self)
self.log_mock.debug.assert_any_call(
"Application : Configuration file taken from the input file(s) '%s'"
% self.fccAna.SteeringFile)
def test_preparemarlindll_swaplibpositions(self):
self.marAna.ProcessorListToExclude = ['mytestlibrary.so']
exists_dict = {'./lib/marlin_dll': True}
def replace_exists(path):
return exists_dict[path]
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \
patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL('some_path')
open_mock.assert_called_once_with('temp.sh', 'w')
assertMockCalls(
handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path\necho $MARLIN_DLL'], self)
assertEqualsImproved(len(handles), 1, self)
chmod_mock.assert_called_once_with('temp.sh', 0755)
shell_mock.assert_called_once_with(0, './temp.sh')
remove_mock.assert_called_once_with('temp.sh')
exists_mock.assert_called_once_with('./lib/marlin_dll')
glob_mock.assert_called_once_with('./lib/marlin_dll/*.so')
assertDiracSucceedsWith(
result,
'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so',
self)
def test_applicationspecificinputs_othercases(self):
self.rod.step_commons['Enable'] = {'mydict': True}
assertDiracSucceedsWith(self.rod.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved((self.rod.enable, self.rod.sel_eff,
self.rod.cut_eff, self.rod.add_info),
(False, 0, 0, ''), self)
def test_applicationSpecificInputs(self):
self.upod.step_commons['TestFailover'] = 'something'
self.upod.workflow_commons['outputList'] = [{
'appdict': True,
'myOutput': 'yes',
'outputFile': ''
}, {
'outputFile': True
}]
self.upod.workflow_commons['PRODUCTION_ID'] = 1834
self.upod.workflow_commons['JOB_ID'] = 418
self.upod.workflow_commons['outputDataFileMask'] = ['mycoollist']
with patch('%s.getProdFilename' % MODULE_NAME,
new=Mock(side_effect=['myOutputF_1', 'other_file.txt'])):
assertDiracSucceedsWith(self.upod.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved(self.upod.failoverTest, False, self)
assertEqualsImproved(self.upod.outputList,
[{
'appdict': True,
'myOutput': 'yes',
'outputFile': 'myOutputF_1'
}, {
'outputFile': 'other_file.txt'
}], self)
def test_execute_maximal_othercase(self):
self.rod.nbofevents = 1389
self.rod.luminosity = 9814.2
self.rod.sel_eff = 184.2
self.rod.cut_eff = 13.1
self.rod.InputData = 'myTestInputFiles.rec'
self.rod.inputdataMeta['CrossSection'] = 'myTestCrosssection'
self.rod.inputdataMeta['AdditionalInfo'] = 'more_information_testme'
self.rod.WorkflowStartFrom = 'EventZerotest'
self.rod.prodOutputLFNs = ['/some/test/dir/mytestfile.txt']
fcc_mock = Mock()
fcc_mock.setMetadata.return_value = S_OK()
fcc_mock.addFileAncestors.return_value = S_OK()
self.rod.filecatalog = fcc_mock
with patch('%s.RegisterOutputData.resolveInputVariables' % MODULE_NAME,
new=Mock(return_value=S_OK())):
assertDiracSucceedsWith(
self.rod.execute(),
'Output data metadata registered in catalog', self)
fcc_mock.setMetadata.assert_called_once_with(
'/some/test/dir/mytestfile.txt', {
'NumberOfEvents': 1389,
'Luminosity': 9814.2,
'Reduction': 184.2,
'CutEfficiency': 13.1,
'AdditionalInfo': 'more_information_testme',
'CrossSection': 'myTestCrosssection',
'FirstEventFromInput': 'EventZerotest'
})
fcc_mock.addFileAncestors.assert_called_once_with({
'/some/test/dir/mytestfile.txt': {
'Ancestors': 'myTestInputFiles.rec'
}
})
def test_preparemarlindll_swaplibpositions( self ):
self.marAna.ProcessorListToExclude = [ 'mytestlibrary.so' ]
exists_dict = { './lib/marlin_dll' : True }
def replace_exists( path ):
return exists_dict[path]
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles( file_contents )
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \
patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL( 'some_path' )
open_mock.assert_called_once_with( 'temp.sh', 'w' )
assertMockCalls( handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path > /dev/null\necho $MARLIN_DLL'], self)
assertEqualsImproved( len( handles ), 1, self )
chmod_mock.assert_called_once_with('temp.sh', 0o755)
shell_mock.assert_called_once_with( 0, './temp.sh' )
remove_mock.assert_called_once_with( 'temp.sh' )
exists_mock.assert_called_once_with( './lib/marlin_dll' )
glob_mock.assert_called_once_with( './lib/marlin_dll/*.so' )
assertDiracSucceedsWith( result, 'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so', self )
def test_runit_without_current_rootfiles(self):
getctime_dict = {
os.path.realpath(self.root_files[1]): 1501667507.9749944,
os.path.realpath(self.root_files[0]): 1501667510.7510207,
os.path.realpath(self.applicationScript): 1501667512.7510207
}
def replace_getctime(path):
return getctime_dict[os.path.realpath(path)]
with patch('os.makedirs'), \
patch("%s.glob.glob" % MODULE_NAME) as mock_glob, \
patch("os.path.getctime") as mock_getctime, \
patch('os.path.exists') as mock_exists :
mock_exists.side_effect = self.replace_exists
mock_getctime.side_effect = replace_getctime
mock_glob.return_value = self.root_files
assertDiracSucceedsWith(
self.fccAna.runIt(),
"Execution of the FCC application successfull", self)
self.log_mock.warn.assert_called_once_with(
"Application : This application did not generate any root files, was that intentional ?"
)
def test_checkconsistency_run_read_failed( self ):
build_folder = 'build.x86_64-slc6-gcc49-opt'
build_path = os.path.join(self.fcc.fccSwPath, build_folder)
executable_path = os.path.join(build_path, 'run')
info_message = (
"Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {'name':self.fcc.appname}
)
with patch('os.path.exists') as mock_exists, \
patch('os.listdir') as mock_listdir, \
patch('os.path.isfile') as mock_isfile, \
patch("%s._checkConsistency" % MODULE_NAME) as mock_check, \
patch("%s._readFromFile" % MODULE_NAME) as mock_read, \
patch.object(inspect.getmodule(FccSw), 'LOG', new=self.log_mock):
mock_exists.return_value = True
mock_listdir.return_value = [build_folder]
mock_isfile.return_value = False
mock_read.return_value = (None, 'DO NOT READ ME !')
mock_check.return_value = S_OK(info_message)
assertDiracSucceedsWith( self.fcc._checkConsistency(), info_message, self )
self.log_mock.debug.assert_any_call( "FCCSW specific consistency : The temporary folder 'temp_fcc_dirac' already exists" )
mock_listdir.assert_called_once_with( self.fcc.fccSwPath )
mock_isfile.assert_called_once_with( build_path )
mock_exists.assert_any_call( executable_path )
mock_read.assert_called_once_with( executable_path )
self.log_mock.warn.assert_called_once_with( 'DO NOT READ ME !' )
self.log_mock.debug.assert_any_call( 'FCCSW specific consistency : Using by default the command of the 0.8.1 release !' )
def test_runit_oldversion( self ):
exists_dict = { 'secret/dir/mySuperDetector_pandora.xml' : False,
'secret/dir/mySuperDetector.zip' : False,
'/my/curdir/test/secret/dir/mySuperDetector_pandora.xml' : True,
'SLICPandora_V2_Run_465.sh' : False, './lib' : False,
'applogFile.test' : False }
self.spa.detectorxml = 'secret/dir/mySuperDetector'
self.spa.applicationVersion = 'V2'
self.spa.InputFile = 'testInput.file'
self.spa.STEP_NUMBER = 465
ops_mock = Mock()
ops_mock.getValue.return_value = [ 'http://some_url/' ]
self.spa.ops = ops_mock
mo = mock_open( read_data='some_log_data\n Missing PandoraSettings file, but ignore this error :)')
def replace_exists( path ):
""" Mock implementation of exists """
result = exists_dict[path]
if path == 'secret/dir/mySuperDetector.zip' or path == 'applogFile.test':
exists_dict[path] = True
return result
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('myenvscriptpathtestme'))), \
patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK([ 'ruinonslcio.test' ]))), \
patch('%s.unzip_file_into_dir' % MODULE_NAME) as unzip_mock, \
patch('%s.os.unlink' % MODULE_NAME) as unlink_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.open' % MODULE_NAME, mo ) as open_mock, \
patch('%s.urllib.urlretrieve' % MODULE_NAME) as urlretrieve_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([0, 'Disabled execution', '']))) as shell_mock, \
patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/my/curdir/test')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, patch('%s.os.remove' % MODULE_NAME) as remove_mock:
assertDiracSucceedsWith( self.spa.runIt(), 'SLICPandora V2 Successful', self )
assertMockCalls( exists_mock, [ 'secret/dir/mySuperDetector.zip', 'secret/dir/mySuperDetector.zip',
'/my/curdir/test/secret/dir/mySuperDetector_pandora.xml',
'SLICPandora_V2_Run_465.sh', './lib', 'applogFile.test',
'applogFile.test' ], self )
assertMockCalls( open_mock, [ ( 'SLICPandora_V2_Run_465.sh', 'w' ), ( 'applogFile.test', 'r') ],
self, only_these_calls = False )
mo = mo()
open_mock = open_mock()
assertMockCalls( open_mock.write, [
'#!/bin/bash \n', '#####################################################################\n',
'# Dynamically generated script to run a production or analysis job. #\n',
'#####################################################################\n',
'source myenvscriptpathtestme\n', 'declare -x file=./Settings/\n',
'\nif [ -e "${file}" ]\nthen\n declare -x PANDORASETTINGS=$file\nelse\n if [ -d "${PANDORASETTINGSDIR}" ]\n then\n cp $PANDORASETTINGSDIR/*.xml .\n declare -x PANDORASETTINGS=\n fi\nfi\nif [ ! -e "${PANDORASETTINGS}" ]\nthen\n echo "Missing PandoraSettings file"\n exit 1\nfi \n',
'echo =============================\n', 'echo PATH is \n', 'echo $PATH | tr ":" "\n" \n',
'echo ==============\n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is \n',
'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo ============================= \n',
'env | sort >> localEnv.log\n',
'PandoraFrontend /my/curdir/test/secret/dir/mySuperDetector_pandora.xml $PANDORASETTINGS ruinonslcio.test 0 \n',
'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self )
chmod_mock.assert_called_once_with('SLICPandora_V2_Run_465.sh', 0o755)
shell_mock.assert_called_once_with( 0, 'sh -c "./SLICPandora_V2_Run_465.sh"',
callbackFunction = self.spa.redirectLogOutput,
bufferLimit = 20971520 )
self.assertFalse( urlretrieve_mock.called )
self.assertFalse( unzip_mock.called )
self.assertFalse( unlink_mock.called )
self.assertFalse( remove_mock.called )
def test_execute_minimal(self):
self.rod.enable = False
self.rod.prodOutputLFNs = ['myOutput.lfn.stdhep']
with patch('%s.RegisterOutputData.resolveInputVariables' % MODULE_NAME,
new=Mock(return_value=S_OK())):
assertDiracSucceedsWith(
self.rod.execute(),
'Output data metadata registered in catalog', self)
def test_execute_disabledmodule(self):
self.upod.prodOutputLFNs = ['/ilc/prod/ilc/mc-dbd/example_file']
with patch.object(self.upod, 'getCandidateFiles', new=Mock(return_value=S_OK({}))), \
patch.object(self.upod, 'getFileMetadata', new=Mock(return_value=S_OK( { 'fileTestName' : { 'workflowSE' : 'testSE', 'otherTestMetadata' : True } } ))), \
patch('%s.getDestinationSEList' % MODULE_NAME, new=Mock(return_value=S_OK('myTestSE'))):
assertDiracSucceedsWith(self.upod.execute(),
'Module is disabled by control flag', self)
assertEqualsImproved(self.upod.experiment, 'ILC_ILD', self)
def test_execute_status_not_ok( self ):
log = 'my_123_log.txt'
self.over.applicationLog = log
self.over.workflowStatus = S_ERROR('myerror167')
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=2)):
result = self.over.execute()
assertDiracSucceedsWith( result, 'OverlayInput should not proceed', self )
assertEqualsImproved( self.over.applicationLog, os.getcwd() + '/' + log, self )
def test_applicationspecificinputs_nonset(self):
self.rod.workflow_commons['Info'] = {'SomeKey': 'someValue'}
assertDiracSucceedsWith(self.rod.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved(
(self.rod.enable, self.rod.prodOutputLFNs, self.rod.luminosity,
self.rod.nbofevents, self.rod.sel_eff, self.rod.cut_eff,
self.rod.add_info),
(True, [], 0, 0, 0, 0, 'ds7:SomeKeys9:someValuee'), self)
def test_construct_user_lfns_no_outputpath( self ):
today_mock = Mock()
today_mock.timetuple.return_value=[ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value = 'testpre')):
result = constructUserLFNs( 1234567, 'mytestVO', 'mytestowner', '/ignore/this/myOutputString.md5', None )
assertDiracSucceedsWith( result, '/mytestVO/testpre/m/mytestowner/1999_27/1234/1234567/myOutputString.md5', self )
def test_runit_without_inputfile(self):
with patch('os.makedirs'), \
patch('os.path.exists') as mock_exists :
mock_exists.side_effect = self.replace_exists
assertDiracSucceedsWith(
self.fccAna.runIt(),
"Execution of the FCC application successfull", self)
assertEqualsImproved(self.fccAna.InputFile, [], self)
def test_runit_status_not_ok( self ):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = '/my/test/script.sh'
self.rea.workflowStatus['OK'] = True
self.rea.stepStatus['OK'] = False
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))):
assertDiracSucceedsWith( self.rea.runIt(),
'ROOT should not proceed as previous step did not end properly', self )
def test_construct_user_lfns_no_outputpath( self ):
today_mock = Mock()
today_mock.timetuple.return_value=[ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value = 'testpre')):
result = constructUserLFNs( 1234567, 'mytestVO', 'mytestowner', '/ignore/this/myOutputString.md5', None )
assertDiracSucceedsWith(
result, '/mytestVO/testpre/m/mytestowner/1999_27/1234/1234567/myOutputString.md5', self )
def test_finalstatusreport_ignorefail(self):
log_mock = Mock()
self.moba.log = log_mock
self.moba.ignoreapperrors = True
with patch('%s.ModuleBase.setApplicationStatus' %
MODULE_NAME) as appstat_mock:
assertDiracSucceedsWith(self.moba.finalStatusReport(1),
' exited With Status 1', self)
appstat_mock.assert_called_once_with(' exited With Status 1')
self.assertTrue(log_mock.error.called)
def test_construct_user_lfns( self ):
today_mock = Mock()
today_mock.timetuple.return_value=[ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='testpre')):
result = constructUserLFNs( 1234567, 'mytestVO', 'mytestowner', [ 'myTestOutputFile1.txt', 'myTestOutputFile2.ppt' ], '/mydir/dir//MyTestOutputPath1' )
assertDiracSucceedsWith( result, '/mytestVO/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile1.txt', self )
assertDiracSucceedsWith( result, '/mytestVO/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile2.ppt', self )
def test_finalstatusreport(self):
log_mock = Mock()
self.moba.log = log_mock
self.moba.ignoreapperrors = False
with patch('%s.ModuleBase.setApplicationStatus' %
MODULE_NAME) as appstat_mock:
assertDiracSucceedsWith(self.moba.finalStatusReport(0),
' Successful', self)
appstat_mock.assert_called_once_with(' Successful')
self.assertFalse(log_mock.error.called)
def test_getdestinationse_alias_found( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' :
'myTest_Alias_value' }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' )
assertDiracSucceedsWith( result, 'myTest_Alias_value', self )
def test_checkconsistency( self ):
with patch.object(self.fcc, '_importToSandbox', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(FccSw), 'LOG', new=self.log_mock):
info_message = ("Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {'name': self.fcc.appname}
)
assertDiracSucceedsWith( self.fcc._checkConsistency(), info_message, self )
self.log_mock.info.assert_called_with( info_message )
self.log_mock.info.assert_any_call( "Sandboxing : Sandboxing successfull" )
def test_execute_status_not_ok(self):
log = 'my_123_log.txt'
self.over.applicationLog = log
self.over.workflowStatus = S_ERROR('myerror167')
with patch('%s.Operations.getValue' % MODULE_NAME,
new=Mock(return_value=2)):
result = self.over.execute()
assertDiracSucceedsWith(result, 'OverlayInput should not proceed',
self)
assertEqualsImproved(self.over.applicationLog,
os.getcwd() + '/' + log, self)
def test_applicationspecificinputs( self ):
self.rod.step_commons[ 'Enable' ] = True
self.rod.workflow_commons[ 'ProductionOutputData' ] = 'MyOutputLFN1.php;other__lfn.stdio;;last_file.stdhep'
self.rod.workflow_commons[ 'Luminosity' ] = 1391.2
self.rod.workflow_commons[ 'Info' ] = { 'stdhepcut' : { 'Reduction' : 12.2, 'CutEfficiency': 0.92 } }
assertDiracSucceedsWith( self.rod.applicationSpecificInputs(), 'Parameters resolved', self )
assertEqualsImproved( ( self.rod.enable, self.rod.prodOutputLFNs, self.rod.luminosity,
self.rod.sel_eff, self.rod.cut_eff, self.rod.add_info ),
( True, [ 'MyOutputLFN1.php', 'other__lfn.stdio', '', 'last_file.stdhep' ],
1391.2, 12.2, 0.92, 'de' ), self )
assert 'stdhepcut' not in self.rod.workflow_commons[ 'Info' ]
def test_constructoutputlfns( self ):
self.ujf.jobID = 0
self.ujf.workflow_commons = {}
self.ujf.userOutputData = 'testOutputData'
self.ujf.userOutputPath = 'testOutputPath'
with patch('%s.getProxyInfo' % MODULE_NAME, new=Mock(return_value=S_OK({ 'username' : 'myTestUser123', 'group' : 'TeILCst_bla'}))), \
patch('%s.constructUserLFNs' % MODULE_NAME, new=Mock(return_value=S_OK('constructedLFNs'))) as construct_mock:
result = self.ujf.constructOutputLFNs()
assertDiracSucceedsWith( result, 'constructedLFNs', self )
construct_mock.assert_called_once_with( 12345, 'TeILCst', 'myTestUser123',
'testOutputData', 'testOutputPath' )
assertEqualsImproved( self.ujf.jobID, 12345, self )
def test_runit_status_not_ok(self):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = '/my/test/script.sh'
self.rea.workflowStatus['OK'] = True
self.rea.stepStatus['OK'] = False
with patch('%s.getEnvironmentScript' % MODULE_NAME,
new=Mock(return_value=S_OK('/mytestenvscript'))):
assertDiracSucceedsWith(
self.rea.runIt(),
'ROOT should not proceed as previous step did not end properly',
self)
def test_construct_user_lfns_getvo_fails( self ):
today_mock = Mock()
today_mock.timetuple.return_value = [ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='testpre')), \
patch('%s.getVOfromProxyGroup' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))), \
patch('%s.gLogger.error' % MODULE_NAME) as log_mock:
result = constructUserLFNs( 1234567, None, 'mytestowner', [ 'myTestOutputFile1.txt', 'myTestOutputFile2.ppt' ], '/mydir/dir//MyTestOutputPath1' )
assertDiracSucceedsWith( result, '/ilc/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile1.txt', self )
assertDiracSucceedsWith( result, '/ilc/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile2.ppt', self )
log_mock.assert_called_once_with( 'Could not get VO from CS, assuming ilc' )
def test_construct_user_lfns( self ):
today_mock = Mock()
today_mock.timetuple.return_value=[ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='testpre')):
result = constructUserLFNs( 1234567, 'mytestVO', 'mytestowner',
[ 'myTestOutputFile1.txt', 'myTestOutputFile2.ppt' ],
'/mydir/dir//MyTestOutputPath1' )
assertDiracSucceedsWith(
result, '/mytestVO/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile1.txt', self )
assertDiracSucceedsWith(
result, '/mytestVO/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile2.ppt', self )
def test_getenvscript( self ):
moduleName = "ILCDIRAC.Workflow.Modules.MarlinAnalysis"
file_contents = []
text_file_data = '\n'.join(file_contents)
result = {}
with patch('%s.open' % moduleName, mock_open(read_data=text_file_data), create=True) as file_mocker:
result = self.marAna.getEnvScript(None, None, None)
assertDiracSucceedsWith( result, '/MarlinEnv.sh', self )
check_in_script = [ 'declare -x PATH=aFolder/Executable:$PATH\n', 'declare -x ROOTSYS=aFolder/ROOT\n', 'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:aFolder/LDLibs:bFolder\n', "declare -x MARLIN_DLL=aFolder/MARLIN_DLL/testdir:\n", "declare -x PANDORASETTINGS=aFolder/Settings/PandoraSettings.xml" ]
file_mocker.assert_called_with('MarlinEnv.sh', 'w')
mocker_handle = file_mocker()
for expected in check_in_script:
mocker_handle.write.assert_any_call(expected)
self.assertEquals(len(check_in_script), mocker_handle.__enter__.return_value.write.call_count - 4)
def test_runit_complete(self):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = '/my/test/my_testscript123.sh'
self.rea.workflowStatus['OK'] = True
self.rea.stepStatus['OK'] = True
self.rea.applicationVersion = 'v123'
self.rea.STEP_NUMBER = 13
self.rea.arguments = 'mytest args 123'
self.rea.extraCLIarguments = 'extra test_args'
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[False,False,False,False,True])) as exists_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK((0, 'my_statusmsg')))) as cmd_mock:
assertDiracSucceedsWith(self.rea.runIt(), 'ROOT v123 Successful',
self)
assertMockCalls(exists_mock, [
'Root_v123_Run_13.sh', './lib', 'my_testscript123.sh',
'/my/applog/test.log', '/my/applog/test.log'
], self)
self.assertFalse(remove_mock.called)
open_mock.assert_called_once_with('Root_v123_Run_13.sh', 'w')
open_mock = open_mock()
open_mock.close.assert_called_once_with()
assertMockCalls(open_mock.write, [
'#!/bin/sh \n',
'#####################################################################\n',
'# Dynamically generated script to run a production or analysis job. #\n',
'#####################################################################\n',
'source /mytestenvscript\n',
'echo =============================\n',
'echo LD_LIBRARY_PATH is\n',
'echo $LD_LIBRARY_PATH | tr ":" "\n"\n',
'echo =============================\n', 'echo PATH is\n',
'echo $PATH | tr ":" "\n"\n',
'echo =============================\n',
'env | sort >> localEnv.log\n',
'echo =============================\n',
'my_testscript123.sh mytest args 123 extra test_args\n',
'declare -x appstatus=$?\n', 'exit $appstatus\n'
], self)
chmod_mock.assert_called_once_with('Root_v123_Run_13.sh', 0755)
cmd_mock.assert_called_once_with(
0,
'sh -c "./Root_v123_Run_13.sh"',
callbackFunction=self.rea.redirectLogOutput,
bufferLimit=20971520)
def test_checkconsistency(self):
with patch.object(self.fcc,
'_importToSandbox',
new=Mock(return_value=True)):
info_message = (
"Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {
'name': self.fcc.appname
})
assertDiracSucceedsWith(self.fcc._checkConsistency(), info_message,
self)
self.log_mock.info.assert_called_with(info_message)
self.log_mock.info.assert_any_call(
"Sandboxing : Sandboxing successfull")
def test_execute_success(self):
trans_mock = Mock()
trans_mock.transferAndRegisterFile.return_value = S_OK('bla')
trans_mock.transferAndRegisterFileFailover.return_value = S_OK('bla')
self.upod.enable = True
self.upod.jobID = 13831
self.upod.failoverTest = True
self.upod.prodOutputLFNs = ['/ilc/prod/ilc/mc-dbd/example_file']
with patch.object(self.upod, 'getCandidateFiles', new=Mock(return_value=S_OK({}))), \
patch.object(self.upod, 'getFileMetadata', new=Mock(return_value=S_OK( { 'fileTestName' : { 'workflowSE' : 'testSE', 'otherTestMetadata' : True, 'localpath' : None, 'lfn' : None, 'resolvedSE' : None, 'filedict' : None } } ))), \
patch('%s.getDestinationSEList' % MODULE_NAME, new=Mock(return_value=S_OK('myTestSE'))), \
patch('%s.FailoverTransfer' % MODULE_NAME, new=Mock(return_value=trans_mock)):
assertDiracSucceedsWith(self.upod.execute(),
'Output data uploaded', self)
assertEqualsImproved(self.upod.experiment, 'ILC_ILD', self)
def test_getdestinationse_alias_found(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
'myTest_Alias_value'
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country',
'AnyOutputmodeTest')
assertDiracSucceedsWith(result, 'myTest_Alias_value', self)
def test_construct_user_lfns_getvo_fails( self ):
today_mock = Mock()
today_mock.timetuple.return_value = [ 1999, 27 ]
date_mock = Mock()
date_mock.today.return_value = today_mock
with patch('%s.datetime.date' % MODULE_NAME, new=date_mock), \
patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='testpre')), \
patch('%s.getVOfromProxyGroup' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))), \
patch('%s.gLogger.error' % MODULE_NAME) as log_mock:
result = constructUserLFNs( 1234567, None, 'mytestowner',
[ 'myTestOutputFile1.txt', 'myTestOutputFile2.ppt' ],
'/mydir/dir//MyTestOutputPath1' )
assertDiracSucceedsWith(
result, '/ilc/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile1.txt', self )
assertDiracSucceedsWith(
result, '/ilc/testpre/m/mytestowner/mydir/dir/MyTestOutputPath1/myTestOutputFile2.ppt', self )
log_mock.assert_called_once_with( 'Could not get VO from CS, assuming ilc' )
def test_applicationspecificinputs(self):
self.rod.step_commons['Enable'] = True
self.rod.workflow_commons[
'ProductionOutputData'] = 'MyOutputLFN1.php;other__lfn.stdio;;last_file.stdhep'
self.rod.workflow_commons['Luminosity'] = 1391.2
self.rod.workflow_commons['Info'] = {
'stdhepcut': {
'Reduction': 12.2,
'CutEfficiency': 0.92
}
}
assertDiracSucceedsWith(self.rod.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved(
(self.rod.enable, self.rod.prodOutputLFNs, self.rod.luminosity,
self.rod.sel_eff, self.rod.cut_eff, self.rod.add_info), (True, [
'MyOutputLFN1.php', 'other__lfn.stdio', '', 'last_file.stdhep'
], 1391.2, 12.2, 0.92, 'de'), self)
assert 'stdhepcut' not in self.rod.workflow_commons['Info']
def test_checkconsistency_run_read(self):
file_content = (
'#!/bin/sh\n'
'exec /cvmfs/sft.cern.ch/lcg/views/LCG_88/x86_64-slc6-gcc49-opt/bin/python'
' /cvmfs/fcc.cern.ch/sw/0.8/gaudi/v28r2/x86_64-slc6-gcc49-opt/scripts/xenv '
'--xml /afs/cern.ch/user/s/sfernana/FCC/FCCSW/build.x86_64-slc6-gcc49-opt/config/FCCSW-build.xenv "$@"\n'
)
build_folder = 'build.x86_64-slc6-gcc49-opt'
build_path = os.path.join(self.fcc.fccSwPath, build_folder)
executable_path = os.path.join(build_path, 'run')
info_message = ("Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {
'name': self.fcc.appname
})
read_message = 'Sandboxing : FCC file reading successfull'
with patch('os.path.exists') as mock_exists, \
patch('os.listdir') as mock_listdir, \
patch('os.path.isfile') as mock_isfile, \
patch("%s._checkConsistency" % MODULE_NAME) as mock_check, \
patch("%s._readFromFile" % MODULE_NAME) as mock_read:
mock_exists.return_value = True
mock_listdir.return_value = [build_folder]
mock_isfile.return_value = False
mock_read.return_value = (file_content, read_message)
mock_check.return_value = S_OK(info_message)
assertDiracSucceedsWith(self.fcc._checkConsistency(), info_message,
self)
self.log_mock.debug.assert_any_call(
"FCCSW specific consistency : The temporary folder 'temp_fcc_dirac' already exists"
)
mock_listdir.assert_called_once_with(self.fcc.fccSwPath)
mock_isfile.assert_called_once_with(build_path)
mock_exists.assert_any_call(executable_path)
mock_read.assert_called_once_with(executable_path)
self.log_mock.debug.assert_any_call(read_message)
def test_upload_putrequest_works( self ):
self.module_patcher.stop()
req_mock = Mock()
reqval_mock = Mock()
reqval_mock.RequestValidator().validate.return_value = S_OK()
reqclient_mock = Mock()
reqclient_mock.ReqClient().putRequest.return_value = S_OK()
op_mock = Mock()
op_list = [ Mock(), Mock(), Mock(), Mock() ]
file_list = [ Mock(), Mock(), Mock(), Mock() ]
copies_at_list = [ 'MyCopySE1', 'OtherSE', 'LastSE', '' ]
op_mock.Operation.side_effect = op_list
op_mock.File.side_effect = file_list
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : self.dm_mock,
'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : self.ops_mock,
'DIRAC.RequestManagementSystem.Client.Request' : req_mock,
'DIRAC.RequestManagementSystem.private.RequestValidator' : reqval_mock,
'DIRAC.RequestManagementSystem.Client.ReqClient' : reqclient_mock,
'DIRAC.RequestManagementSystem.Client.Operation' : op_mock }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
self.dm_mock.DataManager().putAndRegister.return_value = S_OK()
self.ops_mock.Operations().getValue.side_effect = [ None, [ 'MyCopySE1', 'OtherSE', 'LastSE', '' ] ]
from ILCDIRAC.Core.Utilities.FileUtils import upload
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=True)):
assertDiracSucceedsWith( upload( '/some/local/path/', 'appTarTest.tgz' ), 'Application uploaded', self )
assertMockCalls( self.ops_mock.Operations().getValue,
[ ( 'Software/BaseStorageElement', 'CERN-SRM' ), ( 'Software/CopiesAt', [] ) ], self )
request_to_test = req_mock.Request()
assertEqualsImproved( request_to_test.RequestName, 'copy_appTarTest', self )
assertEqualsImproved( request_to_test.SourceComponent, 'ReplicateILCSoft', self )
assertMockCalls( request_to_test.addOperation, op_list, self )
assertEqualsImproved( len( op_list ), len( copies_at_list ), self )
index = 0
for operation, se in zip(op_list, copies_at_list):
assertEqualsImproved( operation.Type, 'ReplicateAndRegister', self )
assertEqualsImproved( operation.TargetSE, se, self )
operation.addFile.assert_called_once_with( file_list[ index ] )
assertEqualsImproved( ( file_list[ index ].LFN, file_list[ index ].GUID ),
( '/some/local/path/appTarTest.tgz', '' ), self )
index += 1
def test_execute_maximal_othercase( self ):
self.rod.nbofevents = 1389
self.rod.luminosity = 9814.2
self.rod.sel_eff = 184.2
self.rod.cut_eff = 13.1
self.rod.InputData = 'myTestInputFiles.rec'
self.rod.inputdataMeta[ 'CrossSection' ] = 'myTestCrosssection'
self.rod.inputdataMeta[ 'AdditionalInfo' ] = 'more_information_testme'
self.rod.WorkflowStartFrom = 'EventZerotest'
self.rod.prodOutputLFNs = [ '/some/test/dir/mytestfile.txt' ]
fcc_mock = Mock()
fcc_mock.setMetadata.return_value = S_OK()
fcc_mock.addFileAncestors.return_value = S_OK()
self.rod.filecatalog = fcc_mock
with patch('%s.RegisterOutputData.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracSucceedsWith( self.rod.execute(), 'Output data metadata registered in catalog', self )
fcc_mock.setMetadata.assert_called_once_with( '/some/test/dir/mytestfile.txt', {
'NumberOfEvents' : 1389, 'Luminosity' : 9814.2, 'Reduction' : 184.2,
'CutEfficiency' : 13.1, 'AdditionalInfo' : 'more_information_testme',
'CrossSection' : 'myTestCrosssection', 'FirstEventFromInput' : 'EventZerotest' } )
fcc_mock.addFileAncestors.assert_called_once_with( { '/some/test/dir/mytestfile.txt' : { 'Ancestors' : 'myTestInputFiles.rec' } } )
def test_checkconsistency_run_read( self ):
file_content = (
'#!/bin/sh\n'
'exec /cvmfs/sft.cern.ch/lcg/views/LCG_88/x86_64-slc6-gcc49-opt/bin/python'
' /cvmfs/fcc.cern.ch/sw/0.8/gaudi/v28r2/x86_64-slc6-gcc49-opt/scripts/xenv '
'--xml /afs/cern.ch/user/s/sfernana/FCC/FCCSW/build.x86_64-slc6-gcc49-opt/config/FCCSW-build.xenv "$@"\n'
)
build_folder = 'build.x86_64-slc6-gcc49-opt'
build_path = os.path.join(self.fcc.fccSwPath, build_folder)
executable_path = os.path.join(build_path, 'run')
info_message = (
"Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {'name':self.fcc.appname}
)
read_message = 'Sandboxing : FCC file reading successfull'
with patch('os.path.exists') as mock_exists, \
patch('os.listdir') as mock_listdir, \
patch('os.path.isfile') as mock_isfile, \
patch("%s._checkConsistency" % MODULE_NAME) as mock_check, \
patch("%s._readFromFile" % MODULE_NAME) as mock_read, \
patch.object(inspect.getmodule(FccSw), 'LOG', new=self.log_mock):
mock_exists.return_value = True
mock_listdir.return_value = [build_folder]
mock_isfile.return_value = False
mock_read.return_value = (file_content, read_message)
mock_check.return_value = S_OK(info_message)
assertDiracSucceedsWith( self.fcc._checkConsistency(), info_message, self )
self.log_mock.debug.assert_any_call( "FCCSW specific consistency : The temporary folder 'temp_fcc_dirac' already exists" )
mock_listdir.assert_called_once_with( self.fcc.fccSwPath )
mock_isfile.assert_called_once_with( build_path )
mock_exists.assert_any_call( executable_path )
mock_read.assert_called_once_with( executable_path )
self.log_mock.debug.assert_any_call( read_message )
def test_getenvscript(self):
file_contents = []
text_file_data = '\n'.join(file_contents)
result = {}
with patch('%s.open' % MODULE_NAME,
mock_open(read_data=text_file_data),
create=True) as file_mocker:
result = self.marAna.getEnvScript(None, None, None)
assertDiracSucceedsWith(result, '/MarlinEnv.sh', self)
check_in_script = [
'declare -x PATH=aFolder/Executable:$PATH\n',
'declare -x ROOTSYS=aFolder/ROOT\n',
'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:aFolder/LDLibs:bFolder\n',
"declare -x MARLIN_DLL=aFolder/MARLIN_DLL/testdir:\n",
"declare -x PANDORASETTINGS=aFolder/Settings/PandoraSettings.xml"
]
file_mocker.assert_called_with('MarlinEnv.sh', 'w')
mocker_handle = file_mocker()
for expected in check_in_script:
mocker_handle.write.assert_any_call(expected)
self.assertEquals(
len(check_in_script),
mocker_handle.__enter__.return_value.write.call_count - 4)
def test_runit_complete( self ):
self.rea.platform = 'myTestPlatform'
self.rea.applicationLog = '/my/applog/test.log'
self.rea.script = '/my/test/my_testscript123.sh'
self.rea.workflowStatus['OK'] = True
self.rea.stepStatus['OK'] = True
self.rea.applicationVersion = 'v123'
self.rea.STEP_NUMBER = 13
self.rea.arguments = 'mytest args 123'
self.rea.extraCLIarguments = 'extra test_args'
with patch('%s.getEnvironmentScript' % MODULE_NAME, new=Mock(return_value=S_OK('/mytestenvscript'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[False,False,False,False,True])) as exists_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK((0, 'my_statusmsg')))) as cmd_mock:
assertDiracSucceedsWith( self.rea.runIt(), 'ROOT v123 Successful', self )
assertMockCalls( exists_mock, [ 'Root_v123_Run_13.sh', './lib', 'my_testscript123.sh',
'/my/applog/test.log', '/my/applog/test.log' ], self )
self.assertFalse( remove_mock.called )
open_mock.assert_called_once_with( 'Root_v123_Run_13.sh', 'w' )
open_mock = open_mock()
open_mock.close.assert_called_once_with()
assertMockCalls( open_mock.write, [
'#!/bin/sh \n', '#####################################################################\n',
'# Dynamically generated script to run a production or analysis job. #\n',
'#####################################################################\n',
'source /mytestenvscript\n', 'echo =============================\n', 'echo LD_LIBRARY_PATH is\n',
'echo $LD_LIBRARY_PATH | tr ":" "\n"\n', 'echo =============================\n',
'echo PATH is\n', 'echo $PATH | tr ":" "\n"\n', 'echo =============================\n',
'env | sort >> localEnv.log\n', 'echo =============================\n',
'my_testscript123.sh mytest args 123 extra test_args\n', 'declare -x appstatus=$?\n',
'exit $appstatus\n' ], self )
chmod_mock.assert_called_once_with('Root_v123_Run_13.sh', 0o755)
cmd_mock.assert_called_once_with( 0, 'sh -c "./Root_v123_Run_13.sh"',
callbackFunction = self.rea.redirectLogOutput,
bufferLimit = 20971520 )
def test_execute_maximal( self ):
self.rod.nbofevents = 1389
self.rod.luminosity = 9814.2
self.rod.sel_eff = 184.2
self.rod.cut_eff = 13.1
self.rod.InputData = 'myTestInputFiles.rec'
self.rod.inputdataMeta[ 'CrossSection' ] = 'myTestCrosssection'
self.rod.WorkflowStartFrom = 'EventZerotest'
self.rod.add_info = 'MoreInfo.additional_testme'
self.rod.workflow_commons[ 'file_number_of_event_relation' ] = { 'myOutput.lfn.stdhep' : 2148 }
self.rod.prodOutputLFNs = [ '/some/dir/myOutput.lfn.stdhep', 'some_other.file', '/other/dir/lastOne', '' ]
fcc_mock = Mock()
fcc_mock.setMetadata.return_value = S_OK()
fcc_mock.addFileAncestors.return_value = S_OK()
self.rod.filecatalog = fcc_mock
with patch('%s.RegisterOutputData.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracSucceedsWith( self.rod.execute(), 'Output data metadata registered in catalog', self )
assertMockCalls( fcc_mock.setMetadata, [
( '/some/dir/myOutput.lfn.stdhep', { 'NumberOfEvents' : 2148, 'Luminosity' : 9814.2,
'Reduction' : 184.2, 'CutEfficiency' : 13.1, 'AdditionalInfo' : 'MoreInfo.additional_testme',
'CrossSection' : 'myTestCrosssection',
'FirstEventFromInput' : 'EventZerotest' } ),
( 'some_other.file', { 'NumberOfEvents' : 1389, 'Luminosity' : 9814.2,
'Reduction' : 184.2, 'CutEfficiency' : 13.1, 'AdditionalInfo' : 'MoreInfo.additional_testme',
'CrossSection' : 'myTestCrosssection', 'FirstEventFromInput' : 'EventZerotest' } ),
( '/other/dir/lastOne', { 'NumberOfEvents' : 1389, 'Luminosity' : 9814.2, 'Reduction' : 184.2,
'CutEfficiency' : 13.1, 'AdditionalInfo' : 'MoreInfo.additional_testme',
'CrossSection' : 'myTestCrosssection', 'FirstEventFromInput' :
'EventZerotest' } ),
( '', { 'NumberOfEvents' : 1389, 'Luminosity' : 9814.2, 'Reduction' : 184.2, 'CutEfficiency' : 13.1,
'AdditionalInfo' : 'MoreInfo.additional_testme', 'CrossSection' : 'myTestCrosssection',
'FirstEventFromInput' : 'EventZerotest' } ) ], self )
assertMockCalls( fcc_mock.addFileAncestors, [
{ '/some/dir/myOutput.lfn.stdhep' : { 'Ancestors' : 'myTestInputFiles.rec' } },
{ 'some_other.file' : { 'Ancestors' : 'myTestInputFiles.rec' } },
{ '/other/dir/lastOne' : { 'Ancestors' : 'myTestInputFiles.rec' } },
{ '' : { 'Ancestors' : 'myTestInputFiles.rec' } } ], self )
def test_checkconsistency_run_read_failed(self):
build_folder = 'build.x86_64-slc6-gcc49-opt'
build_path = os.path.join(self.fcc.fccSwPath, build_folder)
executable_path = os.path.join(build_path, 'run')
info_message = ("Application general consistency : _checkConsistency()"
" on '%(name)s' successfull" % {
'name': self.fcc.appname
})
with patch('os.path.exists') as mock_exists, \
patch('os.listdir') as mock_listdir, \
patch('os.path.isfile') as mock_isfile, \
patch("%s._checkConsistency" % MODULE_NAME) as mock_check, \
patch("%s._readFromFile" % MODULE_NAME) as mock_read:
mock_exists.return_value = True
mock_listdir.return_value = [build_folder]
mock_isfile.return_value = False
mock_read.return_value = (None, 'DO NOT READ ME !')
mock_check.return_value = S_OK(info_message)
assertDiracSucceedsWith(self.fcc._checkConsistency(), info_message,
self)
self.log_mock.debug.assert_any_call(
"FCCSW specific consistency : The temporary folder 'temp_fcc_dirac' already exists"
)
mock_listdir.assert_called_once_with(self.fcc.fccSwPath)
mock_isfile.assert_called_once_with(build_path)
mock_exists.assert_any_call(executable_path)
mock_read.assert_called_once_with(executable_path)
self.log_mock.warn.assert_called_once_with('DO NOT READ ME !')
self.log_mock.debug.assert_any_call(
'FCCSW specific consistency : Using by default the command of the 0.8.1 release !'
)
def test_applicationspecificinputs_othercases( self ):
self.rod.step_commons[ 'Enable' ] = { 'mydict' : True }
assertDiracSucceedsWith( self.rod.applicationSpecificInputs(), 'Parameters resolved', self )
assertEqualsImproved( ( self.rod.enable, self.rod.sel_eff, self.rod.cut_eff, self.rod.add_info ),
( False, 0, 0, '' ), self )
def test_execute_minimal( self ):
self.rod.enable = False
self.rod.prodOutputLFNs = [ 'myOutput.lfn.stdhep' ]
with patch('%s.RegisterOutputData.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracSucceedsWith( self.rod.execute(), 'Output data metadata registered in catalog', self )
def test_execute_no_production_output_data( self ): assertDiracSucceedsWith( self.rod.execute(), "No files' metadata to be registered", self )
def test_getcurrentvo( self ):
with patch('%s.getProxyInfo' % MODULE_NAME, new=Mock(return_value=S_OK({ 'myproxydict' : True, 'group' : 'testilc_blabla' }))):
result = self.ujf.getCurrentVO()
assertDiracSucceedsWith( result, 'testilc', self )
def test_applicationspecificinputs( self ): self.rea.script = 'myTestscript' assertDiracSucceedsWith( self.rea.applicationSpecificInputs(), 'Parameters resolved', self )
def test_checkparams( self ):
job_mock = Mock()
job_mock.errorDict = {}
with patch('%s.DiracILC.preSubmissionChecks' % MODULE_NAME, new=Mock(return_value=S_OK('mytest'))) as check_mock:
assertDiracSucceedsWith( self.dilc.checkparams( job_mock ), 'mytest', self )
check_mock.assert_called_once_with( job_mock, mode = '' )
def test_getcurrentowner( self ):
with patch('%s.getProxyInfo' % MODULE_NAME, new=Mock(return_value=S_OK({ 'myproxydict' : True, 'username' : 'readThisOut' }))):
result = self.ujf.getCurrentOwner()
assertDiracSucceedsWith( result, 'readThisOut', self )
def test_runit_stepbad( self ):
self.marAna.platform = "Testplatform123"
self.marAna.applicationLog = "testlog123"
self.marAna.stepStatus = { 'OK' : False }
result = self.marAna.runIt()
assertDiracSucceedsWith( result, 'should not proceed', self )
def test_applicationspecificinputs_nonset( self ):
self.rod.workflow_commons[ 'Info' ] = { 'SomeKey' : 'someValue' }
assertDiracSucceedsWith( self.rod.applicationSpecificInputs(), 'Parameters resolved', self )
assertEqualsImproved( ( self.rod.enable, self.rod.prodOutputLFNs, self.rod.luminosity,
self.rod.nbofevents, self.rod.sel_eff, self.rod.cut_eff, self.rod.add_info ),
( True, [], 0, 0, 0, 0, 'ds7:SomeKeys9:someValuee' ), self )
def test_finalstatusreport_ignorefail( self ):
self.moba.ignoreapperrors = True
with patch('%s.ModuleBase.setApplicationStatus' % MODULE_NAME) as appstat_mock:
assertDiracSucceedsWith( self.moba.finalStatusReport( 1 ), ' exited With Status 1', self )
appstat_mock.assert_called_once_with( ' exited With Status 1' )
self.assertTrue(self.log_mock.error.called)
def test_finalstatusreport( self ):
self.moba.ignoreapperrors = False
with patch('%s.ModuleBase.setApplicationStatus' % MODULE_NAME) as appstat_mock:
assertDiracSucceedsWith( self.moba.finalStatusReport( 0 ), ' Successful', self )
appstat_mock.assert_called_once_with( ' Successful' )
self.assertFalse(self.log_mock.error.called)
def test_execute_status_not_ok( self ): self.rod.workflowStatus[ 'OK' ] = True self.rod.stepStatus[ 'OK' ] = False assertDiracSucceedsWith( self.rod.execute(), 'No registration of output data metadata attempted', self )
