def check_scriptwriting_method( self, mylfn, scriptmethod, expected, should_fail_with = '',
exists_sideeff = None, unlink_called = True,
file_contents = None, expected_opens = None,
environ_dict = None, is_ral = False ):
""" Helper method that checks one of the methods provided by OverlayInput
:param str mylfn: file path (LFN)
:param method scriptmethod: getLyonFile etc
:param list of list of strings expected: Expected output in the script file (list of lists with all touched files)
:param str should_fail_with: Error message the method should return (in a S_ERROR structure)
:param list of bool exists_sideeff: Return values of the os.path.exists method
:param bool unlink_called: indicates whether os.unlink is expected to be called or not
:param list of pair of string expected_opens: a list of (filename, mode) pairs of all opened files
"""
# Set default list values
if exists_sideeff is None:
exists_sideeff = [ True, True, True ]
if file_contents is None:
file_contents = [[]]
if environ_dict is None:
environ_dict = {}
if expected_opens is None:
expected_opens = [('overlayinput.sh', 'w')]
handles = FileUtil.getMultipleReadHandles( file_contents )
# Variable mocks: exists return value, open values
with patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=0)) as shell_mock, \
patch('%s.os.unlink' % MODULE_NAME, new=Mock(return_value=True)) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=exists_sideeff)), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock(return_value=True)) as chmod_mock, \
patch.dict(os.environ, environ_dict, True), \
patch('%s.os.getuid' % MODULE_NAME, new=Mock(return_value='mytestuserid')):
mo.side_effect = (h for h in handles)
result = scriptmethod( mylfn )
if should_fail_with:
assertDiracFailsWith( result, should_fail_with, self )
else:
assertDiracSucceedsWith_equals( result, 'testfile.txt', self )
shell_mock.assert_called_with( 600, 'sh -c "./overlayinput.sh"',
bufferLimit = 20971520,
callbackFunction = self.over.redirectLogOutput )
chmod_mock.assert_called_with('overlayinput.sh', 0o755)
if unlink_called:
remove_mock.assert_called_with( 'overlayinput.sh' )
else:
self.assertFalse( remove_mock.called )
# Check if output to files is correct
FileUtil.checkFileInteractions( self, mo, expected_opens, expected, handles )
if is_ral:
self.assertIn( 'CNS_HOST', os.environ )
self.assertIn( 'STAGE_SVCCLASS', os.environ )
self.assertIn( 'STAGE_HOST', os.environ )
def test_fromwhizarddotin(self):
self.whop.paramdict = {}
self.whop.whizardxml = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float"> </beam_intensity> <beam_density type="0/1/2/3"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string"> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray"> </myfloatarr> <lasttestvalue type="T/F"> </lasttestvalue> </lasttestkey> </whizardxml>'
)
expected_tree = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float" value="84.2"> </beam_intensity> <beam_density type="0/1/2/3" value="0"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string" value="TmyFstring"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string" value="teststring_dontloseme."> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray" value="943 0.1 01.2 9024.4"> </myfloatarr> <lasttestvalue type="T/F" value="F"> </lasttestvalue> </lasttestkey> </whizardxml>'
)
expected_tree.find(
'beam_input_1/beam_intensity').attrib['value'] = 84.2
expected_tree.find('beam_input_1/beam_density').attrib['value'] = 0
file_contents = [[
' !initial Comment, ignore this line ',
'/ Other comment. ',
' ', 'ignoreThisLineTOO',
'&beam_input', 'beam_intensity=84.2', 'beam_density=0',
'&myotherkey', '&testkey', 'testcurkey="teststring_dontloseme."',
'&beam_input ', ' FcoolTtestkey = "TmyFstring" ',
'&lasttestkey', 'myfloatarr= 943 0.1 01.2 9024.4',
'lasttestvalue=F'
]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo:
mo.side_effect = (h for h in handles)
result = self.whop.fromWhizardDotIn('filename.txt')
assertDiracSucceeds(result, self)
assertEqualsXmlTree(result['Value'], expected_tree, self)
def test_preparescript( self ):
exists_dict = { 'TestApp_vT_Run_148.sh' : True, './lib' : True }
( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion, self.shc.STEP_NUMBER, self.shc.MaxNbEvts, self.shc.OutputFile, self.shc.SteeringFile ) = ( 'testPlatformV1', 'TestApp', 'vT', 148, 13, 'test_OF.ile', 'steer_test.file' )
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/my/testsoft/dir1/')) as getldlibs_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo:
mo.side_effect = ( h for h in handles )
self.shc.prepareScript( 'test_software/dir' )
remove_mock.assert_called_once_with( 'TestApp_vT_Run_148.sh' )
getldlibs_mock.assert_called_once_with( 'testPlatformV1', 'TestApp', 'vT' )
assertMockCalls( exists_mock, [ './lib', 'TestApp_vT_Run_148.sh' ], self )
mo.assert_called_once_with( 'TestApp_vT_Run_148.sh', 'w' )
assertEqualsImproved( len( handles ), 1, self )
assertMockCalls( handles[0].write, [
'#!/bin/sh \n', '#####################################################################\n',
'# Dynamically generated script to run a production or analysis job. #\n',
'#####################################################################\n',
'declare -x PATH=test_software/dir:$PATH\n',
'declare -x LD_LIBRARY_PATH=./lib:test_software/dir/lib:/my/testsoft/dir1/\n',
'env | sort >> localEnv.log\n', 'echo =============================\n',
"stdhepCut -m 13 -o test_OF.ile -c steer_test.file ../*.stdhep\n",
'declare -x appstatus=$?\n', 'exit $appstatus\n' ], self )
assert handles[0].close.called
def test_preparescript(self):
exists_dict = {'TestApp_vT_Run_148.sh': True, './lib': True}
(self.shc.platform, self.shc.applicationName,
self.shc.applicationVersion, self.shc.STEP_NUMBER, self.shc.MaxNbEvts,
self.shc.OutputFile,
self.shc.SteeringFile) = ('testPlatformV1', 'TestApp', 'vT', 148, 13,
'test_OF.ile', 'steer_test.file')
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/my/testsoft/dir1/')) as getldlibs_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo:
mo.side_effect = (h for h in handles)
self.shc.prepareScript('test_software/dir')
remove_mock.assert_called_once_with('TestApp_vT_Run_148.sh')
getldlibs_mock.assert_called_once_with('testPlatformV1', 'TestApp',
'vT')
assertMockCalls(exists_mock, ['./lib', 'TestApp_vT_Run_148.sh'],
self)
mo.assert_called_once_with('TestApp_vT_Run_148.sh', 'w')
assertEqualsImproved(len(handles), 1, self)
assertMockCalls(handles[0].write, [
'#!/bin/sh \n',
'#####################################################################\n',
'# Dynamically generated script to run a production or analysis job. #\n',
'#####################################################################\n',
'declare -x PATH=test_software/dir:$PATH\n',
'declare -x LD_LIBRARY_PATH=./lib:test_software/dir/lib:/my/testsoft/dir1/\n',
'env | sort >> localEnv.log\n',
'echo =============================\n',
"stdhepCut -m 13 -o test_OF.ile -c steer_test.file ../*.stdhep\n",
'declare -x appstatus=$?\n', 'exit $appstatus\n'
], self)
assert handles[0].close.called
def test_preparemarlindll_swaplibpositions( self ):
self.marAna.ProcessorListToExclude = [ 'mytestlibrary.so' ]
exists_dict = { './lib/marlin_dll' : True }
def replace_exists( path ):
return exists_dict[path]
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles( file_contents )
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \
patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL( 'some_path' )
open_mock.assert_called_once_with( 'temp.sh', 'w' )
assertMockCalls( handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path > /dev/null\necho $MARLIN_DLL'], self)
assertEqualsImproved( len( handles ), 1, self )
chmod_mock.assert_called_once_with('temp.sh', 0o755)
shell_mock.assert_called_once_with( 0, './temp.sh' )
remove_mock.assert_called_once_with( 'temp.sh' )
exists_mock.assert_called_once_with( './lib/marlin_dll' )
glob_mock.assert_called_once_with( './lib/marlin_dll/*.so' )
assertDiracSucceedsWith( result, 'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so', self )
def test_preparemarlindll_swaplibpositions(self):
self.marAna.ProcessorListToExclude = ['mytestlibrary.so']
exists_dict = {'./lib/marlin_dll': True}
def replace_exists(path):
return exists_dict[path]
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK([ 0, 'MARlin_DLL/path', 'other_return_value_from_shell' ]))) as shell_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock(side_effect=[ True ])) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=replace_exists)) as exists_mock, \
patch('%s.glob.glob' % MODULE_NAME, new=Mock(return_value=[ 'testlibLCFIPlus.so', 'testlibLCFIVertex.1.so' ])) as glob_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL('some_path')
open_mock.assert_called_once_with('temp.sh', 'w')
assertMockCalls(
handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path\necho $MARLIN_DLL'], self)
assertEqualsImproved(len(handles), 1, self)
chmod_mock.assert_called_once_with('temp.sh', 0755)
shell_mock.assert_called_once_with(0, './temp.sh')
remove_mock.assert_called_once_with('temp.sh')
exists_mock.assert_called_once_with('./lib/marlin_dll')
glob_mock.assert_called_once_with('./lib/marlin_dll/*.so')
assertDiracSucceedsWith(
result,
'MARlin_DLL/path:testlibLCFIVertex.1.so:testlibLCFIPlus.so',
self)
def test_main_wrong_paraminput( self ):
print_mock = Mock()
pprint_mock = Mock(return_value=print_mock)
file_contents = [ [ '&process_input', 'parameter_input=123', 'process_id=myprocessid', 'cm_frame=F',
'&integration_input', '&simulation_input', 'unweighted=F', '&diagnostics_input ',
' write_logfile_file = "my_diagnostics_logfile.txt" ' ] ]
handles = FileUtil.getMultipleReadHandles( file_contents )
with patch('sys.argv', [ 'scriptname', 'filename.xml', 'mymodel' ]), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo, \
patch.dict( sys.modules, { 'pprint' : pprint_mock } ):
mo.side_effect = (h for h in handles)
assertEqualsImproved( main(), 1, self )
mo.assert_called_once_with( 'filename.xml', 'r' )
def test_applicationspecificinputs( self ):
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
self.shc.inlineCuts = 'first_line;Some more content of this file; @end \n of% fi/le'
with patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'something.txt', 'myoutputfile.stdhep', 'stdhep.io' ])), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo:
mo.side_effect = ( h for h in handles )
assertDiracSucceeds( self.shc.applicationSpecificInputs(), self )
assertEqualsImproved( ( self.shc.OutputFile, self.shc.SteeringFile ),
( 'myoutputfile_reduced.stdhep', 'cuts_local.txt' ), self )
mo.assert_called_once_with( 'cuts_local.txt', 'w' )
assertEqualsImproved( len( handles ), 1, self )
handles[0].write.assert_any_call( 'first_line\nSome more content of this file\n @end \n of% fi/le' )
assert handles[0].close.called
def test_preparemarlindll_shellcall_fails( self ):
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles( file_contents )
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))) as shell_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL( 'some_path' )
open_mock.assert_called_once_with( 'temp.sh', 'w' )
assertMockCalls( handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path > /dev/null\necho $MARLIN_DLL'], self)
assertEqualsImproved( len( handles ), 1, self )
chmod_mock.assert_called_once_with('temp.sh', 0o755)
shell_mock.assert_called_once_with( 0, './temp.sh' )
assertDiracFailsWith( result, 'failed getting the marlin_dll', self )
def test_main_wrong_paraminput(self):
print_mock = Mock()
pprint_mock = Mock(return_value=print_mock)
file_contents = [[
'&process_input', 'parameter_input=123', 'process_id=myprocessid',
'cm_frame=F', '&integration_input', '&simulation_input',
'unweighted=F', '&diagnostics_input ',
' write_logfile_file = "my_diagnostics_logfile.txt" '
]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('sys.argv', [ 'scriptname', 'filename.xml', 'mymodel' ]), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True ) as mo, \
patch.dict( sys.modules, { 'pprint' : pprint_mock } ):
mo.side_effect = (h for h in handles)
assertEqualsImproved(main(), 1, self)
mo.assert_called_once_with('filename.xml', 'r')
def test_applicationspecificinputs(self):
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
self.shc.inlineCuts = 'first_line;Some more content of this file; @end \n of% fi/le'
with patch('%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'something.txt', 'myoutputfile.stdhep', 'stdhep.io' ])), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo:
mo.side_effect = (h for h in handles)
assertDiracSucceeds(self.shc.applicationSpecificInputs(), self)
assertEqualsImproved(
(self.shc.OutputFile, self.shc.SteeringFile),
('myoutputfile_reduced.stdhep', 'cuts_local.txt'), self)
mo.assert_called_once_with('cuts_local.txt', 'w')
assertEqualsImproved(len(handles), 1, self)
handles[0].write.assert_any_call(
'first_line\nSome more content of this file\n @end \n of% fi/le')
assert handles[0].close.called
def test_preparemarlindll_shellcall_fails(self):
file_contents = [[]]
handles = FileUtil.getMultipleReadHandles(file_contents)
with patch('%s.open' % MODULE_NAME, mock_open()) as open_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))) as shell_mock:
open_mock.side_effect = (h for h in handles)
result = self.marAna.prepareMARLIN_DLL('some_path')
open_mock.assert_called_once_with('temp.sh', 'w')
assertMockCalls(
handles[0].__enter__().write,
['#!/bin/bash\n', 'source some_path\necho $MARLIN_DLL'], self)
assertEqualsImproved(len(handles), 1, self)
chmod_mock.assert_called_once_with('temp.sh', 0755)
shell_mock.assert_called_once_with(0, './temp.sh')
assertDiracFailsWith(result, 'failed getting the marlin_dll', self)
def test_fromwhizarddotin( self ):
self.whop.paramdict = {}
self.whop.whizardxml = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float"> </beam_intensity> <beam_density type="0/1/2/3"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string"> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray"> </myfloatarr> <lasttestvalue type="T/F"> </lasttestvalue> </lasttestkey> </whizardxml>' )
expected_tree = fromstring(
'<whizardxml> <beam_input_1> <beam_intensity type="float" value="84.2"> </beam_intensity> <beam_density type="0/1/2/3" value="0"> </beam_density> </beam_input_1> <beam_input_2> <FcoolTtestkey type="string" value="TmyFstring"> </FcoolTtestkey> </beam_input_2> <myotherkey> </myotherkey> <testkey> <testcurkey type="string" value="teststring_dontloseme."> </testcurkey> </testkey> <lasttestkey> <myfloatarr type="floatarray" value="943 0.1 01.2 9024.4"> </myfloatarr> <lasttestvalue type="T/F" value="F"> </lasttestvalue> </lasttestkey> </whizardxml>' )
expected_tree.find( 'beam_input_1/beam_intensity' ).attrib['value'] = 84.2
expected_tree.find( 'beam_input_1/beam_density' ).attrib['value'] = 0
file_contents = [ [ ' !initial Comment, ignore this line ', '/ Other comment. ',
' ', 'ignoreThisLineTOO', '&beam_input',
'beam_intensity=84.2', 'beam_density=0', '&myotherkey', '&testkey',
'testcurkey="teststring_dontloseme."', '&beam_input ',
' FcoolTtestkey = "TmyFstring" ', '&lasttestkey',
'myfloatarr= 943 0.1 01.2 9024.4', 'lasttestvalue=F' ] ]
handles = FileUtil.getMultipleReadHandles( file_contents )
with patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo:
mo.side_effect = (h for h in handles)
result = self.whop.fromWhizardDotIn( 'filename.txt' )
assertDiracSucceeds( result, self )
assertEqualsXmlTree( result['Value'], expected_tree, self )
def check_scriptwriting_method(self,
mylfn,
scriptmethod,
expected,
should_fail_with='',
exists_sideeff=None,
unlink_called=True,
file_contents=None,
expected_opens=None,
environ_dict=None,
is_ral=False):
""" Helper method that checks one of the methods provided by OverlayInput
:param str mylfn: file path (LFN)
:param method scriptmethod: getLyonFile etc
:param list of list of strings expected: Expected output in the script file (list of lists with all touched files)
:param str should_fail_with: Error message the method should return (in a S_ERROR structure)
:param list of bool exists_sideeff: Return values of the os.path.exists method
:param bool unlink_called: indicates whether os.unlink is expected to be called or not
:param list of pair of string expected_opens: a list of (filename, mode) pairs of all opened files
"""
# Set default list values
if exists_sideeff is None:
exists_sideeff = [True, True, True]
if file_contents is None:
file_contents = [[]]
if environ_dict is None:
environ_dict = {}
if expected_opens is None:
expected_opens = [('overlayinput.sh', 'w')]
handles = FileUtil.getMultipleReadHandles(file_contents)
# Variable mocks: exists return value, open values
with patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=0)) as shell_mock, \
patch('%s.os.unlink' % MODULE_NAME, new=Mock(return_value=True)) as remove_mock, \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=exists_sideeff)), \
patch('%s.open' % MODULE_NAME, mock_open(), create=True) as mo, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock(return_value=True)) as chmod_mock, \
patch.dict(os.environ, environ_dict, True), \
patch('%s.os.getuid' % MODULE_NAME, new=Mock(return_value='mytestuserid')):
mo.side_effect = (h for h in handles)
result = scriptmethod(mylfn)
if should_fail_with:
assertDiracFailsWith(result, should_fail_with, self)
else:
assertDiracSucceedsWith_equals(result, 'testfile.txt', self)
shell_mock.assert_called_with(
600,
'sh -c "./overlayinput.sh"',
bufferLimit=20971520,
callbackFunction=self.over.redirectLogOutput)
chmod_mock.assert_called_with('overlayinput.sh', 0755)
if unlink_called:
remove_mock.assert_called_with('overlayinput.sh')
else:
self.assertFalse(remove_mock.called)
# Check if output to files is correct
FileUtil.checkFileInteractions(self, mo, expected_opens, expected,
handles)
if is_ral:
self.assertIn('CNS_HOST', os.environ)
self.assertIn('STAGE_SVCCLASS', os.environ)
self.assertIn('STAGE_HOST', os.environ)