def test_broadcast_processed_checkchunking(self):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
import copy
files = {}
for i in xrange(0, 599):
files[('/file/dir/input%s.txt' % i)] = ['child']
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK({
'Successful': files,
'Failed': {}
})
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('BroadcastProcessed', dataman_mock,
trans_mock)
self.tfp.params['Status'] = ''
self.tfp.util = util_mock
self.tfp.setInputData(copy.deepcopy(files))
del self.tfp.data['/file/dir/input542.txt']
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME,
new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals(self.tfp.run(), 98124, self)
expected = {}
for i in xrange(0, 599):
if i != 542:
expected[('/file/dir/input%s.txt' % i)] = ['child']
assertListContentEquals(self.tfp.data, expected, self)
assertEqualsImproved(len(util_mock.fc.getFileDescendents.mock_calls),
3, self)
def test_execute_getfilemetadata_empty( self ):
exists_dict = { 'list_of.txt' : True, 'something' : True }
self.ujf.jobID = 512
request_mock = Mock()
request_mock.RequestName = 'job_512_request.xml'
request_mock.JobID = 512
request_mock.SourceComponent = "Job_512"
report_mock = Mock()
self.ujf.jobReport = report_mock
self.ujf.workflow_commons = { 'TotalSteps' : '42', 'Request' : request_mock,
'JOB_ID' : '512', 'IS_PROD' : True,
'PRODUCTION_ID' : '98245', 'Platform' : 'myTestPlatform',
'Owner' : 'myTestOwner123RichGuy', 'VO' : 'myTestVirtualOrga',
'JobReport' : report_mock }
self.ujf.workflowStatus = S_OK()
self.ujf.stepStatus = S_OK()
self.ujf.userOutputData = [ 'something' ]
self.ujf.userOutputSE = 'myTestReceivingSE'
self.ujf.userOutputPath = 'my/User/OPPath'
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.open' % MODULE_NAME, mock_open(read_data='myfilecontent,makeniceadlerchecksum')), \
patch('%s.os.path.getsize' % MODULE_NAME, new=Mock(return_value=3048)), \
patch('%s.os.getcwd' % MODULE_NAME, new=Mock(return_value='/mycurdirTestMe')), \
patch('%s.getDestinationSEList' % MODULE_NAME, new=Mock(return_value=S_OK(['CERN-DIP-4']))), \
patch('%s.UserJobFinalization.getFileMetadata' % MODULE_NAME, new=Mock(return_value=S_OK())):
result = self.ujf.execute()
assertDiracSucceedsWith_equals( result, None, self )
report_mock.setApplicationStatus.assert_called_once_with(
'No Output Data Files To Upload', True )
def test_getsoftwarefolder_from_cvmfs(self):
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_OK(('mycvmfsfolder/txt',
'otherentry')))) as cvmfs_mock:
result = getSoftwareFolder('a', 'b', 'c')
cvmfs_mock.assert_called_once_with('a', ['b', 'c'])
assertDiracSucceedsWith_equals(result, 'mycvmfsfolder/txt', self)
def test_checkcvmfs_exists( self ):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(side_effect=['testcvmfspath', 'testenvscript'])), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=True)):
result = checkCVMFS( 'mytestplatform', ('appnametesttest123',
'appversionv77testp2') )
assertDiracSucceedsWith_equals( result, ('testcvmfspath',
'testenvscript'), self )
def test_getEnvironmentScript(self):
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_OK(('/otherfolder/otherfile',
'/cvmfsfolder/myenvscript')))):
result = getEnvironmentScript('a', 'b', 'c', None)
assertDiracSucceedsWith_equals(result, '/cvmfsfolder/myenvscript',
self)
def test_getfilesfromcastor(self):
self.over.machine = 'testMach12'
import subprocess
popen_mock = Mock()
popen_mock.communicate.side_effect = [
('myfile1\nfile1923\n813tev_collision ', 'ignoreme'),
('file1.stdhep\nsome_other_file', 'ignoreme'), ('', 'ignoreme'),
('\nlast_file.txt', 'ignoreme')
]
castor_dict = {
'ProdID': 121345,
'Energy': '813',
'EvtType': 'myTestEvt',
'DetectorType': 'myTestDetectorv3'
}
with patch('subprocess.Popen',
new=Mock(return_value=popen_mock)) as proc_mock:
proc_command_dir = '/castor/cern.ch/grid/ilc/prod/testMach12/813/myTestEvt/myTestDetectorv3/SIM/00121345/'
assertDiracSucceedsWith_equals(
self.over._OverlayInput__getFilesFromCastor(castor_dict), [
proc_command_dir + 'myfile1/file1.stdhep',
proc_command_dir + 'myfile1/some_other_file',
proc_command_dir + 'file1923/',
proc_command_dir + '813tev_collision/',
proc_command_dir + '813tev_collision/last_file.txt'
], self)
assertEqualsImproved(proc_mock.mock_calls, [
call(['nsls', proc_command_dir], stdout=subprocess.PIPE),
call(['nsls', proc_command_dir + 'myfile1'],
stdout=subprocess.PIPE),
call(['nsls', proc_command_dir + 'file1923'],
stdout=subprocess.PIPE),
call(['nsls', proc_command_dir + '813tev_collision'],
stdout=subprocess.PIPE)
], self)
def test_getnumberofevents_othercases( self ):
# Expected behavior:
# If one file in a directory, get its tags, if Number of events defined go to next entry
# Else go to directory and check there. if nbevts go to next
# Else iterate over all files in directory.
# If numberevents defined nowhere, method fails
file_meta_dict = { '/unique/dir/file3' : S_OK( { 'Luminosity' : '49.2' } ),
'/one/file/myfile' : S_OK( { 'NumberOfEvents' : '14' } ),
'/other/myfile2' : S_OK( { 'Luminosity' : 1489, 'NumberOfEvents' : 941.2 } ),
'/a/b/c/Dir1/someFile' : S_OK( { 'NumberOfEvents' : '14' } ),
'/a/b/c/Dir1/other_file' : S_OK( { 'NumberOfEvents' : '14' } ),
'/a/b/c/Dir1/dontforget_me' : S_OK( { 'NumberOfEvents' : '14' } ) }
directory_meta_dict = { '/a/b/c/Dir1' : S_OK( { 'Luminosity' : 84.1, 'evttype' : 'testEvt' } ),
'/unique/dir' : S_OK( { 'NumberOfEvents' : 814, 'Luminosity' : None } ),
'/other' : S_OK( { 'NumberOfEvents' : None, 'Luminosity' : None } ),
'/one/file' : S_OK( {} ) }
fcMock = Mock()
fcMock.getDirectoryUserMetadata = Mock(side_effect=lambda path: directory_meta_dict[path])
fcMock.getFileUserMetadata = Mock(side_effect=lambda filename : file_meta_dict[filename] )
with patch( "%s.FileCatalogClient" % MODULE_NAME, new=Mock(return_value=fcMock)):
assertDiracSucceedsWith_equals( getNumberOfEvents( [ '/a/b/c/Dir1/someFile', '', '/a/b/c/Dir1/other_file',
'/unique/dir/file3', '', '', '',
'/a/b/c/Dir1/dontforget_me', '/one/file/myfile',
'/other/myfile2' ] ),
{ 'AdditionalMeta': { 'evttype' : 'testEvt' }, 'EvtType' : '',
'lumi' : 1790.5, 'nbevts' : 1811 }, self )
def test_setfilestatus_useexistingfilereport( self ):
report_mock = Mock()
report_mock.setFileStatus.return_value = S_OK('my_report_test_returnval')
self.moba.workflow_commons['FileReport'] = report_mock
assertDiracSucceedsWith_equals( self.moba.setFileStatus( 'production', 'lfn', 'status' ),
'my_report_test_returnval', self )
assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self )
def test_limited_add_up_to_maximum( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
util_mock = Mock()
util_mock.groupByReplicas.return_value = S_OK(
[ ( 'testSE', [ 'mylfn1', 'testLFN2' ] ), ( 'testSE', [ 'other_test_lfn' ] ),
( 'testSE', [ 'mylfn' ] ), ( 'testSE', [ 'MyLFN' ] ) ] )
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK(
[ ( { 'Status' : 'Processed' }, 3 ), ( { 'Status' : 'junk' }, 6 ), ( { 'Status' : 'Ignore_me' }, 8 ),
( { 'Status' : 'Assigned' }, 1 ) ] )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'Limited', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = 'Processed'
self.tfp.params[ 'MaxNumberOfTasks' ] = 6
self.tfp.params[ 'TransformationID' ] = 78456
self.tfp.util = util_mock
test_data = [
( '/my/file/here_abc', [ 'myTestSE' ] ),
( '/my/file/other_file.txt', [ 'secretSE', 'StorageSE', 'TestSE' ] ),
( '/my/file/newfile.pdf', [ 'TestSE' ] ),
( '/my/file/a', [] ), ( '/dir/somefile', [ '' ] ) ]
self.tfp.setInputData( test_data )
assertDiracSucceedsWith_equals( self.tfp.run(), [
( '', [ 'mylfn1', 'testLFN2' ] ), ( '', [ 'other_test_lfn' ] ) ], self )
util_mock.groupByReplicas.assert_called_once_with( test_data, 'Processed' )
trans_mock.getCounters.assert_called_once_with( 'TransformationFiles', [ 'Status' ],
{ 'TransformationID' : 78456 } )
def test_writeproclist(self):
expected_write = 'Processes\n{\n mytestprocess123\n {\n TarBallCSPath = /test/cs/path/bal.tarr\n Detail = TestNoDetails\n Generator = mytestGen21\n Model = testmodel3001\n Restrictions = \n InFile = my/file.in\n CrossSection = 0\n }\n}\n'
self.prol._addEntry(
'mytestprocess123', {
'TarBallCSPath': '/test/cs/path/bal.tarr',
'Detail': 'TestNoDetails',
'Generator': 'mytestGen21',
'Model': 'testmodel3001',
'Restrictions': '',
'InFile': 'my/file.in'
})
exists_dict = {
'/temp/dir': False,
'/temp/dir/mytempfile.txt': True,
'/my/folder/testpath.xml': True
}
fhandle_mock = Mock()
with patch('tempfile.mkstemp', new=Mock(return_value=('handle', '/temp/dir/mytempfile.txt'))), \
patch('__builtin__.file', new=Mock(return_value=fhandle_mock)) as file_mock, \
patch('os.makedirs') as mkdir_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('shutil.move') as move_mock, \
patch('os.close') as close_mock:
assertDiracSucceedsWith_equals(
self.prol.writeProcessList('/my/folder/testpath.xml'),
'/my/folder/testpath.xml', self)
mkdir_mock.assert_called_once_with('/temp/dir')
file_mock.assert_called_once_with('/temp/dir/mytempfile.txt', 'w')
fhandle_mock.write.assert_called_once_with(expected_write)
close_mock.assert_called_once_with('handle')
move_mock.assert_called_once_with('/temp/dir/mytempfile.txt',
'/my/folder/testpath.xml')
def test_sliced_empty( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
self.tfp = TransformationPlugin( 'Sliced', dataman_mock, trans_mock )
self.tfp.setInputData( {} )
assertDiracSucceedsWith_equals( self.tfp.run(), [], self )
def test_applicationSpecificInputs_with_setters(self):
tmp_dict = {
'Detector': 'othertestdetectorv3000',
'Energy': '10000GeV',
'BXOverlay': '651',
'ggtohadint': 9.5,
'ProdID': 429875,
'NbSigEvtsPerJob': 94,
'BkgEvtType': 'bgoijaf',
'STEP_NUMBER': 1,
}
self.over.step_commons = tmp_dict
self.over.InputData = ['abc']
self.over.NumberOfEvents = 15
with patch('%s.Operations.getValue' % MODULE_NAME,
new=Mock(return_value=2)):
result = self.over.applicationSpecificInputs()
assertDiracSucceedsWith_equals(result,
TestOverlayUnittests.GOOD_EXIT,
self)
assertEqualsImproved(
(self.over.detectormodel, self.over.energytouse,
self.over.BXOverlay, self.over.ggtohadint, self.over.prodid,
self.over.NbSigEvtsPerJob, self.over.BkgEvtType),
('othertestdetectorv3000', '10000GeV', '651', 9.5, 429875, 94,
'bgoijaf'), self)
assertEqualsImproved(self.over.nbsigeventsperfile, 15, self)
assertEqualsImproved(self.over.nbinputsigfile, 1, self)
def test_broadcast_processed_checkchunking( self ):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
import copy
files = {}
for i in xrange(0, 599):
files[ ( '/file/dir/input%s.txt' % i ) ] = [ 'child' ]
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK( { 'Successful' : files, 'Failed' : {} } )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'BroadcastProcessed', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = ''
self.tfp.util = util_mock
self.tfp.setInputData( copy.deepcopy( files ) )
del self.tfp.data[ '/file/dir/input542.txt' ]
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME, new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals( self.tfp.run(), 98124, self )
expected = {}
for i in xrange(0, 599):
if i != 542:
expected[ ( '/file/dir/input%s.txt' % i ) ] = [ 'child' ]
assertListContentEquals( self.tfp.data, expected, self )
assertEqualsImproved( len( util_mock.fc.getFileDescendents.mock_calls ), 3, self )
def test_getfcfiles(self):
ops_dict = {
'/Overlay/clic_cdr/200TeV/testdetectorv2000/myTestBkgEvt/ProdID':
98421,
'/Overlay/clic_cdr/200TeV/testdetectorv2000/myTestBkgEvt/NbEvts':
482,
'/Overlay/clic_cdr/200TeV/testdetectorv2000/myTestBkgEvt/EvtType':
'someTestEventType'
}
self.over.energy = 123
self.over.useEnergyForFileLookup = True
self.over.BkgEvtType = 'myTestBkgEvt'
self.over.machine = 'clic_cdr'
ops_mock = Mock()
ops_mock.getValue.side_effect = lambda key, default: ops_dict[key]
self.over.ops = ops_mock
fcc_mock = Mock()
fcc_mock.findFilesByMetadata.return_value = S_OK(9824)
self.over.fcc = fcc_mock
result = self.over._OverlayInput__getFilesFromFC()
assertDiracSucceedsWith_equals(result, 9824, self)
fcc_mock.findFilesByMetadata.assert_called_once_with({
'Energy': '123',
'EvtType': 'someTestEventType',
'ProdID': 98421,
'Datatype': 'SIM',
'DetectorModel': 'testdetectorv2000',
'Machine': 'clic'
})
def test_applicationspecificinputs_isprod_false( self ):
self.shs.OutputFile = 'something'
self.shs.InputFile = 'input.stdhep'
self.shs.InputData = []
assertDiracSucceedsWith_equals( self.shs.applicationSpecificInputs(), 'Parameters resolved', self )
assertEqualsImproved( self.shs.OutputFile, 'something', self )
assertEqualsImproved( self.shs.listoutput, {}, self )
def test_getjobsatsite_no_site_found(self):
con_mock = Mock()
with patch('%s.OverlayDB._query' % MODULE_NAME,
new=Mock(return_value=S_OK([]))):
assertDiracSucceedsWith_equals(
self.odb.getJobsAtSite('nonexistent_site_testme', con_mock), 0,
self)
def test_applicationSpecificInputs(self):
# Compare to this. Get outside of patch, else constructor fails
reference = OverlayInput()
with patch('%s.Operations.getValue' % MODULE_NAME,
new=Mock(return_value=2)):
result = self.over.applicationSpecificInputs()
assertDiracSucceedsWith_equals(result,
TestOverlayUnittests.GOOD_EXIT,
self)
# Assert nothing has been changed, except the values in setUp (and DataManager/FileCatalogClient since theyre created anew for every object)
assertEqualsImproved(
(self.over.enable, self.over.STEP_NUMBER, self.over.log,
self.over.applicationName, self.over.curdir,
self.over.applicationLog, self.over.printoutflag,
self.over.prodid, self.over.detector, self.over.energy,
self.over.nbofeventsperfile, self.over.lfns,
self.over.nbfilestoget, self.over.BkgEvtType,
self.over.ggtohadint, self.over.nbsigeventsperfile,
self.over.nbinputsigfile, self.over.site,
self.over.useEnergyForFileLookup, self.over.machine,
self.over.pathToOverlayFiles),
(reference.enable, reference.STEP_NUMBER, reference.log,
reference.applicationName, reference.curdir,
reference.applicationLog, reference.printoutflag,
reference.prodid, reference.detector, reference.energy,
reference.nbofeventsperfile, reference.lfns,
reference.nbfilestoget, reference.BkgEvtType,
reference.ggtohadint, reference.nbsigeventsperfile,
reference.nbinputsigfile, reference.site,
reference.useEnergyForFileLookup, reference.machine,
reference.pathToOverlayFiles), self)
if self.over.fcc is None:
self.fail('FCC not initialized')
if self.over.datMan is None:
self.fail('DataManager not initialized')
def test_getdestinationse_noassoc_country(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
option_dict = {
'/Resources/Countries/country/AssignedTo':
S_OK('MyTestCountry.Assigned'),
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs':
S_OK('something')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
[],
'/Resources/StorageElementGroups/myOutputSE':
['some_elements', 'ignore_me'],
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE':
['myassociated_alias_se'],
'/Resources/Countries/country/AssociatedSEs/myOutputSE':
''
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getOption.side_effect = lambda path: option_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country', 'Local')
assertDiracSucceedsWith_equals(result, ['myassociated_alias_se'],
self)
def test_runit_complete( self ):
exists_dict = { '/my/steer/dir/SteerFile.testme' : False, 'SteerFile.testme' : False,
'myAppTestLog.log' : True }
open_mock = Mock()
open_mock.readlines.side_effect = [ 'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n' ]
open_mock.__enter__.return_value = [ 'Events kept 12', 'Events passing cuts 2984', 'Events total 2996' ]
self.shc.applicationLog = 'myAppTestLog.log'
self.shc.OutputFile = 'something'
self.shc.SteeringFile = '/my/dir/SteerFile.testme'
self.shc.workflow_commons[ 'Luminosity' ] = 13.0
( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion ) = ( 'testPlatform',
'AppTestName', 'vT' )
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 0, )))), \
patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()), \
patch('%s.getSteeringFileDirName' % MODULE_NAME, new=Mock(return_value=S_OK('/my/steer/dir'))):
assertDiracSucceedsWith_equals( self.shc.runIt(), 'AppTestName vT Successful', self )
remove_mock.assert_called_once_with( 'myAppTestLog.log' )
assertEqualsImproved( self.shc.workflow_commons[ 'Luminosity' ], 13.0 * ( 1. * 12 / 2984 ), self )
assertEqualsImproved( self.shc.workflow_commons[ 'Info' ],
{ 'stdhepcut' : { 'Reduction' : 1. * 12 / 2984,
'CutEfficiency' : 1. * 2984 / 2996 } }, self )
def test_runit_othercases( self ):
exists_dict = { 'SteerFile.testme' : True, 'myAppTestLog.log' : False }
open_mock = Mock()
open_mock.readlines.side_effect = [ 'line1\n', 'line2\n', 'newline\n', '\n', 'ok\n' ]
open_mock.__enter__.return_value = [ 'Events kept 23', 'Events passing cuts 14', 'Events total 37' ]
self.shc.scriptName = 'my_test_script.sh'
self.shc.ignoreapperrors = True
self.shc.OutputFile = 'something'
self.shc.applicationLog = 'myAppTestLog.log'
self.shc.SteeringFile = '/my/dir/SteerFile.testme'
self.shc.workflow_commons[ 'Info' ] = { 'some_entry' : 'rememberMe' }
self.shc.MaxNbEvts = 100
( self.shc.platform, self.shc.applicationName, self.shc.applicationVersion ) = ( 'testPlatform',
'AppTestName', 'vT' )
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('SoftDir'))), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)) as open_mock, \
patch('%s.os.remove' % MODULE_NAME, new=Mock()), \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 0, )))), \
patch('%s.StdHepCut.prepareScript' % MODULE_NAME, new=Mock()):
assertDiracSucceedsWith_equals( self.shc.runIt(), 'AppTestName vT Successful', self )
assertEqualsImproved( self.shc.workflow_commons[ 'Info'],
{ 'some_entry' : 'rememberMe', 'stdhepcut' :
{ 'Reduction' : 1. * 23 / 14, 'CutEfficiency' : 1. * 14 / 37 } }, self )
assert 'Luminosity' not in self.shc.workflow_commons
def test_getfcfiles_othercase(self):
ops_dict = {
'/Overlay/ilc_dbd/TestILCDetectorv1/200TeV/otherTestEvt/ProdID':
139,
'/Overlay/ilc_dbd/200TeV/TestILCDetectorv1/otherTestEvt/NbEvts':
2145,
'/Overlay/ilc_dbd/200TeV/TestILCDetectorv1/otherTestEvt/EvtType':
'ilc_evt_testme'
}
self.over.energy = 0
self.over.useEnergyForFileLookup = False
self.over.detectormodel = ''
self.over.BkgEvtType = 'otherTestEvt'
self.over.machine = 'ilc_dbd'
self.over.detector = 'TestILCDetectorv1'
self.over.prodid = 82492
ops_mock = Mock()
ops_mock.getValue.side_effect = lambda key, default: ops_dict[key]
self.over.ops = ops_mock
fcc_mock = Mock()
fcc_mock.findFilesByMetadata.return_value = S_OK(2948)
self.over.fcc = fcc_mock
result = self.over._OverlayInput__getFilesFromFC()
assertDiracSucceedsWith_equals(result, 2948, self)
fcc_mock.findFilesByMetadata.assert_called_once_with({
'EvtType': 'ilc_evt_testme',
'ProdID': 82492,
'Datatype': 'SIM',
'Machine': 'ilc'
})
def test_broadcast_processed_selectsome( self ):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK(
{ 'Successful' :
{ '/some/deep/dir/structure/file1.stdio' : [ 'child' ], '/file/dir/input1.txt' : [],
'/dir/file.xml' : [ 'child' ], '/nodir.txt' : [ 'child' ], 'input_file.txt' : [ 'child' ] },
'Failed' : { '/my/dir/pphys.ics' : [] } } )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'BroadcastProcessed', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = ''
self.tfp.util = util_mock
self.tfp.setInputData( { '/some/deep/dir/structure/file1.stdio' : [ 'myTestSE1' ],
'/file/dir/input1.txt' : [ 'other_SE' ],
'/dir/file.xml' : [ 'myTestSE1', 'SecondTestSE' ],
'/nodir.txt' : [ 'secretSE' ],
'input_file.txt' : [ 'SE_to_test' ],
'/my/dir/pphys.ics' : [ '' ] } )
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME, new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals( self.tfp.run(), 98124, self )
assertListContentEquals( self.tfp.data,
[ '/some/deep/dir/structure/file1.stdio', '/dir/file.xml', '/nodir.txt',
'input_file.txt' ], self )
util_mock.fc.getFileDescendents.assert_called_once()
def test_limited_add_all(self):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
new_tasks = [('otherSE', ['unique_lfn', 'testme_lfn']),
('testSE', ['LFN_to_add']),
('testSE', ['LFN:/mydir/subdir/file.stdhep']),
('testSE', ['testSE/storage/LFN.txt'])]
util_mock = Mock()
util_mock.groupByReplicas.return_value = S_OK(new_tasks)
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK([({
'Status': 'Processed'
}, 3), ({
'Status': 'junk'
}, 6), ({
'Status': 'Ignore_me'
}, 8), ({
'Status': 'Assigned'
}, 1)])
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('Limited', dataman_mock, trans_mock)
self.tfp.params['Status'] = 'Processed'
self.tfp.params['MaxNumberOfTasks'] = 12
self.tfp.params['TransformationID'] = 78456
self.tfp.util = util_mock
self.tfp.setInputData([('/my/file/here_abc', ['myTestSE']),
('/my/file/other_file.txt',
['secretSE', 'StorageSE', 'TestSE']),
('/my/file/newfile.pdf', ['TestSE']),
('/my/file/a', []), ('/dir/somefile', [''])])
assertDiracSucceedsWith_equals(self.tfp.run(),
[('', tup) for (_se, tup) in new_tasks],
self)
def test_setfilestatus( self ):
report_mock = Mock()
report_mock.setFileStatus.return_value = S_OK('other_my_report_test')
with patch('%s.FileReport' % MODULE_NAME, new=Mock(return_value=report_mock)):
assertDiracSucceedsWith_equals( self.moba.setFileStatus( 'production', 'lfn', 'status' ),
'other_my_report_test', self )
assertEqualsImproved( self.moba.workflow_commons['FileReport'], report_mock, self )
def test_resolveinput_productionjob4(self):
self.marAna.workflow_commons["IS_PROD"] = True
self.marAna.OutputFile = ''
prodId = "123"
self.marAna.workflow_commons["PRODUCTION_ID"] = prodId
jobId = "456"
self.marAna.workflow_commons["JOB_ID"] = jobId
self.marAna.workflow_commons.pop(
'ProductionOutputData',
None) #default value needed, else maybe KeyError
self.marAna.InputFile = 'in3.slcio'
self.marAna.outputREC = 'out1.stdhep'
self.marAna.outputDST = 'test2.stdhep'
assertDiracSucceedsWith_equals(self.marAna.applicationSpecificInputs(),
"Parameters resolved", self)
files = [
self.marAna.outputREC, self.marAna.outputDST,
self.marAna.InputFile[0]
]
for filename in files:
# TODO: check for file extension, differentiate 'test'/'test2' in filename...
self.assertInMultiple([prodId, jobId], filename)
self.assertInMultiple(['in3', '.slcio'], self.marAna.InputFile[0])
self.assertInMultiple(['out1', '.stdhep'], self.marAna.outputREC)
self.assertInMultiple(['test2', '.stdhep'], self.marAna.outputDST)
def test_applicationSpecificInputs(self):
assertDiracSucceedsWith_equals(self.spa.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved(self.spa.pandorasettings, 'PandoraSettings.xml',
self)
assertEqualsImproved(self.spa.InputFile, [], self)
assertEqualsImproved(self.spa.InputData, [], self)
def test_getenvscript_other_prefixpath( self ):
def replace_abspath( path ):
""" Mock implementation of os.path.abspath """
if path == 'SLICPandora.sh':
return '/abs/test/path/SLICPandora.sh'
else:
sys.exit()
exists_dict = { 'PandoraFrontend' : False, '/my/dir/test/me/Executable/PandoraFrontend' : True }
with patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('/my/dir/test/me'))) as getsoft_mock, \
patch('%s.removeLibc' % MODULE_NAME) as removelib_mock, \
patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/new/ldpath')) as getlib_mock, \
patch('%s.getNewPATH' % MODULE_NAME, new=Mock(return_value='/new/test/path')) as getpath_mock, \
patch('%s.open' % MODULE_NAME, mock_open()) as open_mock:
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock, \
patch('%s.os.chmod' % MODULE_NAME) as chmod_mock, \
patch('%s.os.path.abspath' % MODULE_NAME, new=Mock(side_effect=replace_abspath)) as abspath_mock:
result = self.spa.getEnvScript( 'mytestsysconfig', 'SLIC Pandora', 'V2' )
chmod_mock.assert_called_once_with('SLICPandora.sh', 0o755)
assertMockCalls( exists_mock, [ 'PandoraFrontend', '/my/dir/test/me/Executable/PandoraFrontend' ], self )
abspath_mock.assert_called_once_with( 'SLICPandora.sh' )
assertDiracSucceedsWith_equals( result, '/abs/test/path/SLICPandora.sh', self )
getsoft_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' )
removelib_mock.assert_called_once_with( '/my/dir/test/me/LDLibs' )
getlib_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' )
getpath_mock.assert_called_once_with( 'mytestsysconfig', 'SLIC Pandora', 'V2' )
open_mock.assert_any_call( 'SLICPandora.sh', 'w' )
open_mock = open_mock()
assertMockCalls( open_mock.write, [
'#!/bin/sh \n', '############################################################\n',
'# Dynamically generated script to get the SLICPandora env. #\n',
'############################################################\n',
"declare -x PATH=/new/test/path:$PATH\n", 'declare -x ROOTSYS=/my/dir/test/me/ROOT\n',
'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:/my/dir/test/me/LDLibs:/new/ldpath\n',
'declare -x PANDORASETTINGSDIR=/my/dir/test/me/Settings\n', "declare -x PATH=/my/dir/test/me/Executable:$PATH\n" ], self )
def test_execute_noinput(self):
self.shs.platform = 'TestPlatV1'
self.shs.InputFile = ''
with patch('%s.StdHepSplit.resolveInputVariables' % MODULE_NAME,
new=Mock(return_value=S_OK())):
assertDiracSucceedsWith_equals(self.shs.execute(),
'No files found to process', self)
def test_application_specific_inputs_jobid_zero( self ): self.ujf.jobID = 0 self.ujf.enable = True result = self.ujf.applicationSpecificInputs() assertDiracSucceedsWith_equals( result, 'Parameters resolved', self ) assertEqualsImproved( self.ujf.jobID, 0, self ) assertEqualsImproved( self.ujf.enable, False, self )
def test_sliced_empty(self):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
self.tfp = TransformationPlugin('Sliced', dataman_mock, trans_mock)
self.tfp.setInputData({})
assertDiracSucceedsWith_equals(self.tfp.run(), [], self)
def test_getmainfields(self):
result = self.whop.getMainFields()
assertDiracSucceedsWith_equals(result, [
'process_input', 'integration_input', 'simulation_input',
'diagnostics_input', 'parameter_input', 'beam_input_1',
'beam_input_2'
], self)
def test_applicationspecificinputs_isprod_false(self):
self.shs.OutputFile = 'something'
self.shs.InputFile = 'input.stdhep'
self.shs.InputData = []
assertDiracSucceedsWith_equals(self.shs.applicationSpecificInputs(),
'Parameters resolved', self)
assertEqualsImproved(self.shs.OutputFile, 'something', self)
assertEqualsImproved(self.shs.listoutput, {}, self)
def test_runit_statusbad(self):
self.shc.workflowStatus['OK'] = True
self.shc.workflowStatus['OK'] = False
self.shc.applicationName = 'testApp'
assertDiracSucceedsWith_equals(
self.shc.runIt(),
'testApp should not proceed as previous step did not end properly',
self)
def test_execute_disabled( self ):
self.ulf.enable = False
with patch.object(self.ulf, 'resolveInputVariables', new=Mock(return_value=S_OK())), \
patch.object(self.ulf, '_determineRelevantFiles', new=Mock(return_value=S_OK([ 'first_selected_file', 'other_files' ]))), \
patch.object(self.ulf, '_populateLogDirectory', new=Mock(return_value=S_OK())), \
patch('%s.UploadLogFile.logWorkingDirectory' % MODULE_NAME, new=Mock()):
assertDiracSucceedsWith_equals( self.ulf.execute(), 'Module is disabled by control flag', self )
self.assertFalse(self.log_mock.error.called)
def test_checkcvmfs_exists(self):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(side_effect=['testcvmfspath', 'testenvscript'])), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=True)):
result = checkCVMFS('mytestplatform',
('appnametesttest123', 'appversionv77testp2'))
assertDiracSucceedsWith_equals(result,
('testcvmfspath', 'testenvscript'),
self)
def test_getsites(self):
con_mock = Mock()
with patch('%s.OverlayDB._getConnection' % MODULE_NAME, new=Mock(return_value=S_OK(con_mock))), \
patch('%s.OverlayDB._query' % MODULE_NAME, new=Mock(return_value=S_OK([ ('some_row',), ('other_row', )]))) as query_mock:
assertDiracSucceedsWith_equals(self.odb.getSites(),
['some_row', 'other_row'], self)
query_mock.assert_called_once_with('SELECT Site From OverlayData;',
con_mock)
def test_resolveinput_productionjob3(self):
self.marAna.workflow_commons["IS_PROD"] = True
self.marAna.OutputFile = "c.c"
self.marAna.InputFile = []
inputlist = ["a.slcio", "b.slcio", "c.exe"]
self.marAna.InputData = inputlist
assertDiracSucceedsWith_equals(self.marAna.applicationSpecificInputs(),
"Parameters resolved", self)
self.assertEquals([inputlist[0], inputlist[1]], self.marAna.InputFile)
def test_resolveinput_productionjob2( self ):
self.marAna.workflow_commons[ 'ParametricInputSandbox' ] = []
self.marAna.workflow_commons[ "IS_PROD" ] = False
self.marAna.workflow_commons[ "PRODUCTION_ID" ] = "123"
self.marAna.workflow_commons[ "JOB_ID" ] = 456
assertDiracSucceedsWith_equals( self.marAna.applicationSpecificInputs(),
"Parameters resolved", self )
self.assertEquals( self.marAna.InputFile, [] )
self.assertEquals( self.marAna.InputData, [] )
def test_resolveinput_productionjob3( self ):
self.marAna.workflow_commons[ "IS_PROD" ] = True
self.marAna.OutputFile = "c.c"
self.marAna.InputFile = []
inputlist = [ "a.slcio", "b.slcio", "c.exe" ]
self.marAna.InputData = inputlist
assertDiracSucceedsWith_equals( self.marAna.applicationSpecificInputs(),
"Parameters resolved", self )
self.assertEquals([inputlist[0], inputlist[1]], self.marAna.InputFile )
def test_canrun_toomanyjobs(self):
con_mock = Mock()
with patch('%s.OverlayDB._query' % MODULE_NAME,
new=Mock(return_value=S_OK([[2]]))) as query_mock:
assertDiracSucceedsWith_equals(
self.odb.canRun('testSite1', con_mock), False, self)
query_mock.assert_called_once_with(
"SELECT NumberOfJobs FROM OverlayData WHERE Site='testSite1';",
con_mock)
def test_getjobsatsite(self):
con_mock = Mock()
with patch('%s.OverlayDB._query' % MODULE_NAME,
new=Mock(return_value=S_OK([[123]]))) as query_mock:
assertDiracSucceedsWith_equals(
self.odb.getJobsAtSite('myTest_Site1', con_mock), 123, self)
query_mock.assert_called_once_with(
"SELECT NumberOfJobs FROM OverlayData WHERE Site='myTest_Site1';",
con_mock)
def test_submit_createnew_dirac_instance( self ):
ilc_mock = Mock()
ilc_mock().submitJob.return_value = S_OK('test_submission_successful')
self.ujo.proxyinfo = S_OK( { 'group' : 'ilc_user' } )
with patch('%s.UserJob._addToWorkflow' % MODULE_NAME, new=Mock(return_value=S_OK())), \
patch('%s.DiracILC' % MODULE_NAME, new=ilc_mock):
assertDiracSucceedsWith_equals( self.ujo.submit(), 'test_submission_successful', self )
ilc_mock().submitJob.assert_called_once_with(self.ujo, 'wms')
assert self.ujo.oktosubmit
def test_submit_existing_dirac_instance(self):
"""Test submit with dirac instance."""
ilc_mock = Mock()
ilc_mock.submitJob.return_value = S_OK('test_submission_successful')
self.ujo.proxyinfo = S_OK({'group': 'ilc_user'})
with patch('%s.UserJob._addToWorkflow' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracSucceedsWith_equals(self.ujo.submit(diracinstance=ilc_mock), 'test_submission_successful', self)
ilc_mock.submitJob.assert_called_once_with(self.ujo, 'wms')
assert self.ujo.oktosubmit
def test_overlayinput_getEosFile_lfn_success( self ):
""" test success when getting an lfn to copy from eos """
testLFN = "/lfn/to/overlay/overlayFile.slcio"
res = self.over.getEOSFile( testLFN )
print(res)
print("self result", self.over.result)
assertDiracSucceedsWith_equals( res, os.path.basename( testLFN ), self )
with open("overlayinput.sh") as overscript:
self.assertIn( "xrdcp -s root://eospublic.cern.ch//eos/experiment/clicdp/grid%s" % testLFN , overscript.read() )
def test_overlayinput_getEosFile_fullpath_success( self ):
""" test that we don't predent if we get a fullpath for eos, however that might happen"""
testLFN = "/eos/experiment/clicdp/grid/lfn/to/overlay/overlayFile.slcio"
res = self.over.getEOSFile( testLFN )
print(res)
print("self result", self.over.result)
assertDiracSucceedsWith_equals( res, os.path.basename( testLFN ), self )
with open("overlayinput.sh") as overscript:
self.assertIn( "xrdcp -s root://eospublic.cern.ch/%s" % testLFN , overscript.read() )
def test_getsteerfiledir_cvmfs_success(self):
cvmfs_mock = Mock(return_value=S_OK(
['mytestListEntry#1', 'other_entry_dontusethis', '', '18319']))
with patch('%s.checkCVMFS' % MODULE_NAME, new=cvmfs_mock):
assertDiracSucceedsWith_equals(
getSteeringFileDir('myTestPlatform_1', 'v123Test'),
'mytestListEntry#1', self)
cvmfs_mock.assert_called_once_with('myTestPlatform_1',
['steeringfiles', 'v123Test'])
def test_broadcast_processed_flush( self ):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
self.tfp = TransformationPlugin( 'BroadcastProcessed', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = 'Flush'
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME, new=Mock(return_value=S_OK(8124))):
assertDiracSucceedsWith_equals( self.tfp.run(), 8124, self )
def test_applicationSpecificInputs_1( self ):
self.spa.pandorasettings = 'something'
self.spa.InputData = [ 'myfile.slcio', 'ignorethisfile', 'testPart1.slcio',
'myarchive.slcio.tar.gz' ]
assertDiracSucceedsWith_equals( self.spa.applicationSpecificInputs(),
'Parameters resolved', self )
assertEqualsImproved( self.spa.InputFile,
[ 'myfile.slcio', 'testPart1.slcio',
'myarchive.slcio.tar.gz' ], self )
def test_resolveinput_productionjob2(self):
self.marAna.workflow_commons['ParametricInputSandbox'] = []
self.marAna.workflow_commons["IS_PROD"] = False
self.marAna.workflow_commons["PRODUCTION_ID"] = "123"
self.marAna.workflow_commons["JOB_ID"] = 456
assertDiracSucceedsWith_equals(self.marAna.applicationSpecificInputs(),
"Parameters resolved", self)
self.assertEquals(self.marAna.InputFile, [])
self.assertEquals(self.marAna.InputData, [])
def test_getsoftwarefolder_uselocal( self ):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[ False, True, False ])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
exists_mock.assert_called_with( '/mylocalarea/test/myapparchivev2.test' )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertEqualsImproved( len(exists_mock.mock_calls), 2, self ) #One exists call in checkCVMFS
assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self )
def test_setfilestatus(self):
report_mock = Mock()
report_mock.setFileStatus.return_value = S_OK('other_my_report_test')
with patch('%s.FileReport' % MODULE_NAME,
new=Mock(return_value=report_mock)):
assertDiracSucceedsWith_equals(
self.moba.setFileStatus('production', 'lfn', 'status'),
'other_my_report_test', self)
assertEqualsImproved(self.moba.workflow_commons['FileReport'],
report_mock, self)
def test_sendstoredstatinfo( self ):
self.moba.jobID = 24986
report_mock = Mock()
report_mock.sendStoredStatusInfo.return_value = S_OK('mytest_success!!!')
self.moba.workflow_commons['JobReport'] = report_mock
assertDiracSucceedsWith_equals( self.moba.sendStoredStatusInfo(), 'mytest_success!!!', self )
self.assertFalse(self.log_mock.called)
self.assertFalse(self.log_mock.warn.called)
self.assertFalse(self.log_mock.error.called)
report_mock.sendStoredStatusInfo.assert_called_once_with()
def test_overlayinput_getEosFile_fullpath_success(self):
""" test that we don't predent if we get a fullpath for eos, however that might happen"""
testLFN = "/eos/experiment/clicdp/grid/lfn/to/overlay/overlayFile.slcio"
res = self.over.getEOSFile(testLFN)
print res
print "self result", self.over.result
assertDiracSucceedsWith_equals(res, os.path.basename(testLFN), self)
with open("overlayinput.sh") as overscript:
self.assertIn("xrdcp -s root://eospublic.cern.ch/%s" % testLFN,
overscript.read())
def test_getoptionsforfield( self ):
assertDiracSucceedsWith_equals( self.whop.getOptionsForField( 'process_input' ),
[ 'process_id', 'cm_frame', 'sqrts', 'luminosity', 'polarized_beams',
'structured_beams', 'beam_recoil', 'recoil_conserve_momentum',
'filename','directory', 'input_file', 'input_slha_format' ], self )
assertDiracSucceedsWith_equals( self.whop.getOptionsForField( 'parameter_input' ),
['GF', 'mZ', 'mW', 'mH', 'alphas', 'me', 'mmu', 'mtau', 'ms', 'mc', 'mb',
'mtop', 'wtop', 'wZ', 'wW', 'wH', 'vckm11', 'vckm12', 'vckm13',
'vckm21', 'vckm22', 'vckm23', 'vckm31', 'vckm32', 'vckm33', 'khgaz',
'khgaga', 'khgg'], self )
def test_getsoftwarefolder_uselocal( self ):
exists_dict = { 'myapparchivev2.test.tar.gz' : False, '/mylocalarea/test/myapparchivev2.test' : True }
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
assertMockCalls( exists_mock, [ '/mylocalarea/test/myapparchivev2.test', 'myapparchivev2.test.tar.gz' ],
self )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self )
def test_resolveinput_productionjob1( self ):
self.marAna.workflow_commons[ "IS_PROD" ] = True
outputfile1 = "/dir/a_REC_.a"
outputfile2 = "/otherdir/b_DST_.b"
inputfile = "/inputdir/input_SIM_.i"
self.marAna.workflow_commons[ "ProductionOutputData" ] = ";".join([outputfile1, outputfile2, inputfile])
assertDiracSucceedsWith_equals( self.marAna.applicationSpecificInputs(),
"Parameters resolved", self )
assertEqualsImproved( ( self.marAna.outputREC, self.marAna.outputDST,
self.marAna.InputFile ),
( "a_REC_.a", "b_DST_.b", [ "input_SIM_.i" ] ), self )
def test_applicationspecificinputs_2( self ):
self.shs.OutputFile = 'something'
self.shs.workflow_commons.update( { 'IS_PROD' : True, 'PRODUCTION_ID' : '1248', 'JOB_ID' : 173 } )
self.shs.InputFile = 'input.stdhep'
self.shs.InputData = []
self.shs.listoutput = 'my_test_list'
with patch('%s.getProdFilename' % MODULE_NAME, new=Mock(return_value='myoutputfile.stdio')) as getfile_mock:
assertDiracSucceedsWith_equals( self.shs.applicationSpecificInputs(), 'Parameters resolved', self )
assertEqualsImproved( self.shs.OutputFile, 'myoutputfile.stdio', self )
assertEqualsImproved( self.shs.listoutput, 'my_test_list', self )
getfile_mock.assert_called_once_with( 'something', 1248, 173 )
def test_applicationSpecificInputs_with_setters( self ):
tmp_dict = { 'Detector' : 'othertestdetectorv3000', 'Energy' : '10000GeV', 'BXOverlay' : '651', 'ggtohadint' : 9.5, 'ProdID' : 429875, 'NbSigEvtsPerJob' : 94, 'BkgEvtType' : 'bgoijaf' }
self.over.step_commons = tmp_dict
self.over.InputData = [ 'abc' ]
self.over.NumberOfEvents = 15
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value=2)):
result = self.over.applicationSpecificInputs()
assertDiracSucceedsWith_equals( result, TestOverlayUnittests.GOOD_EXIT, self )
assertEqualsImproved( ( self.over.detectormodel, self.over.energytouse, self.over.BXOverlay, self.over.ggtohadint, self.over.prodid, self.over.NbSigEvtsPerJob, self.over.BkgEvtType ), ( 'othertestdetectorv3000', '10000GeV', '651', 9.5, 429875, 94, 'bgoijaf' ), self )
assertEqualsImproved( self.over.nbsigeventsperfile, 15, self )
assertEqualsImproved( self.over.nbinputsigfile, 1, self )
def test_getdestinationse( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [],
'/Resources/StorageElementGroups/myOutputSE' : [ 'some_grp_member', 'GroupElement', 'StorEl', 'LocalElement' ] }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'StorEl', 'LocalElement' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' )
assertDiracSucceedsWith_equals( result, [ 'StorEl', 'LocalElement', 'some_grp_member', 'GroupElement' ], self )
def test_setjobparameter( self ):
self.moba.jobID = 24986
report_mock = Mock()
report_mock.setJobParameter.return_value = S_OK('mytest_success!!!')
self.moba.workflow_commons['JobReport'] = report_mock
assertDiracSucceedsWith_equals( self.moba.setJobParameter( 'mytestName', 135 ),
'mytest_success!!!', self )
self.assertFalse(self.log_mock.called)
self.assertFalse(self.log_mock.warn.called)
self.assertFalse(self.log_mock.error.called)
report_mock.setJobParameter.assert_called_once_with( 'mytestName', '135', True )
