def test_broadcast_processed_selectsome( self ):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK(
{ 'Successful' :
{ '/some/deep/dir/structure/file1.stdio' : [ 'child' ], '/file/dir/input1.txt' : [],
'/dir/file.xml' : [ 'child' ], '/nodir.txt' : [ 'child' ], 'input_file.txt' : [ 'child' ] },
'Failed' : { '/my/dir/pphys.ics' : [] } } )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'BroadcastProcessed', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = ''
self.tfp.util = util_mock
self.tfp.setInputData( { '/some/deep/dir/structure/file1.stdio' : [ 'myTestSE1' ],
'/file/dir/input1.txt' : [ 'other_SE' ],
'/dir/file.xml' : [ 'myTestSE1', 'SecondTestSE' ],
'/nodir.txt' : [ 'secretSE' ],
'input_file.txt' : [ 'SE_to_test' ],
'/my/dir/pphys.ics' : [ '' ] } )
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME, new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals( self.tfp.run(), 98124, self )
assertListContentEquals( self.tfp.data,
[ '/some/deep/dir/structure/file1.stdio', '/dir/file.xml', '/nodir.txt',
'input_file.txt' ], self )
util_mock.fc.getFileDescendents.assert_called_once()
def test_broadcast_processed_checkchunking( self ):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
import copy
files = {}
for i in xrange(0, 599):
files[ ( '/file/dir/input%s.txt' % i ) ] = [ 'child' ]
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK( { 'Successful' : files, 'Failed' : {} } )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'BroadcastProcessed', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = ''
self.tfp.util = util_mock
self.tfp.setInputData( copy.deepcopy( files ) )
del self.tfp.data[ '/file/dir/input542.txt' ]
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME, new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals( self.tfp.run(), 98124, self )
expected = {}
for i in xrange(0, 599):
if i != 542:
expected[ ( '/file/dir/input%s.txt' % i ) ] = [ 'child' ]
assertListContentEquals( self.tfp.data, expected, self )
assertEqualsImproved( len( util_mock.fc.getFileDescendents.mock_calls ), 3, self )
def test_updateproclist_and_getters( self ):
self.prol.cfg.createNewSection( 'Processes/myTestProcDeleteMe' )
self.prol.cfg.setOption( 'Processes/myTestProcDeleteMe/someRandomOption', True )
dict_1 = { 'CrossSection' : 'someCross' }
dict_1.update( STD_PROC_DICT )
dict_2 = { 'CrossSection' : 'some_other_val' }
dict_2.update( STD_PROC_DICT )
process_dict = {}
process_dict[ 'MytestProcess' ] = dict_1
process_dict[ 'myTestProcDeleteMe' ] = dict_2
result = self.prol.updateProcessList( process_dict )
assertDiracSucceeds( result, self )
conf = self.prol.cfg
self.assertFalse( conf.existsKey( 'Processes/myTestProcDeleteMe/someRandomOption' ) )
options = [ 'Processes/MytestProcess/CrossSection', 'Processes/myTestProcDeleteMe/CrossSection' ]
assertEqualsImproved( ( map( conf.getOption, options ) ), ( [ 'someCross', 'some_other_val' ] ), self )
assertEqualsImproved( ( self.prol.getCSPath( 'myTestProcDeleteMe' ),
self.prol.getInFile( 'myTestProcDeleteMe' ),
self.prol.existsProcess( 'myTestProcDeleteMe' ),
self.prol.existsProcess( '' ), self.prol.existsProcess( 'invalidProcess' ),
self.prol.existsProcess( 'myTestProcDeleteMeToo' ) ),
( '/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True), S_OK(False),
S_OK(False) ), self )
assertListContentEquals( self.prol.getProcesses(), [ 'myTestProcDeleteMe', 'MytestProcess' ], self )
all_processes_dict = self.prol.getProcessesDict()
assertEqualsImproved( len(all_processes_dict), 2, self )
assertEqualsImproved( ('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess' in all_processes_dict),
( True, True ), self )
self.prol.printProcesses()
def test_sliced_limited_add_all( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
util_mock = Mock()
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK(
[ ( { 'Status' : 'Processed' }, 4 ), ( { 'Status' : 'junk' }, 6 ), ( { 'Status' : 'Ignore_me' }, 8 ),
( { 'Status' : 'Assigned' }, 6 ) ] )
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin( 'SlicedLimited', dataman_mock, trans_mock )
self.tfp.params[ 'Status' ] = 'Processed'
self.tfp.params[ 'MaxNumberOfTasks' ] = 16
self.tfp.params[ 'TransformationID' ] = 78456
self.tfp.util = util_mock
self.tfp.setInputData( {
'/my/file/here_abc' : [ 'myTestSE' ],
'/my/file/other_file.txt' : [ 'secretSE', 'StorageSE', 'TestSE' ],
'/my/file/newfile.pdf' : [ 'TestSE' ],
'/my/file/a' : [], '/dir/somefile': [ '' ] } )
result = self.tfp.run()
assertDiracSucceeds( result, self )
expected = [ ( '', [ '/my/file/here_abc' ] ), ( '', [ '/my/file/other_file.txt' ] ),
( '', [ '/my/file/newfile.pdf' ] ), ( '', [ '/my/file/a' ] ), ( '', [ '/dir/somefile' ] ) ]
assertListContentEquals( result[ 'Value' ], expected, self )
trans_mock.getCounters.assert_called_once_with( 'TransformationFiles', [ 'Status' ],
{ 'TransformationID' : 78456 } )
def test_broadcast_processed_checkchunking(self):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
import copy
files = {}
for i in xrange(0, 599):
files[('/file/dir/input%s.txt' % i)] = ['child']
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK({
'Successful': files,
'Failed': {}
})
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('BroadcastProcessed', dataman_mock,
trans_mock)
self.tfp.params['Status'] = ''
self.tfp.util = util_mock
self.tfp.setInputData(copy.deepcopy(files))
del self.tfp.data['/file/dir/input542.txt']
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME,
new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals(self.tfp.run(), 98124, self)
expected = {}
for i in xrange(0, 599):
if i != 542:
expected[('/file/dir/input%s.txt' % i)] = ['child']
assertListContentEquals(self.tfp.data, expected, self)
assertEqualsImproved(len(util_mock.fc.getFileDescendents.mock_calls),
3, self)
def test_getfiles( self ):
walk_list = [ ( '/testRoot', 'nodirs', [ 'file1.txt', 'deletethislib.so', '', 'ignorefile.stdhep' ] ),
( '/otherRoot/mydir/', '', [ 'particle.stdhep', 'subfolder/otherlib.stdhep', 'library.dll' ] ) ]
with patch('%s.os.walk' % MODULE_NAME, new=Mock(return_value=walk_list)) as walk_mock:
res = getFiles( 'myTest_folder', '.stdhep' )
walk_mock.assert_called_once_with( 'myTest_folder', followlinks=True )
expected_files = [ '/testRoot/ignorefile.stdhep', '/otherRoot/mydir/particle.stdhep',
'/otherRoot/mydir/subfolder/otherlib.stdhep' ]
assertListContentEquals( res, expected_files, self )
def test_sliced( self ):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
self.tfp = TransformationPlugin( 'Sliced', dataman_mock, trans_mock )
self.tfp.setInputData( {
'/my/file/here_abc' : [ 'myTestSE' ],
'/my/file/other_file.txt' : [ 'secretSE', 'StorageSE', 'TestSE' ],
'/my/file/newfile.pdf' : [ 'TestSE' ],
'/my/file/a' : [], '/dir/somefile': [ '' ] } )
result = self.tfp.run()
assertDiracSucceeds( result, self )
assertListContentEquals( result[ 'Value' ], [
( '', [ '/my/file/here_abc' ] ), ( '', [ '/my/file/other_file.txt' ] ),
( '', [ '/my/file/newfile.pdf' ] ), ( '', [ '/my/file/a' ] ), ( '', [ '/dir/somefile' ] ) ], self )
def test_getfiles(self):
walk_list = [
('/testRoot', 'nodirs',
['file1.txt', 'deletethislib.so', '', 'ignorefile.stdhep']),
('/otherRoot/mydir/', '',
['particle.stdhep', 'subfolder/otherlib.stdhep', 'library.dll'])
]
with patch('%s.os.walk' % MODULE_NAME,
new=Mock(return_value=walk_list)) as walk_mock:
res = getFiles('myTest_folder', '.stdhep')
walk_mock.assert_called_once_with('myTest_folder',
followlinks=True)
expected_files = [
'/testRoot/ignorefile.stdhep',
'/otherRoot/mydir/particle.stdhep',
'/otherRoot/mydir/subfolder/otherlib.stdhep'
]
assertListContentEquals(res, expected_files, self)
def test_sliced(self):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
self.tfp = TransformationPlugin('Sliced', dataman_mock, trans_mock)
self.tfp.setInputData({
'/my/file/here_abc': ['myTestSE'],
'/my/file/other_file.txt': ['secretSE', 'StorageSE', 'TestSE'],
'/my/file/newfile.pdf': ['TestSE'],
'/my/file/a': [],
'/dir/somefile': ['']
})
result = self.tfp.run()
assertDiracSucceeds(result, self)
assertListContentEquals(result['Value'],
[('', ['/my/file/here_abc']),
('', ['/my/file/other_file.txt']),
('', ['/my/file/newfile.pdf']),
('', ['/my/file/a']),
('', ['/dir/somefile'])], self)
def test_updateproclist_and_getters(self):
self.prol.cfg.createNewSection('Processes/myTestProcDeleteMe')
self.prol.cfg.setOption(
'Processes/myTestProcDeleteMe/someRandomOption', True)
dict_1 = {'CrossSection': 'someCross'}
dict_1.update(STD_PROC_DICT)
dict_2 = {'CrossSection': 'some_other_val'}
dict_2.update(STD_PROC_DICT)
process_dict = {}
process_dict['MytestProcess'] = dict_1
process_dict['myTestProcDeleteMe'] = dict_2
result = self.prol.updateProcessList(process_dict)
assertDiracSucceeds(result, self)
conf = self.prol.cfg
self.assertFalse(
conf.existsKey('Processes/myTestProcDeleteMe/someRandomOption'))
options = [
'Processes/MytestProcess/CrossSection',
'Processes/myTestProcDeleteMe/CrossSection'
]
assertEqualsImproved((map(conf.getOption, options)),
(['someCross', 'some_other_val']), self)
assertEqualsImproved(
(self.prol.getCSPath('myTestProcDeleteMe'),
self.prol.getInFile('myTestProcDeleteMe'),
self.prol.existsProcess('myTestProcDeleteMe'),
self.prol.existsProcess(''),
self.prol.existsProcess('invalidProcess'),
self.prol.existsProcess('myTestProcDeleteMeToo')),
('/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True),
S_OK(False), S_OK(False)), self)
assertListContentEquals(self.prol.getProcesses(),
['myTestProcDeleteMe', 'MytestProcess'], self)
all_processes_dict = self.prol.getProcessesDict()
assertEqualsImproved(len(all_processes_dict), 2, self)
assertEqualsImproved(
('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess'
in all_processes_dict), (True, True), self)
self.prol.printProcesses()
def test_broadcast_processed_selectsome(self):
from DIRAC import S_OK
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
dataman_mock = Mock()
trans_mock = Mock()
util_mock = Mock()
util_mock.fc.getFileDescendents.return_value = S_OK({
'Successful': {
'/some/deep/dir/structure/file1.stdio': ['child'],
'/file/dir/input1.txt': [],
'/dir/file.xml': ['child'],
'/nodir.txt': ['child'],
'input_file.txt': ['child']
},
'Failed': {
'/my/dir/pphys.ics': []
}
})
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('BroadcastProcessed', dataman_mock,
trans_mock)
self.tfp.params['Status'] = ''
self.tfp.util = util_mock
self.tfp.setInputData({
'/some/deep/dir/structure/file1.stdio': ['myTestSE1'],
'/file/dir/input1.txt': ['other_SE'],
'/dir/file.xml': ['myTestSE1', 'SecondTestSE'],
'/nodir.txt': ['secretSE'],
'input_file.txt': ['SE_to_test'],
'/my/dir/pphys.ics': ['']
})
with patch('%s.TransformationPlugin._Broadcast' % MODULE_NAME,
new=Mock(return_value=S_OK(98124))):
assertDiracSucceedsWith_equals(self.tfp.run(), 98124, self)
assertListContentEquals(self.tfp.data, [
'/some/deep/dir/structure/file1.stdio', '/dir/file.xml',
'/nodir.txt', 'input_file.txt'
], self)
util_mock.fc.getFileDescendents.assert_called_once()
def test_sliced_limited_add_all(self):
from ILCDIRAC.ILCTransformationSystem.Agent.TransformationPlugin import TransformationPlugin
from DIRAC import S_OK
dataman_mock = Mock()
util_mock = Mock()
trans_mock = Mock()
trans_mock.getCounters.return_value = S_OK([({
'Status': 'Processed'
}, 4), ({
'Status': 'junk'
}, 6), ({
'Status': 'Ignore_me'
}, 8), ({
'Status': 'Assigned'
}, 6)])
util_mock.transClient = trans_mock
self.tfp = TransformationPlugin('SlicedLimited', dataman_mock,
trans_mock)
self.tfp.params['Status'] = 'Processed'
self.tfp.params['MaxNumberOfTasks'] = 16
self.tfp.params['TransformationID'] = 78456
self.tfp.util = util_mock
self.tfp.setInputData({
'/my/file/here_abc': ['myTestSE'],
'/my/file/other_file.txt': ['secretSE', 'StorageSE', 'TestSE'],
'/my/file/newfile.pdf': ['TestSE'],
'/my/file/a': [],
'/dir/somefile': ['']
})
result = self.tfp.run()
assertDiracSucceeds(result, self)
expected = [('', ['/my/file/here_abc']),
('', ['/my/file/other_file.txt']),
('', ['/my/file/newfile.pdf']), ('', ['/my/file/a']),
('', ['/dir/somefile'])]
assertListContentEquals(result['Value'], expected, self)
trans_mock.getCounters.assert_called_once_with(
'TransformationFiles', ['Status'], {'TransformationID': 78456})
def test_execute( self ):
exists_dict = { 'StdHepSplit_12_Run_4.tcl' : False, 'mytest_applog.log' : True }
self.shs.applicationVersion = 12
self.shs.STEP_NUMBER = 4
self.shs.nbEventsPerSlice = 16
self.shs.workflow_commons[ 'ProductionOutputData' ] = 'first_entry;/some/dir/mytestOutput_file/new_folder;a;;dontdeleteme'
self.shs.platform = 'TestPlatV1'
self.shs.InputFile = 'something'
self.shs.applicationLog = 'mytest_applog.log'
self.shs.listoutput = { 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' }
open_mock = Mock()
open_mock.__enter__.return_value = [ 'Open output file opfile1 ', 'Record = 41298 ',
'Open output file protonpeter', 'Record = 48',
'Record = 2 Output Begin Run',
'Open output file /mydir/files/run1.stdhep', 'Record=172',
'Open output file ignored_file',
'Open output file /last/file.stdhep', 'Record =2941 ' ]
self.shs.OutputFile = '/some/dir/mytestOutput_file.stdhep'
with patch('%s.StdHepSplit.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())), \
patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK(['runonstd_test_hep']))), \
patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('mySplitDir/test/'))), \
patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/mydir/ldlibs/library')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)), \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 0, ) ))), \
patch('%s.StdHepSplit.listDir' % MODULE_NAME, new=Mock()):
assertDiracSucceeds( self.shs.execute(), self )
remove_mock.assert_called_once_with( 'mytest_applog.log' )
assertListContentEquals( self.shs.step_commons[ 'listoutput' ], [
{ 'outputFile' : 'opfile1', 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' },
{ 'outputFile' : 'protonpeter', 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' },
{ 'outputFile' : '/mydir/files/run1.stdhep', 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' },
{ 'outputFile' : 'ignored_file', 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' },
{ 'outputFile' : '/last/file.stdhep', 'outputPath' : 123, 'outputDataSE' : 'myTestDataSE' } ], self )
assertListContentEquals( self.shs.workflow_commons[ 'ProductionOutputData' ].split(';'),
'first_entry;a;;dontdeleteme;/some/dir/mytestOutput_file/ignored_file;/some/dir/mytestOutput_file/opfile1;/some/dir/mytestOutput_file/protonpeter;/last/file.stdhep;/mydir/files/run1.stdhep'.split(';'), self )
def test_execute(self):
exists_dict = {
'StdHepSplit_12_Run_4.tcl': False,
'mytest_applog.log': True
}
self.shs.applicationVersion = 12
self.shs.STEP_NUMBER = 4
self.shs.nbEventsPerSlice = 16
self.shs.workflow_commons[
'ProductionOutputData'] = 'first_entry;/some/dir/mytestOutput_file/new_folder;a;;dontdeleteme'
self.shs.platform = 'TestPlatV1'
self.shs.InputFile = 'something'
self.shs.applicationLog = 'mytest_applog.log'
self.shs.listoutput = {
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}
open_mock = Mock()
open_mock.__enter__.return_value = [
'Open output file opfile1 ', 'Record = 41298 ',
'Open output file protonpeter', 'Record = 48',
'Record = 2 Output Begin Run',
'Open output file /mydir/files/run1.stdhep', 'Record=172',
'Open output file ignored_file',
'Open output file /last/file.stdhep', 'Record =2941 '
]
self.shs.OutputFile = '/some/dir/mytestOutput_file.stdhep'
with patch('%s.StdHepSplit.resolveInputVariables' % MODULE_NAME, new=Mock(return_value=S_OK())), \
patch('%s.resolveIFpaths' % MODULE_NAME, new=Mock(return_value=S_OK(['runonstd_test_hep']))), \
patch('%s.getSoftwareFolder' % MODULE_NAME, new=Mock(return_value=S_OK('mySplitDir/test/'))), \
patch('%s.getNewLDLibs' % MODULE_NAME, new=Mock(return_value='/mydir/ldlibs/library')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('%s.os.remove' % MODULE_NAME, new=Mock()) as remove_mock, \
patch('%s.open' % MODULE_NAME, new=Mock(return_value=open_mock)), \
patch('%s.os.chmod' % MODULE_NAME, new=Mock()), \
patch('%s.shellCall' % MODULE_NAME, new=Mock(return_value=S_OK( ( 0, ) ))), \
patch('%s.StdHepSplit.listDir' % MODULE_NAME, new=Mock()):
assertDiracSucceeds(self.shs.execute(), self)
remove_mock.assert_called_once_with('mytest_applog.log')
assertListContentEquals(
self.shs.step_commons['listoutput'],
[{
'outputFile': 'opfile1',
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}, {
'outputFile': 'protonpeter',
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}, {
'outputFile': '/mydir/files/run1.stdhep',
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}, {
'outputFile': 'ignored_file',
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}, {
'outputFile': '/last/file.stdhep',
'outputPath': 123,
'outputDataSE': 'myTestDataSE'
}], self)
assertListContentEquals(
self.shs.workflow_commons['ProductionOutputData'].split(';'),
'first_entry;a;;dontdeleteme;/some/dir/mytestOutput_file/ignored_file;/some/dir/mytestOutput_file/opfile1;/some/dir/mytestOutput_file/protonpeter;/last/file.stdhep;/mydir/files/run1.stdhep'
.split(';'), self)