def doTheWhizardInstallation():
"""Do the instalation for new whizard version Copy libraries, create tarball,
upload processList file add entry in configuration system
"""
res = checkSLCVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
res = checkGFortranVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors= False)
whizardResultFolder = cliParams.path
platform = cliParams.platform
whizard_version = cliParams.version
appVersion = whizard_version
beam_spectra_version = cliParams.beam_spectra
if not whizardResultFolder or not whizard_version or not beam_spectra_version:
Script.showHelp()
dexit(2)
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
diracAdmin = DiracAdmin()
modifiedCS = False
softwareSection = "/Operations/Defaults/AvailableTarBalls"
processlistLocation = "ProcessList/Location"
appName = "whizard"
ops = Operations()
path_to_process_list = ops.getValue(processlistLocation, "")
if not path_to_process_list:
gLogger.error("Could not find process list location in CS")
dexit(2)
gLogger.verbose("Getting process list from file catalog")
datMan = DataManager()
res = datMan.getFile(path_to_process_list)
if not res['OK']:
gLogger.error("Error while getting process list from storage")
dexit(2)
gLogger.verbose("done")
##just the name of the local file in current working directory
processlist = os.path.basename(path_to_process_list)
if not os.path.exists(processlist):
gLogger.error("Process list does not exist locally")
dexit(2)
pl = ProcessList(processlist)
startDir = os.getcwd()
inputlist = {}
os.chdir(whizardResultFolder)
folderlist = os.listdir(whizardResultFolder)
whiz_here = folderlist.count("whizard")
if whiz_here == 0:
gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizprc_here = folderlist.count("whizard.prc")
if whizprc_here == 0:
gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizmdl_here = folderlist.count("whizard.mdl")
if whizmdl_here == 0:
gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
gLogger.verbose("Preparing process list")
## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS
for f in folderlist:
if f.count(".in"):
infile = open(f, "r")
found_detail = False
for line in infile:
if line.count("decay_description"):
currprocess = f.split(".template.in")[0]
inputlist[currprocess] = {}
inputlist[currprocess]["InFile"] = f.rstrip("~")
inputlist[currprocess]["Detail"] = line.split("\"")[1]
found_detail = True
if line.count("process_id") and found_detail:
process_id = line.split("\"")[1]
inputlist[currprocess]["Model"] = ""
inputlist[currprocess]["Generator"] = ""
inputlist[currprocess]["Restrictions"] = ""
for process in process_id.split():
print("Looking for detail of process %s" % (process))
process_detail = getDetailsFromPRC("whizard.prc", process)
inputlist[currprocess]["Model"] = process_detail["Model"]
inputlist[currprocess]["Generator"] = process_detail["Generator"]
if len(inputlist[currprocess]["Restrictions"]):
inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"]
else:
inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"]
#if len(inputlist[currprocess].items()):
# inputlist.append(processdict)
## END FIXEME
##Update inputlist with what was found looking in the prc file
processes = readPRCFile("whizard.prc")
inputlist.update(processes)
##get from cross section files the cross sections for the processes in inputlist
#Need full process list
for f in folderlist:
if f.count("cross_sections_"):
crossfile = open(f, "r")
for line in crossfile:
line = line.rstrip().lstrip()
if not len(line):
continue
if line[0] == "#" or line[0] == "!":
continue
if len(line.split()) < 2:
continue
currprocess = line.split()[0]
if currprocess in inputlist:
inputlist[currprocess]['CrossSection'] = line.split()[1]
gLogger.notice("Preparing Tarball")
##Make a folder in the current directory of the user to store the whizard libraries, executable et al.
localWhizardFolderRel = ("whizard" + whizard_version) # relative path
localWhizardFolder = os.path.join(startDir, localWhizardFolderRel)
if not os.path.exists(localWhizardFolder):
os.makedirs(localWhizardFolder)
localWhizardLibFolder = os.path.join(localWhizardFolder,'lib')
if os.path.exists(localWhizardLibFolder):
shutil.rmtree(localWhizardLibFolder)
os.makedirs(localWhizardLibFolder) ##creates the lib folder
whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard"))
copyLibsCall = ["rsync","-avzL"]
for lib in whizardLibraries:
copyLibsCall.append(lib)
copyLibsCall.append(localWhizardLibFolder)
subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for fileName in folderlist:
shutil.copy(fileName, localWhizardFolder)
##Get the list of md5 sums for all the files in the folder to be tarred
os.chdir( localWhizardFolder )
subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True)
os.chdir(startDir)
##Create the Tarball
gLogger.notice("Creating Tarball...")
appTar = localWhizardFolder + ".tgz"
myappTar = tarfile.open(appTar, "w:gz")
myappTar.add(localWhizardFolderRel)
myappTar.close()
md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
gLogger.notice("Registering new Tarball in CS")
tarballurl = {}
av_platforms = gConfig.getSections(softwareSection, [])
if av_platforms['OK']:
if platform not in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value'])))
gLogger.error("If yours is missing add it in CS")
dexit(255)
else:
gLogger.error("Could not find all platforms available in CS")
dexit(255)
av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), [])
if not av_apps['OK']:
gLogger.error("Could not find all applications available in CS")
dexit(255)
if appName.lower() in av_apps['Value']:
versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()),
[])
if not versions['OK']:
gLogger.error("Could not find all versions available in CS")
dexit(255)
if appVersion in versions['Value']:
gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform))
dexit(0)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
if result['OK']:
modifiedCS = True
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform,
appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()),
"")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
gLogger.verbose("Done uploading the tar ball")
os.remove(appTar)
#Set for all new processes the TarBallURL
for process in inputlist.keys():
inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar)
pl.updateProcessList(inputlist)
pl.writeProcessList()
raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!")
pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion)
#Commit the changes if nothing has failed and the CS has been modified
if modifiedCS:
result = diracAdmin.csCommitChanges(False)
gLogger.verbose(result)
gLogger.notice('All done OK!')
dexit(0)
class ProcessListComplexTestCase(unittest.TestCase):
""" Test the different methods of the class, providing a usable CFG
"""
def setUp(self):
with patch('%s.os.path.exists' % MODULE_NAME,
new=Mock(return_value=False)):
self.prol = ProcessList('myTestProcess.list')
self.prol.cfg.createNewSection('Processes')
def test_updateproclist_and_getters(self):
self.prol.cfg.createNewSection('Processes/myTestProcDeleteMe')
self.prol.cfg.setOption(
'Processes/myTestProcDeleteMe/someRandomOption', True)
dict_1 = {'CrossSection': 'someCross'}
dict_1.update(STD_PROC_DICT)
dict_2 = {'CrossSection': 'some_other_val'}
dict_2.update(STD_PROC_DICT)
process_dict = {}
process_dict['MytestProcess'] = dict_1
process_dict['myTestProcDeleteMe'] = dict_2
result = self.prol.updateProcessList(process_dict)
assertDiracSucceeds(result, self)
conf = self.prol.cfg
self.assertFalse(
conf.existsKey('Processes/myTestProcDeleteMe/someRandomOption'))
options = [
'Processes/MytestProcess/CrossSection',
'Processes/myTestProcDeleteMe/CrossSection'
]
assertEqualsImproved((map(conf.getOption, options)),
(['someCross', 'some_other_val']), self)
assertEqualsImproved(
(self.prol.getCSPath('myTestProcDeleteMe'),
self.prol.getInFile('myTestProcDeleteMe'),
self.prol.existsProcess('myTestProcDeleteMe'),
self.prol.existsProcess(''),
self.prol.existsProcess('invalidProcess'),
self.prol.existsProcess('myTestProcDeleteMeToo')),
('/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True),
S_OK(False), S_OK(False)), self)
assertListContentEquals(self.prol.getProcesses(),
['myTestProcDeleteMe', 'MytestProcess'], self)
all_processes_dict = self.prol.getProcessesDict()
assertEqualsImproved(len(all_processes_dict), 2, self)
assertEqualsImproved(
('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess'
in all_processes_dict), (True, True), self)
self.prol.printProcesses()
def test_writeproclist(self):
expected_write = 'Processes\n{\n mytestprocess123\n {\n TarBallCSPath = /test/cs/path/bal.tarr\n Detail = TestNoDetails\n Generator = mytestGen21\n Model = testmodel3001\n Restrictions = \n InFile = my/file.in\n CrossSection = 0\n }\n}\n'
self.prol._addEntry(
'mytestprocess123', {
'TarBallCSPath': '/test/cs/path/bal.tarr',
'Detail': 'TestNoDetails',
'Generator': 'mytestGen21',
'Model': 'testmodel3001',
'Restrictions': '',
'InFile': 'my/file.in'
})
exists_dict = {
'/temp/dir': False,
'/temp/dir/mytempfile.txt': True,
'/my/folder/testpath.xml': True
}
fhandle_mock = Mock()
with patch('tempfile.mkstemp', new=Mock(return_value=('handle', '/temp/dir/mytempfile.txt'))), \
patch('__builtin__.file', new=Mock(return_value=fhandle_mock)) as file_mock, \
patch('os.makedirs') as mkdir_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('shutil.move') as move_mock, \
patch('os.close') as close_mock:
assertDiracSucceedsWith_equals(
self.prol.writeProcessList('/my/folder/testpath.xml'),
'/my/folder/testpath.xml', self)
mkdir_mock.assert_called_once_with('/temp/dir')
file_mock.assert_called_once_with('/temp/dir/mytempfile.txt', 'w')
fhandle_mock.write.assert_called_once_with(expected_write)
close_mock.assert_called_once_with('handle')
move_mock.assert_called_once_with('/temp/dir/mytempfile.txt',
'/my/folder/testpath.xml')
def test_writeproclist_notwritten(self):
exists_dict = {'myTmpNameTestme': True}
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = False
self.prol.cfg = cfg_mock
self.prol.location = '/my/folder/testpath2.txt'
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))):
assertDiracFailsWith(self.prol.writeProcessList(),
'failed to write repo', self)
close_mock.assert_called_once_with('myhandle')
remove_mock.assert_called_once_with('myTmpNameTestme')
def test_writeproclist_notwritten_noremove(self):
exists_dict = {'myTmpNameTestme': False}
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = False
self.prol.cfg = cfg_mock
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))):
assertDiracFailsWith(
self.prol.writeProcessList('/my/folder/testpath2.txt'),
'failed to write repo', self)
close_mock.assert_called_once_with('myhandle')
self.assertFalse(remove_mock.called)
def test_writeproclist_move_fails(self):
exists_dict = {'/my/folder/testpath2.txt': False}
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = True
self.prol.cfg = cfg_mock
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))), \
patch('shutil.move', new=Mock(side_effect=OSError('mytestErr_os'))):
assertDiracFailsWith(
self.prol.writeProcessList('/my/folder/testpath2.txt'),
'failed to write repo', self)
close_mock.assert_called_once_with('myhandle')
self.assertFalse(remove_mock.called)
def test_uploadproclist(self):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('/local/path/proc.list')
mocked_modules = {
'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock
}
module_patcher = patch.dict(sys.modules, mocked_modules)
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
with patch('shutil.copy') as copy_mock, \
patch('subprocess.call') as proc_mock:
self.prol.uploadProcessListToFileCatalog(
'/my/secret/path/processlist.whiz', 'v120')
assertMockCalls(
copy_mock,
[('myTestProcess.list',
'/afs/cern.ch/eng/clic/software/whizard/whizard_195/'),
('myTestProcess.list', '/local/path/proc.list')], self)
proc_mock.assert_called_once_with([
'svn', 'ci',
'/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list',
"-m'Process list for whizard version v120'"
],
shell=False)
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_remove_fails(self):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_ERROR('my_test_err')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('somepath')
mocked_modules = {
'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock
}
module_patcher = patch.dict(sys.modules, mocked_modules)
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with self.assertRaises(KeyboardInterrupt) as ki:
self.prol.uploadProcessListToFileCatalog('asd', 'v1')
key_interrupt = ki.exception
assertEqualsImproved(key_interrupt.args, ('abort_my_test', ), self)
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_upload_fails(self):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_ERROR('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('somepath')
mocked_modules = {
'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock
}
module_patcher = patch.dict(sys.modules, mocked_modules)
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with self.assertRaises(KeyboardInterrupt) as ki:
self.prol.uploadProcessListToFileCatalog('asd', 'v1')
key_interrupt = ki.exception
assertEqualsImproved(key_interrupt.args, ('abort_my_test', ), self)
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_copy_and_commit_fail(self):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('somepath')
mocked_modules = {
'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock
}
module_patcher = patch.dict(sys.modules, mocked_modules)
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with patch('shutil.copy', new=Mock(side_effect=OSError('oserr_testme_keeprunning'))), \
patch('subprocess.call', new=Mock(side_effect=OSError('subproc_test_err'))):
self.prol.uploadProcessListToFileCatalog(
'/my/secret/path/processlist.whiz', 'v120')
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_skip_copy(self):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('')
mocked_modules = {
'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock
}
module_patcher = patch.dict(sys.modules, mocked_modules)
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with patch('shutil.copy', new=Mock(side_effect=IOError('dont_call_me'))), \
patch('subprocess.call', new=Mock(side_effect=IOError('dont_call_me_either'))):
self.prol.uploadProcessListToFileCatalog(
'/my/secret/path/processlist.whiz', 'v120')
DIRAC.gConfig = backup_conf
module_patcher.stop()
os.remove(appTar)
#Set for all new processes the TarBallURL
for process in inputlist.keys():
inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar)
gLogger.verbose("Updating process list:")
knownprocess = pl.getProcessesDict()
knownprocess.update(inputlist)
pl.updateProcessList(knownprocess)
gLogger.verbose("Done Updating process list")
#Return to initial location
os.chdir(startdir)
pl.writeProcessList()
gLogger.verbose("Removing process list from storage")
res = rm.removeFile(path_to_process_list)
if not res['OK']:
gLogger.error("Could not remove process list from storage, do it by hand")
dexit(2)
res = upload(os.path.dirname(path_to_process_list) + "/", processlist)
if not res['OK']:
gLogger.error("something went wrong in the copy")
dexit(2)
gLogger.verbose("Done Removing process list from storage")
gLogger.verbose("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
class ProcessListComplexTestCase( unittest.TestCase ):
""" Test the different methods of the class, providing a usable CFG
"""
def setUp( self ):
with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)):
self.prol = ProcessList( 'myTestProcess.list' )
self.prol.cfg.createNewSection( 'Processes' )
def test_updateproclist_and_getters( self ):
self.prol.cfg.createNewSection( 'Processes/myTestProcDeleteMe' )
self.prol.cfg.setOption( 'Processes/myTestProcDeleteMe/someRandomOption', True )
dict_1 = { 'CrossSection' : 'someCross' }
dict_1.update( STD_PROC_DICT )
dict_2 = { 'CrossSection' : 'some_other_val' }
dict_2.update( STD_PROC_DICT )
process_dict = {}
process_dict[ 'MytestProcess' ] = dict_1
process_dict[ 'myTestProcDeleteMe' ] = dict_2
result = self.prol.updateProcessList( process_dict )
assertDiracSucceeds( result, self )
conf = self.prol.cfg
self.assertFalse( conf.existsKey( 'Processes/myTestProcDeleteMe/someRandomOption' ) )
options = [ 'Processes/MytestProcess/CrossSection', 'Processes/myTestProcDeleteMe/CrossSection' ]
assertEqualsImproved( ( map( conf.getOption, options ) ), ( [ 'someCross', 'some_other_val' ] ), self )
assertEqualsImproved( ( self.prol.getCSPath( 'myTestProcDeleteMe' ),
self.prol.getInFile( 'myTestProcDeleteMe' ),
self.prol.existsProcess( 'myTestProcDeleteMe' ),
self.prol.existsProcess( '' ), self.prol.existsProcess( 'invalidProcess' ),
self.prol.existsProcess( 'myTestProcDeleteMeToo' ) ),
( '/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True), S_OK(False),
S_OK(False) ), self )
assertListContentEquals( self.prol.getProcesses(), [ 'myTestProcDeleteMe', 'MytestProcess' ], self )
all_processes_dict = self.prol.getProcessesDict()
assertEqualsImproved( len(all_processes_dict), 2, self )
assertEqualsImproved( ('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess' in all_processes_dict),
( True, True ), self )
self.prol.printProcesses()
def test_writeproclist( self ):
expected_write = 'Processes\n{\n mytestprocess123\n {\n TarBallCSPath = /test/cs/path/bal.tarr\n Detail = TestNoDetails\n Generator = mytestGen21\n Model = testmodel3001\n Restrictions = \n InFile = my/file.in\n CrossSection = 0\n }\n}\n'
self.prol._addEntry( 'mytestprocess123', { 'TarBallCSPath' : '/test/cs/path/bal.tarr', 'Detail' :
'TestNoDetails', 'Generator' : 'mytestGen21', 'Model' :
'testmodel3001', 'Restrictions' : '', 'InFile' : 'my/file.in'
} )
exists_dict = { '/temp/dir' : False, '/temp/dir/mytempfile.txt' : True, '/my/folder/testpath.xml' : True }
fhandle_mock = Mock()
file_mock = Mock(return_value=fhandle_mock)
with patch('tempfile.mkstemp', new=Mock(return_value=('handle', '/temp/dir/mytempfile.txt'))), \
patch('__builtin__.file', new=file_mock), \
patch('__builtin__.open', new=file_mock), \
patch('os.makedirs') as mkdir_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('shutil.move') as move_mock, \
patch('os.close') as close_mock:
assertDiracSucceedsWith_equals( self.prol.writeProcessList( '/my/folder/testpath.xml' ),
'/my/folder/testpath.xml', self )
mkdir_mock.assert_called_once_with( '/temp/dir' )
file_mock.assert_called_once_with( '/temp/dir/mytempfile.txt', 'w' )
fhandle_mock.write.assert_called_once_with( expected_write )
close_mock.assert_called_once_with( 'handle' )
move_mock.assert_called_once_with( '/temp/dir/mytempfile.txt', '/my/folder/testpath.xml' )
def test_writeproclist_notwritten( self ):
exists_dict = { 'myTmpNameTestme' : True }
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = False
self.prol.cfg = cfg_mock
self.prol.location = '/my/folder/testpath2.txt'
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))):
assertDiracFailsWith( self.prol.writeProcessList(), 'failed to write repo', self )
close_mock.assert_called_once_with( 'myhandle' )
remove_mock.assert_called_once_with( 'myTmpNameTestme')
def test_writeproclist_notwritten_noremove( self ):
exists_dict = { 'myTmpNameTestme' : False }
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = False
self.prol.cfg = cfg_mock
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))):
assertDiracFailsWith( self.prol.writeProcessList( '/my/folder/testpath2.txt' ),
'failed to write repo', self )
close_mock.assert_called_once_with( 'myhandle' )
self.assertFalse( remove_mock.called )
def test_writeproclist_move_fails( self ):
exists_dict = { '/my/folder/testpath2.txt' : False }
cfg_mock = Mock()
cfg_mock.writeToFile.return_value = True
self.prol.cfg = cfg_mock
with patch('os.close') as close_mock, \
patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \
patch('os.remove') as remove_mock, \
patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))), \
patch('shutil.move', new=Mock(side_effect=OSError('mytestErr_os'))):
assertDiracFailsWith( self.prol.writeProcessList( '/my/folder/testpath2.txt' ),
'failed to write repo', self )
close_mock.assert_called_once_with( 'myhandle' )
self.assertFalse( remove_mock.called )
def test_uploadproclist( self ):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK( '/local/path/proc.list' )
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock }
module_patcher = patch.dict( sys.modules, mocked_modules )
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
with patch('shutil.copy') as copy_mock, \
patch('subprocess.call') as proc_mock:
self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' )
assertMockCalls( copy_mock, [
( 'myTestProcess.list', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/' ),
( 'myTestProcess.list', '/local/path/proc.list' ) ], self )
proc_mock.assert_called_once_with(
[ 'svn', 'ci', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list',
"-m'Process list for whizard version v120'" ], shell=False )
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_remove_fails( self ):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_ERROR('my_test_err')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK( 'somepath' )
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock }
module_patcher = patch.dict( sys.modules, mocked_modules )
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with self.assertRaises( KeyboardInterrupt ) as ki:
self.prol.uploadProcessListToFileCatalog( 'asd', 'v1' )
key_interrupt = ki.exception
assertEqualsImproved( key_interrupt.args, ( 'abort_my_test', ), self )
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_upload_fails( self ):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_ERROR('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK( 'somepath' )
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock }
module_patcher = patch.dict( sys.modules, mocked_modules )
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with self.assertRaises( KeyboardInterrupt ) as ki:
self.prol.uploadProcessListToFileCatalog( 'asd', 'v1' )
key_interrupt = ki.exception
assertEqualsImproved( key_interrupt.args, ( 'abort_my_test', ), self )
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_copy_and_commit_fail( self ):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK( 'somepath' )
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock }
module_patcher = patch.dict( sys.modules, mocked_modules )
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with patch('shutil.copy', new=Mock(side_effect=OSError('oserr_testme_keeprunning'))), \
patch('subprocess.call', new=Mock(side_effect=OSError('subproc_test_err'))):
self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' )
DIRAC.gConfig = backup_conf
module_patcher.stop()
def test_uploadproclist_skip_copy( self ):
import sys
import DIRAC
datman_mock = Mock()
datman_mock.removeFile.return_value = S_OK('something')
datmodule_mock = Mock()
datmodule_mock.DataManager.return_value = datman_mock
fileutil_mock = Mock()
fileutil_mock.upload.return_value = S_OK('something')
conf_mock = Mock()
conf_mock.getOption.return_value = S_OK('')
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock,
'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock }
module_patcher = patch.dict( sys.modules, mocked_modules )
module_patcher.start()
backup_conf = DIRAC.gConfig
DIRAC.gConfig = conf_mock
DIRAC.exit = abort_test
with patch('shutil.copy', new=Mock(side_effect=IOError('dont_call_me'))), \
patch('subprocess.call', new=Mock(side_effect=IOError('dont_call_me_either'))):
self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' )
DIRAC.gConfig = backup_conf
module_patcher.stop()
def doTheWhizardInstallation():
"""Do the instalation for new whizard version Copy libraries, create tarball,
upload processList file add entry in configuration system
"""
res = checkSLCVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
res = checkGFortranVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors= False)
whizardResultFolder = cliParams.path
platform = cliParams.platform
whizard_version = cliParams.version
appVersion = whizard_version
beam_spectra_version = cliParams.beam_spectra
if not whizardResultFolder or not whizard_version or not beam_spectra_version:
Script.showHelp()
dexit(2)
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
diracAdmin = DiracAdmin()
modifiedCS = False
softwareSection = "/Operations/Defaults/AvailableTarBalls"
processlistLocation = "ProcessList/Location"
appName = "whizard"
ops = Operations()
path_to_process_list = ops.getValue(processlistLocation, "")
if not path_to_process_list:
gLogger.error("Could not find process list location in CS")
dexit(2)
gLogger.verbose("Getting process list from file catalog")
datMan = DataManager()
res = datMan.getFile(path_to_process_list)
if not res['OK']:
gLogger.error("Error while getting process list from storage")
dexit(2)
gLogger.verbose("done")
##just the name of the local file in current working directory
processlist = os.path.basename(path_to_process_list)
if not os.path.exists(processlist):
gLogger.error("Process list does not exist locally")
dexit(2)
pl = ProcessList(processlist)
startDir = os.getcwd()
inputlist = {}
os.chdir(whizardResultFolder)
folderlist = os.listdir(whizardResultFolder)
whiz_here = folderlist.count("whizard")
if whiz_here == 0:
gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizprc_here = folderlist.count("whizard.prc")
if whizprc_here == 0:
gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizmdl_here = folderlist.count("whizard.mdl")
if whizmdl_here == 0:
gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
gLogger.verbose("Preparing process list")
## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS
for f in folderlist:
if f.count(".in"):
infile = open(f, "r")
found_detail = False
for line in infile:
if line.count("decay_description"):
currprocess = f.split(".template.in")[0]
inputlist[currprocess] = {}
inputlist[currprocess]["InFile"] = f.rstrip("~")
inputlist[currprocess]["Detail"] = line.split("\"")[1]
found_detail = True
if line.count("process_id") and found_detail:
process_id = line.split("\"")[1]
inputlist[currprocess]["Model"] = ""
inputlist[currprocess]["Generator"] = ""
inputlist[currprocess]["Restrictions"] = ""
for process in process_id.split():
print "Looking for detail of process %s" % (process)
process_detail = getDetailsFromPRC("whizard.prc", process)
inputlist[currprocess]["Model"] = process_detail["Model"]
inputlist[currprocess]["Generator"] = process_detail["Generator"]
if len(inputlist[currprocess]["Restrictions"]):
inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"]
else:
inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"]
#if len(inputlist[currprocess].items()):
# inputlist.append(processdict)
## END FIXEME
##Update inputlist with what was found looking in the prc file
processes = readPRCFile("whizard.prc")
inputlist.update(processes)
##get from cross section files the cross sections for the processes in inputlist
#Need full process list
for f in folderlist:
if f.count("cross_sections_"):
crossfile = open(f, "r")
for line in crossfile:
line = line.rstrip().lstrip()
if not len(line):
continue
if line[0] == "#" or line[0] == "!":
continue
if len(line.split()) < 2:
continue
currprocess = line.split()[0]
if currprocess in inputlist:
inputlist[currprocess]['CrossSection'] = line.split()[1]
gLogger.notice("Preparing Tarball")
##Make a folder in the current directory of the user to store the whizard libraries, executable et al.
localWhizardFolderRel = ("whizard" + whizard_version) # relative path
localWhizardFolder = os.path.join(startDir, localWhizardFolderRel)
if not os.path.exists(localWhizardFolder):
os.makedirs(localWhizardFolder)
localWhizardLibFolder = os.path.join(localWhizardFolder,'lib')
if os.path.exists(localWhizardLibFolder):
shutil.rmtree(localWhizardLibFolder)
os.makedirs(localWhizardLibFolder) ##creates the lib folder
whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard"))
copyLibsCall = ["rsync","-avzL"]
for lib in whizardLibraries:
copyLibsCall.append(lib)
copyLibsCall.append(localWhizardLibFolder)
subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for fileName in folderlist:
shutil.copy(fileName, localWhizardFolder)
##Get the list of md5 sums for all the files in the folder to be tarred
os.chdir( localWhizardFolder )
subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True)
os.chdir(startDir)
##Create the Tarball
gLogger.notice("Creating Tarball...")
appTar = localWhizardFolder + ".tgz"
myappTar = tarfile.open(appTar, "w:gz")
myappTar.add(localWhizardFolderRel)
myappTar.close()
md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
gLogger.notice("Registering new Tarball in CS")
tarballurl = {}
av_platforms = gConfig.getSections(softwareSection, [])
if av_platforms['OK']:
if platform not in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value'])))
gLogger.error("If yours is missing add it in CS")
dexit(255)
else:
gLogger.error("Could not find all platforms available in CS")
dexit(255)
av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), [])
if not av_apps['OK']:
gLogger.error("Could not find all applications available in CS")
dexit(255)
if appName.lower() in av_apps['Value']:
versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()),
[])
if not versions['OK']:
gLogger.error("Could not find all versions available in CS")
dexit(255)
if appVersion in versions['Value']:
gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform))
dexit(0)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
if result['OK']:
modifiedCS = True
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform,
appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()),
"")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
gLogger.verbose("Done uploading the tar ball")
os.remove(appTar)
#Set for all new processes the TarBallURL
for process in inputlist.keys():
inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar)
pl.updateProcessList(inputlist)
pl.writeProcessList()
raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!")
pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion)
#Commit the changes if nothing has failed and the CS has been modified
if modifiedCS:
result = diracAdmin.csCommitChanges(False)
gLogger.verbose(result)
gLogger.notice('All done OK!')
dexit(0)
