def doTheWhizardInstallation(): """Do the instalation for new whizard version Copy libraries, create tarball, upload processList file add entry in configuration system """ res = checkSLCVersion() if not res['OK']: gLogger.error(res['Message']) dexit(1) res = checkGFortranVersion() if not res['OK']: gLogger.error(res['Message']) dexit(1) cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine( ignoreErrors= False) whizardResultFolder = cliParams.path platform = cliParams.platform whizard_version = cliParams.version appVersion = whizard_version beam_spectra_version = cliParams.beam_spectra if not whizardResultFolder or not whizard_version or not beam_spectra_version: Script.showHelp() dexit(2) from ILCDIRAC.Core.Utilities.ProcessList import ProcessList from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from ILCDIRAC.Core.Utilities.FileUtils import upload from DIRAC.DataManagementSystem.Client.DataManager import DataManager diracAdmin = DiracAdmin() modifiedCS = False softwareSection = "/Operations/Defaults/AvailableTarBalls" processlistLocation = "ProcessList/Location" appName = "whizard" ops = Operations() path_to_process_list = ops.getValue(processlistLocation, "") if not path_to_process_list: gLogger.error("Could not find process list location in CS") dexit(2) gLogger.verbose("Getting process list from file catalog") datMan = DataManager() res = datMan.getFile(path_to_process_list) if not res['OK']: gLogger.error("Error while getting process list from storage") dexit(2) gLogger.verbose("done") ##just the name of the local file in current working directory processlist = os.path.basename(path_to_process_list) if not os.path.exists(processlist): gLogger.error("Process list does not exist locally") dexit(2) pl = ProcessList(processlist) startDir = os.getcwd() inputlist = {} os.chdir(whizardResultFolder) folderlist = os.listdir(whizardResultFolder) whiz_here = folderlist.count("whizard") if whiz_here == 0: gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) whizprc_here = folderlist.count("whizard.prc") if whizprc_here == 0: gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) whizmdl_here = folderlist.count("whizard.mdl") if whizmdl_here == 0: gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) gLogger.verbose("Preparing process list") ## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS for f in folderlist: if f.count(".in"): infile = open(f, "r") found_detail = False for line in infile: if line.count("decay_description"): currprocess = f.split(".template.in")[0] inputlist[currprocess] = {} inputlist[currprocess]["InFile"] = f.rstrip("~") inputlist[currprocess]["Detail"] = line.split("\"")[1] found_detail = True if line.count("process_id") and found_detail: process_id = line.split("\"")[1] inputlist[currprocess]["Model"] = "" inputlist[currprocess]["Generator"] = "" inputlist[currprocess]["Restrictions"] = "" for process in process_id.split(): print("Looking for detail of process %s" % (process)) process_detail = getDetailsFromPRC("whizard.prc", process) inputlist[currprocess]["Model"] = process_detail["Model"] inputlist[currprocess]["Generator"] = process_detail["Generator"] if len(inputlist[currprocess]["Restrictions"]): inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"] else: inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"] #if len(inputlist[currprocess].items()): # inputlist.append(processdict) ## END FIXEME ##Update inputlist with what was found looking in the prc file processes = readPRCFile("whizard.prc") inputlist.update(processes) ##get from cross section files the cross sections for the processes in inputlist #Need full process list for f in folderlist: if f.count("cross_sections_"): crossfile = open(f, "r") for line in crossfile: line = line.rstrip().lstrip() if not len(line): continue if line[0] == "#" or line[0] == "!": continue if len(line.split()) < 2: continue currprocess = line.split()[0] if currprocess in inputlist: inputlist[currprocess]['CrossSection'] = line.split()[1] gLogger.notice("Preparing Tarball") ##Make a folder in the current directory of the user to store the whizard libraries, executable et al. localWhizardFolderRel = ("whizard" + whizard_version) # relative path localWhizardFolder = os.path.join(startDir, localWhizardFolderRel) if not os.path.exists(localWhizardFolder): os.makedirs(localWhizardFolder) localWhizardLibFolder = os.path.join(localWhizardFolder,'lib') if os.path.exists(localWhizardLibFolder): shutil.rmtree(localWhizardLibFolder) os.makedirs(localWhizardLibFolder) ##creates the lib folder whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard")) copyLibsCall = ["rsync","-avzL"] for lib in whizardLibraries: copyLibsCall.append(lib) copyLibsCall.append(localWhizardLibFolder) subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for fileName in folderlist: shutil.copy(fileName, localWhizardFolder) ##Get the list of md5 sums for all the files in the folder to be tarred os.chdir( localWhizardFolder ) subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True) os.chdir(startDir) ##Create the Tarball gLogger.notice("Creating Tarball...") appTar = localWhizardFolder + ".tgz" myappTar = tarfile.open(appTar, "w:gz") myappTar.add(localWhizardFolderRel) myappTar.close() md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest() gLogger.notice("...Done") gLogger.notice("Registering new Tarball in CS") tarballurl = {} av_platforms = gConfig.getSections(softwareSection, []) if av_platforms['OK']: if platform not in av_platforms['Value']: gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value']))) gLogger.error("If yours is missing add it in CS") dexit(255) else: gLogger.error("Could not find all platforms available in CS") dexit(255) av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), []) if not av_apps['OK']: gLogger.error("Could not find all applications available in CS") dexit(255) if appName.lower() in av_apps['Value']: versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()), []) if not versions['OK']: gLogger.error("Could not find all versions available in CS") dexit(255) if appVersion in versions['Value']: gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform)) dexit(0) else: result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion), os.path.basename(appTar)) if result['OK']: modifiedCS = True tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "") if len(tarballurl['Value']) > 0: res = upload(tarballurl['Value'], appTar) if not res['OK']: gLogger.error("Upload to %s failed" % tarballurl['Value']) dexit(255) result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion), md5sum) if result['OK']: modifiedCS = True result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection, platform, appName.lower(), appVersion), beam_spectra_version) else: result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion), os.path.basename(appTar)) if result['OK']: modifiedCS = True tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "") if len(tarballurl['Value']) > 0: res = upload(tarballurl['Value'], appTar) if not res['OK']: gLogger.error("Upload to %s failed" % tarballurl['Value']) dexit(255) result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion), md5sum) result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection, platform, appName.lower(), appVersion), beam_spectra_version) gLogger.verbose("Done uploading the tar ball") os.remove(appTar) #Set for all new processes the TarBallURL for process in inputlist.keys(): inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar) pl.updateProcessList(inputlist) pl.writeProcessList() raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!") pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion) #Commit the changes if nothing has failed and the CS has been modified if modifiedCS: result = diracAdmin.csCommitChanges(False) gLogger.verbose(result) gLogger.notice('All done OK!') dexit(0)
class ProcessListComplexTestCase( unittest.TestCase ): """ Test the different methods of the class, providing a usable CFG """ def setUp( self ): with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)): self.prol = ProcessList( 'myTestProcess.list' ) self.prol.cfg.createNewSection( 'Processes' ) def test_updateproclist_and_getters( self ): self.prol.cfg.createNewSection( 'Processes/myTestProcDeleteMe' ) self.prol.cfg.setOption( 'Processes/myTestProcDeleteMe/someRandomOption', True ) dict_1 = { 'CrossSection' : 'someCross' } dict_1.update( STD_PROC_DICT ) dict_2 = { 'CrossSection' : 'some_other_val' } dict_2.update( STD_PROC_DICT ) process_dict = {} process_dict[ 'MytestProcess' ] = dict_1 process_dict[ 'myTestProcDeleteMe' ] = dict_2 result = self.prol.updateProcessList( process_dict ) assertDiracSucceeds( result, self ) conf = self.prol.cfg self.assertFalse( conf.existsKey( 'Processes/myTestProcDeleteMe/someRandomOption' ) ) options = [ 'Processes/MytestProcess/CrossSection', 'Processes/myTestProcDeleteMe/CrossSection' ] assertEqualsImproved( ( map( conf.getOption, options ) ), ( [ 'someCross', 'some_other_val' ] ), self ) assertEqualsImproved( ( self.prol.getCSPath( 'myTestProcDeleteMe' ), self.prol.getInFile( 'myTestProcDeleteMe' ), self.prol.existsProcess( 'myTestProcDeleteMe' ), self.prol.existsProcess( '' ), self.prol.existsProcess( 'invalidProcess' ), self.prol.existsProcess( 'myTestProcDeleteMeToo' ) ), ( '/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True), S_OK(False), S_OK(False) ), self ) assertListContentEquals( self.prol.getProcesses(), [ 'myTestProcDeleteMe', 'MytestProcess' ], self ) all_processes_dict = self.prol.getProcessesDict() assertEqualsImproved( len(all_processes_dict), 2, self ) assertEqualsImproved( ('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess' in all_processes_dict), ( True, True ), self ) self.prol.printProcesses() def test_writeproclist( self ): expected_write = 'Processes\n{\n mytestprocess123\n {\n TarBallCSPath = /test/cs/path/bal.tarr\n Detail = TestNoDetails\n Generator = mytestGen21\n Model = testmodel3001\n Restrictions = \n InFile = my/file.in\n CrossSection = 0\n }\n}\n' self.prol._addEntry( 'mytestprocess123', { 'TarBallCSPath' : '/test/cs/path/bal.tarr', 'Detail' : 'TestNoDetails', 'Generator' : 'mytestGen21', 'Model' : 'testmodel3001', 'Restrictions' : '', 'InFile' : 'my/file.in' } ) exists_dict = { '/temp/dir' : False, '/temp/dir/mytempfile.txt' : True, '/my/folder/testpath.xml' : True } fhandle_mock = Mock() file_mock = Mock(return_value=fhandle_mock) with patch('tempfile.mkstemp', new=Mock(return_value=('handle', '/temp/dir/mytempfile.txt'))), \ patch('__builtin__.file', new=file_mock), \ patch('__builtin__.open', new=file_mock), \ patch('os.makedirs') as mkdir_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('shutil.move') as move_mock, \ patch('os.close') as close_mock: assertDiracSucceedsWith_equals( self.prol.writeProcessList( '/my/folder/testpath.xml' ), '/my/folder/testpath.xml', self ) mkdir_mock.assert_called_once_with( '/temp/dir' ) file_mock.assert_called_once_with( '/temp/dir/mytempfile.txt', 'w' ) fhandle_mock.write.assert_called_once_with( expected_write ) close_mock.assert_called_once_with( 'handle' ) move_mock.assert_called_once_with( '/temp/dir/mytempfile.txt', '/my/folder/testpath.xml' ) def test_writeproclist_notwritten( self ): exists_dict = { 'myTmpNameTestme' : True } cfg_mock = Mock() cfg_mock.writeToFile.return_value = False self.prol.cfg = cfg_mock self.prol.location = '/my/folder/testpath2.txt' with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))): assertDiracFailsWith( self.prol.writeProcessList(), 'failed to write repo', self ) close_mock.assert_called_once_with( 'myhandle' ) remove_mock.assert_called_once_with( 'myTmpNameTestme') def test_writeproclist_notwritten_noremove( self ): exists_dict = { 'myTmpNameTestme' : False } cfg_mock = Mock() cfg_mock.writeToFile.return_value = False self.prol.cfg = cfg_mock with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))): assertDiracFailsWith( self.prol.writeProcessList( '/my/folder/testpath2.txt' ), 'failed to write repo', self ) close_mock.assert_called_once_with( 'myhandle' ) self.assertFalse( remove_mock.called ) def test_writeproclist_move_fails( self ): exists_dict = { '/my/folder/testpath2.txt' : False } cfg_mock = Mock() cfg_mock.writeToFile.return_value = True self.prol.cfg = cfg_mock with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))), \ patch('shutil.move', new=Mock(side_effect=OSError('mytestErr_os'))): assertDiracFailsWith( self.prol.writeProcessList( '/my/folder/testpath2.txt' ), 'failed to write repo', self ) close_mock.assert_called_once_with( 'myhandle' ) self.assertFalse( remove_mock.called ) def test_uploadproclist( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK( '/local/path/proc.list' ) mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock with patch('shutil.copy') as copy_mock, \ patch('subprocess.call') as proc_mock: self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' ) assertMockCalls( copy_mock, [ ( 'myTestProcess.list', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/' ), ( 'myTestProcess.list', '/local/path/proc.list' ) ], self ) proc_mock.assert_called_once_with( [ 'svn', 'ci', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list', "-m'Process list for whizard version v120'" ], shell=False ) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_remove_fails( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_ERROR('my_test_err') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() conf_mock = Mock() conf_mock.getOption.return_value = S_OK( 'somepath' ) mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with self.assertRaises( KeyboardInterrupt ) as ki: self.prol.uploadProcessListToFileCatalog( 'asd', 'v1' ) key_interrupt = ki.exception assertEqualsImproved( key_interrupt.args, ( 'abort_my_test', ), self ) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_upload_fails( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_ERROR('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK( 'somepath' ) mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with self.assertRaises( KeyboardInterrupt ) as ki: self.prol.uploadProcessListToFileCatalog( 'asd', 'v1' ) key_interrupt = ki.exception assertEqualsImproved( key_interrupt.args, ( 'abort_my_test', ), self ) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_copy_and_commit_fail( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK( 'somepath' ) mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with patch('shutil.copy', new=Mock(side_effect=OSError('oserr_testme_keeprunning'))), \ patch('subprocess.call', new=Mock(side_effect=OSError('subproc_test_err'))): self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' ) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_skip_copy( self ): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils' : fileutil_mock } module_patcher = patch.dict( sys.modules, mocked_modules ) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with patch('shutil.copy', new=Mock(side_effect=IOError('dont_call_me'))), \ patch('subprocess.call', new=Mock(side_effect=IOError('dont_call_me_either'))): self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120' ) DIRAC.gConfig = backup_conf module_patcher.stop()
class ProcessListComplexTestCase(unittest.TestCase): """ Test the different methods of the class, providing a usable CFG """ def setUp(self): with patch('%s.os.path.exists' % MODULE_NAME, new=Mock(return_value=False)): self.prol = ProcessList('myTestProcess.list') self.prol.cfg.createNewSection('Processes') def test_updateproclist_and_getters(self): self.prol.cfg.createNewSection('Processes/myTestProcDeleteMe') self.prol.cfg.setOption( 'Processes/myTestProcDeleteMe/someRandomOption', True) dict_1 = {'CrossSection': 'someCross'} dict_1.update(STD_PROC_DICT) dict_2 = {'CrossSection': 'some_other_val'} dict_2.update(STD_PROC_DICT) process_dict = {} process_dict['MytestProcess'] = dict_1 process_dict['myTestProcDeleteMe'] = dict_2 result = self.prol.updateProcessList(process_dict) assertDiracSucceeds(result, self) conf = self.prol.cfg self.assertFalse( conf.existsKey('Processes/myTestProcDeleteMe/someRandomOption')) options = [ 'Processes/MytestProcess/CrossSection', 'Processes/myTestProcDeleteMe/CrossSection' ] assertEqualsImproved((map(conf.getOption, options)), (['someCross', 'some_other_val']), self) assertEqualsImproved( (self.prol.getCSPath('myTestProcDeleteMe'), self.prol.getInFile('myTestProcDeleteMe'), self.prol.existsProcess('myTestProcDeleteMe'), self.prol.existsProcess(''), self.prol.existsProcess('invalidProcess'), self.prol.existsProcess('myTestProcDeleteMeToo')), ('/test/cs/path/ball.tar', 'my/file.in', S_OK(True), S_OK(True), S_OK(False), S_OK(False)), self) assertListContentEquals(self.prol.getProcesses(), ['myTestProcDeleteMe', 'MytestProcess'], self) all_processes_dict = self.prol.getProcessesDict() assertEqualsImproved(len(all_processes_dict), 2, self) assertEqualsImproved( ('myTestProcDeleteMe' in all_processes_dict, 'MytestProcess' in all_processes_dict), (True, True), self) self.prol.printProcesses() def test_writeproclist(self): expected_write = 'Processes\n{\n mytestprocess123\n {\n TarBallCSPath = /test/cs/path/bal.tarr\n Detail = TestNoDetails\n Generator = mytestGen21\n Model = testmodel3001\n Restrictions = \n InFile = my/file.in\n CrossSection = 0\n }\n}\n' self.prol._addEntry( 'mytestprocess123', { 'TarBallCSPath': '/test/cs/path/bal.tarr', 'Detail': 'TestNoDetails', 'Generator': 'mytestGen21', 'Model': 'testmodel3001', 'Restrictions': '', 'InFile': 'my/file.in' }) exists_dict = { '/temp/dir': False, '/temp/dir/mytempfile.txt': True, '/my/folder/testpath.xml': True } fhandle_mock = Mock() with patch('tempfile.mkstemp', new=Mock(return_value=('handle', '/temp/dir/mytempfile.txt'))), \ patch('__builtin__.file', new=Mock(return_value=fhandle_mock)) as file_mock, \ patch('os.makedirs') as mkdir_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('shutil.move') as move_mock, \ patch('os.close') as close_mock: assertDiracSucceedsWith_equals( self.prol.writeProcessList('/my/folder/testpath.xml'), '/my/folder/testpath.xml', self) mkdir_mock.assert_called_once_with('/temp/dir') file_mock.assert_called_once_with('/temp/dir/mytempfile.txt', 'w') fhandle_mock.write.assert_called_once_with(expected_write) close_mock.assert_called_once_with('handle') move_mock.assert_called_once_with('/temp/dir/mytempfile.txt', '/my/folder/testpath.xml') def test_writeproclist_notwritten(self): exists_dict = {'myTmpNameTestme': True} cfg_mock = Mock() cfg_mock.writeToFile.return_value = False self.prol.cfg = cfg_mock self.prol.location = '/my/folder/testpath2.txt' with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))): assertDiracFailsWith(self.prol.writeProcessList(), 'failed to write repo', self) close_mock.assert_called_once_with('myhandle') remove_mock.assert_called_once_with('myTmpNameTestme') def test_writeproclist_notwritten_noremove(self): exists_dict = {'myTmpNameTestme': False} cfg_mock = Mock() cfg_mock.writeToFile.return_value = False self.prol.cfg = cfg_mock with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))): assertDiracFailsWith( self.prol.writeProcessList('/my/folder/testpath2.txt'), 'failed to write repo', self) close_mock.assert_called_once_with('myhandle') self.assertFalse(remove_mock.called) def test_writeproclist_move_fails(self): exists_dict = {'/my/folder/testpath2.txt': False} cfg_mock = Mock() cfg_mock.writeToFile.return_value = True self.prol.cfg = cfg_mock with patch('os.close') as close_mock, \ patch('os.path.exists', new=Mock(side_effect=lambda path: exists_dict[path])), \ patch('os.remove') as remove_mock, \ patch('tempfile.mkstemp', new=Mock(return_value=('myhandle', 'myTmpNameTestme'))), \ patch('shutil.move', new=Mock(side_effect=OSError('mytestErr_os'))): assertDiracFailsWith( self.prol.writeProcessList('/my/folder/testpath2.txt'), 'failed to write repo', self) close_mock.assert_called_once_with('myhandle') self.assertFalse(remove_mock.called) def test_uploadproclist(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('/local/path/proc.list') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock with patch('shutil.copy') as copy_mock, \ patch('subprocess.call') as proc_mock: self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120') assertMockCalls( copy_mock, [('myTestProcess.list', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/'), ('myTestProcess.list', '/local/path/proc.list')], self) proc_mock.assert_called_once_with([ 'svn', 'ci', '/afs/cern.ch/eng/clic/software/whizard/whizard_195/proc.list', "-m'Process list for whizard version v120'" ], shell=False) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_remove_fails(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_ERROR('my_test_err') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() conf_mock = Mock() conf_mock.getOption.return_value = S_OK('somepath') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with self.assertRaises(KeyboardInterrupt) as ki: self.prol.uploadProcessListToFileCatalog('asd', 'v1') key_interrupt = ki.exception assertEqualsImproved(key_interrupt.args, ('abort_my_test', ), self) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_upload_fails(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_ERROR('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('somepath') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with self.assertRaises(KeyboardInterrupt) as ki: self.prol.uploadProcessListToFileCatalog('asd', 'v1') key_interrupt = ki.exception assertEqualsImproved(key_interrupt.args, ('abort_my_test', ), self) DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_copy_and_commit_fail(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('somepath') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with patch('shutil.copy', new=Mock(side_effect=OSError('oserr_testme_keeprunning'))), \ patch('subprocess.call', new=Mock(side_effect=OSError('subproc_test_err'))): self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120') DIRAC.gConfig = backup_conf module_patcher.stop() def test_uploadproclist_skip_copy(self): import sys import DIRAC datman_mock = Mock() datman_mock.removeFile.return_value = S_OK('something') datmodule_mock = Mock() datmodule_mock.DataManager.return_value = datman_mock fileutil_mock = Mock() fileutil_mock.upload.return_value = S_OK('something') conf_mock = Mock() conf_mock.getOption.return_value = S_OK('') mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager': datmodule_mock, 'ILCDIRAC.Core.Utilities.FileUtils': fileutil_mock } module_patcher = patch.dict(sys.modules, mocked_modules) module_patcher.start() backup_conf = DIRAC.gConfig DIRAC.gConfig = conf_mock DIRAC.exit = abort_test with patch('shutil.copy', new=Mock(side_effect=IOError('dont_call_me'))), \ patch('subprocess.call', new=Mock(side_effect=IOError('dont_call_me_either'))): self.prol.uploadProcessListToFileCatalog( '/my/secret/path/processlist.whiz', 'v120') DIRAC.gConfig = backup_conf module_patcher.stop()
def doTheWhizardInstallation(): """Do the instalation for new whizard version Copy libraries, create tarball, upload processList file add entry in configuration system """ res = checkSLCVersion() if not res['OK']: gLogger.error(res['Message']) dexit(1) res = checkGFortranVersion() if not res['OK']: gLogger.error(res['Message']) dexit(1) cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine( ignoreErrors= False) whizardResultFolder = cliParams.path platform = cliParams.platform whizard_version = cliParams.version appVersion = whizard_version beam_spectra_version = cliParams.beam_spectra if not whizardResultFolder or not whizard_version or not beam_spectra_version: Script.showHelp() dexit(2) from ILCDIRAC.Core.Utilities.ProcessList import ProcessList from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from ILCDIRAC.Core.Utilities.FileUtils import upload from DIRAC.DataManagementSystem.Client.DataManager import DataManager diracAdmin = DiracAdmin() modifiedCS = False softwareSection = "/Operations/Defaults/AvailableTarBalls" processlistLocation = "ProcessList/Location" appName = "whizard" ops = Operations() path_to_process_list = ops.getValue(processlistLocation, "") if not path_to_process_list: gLogger.error("Could not find process list location in CS") dexit(2) gLogger.verbose("Getting process list from file catalog") datMan = DataManager() res = datMan.getFile(path_to_process_list) if not res['OK']: gLogger.error("Error while getting process list from storage") dexit(2) gLogger.verbose("done") ##just the name of the local file in current working directory processlist = os.path.basename(path_to_process_list) if not os.path.exists(processlist): gLogger.error("Process list does not exist locally") dexit(2) pl = ProcessList(processlist) startDir = os.getcwd() inputlist = {} os.chdir(whizardResultFolder) folderlist = os.listdir(whizardResultFolder) whiz_here = folderlist.count("whizard") if whiz_here == 0: gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) whizprc_here = folderlist.count("whizard.prc") if whizprc_here == 0: gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) whizmdl_here = folderlist.count("whizard.mdl") if whizmdl_here == 0: gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder) os.chdir(startDir) dexit(2) gLogger.verbose("Preparing process list") ## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS for f in folderlist: if f.count(".in"): infile = open(f, "r") found_detail = False for line in infile: if line.count("decay_description"): currprocess = f.split(".template.in")[0] inputlist[currprocess] = {} inputlist[currprocess]["InFile"] = f.rstrip("~") inputlist[currprocess]["Detail"] = line.split("\"")[1] found_detail = True if line.count("process_id") and found_detail: process_id = line.split("\"")[1] inputlist[currprocess]["Model"] = "" inputlist[currprocess]["Generator"] = "" inputlist[currprocess]["Restrictions"] = "" for process in process_id.split(): print "Looking for detail of process %s" % (process) process_detail = getDetailsFromPRC("whizard.prc", process) inputlist[currprocess]["Model"] = process_detail["Model"] inputlist[currprocess]["Generator"] = process_detail["Generator"] if len(inputlist[currprocess]["Restrictions"]): inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"] else: inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"] #if len(inputlist[currprocess].items()): # inputlist.append(processdict) ## END FIXEME ##Update inputlist with what was found looking in the prc file processes = readPRCFile("whizard.prc") inputlist.update(processes) ##get from cross section files the cross sections for the processes in inputlist #Need full process list for f in folderlist: if f.count("cross_sections_"): crossfile = open(f, "r") for line in crossfile: line = line.rstrip().lstrip() if not len(line): continue if line[0] == "#" or line[0] == "!": continue if len(line.split()) < 2: continue currprocess = line.split()[0] if currprocess in inputlist: inputlist[currprocess]['CrossSection'] = line.split()[1] gLogger.notice("Preparing Tarball") ##Make a folder in the current directory of the user to store the whizard libraries, executable et al. localWhizardFolderRel = ("whizard" + whizard_version) # relative path localWhizardFolder = os.path.join(startDir, localWhizardFolderRel) if not os.path.exists(localWhizardFolder): os.makedirs(localWhizardFolder) localWhizardLibFolder = os.path.join(localWhizardFolder,'lib') if os.path.exists(localWhizardLibFolder): shutil.rmtree(localWhizardLibFolder) os.makedirs(localWhizardLibFolder) ##creates the lib folder whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard")) copyLibsCall = ["rsync","-avzL"] for lib in whizardLibraries: copyLibsCall.append(lib) copyLibsCall.append(localWhizardLibFolder) subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for fileName in folderlist: shutil.copy(fileName, localWhizardFolder) ##Get the list of md5 sums for all the files in the folder to be tarred os.chdir( localWhizardFolder ) subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True) os.chdir(startDir) ##Create the Tarball gLogger.notice("Creating Tarball...") appTar = localWhizardFolder + ".tgz" myappTar = tarfile.open(appTar, "w:gz") myappTar.add(localWhizardFolderRel) myappTar.close() md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest() gLogger.notice("...Done") gLogger.notice("Registering new Tarball in CS") tarballurl = {} av_platforms = gConfig.getSections(softwareSection, []) if av_platforms['OK']: if platform not in av_platforms['Value']: gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value']))) gLogger.error("If yours is missing add it in CS") dexit(255) else: gLogger.error("Could not find all platforms available in CS") dexit(255) av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), []) if not av_apps['OK']: gLogger.error("Could not find all applications available in CS") dexit(255) if appName.lower() in av_apps['Value']: versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()), []) if not versions['OK']: gLogger.error("Could not find all versions available in CS") dexit(255) if appVersion in versions['Value']: gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform)) dexit(0) else: result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion), os.path.basename(appTar)) if result['OK']: modifiedCS = True tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "") if len(tarballurl['Value']) > 0: res = upload(tarballurl['Value'], appTar) if not res['OK']: gLogger.error("Upload to %s failed" % tarballurl['Value']) dexit(255) result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion), md5sum) if result['OK']: modifiedCS = True result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection, platform, appName.lower(), appVersion), beam_spectra_version) else: result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion), os.path.basename(appTar)) if result['OK']: modifiedCS = True tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "") if len(tarballurl['Value']) > 0: res = upload(tarballurl['Value'], appTar) if not res['OK']: gLogger.error("Upload to %s failed" % tarballurl['Value']) dexit(255) result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion), md5sum) result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection, platform, appName.lower(), appVersion), beam_spectra_version) gLogger.verbose("Done uploading the tar ball") os.remove(appTar) #Set for all new processes the TarBallURL for process in inputlist.keys(): inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar) pl.updateProcessList(inputlist) pl.writeProcessList() raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!") pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion) #Commit the changes if nothing has failed and the CS has been modified if modifiedCS: result = diracAdmin.csCommitChanges(False) gLogger.verbose(result) gLogger.notice('All done OK!') dexit(0)