def getNewLDLibs(platform, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new LD_LIBRARY_PATH """ log = gLogger.getSubLogger("GetLDLibs") log.verbose("Getting all lib folders") new_ld_lib_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep["app"], dep['version']) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if "LD_LIBRARY_PATH" in os.environ: if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def GetNewLDLibs(systemConfig, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included @param systemConfig: System config used for the job @param application: name of the application considered @param applicationVersion: version of the application considered @return: new LD_LIBRARY_PATH """ new_ld_lib_path = "" deps = resolveDepsTar(systemConfig, application, applicationVersion) for dep in deps: depfolder = dep.replace(".tgz","").replace(".tar.gz","") res = getSoftwareFolder(depfolder) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): gLogger.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): gLogger.verbose("Found lib folder in %s" % (depfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.environ.has_key("LD_LIBRARY_PATH"): if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def configure(app, area, res_from_check): """ Configure our applications: set the proper env variables """ ########################################### ###Go where the software is to be installed os.chdir(area) #We go back to the initial place either at the end of the installation or at any error ########################################### appName = app[0].lower() ### Set env variables basefolder = res_from_check[0] removeLibc(os.path.join(os.getcwd(), basefolder, "LDLibs")) libFolder = os.path.join(os.getcwd(), basefolder, "lib") if os.path.isdir(libFolder): removeLibc(os.path.join(libFolder)) addFolderToLdLibraryPath(libFolder) if appName == "slic": configureSlic(basefolder) elif appName == "root": configureRoot(basefolder) elif appName == 'java': configureJava(basefolder) elif appName == "lcio": res = checkJava() if not res['OK']: return res configureLCIO(basefolder) elif appName in ["lcsim", 'stdhepcutjava']: res = checkJava() if not res['OK']: return res return S_OK()
def configure(app, area, res_from_check): """ Configure our applications: set the proper env variables """ ########################################### ###Go where the software is to be installed os.chdir(area) #We go back to the initial place either at the end of the installation or at any error ########################################### appName = app[0].lower() ### Set env variables basefolder = res_from_check[0] removeLibc( os.path.join( os.getcwd(), basefolder, "LDLibs" ) ) libFolder = os.path.join( os.getcwd(), basefolder, "lib" ) if os.path.isdir( libFolder ): removeLibc( os.path.join( libFolder ) ) addFolderToLdLibraryPath( libFolder ) if appName == "slic": configureSlic(basefolder) elif appName == "root": configureRoot(basefolder) elif appName == 'java': configureJava(basefolder) elif appName == "lcio": res = checkJava() if not res['OK']: return res configureLCIO(basefolder) elif appName in ["lcsim",'stdhepcutjava']: res = checkJava() if not res['OK']: return res return S_OK()
def getEnvScript(self, sysconfig, appname, appversion): """ Produce the environment file in case CVMFS is not here """ env_script_name = "SLICPandora.sh" res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: self.setApplicationStatus( 'SLICPandora: Could not find neither local area not shared area install' ) return res myslicPandoraDir = res['Value'] ##Remove libc lib removeLibc(myslicPandoraDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion) new_path = getNewPATH(sysconfig, appname, appversion) prefixpath = "" if os.path.exists("PandoraFrontend"): prefixpath = "." elif os.path.exists("%s/Executable/PandoraFrontend" % myslicPandoraDir): prefixpath = "%s/Executable" % myslicPandoraDir else: return S_ERROR("Missing PandoraFrontend binary") with open(env_script_name, "w") as script: script.write('#!/bin/sh \n') script.write( '############################################################\n' ) script.write( '# Dynamically generated script to get the SLICPandora env. #\n' ) script.write( '############################################################\n' ) script.write("declare -x PATH=%s:$PATH\n" % new_path) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir)) script.write( 'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myslicPandoraDir, new_ld_lib_path)) script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' % myslicPandoraDir) script.write("declare -x PATH=%s:$PATH\n" % prefixpath) os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def getEnvScript(self, sysconfig, appname, appversion): """ Produce the environment file in case CVMFS is not here """ env_script_name = "SLICPandora.sh" res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install') return res myslicPandoraDir = res['Value'] ##Remove libc lib removeLibc(myslicPandoraDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion) new_path = getNewPATH(sysconfig, appname, appversion) script = open(env_script_name, "w") script.write('#!/bin/sh \n') script.write('############################################################\n') script.write('# Dynamically generated script to get the SLICPandora env. #\n') script.write('############################################################\n') script.write("declare -x PATH=%s:$PATH\n" % new_path ) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir)) script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myslicPandoraDir, new_ld_lib_path)) script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' % myslicPandoraDir) prefixpath = "" if os.path.exists("PandoraFrontend"): prefixpath = "." elif os.path.exists("%s/Executable/PandoraFrontend" % myslicPandoraDir): prefixpath ="%s/Executable" % myslicPandoraDir else: return S_ERROR("Missing PandoraFrontend binary") script.write("declare -x PATH=%s:$PATH\n" % prefixpath ) script.close() os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def test_remove_libc_nothing_to_remove(self): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock: result = removeLibc('mytestpath') self.assertTrue(result) assertMockCalls(chdir_mock, ['mytestpath', 'current_dir'], self) listdir_mock.assert_called_once_with('mytestpath') self.assertFalse(remove_mock.called) assertMockCalls(getcwd_mock, [()] * 2, self)
def test_remove_libc_remove_fails(self): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock: result = removeLibc('mytestpath') self.assertFalse(result) assertMockCalls(chdir_mock, ['current_dir', 'mytestpath'], self) listdir_mock.assert_called_once_with('mytestpath') remove_mock.assert_called_once_with('mytestpath/libc.so') assertMockCalls(getcwd_mock, [()] * 3, self)
def test_remove_libc_remove_fails( self ): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock: result = removeLibc( 'mytestpath' ) self.assertFalse( result ) assertMockCalls( chdir_mock, [ 'current_dir', 'mytestpath' ], self ) listdir_mock.assert_called_once_with( 'mytestpath' ) remove_mock.assert_called_once_with( 'mytestpath/libc.so' ) assertMockCalls( getcwd_mock, [ () ] * 3, self )
def test_remove_libc_nothing_to_remove( self ): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock: result = removeLibc( 'mytestpath' ) self.assertTrue( result ) assertMockCalls( chdir_mock, [ 'mytestpath', 'current_dir' ], self ) listdir_mock.assert_called_once_with( 'mytestpath' ) self.assertFalse( remove_mock.called ) assertMockCalls( getcwd_mock, [ () ] * 2, self )
def getEnvScript(self, sysconfig, appname, appversion): """ Called if CVMFS is not available """ res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: self.setApplicationStatus('Marlin: Could not find neither local area not shared area install') return res myMarlinDir = res['Value'] ##Remove libc removeLibc(myMarlinDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion) marlindll = "" if os.path.exists("%s/MARLIN_DLL" % myMarlinDir): for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir): marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":" marlindll = "%s" % (marlindll) else: self.log.error('MARLIN_DLL folder not found, cannot proceed') return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir) env_script_name = "MarlinEnv.sh" script = open(env_script_name, "w") script.write("#!/bin/sh\n") script.write('##########################################################\n') script.write('# Dynamically generated script to create env for Marlin. #\n') script.write('##########################################################\n') script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir)) script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path)) script.write("declare -x MARLIN_DLL=%s\n" % marlindll) script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir) script.close() #os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def getEnvScript(self, sysconfig, appname, appversion): """ Called if CVMFS is not available """ res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: self.setApplicationStatus('Marlin: Could not find neither local area not shared area install') return res myMarlinDir = res['Value'] ##Remove libc removeLibc(myMarlinDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion) marlindll = "" if os.path.exists("%s/MARLIN_DLL" % myMarlinDir): for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir): marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":" marlindll = "%s" % (marlindll) else: LOG.error('MARLIN_DLL folder not found, cannot proceed') return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir) env_script_name = "MarlinEnv.sh" script = open(env_script_name, "w") script.write("#!/bin/sh\n") script.write('##########################################################\n') script.write('# Dynamically generated script to create env for Marlin. #\n') script.write('##########################################################\n') script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir)) script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path)) script.write("declare -x MARLIN_DLL=%s\n" % marlindll) script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir) script.close() #os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def getNewLDLibs(platform, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new LD_LIBRARY_PATH """ log = gLogger.getSubLogger("GetLDLibs") log.verbose("Getting all lib folders") new_ld_lib_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep["app"], dep['version']) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if "LD_LIBRARY_PATH" in os.environ: if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ[ "LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def test_remove_libc( self ): with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \ patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \ patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \ patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock: result = removeLibc( 'mytestpath' ) self.assertTrue( result ) chdir_mock.assert_any_call( 'mytestpath' ) chdir_mock.assert_called_with( 'current_dir' ) assertEqualsImproved( len( chdir_mock.mock_calls ), 2, self ) listdir_mock.assert_called_once_with( 'mytestpath' ) remove_mock.assert_any_call( 'mytestpath/libc.so' ) remove_mock.assert_called_with( 'mytestpath/libstdc++.so' ) assertEqualsImproved( len( remove_mock.mock_calls ), 2, self ) getcwd_mock.assert_called_with() assertEqualsImproved( len( getcwd_mock.mock_calls ), 4, self )
def runIt(self): """ Called by Agent Executes the following - resolve input variables - resolve installation location - resolve dependencies location (beam_spectra) - get processlist if needed - define output file name - prepare whizard.in - make magic :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'Whizard should not proceed as previous step did not end properly' ) #if self.debug: # self.excludeAllButEventString = False res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res mySoftDir = res['Value'] ###Remove libc removeLibc(mySoftDir + "/lib") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) #Don't forget to prepend the application's libs new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path ### Resolve dependencies (look for beam_spectra) deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion) path_to_beam_spectra = "" path_to_gridfiles = "" for dep in deps: res = getSoftwareFolder(self.platform, dep["app"], dep['version']) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res depfolder = res['Value'] if dep["app"] == "beam_spectra": path_to_beam_spectra = depfolder elif dep["app"] == "gridfiles": path_to_gridfiles = depfolder ##Env variables needed to run whizard: avoids hard coded locations os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000" os.environ[ 'PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000" os.environ[ 'PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000" os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000" os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000" os.environ[ 'LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000" os.environ[ 'LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000" os.environ[ 'LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000" os.environ[ 'LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000" list_of_gridfiles = [] if path_to_gridfiles and self.useGridFiles: tmp_list_of_gridfiles = [ os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles) ] gridfilesfound = False for path in tmp_list_of_gridfiles: if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files list_of_gridfiles = [ os.path.join(path, item) for item in os.listdir(path) ] gridfilesfound = True self.log.info('Found grid files specific for energy %s' % self.energy) break if not gridfilesfound: self.log.info( "Will use generic grid files found, hope the energy is set right" ) list_of_gridfiles = [ item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc")) ] template = False if self.SteeringFile.count("template"): template = True ## Get from process file the proper whizard.in file if self.getProcessInFile: whizardin = "" res = self.obtainProcessList() if not res['OK']: self.log.error("Could not obtain process list") self.setApplicationStatus('Failed getting processlist') return res whizardin = self.processlist.getInFile(self.evttype) if not whizardin: self.log.error( "Whizard input file was not found in process list, cannot proceed" ) self.setApplicationStatus('Whizard input file was not found') return S_ERROR("Error while resolving whizard input file") if whizardin.count("template"): template = True try: shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in") self.SteeringFile = "whizardnew.in" except EnvironmentError: self.log.error("Could not copy %s from %s" % (whizardin, mySoftDir)) self.setApplicationStatus('Failed getting whizard.in file') return S_ERROR("Failed to obtain %s" % whizardin) ##Check existence of Les Houches input file leshouchesfiles = '' if not os.path.exists("LesHouches.msugra_1.in"): if self.susymodel: if self.susymodel == 1: if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % ( mySoftDir) if self.susymodel == 2: if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % ( mySoftDir) if self.Model: if self.genmodel.hasModel(self.Model)['OK']: if self.genmodel.getFile(self.Model)['OK']: if os.path.exists( "%s/%s" % (mySoftDir, self.genmodel.getFile( self.Model)['Value'])): leshouchesfiles = "%s/%s" % ( mySoftDir, self.genmodel.getFile( self.Model)['Value']) else: self.log.error( "Request LesHouches file is missing, cannot proceed" ) self.setApplicationStatus( "LesHouches file missing") return S_ERROR( "The LesHouches file was not found. Probably you are using a wrong version of whizard." ) else: self.log.warn("No file found attached to model %s" % self.Model) else: self.log.error("Model undefined:", self.Model) self.setApplicationStatus("Model undefined") return S_ERROR("No Model %s defined" % self.Model) else: leshouchesfiles = "LesHouches.msugra_1.in" outputfilename = self.evttype if self.optionsdict: self.log.info("Using: %s" % self.optionsdict) self.options = WhizardOptions(self.Model) res = self.options.changeAndReturn(self.optionsdict) if not res['OK']: return res res = self.options.toWhizardDotIn("whizard.in") elif not template: res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy, self.RandomSeed, self.NumberOfEvents, self.Lumi, "whizard.in") else: res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename, self.parameters, "whizard.in") if not res['OK']: self.log.error('Something went wrong with input file generation') self.setApplicationStatus( 'Whizard: something went wrong with input file generation') return S_ERROR( 'Something went wrong with whizard.in file generation') foundproceesinwhizardin = res['Value'] scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write('declare -x PATH=%s:$PATH\n' % mySoftDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path) script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write('echo Printing content of whizard.in \n') script.write('cat whizard.in\n') script.write('echo =============================\n') script.write('cp %s/whizard.mdl ./\n' % mySoftDir) if leshouchesfiles: if not leshouchesfiles == 'LesHouches.msugra_1.in': script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles)) script.write('ln -s LesHouches.msugra_1.in fort.71\n') if len(list_of_gridfiles): for gridfile in list_of_gridfiles: script.write('cp %s ./\n' % (gridfile)) script.write('cp %s/whizard.prc ./\n' % mySoftDir) if self.genlevelcuts: res = self.makeWhizardDotCut1() if not res['OK']: script.close() self.log.error("Could not create the cut1 file") return S_ERROR("Could not create the cut1 file") script.write('echo =============================\n') script.write('echo Printing content of whizard.prc \n') script.write('cat whizard.prc\n') script.write('echo =============================\n') extracmd = "" if not self.debug: extracmd = "2>/dev/null" comm = "" if foundproceesinwhizardin: comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % ( outputfilename) else: comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % ( self.evttype, outputfilename) comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd) self.log.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('Whizard %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=209715200) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} if not self.result['OK']: self.log.error("Failed with error %s" % self.result['Message']) if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) lumi = '' message = "" success = False ###Analyse log file with open(self.applicationLog) as logfile: for line in logfile: if line.count('! Event sample corresponds to luminosity'): elems = line.split() lumi = elems[-1] if line.count("*** Fatal error:"): status = 1 message = line break elif line.count("PYSTOP"): status = 1 message = line break elif line.count("No matrix element available"): status = 1 message = line break elif line.count("Floating point exception"): status = 1 message = line break elif line.count("Event generation finished."): success = True else: status = 0 if success: status = 0 else: status = 1 self.log.info( 'The sample generated has an equivalent luminosity of %s' % lumi) if lumi: self.workflow_commons['Luminosity'] = float(lumi) else: status = 1 ##Now care for the cross sections info = {} res = self.options.getAsDict() if os.path.exists("whizard.out") and res['OK']: full_opts_dict = res['Value'] processes = full_opts_dict['process_input']['process_id'].split() info = {} info['xsection'] = {} processes.append('sum') with open("whizard.out", "r") as inf: for line in inf: line = line.rstrip() for process in processes: if not process: continue if line.count(" %s " % process): info['xsection'][process] = {} line = line.lstrip() crosssection = line.split()[1] err_crosssection = line.split()[2] frac = line.split()[4] info['xsection'][process]['xsection'] = float( crosssection) info['xsection'][process]['err_xsection'] = float( err_crosssection) info['xsection'][process]['fraction'] = float(frac) if info: if 'Info' not in self.workflow_commons: self.workflow_commons['Info'] = info else: self.workflow_commons['Info'].update(info) self.log.info("Status after the application execution is %s" % str(status)) messageout = 'Whizard %s Successful' % (self.applicationVersion) failed = False if status != 0: self.log.error("Whizard execution completed with errors:") failed = True else: self.log.info("Whizard execution completed successfully") ###Deal with output file if len(self.OutputFile): if os.path.exists(outputfilename + ".001.stdhep"): self.log.notice("Looking for output files") ofnames = glob.glob(outputfilename + '*.stdhep') if len(ofnames) > 1: basename = self.OutputFile.split(".stdhep")[0] i = 0 for of in ofnames: i += 1 name = basename + "_" + str(i) + ".stdhep" os.rename(of, name) else: os.rename(outputfilename + ".001.stdhep", self.OutputFile) else: self.log.error( "Whizard execution did not produce a stdhep file") self.setApplicationStatus( 'Whizard %s Failed to produce STDHEP file' % (self.applicationVersion)) messageout = 'Whizard Failed to produce STDHEP file' if not self.ignoreapperrors: return S_ERROR(messageout) if failed is True: self.log.error("==================================\n StdError:\n") self.log.error(message) self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status)) self.log.error('Whizard Exited With Status %s' % (status)) messageout = 'Whizard Exited With Status %s' % (status) if not self.ignoreapperrors: return S_ERROR(messageout) else: self.setApplicationStatus(messageout) return S_OK({"OutputFile": self.OutputFile})
def runIt(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using :any:`prepareLCSIMFile` - run java and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('LCSIM should not proceed as previous step did not end properly') res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error('LCSIM was not found in either the local area or shared area:', res['Message']) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # LOG.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) retMod = self.downloadDetectorZip() if not retMod: return retMod ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: LOG.error("Could not create .lcsim folder !", str(x))
def execute(self): """ Called by Agent Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background, via a call to L{SQLWrapper} - prepare the steering fie using L{PrepareSteeringFile} - run Mokka and catch its return status @return: S_OK(), S_ERROR() """ result = self.resolveInputVariables() if not result['OK']: return result #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # self.log.info('Skip this module, failure detected in a previous step :') # self.log.info('Workflow status : %s' %(self.workflowStatus)) # self.log.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Mokka should not proceed as previous step did not end properly') cwd = os.getcwd() root = gConfig.getValue('/LocalSite/Root', cwd) self.log.info( "Executing Mokka %s" % ( self.applicationVersion )) self.log.info("Platform for job is %s" % ( self.systemConfig ) ) self.log.info("Root directory for job is %s" % ( root ) ) mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka", self.applicationVersion), '') if not mokkaDir: self.log.error('Could not get Tar ball name') return S_ERROR('Failed finding software directory') mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "") #mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone. mySoftwareRoot = '' localArea = LocalArea() sharedArea = SharedArea() if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)): mySoftwareRoot = localArea elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)): mySoftwareRoot = sharedArea else: self.log.error("Mokka: could not find installation directory!") return S_ERROR("Mokka installation could not be found") ##This is a duplication of what is above, but the above cannot easily be removed... res = getSoftwareFolder(mokkaDir) if not res['OK']: self.log.error("Mokka: could not find installation directory!") return res myMokkaDir = res['Value'] if not mySoftwareRoot: self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea, sharedArea)) return S_ERROR('Failed to discover software') ####Setup MySQL instance MokkaDBrandomName = '/tmp/MokkaDBRoot-' + GenRandString(8) #sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot) sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot) res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice) if not res['OK']: self.log.error("Failed to find the DB slice") return res result = sqlwrapper.makedirs() if not result['OK']: self.setApplicationStatus('MySQL setup failed to create directories.') return result result = sqlwrapper.mysqlSetup() if not result['OK']: self.setApplicationStatus('MySQL setup failed.') return result ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion) ##Remove libc removeLibc(myMokkaDir) ##Get the particle.tbl, if any path_to_particle_tbl = '' dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles') if os.path.exists(dir_to_particletbl): configdir = os.listdir(dir_to_particletbl) if 'particle.tbl' in configdir: path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl') if os.path.exists('./particle.tbl'): path_to_particle_tbl = "./particle.tbl" ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: shutil.move("mokka.steer", "mymokka.steer") except: self.log.error("Failed renaming the steering file") self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") result = sqlwrapper.mysqlCleanUp() return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion) if not res['OK']: result = sqlwrapper.mysqlCleanUp() return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except Exception, x: result = sqlwrapper.mysqlCleanUp() return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self): """ Called by Agent Executes the following - resolve input variables - resolve installation location - resolve dependencies location (beam_spectra) - get processlist if needed - define output file name - prepare whizard.in - make magic :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Whizard should not proceed as previous step did not end properly') #if self.debug: # self.excludeAllButEventString = False res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res mySoftDir = res['Value'] ###Remove libc removeLibc(mySoftDir + "/lib") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) #Don't forget to prepend the application's libs new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path ### Resolve dependencies (look for beam_spectra) deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion) path_to_beam_spectra = "" path_to_gridfiles = "" for dep in deps: res = getSoftwareFolder(self.platform, dep[ "app" ], dep['version']) if not res['OK']: LOG.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res depfolder = res['Value'] if dep["app"] == "beam_spectra": path_to_beam_spectra = depfolder elif dep["app"] == "gridfiles": path_to_gridfiles = depfolder ##Env variables needed to run whizard: avoids hard coded locations os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000" os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000" os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000" os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000" os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000" os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000" os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000" os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000" os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000" list_of_gridfiles = [] if path_to_gridfiles and self.useGridFiles: tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)] gridfilesfound = False for path in tmp_list_of_gridfiles: if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)] gridfilesfound = True LOG.info('Found grid files specific for energy %s' % self.energy) break if not gridfilesfound: LOG.info("Will use generic grid files found, hope the energy is set right") list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))] template = False if self.SteeringFile.count("template"): template = True ## Get from process file the proper whizard.in file if self.getProcessInFile: whizardin = "" res = self.obtainProcessList() if not res['OK']: LOG.error("Could not obtain process list") self.setApplicationStatus('Failed getting processlist') return res whizardin = self.processlist.getInFile(self.evttype) if not whizardin: LOG.error("Whizard input file was not found in process list, cannot proceed") self.setApplicationStatus('Whizard input file was not found') return S_ERROR("Error while resolving whizard input file") if whizardin.count("template"): template = True try: shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in") self.SteeringFile = "whizardnew.in" except EnvironmentError: LOG.error("Could not copy %s from %s" % (whizardin, mySoftDir)) self.setApplicationStatus('Failed getting whizard.in file') return S_ERROR("Failed to obtain %s" % whizardin) ##Check existence of Les Houches input file leshouchesfiles = '' if not os.path.exists("LesHouches.msugra_1.in"): if self.susymodel: if self.susymodel == 1: if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir) if self.susymodel == 2: if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir) if self.Model: if self.genmodel.hasModel(self.Model)['OK']: if self.genmodel.getFile(self.Model)['OK']: if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])): leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value']) else: LOG.error("Request LesHouches file is missing, cannot proceed") self.setApplicationStatus("LesHouches file missing") return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.") else: LOG.warn("No file found attached to model %s" % self.Model) else: LOG.error("Model undefined:", self.Model) self.setApplicationStatus("Model undefined") return S_ERROR("No Model %s defined" % self.Model) else: leshouchesfiles = "LesHouches.msugra_1.in" outputfilename = self.evttype if self.optionsdict: LOG.info("Using: %s" % self.optionsdict) self.options = WhizardOptions(self.Model) res = self.options.changeAndReturn(self.optionsdict) if not res['OK']: return res res = self.options.toWhizardDotIn("whizard.in") elif not template: res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy, self.RandomSeed, self.NumberOfEvents, self.Lumi, "whizard.in") else: res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename, self.parameters, "whizard.in") if not res['OK']: LOG.error('Something went wrong with input file generation') self.setApplicationStatus('Whizard: something went wrong with input file generation') return S_ERROR('Something went wrong with whizard.in file generation') foundproceesinwhizardin = res['Value'] scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write('declare -x PATH=%s:$PATH\n' % mySoftDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path) script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write('echo Printing content of whizard.in \n') script.write('cat whizard.in\n') script.write('echo =============================\n') script.write('cp %s/whizard.mdl ./\n' % mySoftDir) if leshouchesfiles: if not leshouchesfiles == 'LesHouches.msugra_1.in': script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles)) script.write('ln -s LesHouches.msugra_1.in fort.71\n') if len(list_of_gridfiles): for gridfile in list_of_gridfiles: script.write('cp %s ./\n' % (gridfile)) script.write('cp %s/whizard.prc ./\n' % mySoftDir) if self.genlevelcuts: res = self.makeWhizardDotCut1() if not res['OK']: script.close() LOG.error("Could not create the cut1 file") return S_ERROR("Could not create the cut1 file") script.write('echo =============================\n') script.write('echo Printing content of whizard.prc \n') script.write('cat whizard.prc\n') script.write('echo =============================\n') extracmd = "" if not self.debug: extracmd = "2>/dev/null" comm = "" if foundproceesinwhizardin: comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (outputfilename) else: comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (self.evttype, outputfilename) comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd) LOG.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} if not self.result['OK']: LOG.error("Failed with error %s" % self.result['Message']) if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) lumi = '' message = "" success = False ###Analyse log file with open(self.applicationLog) as logfile: for line in logfile: if line.count('! Event sample corresponds to luminosity'): elems = line.split() lumi = elems[-1] if line.count("*** Fatal error:"): status = 1 message = line break elif line.count("PYSTOP"): status = 1 message = line break elif line.count("No matrix element available"): status = 1 message = line break elif line.count("Floating point exception"): status = 1 message = line break elif line.count("Event generation finished."): success = True else: status = 0 if success: status = 0 else: status = 1 LOG.info('The sample generated has an equivalent luminosity of %s' % lumi) if lumi: self.workflow_commons['Luminosity'] = float(lumi) else: status = 1 ##Now care for the cross sections info = {} res = self.options.getAsDict() if os.path.exists("whizard.out") and res['OK']: full_opts_dict = res['Value'] processes = full_opts_dict['process_input']['process_id'].split() info = {} info['xsection'] = {} processes.append('sum') with open("whizard.out", "r") as inf: for line in inf: line = line.rstrip() for process in processes: if not process: continue if line.count(" %s " % process): info['xsection'][process] = {} line = line.lstrip() crosssection = line.split()[1] err_crosssection = line.split()[2] frac = line.split()[4] info['xsection'][process]['xsection'] = float(crosssection) info['xsection'][process]['err_xsection'] = float(err_crosssection) info['xsection'][process]['fraction'] = float(frac) if info: if 'Info' not in self.workflow_commons: self.workflow_commons['Info'] = info else: self.workflow_commons['Info'].update(info) LOG.info("Status after the application execution is %s" % str(status)) messageout = 'Whizard %s Successful' % (self.applicationVersion) failed = False if status != 0: LOG.error("Whizard execution completed with errors:") failed = True else: LOG.info("Whizard execution completed successfully") ###Deal with output file if len(self.OutputFile): if os.path.exists(outputfilename + ".001.stdhep"): LOG.notice("Looking for output files") ofnames = glob.glob(outputfilename+'*.stdhep') if len(ofnames) > 1: basename = self.OutputFile.split(".stdhep")[0] i = 0 for of in ofnames: i += 1 name = basename + "_" + str(i) + ".stdhep" os.rename(of, name) else: os.rename(outputfilename + ".001.stdhep", self.OutputFile) else: LOG.error("Whizard execution did not produce a stdhep file") self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion)) messageout = 'Whizard Failed to produce STDHEP file' if not self.ignoreapperrors: return S_ERROR(messageout) if failed is True: LOG.error("==================================\n StdError:\n") LOG.error(message) self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status)) LOG.error('Whizard Exited With Status %s' % (status)) messageout = 'Whizard Exited With Status %s' % (status) if not self.ignoreapperrors: return S_ERROR(messageout) else: self.setApplicationStatus(messageout) return S_OK( { "OutputFile": self.OutputFile } )
def execute(self): """ Run the thing """ # Get input variables result = self.resolveInputVariables() # Checks if not self.platform: result = S_ERROR( 'No ILC platform selected' ) if not os.environ.has_key("LCIO"): self.log.error("Environment variable LCIO was not defined, cannot do anything") result = S_ERROR("Environment variable LCIO was not defined, cannot do anything") if not result['OK']: self.log.error("Failed to resolve the input parameters:", self.result["Message"]) return result # removeLibc removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) # Setting up script LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" ) if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = Template(''' #!/bin/sh #------------------------------------------------------------------------------# # Dynamically generated script by CheckCollections module # #------------------------------------------------------------------------------# declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH_ declare -x PATH=$PATH_ python <<PYTHONSCRIPT import sys, subprocess exitStatus = 0 for file in $files: cmdResult = subprocess.Popen( ["lcio", "count", file], stdout=subprocess.PIPE ).communicate()[ 0 ] numberOfEvents = int( cmdResult.strip().split()[1] ) cmdAnajobResult = subprocess.Popen( ["anajob", file], stdout=subprocess.PIPE ).communicate()[ 0 ] for collection in $collections: cmdResult = subprocess.Popen( ["grep", "-c", collection], stdin=subprocess.PIPE, stdout=subprocess.PIPE ).communicate( cmdAnajobResult )[ 0 ] numberOfCollections = int( cmdResult.strip() ) if numberOfEvents != numberOfCollections: print 'Inconsistency in %s: %i events vs %i collections (%s)' % ( file, numberOfEvents, numberOfCollections, collection ) exitStatus = 1 #sys.exit( exitStatus ) sys.exit( exitStatus ) PYTHONSCRIPT declare -x appstatus=$$? exit $$appstatus ''') scriptContent = scriptContent.substitute( LD_LIBRARY_PATH_ = LD_LIBRARY_PATH, PATH_ = PATH, files = self.InputFile, collections = self.collections ) # Write script to file scriptPath = 'CheckCollections_%s_Run_%s' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists( self.applicationLog ): os.remove( self.applicationLog ) # Run code os.chmod( scriptPath, 0755 ) command = '"./%s"' % ( scriptPath ) self.setApplicationStatus( 'CheckCollections %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' self.result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = self.result['Value'] status = resultTuple[0] self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def execute(self): """ Called from Workflow """ # Get input variables self.result = self.resolveInputVariables() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) if not self.result['OK']: self.log.error('Failed to resolve input parameters:', self.result['Message']) return self.result if not os.environ.has_key("LCIO"): self.log.error("Environment variable LCIO was not defined, cannot do anything") return S_ERROR("Environment variable LCIO was not defined, cannot do anything") # removeLibc removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) # Setting up script LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" ) if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by StdHepConverter module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s for STDHEPFILE in *.stdhep; do stdhepjob $STDHEPFILE ${STDHEPFILE/.stdhep/.slcio} -1 done exit $? """ % ( LD_LIBRARY_PATH, PATH ) # Write script to file scriptPath = 'StdHepConverter_%s_Run_%s.sh' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod( scriptPath, 0755 ) command = 'sh -c "./%s"' % ( scriptPath ) self.setApplicationStatus( 'StdHepConverter %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' self.result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = self.result['Value'] status = resultTuple[0] return self.finalStatusReport(status)
def execute(self): """ Execute the module, called by JobAgent """ # Checks resultRIV = self.resolveInputVariables() if not resultRIV['OK']: self.log.error("Failed to resolve input variables", resultRIV['Message']) return resultRIV if not self.platform: return S_ERROR('No ILC platform selected') if "LCIO" not in os.environ: self.log.error( "Environment variable LCIO was not defined, cannot do anything" ) return S_ERROR( "Environment variable LCIO was not defined, cannot do anything" ) if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Missing slcio file!") self.setApplicationStatus( 'LCIOSplit: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] else: return S_OK("No files found to process") removeLibc(os.path.join(os.environ["LCIO"], "lib")) # Setting up script LD_LIBRARY_PATH = os.path.join("$LCIO", "lib") if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s lcio split -i %s -n %s exit $? """ % (LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice) # Write script to file scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptPath): os.remove(scriptPath) script = open(scriptPath, 'w') script.write(scriptContent) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod(scriptPath, 0755) command = '"./%s"' % (scriptPath) self.setApplicationStatus('LCIOSplit %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' result = shellCall(0, command, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) # Check results status = result['Value'][0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0] output_file_base_name = '' if self.OutputFile: output_file_base_name = self.OutputFile.split('.slcio')[0] self.log.info("Will rename all files using '%s' as base." % output_file_base_name) numberofeventsdict = {} fname = '' with open(self.applicationLog, "r") as logf: for line in logf: line = line.rstrip() if line.count(baseinputfilename): #First, we need to rename those guys current_file = os.path.basename(line).replace(".slcio", "") current_file_extension = current_file.replace( baseinputfilename, "") newfile = output_file_base_name + current_file_extension + ".slcio" os.rename(line, newfile) fname = newfile numberofeventsdict[fname] = 0 elif line.count("events"): numberofeventsdict[fname] = int(line.split()[0]) self.log.verbose("Number of eventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: needed for #the registerOutputData self.workflow_commons[ 'file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for fileName in numberofeventsdict: item = {} item['outputFile'] = fileName item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if 'ProductionOutputData' in self.workflow_commons: proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(output_file_base_name): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for fileName in numberofeventsdict: finalproddata.append(os.path.join(path, fileName)) self.workflow_commons['ProductionOutputData'] = ";".join( finalproddata) self.log.info("Status after the application execution is %s" % str(status)) self.listDir() return self.finalStatusReport(status)
def configure(app, area, res_from_check): """ Configure our applications: set the proper env variables """ ########################################### ###Go where the software is to be installed os.chdir(area) #We go back to the initial place either at the end of the installation or at any error ########################################### appName = app[0].lower() ### Set env variables basefolder = res_from_check[0] removeLibc(os.path.join(os.getcwd(), basefolder) + "/LDLibs") if os.path.isdir(os.path.join(os.getcwd(), basefolder) + "/lib"): removeLibc(os.path.join(os.getcwd(), basefolder) + "/lib") if os.environ.has_key('LD_LIBRARY_PATH'): os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib:" + os.environ['LD_LIBRARY_PATH'] else: os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib" if appName == "slic": os.environ['SLIC_DIR'] = basefolder slicv = '' lcddv = '' xercesv = '' try: slicv = os.listdir(os.path.join(basefolder, 'packages/slic/'))[0] lcddv = os.listdir(os.path.join(basefolder, 'packages/lcdd/'))[0] if os.path.exists(os.path.join(basefolder, 'packages/xerces/')): xercesv = os.listdir(os.path.join(basefolder, 'packages/xerces/'))[0] except: return S_ERROR("Could not resolve slic env variables, folder content does not match usual pattern") #for mem in members: # if mem.name.find('/packages/slic/')>0: # slicv = mem.name.split("/")[3] # if mem.name.find('/packages/lcdd/')>0: # lcddv = mem.name.split("/")[3] # if mem.name.find('/packages/xerces/')>0: # xercesv = mem.name.split("/")[3] if slicv: os.environ['SLIC_VERSION'] = slicv if xercesv: os.environ['XERCES_VERSION'] = xercesv if lcddv: os.environ['LCDD_VERSION'] = lcddv elif appName == "root": #members = app_tar_to_untar.getmembers() #fileexample = members[0].name #fileexample.split("/")[0] os.environ['ROOTSYS'] = os.path.join(os.getcwd(), basefolder) if os.environ.has_key('LD_LIBRARY_PATH'): os.environ['LD_LIBRARY_PATH'] = os.environ['ROOTSYS'] + "/lib:" + os.environ['LD_LIBRARY_PATH'] else: os.environ['LD_LIBRARY_PATH'] = os.environ['ROOTSYS'] + "/lib" os.environ['PATH'] = os.environ['ROOTSYS'] + "/bin:" + os.environ['PATH'] os.environ['PYTHONPATH'] = os.environ['ROOTSYS'] + "/lib:" + os.environ["PYTHONPATH"] elif appName == 'java': os.environ['PATH'] = os.path.join(os.getcwd(), basefolder) + "/bin:" + os.environ['PATH'] if os.environ.has_key('LD_LIBRARY_PATH'): os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib:" + os.environ['LD_LIBRARY_PATH'] else: os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib" elif appName == "lcio": os.environ['LCIO'] = os.path.join(os.getcwd(), basefolder) os.environ['PATH'] = os.path.join(os.getcwd(), basefolder) + "/bin:" + os.environ['PATH'] res = checkJava() if not res['OK']: return res elif appName == "lcsim": res = checkJava() if not res['OK']: return res return S_OK()
def execute(self): """ Called by Agent Execute the following: - resolve where the soft was installed - prepare the list of file to feed Marlin with - create the XML file on which Marlin has to run, done by L{PrepareXMLFile} - run Marlin and catch the exit code @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) marlinDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "marlin", self.applicationVersion), '') marlinDir = marlinDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(marlinDir) if not res['OK']: self.setApplicationStatus('Marlin: Could not find neither local area not shared area install') return res myMarlinDir = res['Value'] ##Remove libc removeLibc(myMarlinDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "marlin", self.applicationVersion) res = self.GetInputFiles() if not res['OK']: self.log.error(res['Message']) return res listofslcio = res['Value'] finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml" steeringfiledirname = '' res = getSteeringFileDirName(self.systemConfig, "marlin", self.applicationVersion) if res['OK']: steeringfiledirname = res['Value'] else: self.log.warn('Could not find the steering file directory') ##Handle PandoraSettings.xml pandorasettings = 'PandoraSettings.xml' if not os.path.exists(pandorasettings): if os.path.exists(os.path.join(myMarlinDir, 'Settings', pandorasettings)): try: shutil.copy(os.path.join(myMarlinDir, 'Settings', pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except Exception, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x) elif steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname,pandorasettings)): try: shutil.copy(os.path.join(steeringfiledirname, pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except Exception, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
# //_^_/ \/_ // | \ \ # ( //) | \/// | \ \ # ( / /) _|_ / ) // | \ _\ # ( // /) '/,_ _ _/ ( ; -. | _ _\.-~ .-~~~^-. # (( / / )) ,-{ _ `-.|.-~-. .~ `. # (( // / )) '/\ / ~-. _ .-~ .-~^-. \ # (( /// )) `. { } / \ \ # (( / )) .----~-.\ \-' .~ \ `. \^-. # ///.----..> \ _ -~ `. ^-` ^-_ # ///-._ _ _ _ _ _ _}^ - - - - ~ ~-- ,.-~ # /.-~ ###### #### Hell is the maintenance of the crap below! os.chdir(self.softDir) self.log.verbose('setup local mokka database') removeLibc(self.softDir + "/mysql4grid/lib64/mysql") if os.environ.has_key('LD_LIBRARY_PATH'): os.environ['LD_LIBRARY_PATH'] = '%s/mysql4grid/lib64/mysql:%s/mysql4grid/lib64:%s' % (self.softDir, self.softDir, os.environ['LD_LIBRARY_PATH']) else: os.environ['LD_LIBRARY_PATH'] = '%s/mysql4grid/lib64/mysql:%s/mysql4grid/lib64' % (self.softDir, self.softDir) os.environ['PATH'] = '%s/mysql4grid/bin:%s' % (self.softDir, os.environ['PATH']) self.exeEnv = dict( os.environ ) safe_options = "--no-defaults --skip-networking --socket=%s/mysql.sock --datadir=%s --basedir=%s/mysql4grid --pid-file=%s/mysql.pid --log-error=%s --log=%s" % (self.MokkaTMPDir, self.MokkaDataDir, self.softDir, self.MokkaTMPDir, self.stdError, self.applicationLog)
def execute(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using L{PrepareLCSIMFile} - run java and catch the exit code @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('LCSIM should not proceed as previous step did not end properly') #look for lcsim filename lcsim_name = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig, "lcsim", self.applicationVersion), '') if not lcsim_name: self.log.error("Could not find lcsim file name from CS") return S_ERROR("Could not find lcsim file name from CS") res = getSoftwareFolder(lcsim_name) if not res['OK']: self.log.error('Application %s was not found in either the local area or shared area' % (lcsim_name)) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "lcsim", self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # self.log.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: self.log.error("Could not create .lcsim folder !", str(x))
def execute(self): """ Execute the module, called by JobAgent """ result = S_ERROR() # Get input variables result = self.resolveInputVariables() # Checks if not result['OK']: self.log.error("Failed to resolve input parameters:", result['Message']) return result if not self.platform: result = S_ERROR( 'No ILC platform selected' ) if not result['OK']: self.log.error("Failed to resolve input parameters:", result['Message']) return result if not os.environ.has_key("LCIO"): self.log.error("Environment variable LCIO was not defined, cannot do anything") return S_ERROR("Environment variable LCIO was not defined, cannot do anything") # removeLibc removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) # Setting up script LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" ) if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s lcio concat -f *.slcio -o %s exit $? """ % ( LD_LIBRARY_PATH, PATH, self.OutputFile ) # Write script to file scriptPath = 'LCIOConcatenate_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod( scriptPath, 0755 ) command = '"./%s"' % ( scriptPath ) self.setApplicationStatus( 'LCIOConcatenate %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = result['Value'] status = resultTuple[0] self.log.info( "Status after the application execution is %s" % str( status ) ) self.listDir() return self.finalStatusReport(status)
def execute(self): """ Execute the module, called by JobAgent """ # Get input variables self.result = self.resolveInputVariables() # Checks if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) if not self.result['OK']: return self.result if not os.environ.has_key("LCIO"): self.log.error("Environment variable LCIO was not defined, cannot do anything") return S_ERROR("Environment variable LCIO was not defined, cannot do anything") if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCIOSplit: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] else: return S_OK("No files found to process") # removeLibc removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) # Setting up script LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" ) if os.environ.has_key('LD_LIBRARY_PATH'): LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if os.environ.has_key('PATH'): PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s lcio split -i %s -n %s exit $? """ % ( LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice ) # Write script to file scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod( scriptPath, 0755 ) command = '"./%s"' % ( scriptPath ) self.setApplicationStatus( 'LCIOSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' self.result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = self.result['Value'] status = resultTuple[0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") logf = file(self.applicationLog,"r") baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0] output_file_base_name = '' if self.OutputFile: output_file_base_name = self.OutputFile.split('.slcio')[0] self.log.info("Will rename all files using '%s' as base." % output_file_base_name) numberofeventsdict = {} fname = '' for line in logf: line = line.rstrip() if line.count(baseinputfilename): #First, we need to rename those guys current_file = os.path.basename(line).replace(".slcio", "") current_file_extension = current_file.replace(baseinputfilename, "") newfile = output_file_base_name + current_file_extension + ".slcio" os.rename(line, newfile) fname = newfile numberofeventsdict[fname] = 0 elif line.count("events"): numberofeventsdict[fname] = int(line.split()[0]) self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: needed for #the registerOutputData self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for f in numberofeventsdict.keys(): item = {} item['outputFile'] = f item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(output_file_base_name): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for f in numberofeventsdict.keys(): finalproddata.append(os.path.join(path, f)) self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata) self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def getEnvScript(self, sysconfig, appname, appversion): """ Return the environment script in case CVMFS is not here. """ env_script_name = "MokkaEnv.sh" res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: LOG.error("Mokka: could not find installation directory!") return res myMokkaDir = res['Value'] mySoftwareRoot = os.sep.join(myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1]) ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, "mokka", self.applicationVersion) ##Remove libc removeLibc(myMokkaDir) remoteMysqlInstall = mySoftwareRoot removeLibc(remoteMysqlInstall+'/mysql4grid/lib64/mysql') script = open(env_script_name, "w") script.write('#!/bin/sh \n') script.write('######################################################\n') script.write('# Dynamically generated script to get teh Mokka env. #\n') script.write('######################################################\n') #if(os.path.exists(sharedArea+"/initILCSOFT.sh")): # script.write("%s/initILCSOFT.sh"%sharedArea) script.write("declare -x g4releases=%s\n" % (myMokkaDir)) script.write("declare -x G4SYSTEM=Linux-g++\n") script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n") #script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n") script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n') script.write('declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n') script.write('declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n') script.write('declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n') ###No such data on the GRID (???) #script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n') script.write('declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n') #script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n") script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n') script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir)) script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall)) script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir) if new_ld_lib_path: script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' % (new_ld_lib_path)) script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' % remoteMysqlInstall)#for MySQL script.write('declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n' % remoteMysqlInstall) #for MySQL script.write("if [ -e %s/%s ]; then\n" % (myMokkaDir, self.db_dump_name)) script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name) ) script.write("fi\n") #### Do something with the additional environment variables add_env = self.ops.getOptionsDict("/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" % (self.platform, "mokka", self.applicationVersion)) if add_env['OK']: for key in add_env['Value'].keys(): script.write('declare -x %s=%s/%s\n' % (key, mySoftwareRoot, add_env['Value'][key])) else: LOG.verbose("No additional environment variables needed for this application") script.close() #os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def execute(self): """ Called from Workflow """ # Get input variables self.result = self.resolveInputVariables() if not self.platform: self.result = S_ERROR('No ILC platform selected') if not self.result['OK']: self.log.error('Failed to resolve input parameters:', self.result['Message']) return self.result if 'LCIO' not in os.environ: self.log.error( "Environment variable LCIO was not defined, cannot do anything" ) return S_ERROR( "Environment variable LCIO was not defined, cannot do anything" ) # removeLibc removeLibc(os.path.join(os.environ["LCIO"], "lib")) # Setting up script LD_LIBRARY_PATH = os.path.join("$LCIO", "lib") if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by StdHepConverter module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s for STDHEPFILE in *.stdhep; do stdhepjob $STDHEPFILE ${STDHEPFILE/.stdhep/.slcio} -1 done exit $? """ % (LD_LIBRARY_PATH, PATH) # Write script to file scriptPath = 'StdHepConverter_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptPath): os.remove(scriptPath) script = open(scriptPath, 'w') script.write(scriptContent) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod(scriptPath, 0755) command = 'sh -c "./%s"' % (scriptPath) self.setApplicationStatus('StdHepConverter %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, command, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) # Check results resultTuple = self.result['Value'] status = resultTuple[0] return self.finalStatusReport(status)
def runIt(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using :any:`prepareLCSIMFile` - run java and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'LCSIM should not proceed as previous step did not end properly' ) res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( 'LCSIM was not found in either the local area or shared area:', res['Message']) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # self.log.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) retMod = self.downloadDetectorZip() if not retMod: return retMod ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: self.log.error("Could not create .lcsim folder !", str(x))
def execute(self): """ Run the thing """ # Get input variables result = self.resolveInputVariables() # Checks if not self.platform: result = S_ERROR('No ILC platform selected') if 'LCIO' not in os.environ: self.log.error( "Environment variable LCIO was not defined, cannot do anything" ) result = S_ERROR( "Environment variable LCIO was not defined, cannot do anything" ) if not result['OK']: self.log.error("Failed to resolve the input parameters:", self.result["Message"]) return result removeLibc(os.path.join(os.environ["LCIO"], "lib")) # Setting up script LD_LIBRARY_PATH = os.path.join("$LCIO", "lib") if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = Template(''' #!/bin/sh #------------------------------------------------------------------------------# # Dynamically generated script by CheckCollections module # #------------------------------------------------------------------------------# declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH_ declare -x PATH=$PATH_ python <<PYTHONSCRIPT import sys, subprocess exitStatus = 0 for file in $files: cmdResult = subprocess.Popen( ["lcio", "count", file], stdout=subprocess.PIPE ).communicate()[ 0 ] numberOfEvents = int( cmdResult.strip().split()[1] ) cmdAnajobResult = subprocess.Popen( ["anajob", file], stdout=subprocess.PIPE ).communicate()[ 0 ] for collection in $collections: cmdResult = subprocess.Popen( ["grep", "-c", collection], stdin=subprocess.PIPE, stdout=subprocess.PIPE ).communicate( cmdAnajobResult )[ 0 ] numberOfCollections = int( cmdResult.strip() ) if numberOfEvents != numberOfCollections: print 'Inconsistency in %s: %i events vs %i collections (%s)' % ( file, numberOfEvents, numberOfCollections, collection ) exitStatus = 1 #sys.exit( exitStatus ) sys.exit( exitStatus ) PYTHONSCRIPT declare -x appstatus=$$? exit $$appstatus ''') scriptContent = scriptContent.substitute( LD_LIBRARY_PATH_=LD_LIBRARY_PATH, PATH_=PATH, files=self.InputFile, collections=self.collections) # Write script to file scriptPath = 'CheckCollections_%s_Run_%s' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptPath): os.remove(scriptPath) script = open(scriptPath, 'w') script.write(scriptContent) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod(scriptPath, 0755) command = '"./%s"' % (scriptPath) self.setApplicationStatus('CheckCollections %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, command, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) # Check results resultTuple = self.result['Value'] status = resultTuple[0] self.log.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def getEnvScript(self, sysconfig, appname, appversion): """ Return the environment script in case CVMFS is not here. """ env_script_name = "MokkaEnv.sh" res = getSoftwareFolder(sysconfig, appname, appversion) if not res['OK']: self.log.error("Mokka: could not find installation directory!") return res myMokkaDir = res['Value'] mySoftwareRoot = os.sep.join( myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1]) ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, "mokka", self.applicationVersion) ##Remove libc removeLibc(myMokkaDir) remoteMysqlInstall = mySoftwareRoot removeLibc(remoteMysqlInstall + '/mysql4grid/lib64/mysql') script = open(env_script_name, "w") script.write('#!/bin/sh \n') script.write( '######################################################\n') script.write( '# Dynamically generated script to get teh Mokka env. #\n') script.write( '######################################################\n') #if(os.path.exists(sharedArea+"/initILCSOFT.sh")): # script.write("%s/initILCSOFT.sh"%sharedArea) script.write("declare -x g4releases=%s\n" % (myMokkaDir)) script.write("declare -x G4SYSTEM=Linux-g++\n") script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n") #script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n") script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n') script.write( 'declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n') script.write( 'declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n' ) script.write( 'declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n' ) ###No such data on the GRID (???) #script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n') script.write( 'declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n') #script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n") script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n') script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir)) script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall)) script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir) if new_ld_lib_path: script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' % (new_ld_lib_path)) script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' % remoteMysqlInstall) #for MySQL script.write( 'declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n' % remoteMysqlInstall) #for MySQL script.write("if [ -e %s/%s ]; then\n" % (myMokkaDir, self.db_dump_name)) script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name)) script.write("fi\n") #### Do something with the additional environment variables add_env = self.ops.getOptionsDict( "/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" % (self.platform, "mokka", self.applicationVersion)) if add_env['OK']: for key in add_env['Value'].keys(): script.write('declare -x %s=%s/%s\n' % (key, mySoftwareRoot, add_env['Value'][key])) else: self.log.verbose( "No additional environment variables needed for this application" ) script.close() #os.chmod(env_script_name, 0755) return S_OK(os.path.abspath(env_script_name))
def execute(self): """ Called from Workflow """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC Pandora should not proceed as previous step did not end properly') slicPandoraDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "slicpandora", self.applicationVersion), '') slicPandoraDir = slicPandoraDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(slicPandoraDir) if not res['OK']: self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install') return res myslicPandoraDir = res['Value'] ##Remove libc lib removeLibc(myslicPandoraDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slicpandora", self.applicationVersion) new_path = GetNewPATH(self.systemConfig, "slicpandora", self.applicationVersion) res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('SLICPandora: missing slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)): detmodel = self.detectorxml.replace("_pandora.xml", "") if os.path.exists(detmodel + ".zip"): try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) except: os.unlink(detmodel + ".zip") if not os.path.exists(detmodel + ".zip"): #retrieve detector model from web detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error('Could not find in CS the URL for detector model') return S_ERROR('Could not find in CS the URL for detector model') for detector_url in detector_urls: try: urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip") except: self.log.error("Download of detector model failed") continue try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) break except: os.unlink(detmodel + ".zip") continue #if os.path.exists(detmodel): #and os.path.isdir(detmodel): self.detectorxml = os.path.join(os.getcwd(), self.detectorxml) self.detectorxml = self.detectorxml + "_pandora.xml" if not os.path.exists(self.detectorxml): self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml) return S_ERROR('Detector model xml %s was not found, exiting' % self.detectorxml) if not os.path.exists(self.pandorasettings): if os.path.exists("./Settings/%s" % self.pandorasettings): self.pandorasettings = "./Settings/%s" % self.pandorasettings elif os.path.exists(os.path.join(myslicPandoraDir, 'Settings')): xmllist = glob.glob(os.path.join(myslicPandoraDir, 'Settings', "*.xml")) for f in xmllist: try: shutil.copy(f, os.path.join(os.getcwd(), os.path.basename(f))) except Exception, x: self.log.error('Could not copy %s, exception: %s' % (f, str(x))) return S_ERROR('Could not copy PandoraSettings file') else: self.log.error("Failed to find PandoraSettings anywhere, possibly SLICPandora install broken") return S_ERROR("Failed to find PandoraSettings anywhere")