def getNewLDLibs(platform, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new LD_LIBRARY_PATH """ log = gLogger.getSubLogger("GetLDLibs") log.verbose("Getting all lib folders") new_ld_lib_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep["app"], dep['version']) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if "LD_LIBRARY_PATH" in os.environ: if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def getNewPATH(platform, application, applicationVersion): """ Same as :func:`getNewLDLibs`,but for the PATH :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new PATH """ log = gLogger.getSubLogger("GetPaths") log.verbose("Getting all PATH folders") new_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep['app'], dep['version']) if not res['OK']: continue depfolder = res['Value'] if os.path.exists(os.path.join(depfolder, "bin")): log.verbose("Found bin folder in %s" % (depfolder)) newpathdir = os.path.join(depfolder, "bin") new_path = newpathdir if "PATH" in os.environ: if new_path: new_path = new_path + ":%s" % os.environ["PATH"] else: new_path = os.environ["PATH"] return new_path
def getEnvScript(self, sysconfig, appname, appversion): """ Called if CVMFS install is not here """ res = getSoftwareFolder(sysconfig, appname, appversion) if not res['Value']: self.setApplicationStatus('Tomato: Could not find neither local area not shared area install') return res myTomatoDir = res['Value'] deps = resolveDeps(sysconfig, "tomato", appversion) for dep in deps: if dep["app"].lower() == 'marlin': res = getSoftwareFolder(sysconfig, "marlin", dep["version"]) if not res['OK']: LOG.error('Marlin was not found in software directory') return res else: myMarlinDir = res['Value'] break env_script_name = "TomatoEnv.sh" script = open(env_script_name, "w") script.write("#!/bin/sh\n") script.write('###########################################################\n') script.write('# Dynamically generated script to get the Env for Tomato. #\n') script.write('###########################################################\n') script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir)) script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' % (myMarlinDir)) script.write("declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" % myTomatoDir) script.close() return S_OK(os.path.abspath(env_script_name))
def __init__(self, argumentsDict): """ Standard constructor Defines, from dictionary of job parameters passed, a set of members to hold e.g. the applications and the system config. Also determines the SharedArea and LocalArea. """ self.ops = Operations() self.job = argumentsDict.get('Job', {}) self.ce = argumentsDict.get('CE', {}) self.source = argumentsDict.get('Source', {}) apps = [] if self.job.has_key('SoftwarePackages'): if type( self.job['SoftwarePackages'] ) == type(''): apps = [self.job['SoftwarePackages']] elif type( self.job['SoftwarePackages'] ) == type([]): apps = self.job['SoftwarePackages'] self.jobConfig = '' if 'SystemConfig' in self.job: self.jobConfig = self.job['SystemConfig'] elif 'Platform' in self.job: self.jobConfig = self.job['Platform'] else: self.jobConfig = natOS.CMTSupportedConfig()[0] self.apps = [] for app in apps: DIRAC.gLogger.verbose( 'Requested Package %s' % app ) app = tuple(app.split('.')) if len(app) > 2: tempapp = app app = [] app.append(tempapp[0]) app.append(".".join(tempapp[1:])) deps = resolveDeps(self.jobConfig, app[0], app[1]) for dep in deps: depapp = (dep['app'], dep['version']) self.apps.append(depapp) self.apps.append(app) self.ceConfigs = [] if 'CompatiblePlatforms' in self.ce: self.ceConfigs = self.ce['CompatiblePlatforms'] if type(self.ceConfigs) == type(''): self.ceConfigs = [self.ceConfigs] #else: ### Use always the list of compatible platform. self.ceConfigs = natOS.CMTSupportedConfig() self.sharedArea = getSharedAreaLocation() DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea) self.localArea = getLocalAreaLocation() DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def __init__(self, argumentsDict): """ Standard constructor Defines, from dictionary of job parameters passed, a set of members to hold e.g. the applications and the system config. Also determines the SharedArea and LocalArea. """ self.ops = Operations() self.job = argumentsDict.get('Job', {}) self.ce = argumentsDict.get('CE', {}) self.source = argumentsDict.get('Source', {}) apps = [] if 'SoftwarePackages' in self.job: if isinstance(self.job['SoftwarePackages'], basestring): apps = [self.job['SoftwarePackages']] elif isinstance(self.job['SoftwarePackages'], list): apps = self.job['SoftwarePackages'] self.jobConfig = '' if 'SystemConfig' in self.job: self.jobConfig = self.job['SystemConfig'] elif 'Platform' in self.job: self.jobConfig = self.job['Platform'] else: self.jobConfig = natOS.CMTSupportedConfig()[0] self.apps = [] for app in apps: DIRAC.gLogger.verbose('Requested Package %s' % app) app = tuple(app.split('.')) if len(app) > 2: tempapp = app app = [] app.append(tempapp[0]) app.append(".".join(tempapp[1:])) deps = resolveDeps(self.jobConfig, app[0], app[1]) for dep in deps: depapp = (dep['app'], dep['version']) self.apps.append(depapp) self.apps.append(app) self.ceConfigs = [] if 'CompatiblePlatforms' in self.ce: self.ceConfigs = self.ce['CompatiblePlatforms'] if isinstance(self.ceConfigs, basestring): self.ceConfigs = [self.ceConfigs] #else: ### Use always the list of compatible platform. self.ceConfigs = natOS.CMTSupportedConfig() self.sharedArea = getSharedAreaLocation() DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea) self.localArea = getLocalAreaLocation() DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def installDependencies(app, config, areas): """install dependencies for application""" appName = app[0].lower() appVersion = app[1] deps = resolveDeps(config, appName, appVersion) for dep in deps: depapp = [ dep["app"], dep["version"] ] resDep = installInAnyArea(areas, depapp, config) if not resDep['OK']: return S_ERROR("Failed to install dependency: %s" % str(depapp)) return S_OK()
def installDependencies(app, config, areas): """install dependencies for application""" appName = app[0].lower() appVersion = app[1] deps = resolveDeps(config, appName, appVersion) for dep in deps: depapp = [dep["app"], dep["version"]] resDep = installInAnyArea(areas, depapp, config) if not resDep['OK']: return S_ERROR("Failed to install dependency: %s" % str(depapp)) return S_OK()
def execute(self): """ Run the module """ self.result = self.resolveInputVariables() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationName: self.result = S_ERROR("Pythia version name not given") elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error('Failed finding the software area') self.setApplicationStatus('Could find neither local area nor shared area install') return res myappDir = res['Value'] deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0] res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"]) if not res['OK']: self.log.error("Failed finding the dependency location") return res path = res['Value'] if not os.path.exists("%s.ep" % path): self.log.error('Could not find the lumi files!') return S_ERROR("Lumi files not found") originpath = "%s.ep" % path randomName = '/tmp/LumiFile-' + generateRandomString(8) try: os.mkdir(randomName) except EnvironmentError, x: self.log.error("Failed setting up the temp directory") return S_ERROR("Could not create temp dir: %s" % str(x))
def execute(self): """ Run the module """ self.result = self.resolveInputVariables() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationName: self.result = S_ERROR("Pythia version name not given") elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error('Failed finding the software area') self.setApplicationStatus('Could find neither local area nor shared area install') return res myappDir = res['Value'] deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0] res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"]) if not res['OK']: LOG.error("Failed finding the dependency location") return res path = res['Value'] if not os.path.exists("%s.ep" % path): LOG.error('Could not find the lumi files!') return S_ERROR("Lumi files not found") originpath = "%s.ep" % path randomName = '/tmp/LumiFile-' + generateRandomString(8) try: os.mkdir(randomName) except EnvironmentError, x: LOG.error("Failed setting up the temp directory") return S_ERROR("Could not create temp dir: %s" % str(x))
def getEnvScript(self, sysconfig, appname, appversion): """ Called if CVMFS install is not here """ res = getSoftwareFolder(sysconfig, appname, appversion) if not res['Value']: self.setApplicationStatus( 'Tomato: Could not find neither local area not shared area install' ) return res myTomatoDir = res['Value'] deps = resolveDeps(sysconfig, "tomato", appversion) for dep in deps: if dep["app"].lower() == 'marlin': res = getSoftwareFolder(sysconfig, "marlin", dep["version"]) if not res['OK']: self.log.error( 'Marlin was not found in software directory') return res else: myMarlinDir = res['Value'] break env_script_name = "TomatoEnv.sh" script = open(env_script_name, "w") script.write("#!/bin/sh\n") script.write( '###########################################################\n') script.write( '# Dynamically generated script to get the Env for Tomato. #\n') script.write( '###########################################################\n') script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir) script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir)) script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' % (myMarlinDir)) script.write( "declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" % myTomatoDir) script.close() return S_OK(os.path.abspath(env_script_name))
def getNewLDLibs(platform, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included :param string platform: System config used for the job :param string application: name of the application considered :param string applicationVersion: version of the application considered :return: new LD_LIBRARY_PATH """ log = gLogger.getSubLogger("GetLDLibs") log.verbose("Getting all lib folders") new_ld_lib_path = "" deps = resolveDeps(platform, application, applicationVersion) for dep in deps: res = getSoftwareFolder(platform, dep["app"], dep['version']) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): log.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if "LD_LIBRARY_PATH" in os.environ: if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ[ "LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def runIt(self): """ Called by Agent Executes the following - resolve input variables - resolve installation location - resolve dependencies location (beam_spectra) - get processlist if needed - define output file name - prepare whizard.in - make magic :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Whizard should not proceed as previous step did not end properly') #if self.debug: # self.excludeAllButEventString = False res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res mySoftDir = res['Value'] ###Remove libc removeLibc(mySoftDir + "/lib") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) #Don't forget to prepend the application's libs new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path ### Resolve dependencies (look for beam_spectra) deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion) path_to_beam_spectra = "" path_to_gridfiles = "" for dep in deps: res = getSoftwareFolder(self.platform, dep[ "app" ], dep['version']) if not res['OK']: LOG.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res depfolder = res['Value'] if dep["app"] == "beam_spectra": path_to_beam_spectra = depfolder elif dep["app"] == "gridfiles": path_to_gridfiles = depfolder ##Env variables needed to run whizard: avoids hard coded locations os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000" os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000" os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000" os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000" os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000" os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000" os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000" os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000" os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000" list_of_gridfiles = [] if path_to_gridfiles and self.useGridFiles: tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)] gridfilesfound = False for path in tmp_list_of_gridfiles: if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)] gridfilesfound = True LOG.info('Found grid files specific for energy %s' % self.energy) break if not gridfilesfound: LOG.info("Will use generic grid files found, hope the energy is set right") list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))] template = False if self.SteeringFile.count("template"): template = True ## Get from process file the proper whizard.in file if self.getProcessInFile: whizardin = "" res = self.obtainProcessList() if not res['OK']: LOG.error("Could not obtain process list") self.setApplicationStatus('Failed getting processlist') return res whizardin = self.processlist.getInFile(self.evttype) if not whizardin: LOG.error("Whizard input file was not found in process list, cannot proceed") self.setApplicationStatus('Whizard input file was not found') return S_ERROR("Error while resolving whizard input file") if whizardin.count("template"): template = True try: shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in") self.SteeringFile = "whizardnew.in" except EnvironmentError: LOG.error("Could not copy %s from %s" % (whizardin, mySoftDir)) self.setApplicationStatus('Failed getting whizard.in file') return S_ERROR("Failed to obtain %s" % whizardin) ##Check existence of Les Houches input file leshouchesfiles = '' if not os.path.exists("LesHouches.msugra_1.in"): if self.susymodel: if self.susymodel == 1: if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir) if self.susymodel == 2: if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir) if self.Model: if self.genmodel.hasModel(self.Model)['OK']: if self.genmodel.getFile(self.Model)['OK']: if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])): leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value']) else: LOG.error("Request LesHouches file is missing, cannot proceed") self.setApplicationStatus("LesHouches file missing") return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.") else: LOG.warn("No file found attached to model %s" % self.Model) else: LOG.error("Model undefined:", self.Model) self.setApplicationStatus("Model undefined") return S_ERROR("No Model %s defined" % self.Model) else: leshouchesfiles = "LesHouches.msugra_1.in" outputfilename = self.evttype if self.optionsdict: LOG.info("Using: %s" % self.optionsdict) self.options = WhizardOptions(self.Model) res = self.options.changeAndReturn(self.optionsdict) if not res['OK']: return res res = self.options.toWhizardDotIn("whizard.in") elif not template: res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy, self.RandomSeed, self.NumberOfEvents, self.Lumi, "whizard.in") else: res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename, self.parameters, "whizard.in") if not res['OK']: LOG.error('Something went wrong with input file generation') self.setApplicationStatus('Whizard: something went wrong with input file generation') return S_ERROR('Something went wrong with whizard.in file generation') foundproceesinwhizardin = res['Value'] scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write('declare -x PATH=%s:$PATH\n' % mySoftDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path) script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write('echo Printing content of whizard.in \n') script.write('cat whizard.in\n') script.write('echo =============================\n') script.write('cp %s/whizard.mdl ./\n' % mySoftDir) if leshouchesfiles: if not leshouchesfiles == 'LesHouches.msugra_1.in': script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles)) script.write('ln -s LesHouches.msugra_1.in fort.71\n') if len(list_of_gridfiles): for gridfile in list_of_gridfiles: script.write('cp %s ./\n' % (gridfile)) script.write('cp %s/whizard.prc ./\n' % mySoftDir) if self.genlevelcuts: res = self.makeWhizardDotCut1() if not res['OK']: script.close() LOG.error("Could not create the cut1 file") return S_ERROR("Could not create the cut1 file") script.write('echo =============================\n') script.write('echo Printing content of whizard.prc \n') script.write('cat whizard.prc\n') script.write('echo =============================\n') extracmd = "" if not self.debug: extracmd = "2>/dev/null" comm = "" if foundproceesinwhizardin: comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (outputfilename) else: comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (self.evttype, outputfilename) comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd) LOG.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} if not self.result['OK']: LOG.error("Failed with error %s" % self.result['Message']) if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) lumi = '' message = "" success = False ###Analyse log file with open(self.applicationLog) as logfile: for line in logfile: if line.count('! Event sample corresponds to luminosity'): elems = line.split() lumi = elems[-1] if line.count("*** Fatal error:"): status = 1 message = line break elif line.count("PYSTOP"): status = 1 message = line break elif line.count("No matrix element available"): status = 1 message = line break elif line.count("Floating point exception"): status = 1 message = line break elif line.count("Event generation finished."): success = True else: status = 0 if success: status = 0 else: status = 1 LOG.info('The sample generated has an equivalent luminosity of %s' % lumi) if lumi: self.workflow_commons['Luminosity'] = float(lumi) else: status = 1 ##Now care for the cross sections info = {} res = self.options.getAsDict() if os.path.exists("whizard.out") and res['OK']: full_opts_dict = res['Value'] processes = full_opts_dict['process_input']['process_id'].split() info = {} info['xsection'] = {} processes.append('sum') with open("whizard.out", "r") as inf: for line in inf: line = line.rstrip() for process in processes: if not process: continue if line.count(" %s " % process): info['xsection'][process] = {} line = line.lstrip() crosssection = line.split()[1] err_crosssection = line.split()[2] frac = line.split()[4] info['xsection'][process]['xsection'] = float(crosssection) info['xsection'][process]['err_xsection'] = float(err_crosssection) info['xsection'][process]['fraction'] = float(frac) if info: if 'Info' not in self.workflow_commons: self.workflow_commons['Info'] = info else: self.workflow_commons['Info'].update(info) LOG.info("Status after the application execution is %s" % str(status)) messageout = 'Whizard %s Successful' % (self.applicationVersion) failed = False if status != 0: LOG.error("Whizard execution completed with errors:") failed = True else: LOG.info("Whizard execution completed successfully") ###Deal with output file if len(self.OutputFile): if os.path.exists(outputfilename + ".001.stdhep"): LOG.notice("Looking for output files") ofnames = glob.glob(outputfilename+'*.stdhep') if len(ofnames) > 1: basename = self.OutputFile.split(".stdhep")[0] i = 0 for of in ofnames: i += 1 name = basename + "_" + str(i) + ".stdhep" os.rename(of, name) else: os.rename(outputfilename + ".001.stdhep", self.OutputFile) else: LOG.error("Whizard execution did not produce a stdhep file") self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion)) messageout = 'Whizard Failed to produce STDHEP file' if not self.ignoreapperrors: return S_ERROR(messageout) if failed is True: LOG.error("==================================\n StdError:\n") LOG.error(message) self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status)) LOG.error('Whizard Exited With Status %s' % (status)) messageout = 'Whizard Exited With Status %s' % (status) if not self.ignoreapperrors: return S_ERROR(messageout) else: self.setApplicationStatus(messageout) return S_OK( { "OutputFile": self.OutputFile } )
def runIt(self): """ Called by Agent Executes the following - resolve input variables - resolve installation location - resolve dependencies location (beam_spectra) - get processlist if needed - define output file name - prepare whizard.in - make magic :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'Whizard should not proceed as previous step did not end properly' ) #if self.debug: # self.excludeAllButEventString = False res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res mySoftDir = res['Value'] ###Remove libc removeLibc(mySoftDir + "/lib") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) #Don't forget to prepend the application's libs new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path ### Resolve dependencies (look for beam_spectra) deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion) path_to_beam_spectra = "" path_to_gridfiles = "" for dep in deps: res = getSoftwareFolder(self.platform, dep["app"], dep['version']) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res depfolder = res['Value'] if dep["app"] == "beam_spectra": path_to_beam_spectra = depfolder elif dep["app"] == "gridfiles": path_to_gridfiles = depfolder ##Env variables needed to run whizard: avoids hard coded locations os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000" os.environ[ 'PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000" os.environ[ 'PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000" os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000" os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000" os.environ[ 'LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000" os.environ[ 'LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000" os.environ[ 'LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000" os.environ[ 'LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000" list_of_gridfiles = [] if path_to_gridfiles and self.useGridFiles: tmp_list_of_gridfiles = [ os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles) ] gridfilesfound = False for path in tmp_list_of_gridfiles: if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files list_of_gridfiles = [ os.path.join(path, item) for item in os.listdir(path) ] gridfilesfound = True self.log.info('Found grid files specific for energy %s' % self.energy) break if not gridfilesfound: self.log.info( "Will use generic grid files found, hope the energy is set right" ) list_of_gridfiles = [ item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc")) ] template = False if self.SteeringFile.count("template"): template = True ## Get from process file the proper whizard.in file if self.getProcessInFile: whizardin = "" res = self.obtainProcessList() if not res['OK']: self.log.error("Could not obtain process list") self.setApplicationStatus('Failed getting processlist') return res whizardin = self.processlist.getInFile(self.evttype) if not whizardin: self.log.error( "Whizard input file was not found in process list, cannot proceed" ) self.setApplicationStatus('Whizard input file was not found') return S_ERROR("Error while resolving whizard input file") if whizardin.count("template"): template = True try: shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in") self.SteeringFile = "whizardnew.in" except EnvironmentError: self.log.error("Could not copy %s from %s" % (whizardin, mySoftDir)) self.setApplicationStatus('Failed getting whizard.in file') return S_ERROR("Failed to obtain %s" % whizardin) ##Check existence of Les Houches input file leshouchesfiles = '' if not os.path.exists("LesHouches.msugra_1.in"): if self.susymodel: if self.susymodel == 1: if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % ( mySoftDir) if self.susymodel == 2: if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % ( mySoftDir) if self.Model: if self.genmodel.hasModel(self.Model)['OK']: if self.genmodel.getFile(self.Model)['OK']: if os.path.exists( "%s/%s" % (mySoftDir, self.genmodel.getFile( self.Model)['Value'])): leshouchesfiles = "%s/%s" % ( mySoftDir, self.genmodel.getFile( self.Model)['Value']) else: self.log.error( "Request LesHouches file is missing, cannot proceed" ) self.setApplicationStatus( "LesHouches file missing") return S_ERROR( "The LesHouches file was not found. Probably you are using a wrong version of whizard." ) else: self.log.warn("No file found attached to model %s" % self.Model) else: self.log.error("Model undefined:", self.Model) self.setApplicationStatus("Model undefined") return S_ERROR("No Model %s defined" % self.Model) else: leshouchesfiles = "LesHouches.msugra_1.in" outputfilename = self.evttype if self.optionsdict: self.log.info("Using: %s" % self.optionsdict) self.options = WhizardOptions(self.Model) res = self.options.changeAndReturn(self.optionsdict) if not res['OK']: return res res = self.options.toWhizardDotIn("whizard.in") elif not template: res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy, self.RandomSeed, self.NumberOfEvents, self.Lumi, "whizard.in") else: res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename, self.parameters, "whizard.in") if not res['OK']: self.log.error('Something went wrong with input file generation') self.setApplicationStatus( 'Whizard: something went wrong with input file generation') return S_ERROR( 'Something went wrong with whizard.in file generation') foundproceesinwhizardin = res['Value'] scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write('declare -x PATH=%s:$PATH\n' % mySoftDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path) script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write('echo Printing content of whizard.in \n') script.write('cat whizard.in\n') script.write('echo =============================\n') script.write('cp %s/whizard.mdl ./\n' % mySoftDir) if leshouchesfiles: if not leshouchesfiles == 'LesHouches.msugra_1.in': script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles)) script.write('ln -s LesHouches.msugra_1.in fort.71\n') if len(list_of_gridfiles): for gridfile in list_of_gridfiles: script.write('cp %s ./\n' % (gridfile)) script.write('cp %s/whizard.prc ./\n' % mySoftDir) if self.genlevelcuts: res = self.makeWhizardDotCut1() if not res['OK']: script.close() self.log.error("Could not create the cut1 file") return S_ERROR("Could not create the cut1 file") script.write('echo =============================\n') script.write('echo Printing content of whizard.prc \n') script.write('cat whizard.prc\n') script.write('echo =============================\n') extracmd = "" if not self.debug: extracmd = "2>/dev/null" comm = "" if foundproceesinwhizardin: comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % ( outputfilename) else: comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % ( self.evttype, outputfilename) comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd) self.log.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('Whizard %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=209715200) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} if not self.result['OK']: self.log.error("Failed with error %s" % self.result['Message']) if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) lumi = '' message = "" success = False ###Analyse log file with open(self.applicationLog) as logfile: for line in logfile: if line.count('! Event sample corresponds to luminosity'): elems = line.split() lumi = elems[-1] if line.count("*** Fatal error:"): status = 1 message = line break elif line.count("PYSTOP"): status = 1 message = line break elif line.count("No matrix element available"): status = 1 message = line break elif line.count("Floating point exception"): status = 1 message = line break elif line.count("Event generation finished."): success = True else: status = 0 if success: status = 0 else: status = 1 self.log.info( 'The sample generated has an equivalent luminosity of %s' % lumi) if lumi: self.workflow_commons['Luminosity'] = float(lumi) else: status = 1 ##Now care for the cross sections info = {} res = self.options.getAsDict() if os.path.exists("whizard.out") and res['OK']: full_opts_dict = res['Value'] processes = full_opts_dict['process_input']['process_id'].split() info = {} info['xsection'] = {} processes.append('sum') with open("whizard.out", "r") as inf: for line in inf: line = line.rstrip() for process in processes: if not process: continue if line.count(" %s " % process): info['xsection'][process] = {} line = line.lstrip() crosssection = line.split()[1] err_crosssection = line.split()[2] frac = line.split()[4] info['xsection'][process]['xsection'] = float( crosssection) info['xsection'][process]['err_xsection'] = float( err_crosssection) info['xsection'][process]['fraction'] = float(frac) if info: if 'Info' not in self.workflow_commons: self.workflow_commons['Info'] = info else: self.workflow_commons['Info'].update(info) self.log.info("Status after the application execution is %s" % str(status)) messageout = 'Whizard %s Successful' % (self.applicationVersion) failed = False if status != 0: self.log.error("Whizard execution completed with errors:") failed = True else: self.log.info("Whizard execution completed successfully") ###Deal with output file if len(self.OutputFile): if os.path.exists(outputfilename + ".001.stdhep"): self.log.notice("Looking for output files") ofnames = glob.glob(outputfilename + '*.stdhep') if len(ofnames) > 1: basename = self.OutputFile.split(".stdhep")[0] i = 0 for of in ofnames: i += 1 name = basename + "_" + str(i) + ".stdhep" os.rename(of, name) else: os.rename(outputfilename + ".001.stdhep", self.OutputFile) else: self.log.error( "Whizard execution did not produce a stdhep file") self.setApplicationStatus( 'Whizard %s Failed to produce STDHEP file' % (self.applicationVersion)) messageout = 'Whizard Failed to produce STDHEP file' if not self.ignoreapperrors: return S_ERROR(messageout) if failed is True: self.log.error("==================================\n StdError:\n") self.log.error(message) self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status)) self.log.error('Whizard Exited With Status %s' % (status)) messageout = 'Whizard Exited With Status %s' % (status) if not self.ignoreapperrors: return S_ERROR(messageout) else: self.setApplicationStatus(messageout) return S_OK({"OutputFile": self.OutputFile})
def TARinstall(app, config, area): """ For the specified app, install all dependencies """ curdir = os.getcwd() appName = app[0].lower() appVersion = app[1] deps = resolveDeps(config, appName, appVersion) for dep in deps: depapp = [] depapp.append(dep["app"]) depapp.append(dep["version"]) gLogger.info("Installing dependency %s %s" % (dep["app"], dep["version"])) res = getTarBallLocation(depapp, config, area) if not res['OK']: gLogger.error("Could not install dependency %s %s: %s" % (dep["app"], dep["version"], res['Message'])) return S_ERROR('Failed to install software') res_from_getTarBall = res['Value'] app_tar = res_from_getTarBall[0] TarBallURL = res_from_getTarBall[1] overwrite = res_from_getTarBall[2] md5sum = res_from_getTarBall[3] res = install(depapp, app_tar, TarBallURL, overwrite, md5sum, area) os.chdir(curdir) if not res['OK']: gLogger.error("Could not install dependency %s %s: %s" % (dep["app"], dep["version"], res['Message'])) return S_ERROR('Failed to install software') res_from_install = res['Value'] res = check(depapp, area, res_from_install) os.chdir(curdir) if not res['OK']: gLogger.error("Failed to check dependency %s %s" % (dep["app"], dep["version"])) return S_ERROR('Failed to check integrity of software') res_from_check = res['Value'] res = configure(depapp, area, res_from_check) os.chdir(curdir) if not res['OK']: gLogger.error("Failed to configure dependency %s %s" % (dep["app"], dep["version"])) return S_ERROR('Failed to configure software') res = clean(area, res_from_install) if not res['OK']: gLogger.error("Failed to clean useless tar balls, deal with it: %s %s" % (dep["app"], dep["version"])) os.chdir(curdir) gLogger.notice("Successfully installed %s %s in %s" % (dep["app"], dep["version"], area)) res = getTarBallLocation(app, config, area) if not res['OK']: gLogger.error("Could not install dependency %s %s: %s" % (appName, appVersion, res['Message'])) return S_ERROR('Failed to install software') res_from_getTarBall = res['Value'] app_tar = res_from_getTarBall[0] TarBallURL = res_from_getTarBall[1] overwrite = res_from_getTarBall[2] md5sum = res_from_getTarBall[3] res = install(app, app_tar, TarBallURL, overwrite, md5sum, area) os.chdir(curdir) if not res['OK']: gLogger.error("Could not install software %s %s: %s" % (appName, appVersion, res['Message'])) return S_ERROR('Failed to install software') res_from_install = res['Value'] res = check(app, area, res_from_install) os.chdir(curdir) if not res['OK']: gLogger.error("Failed to check software %s %s" % (appName, appVersion)) return S_ERROR('Failed to check integrity of software') res_from_check = res['Value'] res = configure(app, area, res_from_check) os.chdir(curdir) if not res['OK']: gLogger.error("Failed to configure software %s %s" % (appName, appVersion)) return S_ERROR('Failed to configure software') res = clean(area, res_from_install) os.chdir(curdir) if not res['OK']: gLogger.error("Failed to clean useless tar balls, deal with it") return S_OK() gLogger.notice("Successfully installed %s %s in %s" % (appName, appVersion, area)) return res