def CreateDirectory(self): logging.info("\nWelcome to the expert mode of MadAnalysis") logging.info( "Please enter a folder name for creating an empty SampleAnalyzer job" ) answer = raw_input("Answer: ") answer = answer.replace(' ', '_') answer = answer.replace('-', '_') self.path = os.path.expanduser(answer) if not self.path.startswith('/'): self.path = self.main.currentdir + '/' + self.path self.path = os.path.normpath(self.path) # Checking folder if self.path in self.forbiddenpaths: logging.error("the folder '"+anwser+"' is a MadAnalysis folder. " + \ "You cannot overwrite it. Please choose another folder.") return False # Checking if the job folder exists if os.path.isdir(self.path): logging.warning("A directory called '"+self.path+"' is already "+ \ "defined.\nWould you like to remove it ? (Y/N)") allowed_answers = ['n', 'no', 'y', 'yes'] answer = "" while answer not in allowed_answers: answer = raw_input("Answer: ") answer = answer.lower() if answer == "no" or answer == "n": return False else: if not FolderWriter.RemoveDirectory(self.path, False): return False return True
def Deactivate(self): if self.main.archi_info.delphesMA5tune_lib_paths == []: return True for x in self.main.archi_info.delphesMA5tune_lib_paths: if 'DEACT' in x: return True if os.path.isdir(self.main.archi_info.delphesMA5tune_lib_paths[0]): self.logger.warning( "DelphesMA5tune is installed. Deactivating it.") # Paths delpath = os.path.normpath( self.main.archi_info.delphesMA5tune_lib_paths[0]) deldeac = delpath.replace( delpath.split('/')[-1], "DEACT_" + delpath.split('/')[-1]) self.main.archi_info.toLDPATH1 = [ x for x in self.main.archi_info.toLDPATH1 if not 'MA5tune' in x ] if 'DelphesMA5tune' in self.main.archi_info.libraries.keys(): del self.main.archi_info.libraries['DelphesMA5tune'] # If the deactivated directory already exists -> suppression if os.path.isdir(os.path.normpath(deldeac)): if not FolderWriter.RemoveDirectory(os.path.normpath(deldeac), True): return False # cleaning delphesMA5tune + the samplanalyzer interface to delphesMA5tune shutil.move(delpath, deldeac) myexts = ['so', 'a', 'dylib'] for ext in myexts: myfile = self.main.archi_info.ma5dir + '/tools/SampleAnalyzer/Lib/libdelphesMA5tune_for_ma5.' + ext if os.path.isfile(os.path.normpath(myfile)): os.remove(os.path.normpath(myfile)) ToRemove = [ 'Makefile_delphesMA5tune', 'compilation_delphesMA5tune.log', 'linking_delphesMA5tune.log', 'cleanup_delphesMA5tune.log' ] for myfile in ToRemove: if os.path.isfile( os.path.normpath(self.main.archi_info.ma5dir + '/tools/SampleAnalyzer/Interfaces/' + myfile)): os.remove( os.path.normpath(self.main.archi_info.ma5dir + '/tools/SampleAnalyzer/Interfaces/' + myfile)) self.main.archi_info.has_delphesMA5tune = False self.main.archi_info.delphesMA5tune_priority = False self.main.archi_info.delphesMA5tune_lib_paths = [] self.main.archi_info.delphesMA5tune_inc_paths = [] self.main.archi_info.delphesMA5tune_lib = "" self.main.archi_info.delphesMA5tune_original_libs = [] return True
def Remove(self,question=True): import time bkpname = "pad_forma5tune-v" + time.strftime("%Y%m%d-%Hh%M") + ".tgz" logging.getLogger('MA5').info(" => Backuping the previous installation: " + bkpname) logname = os.path.normpath(self.main.archi_info.ma5dir+'/pad-backup.log') TheCommand = ['tar', 'czf', bkpname, 'PADForMA5tune'] logging.getLogger('MA5').debug('Shell command: '+' '.join(TheCommand)) ok, out= ShellCommand.ExecuteWithLog(TheCommand,logname,self.main.archi_info.ma5dir,silent=False) if not ok: return False logging.getLogger('MA5').info(" => Backup done") from madanalysis.IOinterface.folder_writer import FolderWriter return FolderWriter.RemoveDirectory(self.installdir,question)
def CreateDirectory(self, name): if name == "": logging.getLogger('MA5').debug( "Debug mode: no directory name is specified.") logging.getLogger('MA5').info( "\nWelcome to the expert mode of MadAnalysis") logging.getLogger('MA5').info( "Please enter a folder name for creating an empty SampleAnalyzer job" ) answer = raw_input("Answer: ") answer = answer.replace(' ', '_') name = answer.replace('-', '_') else: logging.getLogger('MA5').debug( "Debug mode: a directory name is specified.") # Getting the full path self.path = os.path.expanduser(name) if not self.path.startswith('/'): if name in ['PAD', 'PADForMA5tune']: self.path = self.main.archi_info.ma5dir + '/' + self.path else: self.path = self.main.currentdir + '/' + self.path self.path = os.path.normpath(self.path) logging.getLogger('MA5').debug("Full path of the working directory: " + self.path) # Checking folder logging.getLogger('MA5').debug( "Check that the path is not forbidden...") if self.path in self.forbiddenpaths: logging.getLogger('MA5').error("the folder '"+anwser+"' is a MadAnalysis folder. " + \ "You cannot overwrite it. Please choose another folder.") return False # Checking if the job folder exists, if not remove it if os.path.isdir(self.path): logging.getLogger('MA5').debug("Remove the previous folder?") question="A directory called '"+self.path+"' is already defined.\n"+\ "Would you like to remove it ? (Y/N)" from madanalysis.IOinterface.folder_writer import FolderWriter test = FolderWriter.RemoveDirectory(self.path, question) if not test[0] or not test[1]: return False return True
def RunPAD(self, PADdir, eventfile): ## input file if os.path.isfile(PADdir + '/Input/PADevents.list'): os.remove(PADdir + '/Input/PADevents.list') infile = open(PADdir + '/Input/PADevents.list', 'w') infile.write(eventfile) infile.close() ## cleaning the output directory if not FolderWriter.RemoveDirectory( os.path.normpath(PADdir + '/Output/PADevents.list')): return False ## running command = ['./MadAnalysis5job', '../Input/PADevents.list'] ok = ShellCommand.Execute(command, PADdir + '/Build') if not ok: self.logger.error('Problem with the run of the PAD on the file: ' + eventfile) return False os.remove(PADdir + '/Input/PADevents.list') return True
def run_pad(self,eventfile): ## input file if os.path.isfile(self.pad+'/Input/PADevents.list'): os.remove(self.pad+'/Input/PADevents.list') infile = open(self.pad+'/Input/PADevents.list','w') infile.write(eventfile) infile.close() ## cleaning the output directory if not FolderWriter.RemoveDirectory(os.path.normpath(self.pad+'/Output/PADevents')): return False ## running command = ['./MadAnalysis5job', '../Input/PADevents.list'] ok = ShellCommand.Execute(command,self.pad+'/Build') ## checks if not ok: logging.getLogger('MA5').error('Problem with the run of the PAD on the file: '+ eventfile) return False os.remove(self.pad+'/Input/PADevents.list') ## exit time.sleep(1.); return True
def generate_events(self,dataset,card): # Preparing the run self.main.recasting.status="off" self.main.fastsim.package=self.detector self.main.fastsim.clustering=0 if self.detector=="delphesMA5tune": self.main.fastsim.delphes=0 self.main.fastsim.delphesMA5tune = DelphesMA5tuneConfiguration() self.main.fastsim.delphesMA5tune.card = os.path.normpath("../../../../PADForMA5tune/Input/Cards/"+card) elif self.detector=="delphes": self.main.fastsim.delphesMA5tune = 0 self.main.fastsim.delphes = DelphesConfiguration() self.main.fastsim.delphes.card = os.path.normpath("../../../../PAD/Input/Cards/"+card) # Execution if not self.run_delphes(dataset,card): logging.getLogger('MA5').error('The '+detector+' problem with the running of the fastsim') return False # Restoring the run self.main.recasting.status="on" self.main.fastsim.package="none" ## Saving the output if not os.path.isdir(self.dirname+'/Output/'+dataset.name): os.mkdir(self.dirname+'/Output/'+dataset.name) if not os.path.isdir(self.dirname+'/Output/'+dataset.name+'/RecoEvents'): os.mkdir(self.dirname+'/Output/'+dataset.name+'/RecoEvents') if self.detector=="delphesMA5tune": shutil.move(self.dirname+'_FastSimRun/Output/_'+dataset.name+'/RecoEvents0_0/DelphesMA5tuneEvents.root',\ self.dirname+'/Output/'+dataset.name+'/RecoEvents/RecoEvents_v1x1_'+card.replace('.tcl','')+'.root') elif self.detector=="delphes": shutil.move(self.dirname+'_FastSimRun/Output/_'+dataset.name+'/RecoEvents0_0/DelphesEvents.root',\ self.dirname+'/Output/'+dataset.name+'/RecoEvents/RecoEvents_v1x2_'+card.replace('.tcl','')+'.root') ## Cleaning the temporary directory if not FolderWriter.RemoveDirectory(os.path.normpath(self.dirname+'_FastSimRun')): return False ## Exit return True
def do_resubmit(self,args,history): # Start time start_time=time.time() # Checking argument number if len(args)!=0: logging.warning("Command 'resubmit' takes no argument. Any argument will be skipped.") # Checking presence of a valid job if self.main.lastjob_name is "": logging.error("an analysis must be defined and ran before using the resubmit command.") return False self.main.lastjob_status = False # Checking if new plots or cuts have been performed newhistory = [] ToAdd = False Inside = False for cmd in history: if 'submit' in cmd and 'resubmit' not in cmd: ToAdd = True Inside = True elif ToAdd: if not Inside: newhistory=[] Inside = True if 'resubmit' in cmd: Inside = False newhistory.append(cmd) ToReAnalyze = False ReAnalyzeCmdList = ['plot','select','reject','set main.clustering', 'set main.merging', 'set main.shower', 'define', 'import', 'set main.isolation'] # Determining if we have to resubmit the job for cmd in newhistory: # Split cmd line into words words = cmd.split() # Creating a line with one whitespace between each word cmd2 = '' for word in words: if word!='': cmd2+=word+' ' # Looping over patterns for pattern in ReAnalyzeCmdList: if cmd2.startswith(pattern): ToReAnalyze = True break # Found? if ToReAnalyze: break if ToReAnalyze: logging.info(" Creating the new histograms and/or applying the new cuts...") # Submission if not self.submit(self.main.lastjob_name,history): return logging.info(" Updating the reports...") else: logging.info(" No new histogram / cut to account for. Updating the reports...") # Reading info from job output layout = Layout(self.main) if not self.extract(self.main.lastjob_name,layout): return # Status = GOOD self.main.lastjob_status = True # Computing layout.Initialize() # Cleaning the directories if not FolderWriter.RemoveDirectory(self.main.lastjob_name+'/HTML',False): return if self.main.pdflatex: if not FolderWriter.RemoveDirectory(self.main.lastjob_name+'/PDF',False): return if self.main.latex: if not FolderWriter.RemoveDirectory(self.main.lastjob_name+'/DVI',False): return # Creating the reports self.CreateReports([self.main.lastjob_name],history,layout) # End time end_time=time.time() logging.info(" Well done! Elapsed time = " + CmdSubmit.chronometer_display(end_time-start_time) )
def submit(self,dirname,history): # checking if the needed version of delphes is activated forced_bkp = self.main.forced self.main.forced=True if self.main.fastsim.package == 'delphes' and not self.main.archi_info.has_delphes: installer=InstallManager(self.main) if self.main.archi_info.has_delphesMA5tune: if not installer.Deactivate('delphesMA5tune'): return False if installer.Activate('delphes')==-1: return False if self.main.fastsim.package == 'delphesMA5tune' and not self.main.archi_info.has_delphesMA5tune: installer=InstallManager(self.main) if self.main.archi_info.has_delphes: if not installer.Deactivate('delphes'): return False if installer.Activate('delphesMA5tune')==-1: return False self.main.forced=forced_bkp # Initializing the JobWriter jobber = JobWriter(self.main,dirname,self.resubmit) # Writing process if not self.resubmit: self.logger.info(" Creating folder '"+dirname.split('/')[-1] \ +"'...") else: self.logger.info(" Checking the structure of the folder '"+\ dirname.split('/')[-1]+"'...") if not jobber.Open(): self.logger.error("job submission aborted.") return False if not self.resubmit: if self.main.recasting.status != 'on': self.logger.info(" Copying 'SampleAnalyzer' source files...") if not jobber.CopyLHEAnalysis(): self.logger.error(" job submission aborted.") return False if self.main.recasting.status != 'on' and not jobber.CreateBldDir(): self.logger.error(" job submission aborted.") return False if self.main.recasting.status == 'on': if not FolderWriter.CreateDirectory(dirname+'/Events'): return False # In the case of recasting, there is no need to create a standard Job if self.main.recasting.status == "on": ### First, the analyses to take care off self.editRecastingCard(dirname) self.logger.info(" Getting the list of delphes simulation to be performed...") ### Second, which delphes run must be performed, and running them if not self.main.recasting.GetDelphesRuns(dirname+"/Input/recasting_card.dat"): return False firstv11 = True firstv13 = True forced_bkp = self.main.forced self.main.forced=True if len(self.main.recasting.delphesruns)==0: self.logger.warning('No recasting to do... Please check the recasting card') return False for mydelphescard in sorted(self.main.recasting.delphesruns): version=mydelphescard[:4] card=mydelphescard[5:] if version=="v1.1": if not self.main.recasting.ma5tune: self.logger.error('The DelphesMA5tune library is not present... v1.1 analyses cannot be used') return False if firstv11: self.logger.info(" **********************************************************") self.logger.info(" "+StringTools.Center('v1.1 detector simulations',57)) self.logger.info(" **********************************************************") firstv11=False ## Deactivating delphes installer=InstallManager(self.main) if not installer.Deactivate('delphes'): return False ## Activating and compile the MA5Tune if installer.Activate('delphesMA5tune')==-1: return False ## running delphesMA5tune for item in self.main.datasets: if not os.path.isfile(dirname + '/Events/' + item.name + '_v1x1_' + \ card.replace('.tcl','')+'.root'): self.main.recasting.status="off" self.main.fastsim.package="delphesMA5tune" self.main.fastsim.clustering=0 self.main.fastsim.delphes=0 self.main.fastsim.delphesMA5tune = DelphesMA5tuneConfiguration() self.main.fastsim.delphesMA5tune.card = os.path.normpath("../../../../PADForMA5tune/Input/Cards/"+card) self.submit(dirname+'_DelphesForMa5tuneRun',[]) self.main.recasting.status="on" self.main.fastsim.package="none" ## saving the output shutil.move(dirname+'_DelphesForMa5tuneRun/Output/_'+item.name+'/TheMouth.root',\ dirname+'/Events/'+item.name+'_v1x1_'+card.replace('.tcl','')+'.root') if not FolderWriter.RemoveDirectory(os.path.normpath(dirname+'_DelphesForMa5tuneRun')): return False elif version in ['v1.2', 'v1.3']: if not self.main.recasting.delphes: self.logger.error('The Delphes library is not present... v1.2+ analyses cannot be used') return False if firstv13: self.logger.info(" **********************************************************") self.logger.info(" "+StringTools.Center('v1.2+ detector simulations',57)) self.logger.info(" **********************************************************") firstv13=False ## Deactivating delphesMA5tune installer=InstallManager(self.main) if not installer.Deactivate('delphesMA5tune'): return False ## Activating and compile Delphes if installer.Activate('delphes')==-1: return False ## running delphesMA5tune for item in self.main.datasets: if not os.path.isfile(dirname+'/Events/' + item.name +\ '_v1x2_'+card.replace('.tcl','')+'.root'): self.main.recasting.status="off" self.main.fastsim.package="delphes" self.main.fastsim.clustering=0 self.main.fastsim.delphesMA5tune=0 self.main.fastsim.delphes = DelphesConfiguration() self.main.fastsim.delphes.card = os.path.normpath("../../../../PAD/Input/Cards/"+card) self.submit(dirname+'_DelphesRun',[]) self.main.recasting.status="on" self.main.fastsim.package="none" ## saving the output shutil.move(dirname+'_DelphesRun/Output/_'+item.name+'/TheMouth.root',\ dirname+'/Events/'+item.name+'_v1x2_'+card.replace('.tcl','')+'.root') if not FolderWriter.RemoveDirectory(os.path.normpath(dirname+'_DelphesRun')): return False else: self.logger.error('An analysis can only be compatible with ma5 v1.1, v1.2 or v1.3...') return False self.main.forced=forced_bkp ### Third, executing the analyses if not self.main.recasting.GetAnalysisRuns(dirname+"/Input/recasting_card.dat"): return False forced_bkp = self.main.forced self.main.forced=True for mydelphescard in list(set(sorted(self.main.recasting.delphesruns))): ## checking the PAD Version myversion = mydelphescard[:4] if myversion=='v1.2': PADdir = self.main.archi_info.ma5dir+'/PAD' elif myversion=='v1.1': PADdir = self.main.archi_info.ma5dir+'/PADForMA5tune' else: self.logger.error('Unknown PAD version') self.main.forced=forced_bkp return False ## current delphes card mycard = mydelphescard[5:] ## print out self.logger.info(" **********************************************************") self.logger.info(" "+StringTools.Center(myversion+' running of the PAD'+\ ' on events generated with',57)) self.logger.info(" "+StringTools.Center(mycard,57)) self.logger.info(" **********************************************************") ## setting up Delphes installer=InstallManager(self.main) if myversion=='v1.1': if not installer.Deactivate('delphes'): self.main.forced=forced_bkp return False if installer.Activate('delphesMA5tune')==-1: self.main.forced=forced_bkp return False elif myversion=='v1.2': if not installer.Deactivate('delphesMA5tune'): self.main.forced=forced_bkp return False if installer.Activate('delphes')==-1: self.main.forced=forced_bkp return False ## Analyses myanalyses = self.main.recasting.analysisruns myanalyses = [ x for x in myanalyses if myversion in x ] myanalyses = [ x.replace(myversion+'_','') for x in myanalyses ] for card,analysislist in self.main.recasting.DelphesDic.items(): if card == mycard: tmpanalyses=analysislist break myanalyses = [ x for x in myanalyses if x in tmpanalyses] ## event file for myset in self.main.datasets: myevents=os.path.normpath(dirname + '/Events/' + myset.name + '_' +\ myversion.replace('.','x')+'_' + mycard.replace('.tcl','')+'.root') ## preparing and running the PAD if not self.main.recasting.UpdatePADMain(myanalyses,PADdir): self.main.forced=forced_bkp return False time.sleep(1.); if not self.main.recasting.MakePAD(PADdir,dirname,self.main): self.main.forced=forced_bkp return False time.sleep(1.); if not self.main.recasting.RunPAD(PADdir,myevents): self.main.forced=forced_bkp return False time.sleep(1.); ## Restoring the PAD as it was before if not self.main.recasting.RestorePADMain(PADdir,dirname,self.main): self.main.forced=forced_bkp return False time.sleep(1.); ## saving the output if not self.main.recasting.SavePADOutput(PADdir,dirname,myanalyses,myset.name): self.main.forced=forced_bkp return False if not self.main.recasting.store_root: os.remove(os.path.normpath(dirname + '/Events/' + myset.name + '_' +\ myversion.replace('.','x')+'_' + mycard.replace('.tcl','')+'.root')) time.sleep(1.); ## Running the CLs exclusion script (if available) if not self.main.recasting.GetCLs(PADdir,dirname,myanalyses,myset.name,myset.xsection,myset.name): self.main.forced=forced_bkp return False ## Saving the results self.main.forced=forced_bkp else: self.logger.info(" Inserting your selection into 'SampleAnalyzer'...") if not jobber.WriteSelectionHeader(self.main): self.logger.error("job submission aborted.") return False if not jobber.WriteSelectionSource(self.main): self.logger.error("job submission aborted.") return False self.logger.info(" Writing the list of datasets...") for item in self.main.datasets: jobber.WriteDatasetList(item) self.logger.info(" Writing the command line history...") jobber.WriteHistory(history,self.main.firstdir) if self.main.recasting.status == "on": self.main.recasting.collect_outputs(dirname,self.main.datasets) self.logger.info(' -> the results can be found in:') self.logger.info(' '+ dirname + '/Output/CLs_output_summary.dat') for item in self.main.datasets: self.logger.info(' '+ dirname + '/Output/'+ item.name + '/CLs_output.dat') else: layouter = LayoutWriter(self.main, dirname) layouter.WriteLayoutConfig() if not self.main.recasting.status=='on' and not self.resubmit: self.logger.info(" Creating Makefiles...") if not jobber.WriteMakefiles(): self.logger.error("job submission aborted.") return False # Edit the delphes or recasting cards if self.main.fastsim.package in ["delphes","delphesMA5tune"]: self.editDelphesCard(dirname) if self.resubmit and not self.main.recasting.status=='on': self.logger.info(" Cleaning 'SampleAnalyzer'...") if not jobber.MrproperJob(): self.logger.error("job submission aborted.") return False if not self.main.recasting.status=='on': self.logger.info(" Compiling 'SampleAnalyzer'...") if not jobber.CompileJob(): self.logger.error("job submission aborted.") return False self.logger.info(" Linking 'SampleAnalyzer'...") if not jobber.LinkJob(): self.logger.error("job submission aborted.") return False for item in self.main.datasets: self.logger.info(" Running 'SampleAnalyzer' over dataset '" +item.name+"'...") self.logger.info(" *******************************************************") if not jobber.RunJob(item): self.logger.error("run over '"+item.name+"' aborted.") self.logger.info(" *******************************************************") return True
def Remove(self,question=True): from madanalysis.IOinterface.folder_writer import FolderWriter return FolderWriter.RemoveDirectory(self.installdir,question)
def ImportJob(self, filename, myinterpreter, history): self.logger.info("SampleAnalyzer job folder is detected") self.logger.info( "Restore MadAnalysis configuration used for this job ...") # Ask question if not self.main.forced: self.logger.warning( "You are going to reinitialize MadAnalysis 5. The current configuration will be lost." ) self.logger.warning("Are you sure to do that ? (Y/N)") allowed_answers = ['n', 'no', 'y', 'yes'] answer = "" while answer not in allowed_answers: answer = raw_input("Answer: ") answer = answer.lower() if answer == "no" or answer == "n": return False self.main.datasets.Reset() # Reset selection self.main.selection.Reset() # Reset main self.main.ResetParameters() # Reset multiparticles self.main.multiparticles.Reset() # Opening a CmdDefine cmd_define = CmdDefine(self.main) # Loading particles input = ParticleReader(self.main.archi_info.ma5dir, cmd_define, self.main.mode) input.Load() input = MultiparticleReader(self.main.archi_info.ma5dir, cmd_define, self.main.mode, self.main.forced) input.Load() # Reset history myinterpreter.history = [] # Load script myinterpreter.load(filename + '/history.ma5') # Saving job name as global variable self.main.lastjob_name = filename self.main.lastjob_status = False # Extract info from ROOT file layout = Layout(self.main) if not self.extract(filename, layout): return # Initialize selection for generating report self.main.selection.RefreshStat() # Status = GOOD self.main.lastjob_status = True # Computing layout.Initialize() # Cleaning the directories if not FolderWriter.RemoveDirectory(self.main.lastjob_name + '/HTML', False): return if self.main.session_info.has_pdflatex: if not FolderWriter.RemoveDirectory( self.main.lastjob_name + '/PDF', False): return if self.main.session_info.has_latex: if not FolderWriter.RemoveDirectory( self.main.lastjob_name + '/DVI', False): return # Creating the reports self.CreateReports([self.main.lastjob_name], history, layout)