def Clean(self,package,folder):
# log file name
if package in ['process','commons','configuration','test']:
logfile = folder+'/cleanup.log'
elif package in ['test_process','test_commons','test_zlib','test_fastjet','test_root','test_delphes','test_delphesMA5tune']:
logfile = folder+'/cleanup_'+package[5:]+'.log'
else:
logfile = folder+'/cleanup_'+package+'.log'
# makefile
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_fastjet','test_root','test_delphes','test_delphesMA5tune']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
# shell command
commands = ['make','clean','--file='+makefile]
# call
result, out = ShellCommand.ExecuteWithLog(commands,logfile,folder)
# return result
if not result:
self.logger.error('impossible to clean the project. For more details, see the log file:')
self.logger.error(logfile)
return result
def MrProper(self,package,folder):
# log file name
if package in ['process','commons','configuration']:
logfile = folder+'/mrproper.log'
elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet','test_delphes','test_delphesMA5tune']:
logfile = folder+'/mrproper_'+package[5:]+'.log'
else:
logfile = folder+'/mrproper_'+package+'.log'
self.logger.debug("LogFile: "+logfile)
# makefile
if package in ['process','commons','test','configuration']:
makefile = 'Makefile'
elif package in ['test_process','test_commons','test_zlib','test_root','test_fastjet','test_delphes','test_delphesMA5tune']:
makefile = 'Makefile_'+package[5:]
else:
makefile = 'Makefile_'+package
self.logger.debug("Makefile: "+makefile)
# shell command
commands = ['make','mrproper','--file='+makefile]
self.logger.debug("Command: "+" ".join(commands))
# call
result, out = ShellCommand.ExecuteWithLog(commands,logfile,folder)
# return result
if not result:
self.logger.error('impossible to clean the project. For more details, see the log file:')
self.logger.error(logfile)
return result
def untar(logname, downloaddir, installdir, tarball):
# Unpacking the folder
theCommands = ['tar', 'xzf', tarball, '-C', installdir]
logging.getLogger('MA5').debug('shell command: ' +
' '.join(theCommands))
logging.getLogger('MA5').debug('exected dir: ' + downloaddir)
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
downloaddir,\
silent=False)
if not ok:
return False, ''
# # Removing the tarball
# toRemove=installdir+'/'+tarball
# logging.getLogger('MA5').debug('removing the file: '+toRemove)
# try:
# os.remove(toRemove)
# except:
# logging.getLogger('MA5').debug('impossible to remove the tarball: '+tarball)
# Getting the good folder
import glob
folder_content = glob.glob(installdir + '/*')
logging.getLogger('MA5').debug('content of ' + installdir + ': ' +
str(folder_content))
if len(folder_content) == 0:
logging.getLogger('MA5').error(
'The content of the tarball is empty')
return False, ''
elif len(folder_content) == 1:
return True, folder_content[0]
else:
return True, installdir
def Build(self):
# Input
theCommands=['make','-j'+str(self.ncores)]
logname=os.path.normpath(self.installdir+'/Build/compilation.log')
# Execute
logging.getLogger('MA5').debug('shell command: '+' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,logname,self.installdir+'/Build',silent=False)
# return result
if not ok:
logging.getLogger('MA5').error('impossible to build the project. For more details, see the log file:')
logging.getLogger('MA5').error(logname)
return ok
def Remove(self,question=True):
import time
bkpname = "pad_forma5tune-v" + time.strftime("%Y%m%d-%Hh%M") + ".tgz"
logging.getLogger('MA5').info(" => Backuping the previous installation: " + bkpname)
logname = os.path.normpath(self.main.archi_info.ma5dir+'/pad-backup.log')
TheCommand = ['tar', 'czf', bkpname, 'PADForMA5tune']
logging.getLogger('MA5').debug('Shell command: '+' '.join(TheCommand))
ok, out= ShellCommand.ExecuteWithLog(TheCommand,logname,self.main.archi_info.ma5dir,silent=False)
if not ok:
return False
logging.getLogger('MA5').info(" => Backup done")
from madanalysis.IOinterface.folder_writer import FolderWriter
return FolderWriter.RemoveDirectory(self.installdir,question)
def Configure(self):
# Input
theCommands=['./configure','--prefix='+self.installdir]
logname=os.path.normpath(self.installdir+'/configuration.log')
# Execute
logging.getLogger('MA5').debug('shell command: '+' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.tmpdir,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error('impossible to configure the project. For more details, see the log file:')
logging.getLogger('MA5').error(logname)
return ok
def Install(self):
# Input
theCommands=['make','install']
logname=os.path.normpath(self.installdir+'/installation_contrib.log')
# Execute
logging.getLogger('MA5').debug('shell command: '+' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.tmpdir,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error('impossible to build the project. For more details, see the log file:')
logging.getLogger('MA5').error(logname)
return ok
def Compile(self, ncores, package, folder):
# number of cores
strcores = ''
if ncores > 1:
strcores = '-j' + str(ncores)
# log file name
if package in ['process', 'commons', 'test', 'configuration']:
logfile = folder + '/compilation.log'
elif package in [
'test_process', 'test_commons', 'test_zlib', 'test_fastjet',
'test_root', 'test_delphes', 'test_delphesMA5tune'
]:
logfile = folder + '/compilation_' + package[5:] + '.log'
else:
logfile = folder + '/compilation_' + package + '.log'
# makefile
if package in ['process', 'commons', 'test', 'configuration']:
makefile = 'Makefile'
elif package in [
'test_process', 'test_commons', 'test_zlib', 'test_fastjet',
'test_root', 'test_delphes', 'test_delphesMA5tune'
]:
makefile = 'Makefile_' + package[5:]
else:
makefile = 'Makefile_' + package
# shell command
if strcores == '':
commands = ['make', 'compile', '--file=' + makefile]
else:
commands = ['make', 'compile', strcores, '--file=' + makefile]
# call
result, out = ShellCommand.ExecuteWithLog(commands, logfile, folder)
# return result
if not result:
self.logger.error(
'impossible to compile the project. For more details, see the log file:'
)
self.logger.error(logfile)
return result
def MakePAD(self, PADdir, dirname, main, silent=False):
if not silent:
self.logger.info(' Compiling the PAD in ' + PADdir)
compiler = LibraryWriter('lib', main)
ncores = compiler.get_ncores2()
if ncores > 1:
strcores = '-j' + str(ncores)
command = ['make', strcores]
logfile = PADdir + '/Build/PADcompilation.log'
result, out = ShellCommand.ExecuteWithLog(command, logfile,
PADdir + '/Build')
if not result:
self.logger.error('Impossible to compile the PAD....'+\
' For more details, see the log file:')
self.logger.error(logfile)
return False
return True
def Clean(self):
# Input
theCommands = ['make', 'clean']
logname = os.path.normpath(self.installdir + '/clean.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to clean the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
def make_pad(self):
# Initializing the compiler
logging.getLogger('MA5').info(' Compiling the PAD located in ' +self.pad)
compiler = LibraryWriter('lib',self.main)
ncores = compiler.get_ncores2()
# compiling
if ncores>1:
strcores='-j'+str(ncores)
command = ['make',strcores]
logfile = self.pad+'/Build/PADcompilation.log'
result, out = ShellCommand.ExecuteWithLog(command,logfile,self.pad+'/Build')
time.sleep(1.);
# Checks and exit
if not result:
logging.getLogger('MA5').error('Impossible to compile the PAD. For more details, see the log file:')
logging.getLogger('MA5').error(logfile)
return False
return True
def Run(self,program,args,folder,silent=False):
# shell command
commands = ['./'+program]
commands.extend(args)
self.logger.debug("Command: "+" ".join(commands))
# logfile
logfile = os.path.normpath(folder+'/'+program+'.log')
self.logger.debug("LogFile: "+logfile)
# call
result, out = ShellCommand.ExecuteWithLog(commands,logfile,folder,silent)
# return result
if not result and not silent:
self.logger.error('impossible to run the project. For more details, see the log file:')
self.logger.error(logfile)
return result
def LaunchInteractiveMatplotlib(self):
# Commands
theCommands = [sys.executable, 'all.py']
# Log file name
logname = os.path.normpath(self.histo_path + '/matplotlib.log')
# Execute
logging.getLogger('MA5').debug('shell command: ' +
' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.histo_path,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error(
'impossible to execute MatPlotLib. For more details, see the log file:'
)
logging.getLogger('MA5').error(logname)
return ok
def Configure(self):
# KNOWn DELPHES ISsues: GENERATE ISSUES BECAuse IT USES TCSLSH COMMAND
# Input
theCommands = ['./configure']
logname = os.path.normpath(self.installdir + '/configuration.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to configure the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
def LinkJob(self):
# folder
folder = self.path + '/Build'
# log file name
logfile = folder + '/Log/linking.log'
# shell command
commands = ['make', 'link']
# call
result, out = ShellCommand.ExecuteWithLog(commands, logfile, folder)
# return result
if not result:
logging.getLogger('MA5').error(
'impossible to link the project. For more details, see the log file:'
)
logging.getLogger('MA5').error(logfile)
return result
def LaunchInteractiveRoot(self):
# Commands
theCommands = ['root', '-l', '-q', '-b']
theCommands.extend(self.filenames)
# Log file name
logname = os.path.normpath(self.histo_path + '/root.log')
# Execute
logging.getLogger('MA5').debug('shell command: ' +
' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.histo_path,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error(
'impossible to execute ROOT. For more details, see the log file:'
)
logging.getLogger('MA5').error(logname)
return ok
def CreatePackageFolder(self):
logname = os.path.normpath(self.main.archi_info.ma5dir +
'/PAD-workingdir.log')
# Initialize the expert mode
logging.getLogger('MA5').debug(
'Calling the expert mode for file cms_sus_13_011')
logging.getLogger('MA5').debug('BEGIN ExpertMode')
from madanalysis.core.expert_mode import ExpertMode
backup = self.main.expertmode
self.main.expertmode = True
expert = ExpertMode(self.main)
dirname = "PADForMA5tune"
if not expert.CreateDirectory(dirname):
return False
filename = "cms_sus_13_011"
if not expert.Copy(filename):
return False
self.main.expertmode = backup
logging.getLogger('MA5').debug('END ExpertMode')
# TheCommand = ['bin/ma5', '-R', '-E', '-f', 'PADForMA5tune', 'cms_sus_13_011']
# logname = os.path.normpath(self.main.archi_info.ma5dir+'/PAD-workingdir.log')
# ok, out= ShellCommand.ExecuteWithLog(TheCommand,logname,self.main.archi_info.ma5dir,silent=False)
# if not ok:
# return False
for analysis in self.analyses:
if "cms_sus_13_011" not in analysis:
TheCommand = ['./newAnalyzer.py', analysis, analysis]
logging.getLogger('MA5').debug(' '.join(TheCommand))
lname = os.path.normpath(self.installdir + '/PAD-' + analysis +
'.log')
ok, out= ShellCommand.ExecuteWithLog(TheCommand,lname,\
self.installdir+'/Build/SampleAnalyzer',silent=False)
if not ok:
return False
TheCommand = ['rm', '-f', self.installdir+'/Build/SampleAnalyzer/User/Analyzer/'+analysis+'.cpp',\
self.installdir+'/Build/SampleAnalyzer/User/Analyzer/'+analysis+'.h']
logging.getLogger('MA5').debug(' '.join(TheCommand))
ok = ShellCommand.Execute(TheCommand, self.main.archi_info.ma5dir)
if not ok:
return False
TheCommand = ['rm', '-f', self.installdir + '/Build/Main/main.bak']
logging.getLogger('MA5').debug(' '.join(TheCommand))
ok = ShellCommand.Execute(TheCommand, self.main.archi_info.ma5dir)
if not ok:
return False
# Logs
logging.debug('Creating folder ' + self.installdir + '/Logs')
try:
os.mkdir(self.installdir + '/Logs')
except:
return False
logging.debug('Move ' + logname + ' in ' + self.installdir)
import shutil
try:
shutil.move(logname,
self.installdir + '/' + os.path.basename(logname))
except:
pass
#bibtex
self.CreateBibtex()
# delphes card directory
TheCommand = ['mkdir', self.installdir + '/Input/Cards']
ok = ShellCommand.Execute(TheCommand, self.main.archi_info.ma5dir)
if not ok:
return False
self.CreateBibtex()
return True
def Unpack(self):
# Logname
logname = os.path.normpath(self.installdir + '/unpack.log')
# Unpacking the tarball
ok, packagedir = InstallService.untar(logname, self.downloaddir,
self.tmpdir,
'delphestune.tar.gz')
if not ok:
return False
# Copying the patch
self.logger.debug('Copying the patch ...')
input = self.toolsdir + '/SampleAnalyzer/Interfaces/delphesMA5tune/patch_delphesMA5tune.tgz'
output = packagedir + '/patch_delphesMA5tune.tgz'
try:
shutil.copy(input, output)
except:
self.logger.error('impossible to copy the patch ' + input +
' to ' + output)
return False
# Unpacking the folder
logname = os.path.normpath(self.installdir + '/unpack_patch.log')
theCommands = ['tar', 'xzf', 'patch_delphesMA5tune.tgz']
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
packagedir,\
silent=False)
if not ok:
self.logger.error('impossible to untar the patch ' + output)
return False
# Applying the patch
logname = os.path.normpath(self.installdir + '/patch.log')
theCommands = [sys.executable, 'patch.py']
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
packagedir,\
silent=False)
if not ok:
self.logger.error('impossible to apply the patch ' + output)
return False
# Getting the list of files
self.logger.debug('Getting the list of files ...')
myfiles = glob.glob(packagedir + '/*')
self.logger.debug('=> ' + str(myfiles))
# Moving files from packagedir to installdir
self.logger.debug('Moving files from ' + packagedir + ' to ' +
self.installdir + ' ...')
for myfile in myfiles:
myfile2 = myfile.split('/')[-1]
if os.path.isdir(myfile):
try:
shutil.copytree(myfile, self.installdir + '/' + myfile2)
except:
self.logger.error('impossible to move the file/folder ' +
myfile + ' from ' + packagedir + ' to ' +
self.installdir)
return False
else:
try:
shutil.copy(myfile, self.installdir + '/' + myfile2)
except:
self.logger.error('impossible to move the file/folder ' +
myfile + ' from ' + packagedir + ' to ' +
self.installdir)
return False
# Ok
return True
def LaunchCompileRoot(self):
import os
import stat
# Commands
output = file(self.histo_path + '/Makefile', 'w')
output.write(
'CXX = `$(MA5_BASE)/tools/SampleAnalyzer/ExternalSymLink/Bin/root-config --cxx`\n'
)
output.write(
'CFLAGS = `$(MA5_BASE)/tools/SampleAnalyzer/ExternalSymLink/Bin/root-config --cflags`\n'
)
output.write(
'LIBS = `$(MA5_BASE)/tools/SampleAnalyzer/ExternalSymLink/Bin/root-config --libs`\n'
)
output.write('SOURCE = all.C\n')
output.write('PROGRAM = goROOT\n')
output.write('\n')
output.write('$(PROGRAM):\n')
output.write('\t$(CXX) $(CFLAGS) $(SOURCE) $(LIBS) -o $(PROGRAM)\n')
output.close()
# Commands
theCommands = ['make']
# Log file name
logname = os.path.normpath(self.histo_path + '/compile_root.log')
# Execute
logging.getLogger('MA5').debug('shell command: ' +
' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.histo_path,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error(
'impossible to execute ROOT. For more details, see the log file:'
)
logging.getLogger('MA5').error(logname)
return ok
# check
if not os.path.isfile(self.histo_path + '/goROOT'):
logging.getLogger('MA5').error('the file ' + self.histo_path +
'/goROOT is not found')
logging.getLogger('MA5').error(logname)
return False
# Commands
theCommands = [self.histo_path + '/goROOT']
# Log file name
logname = os.path.normpath(self.histo_path + '/launch_root.log')
# Execute
logging.getLogger('MA5').debug('shell command: ' +
' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.histo_path,\
silent=False)
# return result
if not ok:
logging.getLogger('MA5').error(
'impossible to execute ROOT. For more details, see the log file:'
)
logging.getLogger('MA5').error(logname)
return ok
# Reading log file
input = file(self.histo_path + '/launch_root.log')
ok1 = False
ok2 = False
for line in input:
if line.startswith('BEGIN-STAMP'):
ok1 = True
if line.startswith('END-STAMP'):
ok2 = True
input.close()
if not (ok1 and ok2):
logging.getLogger('MA5').error(
'wrong behaviour of the ROOT execution. For more details, see the log file:'
)
logging.getLogger('MA5').error(logname)
return ok
return ok
def BuildOld(self):
# Input
theCommands = [
'make', 'ClassesDict_rdict.pcm', 'ExRootAnalysisDict_rdict.pcm',
'ModulesDict_rdict.pcm', 'FastJetDict_rdict.pcm'
]
logname = os.path.normpath(self.installdir + '/compilation_Dict.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
pass # only for ROOT6
# Input
theCommands = ['make', '-j' + str(self.ncores), 'libDelphes.so']
logname = os.path.normpath(self.installdir +
'/compilation_libDelphes.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesSTDHEP']
logname = os.path.normpath(self.installdir + '/compilation_STDHEP.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesLHEF']
logname = os.path.normpath(self.installdir + '/compilation_LHEF.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesHepMC']
logname = os.path.normpath(self.installdir + '/compilation_HepMC.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
def Build(self):
# Input
theCommands = ['make', '-j' + str(self.ncores), 'libDelphesMA5tune.so']
logname = os.path.normpath(self.installdir + '/compilation.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesSTDHEP']
logname = os.path.normpath(self.installdir + '/compilation_STDHEP.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesLHEF']
logname = os.path.normpath(self.installdir + '/compilation_LHEF.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
# Input
theCommands = ['make', 'DelphesHepMC']
logname = os.path.normpath(self.installdir + '/compilation_HepMC.log')
# Execute
self.logger.debug('shell command: ' + ' '.join(theCommands))
ok, out= ShellCommand.ExecuteWithLog(theCommands,\
logname,\
self.installdir,\
silent=False)
# return result
if not ok:
self.logger.error(
'impossible to build the project. For more details, see the log file:'
)
self.logger.error(logname)
return ok
