def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def GetNewLDLibs(systemConfig, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
@param systemConfig: System config used for the job
@param application: name of the application considered
@param applicationVersion: version of the application considered
@return: new LD_LIBRARY_PATH
"""
new_ld_lib_path = ""
deps = resolveDepsTar(systemConfig, application, applicationVersion)
for dep in deps:
depfolder = dep.replace(".tgz","").replace(".tar.gz","")
res = getSoftwareFolder(depfolder)
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
gLogger.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
gLogger.verbose("Found lib folder in %s" % (depfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.environ.has_key("LD_LIBRARY_PATH"):
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def configure(app, area, res_from_check):
""" Configure our applications: set the proper env variables
"""
###########################################
###Go where the software is to be installed
os.chdir(area)
#We go back to the initial place either at the end of the installation or at any error
###########################################
appName = app[0].lower()
### Set env variables
basefolder = res_from_check[0]
removeLibc(os.path.join(os.getcwd(), basefolder, "LDLibs"))
libFolder = os.path.join(os.getcwd(), basefolder, "lib")
if os.path.isdir(libFolder):
removeLibc(os.path.join(libFolder))
addFolderToLdLibraryPath(libFolder)
if appName == "slic":
configureSlic(basefolder)
elif appName == "root":
configureRoot(basefolder)
elif appName == 'java':
configureJava(basefolder)
elif appName == "lcio":
res = checkJava()
if not res['OK']:
return res
configureLCIO(basefolder)
elif appName in ["lcsim", 'stdhepcutjava']:
res = checkJava()
if not res['OK']:
return res
return S_OK()
def configure(app, area, res_from_check):
""" Configure our applications: set the proper env variables
"""
###########################################
###Go where the software is to be installed
os.chdir(area)
#We go back to the initial place either at the end of the installation or at any error
###########################################
appName = app[0].lower()
### Set env variables
basefolder = res_from_check[0]
removeLibc( os.path.join( os.getcwd(), basefolder, "LDLibs" ) )
libFolder = os.path.join( os.getcwd(), basefolder, "lib" )
if os.path.isdir( libFolder ):
removeLibc( os.path.join( libFolder ) )
addFolderToLdLibraryPath( libFolder )
if appName == "slic":
configureSlic(basefolder)
elif appName == "root":
configureRoot(basefolder)
elif appName == 'java':
configureJava(basefolder)
elif appName == "lcio":
res = checkJava()
if not res['OK']:
return res
configureLCIO(basefolder)
elif appName in ["lcsim",'stdhepcutjava']:
res = checkJava()
if not res['OK']:
return res
return S_OK()
def getEnvScript(self, sysconfig, appname, appversion):
""" Produce the environment file in case CVMFS is not here
"""
env_script_name = "SLICPandora.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus(
'SLICPandora: Could not find neither local area not shared area install'
)
return res
myslicPandoraDir = res['Value']
##Remove libc lib
removeLibc(myslicPandoraDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion)
new_path = getNewPATH(sysconfig, appname, appversion)
prefixpath = ""
if os.path.exists("PandoraFrontend"):
prefixpath = "."
elif os.path.exists("%s/Executable/PandoraFrontend" %
myslicPandoraDir):
prefixpath = "%s/Executable" % myslicPandoraDir
else:
return S_ERROR("Missing PandoraFrontend binary")
with open(env_script_name, "w") as script:
script.write('#!/bin/sh \n')
script.write(
'############################################################\n'
)
script.write(
'# Dynamically generated script to get the SLICPandora env. #\n'
)
script.write(
'############################################################\n'
)
script.write("declare -x PATH=%s:$PATH\n" % new_path)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir))
script.write(
'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' %
(myslicPandoraDir, new_ld_lib_path))
script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' %
myslicPandoraDir)
script.write("declare -x PATH=%s:$PATH\n" % prefixpath)
os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getEnvScript(self, sysconfig, appname, appversion):
""" Produce the environment file in case CVMFS is not here
"""
env_script_name = "SLICPandora.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install')
return res
myslicPandoraDir = res['Value']
##Remove libc lib
removeLibc(myslicPandoraDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion)
new_path = getNewPATH(sysconfig, appname, appversion)
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write('############################################################\n')
script.write('# Dynamically generated script to get the SLICPandora env. #\n')
script.write('############################################################\n')
script.write("declare -x PATH=%s:$PATH\n" % new_path )
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myslicPandoraDir, new_ld_lib_path))
script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' % myslicPandoraDir)
prefixpath = ""
if os.path.exists("PandoraFrontend"):
prefixpath = "."
elif os.path.exists("%s/Executable/PandoraFrontend" % myslicPandoraDir):
prefixpath ="%s/Executable" % myslicPandoraDir
else:
return S_ERROR("Missing PandoraFrontend binary")
script.write("declare -x PATH=%s:$PATH\n" % prefixpath )
script.close()
os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def test_remove_libc_nothing_to_remove(self):
with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \
patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \
patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \
patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock:
result = removeLibc('mytestpath')
self.assertTrue(result)
assertMockCalls(chdir_mock, ['mytestpath', 'current_dir'], self)
listdir_mock.assert_called_once_with('mytestpath')
self.assertFalse(remove_mock.called)
assertMockCalls(getcwd_mock, [()] * 2, self)
def test_remove_libc_remove_fails(self):
with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \
patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \
patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \
patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock:
result = removeLibc('mytestpath')
self.assertFalse(result)
assertMockCalls(chdir_mock, ['current_dir', 'mytestpath'], self)
listdir_mock.assert_called_once_with('mytestpath')
remove_mock.assert_called_once_with('mytestpath/libc.so')
assertMockCalls(getcwd_mock, [()] * 3, self)
def test_remove_libc_remove_fails( self ):
with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \
patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \
patch( '%s.os.remove' % MODULE_NAME, new=Mock(side_effect=OSError( 'test_cannot_remove_os_err' )) ) as remove_mock, \
patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock:
result = removeLibc( 'mytestpath' )
self.assertFalse( result )
assertMockCalls( chdir_mock, [ 'current_dir', 'mytestpath' ], self )
listdir_mock.assert_called_once_with( 'mytestpath' )
remove_mock.assert_called_once_with( 'mytestpath/libc.so' )
assertMockCalls( getcwd_mock, [ () ] * 3, self )
def test_remove_libc_nothing_to_remove( self ):
with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \
patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \
patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \
patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'not_a_library.txt', 'dont_delete_me_libsomething.so', 'innocentlibc_.so' ]) ) as listdir_mock:
result = removeLibc( 'mytestpath' )
self.assertTrue( result )
assertMockCalls( chdir_mock, [ 'mytestpath', 'current_dir' ], self )
listdir_mock.assert_called_once_with( 'mytestpath' )
self.assertFalse( remove_mock.called )
assertMockCalls( getcwd_mock, [ () ] * 2, self )
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS is not available
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion)
marlindll = ""
if os.path.exists("%s/MARLIN_DLL" % myMarlinDir):
for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir):
marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":"
marlindll = "%s" % (marlindll)
else:
self.log.error('MARLIN_DLL folder not found, cannot proceed')
return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir)
env_script_name = "MarlinEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('##########################################################\n')
script.write('# Dynamically generated script to create env for Marlin. #\n')
script.write('##########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path))
script.write("declare -x MARLIN_DLL=%s\n" % marlindll)
script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS is not available
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion)
marlindll = ""
if os.path.exists("%s/MARLIN_DLL" % myMarlinDir):
for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir):
marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":"
marlindll = "%s" % (marlindll)
else:
LOG.error('MARLIN_DLL folder not found, cannot proceed')
return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir)
env_script_name = "MarlinEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('##########################################################\n')
script.write('# Dynamically generated script to create env for Marlin. #\n')
script.write('##########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path))
script.write("declare -x MARLIN_DLL=%s\n" % marlindll)
script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ[
"LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def test_remove_libc( self ):
with patch( '%s.os.getcwd' % MODULE_NAME, new=Mock(side_effect=[ 'current_dir', 'mytestpath', 'mytestpath', 'mytestpath' ]) ) as getcwd_mock, \
patch( '%s.os.chdir' % MODULE_NAME, new=Mock(return_value=True) ) as chdir_mock, \
patch( '%s.os.remove' % MODULE_NAME, new=Mock(return_value=True) ) as remove_mock, \
patch( '%s.os.listdir' % MODULE_NAME, new=Mock(return_value=[ 'directory_content1.txt', 'libc.so', 'libstdc++.so' ]) ) as listdir_mock:
result = removeLibc( 'mytestpath' )
self.assertTrue( result )
chdir_mock.assert_any_call( 'mytestpath' )
chdir_mock.assert_called_with( 'current_dir' )
assertEqualsImproved( len( chdir_mock.mock_calls ), 2, self )
listdir_mock.assert_called_once_with( 'mytestpath' )
remove_mock.assert_any_call( 'mytestpath/libc.so' )
remove_mock.assert_called_with( 'mytestpath/libstdc++.so' )
assertEqualsImproved( len( remove_mock.mock_calls ), 2, self )
getcwd_mock.assert_called_with()
assertEqualsImproved( len( getcwd_mock.mock_calls ), 4, self )
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Whizard should not proceed as previous step did not end properly'
)
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName,
self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep["app"], dep['version'])
if not res['OK']:
self.log.error("Failed getting software folder",
res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ[
'PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ[
'PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ[
'LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ[
'LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ[
'LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ[
'LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [
os.path.join(path_to_gridfiles, item)
for item in os.listdir(path_to_gridfiles)
]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [
os.path.join(path, item) for item in os.listdir(path)
]
gridfilesfound = True
self.log.info('Found grid files specific for energy %s' %
self.energy)
break
if not gridfilesfound:
self.log.info(
"Will use generic grid files found, hope the energy is set right"
)
list_of_gridfiles = [
item for item in
glob.glob(os.path.join(path_to_gridfiles, "*.grb")) +
glob.glob(os.path.join(path_to_gridfiles, "*.grc"))
]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
self.log.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
self.log.error(
"Whizard input file was not found in process list, cannot proceed"
)
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin),
"./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
self.log.error("Could not copy %s from %s" %
(whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (
mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (
mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists(
"%s/%s" %
(mySoftDir, self.genmodel.getFile(
self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (
mySoftDir, self.genmodel.getFile(
self.Model)['Value'])
else:
self.log.error(
"Request LesHouches file is missing, cannot proceed"
)
self.setApplicationStatus(
"LesHouches file missing")
return S_ERROR(
"The LesHouches file was not found. Probably you are using a wrong version of whizard."
)
else:
self.log.warn("No file found attached to model %s" %
self.Model)
else:
self.log.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
self.log.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename,
self.energy, self.RandomSeed,
self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
self.log.error('Something went wrong with input file generation')
self.setApplicationStatus(
'Whizard: something went wrong with input file generation')
return S_ERROR(
'Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' %
(leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
self.log.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (
outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (
self.evttype, outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
self.log.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
self.log.info(
'The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(
crosssection)
info['xsection'][process]['err_xsection'] = float(
err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
self.log.info("Status after the application execution is %s" %
str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
self.log.error("Whizard execution completed with errors:")
failed = True
else:
self.log.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
self.log.notice("Looking for output files")
ofnames = glob.glob(outputfilename + '*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep",
self.OutputFile)
else:
self.log.error(
"Whizard execution did not produce a stdhep file")
self.setApplicationStatus(
'Whizard %s Failed to produce STDHEP file' %
(self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
self.log.error("==================================\n StdError:\n")
self.log.error(message)
self.setApplicationStatus('%s Exited With Status %s' %
(self.applicationName, status))
self.log.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK({"OutputFile": self.OutputFile})
def runIt(self):
"""
Called by JobAgent
Execute the following:
- prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
- prepare the list of files to run on
- set the cacheDirectory and put in there the alias.properties
- set the lcsim file using :any:`prepareLCSIMFile`
- run java and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('LCSIM should not proceed as previous step did not end properly')
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error('LCSIM was not found in either the local area or shared area:', res['Message'])
return res
lcsim_name = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)
runonslcio = []
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCSIM: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value']
#for inputfile in inputfilelist:
# LOG.verbose("Will try using %s"%(os.path.basename(inputfile)))
# runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))
retMod = self.downloadDetectorZip()
if not retMod:
return retMod
##Collect jar files to put in classspath
jars = []
if os.path.exists("lib"):
for libs in os.listdir("lib"):
if os.path.basename(libs).find(".jar") > 0:
jars.append(os.path.abspath(os.path.join("lib", libs)))
new_ld_lib_path = "./lib:%s" % new_ld_lib_path
#Remove any libc remaining in .lib
removeLibc("./lib")
###Define cache directory as local folder
aliasproperties = os.path.basename(self.aliasproperties)
cachedir = os.getcwd()
if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
try:
os.mkdir(os.path.join(cachedir, ".lcsim"))
except OSError, x:
LOG.error("Could not create .lcsim folder !", str(x))
def execute(self):
""" Called by Agent
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background, via a call to L{SQLWrapper}
- prepare the steering fie using L{PrepareSteeringFile}
- run Mokka and catch its return status
@return: S_OK(), S_ERROR()
"""
result = self.resolveInputVariables()
if not result['OK']:
return result
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# self.log.info('Skip this module, failure detected in a previous step :')
# self.log.info('Workflow status : %s' %(self.workflowStatus))
# self.log.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Mokka should not proceed as previous step did not end properly')
cwd = os.getcwd()
root = gConfig.getValue('/LocalSite/Root', cwd)
self.log.info( "Executing Mokka %s" % ( self.applicationVersion ))
self.log.info("Platform for job is %s" % ( self.systemConfig ) )
self.log.info("Root directory for job is %s" % ( root ) )
mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka",
self.applicationVersion), '')
if not mokkaDir:
self.log.error('Could not get Tar ball name')
return S_ERROR('Failed finding software directory')
mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "")
#mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone.
mySoftwareRoot = ''
localArea = LocalArea()
sharedArea = SharedArea()
if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)):
mySoftwareRoot = localArea
elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)):
mySoftwareRoot = sharedArea
else:
self.log.error("Mokka: could not find installation directory!")
return S_ERROR("Mokka installation could not be found")
##This is a duplication of what is above, but the above cannot easily be removed...
res = getSoftwareFolder(mokkaDir)
if not res['OK']:
self.log.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
if not mySoftwareRoot:
self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea,
sharedArea))
return S_ERROR('Failed to discover software')
####Setup MySQL instance
MokkaDBrandomName = '/tmp/MokkaDBRoot-' + GenRandString(8)
#sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot)
sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot)
res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice)
if not res['OK']:
self.log.error("Failed to find the DB slice")
return res
result = sqlwrapper.makedirs()
if not result['OK']:
self.setApplicationStatus('MySQL setup failed to create directories.')
return result
result = sqlwrapper.mysqlSetup()
if not result['OK']:
self.setApplicationStatus('MySQL setup failed.')
return result
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
##Get the particle.tbl, if any
path_to_particle_tbl = ''
dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles')
if os.path.exists(dir_to_particletbl):
configdir = os.listdir(dir_to_particletbl)
if 'particle.tbl' in configdir:
path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl')
if os.path.exists('./particle.tbl'):
path_to_particle_tbl = "./particle.tbl"
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
shutil.move("mokka.steer", "mymokka.steer")
except:
self.log.error("Failed renaming the steering file")
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
result = sqlwrapper.mysqlCleanUp()
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion)
if not res['OK']:
result = sqlwrapper.mysqlCleanUp()
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except Exception, x:
result = sqlwrapper.mysqlCleanUp()
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Whizard should not proceed as previous step did not end properly')
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep[ "app" ], dep['version'])
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)]
gridfilesfound = True
LOG.info('Found grid files specific for energy %s' % self.energy)
break
if not gridfilesfound:
LOG.info("Will use generic grid files found, hope the energy is set right")
list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
LOG.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
LOG.error("Whizard input file was not found in process list, cannot proceed")
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
LOG.error("Could not copy %s from %s" % (whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])
else:
LOG.error("Request LesHouches file is missing, cannot proceed")
self.setApplicationStatus("LesHouches file missing")
return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.")
else:
LOG.warn("No file found attached to model %s" % self.Model)
else:
LOG.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
LOG.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy,
self.RandomSeed, self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
LOG.error('Something went wrong with input file generation')
self.setApplicationStatus('Whizard: something went wrong with input file generation')
return S_ERROR('Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
LOG.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (self.evttype,
outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
LOG.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
LOG.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
LOG.info('The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(crosssection)
info['xsection'][process]['err_xsection'] = float(err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
LOG.info("Status after the application execution is %s" % str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
LOG.error("Whizard execution completed with errors:")
failed = True
else:
LOG.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
LOG.notice("Looking for output files")
ofnames = glob.glob(outputfilename+'*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep", self.OutputFile)
else:
LOG.error("Whizard execution did not produce a stdhep file")
self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
LOG.error("==================================\n StdError:\n")
LOG.error(message)
self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status))
LOG.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK( { "OutputFile": self.OutputFile } )
def execute(self):
""" Run the thing
"""
# Get input variables
result = self.resolveInputVariables()
# Checks
if not self.platform:
result = S_ERROR( 'No ILC platform selected' )
if not os.environ.has_key("LCIO"):
self.log.error("Environment variable LCIO was not defined, cannot do anything")
result = S_ERROR("Environment variable LCIO was not defined, cannot do anything")
if not result['OK']:
self.log.error("Failed to resolve the input parameters:", self.result["Message"])
return result
# removeLibc
removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )
# Setting up script
LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" )
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = Template('''
#!/bin/sh
#------------------------------------------------------------------------------#
# Dynamically generated script by CheckCollections module #
#------------------------------------------------------------------------------#
declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH_
declare -x PATH=$PATH_
python <<PYTHONSCRIPT
import sys, subprocess
exitStatus = 0
for file in $files:
cmdResult = subprocess.Popen( ["lcio", "count", file], stdout=subprocess.PIPE ).communicate()[ 0 ]
numberOfEvents = int( cmdResult.strip().split()[1] )
cmdAnajobResult = subprocess.Popen( ["anajob", file], stdout=subprocess.PIPE ).communicate()[ 0 ]
for collection in $collections:
cmdResult = subprocess.Popen( ["grep", "-c", collection], stdin=subprocess.PIPE, stdout=subprocess.PIPE ).communicate( cmdAnajobResult )[ 0 ]
numberOfCollections = int( cmdResult.strip() )
if numberOfEvents != numberOfCollections:
print 'Inconsistency in %s: %i events vs %i collections (%s)' % ( file, numberOfEvents, numberOfCollections, collection )
exitStatus = 1
#sys.exit( exitStatus )
sys.exit( exitStatus )
PYTHONSCRIPT
declare -x appstatus=$$?
exit $$appstatus
''')
scriptContent = scriptContent.substitute( LD_LIBRARY_PATH_ = LD_LIBRARY_PATH,
PATH_ = PATH,
files = self.InputFile,
collections = self.collections
)
# Write script to file
scriptPath = 'CheckCollections_%s_Run_%s' % ( self.applicationVersion, self.STEP_NUMBER )
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open( scriptPath, 'w' )
script.write( scriptContent )
script.close()
# Setup log file for application stdout
if os.path.exists( self.applicationLog ):
os.remove( self.applicationLog )
# Run code
os.chmod( scriptPath, 0755 )
command = '"./%s"' % ( scriptPath )
self.setApplicationStatus( 'CheckCollections %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
self.stdError = ''
self.result = shellCall( 0,
command,
callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520
)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
def execute(self):
""" Called from Workflow
"""
# Get input variables
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
if not self.result['OK']:
self.log.error('Failed to resolve input parameters:', self.result['Message'])
return self.result
if not os.environ.has_key("LCIO"):
self.log.error("Environment variable LCIO was not defined, cannot do anything")
return S_ERROR("Environment variable LCIO was not defined, cannot do anything")
# removeLibc
removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )
# Setting up script
LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" )
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by StdHepConverter module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s
for STDHEPFILE in *.stdhep; do
stdhepjob $STDHEPFILE ${STDHEPFILE/.stdhep/.slcio} -1
done
exit $?
""" % ( LD_LIBRARY_PATH, PATH )
# Write script to file
scriptPath = 'StdHepConverter_%s_Run_%s.sh' % ( self.applicationVersion, self.STEP_NUMBER )
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open( scriptPath, 'w' )
script.write( scriptContent )
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod( scriptPath, 0755 )
command = 'sh -c "./%s"' % ( scriptPath )
self.setApplicationStatus( 'StdHepConverter %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
self.stdError = ''
self.result = shellCall( 0,
command,
callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520
)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
return self.finalStatusReport(status)
def execute(self):
""" Execute the module, called by JobAgent
"""
# Checks
resultRIV = self.resolveInputVariables()
if not resultRIV['OK']:
self.log.error("Failed to resolve input variables",
resultRIV['Message'])
return resultRIV
if not self.platform:
return S_ERROR('No ILC platform selected')
if "LCIO" not in os.environ:
self.log.error(
"Environment variable LCIO was not defined, cannot do anything"
)
return S_ERROR(
"Environment variable LCIO was not defined, cannot do anything"
)
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Missing slcio file!")
self.setApplicationStatus(
'LCIOSplit: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value'][0]
else:
return S_OK("No files found to process")
removeLibc(os.path.join(os.environ["LCIO"], "lib"))
# Setting up script
LD_LIBRARY_PATH = os.path.join("$LCIO", "lib")
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by LCIOConcatenate module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s
lcio split -i %s -n %s
exit $?
""" % (LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice)
# Write script to file
scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open(scriptPath, 'w')
script.write(scriptContent)
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod(scriptPath, 0755)
command = '"./%s"' % (scriptPath)
self.setApplicationStatus('LCIOSplit %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
result = shellCall(0,
command,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
# Check results
status = result['Value'][0]
if not os.path.exists(self.applicationLog):
self.log.error("Cannot access log file, cannot proceed")
return S_ERROR("Failed reading the log file")
baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0]
output_file_base_name = ''
if self.OutputFile:
output_file_base_name = self.OutputFile.split('.slcio')[0]
self.log.info("Will rename all files using '%s' as base." %
output_file_base_name)
numberofeventsdict = {}
fname = ''
with open(self.applicationLog, "r") as logf:
for line in logf:
line = line.rstrip()
if line.count(baseinputfilename):
#First, we need to rename those guys
current_file = os.path.basename(line).replace(".slcio", "")
current_file_extension = current_file.replace(
baseinputfilename, "")
newfile = output_file_base_name + current_file_extension + ".slcio"
os.rename(line, newfile)
fname = newfile
numberofeventsdict[fname] = 0
elif line.count("events"):
numberofeventsdict[fname] = int(line.split()[0])
self.log.verbose("Number of eventsdict dict: %s" % numberofeventsdict)
##Now update the workflow_commons dict with the relation between filename and number of events: needed for
#the registerOutputData
self.workflow_commons[
'file_number_of_event_relation'] = numberofeventsdict
if self.listoutput:
outputlist = []
for fileName in numberofeventsdict:
item = {}
item['outputFile'] = fileName
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
self.step_commons['listoutput'] = outputlist
#Not only the step_commons must be updated
if 'ProductionOutputData' in self.workflow_commons:
proddata = self.workflow_commons['ProductionOutputData'].split(";")
finalproddata = []
this_split_data = ''
for item in proddata:
if not item.count(output_file_base_name):
finalproddata.append(item)
else:
this_split_data = item
path = os.path.dirname(this_split_data)
for fileName in numberofeventsdict:
finalproddata.append(os.path.join(path, fileName))
self.workflow_commons['ProductionOutputData'] = ";".join(
finalproddata)
self.log.info("Status after the application execution is %s" %
str(status))
self.listDir()
return self.finalStatusReport(status)
def configure(app, area, res_from_check):
""" Configure our applications: set the proper env variables
"""
###########################################
###Go where the software is to be installed
os.chdir(area)
#We go back to the initial place either at the end of the installation or at any error
###########################################
appName = app[0].lower()
### Set env variables
basefolder = res_from_check[0]
removeLibc(os.path.join(os.getcwd(), basefolder) + "/LDLibs")
if os.path.isdir(os.path.join(os.getcwd(), basefolder) + "/lib"):
removeLibc(os.path.join(os.getcwd(), basefolder) + "/lib")
if os.environ.has_key('LD_LIBRARY_PATH'):
os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib:" + os.environ['LD_LIBRARY_PATH']
else:
os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib"
if appName == "slic":
os.environ['SLIC_DIR'] = basefolder
slicv = ''
lcddv = ''
xercesv = ''
try:
slicv = os.listdir(os.path.join(basefolder, 'packages/slic/'))[0]
lcddv = os.listdir(os.path.join(basefolder, 'packages/lcdd/'))[0]
if os.path.exists(os.path.join(basefolder, 'packages/xerces/')):
xercesv = os.listdir(os.path.join(basefolder, 'packages/xerces/'))[0]
except:
return S_ERROR("Could not resolve slic env variables, folder content does not match usual pattern")
#for mem in members:
# if mem.name.find('/packages/slic/')>0:
# slicv = mem.name.split("/")[3]
# if mem.name.find('/packages/lcdd/')>0:
# lcddv = mem.name.split("/")[3]
# if mem.name.find('/packages/xerces/')>0:
# xercesv = mem.name.split("/")[3]
if slicv:
os.environ['SLIC_VERSION'] = slicv
if xercesv:
os.environ['XERCES_VERSION'] = xercesv
if lcddv:
os.environ['LCDD_VERSION'] = lcddv
elif appName == "root":
#members = app_tar_to_untar.getmembers()
#fileexample = members[0].name
#fileexample.split("/")[0]
os.environ['ROOTSYS'] = os.path.join(os.getcwd(), basefolder)
if os.environ.has_key('LD_LIBRARY_PATH'):
os.environ['LD_LIBRARY_PATH'] = os.environ['ROOTSYS'] + "/lib:" + os.environ['LD_LIBRARY_PATH']
else:
os.environ['LD_LIBRARY_PATH'] = os.environ['ROOTSYS'] + "/lib"
os.environ['PATH'] = os.environ['ROOTSYS'] + "/bin:" + os.environ['PATH']
os.environ['PYTHONPATH'] = os.environ['ROOTSYS'] + "/lib:" + os.environ["PYTHONPATH"]
elif appName == 'java':
os.environ['PATH'] = os.path.join(os.getcwd(), basefolder) + "/bin:" + os.environ['PATH']
if os.environ.has_key('LD_LIBRARY_PATH'):
os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib:" + os.environ['LD_LIBRARY_PATH']
else:
os.environ['LD_LIBRARY_PATH'] = os.path.join(os.getcwd(), basefolder) + "/lib"
elif appName == "lcio":
os.environ['LCIO'] = os.path.join(os.getcwd(), basefolder)
os.environ['PATH'] = os.path.join(os.getcwd(), basefolder) + "/bin:" + os.environ['PATH']
res = checkJava()
if not res['OK']:
return res
elif appName == "lcsim":
res = checkJava()
if not res['OK']:
return res
return S_OK()
def execute(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by L{PrepareXMLFile}
- run Marlin and catch the exit code
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
marlinDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig,
"marlin",
self.applicationVersion), '')
marlinDir = marlinDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(marlinDir)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "marlin", self.applicationVersion)
res = self.GetInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.systemConfig, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory')
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if os.path.exists(os.path.join(myMarlinDir, 'Settings', pandorasettings)):
try:
shutil.copy(os.path.join(myMarlinDir, 'Settings', pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
elif steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname,pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
# //_^_/ \/_ // | \ \
# ( //) | \/// | \ \
# ( / /) _|_ / ) // | \ _\
# ( // /) '/,_ _ _/ ( ; -. | _ _\.-~ .-~~~^-.
# (( / / )) ,-{ _ `-.|.-~-. .~ `.
# (( // / )) '/\ / ~-. _ .-~ .-~^-. \
# (( /// )) `. { } / \ \
# (( / )) .----~-.\ \-' .~ \ `. \^-.
# ///.----..> \ _ -~ `. ^-` ^-_
# ///-._ _ _ _ _ _ _}^ - - - - ~ ~-- ,.-~
# /.-~
######
#### Hell is the maintenance of the crap below!
os.chdir(self.softDir)
self.log.verbose('setup local mokka database')
removeLibc(self.softDir + "/mysql4grid/lib64/mysql")
if os.environ.has_key('LD_LIBRARY_PATH'):
os.environ['LD_LIBRARY_PATH'] = '%s/mysql4grid/lib64/mysql:%s/mysql4grid/lib64:%s' % (self.softDir,
self.softDir,
os.environ['LD_LIBRARY_PATH'])
else:
os.environ['LD_LIBRARY_PATH'] = '%s/mysql4grid/lib64/mysql:%s/mysql4grid/lib64' % (self.softDir, self.softDir)
os.environ['PATH'] = '%s/mysql4grid/bin:%s' % (self.softDir, os.environ['PATH'])
self.exeEnv = dict( os.environ )
safe_options = "--no-defaults --skip-networking --socket=%s/mysql.sock --datadir=%s --basedir=%s/mysql4grid --pid-file=%s/mysql.pid --log-error=%s --log=%s" % (self.MokkaTMPDir,
self.MokkaDataDir,
self.softDir,
self.MokkaTMPDir,
self.stdError,
self.applicationLog)
def execute(self):
"""
Called by JobAgent
Execute the following:
- prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
- prepare the list of files to run on
- set the cacheDirectory and put in there the alias.properties
- set the lcsim file using L{PrepareLCSIMFile}
- run java and catch the exit code
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('LCSIM should not proceed as previous step did not end properly')
#look for lcsim filename
lcsim_name = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig,
"lcsim",
self.applicationVersion), '')
if not lcsim_name:
self.log.error("Could not find lcsim file name from CS")
return S_ERROR("Could not find lcsim file name from CS")
res = getSoftwareFolder(lcsim_name)
if not res['OK']:
self.log.error('Application %s was not found in either the local area or shared area' % (lcsim_name))
return res
lcsim_name = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "lcsim", self.applicationVersion)
runonslcio = []
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCSIM: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value']
#for inputfile in inputfilelist:
# self.log.verbose("Will try using %s"%(os.path.basename(inputfile)))
# runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))
##Collect jar files to put in classspath
jars = []
if os.path.exists("lib"):
for libs in os.listdir("lib"):
if os.path.basename(libs).find(".jar") > 0:
jars.append(os.path.abspath(os.path.join("lib", libs)))
new_ld_lib_path = "./lib:%s" % new_ld_lib_path
#Remove any libc remaining in .lib
removeLibc("./lib")
###Define cache directory as local folder
aliasproperties = os.path.basename(self.aliasproperties)
cachedir = os.getcwd()
if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
try:
os.mkdir(os.path.join(cachedir, ".lcsim"))
except OSError, x:
self.log.error("Could not create .lcsim folder !", str(x))
def execute(self):
""" Execute the module, called by JobAgent
"""
result = S_ERROR()
# Get input variables
result = self.resolveInputVariables()
# Checks
if not result['OK']:
self.log.error("Failed to resolve input parameters:", result['Message'])
return result
if not self.platform:
result = S_ERROR( 'No ILC platform selected' )
if not result['OK']:
self.log.error("Failed to resolve input parameters:", result['Message'])
return result
if not os.environ.has_key("LCIO"):
self.log.error("Environment variable LCIO was not defined, cannot do anything")
return S_ERROR("Environment variable LCIO was not defined, cannot do anything")
# removeLibc
removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )
# Setting up script
LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" )
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by LCIOConcatenate module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s
lcio concat -f *.slcio -o %s
exit $?
""" % (
LD_LIBRARY_PATH,
PATH,
self.OutputFile
)
# Write script to file
scriptPath = 'LCIOConcatenate_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER )
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open( scriptPath, 'w' )
script.write( scriptContent )
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod( scriptPath, 0755 )
command = '"./%s"' % ( scriptPath )
self.setApplicationStatus( 'LCIOConcatenate %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
self.stdError = ''
result = shellCall( 0,
command,
callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520
)
# Check results
resultTuple = result['Value']
status = resultTuple[0]
self.log.info( "Status after the application execution is %s" % str( status ) )
self.listDir()
return self.finalStatusReport(status)
def execute(self):
""" Execute the module, called by JobAgent
"""
# Get input variables
self.result = self.resolveInputVariables()
# Checks
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
if not self.result['OK']:
return self.result
if not os.environ.has_key("LCIO"):
self.log.error("Environment variable LCIO was not defined, cannot do anything")
return S_ERROR("Environment variable LCIO was not defined, cannot do anything")
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCIOSplit: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value'][0]
else:
return S_OK("No files found to process")
# removeLibc
removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )
# Setting up script
LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" )
if os.environ.has_key('LD_LIBRARY_PATH'):
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if os.environ.has_key('PATH'):
PATH += ":" + os.environ['PATH']
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by LCIOConcatenate module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s
lcio split -i %s -n %s
exit $?
""" % (
LD_LIBRARY_PATH,
PATH,
runonslcio,
self.nbEventsPerSlice
)
# Write script to file
scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER )
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open( scriptPath, 'w' )
script.write( scriptContent )
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod( scriptPath, 0755 )
command = '"./%s"' % ( scriptPath )
self.setApplicationStatus( 'LCIOSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
self.stdError = ''
self.result = shellCall(
0,
command,
callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520
)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
if not os.path.exists(self.applicationLog):
self.log.error("Cannot access log file, cannot proceed")
return S_ERROR("Failed reading the log file")
logf = file(self.applicationLog,"r")
baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0]
output_file_base_name = ''
if self.OutputFile:
output_file_base_name = self.OutputFile.split('.slcio')[0]
self.log.info("Will rename all files using '%s' as base." % output_file_base_name)
numberofeventsdict = {}
fname = ''
for line in logf:
line = line.rstrip()
if line.count(baseinputfilename):
#First, we need to rename those guys
current_file = os.path.basename(line).replace(".slcio", "")
current_file_extension = current_file.replace(baseinputfilename, "")
newfile = output_file_base_name + current_file_extension + ".slcio"
os.rename(line, newfile)
fname = newfile
numberofeventsdict[fname] = 0
elif line.count("events"):
numberofeventsdict[fname] = int(line.split()[0])
self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)
##Now update the workflow_commons dict with the relation between filename and number of events: needed for
#the registerOutputData
self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict
if self.listoutput:
outputlist = []
for f in numberofeventsdict.keys():
item = {}
item['outputFile'] = f
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
self.step_commons['listoutput'] = outputlist
#Not only the step_commons must be updated
if self.workflow_commons.has_key('ProductionOutputData'):
proddata = self.workflow_commons['ProductionOutputData'].split(";")
finalproddata = []
this_split_data = ''
for item in proddata:
if not item.count(output_file_base_name):
finalproddata.append(item)
else:
this_split_data = item
path = os.path.dirname(this_split_data)
for f in numberofeventsdict.keys():
finalproddata.append(os.path.join(path, f))
self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata)
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
def getEnvScript(self, sysconfig, appname, appversion):
""" Return the environment script in case CVMFS is not here.
"""
env_script_name = "MokkaEnv.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
LOG.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
mySoftwareRoot = os.sep.join(myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1])
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, "mokka", self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
remoteMysqlInstall = mySoftwareRoot
removeLibc(remoteMysqlInstall+'/mysql4grid/lib64/mysql')
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write('######################################################\n')
script.write('# Dynamically generated script to get teh Mokka env. #\n')
script.write('######################################################\n')
#if(os.path.exists(sharedArea+"/initILCSOFT.sh")):
# script.write("%s/initILCSOFT.sh"%sharedArea)
script.write("declare -x g4releases=%s\n" % (myMokkaDir))
script.write("declare -x G4SYSTEM=Linux-g++\n")
script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n")
#script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n")
script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n')
script.write('declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n')
script.write('declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n')
script.write('declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n')
###No such data on the GRID (???)
#script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n')
script.write('declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n')
#script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n")
script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n')
script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir))
script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall))
script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir)
if new_ld_lib_path:
script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' % (new_ld_lib_path))
script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' % remoteMysqlInstall)#for MySQL
script.write('declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n' % remoteMysqlInstall)
#for MySQL
script.write("if [ -e %s/%s ]; then\n" % (myMokkaDir, self.db_dump_name))
script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name) )
script.write("fi\n")
#### Do something with the additional environment variables
add_env = self.ops.getOptionsDict("/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" % (self.platform,
"mokka",
self.applicationVersion))
if add_env['OK']:
for key in add_env['Value'].keys():
script.write('declare -x %s=%s/%s\n' % (key, mySoftwareRoot, add_env['Value'][key]))
else:
LOG.verbose("No additional environment variables needed for this application")
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def execute(self):
""" Called from Workflow
"""
# Get input variables
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
if not self.result['OK']:
self.log.error('Failed to resolve input parameters:',
self.result['Message'])
return self.result
if 'LCIO' not in os.environ:
self.log.error(
"Environment variable LCIO was not defined, cannot do anything"
)
return S_ERROR(
"Environment variable LCIO was not defined, cannot do anything"
)
# removeLibc
removeLibc(os.path.join(os.environ["LCIO"], "lib"))
# Setting up script
LD_LIBRARY_PATH = os.path.join("$LCIO", "lib")
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by StdHepConverter module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s
for STDHEPFILE in *.stdhep; do
stdhepjob $STDHEPFILE ${STDHEPFILE/.stdhep/.slcio} -1
done
exit $?
""" % (LD_LIBRARY_PATH, PATH)
# Write script to file
scriptPath = 'StdHepConverter_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open(scriptPath, 'w')
script.write(scriptContent)
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod(scriptPath, 0755)
command = 'sh -c "./%s"' % (scriptPath)
self.setApplicationStatus('StdHepConverter %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
command,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
- prepare the list of files to run on
- set the cacheDirectory and put in there the alias.properties
- set the lcsim file using :any:`prepareLCSIMFile`
- run java and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'LCSIM should not proceed as previous step did not end properly'
)
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
'LCSIM was not found in either the local area or shared area:',
res['Message'])
return res
lcsim_name = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
self.applicationVersion)
runonslcio = []
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCSIM: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value']
#for inputfile in inputfilelist:
# self.log.verbose("Will try using %s"%(os.path.basename(inputfile)))
# runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))
retMod = self.downloadDetectorZip()
if not retMod:
return retMod
##Collect jar files to put in classspath
jars = []
if os.path.exists("lib"):
for libs in os.listdir("lib"):
if os.path.basename(libs).find(".jar") > 0:
jars.append(os.path.abspath(os.path.join("lib", libs)))
new_ld_lib_path = "./lib:%s" % new_ld_lib_path
#Remove any libc remaining in .lib
removeLibc("./lib")
###Define cache directory as local folder
aliasproperties = os.path.basename(self.aliasproperties)
cachedir = os.getcwd()
if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
try:
os.mkdir(os.path.join(cachedir, ".lcsim"))
except OSError, x:
self.log.error("Could not create .lcsim folder !", str(x))
def execute(self):
""" Run the thing
"""
# Get input variables
result = self.resolveInputVariables()
# Checks
if not self.platform:
result = S_ERROR('No ILC platform selected')
if 'LCIO' not in os.environ:
self.log.error(
"Environment variable LCIO was not defined, cannot do anything"
)
result = S_ERROR(
"Environment variable LCIO was not defined, cannot do anything"
)
if not result['OK']:
self.log.error("Failed to resolve the input parameters:",
self.result["Message"])
return result
removeLibc(os.path.join(os.environ["LCIO"], "lib"))
# Setting up script
LD_LIBRARY_PATH = os.path.join("$LCIO", "lib")
if 'LD_LIBRARY_PATH' in os.environ:
LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']
PATH = "$LCIO/bin"
if 'PATH' in os.environ:
PATH += ":" + os.environ['PATH']
scriptContent = Template('''
#!/bin/sh
#------------------------------------------------------------------------------#
# Dynamically generated script by CheckCollections module #
#------------------------------------------------------------------------------#
declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH_
declare -x PATH=$PATH_
python <<PYTHONSCRIPT
import sys, subprocess
exitStatus = 0
for file in $files:
cmdResult = subprocess.Popen( ["lcio", "count", file], stdout=subprocess.PIPE ).communicate()[ 0 ]
numberOfEvents = int( cmdResult.strip().split()[1] )
cmdAnajobResult = subprocess.Popen( ["anajob", file], stdout=subprocess.PIPE ).communicate()[ 0 ]
for collection in $collections:
cmdResult = subprocess.Popen( ["grep", "-c", collection], stdin=subprocess.PIPE, stdout=subprocess.PIPE ).communicate( cmdAnajobResult )[ 0 ]
numberOfCollections = int( cmdResult.strip() )
if numberOfEvents != numberOfCollections:
print 'Inconsistency in %s: %i events vs %i collections (%s)' % ( file, numberOfEvents, numberOfCollections, collection )
exitStatus = 1
#sys.exit( exitStatus )
sys.exit( exitStatus )
PYTHONSCRIPT
declare -x appstatus=$$?
exit $$appstatus
''')
scriptContent = scriptContent.substitute(
LD_LIBRARY_PATH_=LD_LIBRARY_PATH,
PATH_=PATH,
files=self.InputFile,
collections=self.collections)
# Write script to file
scriptPath = 'CheckCollections_%s_Run_%s' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open(scriptPath, 'w')
script.write(scriptContent)
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod(scriptPath, 0755)
command = '"./%s"' % (scriptPath)
self.setApplicationStatus('CheckCollections %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
command,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def getEnvScript(self, sysconfig, appname, appversion):
""" Return the environment script in case CVMFS is not here.
"""
env_script_name = "MokkaEnv.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.log.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
mySoftwareRoot = os.sep.join(
myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1])
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, "mokka",
self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
remoteMysqlInstall = mySoftwareRoot
removeLibc(remoteMysqlInstall + '/mysql4grid/lib64/mysql')
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write(
'######################################################\n')
script.write(
'# Dynamically generated script to get teh Mokka env. #\n')
script.write(
'######################################################\n')
#if(os.path.exists(sharedArea+"/initILCSOFT.sh")):
# script.write("%s/initILCSOFT.sh"%sharedArea)
script.write("declare -x g4releases=%s\n" % (myMokkaDir))
script.write("declare -x G4SYSTEM=Linux-g++\n")
script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n")
#script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n")
script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n')
script.write(
'declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n')
script.write(
'declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n'
)
script.write(
'declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n'
)
###No such data on the GRID (???)
#script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n')
script.write(
'declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n')
#script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n")
script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n')
script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir))
script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall))
script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir)
if new_ld_lib_path:
script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' %
(new_ld_lib_path))
script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' %
remoteMysqlInstall) #for MySQL
script.write(
'declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n'
% remoteMysqlInstall)
#for MySQL
script.write("if [ -e %s/%s ]; then\n" %
(myMokkaDir, self.db_dump_name))
script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name))
script.write("fi\n")
#### Do something with the additional environment variables
add_env = self.ops.getOptionsDict(
"/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" %
(self.platform, "mokka", self.applicationVersion))
if add_env['OK']:
for key in add_env['Value'].keys():
script.write('declare -x %s=%s/%s\n' %
(key, mySoftwareRoot, add_env['Value'][key]))
else:
self.log.verbose(
"No additional environment variables needed for this application"
)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def execute(self):
""" Called from Workflow
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('SLIC Pandora should not proceed as previous step did not end properly')
slicPandoraDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig,
"slicpandora",
self.applicationVersion), '')
slicPandoraDir = slicPandoraDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(slicPandoraDir)
if not res['OK']:
self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install')
return res
myslicPandoraDir = res['Value']
##Remove libc lib
removeLibc(myslicPandoraDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slicpandora", self.applicationVersion)
new_path = GetNewPATH(self.systemConfig, "slicpandora", self.applicationVersion)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('SLICPandora: missing slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value'][0]
if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)):
detmodel = self.detectorxml.replace("_pandora.xml", "")
if os.path.exists(detmodel + ".zip"):
try:
unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
except:
os.unlink(detmodel + ".zip")
if not os.path.exists(detmodel + ".zip"):
#retrieve detector model from web
detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
if len(detector_urls[0]) < 1:
self.log.error('Could not find in CS the URL for detector model')
return S_ERROR('Could not find in CS the URL for detector model')
for detector_url in detector_urls:
try:
urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip")
except:
self.log.error("Download of detector model failed")
continue
try:
unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
break
except:
os.unlink(detmodel + ".zip")
continue
#if os.path.exists(detmodel): #and os.path.isdir(detmodel):
self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
self.detectorxml = self.detectorxml + "_pandora.xml"
if not os.path.exists(self.detectorxml):
self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml)
return S_ERROR('Detector model xml %s was not found, exiting' % self.detectorxml)
if not os.path.exists(self.pandorasettings):
if os.path.exists("./Settings/%s" % self.pandorasettings):
self.pandorasettings = "./Settings/%s" % self.pandorasettings
elif os.path.exists(os.path.join(myslicPandoraDir, 'Settings')):
xmllist = glob.glob(os.path.join(myslicPandoraDir, 'Settings', "*.xml"))
for f in xmllist:
try:
shutil.copy(f, os.path.join(os.getcwd(), os.path.basename(f)))
except Exception, x:
self.log.error('Could not copy %s, exception: %s' % (f, str(x)))
return S_ERROR('Could not copy PandoraSettings file')
else:
self.log.error("Failed to find PandoraSettings anywhere, possibly SLICPandora install broken")
return S_ERROR("Failed to find PandoraSettings anywhere")
