def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS install is not here
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['Value']:
self.setApplicationStatus('Tomato: Could not find neither local area not shared area install')
return res
myTomatoDir = res['Value']
deps = resolveDeps(sysconfig, "tomato", appversion)
for dep in deps:
if dep["app"].lower() == 'marlin':
res = getSoftwareFolder(sysconfig, "marlin", dep["version"])
if not res['OK']:
LOG.error('Marlin was not found in software directory')
return res
else:
myMarlinDir = res['Value']
break
env_script_name = "TomatoEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('###########################################################\n')
script.write('# Dynamically generated script to get the Env for Tomato. #\n')
script.write('###########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' % (myMarlinDir))
script.write("declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" % myTomatoDir)
script.close()
return S_OK(os.path.abspath(env_script_name))
def getNewPATH(platform, application, applicationVersion):
""" Same as :func:`getNewLDLibs`,but for the PATH
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new PATH
"""
log = gLogger.getSubLogger("GetPaths")
log.verbose("Getting all PATH folders")
new_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep['app'], dep['version'])
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
log.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if "PATH" in os.environ:
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def getNewPATH(platform, application, applicationVersion):
""" Same as :func:`getNewLDLibs`,but for the PATH
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new PATH
"""
log = gLogger.getSubLogger("GetPaths")
log.verbose("Getting all PATH folders")
new_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep['app'], dep['version'])
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
log.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if "PATH" in os.environ:
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def treatILDConfigPackage(self):
"""treat the ILDConfig software package"""
config_dir = self.workflow_commons['ILDConfigPackage']
ildConfigVersion = config_dir.replace("ILDConfig", "")
resCVMFS = checkCVMFS(self.platform, ('ildconfig', ildConfigVersion))
if resCVMFS['OK']:
ildConfigPath = resCVMFS['Value'][0]
else:
self.log.error("Cannot find ILDConfig on CVMFS" , ("Version: "+config_dir, resCVMFS['Message']) )
resLoc = getSoftwareFolder(self.platform, "ildconfig", ildConfigVersion)
if resLoc['OK']:
ildConfigPath = resLoc['Value']
else:
self.log.error("Cannot find %s" % config_dir, resLoc['Message'])
return S_ERROR('Failed to locate %s as config dir' % config_dir)
self.log.info("Found ILDConfig here:", ildConfigPath)
list_f = os.listdir(ildConfigPath)
for localFile in list_f:
if os.path.exists("./"+localFile):
self.log.verbose("Found local file, don't overwrite:", localFile)
#Do not overwrite local files with the same name
continue
try:
if os.path.isdir(os.path.join(ildConfigPath, localFile)):
shutil.copytree(os.path.join(ildConfigPath, localFile), "./"+localFile)
else:
shutil.copy2(os.path.join(ildConfigPath, localFile), "./"+localFile)
except EnvironmentError as why:
self.log.error('Could not copy %s here because %s!' % (localFile, str(why)))
return S_OK()
def runIt(self):
""" Called from Workflow
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
if not self.OutputFile:
self.log.error("OutputFile not specified")
return S_ERROR("OutputFile not specified")
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error('Application %s was not found in either the local area or shared area' % self.applicationName)
self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName)
return res
mySoftDir = res['Value']
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose('Getting the steering files directory')
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error("Could not locate the steering file directory")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except EnvironmentError, x:
self.log.error("Failed to get the cuts file")
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def test_getsoftwarefolder_from_cvmfs(self):
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_OK(('mycvmfsfolder/txt',
'otherentry')))) as cvmfs_mock:
result = getSoftwareFolder('a', 'b', 'c')
cvmfs_mock.assert_called_once_with('a', ['b', 'c'])
assertDiracSucceedsWith_equals(result, 'mycvmfsfolder/txt', self)
def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def GetNewLDLibs(systemConfig, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
@param systemConfig: System config used for the job
@param application: name of the application considered
@param applicationVersion: version of the application considered
@return: new LD_LIBRARY_PATH
"""
new_ld_lib_path = ""
deps = resolveDepsTar(systemConfig, application, applicationVersion)
for dep in deps:
depfolder = dep.replace(".tgz","").replace(".tar.gz","")
res = getSoftwareFolder(depfolder)
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
gLogger.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
gLogger.verbose("Found lib folder in %s" % (depfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.environ.has_key("LD_LIBRARY_PATH"):
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def test_getsoftwarefolder_uselocal( self ):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[ False, True, False ])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
exists_mock.assert_called_with( '/mylocalarea/test/myapparchivev2.test' )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertEqualsImproved( len(exists_mock.mock_calls), 2, self ) #One exists call in checkCVMFS
assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self )
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig,
self.applicationName,
self.applicationVersion), '')
appDir = appDir.replace(".tgz","").replace(".tar.gz","")
res = getSoftwareFolder(appDir)
if not res['OK']:
self.setApplicationStatus('Pythia: Could not find neither local area not shared area install')
return res
myappDir = res['Value']
deptar = resolveDepsTar(self.systemConfig, self.applicationName, self.applicationVersion)[0]
depdir = deptar.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(depdir)
if not res['OK']:
return res
path = res['Value']
if not os.path.exists(path + "/%s.ep" % depdir):
return S_ERROR("Lumi files not found")
originpath = os.path.join(path, "/%s.ep" % depdir)
randomName = '/tmp/LumiFile-' + GenRandString(8)
try:
os.mkdir(randomName)
except Exception, x:
return S_ERROR("Could not create temp dir: %s %s" % (Exception, x))
def _getDetectorXML(self):
"""returns the path to the detector XML file
Checks the Configuration System for the Path to DetectorModels or extracts the input sandbox detector xml files
:returns: S_OK(PathToXMLFile), S_ERROR
"""
if os.path.exists(
os.path.join(self.detectorModel, self.detectorModel + ".xml")):
self.log.notice(
"Found detector model: %s" %
os.path.join(self.detectorModel, self.detectorModel + ".xml"))
return S_OK(
os.path.join(self.detectorModel, self.detectorModel + ".xml"))
elif os.path.exists(self.detectorModel + ".zip"):
self.log.notice("Found detector model zipFile: %s" %
self.detectorModel + ".zip")
return self._extractZip()
elif os.path.exists(self.detectorModel + ".tar.gz"):
self.log.notice("Found detector model tarball: %s" %
self.detectorModel + ".tar.gz")
return self._extractTar()
elif os.path.exists(self.detectorModel + ".tgz"):
self.log.notice("Found detector model tarball: %s" %
self.detectorModel + ".tgz")
return self._extractTar(extension=".tgz")
detectorModels = self.ops.getOptionsDict("/DDSimDetectorModels/%s" %
(self.applicationVersion))
if not detectorModels['OK']:
self.log.error(
"Failed to get list of DetectorModels from the ConfigSystem",
detectorModels['Message'])
return S_ERROR(
"Failed to get list of DetectorModels from the ConfigSystem")
softwareFolder = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not softwareFolder['OK']:
return softwareFolder
softwareRoot = softwareFolder['Value']
if self.detectorModel in detectorModels['Value']:
detModelPath = detectorModels['Value'][self.detectorModel]
if not detModelPath.startswith("/"):
detModelPath = os.path.join(softwareRoot, detModelPath)
self.log.info("Found path for DetectorModel %s in CS: %s " %
(self.detectorModel, detModelPath))
return S_OK(detModelPath)
self.log.error(
'Detector model %s was not found neither locally nor on the web, exiting'
% self.detectorModel)
return S_ERROR('Detector model was not found')
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error('Failed finding the software area')
self.setApplicationStatus('Could find neither local area nor shared area install')
return res
myappDir = res['Value']
deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0]
res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"])
if not res['OK']:
LOG.error("Failed finding the dependency location")
return res
path = res['Value']
if not os.path.exists("%s.ep" % path):
LOG.error('Could not find the lumi files!')
return S_ERROR("Lumi files not found")
originpath = "%s.ep" % path
randomName = '/tmp/LumiFile-' + generateRandomString(8)
try:
os.mkdir(randomName)
except EnvironmentError, x:
LOG.error("Failed setting up the temp directory")
return S_ERROR("Could not create temp dir: %s" % str(x))
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error('Failed finding the software area')
self.setApplicationStatus('Could find neither local area nor shared area install')
return res
myappDir = res['Value']
deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0]
res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"])
if not res['OK']:
self.log.error("Failed finding the dependency location")
return res
path = res['Value']
if not os.path.exists("%s.ep" % path):
self.log.error('Could not find the lumi files!')
return S_ERROR("Lumi files not found")
originpath = "%s.ep" % path
randomName = '/tmp/LumiFile-' + generateRandomString(8)
try:
os.mkdir(randomName)
except EnvironmentError, x:
self.log.error("Failed setting up the temp directory")
return S_ERROR("Could not create temp dir: %s" % str(x))
def test_getsoftwarefolder_notfound( self ):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapp_executable')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=[ False, False, False ])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
exists_mock.assert_any_call( '/mylocalarea/test/myapp_executable' )
exists_mock.assert_any_call( '/testshared/area/myapp_executable' )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertEqualsImproved( len(exists_mock.mock_calls), 3, self ) #One exists call in checkCVMFS
assertDiracFailsWith( result, 'missing installation of myapp_executable', self )
def test_getsoftwarefolder_notfound( self ):
exists_dict = { 'myapp_executable' : False, '/mylocalarea/test/myapp_executable' : False, '/testshared/area/myapp_executable' : False }
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapp_executable')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
assertMockCalls( exists_mock, [ '/mylocalarea/test/myapp_executable', '/testshared/area/myapp_executable',
'myapp_executable' ], self )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertDiracFailsWith( result, 'missing installation of myapp_executable', self )
def test_getsoftwarefolder_uselocal( self ):
exists_dict = { 'myapparchivev2.test.tar.gz' : False, '/mylocalarea/test/myapparchivev2.test' : True }
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
assertMockCalls( exists_mock, [ '/mylocalarea/test/myapparchivev2.test', 'myapparchivev2.test.tar.gz' ],
self )
getval_mock.assert_called_with( '/AvailableTarBalls/a/b/c/TarBall', '' )
assertDiracSucceedsWith_equals( result, '/mylocalarea/test/myapparchivev2.test', self )
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS install is not here
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['Value']:
self.setApplicationStatus(
'Tomato: Could not find neither local area not shared area install'
)
return res
myTomatoDir = res['Value']
deps = resolveDeps(sysconfig, "tomato", appversion)
for dep in deps:
if dep["app"].lower() == 'marlin':
res = getSoftwareFolder(sysconfig, "marlin", dep["version"])
if not res['OK']:
self.log.error(
'Marlin was not found in software directory')
return res
else:
myMarlinDir = res['Value']
break
env_script_name = "TomatoEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write(
'###########################################################\n')
script.write(
'# Dynamically generated script to get the Env for Tomato. #\n')
script.write(
'###########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' %
(myMarlinDir))
script.write(
"declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" %
myTomatoDir)
script.close()
return S_OK(os.path.abspath(env_script_name))
def treatConfigPackage( self ):
"""treat the config software package
If a 'ConfigPackage' is part of the workflow_dictionary check for the
package and copy the steering files to the working directory. Existing files
are _not_ overwritten.
"""
## Find if there is a configPackage
configName = None
for key in self.workflow_commons:
if 'ConfigPackage' in key:
configName = key
break
if configName is None:
self.log.verbose( "No ConfigPackage set" )
return S_OK()
configType = configName.split('ConfigPackage')[0]
config_dir = self.workflow_commons[configName]
packageName = str(configType + 'Config').lower()
configVersion = config_dir.split('Config')[1]
resCVMFS = checkCVMFS(self.platform, (packageName, configVersion))
if resCVMFS['OK']:
configPath = resCVMFS['Value'][0]
else:
self.log.error("Cannot find %sConfig on CVMFS" % configType, ("Version: "+config_dir, resCVMFS['Message']) )
resLoc = getSoftwareFolder(self.platform, packageName, configVersion)
if resLoc['OK']:
configPath = resLoc['Value']
else:
self.log.error("Cannot find %s" % config_dir, resLoc['Message'])
return S_ERROR('Failed to locate %s as config dir' % config_dir)
self.log.info("Found %sConfig here:" %configType, configPath)
list_f = os.listdir(configPath)
for localFile in list_f:
if os.path.exists("./"+localFile):
self.log.verbose("Found local file, don't overwrite:", localFile)
#Do not overwrite local files with the same name
continue
try:
if os.path.isdir(os.path.join(configPath, localFile)):
shutil.copytree(os.path.join(configPath, localFile), "./"+localFile)
else:
shutil.copy2(os.path.join(configPath, localFile), "./"+localFile)
except EnvironmentError as why:
self.log.error('Could not copy %s here because %s!' % (localFile, str(why)))
return S_OK()
def treatConfigPackage( self ):
"""treat the config software package
If a 'ConfigPackage' is part of the workflow_dictionary check for the
package and copy the steering files to the working directory. Existing files
are _not_ overwritten.
"""
## Find if there is a configPackage
configName = None
for key in self.workflow_commons:
if 'ConfigPackage' in key:
configName = key
break
if configName is None:
LOG.verbose("No ConfigPackage set")
return S_OK()
configType = configName.split('ConfigPackage')[0]
config_dir = self.workflow_commons[configName]
packageName = str(configType + 'Config').lower()
configVersion = config_dir.split('Config')[1]
resCVMFS = checkCVMFS(self.platform, (packageName, configVersion))
if resCVMFS['OK']:
configPath = resCVMFS['Value'][0]
else:
LOG.error("Cannot find %sConfig on CVMFS" % configType, ("Version: " + config_dir, resCVMFS['Message']))
resLoc = getSoftwareFolder(self.platform, packageName, configVersion)
if resLoc['OK']:
configPath = resLoc['Value']
else:
LOG.error("Cannot find %s" % config_dir, resLoc['Message'])
return S_ERROR('Failed to locate %s as config dir' % config_dir)
LOG.info("Found %sConfig here:" % configType, configPath)
list_f = os.listdir(configPath)
for localFile in list_f:
if os.path.exists("./"+localFile):
LOG.verbose("Found local file, don't overwrite:", localFile)
#Do not overwrite local files with the same name
continue
try:
if os.path.isdir(os.path.join(configPath, localFile)):
shutil.copytree(os.path.join(configPath, localFile), "./"+localFile)
else:
shutil.copy2(os.path.join(configPath, localFile), "./"+localFile)
except EnvironmentError as why:
LOG.error('Could not copy %s here because %s!' % (localFile, str(why)))
return S_OK()
def getEnvScript(self, sysconfig, appname, appversion):
""" Produce the environment file in case CVMFS is not here
"""
env_script_name = "SLICPandora.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus(
'SLICPandora: Could not find neither local area not shared area install'
)
return res
myslicPandoraDir = res['Value']
##Remove libc lib
removeLibc(myslicPandoraDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion)
new_path = getNewPATH(sysconfig, appname, appversion)
prefixpath = ""
if os.path.exists("PandoraFrontend"):
prefixpath = "."
elif os.path.exists("%s/Executable/PandoraFrontend" %
myslicPandoraDir):
prefixpath = "%s/Executable" % myslicPandoraDir
else:
return S_ERROR("Missing PandoraFrontend binary")
with open(env_script_name, "w") as script:
script.write('#!/bin/sh \n')
script.write(
'############################################################\n'
)
script.write(
'# Dynamically generated script to get the SLICPandora env. #\n'
)
script.write(
'############################################################\n'
)
script.write("declare -x PATH=%s:$PATH\n" % new_path)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir))
script.write(
'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' %
(myslicPandoraDir, new_ld_lib_path))
script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' %
myslicPandoraDir)
script.write("declare -x PATH=%s:$PATH\n" % prefixpath)
os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getSteeringFileDir(platform, version):
"""Return directly the directory, without passing by the dependency resolution
"""
res = checkCVMFS(platform, ['steeringfiles', version])
if res['OK']:
return S_OK(res['Value'][0])
#Here means CVMFS is not defined, so we need to rely on the tar ball
res = getSoftwareFolder(platform, 'steeringfiles', version)
if not res['OK']:
return res
mySoftDir = res['Value']
##check that all the files are there: software is not corrupted.
res = check('steeringfiles.%s' % version, '.', [mySoftDir])
if not res['OK']:
return res
return S_OK(mySoftDir)
def runIt(self):
""" Called from Workflow
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'%s should not proceed as previous step did not end properly' %
self.applicationName)
if not self.OutputFile:
self.log.error("OutputFile not specified")
return S_ERROR("OutputFile not specified")
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
'Application %s was not found in either the local area or shared area'
% self.applicationName)
self.setApplicationStatus(
'%s: Could not find neither local area not shared area install'
% self.applicationName)
return res
mySoftDir = res['Value']
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose('Getting the steering files directory')
res = getSteeringFileDirName(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error("Could not locate the steering file directory")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(
os.path.join(steeringfiledirname, self.SteeringFile),
"./" + self.SteeringFile)
except EnvironmentError, x:
self.log.error("Failed to get the cuts file")
return S_ERROR('Failed to access file %s: %s' %
(self.SteeringFile, str(x)))
def test_getsoftwarefolder_uselocal(self):
exists_dict = {
'myapparchivev2.test.tar.gz': False,
'/mylocalarea/test/myapparchivev2.test': True
}
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapparchivev2.test.tar.gz')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock:
result = getSoftwareFolder('a', 'b', 'c')
assertMockCalls(exists_mock, [
'/mylocalarea/test/myapparchivev2.test',
'myapparchivev2.test.tar.gz'
], self)
getval_mock.assert_called_with('/AvailableTarBalls/a/b/c/TarBall',
'')
assertDiracSucceedsWith_equals(
result, '/mylocalarea/test/myapparchivev2.test', self)
def getEnvScript(self, sysconfig, appname, appversion):
""" Produce the environment file in case CVMFS is not here
"""
env_script_name = "SLICPandora.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install')
return res
myslicPandoraDir = res['Value']
##Remove libc lib
removeLibc(myslicPandoraDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, appname, appversion)
new_path = getNewPATH(sysconfig, appname, appversion)
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write('############################################################\n')
script.write('# Dynamically generated script to get the SLICPandora env. #\n')
script.write('############################################################\n')
script.write("declare -x PATH=%s:$PATH\n" % new_path )
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myslicPandoraDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myslicPandoraDir, new_ld_lib_path))
script.write('declare -x PANDORASETTINGSDIR=%s/Settings\n' % myslicPandoraDir)
prefixpath = ""
if os.path.exists("PandoraFrontend"):
prefixpath = "."
elif os.path.exists("%s/Executable/PandoraFrontend" % myslicPandoraDir):
prefixpath ="%s/Executable" % myslicPandoraDir
else:
return S_ERROR("Missing PandoraFrontend binary")
script.write("declare -x PATH=%s:$PATH\n" % prefixpath )
script.close()
os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def test_getsoftwarefolder_notfound(self):
exists_dict = {
'myapp_executable': False,
'/mylocalarea/test/myapp_executable': False,
'/testshared/area/myapp_executable': False
}
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='myapp_executable')) as getval_mock, \
patch('%s.getLocalAreaLocation' % MODULE_NAME, new=Mock(return_value='/mylocalarea/test')), \
patch('%s.getSharedAreaLocation' % MODULE_NAME, new=Mock(return_value='/testshared/area')), \
patch('%s.os.path.exists' % MODULE_NAME, new=Mock(side_effect=lambda path: exists_dict[path])) as exists_mock:
result = getSoftwareFolder('a', 'b', 'c')
assertMockCalls(exists_mock, [
'/mylocalarea/test/myapp_executable',
'/testshared/area/myapp_executable', 'myapp_executable'
], self)
getval_mock.assert_called_with('/AvailableTarBalls/a/b/c/TarBall',
'')
assertDiracFailsWith(result,
'missing installation of myapp_executable',
self)
def _getDetectorXML( self ):
"""returns the path to the detector XML file
Checks the Configurartion System for the Path to DetectorModels or extracts the input sandbox detector xml files
:returns: S_OK(PathToXMLFile), S_ERROR
"""
if os.path.exists( os.path.join( self.detectorModel, self.detectorModel+ ".xml" ) ):
self.log.notice( "Found detector model: %s" % os.path.join( self.detectorModel, self.detectorModel+ ".xml" ) )
return S_OK( os.path.join( self.detectorModel, self.detectorModel+ ".xml" ) )
elif os.path.exists(self.detectorModel + ".zip"):
self.log.notice( "Found detector model zipFile: %s" % self.detectorModel+ ".zip" )
return self._extractZip()
elif os.path.exists(self.detectorModel + ".tar.gz"):
self.log.notice( "Found detector model tarball: %s" % self.detectorModel+ ".tar.gz" )
return self._extractTar()
elif os.path.exists(self.detectorModel + ".tgz"):
self.log.notice( "Found detector model tarball: %s" % self.detectorModel+ ".tgz" )
return self._extractTar( extension=".tgz" )
detectorModels = self.ops.getOptionsDict("/DDSimDetectorModels/%s" % ( self.applicationVersion ) )
if not detectorModels['OK']:
self.log.error("Failed to get list of DetectorModels from the ConfigSystem", detectorModels['Message'])
return S_ERROR("Failed to get list of DetectorModels from the ConfigSystem")
softwareFolder = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not softwareFolder['OK']:
return softwareFolder
softwareRoot = softwareFolder['Value']
if self.detectorModel in detectorModels['Value']:
detModelPath = detectorModels['Value'][self.detectorModel]
if not detModelPath.startswith("/"):
detModelPath = os.path.join( softwareRoot, detModelPath )
self.log.info( "Found path for DetectorModel %s in CS: %s " % ( self.detectorModel, detModelPath ) )
return S_OK(detModelPath)
self.log.error('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel)
return S_ERROR('Detector model was not found')
def getSteeringFileDirName(systemConfig, application, applicationVersion):
""" Locate the path of the steering file directory assigned to the specified application
"""
ops = Operations()
version = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/steeringfiles/version' % (systemConfig,
application,
applicationVersion), '')
if not version:
return S_ERROR("Could not find attached SteeringFile version")
TarBall = ops.getValue('/AvailableTarBalls/%s/steeringfiles/%s/TarBall' % (systemConfig, version), '')
if not TarBall:
return S_ERROR("Could not find tar ball for SteeringFile")
mydir = TarBall.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(mydir)
if not res['OK']:
return res
mySoftDir = res['Value']
res = check('steeringfiles.%s'%version,'.',[mySoftDir])##check that all the files are there: software is not corrupted.
if not res['OK']:
return res
return S_OK(mySoftDir)
def GetNewPATH(systemConfig, application, applicationVersion):
""" Same as L{GetNewLDLibs},but for the PATH
"""
new_path = ""
deps = resolveDepsTar(systemConfig, application, applicationVersion)
for dep in deps:
depfolder = dep.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(depfolder)
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
gLogger.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if os.environ.has_key("PATH"):
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def execute(self):
""" Called from Workflow
"""
self.result = self.resolveInputVariables()
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('StdHepCut should not proceed as previous step did not end properly')
appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig, "stdhepcut",
self.applicationVersion), '')
if not appDir:
self.log.error('Could not get info from CS')
self.setApplicationStatus('Failed finding info from CS')
return S_ERROR('Failed finding info from CS')
appDir = appDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(appDir)
if not res['OK']:
self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName)
return res
mySoftDir = res['Value']
new_ld_lib_path = GetNewLDLibs(self.systemConfig, self.applicationName, self.applicationVersion)
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
if os.path.exists("./lib"):
new_ld_lib_path = "./lib:" + new_ld_lib_path
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.systemConfig, self.applicationName, self.applicationVersion)
if not res['OK']:
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except Exception, x:
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS is not available
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion)
marlindll = ""
if os.path.exists("%s/MARLIN_DLL" % myMarlinDir):
for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir):
marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":"
marlindll = "%s" % (marlindll)
else:
self.log.error('MARLIN_DLL folder not found, cannot proceed')
return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir)
env_script_name = "MarlinEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('##########################################################\n')
script.write('# Dynamically generated script to create env for Marlin. #\n')
script.write('##########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path))
script.write("declare -x MARLIN_DLL=%s\n" % marlindll)
script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS is not available
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(sysconfig, "marlin", appversion)
marlindll = ""
if os.path.exists("%s/MARLIN_DLL" % myMarlinDir):
for library in os.listdir("%s/MARLIN_DLL" % myMarlinDir):
marlindll = marlindll + "%s/MARLIN_DLL/%s" % (myMarlinDir, library) + ":"
marlindll = "%s" % (marlindll)
else:
LOG.error('MARLIN_DLL folder not found, cannot proceed')
return S_ERROR('MARLIN_DLL folder not found in %s' % myMarlinDir)
env_script_name = "MarlinEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('##########################################################\n')
script.write('# Dynamically generated script to create env for Marlin. #\n')
script.write('##########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs:%s\n' % (myMarlinDir, new_ld_lib_path))
script.write("declare -x MARLIN_DLL=%s\n" % marlindll)
script.write("declare -x PANDORASETTINGS=%s/Settings/PandoraSettings.xml" % myMarlinDir)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ[
"LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def getEnvScript(self, platform, appname, appversion):
""" Create an environment script for ddsim. Only used when CVMFS native installation not available
We need ROOTSYS, G4INSTALL and G4DATA as additional environment variables in the CS
We need to set DD4hepINSTALL !
"""
##Need to fetch the new LD_LIBRARY_PATH
newLDLibraryPath = getNewLDLibs(platform, appname, appversion)
softwareFolder = getSoftwareFolder(platform, appname, appversion)
if not softwareFolder['OK']:
return softwareFolder
softwareRoot = softwareFolder['Value']
envName = "DDSimEnv.sh"
script = []
script.append("#!/bin/bash")
script.append("##########################")
script.append("## Env script for DDSim ##")
script.append("##########################")
addEnv = self.ops.getOptionsDict(
"/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" %
(platform, appname, appversion))
if addEnv['OK']:
for variable, value in addEnv['Value'].iteritems():
script.append('declare -x %s=%s' % (variable, value))
else:
self.log.verbose(
"No additional environment variables needed for this application"
)
## if these variables are not given in the configuration system we guess them from the G4DATA folder
for var, folder in {
"G4LEDATA": "G4EMLOW",
"G4LEVELGAMMADATA": "PhotonEvaporation",
"G4NEUTRONXSDATA": "G4NEUTRONXS",
"G4SAIDXSDATA": "G4SAIDDATA",
"G4RADIOACTIVEDATA": "RadioactiveDecay",
"G4NEUTRONHPDATA": "G4NDL",
}.iteritems():
if var not in addEnv.get('Value', {}):
script.append('declare -x %s=$(ls -d $G4DATA/%s*)' %
(var, folder))
init = self.ops.getValue(
"/AvailableTarBalls/%s/%s/%s/InitScript" %
(platform, appname, appversion), None)
if init:
script.append('source %s' % init)
##Executable:
script.append('declare -x PATH=%s/bin:$PATH' % softwareRoot)
script.append('declare -x DD4hepINSTALL=%s' % softwareRoot)
##Python objects, pyroot
script.append('declare -x PYTHONPATH=%s/lib/python:$PYTHONPATH' %
softwareRoot)
script.append('declare -x PYTHONPATH=$ROOTSYS/lib:$PYTHONPATH')
##For ROOT CLING Stuff
script.append('declare -x CPATH=%s/include:$CPATH' % softwareRoot)
##Libraries
if newLDLibraryPath:
script.append('declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH:%s' %
newLDLibraryPath)
## user provided libraries are in lib in the job working directory
if os.path.exists("%s/lib" % os.getcwd()):
script.append(
'declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH' %
os.getcwd())
##Root Path, just in case
script.append('declare -x PATH=$ROOTSYS/bin:$PATH')
##Root and Geant4 Library Path
script.append(
'declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:$LD_LIBRARY_PATH')
script.append(
'declare -x LD_LIBRARY_PATH=$G4INSTALL/lib64:$LD_LIBRARY_PATH')
script.append(
'declare -x LD_LIBRARY_PATH=$DD4hepINSTALL/lib:$LD_LIBRARY_PATH')
with open(envName, "w") as scriptFile:
scriptFile.write("\n".join(script))
scriptFile.write("\n")
os.chmod(envName, 0755)
return S_OK(os.path.abspath(envName))
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Whizard should not proceed as previous step did not end properly'
)
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName,
self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep["app"], dep['version'])
if not res['OK']:
self.log.error("Failed getting software folder",
res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ[
'PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ[
'PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ[
'LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ[
'LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ[
'LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ[
'LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [
os.path.join(path_to_gridfiles, item)
for item in os.listdir(path_to_gridfiles)
]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [
os.path.join(path, item) for item in os.listdir(path)
]
gridfilesfound = True
self.log.info('Found grid files specific for energy %s' %
self.energy)
break
if not gridfilesfound:
self.log.info(
"Will use generic grid files found, hope the energy is set right"
)
list_of_gridfiles = [
item for item in
glob.glob(os.path.join(path_to_gridfiles, "*.grb")) +
glob.glob(os.path.join(path_to_gridfiles, "*.grc"))
]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
self.log.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
self.log.error(
"Whizard input file was not found in process list, cannot proceed"
)
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin),
"./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
self.log.error("Could not copy %s from %s" %
(whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (
mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (
mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists(
"%s/%s" %
(mySoftDir, self.genmodel.getFile(
self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (
mySoftDir, self.genmodel.getFile(
self.Model)['Value'])
else:
self.log.error(
"Request LesHouches file is missing, cannot proceed"
)
self.setApplicationStatus(
"LesHouches file missing")
return S_ERROR(
"The LesHouches file was not found. Probably you are using a wrong version of whizard."
)
else:
self.log.warn("No file found attached to model %s" %
self.Model)
else:
self.log.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
self.log.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename,
self.energy, self.RandomSeed,
self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
self.log.error('Something went wrong with input file generation')
self.setApplicationStatus(
'Whizard: something went wrong with input file generation')
return S_ERROR(
'Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' %
(leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
self.log.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (
outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (
self.evttype, outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
self.log.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
self.log.info(
'The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(
crosssection)
info['xsection'][process]['err_xsection'] = float(
err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
self.log.info("Status after the application execution is %s" %
str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
self.log.error("Whizard execution completed with errors:")
failed = True
else:
self.log.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
self.log.notice("Looking for output files")
ofnames = glob.glob(outputfilename + '*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep",
self.OutputFile)
else:
self.log.error(
"Whizard execution did not produce a stdhep file")
self.setApplicationStatus(
'Whizard %s Failed to produce STDHEP file' %
(self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
self.log.error("==================================\n StdError:\n")
self.log.error(message)
self.setApplicationStatus('%s Exited With Status %s' %
(self.applicationName, status))
self.log.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK({"OutputFile": self.OutputFile})
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Whizard should not proceed as previous step did not end properly')
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep[ "app" ], dep['version'])
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)]
gridfilesfound = True
LOG.info('Found grid files specific for energy %s' % self.energy)
break
if not gridfilesfound:
LOG.info("Will use generic grid files found, hope the energy is set right")
list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
LOG.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
LOG.error("Whizard input file was not found in process list, cannot proceed")
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
LOG.error("Could not copy %s from %s" % (whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])
else:
LOG.error("Request LesHouches file is missing, cannot proceed")
self.setApplicationStatus("LesHouches file missing")
return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.")
else:
LOG.warn("No file found attached to model %s" % self.Model)
else:
LOG.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
LOG.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy,
self.RandomSeed, self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
LOG.error('Something went wrong with input file generation')
self.setApplicationStatus('Whizard: something went wrong with input file generation')
return S_ERROR('Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
LOG.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (self.evttype,
outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
LOG.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
LOG.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
LOG.info('The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(crosssection)
info['xsection'][process]['err_xsection'] = float(err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
LOG.info("Status after the application execution is %s" % str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
LOG.error("Whizard execution completed with errors:")
failed = True
else:
LOG.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
LOG.notice("Looking for output files")
ofnames = glob.glob(outputfilename+'*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep", self.OutputFile)
else:
LOG.error("Whizard execution did not produce a stdhep file")
self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
LOG.error("==================================\n StdError:\n")
LOG.error(message)
self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status))
LOG.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK( { "OutputFile": self.OutputFile } )
def getEnvScript(self, sysconfig, appname, appversion):
""" Return the environment script in case CVMFS is not here.
"""
env_script_name = "MokkaEnv.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
self.log.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
mySoftwareRoot = os.sep.join(
myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1])
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, "mokka",
self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
remoteMysqlInstall = mySoftwareRoot
removeLibc(remoteMysqlInstall + '/mysql4grid/lib64/mysql')
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write(
'######################################################\n')
script.write(
'# Dynamically generated script to get teh Mokka env. #\n')
script.write(
'######################################################\n')
#if(os.path.exists(sharedArea+"/initILCSOFT.sh")):
# script.write("%s/initILCSOFT.sh"%sharedArea)
script.write("declare -x g4releases=%s\n" % (myMokkaDir))
script.write("declare -x G4SYSTEM=Linux-g++\n")
script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n")
#script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n")
script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n')
script.write(
'declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n')
script.write(
'declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n'
)
script.write(
'declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n'
)
###No such data on the GRID (???)
#script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n')
script.write(
'declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n')
#script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n")
script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n')
script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir))
script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall))
script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir)
if new_ld_lib_path:
script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' %
(new_ld_lib_path))
script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' %
remoteMysqlInstall) #for MySQL
script.write(
'declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n'
% remoteMysqlInstall)
#for MySQL
script.write("if [ -e %s/%s ]; then\n" %
(myMokkaDir, self.db_dump_name))
script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name))
script.write("fi\n")
#### Do something with the additional environment variables
add_env = self.ops.getOptionsDict(
"/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" %
(self.platform, "mokka", self.applicationVersion))
if add_env['OK']:
for key in add_env['Value'].keys():
script.write('declare -x %s=%s/%s\n' %
(key, mySoftwareRoot, add_env['Value'][key]))
else:
self.log.verbose(
"No additional environment variables needed for this application"
)
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def runIt(self):
"""
Called by JobAgent
Execute the following:
- prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
- prepare the list of files to run on
- set the cacheDirectory and put in there the alias.properties
- set the lcsim file using :any:`prepareLCSIMFile`
- run java and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'LCSIM should not proceed as previous step did not end properly'
)
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
'LCSIM was not found in either the local area or shared area:',
res['Message'])
return res
lcsim_name = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
self.applicationVersion)
runonslcio = []
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCSIM: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value']
#for inputfile in inputfilelist:
# self.log.verbose("Will try using %s"%(os.path.basename(inputfile)))
# runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))
retMod = self.downloadDetectorZip()
if not retMod:
return retMod
##Collect jar files to put in classspath
jars = []
if os.path.exists("lib"):
for libs in os.listdir("lib"):
if os.path.basename(libs).find(".jar") > 0:
jars.append(os.path.abspath(os.path.join("lib", libs)))
new_ld_lib_path = "./lib:%s" % new_ld_lib_path
#Remove any libc remaining in .lib
removeLibc("./lib")
###Define cache directory as local folder
aliasproperties = os.path.basename(self.aliasproperties)
cachedir = os.getcwd()
if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
try:
os.mkdir(os.path.join(cachedir, ".lcsim"))
except OSError, x:
self.log.error("Could not create .lcsim folder !", str(x))
def test_getsoftwarefolder_apptar_fails(self):
with patch('%s.Operations.getValue' % MODULE_NAME,
new=Mock(return_value='')):
result = getSoftwareFolder('a', 'b', 'c')
assertDiracFailsWith(result, 'could not find b, c name from cs',
self)
def execute(self):
""" Run it now.
"""
self.result = self.resolveInputVariables()
if not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('PostGenSelection should not proceed as previous step did not end properly')
if 'ROOTSYS' not in os.environ:
return S_OK('Root environment is not set')
res = getSoftwareFolder(self.platform, "postgensel", self.applicationVersion)
if not res['OK']:
LOG.error("Failed finding the sofware")
self.setApplicationStatus('PostGenSel: Could not find neither local area not shared area install')
return res
mySoftDir = res['Value']
InputFile = os.path.basename(self.InputFile[0])
base_file = InputFile.replace(".stdhep", "")
scriptName = 'PostGenSel_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName,'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('declare -x DEBUG=ON\n')
script.write('declare -x INDIR=$PWD/\n')
script.write('declare -x MCGEN=WHIZARD\n')
comm = "readstdhep 100000 %s\n" % base_file
LOG.info("Running %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('PostGenSelection_Read %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=20971520)
resultTuple = self.result['Value']
status = resultTuple[0]
if not status == 0:
LOG.error("Reading did not proceed properly")
self.setApplicationStatus('PostGenSelection_Read Exited With Status %s' % (status))
return S_ERROR('PostGenSelection_Read Exited With Status %s' % (status))
if not os.path.exists(base_file + ".dat"):
LOG.error('%s does not exist locally, something went wrong, cannot proceed' % (base_file + ".dat"))
self.setApplicationStatus('%s not there!' % (base_file + ".dat"))
return S_ERROR('%s file does not exist' % (base_file + ".dat"))
os.rename(base_file + ".stdhep", base_file + "-old.stdhep")
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('declare -x DEBUG=ON\n')
script.write('declare -x INDIR=$PWD/\n')
script.write('declare -x MCGEN=WHIZARD\n')
comm = 'writestdhep 100000 %s %s > writestdhep.out\n' % (self.NbEvtsKept, base_file)
LOG.info('Running %s' % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('PostGenSelection_Write %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
resultTuple = self.result['Value']
status = resultTuple[0]
#need to update the number of events kept
self.workflow_commons["NbOfEvts"] = self.NbEvtsKept
return self.finalStatusReport(status)
def test_getsoftwarefolder_from_cvmfs( self ):
with patch('%s.checkCVMFS' % MODULE_NAME, new=Mock(return_value=S_OK(('mycvmfsfolder/txt', 'otherentry')))) as cvmfs_mock:
result = getSoftwareFolder( 'a', 'b', 'c' )
cvmfs_mock.assert_called_with( 'a', ['b', 'c'] )
assertDiracSucceedsWith_equals( result, 'mycvmfsfolder/txt', self )
def test_getsoftwarefolder_apptar_fails( self ):
with patch('%s.Operations.getValue' % MODULE_NAME, new=Mock(return_value='')):
result = getSoftwareFolder( 'a', 'b', 'c' )
assertDiracFailsWith( result, 'could not find b, c name from cs', self )
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
tomatoDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "tomato",
self.applicationVersion), '')
if not tomatoDir:
self.log.error('Could not get Tomato tar ball name, cannot proceed')
return S_ERROR('Problem accessing CS')
tomatoDir = tomatoDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(tomatoDir)
if not res['Value']:
self.setApplicationStatus('Tomato: Could not find neither local area not shared area install')
return res
myTomatoDir = res['Value']
res = self.prepareMARLIN_DLL(myTomatoDir)
if not res['OK']:
self.log.error('Failed building MARLIN_DLL: %s' % res['Message'])
self.setApplicationStatus('Failed to setup MARLIN_DLL')
return S_ERROR('Something wrong with software installation')
self.envdict['MARLIN_DLL'] = res['Value']
deps = resolveDepsTar(self.systemConfig, "tomato", self.applicationVersion)
for dep in deps:
if dep.lower().count('marlin'):
marlindir = dep.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(marlindir)
if not res['OK']:
self.log.error('Marlin was not found in software directory')
return res
else:
self.envdict['MarlinDIR'] = res['Value']
break
new_ldlibs = ''
if os.environ.has_key('LD_LIBRARY_PATH'):
new_ldlibs = os.path.join(myTomatoDir, 'LDLibs') + ":%s" % os.environ['LD_LIBRARY_PATH']
else:
new_ldlibs = os.path.join(myTomatoDir, 'LDLibs')
self.envdict['LD_LIB_PATH'] = new_ldlibs
res = self.GetInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = 'tomato.xml'
res = PrepareTomatoSalad(None, finalXML, self.OutputFile, listofslcio, self.collection)
if not res['OK']:
self.log.error('Could not prepare the Tomato XML: %s' % res['Message'])
self.setApplicationStatus('Failed to setup Tomato')
return S_ERROR('Failed to setup Tomato')
self.result = self.runMarlin(finalXML, self.envdict)
if not self.result['OK']:
self.log.error('Something wrong during running: %s' % self.result['Message'])
self.setApplicationStatus('Error during running %s' % self.applicationName)
return S_ERROR('Failed to run %s' % self.applicationName)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
def execute(self):
""" Run it now.
"""
self.result = self.resolveInputVariables()
if not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'PostGenSelection should not proceed as previous step did not end properly'
)
if 'ROOTSYS' not in os.environ:
return S_OK('Root environment is not set')
res = getSoftwareFolder(self.platform, "postgensel",
self.applicationVersion)
if not res['OK']:
self.log.error("Failed finding the sofware")
self.setApplicationStatus(
'PostGenSel: Could not find neither local area not shared area install'
)
return res
mySoftDir = res['Value']
InputFile = os.path.basename(self.InputFile[0])
base_file = InputFile.replace(".stdhep", "")
scriptName = 'PostGenSel_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('declare -x DEBUG=ON\n')
script.write('declare -x INDIR=$PWD/\n')
script.write('declare -x MCGEN=WHIZARD\n')
comm = "readstdhep 100000 %s\n" % base_file
self.log.info("Running %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('PostGenSelection_Read %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
status = resultTuple[0]
if not status == 0:
self.log.error("Reading did not proceed properly")
self.setApplicationStatus(
'PostGenSelection_Read Exited With Status %s' % (status))
return S_ERROR('PostGenSelection_Read Exited With Status %s' %
(status))
if not os.path.exists(base_file + ".dat"):
self.log.error(
'%s does not exist locally, something went wrong, cannot proceed'
% (base_file + ".dat"))
self.setApplicationStatus('%s not there!' % (base_file + ".dat"))
return S_ERROR('%s file does not exist' % (base_file + ".dat"))
os.rename(base_file + ".stdhep", base_file + "-old.stdhep")
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('declare -x DEBUG=ON\n')
script.write('declare -x INDIR=$PWD/\n')
script.write('declare -x MCGEN=WHIZARD\n')
comm = 'writestdhep 100000 %s %s > writestdhep.out\n' % (
self.NbEvtsKept, base_file)
self.log.info('Running %s' % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('PostGenSelection_Write %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
status = resultTuple[0]
#need to update the number of events kept
self.workflow_commons["NbOfEvts"] = self.NbEvtsKept
return self.finalStatusReport(status)
def getEnvScript(self, sysconfig, appname, appversion):
""" Return the environment script in case CVMFS is not here.
"""
env_script_name = "MokkaEnv.sh"
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['OK']:
LOG.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
mySoftwareRoot = os.sep.join(myMokkaDir.rstrip(os.sep).split(os.sep)[0:-1])
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, "mokka", self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
remoteMysqlInstall = mySoftwareRoot
removeLibc(remoteMysqlInstall+'/mysql4grid/lib64/mysql')
script = open(env_script_name, "w")
script.write('#!/bin/sh \n')
script.write('######################################################\n')
script.write('# Dynamically generated script to get teh Mokka env. #\n')
script.write('######################################################\n')
#if(os.path.exists(sharedArea+"/initILCSOFT.sh")):
# script.write("%s/initILCSOFT.sh"%sharedArea)
script.write("declare -x g4releases=%s\n" % (myMokkaDir))
script.write("declare -x G4SYSTEM=Linux-g++\n")
script.write("declare -x G4INSTALL=$g4releases/share/$g4version\n")
#script.write("export G4SYSTEM G4INSTALL G4LIB CLHEP_BASE_DIR\n")
script.write('declare -x G4LEDATA="$g4releases/G4LEDATA"\n')
script.write('declare -x G4NEUTRONHPDATA="$g4releases/sl4/g4data/g4dataNDL"\n')
script.write('declare -x G4LEVELGAMMADATA="$g4releases/sl4/g4data/g4dataPhotonEvap"\n')
script.write('declare -x G4RADIOACTIVEDATA="$g4releases/sl4/g4data/g4dataRadiativeDecay"\n')
###No such data on the GRID (???)
#script.write('G4ELASTICDATA="$g4releases/share/data/G4ELASTIC1.1"\n')
script.write('declare -x G4ABLADATA="$g4releases/sl4/g4data/g4dataABLA"\n')
#script.write("export G4LEDATA G4NEUTRONHPDATA G4LEVELGAMMADATA G4RADIOACTIVEDATA G4ABLADATA\n")
script.write('declare -x G4NEUTRONHP_NEGLECT_DOPPLER=1\n')
script.write('declare -x PATH=%s:$PATH\n' % myMokkaDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % (myMokkaDir))
script.write('declare -x MYSQL=%s/mysql4grid\n' % (remoteMysqlInstall))
script.write('declare -x MOKKATARBALL=%s\n' % myMokkaDir)
if new_ld_lib_path:
script.write('declare -x LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH\n' % (new_ld_lib_path))
script.write('declare -x PATH=%s/mysql4grid/bin:$PATH\n' % remoteMysqlInstall)#for MySQL
script.write('declare -x LD_LIBRARY_PATH=%s/mysql4grid/lib64/mysql:$LD_LIBRARY_PATH\n' % remoteMysqlInstall)
#for MySQL
script.write("if [ -e %s/%s ]; then\n" % (myMokkaDir, self.db_dump_name))
script.write(" cp %s/%s .\n" % (myMokkaDir, self.db_dump_name) )
script.write("fi\n")
#### Do something with the additional environment variables
add_env = self.ops.getOptionsDict("/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" % (self.platform,
"mokka",
self.applicationVersion))
if add_env['OK']:
for key in add_env['Value'].keys():
script.write('declare -x %s=%s/%s\n' % (key, mySoftwareRoot, add_env['Value'][key]))
else:
LOG.verbose("No additional environment variables needed for this application")
script.close()
#os.chmod(env_script_name, 0755)
return S_OK(os.path.abspath(env_script_name))
def execute(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by L{PrepareXMLFile}
- run Marlin and catch the exit code
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
marlinDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig,
"marlin",
self.applicationVersion), '')
marlinDir = marlinDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(marlinDir)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "marlin", self.applicationVersion)
res = self.GetInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.systemConfig, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory')
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if os.path.exists(os.path.join(myMarlinDir, 'Settings', pandorasettings)):
try:
shutil.copy(os.path.join(myMarlinDir, 'Settings', pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
elif steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname,pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
- prepare the list of files to run on
- set the cacheDirectory and put in there the alias.properties
- set the lcsim file using :any:`prepareLCSIMFile`
- run java and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('LCSIM should not proceed as previous step did not end properly')
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error('LCSIM was not found in either the local area or shared area:', res['Message'])
return res
lcsim_name = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)
runonslcio = []
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.setApplicationStatus('LCSIM: missing input slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value']
#for inputfile in inputfilelist:
# LOG.verbose("Will try using %s"%(os.path.basename(inputfile)))
# runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))
retMod = self.downloadDetectorZip()
if not retMod:
return retMod
##Collect jar files to put in classspath
jars = []
if os.path.exists("lib"):
for libs in os.listdir("lib"):
if os.path.basename(libs).find(".jar") > 0:
jars.append(os.path.abspath(os.path.join("lib", libs)))
new_ld_lib_path = "./lib:%s" % new_ld_lib_path
#Remove any libc remaining in .lib
removeLibc("./lib")
###Define cache directory as local folder
aliasproperties = os.path.basename(self.aliasproperties)
cachedir = os.getcwd()
if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
try:
os.mkdir(os.path.join(cachedir, ".lcsim"))
except OSError, x:
LOG.error("Could not create .lcsim folder !", str(x))
def execute(self):
""" Execute the module, called by JobAgent
"""
# Get input variables
self.result = self.resolveInputVariables()
# Checks
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Cannot find input file")
self.setApplicationStatus('StdHepSplit: missing input stdhep file')
return S_ERROR('Missing stdhep file!')
runonstdhep = res['Value'][0]
else:
self.log.warn("No files found to split")
return S_OK("No files found to process")
# removeLibc
#removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )
prefix = ''
if self.OutputFile:
prefix = self.OutputFile.split('.stdhep')[0]
else:
prefix = "this_split"
#because we need to make sure the files end up in the base directory at the end
self.OutputFile = prefix
self.log.info("Will rename all files using '%s' as base." % prefix)
# Setting up script
res = getSoftwareFolder(self.platform, "stdhepsplit", self.applicationVersion)
if not res['OK']:
self.log.error("Failed to find the software")
self.setApplicationStatus('StdHepSplit: Could not find neither local area not shared area install')
return res
mysplitDir = res['Value']
new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit", self.applicationVersion)
LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by LCIOConcatenate module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
%s/hepsplit --infile %s --nw_per_file %s --outpref %s
exit $?
""" % (
LD_LIBRARY_PATH,
mysplitDir,
runonstdhep,
self.nbEventsPerSlice,
prefix
)
# Write script to file
scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER )
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open( scriptPath, 'w' )
script.write( scriptContent )
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod( scriptPath, 0755 )
command = '"./%s"' % ( scriptPath )
self.setApplicationStatus( 'StdHepSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
self.stdError = ''
self.result = shellCall( 0,
command,
callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520
)
# Check results
resultTuple = self.result['Value']
status = resultTuple[0]
if not os.path.exists(self.applicationLog):
self.log.error("Cannot access log file, cannot proceed")
return S_ERROR("Failed reading the log file")
with open(self.applicationLog, "r") as logf:
numberofeventsdict = {}
fname = ''
for line in logf:
line = line.rstrip()
if line.count('Open output file'):
fname = line.split()[-1].rstrip().rstrip("\0")
numberofeventsdict[fname] = 0
elif line.count("Record") and not line.count('Output Begin Run') :
#print line
val = line.split("=")[1].rstrip().lstrip()
if val != '0':
numberofeventsdict[fname] = int(val)
self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)
##Now update the workflow_commons dict with the relation between filename and number of events:
#needed for the registerOutputData
self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict
if self.listoutput:
outputlist = []
for of in numberofeventsdict.keys():
item = {}
item['outputFile'] = of
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
self.step_commons['listoutput'] = outputlist
#Not only the step_commons must be updated
if self.workflow_commons.has_key('ProductionOutputData'):
proddata = self.workflow_commons['ProductionOutputData'].split(";")
finalproddata = []
this_split_data = ''
for item in proddata:
if not item.count(prefix):
finalproddata.append(item)
else:
this_split_data = item
path = os.path.dirname(this_split_data)
for of in numberofeventsdict.keys():
finalproddata.append(os.path.join(path, of))
self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata)
self.log.info( "Status after the application execution is %s" % str( status ) )
if status == 2:
self.log.info("Reached end of input file")
status = 0
self.listDir()
return self.finalStatusReport(status)
def getEnvScript(self, platform, appname, appversion):
""" Create an environment script for ddsim. Only used when CVMFS native installation not available
We need ROOTSYS, G4INSTALL and G4DATA as additional environment variables in the CS
We need to set DD4hepINSTALL !
"""
##Need to fetch the new LD_LIBRARY_PATH
newLDLibraryPath = getNewLDLibs(platform, appname, appversion)
softwareFolder = getSoftwareFolder(platform, appname, appversion)
if not softwareFolder['OK']:
return softwareFolder
softwareRoot = softwareFolder['Value']
envName = "DDSimEnv.sh"
script = []
script.append("#!/bin/bash")
script.append("##########################")
script.append("## Env script for DDSim ##")
script.append("##########################")
addEnv = self.ops.getOptionsDict("/AvailableTarBalls/%s/%s/%s/AdditionalEnvVar" % (platform,
appname,
appversion))
if addEnv['OK']:
for variable, value in addEnv['Value'].iteritems():
script.append('declare -x %s=%s' % (variable, value))
else:
self.log.verbose("No additional environment variables needed for this application")
## if these variables are not given in the configuration system we guess them from the G4DATA folder
for var, folder in { "G4LEDATA" : "G4EMLOW",
"G4LEVELGAMMADATA" : "PhotonEvaporation",
"G4NEUTRONXSDATA" : "G4NEUTRONXS",
"G4SAIDXSDATA" : "G4SAIDDATA",
"G4RADIOACTIVEDATA" : "RadioactiveDecay",
"G4NEUTRONHPDATA" : "G4NDL",
}.iteritems():
if var not in addEnv.get( 'Value', {} ):
script.append( 'declare -x %s=$(ls -d $G4DATA/%s*)' %( var, folder ) )
init = self.ops.getValue("/AvailableTarBalls/%s/%s/%s/InitScript" % (platform,
appname,
appversion), None)
if init:
script.append( 'source %s' % init )
##Executable:
script.append('declare -x PATH=%s/bin:$PATH' % softwareRoot )
script.append('declare -x DD4hepINSTALL=%s' % softwareRoot )
##Python objects, pyroot
script.append('declare -x PYTHONPATH=%s/lib/python:$PYTHONPATH' % softwareRoot )
script.append('declare -x PYTHONPATH=$ROOTSYS/lib:$PYTHONPATH' )
##For ROOT CLING Stuff
script.append('declare -x CPATH=%s/include:$CPATH' % softwareRoot )
##Libraries
if newLDLibraryPath:
script.append('declare -x LD_LIBRARY_PATH=$LD_LIBRARY_PATH:%s' % newLDLibraryPath)
## user provided libraries are in lib in the job working directory
if os.path.exists( "%s/lib" % os.getcwd() ):
script.append('declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH' % os.getcwd() )
##Root Path, just in case
script.append('declare -x PATH=$ROOTSYS/bin:$PATH')
##Root and Geant4 Library Path
script.append('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:$LD_LIBRARY_PATH')
script.append('declare -x LD_LIBRARY_PATH=$G4INSTALL/lib64:$LD_LIBRARY_PATH')
script.append('declare -x LD_LIBRARY_PATH=$DD4hepINSTALL/lib:$LD_LIBRARY_PATH')
with open(envName,"w") as scriptFile:
scriptFile.write( "\n".join(script) )
scriptFile.write( "\n" )
os.chmod(envName, 0755)
return S_OK(os.path.abspath(envName))
def execute(self):
""" Execute the module, called by JobAgent
"""
# Get input variables
self.result = self.resolveInputVariables()
# Checks
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Cannot find input file")
self.setApplicationStatus(
'StdHepSplit: missing input stdhep file')
return S_ERROR('Missing stdhep file!')
runonstdhep = res['Value'][0]
else:
self.log.warn("No files found to split")
return S_OK("No files found to process")
prefix = ''
if self.OutputFile:
prefix = self.OutputFile.split('.stdhep')[0]
else:
prefix = "this_split"
#because we need to make sure the files end up in the base directory at the end
self.OutputFile = prefix
self.log.info("Will rename all files using '%s' as base." % prefix)
# Setting up script
res = getSoftwareFolder(self.platform, "stdhepsplit",
self.applicationVersion)
if not res['OK']:
self.log.error("Failed to find the software")
self.setApplicationStatus(
'StdHepSplit: Could not find neither local area not shared area install'
)
return res
mysplitDir = res['Value']
new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit",
self.applicationVersion)
LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib
maxReadString = ''
## hepsplit has a off-by-one error so we add 1 to max read to actually read maxRead events
if self.maxRead:
self.maxRead += 1
maxReadString = " --maxread %s" % self.maxRead
scriptContent = """
#!/bin/sh
################################################################################
# Dynamically generated script by LCIOConcatenate module #
################################################################################
declare -x LD_LIBRARY_PATH=%s
%s/hepsplit --infile %s --nw_per_file %s --outpref %s%s
exit $?
""" % (
LD_LIBRARY_PATH,
mysplitDir,
runonstdhep,
self.nbEventsPerSlice,
prefix,
maxReadString,
)
# Write script to file
scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptPath):
os.remove(scriptPath)
script = open(scriptPath, 'w')
script.write(scriptContent)
script.close()
# Setup log file for application stdout
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# Run code
os.chmod(scriptPath, 0755)
command = '"./%s"' % (scriptPath)
self.setApplicationStatus('StdHepSplit %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
command,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
status = resultTuple[0]
if not os.path.exists(self.applicationLog):
self.log.error("Cannot access log file, cannot proceed")
return S_ERROR("Failed reading the log file")
with open(self.applicationLog, "r") as logf:
numberofeventsdict = {}
fname = ''
for line in logf:
line = line.rstrip()
if line.count('Open output file'):
fname = line.split()[-1].rstrip().rstrip("\0")
numberofeventsdict[fname] = 0
elif line.count(
"Record") and not line.count('Output Begin Run'):
#print line
val = line.split("=")[1].rstrip().lstrip()
if val != '0':
numberofeventsdict[fname] = int(val)
self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)
##Now update the workflow_commons dict with the relation between filename and number of events:
#needed for the registerOutputData
self.workflow_commons[
'file_number_of_event_relation'] = numberofeventsdict
if self.listoutput:
outputlist = []
for of in numberofeventsdict:
item = {}
item['outputFile'] = of
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
self.step_commons['listoutput'] = outputlist
#Not only the step_commons must be updated
if 'ProductionOutputData' in self.workflow_commons:
proddata = self.workflow_commons['ProductionOutputData'].split(";")
finalproddata = []
this_split_data = ''
for item in proddata:
if not item.count(prefix):
finalproddata.append(item)
else:
this_split_data = item
path = os.path.dirname(this_split_data)
for of in numberofeventsdict:
finalproddata.append(os.path.join(path, of))
self.workflow_commons['ProductionOutputData'] = ";".join(
finalproddata)
self.log.info("Status after the application execution is %s" %
str(status))
if status == 2:
self.log.info("Reached end of input file")
status = 0
self.listDir()
return self.finalStatusReport(status)