def _get_opts_dict_and_pkl_string(self):
'''Parse the options using gaudirun.py and create a dictionary of the
configuration and pickle the options. The app handler will make a copy
of the .pkl file for each job.'''
tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl')
tmp_py = tempfile.NamedTemporaryFile(suffix='.py')
py_opts = tempfile.NamedTemporaryFile(suffix='.py')
py_opts.write(self._join_opts_files())
py_opts.flush()
gaudirun = 'gaudirun.py -n -v -o %s %s' \
% (tmp_py.name, py_opts.name)
opts_str = ''
err_msg = ''
options = {}
rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
if stdout.find('Gaudi.py') >= 0:
msg = 'The version of gaudirun.py required for your application is not supported.'
raise ValueError(None, msg)
elif stdout.find('no such option: -o') >= 0:
gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name, py_opts.name)
rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
rc = 0
if stdout and rc == 0:
opts_str = stdout
err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name)
err_msg += ' returns valid python syntax'
else:
cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name)
rc, stdout, m = shellEnv_cmd(cmd, self.env)
if rc == 0 and stdout:
opts_str = tmp_py.read()
err_msg = 'Please check gaudirun.py -o file.py produces a ' \
'valid python file.'
if stdout and rc == 0:
try:
options = eval(opts_str)
except Exception as err:
logger.error('Cannot eval() the options file. Exception: %s', err)
from traceback import print_exc
logger.error(' ', print_exc())
raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m)
try:
opts_pkl_string = tmp_pkl.read()
except IOError as err:
logger.error('Cannot read() the temporary pickle file: %s', tmp_pkl.name)
raise err
if not rc == 0:
logger.debug('Failed to run: %s', gaudirun)
raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m)
tmp_pkl.close()
py_opts.close()
tmp_py.close()
return (options, opts_pkl_string)
def get_user_dlls(appname, version, user_release_area, platform, env):
user_ra = user_release_area
update_project_path(user_release_area)
from Ganga.Utility.files import fullpath
full_user_ra = fullpath(user_ra) # expand any symbolic links
# Work our way through the CMTPROJECTPATH until we find a cmt directory
if 'CMTPROJECTPATH' not in env:
return [], [], []
projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
appveruser = os.path.join(appname + '_' + version, 'cmt')
appverrelease = os.path.join(
appname.upper(), appname.upper() + '_' + version, 'cmt')
for projectdir in projectdirs:
projectDir = fullpath(os.path.join(projectdir, appveruser))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
projectDir = fullpath(os.path.join(projectdir, appverrelease))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
logger.debug('Using the CMT directory %s for identifying projects' % projectDir)
# rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
# capture_stderr=True)
from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)
logger.debug(showProj)
libs = []
merged_pys = []
subdir_pys = {}
project_areas = []
py_project_areas = []
for line in showProj.split('\n'):
for entry in line.split():
if entry.startswith(user_ra) or entry.startswith(full_user_ra):
tmp = entry.rstrip('\)')
libpath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'lib'))
logger.debug(libpath)
project_areas.append(libpath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
# savannah 47793 (remove multiple copies of the same areas)
from Ganga.Utility.util import unique
project_areas = unique(project_areas)
py_project_areas = unique(py_project_areas)
ld_lib_path = []
if 'LD_LIBRARY_PATH' in env:
ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
project_areas_dict = {}
for area in project_areas:
if area in ld_lib_path:
project_areas_dict[area] = ld_lib_path.index(area)
else:
project_areas_dict[area] = 666
from operator import itemgetter
sorted_project_areas = []
for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
sorted_project_areas.append(item[0])
lib_names = []
for libpath in sorted_project_areas:
if os.path.exists(libpath):
for f in os.listdir(libpath):
if lib_names.count(f) > 0:
continue
fpath = os.path.join(libpath, f)
if os.path.exists(fpath):
lib_names.append(f)
libs.append(fpath)
else:
logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath))
for pypath in py_project_areas:
if os.path.exists(pypath):
from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
from Ganga.Utility.Config import getConfig
configGaudi = getConfig('GAUDI')
pyFileCollector(
pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth'])
import pprint
logger.debug("%s", pprint.pformat(libs))
logger.debug("%s", pprint.pformat(merged_pys))
logger.debug("%s", pprint.pformat(subdir_pys))
return libs, merged_pys, subdir_pys
def _get_opts_dict_and_pkl_string(self):
'''Parse the options using gaudirun.py and create a dictionary of the
configuration and pickle the options. The app handler will make a copy
of the .pkl file for each job.'''
tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl')
tmp_py = tempfile.NamedTemporaryFile(suffix='.py')
py_opts = tempfile.NamedTemporaryFile(suffix='.py')
py_opts.write(self._join_opts_files())
py_opts.flush()
gaudirun = 'gaudirun.py -n -v -o %s %s' \
% (tmp_py.name, py_opts.name)
opts_str = ''
err_msg = ''
options = {}
rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
if stdout.find('Gaudi.py') >= 0:
msg = 'The version of gaudirun.py required for your application is not supported.'
raise ValueError(None, msg)
elif stdout.find('no such option: -o') >= 0:
gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name,
py_opts.name)
rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
rc = 0
if stdout and rc == 0:
opts_str = stdout
err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name)
err_msg += ' returns valid python syntax'
else:
cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name)
rc, stdout, m = shellEnv_cmd(cmd, self.env)
if rc == 0 and stdout:
opts_str = tmp_py.read()
err_msg = 'Please check gaudirun.py -o file.py produces a ' \
'valid python file.'
if stdout and rc == 0:
try:
options = eval(opts_str)
except Exception as err:
logger.error('Cannot eval() the options file. Exception: %s',
err)
from traceback import print_exc
logger.error(' ', print_exc())
raise ApplicationConfigurationError(None,
stdout + '###SPLIT###' + m)
try:
opts_pkl_string = tmp_pkl.read()
except IOError as err:
logger.error('Cannot read() the temporary pickle file: %s',
tmp_pkl.name)
raise err
if not rc == 0:
logger.debug('Failed to run: %s', gaudirun)
raise ApplicationConfigurationError(None,
stdout + '###SPLIT###' + m)
tmp_pkl.close()
py_opts.close()
tmp_py.close()
return (options, opts_pkl_string)
def get_user_dlls(appname, version, user_release_area, platform, env):
user_ra = user_release_area
update_project_path(user_release_area)
from Ganga.Utility.files import fullpath
full_user_ra = fullpath(user_ra) # expand any symbolic links
# Work our way through the CMTPROJECTPATH until we find a cmt directory
if 'CMTPROJECTPATH' not in env:
return [], [], []
projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
appveruser = os.path.join(appname + '_' + version, 'cmt')
appverrelease = os.path.join(appname.upper(),
appname.upper() + '_' + version, 'cmt')
for projectdir in projectdirs:
projectDir = fullpath(os.path.join(projectdir, appveruser))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
projectDir = fullpath(os.path.join(projectdir, appverrelease))
logger.debug('Looking for projectdir %s' % projectDir)
if os.path.exists(projectDir):
break
logger.debug('Using the CMT directory %s for identifying projects' %
projectDir)
# rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
# capture_stderr=True)
from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)
logger.debug(showProj)
libs = []
merged_pys = []
subdir_pys = {}
project_areas = []
py_project_areas = []
for line in showProj.split('\n'):
for entry in line.split():
if entry.startswith(user_ra) or entry.startswith(full_user_ra):
tmp = entry.rstrip('\)')
libpath = fullpath(
os.path.join(tmp, 'InstallArea', platform, 'lib'))
logger.debug(libpath)
project_areas.append(libpath)
pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
pypath = fullpath(
os.path.join(tmp, 'InstallArea', platform, 'python'))
logger.debug(pypath)
py_project_areas.append(pypath)
# savannah 47793 (remove multiple copies of the same areas)
from Ganga.Utility.util import unique
project_areas = unique(project_areas)
py_project_areas = unique(py_project_areas)
ld_lib_path = []
if 'LD_LIBRARY_PATH' in env:
ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
project_areas_dict = {}
for area in project_areas:
if area in ld_lib_path:
project_areas_dict[area] = ld_lib_path.index(area)
else:
project_areas_dict[area] = 666
from operator import itemgetter
sorted_project_areas = []
for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
sorted_project_areas.append(item[0])
lib_names = []
for libpath in sorted_project_areas:
if os.path.exists(libpath):
for f in os.listdir(libpath):
if lib_names.count(f) > 0:
continue
fpath = os.path.join(libpath, f)
if os.path.exists(fpath):
lib_names.append(f)
libs.append(fpath)
else:
logger.warning("File %s in %s does not exist. Skipping...",
str(f), str(libpath))
for pypath in py_project_areas:
if os.path.exists(pypath):
from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
from Ganga.Utility.Config import getConfig
configGaudi = getConfig('GAUDI')
pyFileCollector(pypath, merged_pys, subdir_pys,
configGaudi['pyFileCollectionDepth'])
import pprint
logger.debug("%s", pprint.pformat(libs))
logger.debug("%s", pprint.pformat(merged_pys))
logger.debug("%s", pprint.pformat(subdir_pys))
return libs, merged_pys, subdir_pys
