def _get_opts_dict_and_pkl_string(self): '''Parse the options using gaudirun.py and create a dictionary of the configuration and pickle the options. The app handler will make a copy of the .pkl file for each job.''' tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl') tmp_py = tempfile.NamedTemporaryFile(suffix='.py') py_opts = tempfile.NamedTemporaryFile(suffix='.py') py_opts.write(self._join_opts_files()) py_opts.flush() gaudirun = 'gaudirun.py -n -v -o %s %s' \ % (tmp_py.name, py_opts.name) opts_str = '' err_msg = '' options = {} rc, stdout, m = shellEnv_cmd(gaudirun, self.env) if stdout.find('Gaudi.py') >= 0: msg = 'The version of gaudirun.py required for your application is not supported.' raise ValueError(None, msg) elif stdout.find('no such option: -o') >= 0: gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name, py_opts.name) rc, stdout, m = shellEnv_cmd(gaudirun, self.env) rc = 0 if stdout and rc == 0: opts_str = stdout err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name) err_msg += ' returns valid python syntax' else: cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name) rc, stdout, m = shellEnv_cmd(cmd, self.env) if rc == 0 and stdout: opts_str = tmp_py.read() err_msg = 'Please check gaudirun.py -o file.py produces a ' \ 'valid python file.' if stdout and rc == 0: try: options = eval(opts_str) except Exception as err: logger.error('Cannot eval() the options file. Exception: %s', err) from traceback import print_exc logger.error(' ', print_exc()) raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m) try: opts_pkl_string = tmp_pkl.read() except IOError as err: logger.error('Cannot read() the temporary pickle file: %s', tmp_pkl.name) raise err if not rc == 0: logger.debug('Failed to run: %s', gaudirun) raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m) tmp_pkl.close() py_opts.close() tmp_py.close() return (options, opts_pkl_string)
def get_user_dlls(appname, version, user_release_area, platform, env): user_ra = user_release_area update_project_path(user_release_area) from Ganga.Utility.files import fullpath full_user_ra = fullpath(user_ra) # expand any symbolic links # Work our way through the CMTPROJECTPATH until we find a cmt directory if 'CMTPROJECTPATH' not in env: return [], [], [] projectdirs = env['CMTPROJECTPATH'].split(os.pathsep) appveruser = os.path.join(appname + '_' + version, 'cmt') appverrelease = os.path.join( appname.upper(), appname.upper() + '_' + version, 'cmt') for projectdir in projectdirs: projectDir = fullpath(os.path.join(projectdir, appveruser)) logger.debug('Looking for projectdir %s' % projectDir) if os.path.exists(projectDir): break projectDir = fullpath(os.path.join(projectdir, appverrelease)) logger.debug('Looking for projectdir %s' % projectDir) if os.path.exists(projectDir): break logger.debug('Using the CMT directory %s for identifying projects' % projectDir) # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects', # capture_stderr=True) from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir) logger.debug(showProj) libs = [] merged_pys = [] subdir_pys = {} project_areas = [] py_project_areas = [] for line in showProj.split('\n'): for entry in line.split(): if entry.startswith(user_ra) or entry.startswith(full_user_ra): tmp = entry.rstrip('\)') libpath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'lib')) logger.debug(libpath) project_areas.append(libpath) pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python')) logger.debug(pypath) py_project_areas.append(pypath) pypath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'python')) logger.debug(pypath) py_project_areas.append(pypath) # savannah 47793 (remove multiple copies of the same areas) from Ganga.Utility.util import unique project_areas = unique(project_areas) py_project_areas = unique(py_project_areas) ld_lib_path = [] if 'LD_LIBRARY_PATH' in env: ld_lib_path = env['LD_LIBRARY_PATH'].split(':') project_areas_dict = {} for area in project_areas: if area in ld_lib_path: project_areas_dict[area] = ld_lib_path.index(area) else: project_areas_dict[area] = 666 from operator import itemgetter sorted_project_areas = [] for item in sorted(project_areas_dict.items(), key=itemgetter(1)): sorted_project_areas.append(item[0]) lib_names = [] for libpath in sorted_project_areas: if os.path.exists(libpath): for f in os.listdir(libpath): if lib_names.count(f) > 0: continue fpath = os.path.join(libpath, f) if os.path.exists(fpath): lib_names.append(f) libs.append(fpath) else: logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath)) for pypath in py_project_areas: if os.path.exists(pypath): from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector from Ganga.Utility.Config import getConfig configGaudi = getConfig('GAUDI') pyFileCollector( pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth']) import pprint logger.debug("%s", pprint.pformat(libs)) logger.debug("%s", pprint.pformat(merged_pys)) logger.debug("%s", pprint.pformat(subdir_pys)) return libs, merged_pys, subdir_pys
def _get_opts_dict_and_pkl_string(self): '''Parse the options using gaudirun.py and create a dictionary of the configuration and pickle the options. The app handler will make a copy of the .pkl file for each job.''' tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl') tmp_py = tempfile.NamedTemporaryFile(suffix='.py') py_opts = tempfile.NamedTemporaryFile(suffix='.py') py_opts.write(self._join_opts_files()) py_opts.flush() gaudirun = 'gaudirun.py -n -v -o %s %s' \ % (tmp_py.name, py_opts.name) opts_str = '' err_msg = '' options = {} rc, stdout, m = shellEnv_cmd(gaudirun, self.env) if stdout.find('Gaudi.py') >= 0: msg = 'The version of gaudirun.py required for your application is not supported.' raise ValueError(None, msg) elif stdout.find('no such option: -o') >= 0: gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name, py_opts.name) rc, stdout, m = shellEnv_cmd(gaudirun, self.env) rc = 0 if stdout and rc == 0: opts_str = stdout err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name) err_msg += ' returns valid python syntax' else: cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name) rc, stdout, m = shellEnv_cmd(cmd, self.env) if rc == 0 and stdout: opts_str = tmp_py.read() err_msg = 'Please check gaudirun.py -o file.py produces a ' \ 'valid python file.' if stdout and rc == 0: try: options = eval(opts_str) except Exception as err: logger.error('Cannot eval() the options file. Exception: %s', err) from traceback import print_exc logger.error(' ', print_exc()) raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m) try: opts_pkl_string = tmp_pkl.read() except IOError as err: logger.error('Cannot read() the temporary pickle file: %s', tmp_pkl.name) raise err if not rc == 0: logger.debug('Failed to run: %s', gaudirun) raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m) tmp_pkl.close() py_opts.close() tmp_py.close() return (options, opts_pkl_string)
def get_user_dlls(appname, version, user_release_area, platform, env): user_ra = user_release_area update_project_path(user_release_area) from Ganga.Utility.files import fullpath full_user_ra = fullpath(user_ra) # expand any symbolic links # Work our way through the CMTPROJECTPATH until we find a cmt directory if 'CMTPROJECTPATH' not in env: return [], [], [] projectdirs = env['CMTPROJECTPATH'].split(os.pathsep) appveruser = os.path.join(appname + '_' + version, 'cmt') appverrelease = os.path.join(appname.upper(), appname.upper() + '_' + version, 'cmt') for projectdir in projectdirs: projectDir = fullpath(os.path.join(projectdir, appveruser)) logger.debug('Looking for projectdir %s' % projectDir) if os.path.exists(projectDir): break projectDir = fullpath(os.path.join(projectdir, appverrelease)) logger.debug('Looking for projectdir %s' % projectDir) if os.path.exists(projectDir): break logger.debug('Using the CMT directory %s for identifying projects' % projectDir) # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects', # capture_stderr=True) from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir) logger.debug(showProj) libs = [] merged_pys = [] subdir_pys = {} project_areas = [] py_project_areas = [] for line in showProj.split('\n'): for entry in line.split(): if entry.startswith(user_ra) or entry.startswith(full_user_ra): tmp = entry.rstrip('\)') libpath = fullpath( os.path.join(tmp, 'InstallArea', platform, 'lib')) logger.debug(libpath) project_areas.append(libpath) pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python')) logger.debug(pypath) py_project_areas.append(pypath) pypath = fullpath( os.path.join(tmp, 'InstallArea', platform, 'python')) logger.debug(pypath) py_project_areas.append(pypath) # savannah 47793 (remove multiple copies of the same areas) from Ganga.Utility.util import unique project_areas = unique(project_areas) py_project_areas = unique(py_project_areas) ld_lib_path = [] if 'LD_LIBRARY_PATH' in env: ld_lib_path = env['LD_LIBRARY_PATH'].split(':') project_areas_dict = {} for area in project_areas: if area in ld_lib_path: project_areas_dict[area] = ld_lib_path.index(area) else: project_areas_dict[area] = 666 from operator import itemgetter sorted_project_areas = [] for item in sorted(project_areas_dict.items(), key=itemgetter(1)): sorted_project_areas.append(item[0]) lib_names = [] for libpath in sorted_project_areas: if os.path.exists(libpath): for f in os.listdir(libpath): if lib_names.count(f) > 0: continue fpath = os.path.join(libpath, f) if os.path.exists(fpath): lib_names.append(f) libs.append(fpath) else: logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath)) for pypath in py_project_areas: if os.path.exists(pypath): from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector from Ganga.Utility.Config import getConfig configGaudi = getConfig('GAUDI') pyFileCollector(pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth']) import pprint logger.debug("%s", pprint.pformat(libs)) logger.debug("%s", pprint.pformat(merged_pys)) logger.debug("%s", pprint.pformat(subdir_pys)) return libs, merged_pys, subdir_pys