Ejemplo n.º 1
0
    def _get_opts_dict_and_pkl_string(self):
        '''Parse the options using gaudirun.py and create a dictionary of the
        configuration and pickle the options. The app handler will make a copy
        of the .pkl file for each job.'''
        tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl')
        tmp_py = tempfile.NamedTemporaryFile(suffix='.py')
        py_opts = tempfile.NamedTemporaryFile(suffix='.py')
        py_opts.write(self._join_opts_files())
        py_opts.flush()

        gaudirun = 'gaudirun.py -n -v -o %s %s' \
                   % (tmp_py.name, py_opts.name)
        opts_str = ''
        err_msg = ''
        options = {}

        rc, stdout, m = shellEnv_cmd(gaudirun, self.env)

        if stdout.find('Gaudi.py') >= 0:
            msg = 'The version of gaudirun.py required for your application is not supported.'
            raise ValueError(None, msg)

        elif stdout.find('no such option: -o') >= 0:
            gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name, py_opts.name)
            rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
            rc = 0

            if stdout and rc == 0:
                opts_str = stdout
                err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name)
                err_msg += ' returns valid python syntax'

        else:
            cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name)
            rc, stdout, m = shellEnv_cmd(cmd, self.env)
            if rc == 0 and stdout:
                opts_str = tmp_py.read()
                err_msg = 'Please check gaudirun.py -o file.py produces a ' \
                          'valid python file.'

        if stdout and rc == 0:
            try:
                options = eval(opts_str)
            except Exception as err:
                logger.error('Cannot eval() the options file. Exception: %s', err)
                from traceback import print_exc
                logger.error(' ', print_exc())
                raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m)
            try:
                opts_pkl_string = tmp_pkl.read()
            except IOError as err:
                logger.error('Cannot read() the temporary pickle file: %s', tmp_pkl.name)
                raise err

        if not rc == 0:
            logger.debug('Failed to run: %s', gaudirun)
            raise ApplicationConfigurationError(None, stdout + '###SPLIT###' + m)

        tmp_pkl.close()
        py_opts.close()
        tmp_py.close()
        return (options, opts_pkl_string)
Ejemplo n.º 2
0
def get_user_dlls(appname, version, user_release_area, platform, env):

    user_ra = user_release_area
    update_project_path(user_release_area)
    from Ganga.Utility.files import fullpath
    full_user_ra = fullpath(user_ra)  # expand any symbolic links

    # Work our way through the CMTPROJECTPATH until we find a cmt directory
    if 'CMTPROJECTPATH' not in env:
        return [], [], []
    projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
    appveruser = os.path.join(appname + '_' + version, 'cmt')
    appverrelease = os.path.join(
        appname.upper(), appname.upper() + '_' + version, 'cmt')

    for projectdir in projectdirs:
        projectDir = fullpath(os.path.join(projectdir, appveruser))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
        projectDir = fullpath(os.path.join(projectdir, appverrelease))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
    logger.debug('Using the CMT directory %s for identifying projects' % projectDir)
    # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
    # capture_stderr=True)
    from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
    rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)

    logger.debug(showProj)

    libs = []
    merged_pys = []
    subdir_pys = {}
    project_areas = []
    py_project_areas = []

    for line in showProj.split('\n'):
        for entry in line.split():
            if entry.startswith(user_ra) or entry.startswith(full_user_ra):
                tmp = entry.rstrip('\)')
                libpath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'lib'))
                logger.debug(libpath)
                project_areas.append(libpath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', platform, 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)

    # savannah 47793 (remove multiple copies of the same areas)
    from Ganga.Utility.util import unique
    project_areas = unique(project_areas)
    py_project_areas = unique(py_project_areas)

    ld_lib_path = []
    if 'LD_LIBRARY_PATH' in env:
        ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
        project_areas_dict = {}
    for area in project_areas:
        if area in ld_lib_path:
            project_areas_dict[area] = ld_lib_path.index(area)
        else:
            project_areas_dict[area] = 666
    from operator import itemgetter
    sorted_project_areas = []
    for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
        sorted_project_areas.append(item[0])

    lib_names = []
    for libpath in sorted_project_areas:
        if os.path.exists(libpath):
            for f in os.listdir(libpath):
                if lib_names.count(f) > 0:
                    continue
                fpath = os.path.join(libpath, f)
                if os.path.exists(fpath):
                    lib_names.append(f)
                    libs.append(fpath)
                else:
                    logger.warning("File %s in %s does not exist. Skipping...", str(f), str(libpath))

    for pypath in py_project_areas:
        if os.path.exists(pypath):
            from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
            from Ganga.Utility.Config import getConfig
            configGaudi = getConfig('GAUDI')
            pyFileCollector(
                pypath, merged_pys, subdir_pys, configGaudi['pyFileCollectionDepth'])

    import pprint
    logger.debug("%s", pprint.pformat(libs))
    logger.debug("%s", pprint.pformat(merged_pys))
    logger.debug("%s", pprint.pformat(subdir_pys))

    return libs, merged_pys, subdir_pys
Ejemplo n.º 3
0
    def _get_opts_dict_and_pkl_string(self):
        '''Parse the options using gaudirun.py and create a dictionary of the
        configuration and pickle the options. The app handler will make a copy
        of the .pkl file for each job.'''
        tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl')
        tmp_py = tempfile.NamedTemporaryFile(suffix='.py')
        py_opts = tempfile.NamedTemporaryFile(suffix='.py')
        py_opts.write(self._join_opts_files())
        py_opts.flush()

        gaudirun = 'gaudirun.py -n -v -o %s %s' \
                   % (tmp_py.name, py_opts.name)
        opts_str = ''
        err_msg = ''
        options = {}

        rc, stdout, m = shellEnv_cmd(gaudirun, self.env)

        if stdout.find('Gaudi.py') >= 0:
            msg = 'The version of gaudirun.py required for your application is not supported.'
            raise ValueError(None, msg)

        elif stdout.find('no such option: -o') >= 0:
            gaudirun = 'gaudirun.py -n -v -p %s %s' % (tmp_pkl.name,
                                                       py_opts.name)
            rc, stdout, m = shellEnv_cmd(gaudirun, self.env)
            rc = 0

            if stdout and rc == 0:
                opts_str = stdout
                err_msg = 'Please check %s -v %s' % (cmdbase, py_opts.name)
                err_msg += ' returns valid python syntax'

        else:
            cmd = 'gaudirun.py -n -p %s %s' % (tmp_pkl.name, py_opts.name)
            rc, stdout, m = shellEnv_cmd(cmd, self.env)
            if rc == 0 and stdout:
                opts_str = tmp_py.read()
                err_msg = 'Please check gaudirun.py -o file.py produces a ' \
                          'valid python file.'

        if stdout and rc == 0:
            try:
                options = eval(opts_str)
            except Exception as err:
                logger.error('Cannot eval() the options file. Exception: %s',
                             err)
                from traceback import print_exc
                logger.error(' ', print_exc())
                raise ApplicationConfigurationError(None,
                                                    stdout + '###SPLIT###' + m)
            try:
                opts_pkl_string = tmp_pkl.read()
            except IOError as err:
                logger.error('Cannot read() the temporary pickle file: %s',
                             tmp_pkl.name)
                raise err

        if not rc == 0:
            logger.debug('Failed to run: %s', gaudirun)
            raise ApplicationConfigurationError(None,
                                                stdout + '###SPLIT###' + m)

        tmp_pkl.close()
        py_opts.close()
        tmp_py.close()
        return (options, opts_pkl_string)
Ejemplo n.º 4
0
def get_user_dlls(appname, version, user_release_area, platform, env):

    user_ra = user_release_area
    update_project_path(user_release_area)
    from Ganga.Utility.files import fullpath
    full_user_ra = fullpath(user_ra)  # expand any symbolic links

    # Work our way through the CMTPROJECTPATH until we find a cmt directory
    if 'CMTPROJECTPATH' not in env:
        return [], [], []
    projectdirs = env['CMTPROJECTPATH'].split(os.pathsep)
    appveruser = os.path.join(appname + '_' + version, 'cmt')
    appverrelease = os.path.join(appname.upper(),
                                 appname.upper() + '_' + version, 'cmt')

    for projectdir in projectdirs:
        projectDir = fullpath(os.path.join(projectdir, appveruser))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
        projectDir = fullpath(os.path.join(projectdir, appverrelease))
        logger.debug('Looking for projectdir %s' % projectDir)
        if os.path.exists(projectDir):
            break
    logger.debug('Using the CMT directory %s for identifying projects' %
                 projectDir)
    # rc, showProj, m = shell.cmd1('cd ' + projectDir +';cmt show projects',
    # capture_stderr=True)
    from GangaGaudi.Lib.Applications.GaudiUtils import shellEnv_cmd
    rc, showProj, m = shellEnv_cmd('cmt show projects', env, projectDir)

    logger.debug(showProj)

    libs = []
    merged_pys = []
    subdir_pys = {}
    project_areas = []
    py_project_areas = []

    for line in showProj.split('\n'):
        for entry in line.split():
            if entry.startswith(user_ra) or entry.startswith(full_user_ra):
                tmp = entry.rstrip('\)')
                libpath = fullpath(
                    os.path.join(tmp, 'InstallArea', platform, 'lib'))
                logger.debug(libpath)
                project_areas.append(libpath)
                pypath = fullpath(os.path.join(tmp, 'InstallArea', 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)
                pypath = fullpath(
                    os.path.join(tmp, 'InstallArea', platform, 'python'))
                logger.debug(pypath)
                py_project_areas.append(pypath)

    # savannah 47793 (remove multiple copies of the same areas)
    from Ganga.Utility.util import unique
    project_areas = unique(project_areas)
    py_project_areas = unique(py_project_areas)

    ld_lib_path = []
    if 'LD_LIBRARY_PATH' in env:
        ld_lib_path = env['LD_LIBRARY_PATH'].split(':')
        project_areas_dict = {}
    for area in project_areas:
        if area in ld_lib_path:
            project_areas_dict[area] = ld_lib_path.index(area)
        else:
            project_areas_dict[area] = 666
    from operator import itemgetter
    sorted_project_areas = []
    for item in sorted(project_areas_dict.items(), key=itemgetter(1)):
        sorted_project_areas.append(item[0])

    lib_names = []
    for libpath in sorted_project_areas:
        if os.path.exists(libpath):
            for f in os.listdir(libpath):
                if lib_names.count(f) > 0:
                    continue
                fpath = os.path.join(libpath, f)
                if os.path.exists(fpath):
                    lib_names.append(f)
                    libs.append(fpath)
                else:
                    logger.warning("File %s in %s does not exist. Skipping...",
                                   str(f), str(libpath))

    for pypath in py_project_areas:
        if os.path.exists(pypath):
            from GangaGaudi.Lib.Applications.GaudiUtils import pyFileCollector
            from Ganga.Utility.Config import getConfig
            configGaudi = getConfig('GAUDI')
            pyFileCollector(pypath, merged_pys, subdir_pys,
                            configGaudi['pyFileCollectionDepth'])

    import pprint
    logger.debug("%s", pprint.pformat(libs))
    logger.debug("%s", pprint.pformat(merged_pys))
    logger.debug("%s", pprint.pformat(subdir_pys))

    return libs, merged_pys, subdir_pys