Пример #1
0
    def install_step(self):
        "Install NVIDIA libs simply by copying files. We can't user the installer because it requires root privileges."

        # list of libs
        libs = expand_glob_paths([os.path.join(self.libsdir, 'lib*.so*')])
        libs += expand_glob_paths([os.path.join(self.libsdir, '*.la')])
        libs += [os.path.join(self.libsdir, 'nvidia_drv.so')]

        # list of binaries
        binaries = ['nvidia-bug-report.sh',
                    'nvidia-cuda-mps-control',
                    'nvidia-cuda-mps-server',
                    'nvidia-debugdump',
                    'nvidia-settings',
                    'nvidia-smi',
                    'nvidia-xconfig']
        binaries = [os.path.join(self.libsdir, x) for x in binaries]

        # list of manpages
        manpages = ['nvidia-settings.1.gz',
                    'nvidia-cuda-mps-control.1.gz',
                    'nvidia-xconfig.1.gz',
                    'nvidia-smi.1.gz']
        manpages = [os.path.join(self.libsdir, x) for x in manpages]

        copy(libs, os.path.join(self.installdir, 'lib64'))
        copy(binaries, os.path.join(self.installdir, 'bin'))
        copy(manpages, os.path.join(self.installdir, 'man', 'man1'))
Пример #2
0
    def test_expand_glob_paths(self):
        """Test expand_glob_paths function."""
        for dirname in ['empty_dir', 'test_dir']:
            ft.mkdir(os.path.join(self.test_prefix, dirname), parents=True)
        for filename in [
                'file1.txt', 'test_dir/file2.txt', 'test_dir/file3.txt',
                'test_dir2/file4.dat'
        ]:
            ft.write_file(os.path.join(self.test_prefix, filename),
                          'gibberish')

        globs = [
            os.path.join(self.test_prefix, '*.txt'),
            os.path.join(self.test_prefix, '*', '*')
        ]
        expected = [
            os.path.join(self.test_prefix, 'file1.txt'),
            os.path.join(self.test_prefix, 'test_dir', 'file2.txt'),
            os.path.join(self.test_prefix, 'test_dir', 'file3.txt'),
            os.path.join(self.test_prefix, 'test_dir2', 'file4.dat'),
        ]
        self.assertEqual(sorted(ft.expand_glob_paths(globs)), sorted(expected))

        # passing non-glob patterns is fine too
        file2 = os.path.join(self.test_prefix, 'test_dir', 'file2.txt')
        self.assertEqual(ft.expand_glob_paths([file2]), [file2])

        # test expanding of '~' into $HOME value
        # hard overwrite $HOME in environment (used by os.path.expanduser) so we can reliably test this
        new_home = os.path.join(self.test_prefix, 'home')
        ft.mkdir(new_home, parents=True)
        ft.write_file(os.path.join(new_home, 'test.txt'), 'test')
        os.environ['HOME'] = new_home
        self.assertEqual(ft.expand_glob_paths(['~/*.txt']),
                         [os.path.join(new_home, 'test.txt')])
Пример #3
0
    def test_expand_glob_paths(self):
        """Test expand_glob_paths function."""
        for dirname in ['empty_dir', 'test_dir']:
            ft.mkdir(os.path.join(self.test_prefix, dirname), parents=True)
        for filename in ['file1.txt', 'test_dir/file2.txt', 'test_dir/file3.txt', 'test_dir2/file4.dat']:
            ft.write_file(os.path.join(self.test_prefix, filename), 'gibberish')

        globs = [os.path.join(self.test_prefix, '*.txt'), os.path.join(self.test_prefix, '*', '*')]
        expected = [
            os.path.join(self.test_prefix, 'file1.txt'),
            os.path.join(self.test_prefix, 'test_dir', 'file2.txt'),
            os.path.join(self.test_prefix, 'test_dir', 'file3.txt'),
            os.path.join(self.test_prefix, 'test_dir2', 'file4.dat'),
        ]
        self.assertEqual(sorted(ft.expand_glob_paths(globs)), sorted(expected))

        # passing non-glob patterns is fine too
        file2 = os.path.join(self.test_prefix, 'test_dir', 'file2.txt')
        self.assertEqual(ft.expand_glob_paths([file2]), [file2])

        # test expanding of '~' into $HOME value
        # hard overwrite $HOME in environment (used by os.path.expanduser) so we can reliably test this
        new_home = os.path.join(self.test_prefix, 'home')
        ft.mkdir(new_home, parents=True)
        ft.write_file(os.path.join(new_home, 'test.txt'), 'test')
        os.environ['HOME'] = new_home
        self.assertEqual(ft.expand_glob_paths(['~/*.txt']), [os.path.join(new_home, 'test.txt')])

        # check behaviour if glob that has no (file) matches is passed
        glob_pat = os.path.join(self.test_prefix, 'test_*')
        self.assertErrorRegex(EasyBuildError, "No files found using glob pattern", ft.expand_glob_paths, [glob_pat])
Пример #4
0
    def install_step(self):
        """Symlink target OpenSSL installation"""
        if all(self.system_ssl[key] for key in ('bin', 'engines', 'include', 'libs')):
            # note: symlink to individual files, not directories,
            # since directory symlinks get resolved easily...

            # link OpenSSL libraries in system
            lib64_dir = os.path.join(self.installdir, 'lib64')
            lib64_engines_dir = os.path.join(lib64_dir, os.path.basename(self.system_ssl['engines']))
            mkdir(lib64_engines_dir, parents=True)

            # link existing known libraries
            for libso in self.system_ssl['libs']:
                symlink(libso, os.path.join(lib64_dir, os.path.basename(libso)))

            # link engines library files
            engine_lib_pattern = [os.path.join(self.system_ssl['engines'], '*')]
            for engine_lib in expand_glob_paths(engine_lib_pattern):
                symlink(engine_lib, os.path.join(lib64_engines_dir, os.path.basename(engine_lib)))

            # relative symlink for unversioned libraries
            cwd = change_dir(lib64_dir)
            for libso in self.system_ssl['libs']:
                libso = os.path.basename(libso)
                unversioned_lib = '%s.%s' % (libso.split('.')[0], get_shared_lib_ext())
                symlink(libso, unversioned_lib, use_abspath_source=False)
            change_dir(cwd)

            # link OpenSSL headers in system
            include_dir = os.path.join(self.installdir, 'include', self.name.lower())
            mkdir(include_dir, parents=True)
            include_pattern = [os.path.join(self.system_ssl['include'], '*')]
            for header_file in expand_glob_paths(include_pattern):
                symlink(header_file, os.path.join(include_dir, os.path.basename(header_file)))

            # link OpenSSL binary in system
            bin_dir = os.path.join(self.installdir, 'bin')
            mkdir(bin_dir)
            symlink(self.system_ssl['bin'], os.path.join(bin_dir, self.name.lower()))

            # install pkg-config files
            self.install_pc_files()

        elif self.cfg.get('wrap_system_openssl'):
            # install OpenSSL component due to lack of OpenSSL in host system
            print_warning("Not all OpenSSL components found in host system, falling back to OpenSSL in EasyBuild!")
            super(EB_OpenSSL_wrapper, self).install_step()
        else:
            # install OpenSSL component by user request
            warn_msg = "Installing OpenSSL from source in EasyBuild by user request ('wrap_system_openssl=%s')"
            print_warning(warn_msg, self.cfg.get('wrap_system_openssl'))
            super(EB_OpenSSL_wrapper, self).install_step()
Пример #5
0
def include_module_naming_schemes(tmpdir, paths):
    """Include module naming schemes at specified locations."""
    mns_path = os.path.join(tmpdir, 'included-module-naming-schemes')

    set_up_eb_package(mns_path, 'easybuild.tools.module_naming_scheme')

    mns_dir = os.path.join(mns_path, 'easybuild', 'tools', 'module_naming_scheme')

    allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py']
    for mns_module in allpaths:
        filename = os.path.basename(mns_module)
        target_path = os.path.join(mns_dir, filename)
        symlink(mns_module, target_path)

    included_mns = [x for x in os.listdir(mns_dir) if x not in ['__init__.py']]
    _log.debug("Included module naming schemes: %s", included_mns)

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, mns_path)

    # hard inject location to included module naming schemes into Python search path
    # only prepending to sys.path is not enough due to 'declare_namespace' in module_naming_scheme/__init__.py
    new_path = os.path.join(mns_path, 'easybuild', 'tools', 'module_naming_scheme')
    easybuild.tools.module_naming_scheme.__path__.insert(0, new_path)

    # sanity check: verify that included module naming schemes can be imported (from expected location)
    verify_imports([os.path.splitext(mns)[0] for mns in included_mns], 'easybuild.tools.module_naming_scheme', mns_dir)

    return mns_path
Пример #6
0
def include_easyblocks(tmpdir, paths):
    """Include generic and software-specific easyblocks found in specified locations."""
    easyblocks_path = os.path.join(tmpdir, 'included-easyblocks')

    set_up_eb_package(easyblocks_path,
                      'easybuild.easyblocks',
                      subpkgs=['generic'],
                      pkg_init_body=EASYBLOCKS_PKG_INIT_BODY)

    easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')

    allpaths = [
        p for p in expand_glob_paths(paths)
        if os.path.basename(p) != '__init__.py'
    ]
    for easyblock_module in allpaths:
        filename = os.path.basename(easyblock_module)

        # generic easyblocks are expected to be in a directory named 'generic'
        parent_dir = os.path.basename(os.path.dirname(easyblock_module))
        if parent_dir == 'generic':
            target_path = os.path.join(easyblocks_dir, 'generic', filename)
        else:
            target_path = os.path.join(easyblocks_dir, filename)

        if not os.path.exists(target_path):
            symlink(easyblock_module, target_path)

    included_ebs = [
        x for x in os.listdir(easyblocks_dir)
        if x not in ['__init__.py', 'generic']
    ]
    included_generic_ebs = [
        x for x in os.listdir(os.path.join(easyblocks_dir, 'generic'))
        if x != '__init__.py'
    ]
    _log.debug("Included generic easyblocks: %s", included_generic_ebs)
    _log.debug("Included software-specific easyblocks: %s", included_ebs)

    # prepend new location to Python search path
    sys.path.insert(0, easyblocks_path)

    # make sure easybuild.easyblocks(.generic)
    import easybuild.easyblocks
    import easybuild.easyblocks.generic

    # hard inject location to included (generic) easyblocks into Python search path
    # only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
    new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')
    easybuild.easyblocks.__path__.insert(0, new_path)
    new_path = os.path.join(new_path, 'generic')
    easybuild.easyblocks.generic.__path__.insert(0, new_path)

    # sanity check: verify that included easyblocks can be imported (from expected location)
    for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
        pkg = '.'.join(['easybuild', 'easyblocks', subdir]).strip('.')
        loc = os.path.join(easyblocks_dir, subdir)
        verify_imports([os.path.splitext(eb)[0] for eb in ebs], pkg, loc)

    return easyblocks_path
Пример #7
0
    def post_install_step(self):
        """
        Create wrappers for the specified host compilers, generate the appropriate stub symlinks,
        and create version independent pkgconfig files
        """
        def create_wrapper(wrapper_name, wrapper_comp):
            """Create for a particular compiler, with a particular name"""
            wrapper_f = os.path.join(self.installdir, 'bin', wrapper_name)
            write_file(wrapper_f, WRAPPER_TEMPLATE % wrapper_comp)
            perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH
            adjust_permissions(wrapper_f, perms)

        # Prepare wrappers to handle a default host compiler other than g++
        for comp in (self.cfg['host_compilers'] or []):
            create_wrapper('nvcc_%s' % comp, comp)

        ldconfig = which('ldconfig', log_ok=False, on_error=IGNORE)
        sbin_dirs = ['/sbin', '/usr/sbin']
        if not ldconfig:
            # ldconfig is usually in /sbin or /usr/sbin
            for cand_path in sbin_dirs:
                if os.path.exists(os.path.join(cand_path, 'ldconfig')):
                    ldconfig = os.path.join(cand_path, 'ldconfig')
                    break

        # fail if we couldn't find ldconfig, because it's really needed
        if ldconfig:
            self.log.info("ldconfig found at %s", ldconfig)
        else:
            path = os.environ.get('PATH', '')
            raise EasyBuildError("Unable to find 'ldconfig' in $PATH (%s), nor in any of %s", path, sbin_dirs)

        stubs_dir = os.path.join(self.installdir, 'lib64', 'stubs')
        # Run ldconfig to create missing symlinks in the stubs directory (libcuda.so.1, etc)
        cmd = ' '.join([ldconfig, '-N', stubs_dir])
        run_cmd(cmd)

        # GCC searches paths in LIBRARY_PATH and the system paths suffixed with ../lib64 or ../lib first
        # This means stubs/../lib64 is searched before the system /lib64 folder containing a potentially older libcuda.
        # See e.g. https://github.com/easybuilders/easybuild-easyconfigs/issues/12348
        # Workaround: Create a copy that matches this pattern
        new_stubs_dir = os.path.join(self.installdir, 'stubs')
        copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64'))
        # Also create the lib dir as a symlink
        symlink('lib64', os.path.join(new_stubs_dir, 'lib'), use_abspath_source=False)

        # Packages like xpra look for version independent pc files.
        # See e.g. https://github.com/Xpra-org/xpra/blob/master/setup.py#L206
        # Distros provide these files, so let's do it here too
        pkgconfig_dir = os.path.join(self.installdir, 'pkgconfig')
        if os.path.exists(pkgconfig_dir):
            pc_files = expand_glob_paths([os.path.join(pkgconfig_dir, '*.pc')])
            cwd = change_dir(pkgconfig_dir)
            for pc_file in pc_files:
                pc_file = os.path.basename(pc_file)
                link = re.sub('-[0-9]*.?[0-9]*(.[0-9]*)?.pc', '.pc', pc_file)
                symlink(pc_file, link, use_abspath_source=False)
            change_dir(cwd)

        super(EB_CUDA, self).post_install_step()
Пример #8
0
def include_module_naming_schemes(tmpdir, paths):
    """Include module naming schemes at specified locations."""
    mns_path = os.path.join(tmpdir, 'included-module-naming-schemes')

    set_up_eb_package(mns_path, 'easybuild.tools.module_naming_scheme')

    mns_dir = os.path.join(mns_path, 'easybuild', 'tools',
                           'module_naming_scheme')

    allpaths = [
        p for p in expand_glob_paths(paths)
        if os.path.basename(p) != '__init__.py'
    ]
    for mns_module in allpaths:
        filename = os.path.basename(mns_module)
        target_path = os.path.join(mns_dir, filename)
        symlink(mns_module, target_path)

    included_mns = [x for x in os.listdir(mns_dir) if x not in ['__init__.py']]
    _log.debug("Included module naming schemes: %s", included_mns)

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, mns_path)

    # hard inject location to included module naming schemes into Python search path
    # only prepending to sys.path is not enough due to 'declare_namespace' in module_naming_scheme/__init__.py
    new_path = os.path.join(mns_path, 'easybuild', 'tools',
                            'module_naming_scheme')
    easybuild.tools.module_naming_scheme.__path__.insert(0, new_path)

    # sanity check: verify that included module naming schemes can be imported (from expected location)
    verify_imports([os.path.splitext(mns)[0] for mns in included_mns],
                   'easybuild.tools.module_naming_scheme', mns_dir)

    return mns_path
Пример #9
0
    def install_step(self):
        """Custom install procedure for Siesta: copy binaries."""
        bindir = os.path.join(self.installdir, 'bin')
        copy_dir(os.path.join(self.cfg['start_dir'], 'bin'), bindir)
        if self.cfg['with_libsiesta']:
            libd = os.path.join(self.installdir, 'lib')
            pkgconfd = os.path.join(libd, 'pkgconfig')
            incdir = os.path.join(self.installdir, "include", "Siesta")
            moddir = os.path.join(incdir, 'modules')

            # Library
            copy_file(os.path.join(self.obj_dir, 'libsiesta.a'),
                      os.path.join(libd, 'libsiesta.a'))
            # Modules
            mkdir(moddir, parents=True)
            modules = expand_glob_paths([os.path.join(self.obj_dir, '*.mod')])
            for module in modules:
                copy_file(module, os.path.join(moddir,
                                               os.path.basename(module)))
            # Pkgconf file
            mkdir(pkgconfd)
            write_file(
                os.path.join(pkgconfd, "Siesta.pc"),
                PKGCONF_TEXT.format(
                    prefix=self.installdir,
                    version=self.version,
                    libnames=" ".join(self.pkg_conf_requires),
                ))
Пример #10
0
def include_toolchains(tmpdir, paths):
    """Include toolchains and toolchain components at specified locations."""
    toolchains_path = os.path.join(tmpdir, 'included-toolchains')
    toolchain_subpkgs = ['compiler', 'fft', 'linalg', 'mpi']

    set_up_eb_package(toolchains_path,
                      'easybuild.toolchains',
                      subpkgs=toolchain_subpkgs)

    tcs_dir = os.path.join(toolchains_path, 'easybuild', 'toolchains')

    allpaths = expand_glob_paths(paths)
    for toolchain_module in allpaths:
        filename = os.path.basename(toolchain_module)

        parent_dir = os.path.basename(os.path.dirname(toolchain_module))

        # toolchain components are expected to be in a directory named according to the type of component
        if parent_dir in toolchain_subpkgs:
            target_path = os.path.join(tcs_dir, parent_dir, filename)
        else:
            target_path = os.path.join(tcs_dir, filename)

        symlink(toolchain_module, target_path)

    included_toolchains = [
        x for x in os.listdir(tcs_dir)
        if x not in ['__init__.py'] + toolchain_subpkgs
    ]
    _log.debug("Included toolchains: %s", included_toolchains)

    included_subpkg_modules = {}
    for subpkg in toolchain_subpkgs:
        included_subpkg_modules[subpkg] = [
            x for x in os.listdir(os.path.join(tcs_dir, subpkg))
            if x != '__init__.py'
        ]
        _log.debug("Included toolchain %s components: %s", subpkg,
                   included_subpkg_modules[subpkg])

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, toolchains_path)
    reload(easybuild.toolchains)
    for subpkg in toolchain_subpkgs:
        reload(sys.modules['easybuild.toolchains.%s' % subpkg])

    # sanity check: verify that included toolchain modules can be imported (from expected location)
    verify_imports([os.path.splitext(mns)[0] for mns in included_toolchains],
                   'easybuild.toolchains', tcs_dir)
    for subpkg in toolchain_subpkgs:
        pkg = '.'.join(['easybuild', 'toolchains', subpkg])
        loc = os.path.join(tcs_dir, subpkg)
        verify_imports([
            os.path.splitext(tcmod)[0]
            for tcmod in included_subpkg_modules[subpkg]
        ], pkg, loc)

    return toolchains_path
Пример #11
0
def include_toolchains(tmpdir, paths):
    """Include toolchains and toolchain components at specified locations."""
    toolchains_path = os.path.join(tmpdir, 'included-toolchains')
    toolchain_subpkgs = ['compiler', 'fft', 'linalg', 'mpi']

    set_up_eb_package(toolchains_path, 'easybuild.toolchains', subpkgs=toolchain_subpkgs)

    tcs_dir = os.path.join(toolchains_path, 'easybuild', 'toolchains')

    allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py']
    for toolchain_module in allpaths:
        filename = os.path.basename(toolchain_module)

        parent_dir = os.path.basename(os.path.dirname(toolchain_module))

        # toolchain components are expected to be in a directory named according to the type of component
        if parent_dir in toolchain_subpkgs:
            target_path = os.path.join(tcs_dir, parent_dir, filename)
        else:
            target_path = os.path.join(tcs_dir, filename)

        if not os.path.exists(target_path):
            symlink(toolchain_module, target_path)

    included_toolchains = [x for x in os.listdir(tcs_dir) if x not in ['__init__.py'] + toolchain_subpkgs]
    _log.debug("Included toolchains: %s", included_toolchains)

    included_subpkg_modules = {}
    for subpkg in toolchain_subpkgs:
        included_subpkg_modules[subpkg] = [x for x in os.listdir(os.path.join(tcs_dir, subpkg)) if x != '__init__.py']
        _log.debug("Included toolchain %s components: %s", subpkg, included_subpkg_modules[subpkg])

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, toolchains_path)

    # reload toolchain modules and hard inject location to included toolchains into Python search path
    # only prepending to sys.path is not enough due to 'declare_namespace' in toolchains/*/__init__.py
    easybuild.toolchains.__path__.insert(0, os.path.join(toolchains_path, 'easybuild', 'toolchains'))
    for subpkg in toolchain_subpkgs:
        tcpkg = 'easybuild.toolchains.%s' % subpkg
        sys.modules[tcpkg].__path__.insert(0, os.path.join(toolchains_path, 'easybuild', 'toolchains', subpkg))

    # sanity check: verify that included toolchain modules can be imported (from expected location)
    verify_imports([os.path.splitext(mns)[0] for mns in included_toolchains], 'easybuild.toolchains', tcs_dir)
    for subpkg in toolchain_subpkgs:
        pkg = '.'.join(['easybuild', 'toolchains', subpkg])
        loc = os.path.join(tcs_dir, subpkg)
        verify_imports([os.path.splitext(tcmod)[0] for tcmod in included_subpkg_modules[subpkg]], pkg, loc)

    return toolchains_path
Пример #12
0
def include_easyblocks(tmpdir, paths):
    """Include generic and software-specific easyblocks found in specified locations."""
    easyblocks_path = os.path.join(tmpdir, 'included-easyblocks')

    set_up_eb_package(easyblocks_path,
                      'easybuild.easyblocks',
                      subpkgs=['generic'],
                      pkg_init_body=EASYBLOCKS_PKG_INIT_BODY)

    easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')

    allpaths = expand_glob_paths(paths)
    for easyblock_module in allpaths:
        filename = os.path.basename(easyblock_module)

        # generic easyblocks are expected to be in a directory named 'generic'
        parent_dir = os.path.basename(os.path.dirname(easyblock_module))
        if parent_dir == 'generic':
            target_path = os.path.join(easyblocks_dir, 'generic', filename)
        else:
            target_path = os.path.join(easyblocks_dir, filename)

        symlink(easyblock_module, target_path)

    included_ebs = [
        x for x in os.listdir(easyblocks_dir)
        if x not in ['__init__.py', 'generic']
    ]
    included_generic_ebs = [
        x for x in os.listdir(os.path.join(easyblocks_dir, 'generic'))
        if x != '__init__.py'
    ]
    _log.debug("Included generic easyblocks: %s", included_generic_ebs)
    _log.debug("Included software-specific easyblocks: %s", included_ebs)

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, easyblocks_path)
    reload(easybuild)
    if 'easybuild.easyblocks' in sys.modules:
        reload(easybuild.easyblocks)
        reload(easybuild.easyblocks.generic)

    # sanity check: verify that included easyblocks can be imported (from expected location)
    for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
        pkg = '.'.join(['easybuild', 'easyblocks', subdir]).strip('.')
        loc = os.path.join(easyblocks_dir, subdir)
        verify_imports([os.path.splitext(eb)[0] for eb in ebs], pkg, loc)

    return easyblocks_path
Пример #13
0
def include_easyblocks(tmpdir, paths):
    """Include generic and software-specific easyblocks found in specified locations."""
    easyblocks_path = os.path.join(tmpdir, 'included-easyblocks')

    set_up_eb_package(easyblocks_path, 'easybuild.easyblocks',
                      subpkgs=['generic'], pkg_init_body=EASYBLOCKS_PKG_INIT_BODY)

    easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')

    allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py']
    for easyblock_module in allpaths:
        filename = os.path.basename(easyblock_module)

        # generic easyblocks are expected to be in a directory named 'generic'
        parent_dir = os.path.basename(os.path.dirname(easyblock_module))
        if parent_dir == 'generic':
            target_path = os.path.join(easyblocks_dir, 'generic', filename)
        else:
            target_path = os.path.join(easyblocks_dir, filename)

        symlink(easyblock_module, target_path)

    included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']]
    included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py']
    _log.debug("Included generic easyblocks: %s", included_generic_ebs)
    _log.debug("Included software-specific easyblocks: %s", included_ebs)

    # prepend new location to Python search path
    sys.path.insert(0, easyblocks_path)

    # make sure easybuild.easyblocks(.generic)
    import easybuild.easyblocks
    import easybuild.easyblocks.generic

    # hard inject location to included (generic) easyblocks into Python search path
    # only prepending to sys.path is not enough due to 'declare_namespace' in easybuild/easyblocks/__init__.py
    new_path = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')
    easybuild.easyblocks.__path__.insert(0, new_path)
    new_path = os.path.join(new_path, 'generic')
    easybuild.easyblocks.generic.__path__.insert(0, new_path)

    # sanity check: verify that included easyblocks can be imported (from expected location)
    for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
        pkg = '.'.join(['easybuild', 'easyblocks', subdir]).strip('.')
        loc = os.path.join(easyblocks_dir, subdir)
        verify_imports([os.path.splitext(eb)[0] for eb in ebs], pkg, loc)

    return easyblocks_path
Пример #14
0
def include_easyblocks(tmpdir, paths):
    """Include generic and software-specific easyblocks found in specified locations."""
    easyblocks_path = os.path.join(tmpdir, 'included-easyblocks')

    set_up_eb_package(easyblocks_path, 'easybuild.easyblocks',
                      subpkgs=['generic'], pkg_init_body=EASYBLOCKS_PKG_INIT_BODY)

    easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks')

    allpaths = expand_glob_paths(paths)
    for easyblock_module in allpaths:
        filename = os.path.basename(easyblock_module)

        # generic easyblocks are expected to be in a directory named 'generic'
        parent_dir = os.path.basename(os.path.dirname(easyblock_module))
        if parent_dir == 'generic':
            target_path = os.path.join(easyblocks_dir, 'generic', filename)
        else:
            target_path = os.path.join(easyblocks_dir, filename)

        symlink(easyblock_module, target_path)

    included_ebs = [x for x in os.listdir(easyblocks_dir) if x not in ['__init__.py', 'generic']]
    included_generic_ebs = [x for x in os.listdir(os.path.join(easyblocks_dir, 'generic')) if x != '__init__.py']
    _log.debug("Included generic easyblocks: %s", included_generic_ebs)
    _log.debug("Included software-specific easyblocks: %s", included_ebs)

    # inject path into Python search path, and reload modules to get it 'registered' in sys.modules
    sys.path.insert(0, easyblocks_path)
    reload(easybuild)
    if 'easybuild.easyblocks' in sys.modules:
        reload(easybuild.easyblocks)
        reload(easybuild.easyblocks.generic)

    # sanity check: verify that included easyblocks can be imported (from expected location)
    for subdir, ebs in [('', included_ebs), ('generic', included_generic_ebs)]:
        pkg = '.'.join(['easybuild', 'easyblocks', subdir]).strip('.')
        loc = os.path.join(easyblocks_dir, subdir)
        verify_imports([os.path.splitext(eb)[0] for eb in ebs], pkg, loc)

    return easyblocks_path
Пример #15
0
    def install_step(self):
        """Symlink target OpenSSL installation"""

        if all(self.system_ssl[key]
               for key in ('bin', 'engines', 'include', 'lib')):
            # note: symlink to individual files, not directories,
            # since directory symlinks get resolved easily...

            # link OpenSSL libraries in system
            lib64_dir = os.path.join(self.installdir, 'lib64')
            lib64_engines_dir = os.path.join(
                lib64_dir, os.path.basename(self.system_ssl['engines']))
            mkdir(lib64_engines_dir, parents=True)

            # link existing known libraries
            ssl_syslibdir = os.path.dirname(self.system_ssl['lib'])
            lib_files = [
                os.path.join(ssl_syslibdir, x) for x in self.target_ssl_libs
            ]
            for libso in lib_files:
                symlink(libso, os.path.join(lib64_dir,
                                            os.path.basename(libso)))

            # link engines library files
            engine_lib_pattern = [
                os.path.join(self.system_ssl['engines'], '*')
            ]
            for engine_lib in expand_glob_paths(engine_lib_pattern):
                symlink(
                    engine_lib,
                    os.path.join(lib64_engines_dir,
                                 os.path.basename(engine_lib)))

            # relative symlink for unversioned libraries
            cwd = change_dir(lib64_dir)
            for libso in self.target_ssl_libs:
                unversioned_lib = '%s.%s' % (libso.split('.')[0],
                                             get_shared_lib_ext())
                symlink(libso, unversioned_lib, use_abspath_source=False)
            change_dir(cwd)

            # link OpenSSL headers in system
            include_dir = os.path.join(self.installdir, 'include',
                                       self.name.lower())
            mkdir(include_dir, parents=True)
            include_pattern = [os.path.join(self.system_ssl['include'], '*')]
            for header_file in expand_glob_paths(include_pattern):
                symlink(
                    header_file,
                    os.path.join(include_dir, os.path.basename(header_file)))

            # link OpenSSL binary in system
            bin_dir = os.path.join(self.installdir, 'bin')
            mkdir(bin_dir)
            symlink(self.system_ssl['bin'],
                    os.path.join(bin_dir, self.name.lower()))

        else:
            # install OpenSSL component
            print_warning(
                "Not all OpenSSL components found, falling back to OpenSSL in EasyBuild!"
            )
            super(EB_OpenSSL_wrapper, self).install_step()