Exemplo n.º 1
0
def build(m, bld_bat, stats):
    with path_prepended(m.config.build_prefix):
        with path_prepended(m.config.host_prefix):
            env = environ.get_dict(m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = False
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    #    See note above about inverted logic on "NO" variables
    env["PIP_NO_DEPENDENCIES"] = False
    env["PIP_IGNORE_INSTALLED"] = True
    # disable use of pip's cache directory.
    #    See note above about inverted logic on "NO" variables
    env["PIP_NO_CACHE_DIR"] = False

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = m.config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                if value:
                    fo.write('set "{key}={value}"\n'.format(key=key,
                                                            value=value))
            if not m.uses_new_style_compiler_activation:
                fo.write(
                    msvc_env_cmd(bits=m.config.host_arch,
                                 config=m.config,
                                 override=m.get_value('build/msvc_compiler',
                                                      None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if m.config.activate and m.name() != 'conda':
                _write_bat_activation_text(fo, m)
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        check_call_env(cmd, cwd=src_dir, stats=stats)

    fix_staged_scripts(join(m.config.host_prefix, 'Scripts'), config=m.config)
Exemplo n.º 2
0
def build(m):
    env = dict(os.environ)
    env.update(environ.get_dict(m))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    bld_bat = join(m.path, 'bld.bat')
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            fo.write(msvc_env_cmd(override=m.get_value('build/msvc_compiler', None)))
            fo.write('\n')
            # more debuggable with echo on
            fo.write('@echo on\n')
            fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
            fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd, cwd=src_dir, env={str(k): str(v) for k, v in env.items()})
        kill_processes()
        fix_staged_scripts()
def build(m, bld_bat):
    env = dict(os.environ)
    env.update(environ.get_dict(m))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in "BIN", "INC", "LIB":
        path = env["LIBRARY_" + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, "bld.bat"), "w") as fo:
            # more debuggable with echo on
            fo.write("@echo on\n")
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
            fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
            fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value("build/msvc_compiler", None)))
            fo.write("\n")
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ["COMSPEC"], "/c", "call", "bld.bat"]
        _check_call(cmd, cwd=src_dir)
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 4
0
def build(m):
    env = dict(os.environ)
    env.update(environ.get_dict(m))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    bld_bat = join(m.path, 'bld.bat')
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            fo.write(msvc_env_cmd())
            for kv in iteritems(env):
                fo.write('set "%s=%s"\n' % kv)
            # more debuggable with echo on
            fo.write('@echo on\n')
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 5
0
def build(m, bld_bat, dirty=False, activate=True):
    env = environ.get_dict(m, dirty=dirty)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if activate:
                fo.write("call activate.bat {0}\n".format(config.build_prefix))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
        fix_staged_scripts()
Exemplo n.º 6
0
def build(m, bld_bat, dirty=False, activate=True):
    env = environ.get_dict(m, dirty=dirty)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
            fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
            fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)))
            if activate:
                fo.write("call activate _build\n")
            fo.write('\n')
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 7
0
def build(m, bld_bat, dirty=False):
    env = dict(os.environ)
    env.update(environ.get_dict(m, dirty=dirty))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
            fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
            fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)))
            fo.write('\n')
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 8
0
def build(m, bld_bat):
    env = dict(os.environ)
    env.update(environ.get_dict(m))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            fo.write(
                msvc_env_cmd(
                    override=m.get_value('build/msvc_compiler', None)))
            fo.write('\n')
            # more debuggable with echo on
            fo.write('@echo on\n')
            fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
            fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd,
                    cwd=src_dir,
                    env={str(k): str(v)
                         for k, v in env.items()})
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 9
0
def build(m, bld_bat, config):
    with path_prepended(config.build_prefix):
        env = environ.get_dict(config=config, m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write(msvc_env_cmd(bits=bits, config=config,
                                  override=m.get_value('build/msvc_compiler', None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if config.activate:
                fo.write("call {conda_root}\\activate.bat {prefix}\n".format(
                    conda_root=root_script_dir,
                    prefix=config.build_prefix))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
Exemplo n.º 10
0
def build(m):
    env = dict(os.environ)
    env.update(environ.get_dict(m))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    bld_bat = join(m.path, 'bld.bat')
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            fo.write(msvc_env_cmd())
            # more debuggable with echo on
            fo.write('@echo on\n')
            for kv in iteritems(env):
                fo.write('set %s=%s\n' % kv)
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
        _check_call(cmd, cwd=src_dir)
        kill_processes()
        fix_staged_scripts()
Exemplo n.º 11
0
def build(m, bld_bat, stats):
    with path_prepended(m.config.build_prefix):
        with path_prepended(m.config.host_prefix):
            env = environ.get_dict(m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = False
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    #    See note above about inverted logic on "NO" variables
    env["PIP_NO_DEPENDENCIES"] = False
    env["PIP_IGNORE_INSTALLED"] = True

    # pip's cache directory (PIP_NO_CACHE_DIR) should not be
    # disabled as this results in .egg-info rather than
    # .dist-info directories being created, see gh-3094
    # set PIP_CACHE_DIR to a path in the work dir that does not exist.
    env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = m.config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                if value:
                    fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            if not m.uses_new_style_compiler_activation:
                fo.write(msvc_env_cmd(bits=m.config.host_arch, config=m.config,
                                    override=m.get_value('build/msvc_compiler', None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if m.config.activate and m.name() != 'conda':
                _write_bat_activation_text(fo, m)
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        check_call_env(cmd, cwd=src_dir, stats=stats)

    fix_staged_scripts(join(m.config.host_prefix, 'Scripts'), config=m.config)
Exemplo n.º 12
0
Arquivo: build.py Projeto: ocefpaf/boa
def build(m, stats=None, from_interactive=False, allow_interactive=False):
    try:
        if not stats:
            stats = {}

        if m.skip():
            console.print(utils.get_skip_message(m))
            return {}

        with utils.path_prepended(m.config.build_prefix):
            env = environ.get_dict(m=m)

        env["CONDA_BUILD_STATE"] = "BUILD"
        if env_path_backup_var_exists:
            env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]

        m.output.sections["package"]["name"] = m.output.name
        env["PKG_NAME"] = m.get_value("package/name")

        src_dir = m.config.work_dir
        if isdir(src_dir):
            if m.config.verbose:
                console.print("source tree in:", src_dir)
        else:
            if m.config.verbose:
                console.print("no source - creating empty work folder")
            os.makedirs(src_dir)

        utils.rm_rf(m.config.info_dir)
        files_before_script = utils.prefix_files(prefix=m.config.host_prefix)

        with open(join(m.config.build_folder, "prefix_files.txt"), "w") as f:
            f.write("\n".join(sorted(list(files_before_script))))
            f.write("\n")

        execute_build_script(m, src_dir, env)

        if m.output.sections["build"].get("intermediate"):
            utils.rm_rf(m.config.host_prefix)
            return

        bundle_conda(m, files_before_script, env, m.output.sections["files"])
    except subprocess.CalledProcessError:
        ext = "bat" if utils.on_win else "sh"
        work_dir = pathlib.Path(m.config.build_prefix).parent / "work"
        build_cmd = work_dir / f"conda_build.{ext}"

        console.print("\n")
        console.print(f"Work directory: {work_dir}")
        console.print(f"Try building again with {build_cmd}")

        if not from_interactive and allow_interactive:
            console.print("[red]Build went wrong, entering interactive mode![/red]")
            from boa.tui import tui
            import asyncio

            asyncio.run(tui.enter_tui(m))
Exemplo n.º 13
0
def write_build_scripts(m, script, build_file):

    with utils.path_prepended(m.config.host_prefix):
        with utils.path_prepended(m.config.build_prefix):
            env = environ.get_dict(m=m, variant={'no': 'variant'})

    env["CONDA_BUILD_STATE"] = "BUILD"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = 'False'
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    env["PIP_NO_DEPENDENCIES"] = True
    env["PIP_IGNORE_INSTALLED"] = True
    # pip's cache directory (PIP_NO_CACHE_DIR) should not be
    # disabled as this results in .egg-info rather than
    # .dist-info directories being created, see gh-3094

    # set PIP_CACHE_DIR to a path in the work dir that does not exist.
    env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

    # tell pip to not get anything from PyPI, please.  We have everything we need
    # locally, and if we don't, it's a problem.
    env["PIP_NO_INDEX"] = True

    if m.noarch == "python":
        env["PYTHONDONTWRITEBYTECODE"] = True

    work_file = join(m.config.work_dir, 'conda_build.sh')
    env_file = join(m.config.work_dir, 'build_env_setup.sh')

    with open(env_file, 'w') as bf:
        for k, v in env.items():
            if v != '' and v is not None:
                bf.write('export {0}="{1}"\n'.format(k, v))

        if m.activate_build_script:
            _write_sh_activation_text(bf, m)

    with open(work_file, 'w') as bf:
        # bf.write('set -ex\n')
        bf.write('if [ -z ${CONDA_BUILD+x} ]; then\n')
        bf.write("    source {}\n".format(env_file))
        bf.write("fi\n")

        if isfile(build_file):
            bf.write(open(build_file).read())
        elif script:
            bf.write(script)

    os.chmod(work_file, 0o766)
    return work_file, env_file
Exemplo n.º 14
0
def build(m, bld_bat, stats, provision_only=False):
    with path_prepended(m.config.host_prefix):
        with path_prepended(m.config.build_prefix):
            env = environ.get_dict(m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = 'False'
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    #    See note above about inverted logic on "NO" variables
    env["PIP_NO_DEPENDENCIES"] = True
    env["PIP_IGNORE_INSTALLED"] = True

    # pip's cache directory (PIP_NO_CACHE_DIR) should not be
    # disabled as this results in .egg-info rather than
    # .dist-info directories being created, see gh-3094
    # set PIP_CACHE_DIR to a path in the work dir that does not exist.
    env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

    # tell pip to not get anything from PyPI, please.  We have everything we need
    # locally, and if we don't, it's a problem.
    env["PIP_NO_INDEX"] = True

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    work_script, env_script = write_build_scripts(m, env, bld_bat)

    if not provision_only and os.path.isfile(work_script):
        cmd = ['cmd.exe', '/c', os.path.basename(work_script)]
        # rewrite long paths in stdout back to their env variables
        if m.config.debug:
            rewrite_env = None
        else:
            rewrite_env = {
                k: env[k]
                for k in ['PREFIX', 'BUILD_PREFIX', 'SRC_DIR'] if k in env
            }
            print("Rewriting env in output: %s" % pprint.pformat(rewrite_env))
        check_call_env(cmd,
                       cwd=m.config.work_dir,
                       stats=stats,
                       rewrite_stdout_env=rewrite_env)
        fix_staged_scripts(join(m.config.host_prefix, 'Scripts'),
                           config=m.config)
Exemplo n.º 15
0
def build(m, bld_bat, stats, provision_only=False):
    with path_prepended(m.config.host_prefix):
        with path_prepended(m.config.build_prefix):
            env = environ.get_dict(m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = 'False'
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    #    See note above about inverted logic on "NO" variables
    env["PIP_NO_DEPENDENCIES"] = True
    env["PIP_IGNORE_INSTALLED"] = True

    # pip's cache directory (PIP_NO_CACHE_DIR) should not be
    # disabled as this results in .egg-info rather than
    # .dist-info directories being created, see gh-3094
    # set PIP_CACHE_DIR to a path in the work dir that does not exist.
    env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

    # tell pip to not get anything from PyPI, please.  We have everything we need
    # locally, and if we don't, it's a problem.
    env["PIP_NO_INDEX"] = True

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    work_script, env_script = write_build_scripts(m, env, bld_bat)

    if not provision_only and os.path.isfile(work_script):
        cmd = ['cmd.exe', '/d', '/c', os.path.basename(work_script)]
        # rewrite long paths in stdout back to their env variables
        if m.config.debug:
            rewrite_env = None
        else:
            rewrite_env = {
                k: env[k]
                for k in ['PREFIX', 'BUILD_PREFIX', 'SRC_DIR'] if k in env
            }
            print("Rewriting env in output: %s" % pprint.pformat(rewrite_env))
        check_call_env(cmd, cwd=m.config.work_dir, stats=stats, rewrite_stdout_env=rewrite_env)
        fix_staged_scripts(join(m.config.host_prefix, 'Scripts'), config=m.config)
Exemplo n.º 16
0
def build(m, get_src=True, pypi=False):
    rm_rf(prefix)
    create_env(prefix, [ms.spec for ms in m.ms_depends('build')], pypi)

    print("BUILD START:", m.dist())

    if get_src:
        source.provide(m.path, m.get_section('source'))
    assert isdir(source.WORK_DIR)
    if os.listdir(source.get_dir()):
        print("source tree in:", source.get_dir())
    else:
        print("no source")

    rm_rf(info_dir)
    files1 = prefix_files()

    if sys.platform == 'win32':
        import conda_build.windows as windows
        windows.build(m)
    else:
        env = environ.get_dict(m)
        cmd = ['/bin/bash', '-x', '-e', join(m.path, 'build.sh')]
        _check_call(cmd, env=env, cwd=source.get_dir())

    create_post_scripts(m)
    create_entry_points(m.get_value('build/entry_points'))
    post_process(preserve_egg_dir=bool(
            m.get_value('build/preserve_egg_dir')))

    assert not exists(info_dir)
    files2 = prefix_files()

    post_build(sorted(files2 - files1))
    create_info_files(m, sorted(files2 - files1))
    files3 = prefix_files()
    fix_permissions(files3 - files1)

    path = bldpkg_path(m)
    t = tarfile.open(path, 'w:bz2')
    for f in sorted(files3 - files1):
        t.add(join(prefix, f), f)
    t.close()

    print("BUILD END:", m.dist())

    # we're done building, perform some checks
    tarcheck.check_all(path)
    update_index(bldpkgs_dir)
Exemplo n.º 17
0
def build(m, stats={}):

    if m.skip():
        print(utils.get_skip_message(m))
        return {}

    log = utils.get_logger(__name__)

    with utils.path_prepended(m.config.build_prefix):
        env = environ.get_dict(m=m)

    env["CONDA_BUILD_STATE"] = "BUILD"
    if env_path_backup_var_exists:
        env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]

    m.output.sections["package"]["name"] = m.output.name
    env["PKG_NAME"] = m.get_value('package/name')

    src_dir = m.config.work_dir
    if isdir(src_dir):
        if m.config.verbose:
            print("source tree in:", src_dir)
    else:
        if m.config.verbose:
            print("no source - creating empty work folder")
        os.makedirs(src_dir)

    utils.rm_rf(m.config.info_dir)
    files_before_script = utils.prefix_files(prefix=m.config.host_prefix)

    with open(join(m.config.build_folder, "prefix_files.txt"), "w") as f:
        f.write("\n".join(sorted(list(files_before_script))))
        f.write("\n")

    execute_build_script(m, src_dir, env)

    files_after_script = utils.prefix_files(prefix=m.config.host_prefix)

    files_difference = files_after_script - files_before_script

    if m.output.sections['build'].get('intermediate') == True:
        utils.rm_rf(m.config.host_prefix)
        return

    bundle_conda(m, files_before_script, env, m.output.sections['files'])
Exemplo n.º 18
0
def build(m, bld_bat, stats):
    with path_prepended(m.config.build_prefix):
        with path_prepended(m.config.host_prefix):
            env = environ.get_dict(config=m.config, m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = m.config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                if value:
                    fo.write('set "{key}={value}"\n'.format(key=key,
                                                            value=value))
            if not m.uses_new_style_compiler_activation:
                fo.write(
                    msvc_env_cmd(bits=m.config.host_arch,
                                 config=m.config,
                                 override=m.get_value('build/msvc_compiler',
                                                      None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if m.config.activate and m.name() != 'conda':
                _write_bat_activation_text(fo, m)
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        check_call_env(cmd, cwd=src_dir, stats=stats)

    fix_staged_scripts(join(m.config.host_prefix, 'Scripts'), config=m.config)
Exemplo n.º 19
0
def build(m, bld_bat, stats):
    with path_prepended(m.config.build_prefix):
        with path_prepended(m.config.host_prefix):
            env = environ.get_dict(config=m.config, m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = m.config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                if value:
                    fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            if not m.uses_new_style_compiler_activation:
                fo.write(msvc_env_cmd(bits=m.config.host_arch, config=m.config,
                                    override=m.get_value('build/msvc_compiler', None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if m.config.activate and m.name() != 'conda':
                _write_bat_activation_text(fo, m)
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        check_call_env(cmd, cwd=src_dir, stats=stats)

    fix_staged_scripts(join(m.config.host_prefix, 'Scripts'), config=m.config)
Exemplo n.º 20
0
def build(m, bld_bat, config):
    with path_prepended(config.build_prefix):
        env = environ.get_dict(config=config, m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            fo.write(
                msvc_env_cmd(bits=bits,
                             config=config,
                             override=m.get_value('build/msvc_compiler',
                                                  None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if config.activate:
                fo.write(
                    'call "{conda_root}\\activate.bat" "{prefix}"\n'.format(
                        conda_root=root_script_dir,
                        prefix=config.build_prefix))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        _check_call(cmd, cwd=src_dir)

    fix_staged_scripts(join(config.build_prefix, 'Scripts'))
Exemplo n.º 21
0
def windows_build(m):
    from conda_build.windows import msvc_env_cmd, kill_processes

    env = dict(os.environ)
    env.update(environ.get_dict(m))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    bld_bat = join(m.path, 'bld_wheel.bat')
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
    else:
        print("Using plain 'python setup.py bdist_wheel'  as build script")
        data = "\n:: Autogenerated build command:\npython setup.py bdist_wheel\n"

    with open(join(src_dir, 'bld.bat'), 'w') as fo:
        fo.write('@echo on\n')
        fo.write(
            msvc_env_cmd(override=m.get_value('build/msvc_compiler', None)))
        fo.write('\n')
        # more debuggable with echo on
        fo.write('set\n')
        fo.write('where python\n')
        fo.write('@echo on\n')
        fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
        fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
        fo.write("REM ===== end generated header =====\n")
        fo.write(data)

    cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
    print("build cmd: %s" % cmd)
    _check_call(cmd, cwd=src_dir, env={str(k): str(v) for k, v in env.items()})
    kill_processes()
Exemplo n.º 22
0
def windows_build(m):
    from conda_build.windows import msvc_env_cmd, kill_processes

    env = dict(os.environ)
    env.update(environ.get_dict(m))
    env = environ.prepend_bin_path(env, config.build_prefix, True)

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = source.get_dir()
    bld_bat = join(m.path, 'bld_wheel.bat')
    if exists(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
    else:
        print("Using plain 'python setup.py bdist_wheel'  as build script")
        data = "\n:: Autogenerated build command:\npython setup.py bdist_wheel\n"

    with open(join(src_dir, 'bld.bat'), 'w') as fo:
        fo.write('@echo on\n')
        fo.write(msvc_env_cmd(override=m.get_value('build/msvc_compiler', None)))
        fo.write('\n')
        # more debuggable with echo on
        fo.write('set\n')
        fo.write('where python\n')
        fo.write('@echo on\n')
        fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
        fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
        fo.write("REM ===== end generated header =====\n")
        fo.write(data)

    cmd = [os.environ['COMSPEC'], '/c', 'call', 'bld.bat']
    print("build cmd: %s" % cmd)
    _check_call(cmd, cwd=src_dir, env={str(k): str(v) for k, v in env.items()})
    kill_processes()
Exemplo n.º 23
0
def build(m, get_src=True, verbose=True):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    '''
    rm_rf(prefix)

    print("BUILD START:", m.dist())
    create_env(prefix, [ms.spec for ms in m.ms_depends('build')],
               verbose=verbose)

    if get_src:
        source.provide(m.path, m.get_section('source'))
    assert isdir(source.WORK_DIR)
    if os.listdir(source.get_dir()):
        print("source tree in:", source.get_dir())
    else:
        print("no source")

    rm_rf(info_dir)
    files1 = prefix_files()

    if sys.platform == 'win32':
        import conda_build.windows as windows
        windows.build(m)
    else:
        env = environ.get_dict(m)
        build_file = join(m.path, 'build.sh')
        if exists(build_file):
            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                with open(build_file, 'w', encoding='utf-8') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)
            cmd = ['/bin/bash', '-x', '-e', build_file]

            _check_call(cmd, env=env, cwd=source.get_dir())

    get_build_metadata(m)
    create_post_scripts(m)
    create_entry_points(m.get_value('build/entry_points'))
    post_process(preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

    assert not exists(info_dir)
    files2 = prefix_files()

    post_build(sorted(files2 - files1),
               binary_relocation=bool(
                   m.get_value('build/binary_relocation', True)))
    create_info_files(m, sorted(files2 - files1), include_recipe=bool(m.path))
    files3 = prefix_files()
    fix_permissions(files3 - files1)

    path = bldpkg_path(m)
    t = tarfile.open(path, 'w:bz2')
    for f in sorted(files3 - files1):
        t.add(join(prefix, f), f)
    t.close()

    print("BUILD END:", m.dist())

    # we're done building, perform some checks
    tarcheck.check_all(path)
    update_index(config.bldpkgs_dir)
Exemplo n.º 24
0
def test(m, verbose=True, channel_urls=(), override_channels=False):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''
    # remove from package cache
    rm_pkgs_cache(m.dist())

    tmp_dir = join(config.croot, 'test-tmp_dir')
    rm_rf(tmp_dir)
    os.makedirs(tmp_dir)
    create_files(tmp_dir, m)
    # Make Perl or Python-specific test files
    if m.name().startswith('perl-'):
        pl_files = create_pl_files(tmp_dir, m)
        py_files = False
    else:
        py_files = create_py_files(tmp_dir, m)
        pl_files = False
    shell_files = create_shell_files(tmp_dir, m)
    if not (py_files or shell_files or pl_files):
        print("Nothing to test for:", m.dist())
        return

    print("TEST START:", m.dist())
    rm_rf(config.build_prefix)
    rm_rf(config.test_prefix)
    specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

    if py_files:
        # as the tests are run by python, we need to specify it
        specs += ['python %s*' % environ.get_py_ver()]
    if pl_files:
        # as the tests are run by perl, we need to specify it
        specs += ['perl %s*' % environ.get_perl_ver()]
    # add packages listed in test/requires
    for spec in m.get_value('test/requires', []):
        specs.append(spec)

    create_env(config.test_prefix, specs, verbose=verbose,
        channel_urls=channel_urls, override_channels=override_channels)

    env = dict(os.environ)
    # TODO: Include all the same environment variables that are used in
    # building.
    env.update(environ.get_dict(m, prefix=config.test_prefix))

    # prepend bin (or Scripts) directory
    env['PATH'] = (join(config.test_prefix, bin_dirname) + os.pathsep +
                   os.getenv('PATH'))

    for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL':
        env[varname] = str(getattr(config, varname))
    env['PREFIX'] = config.test_prefix

    # Python 2 Windows requires that envs variables be string, not unicode
    env = {str(key): str(value) for key, value in env.items()}
    if py_files:
        try:
            subprocess.check_call([config.test_python, '-s',
                                   join(tmp_dir, 'run_test.py')],
                                  env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if pl_files:
        try:
            subprocess.check_call([config.test_perl,
                                   join(tmp_dir, 'run_test.pl')],
                                  env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if shell_files:
        if sys.platform == 'win32':
            test_file = join(tmp_dir, 'run_test.bat')
            cmd = [os.environ['COMSPEC'], '/c', 'call', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)
        else:
            test_file = join(tmp_dir, 'run_test.sh')
            # TODO: Run the test/commands here instead of in run_test.py
            cmd = ['/bin/bash', '-x', '-e', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)

    print("TEST END:", m.dist())
Exemplo n.º 25
0
def test(m, verbose=True, channel_urls=(), override_channels=False):
    """
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    """
    # remove from package cache
    rm_pkgs_cache(m.dist())

    tmp_dir = join(config.croot, "test-tmp_dir")
    rm_rf(tmp_dir)
    os.makedirs(tmp_dir)
    create_files(tmp_dir, m)
    # Make Perl or Python-specific test files
    if m.name().startswith("perl-"):
        pl_files = create_pl_files(tmp_dir, m)
        py_files = False
    else:
        py_files = create_py_files(tmp_dir, m)
        pl_files = False
    shell_files = create_shell_files(tmp_dir, m)
    if not (py_files or shell_files or pl_files):
        print("Nothing to test for:", m.dist())
        return

    print("TEST START:", m.dist())
    rm_rf(config.build_prefix)
    rm_rf(config.test_prefix)
    specs = ["%s %s %s" % (m.name(), m.version(), m.build_id())]

    # add packages listed in test/requires
    specs_include_python = False
    for spec in m.get_value("test/requires", []):
        specs.append(spec)
        if spec.startswith("python ") or spec == "python":
            specs_include_python = True

    if py_files and not specs_include_python:
        # as the tests are run by python, we need to specify it
        specs += ["python %s*" % environ.get_py_ver()]
    if pl_files:
        # as the tests are run by perl, we need to specify it
        specs += ["perl %s*" % environ.get_perl_ver()]

    create_env(
        config.test_prefix, specs, verbose=verbose, channel_urls=channel_urls, override_channels=override_channels
    )

    env = dict(os.environ)
    # TODO: Include all the same environment variables that are used in
    # building.
    env.update(environ.get_dict(m, prefix=config.test_prefix))

    # prepend bin (or Scripts) directory
    env["PATH"] = join(config.test_prefix, bin_dirname) + os.pathsep + os.getenv("PATH")

    for varname in "CONDA_PY", "CONDA_NPY", "CONDA_PERL":
        env[varname] = str(getattr(config, varname))
    env["PREFIX"] = config.test_prefix

    # Python 2 Windows requires that envs variables be string, not unicode
    env = {str(key): str(value) for key, value in env.items()}
    if py_files:
        try:
            subprocess.check_call([config.test_python, "-s", join(tmp_dir, "run_test.py")], env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if pl_files:
        try:
            subprocess.check_call([config.test_perl, join(tmp_dir, "run_test.pl")], env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if shell_files:
        if sys.platform == "win32":
            test_file = join(tmp_dir, "run_test.bat")
            cmd = [os.environ["COMSPEC"], "/c", "call", test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)
        else:
            test_file = join(tmp_dir, "run_test.sh")
            # TODO: Run the test/commands here instead of in run_test.py
            cmd = ["/bin/bash", "-x", "-e", test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)

    print("TEST END:", m.dist())
Exemplo n.º 26
0
def build(m,
          get_src=True,
          verbose=True,
          post=None,
          channel_urls=(),
          override_channels=False,
          include_recipe=True):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''

    if (m.get_value('build/detect_binary_files_with_prefix')
            or m.binary_has_prefix_files()):
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if post in [False, None]:
        print("Removing old build environment")
        if on_win:
            if isdir(config.short_build_prefix):
                move_to_trash(config.short_build_prefix, '')
            if isdir(config.long_build_prefix):
                move_to_trash(config.long_build_prefix, '')
        else:
            rm_rf(config.short_build_prefix)
            rm_rf(config.long_build_prefix)
        print("Removing old work directory")
        if on_win:
            if isdir(source.WORK_DIR):
                move_to_trash(source.WORK_DIR, '')
        else:
            rm_rf(source.WORK_DIR)

        # Display the name only
        # Version number could be missing due to dependency on source info.
        print("BUILD START:", m.dist())
        create_env(config.build_prefix,
                   [ms.spec for ms in m.ms_depends('build')],
                   verbose=verbose,
                   channel_urls=channel_urls,
                   override_channels=override_channels)

        if m.name() in [
                i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
        ]:
            print("%s is installed as a build dependency. Removing." %
                  m.name())
            index = get_build_index(clear_cache=False,
                                    channel_urls=channel_urls,
                                    override_channels=override_channels)
            actions = plan.remove_actions(config.build_prefix, [m.name()],
                                          index=index)
            assert not plan.nothing_to_do(actions), actions
            plan.display_actions(actions, index)
            plan.execute_actions(actions, index)

        if get_src:
            source.provide(m.path, m.get_section('source'))
            # Parse our metadata again because we did not initialize the source
            # information before.
            m.parse_again()

        print("Package:", m.dist())

        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(config.info_dir)
        files1 = prefix_files()
        for pat in m.always_include_files():
            has_matches = False
            for f in set(files1):
                if fnmatch.fnmatch(f, pat):
                    print("Including in package existing file", f)
                    files1.discard(f)
                    has_matches = True
            if not has_matches:
                sys.exit(
                    "Error: Glob %s from always_include_files does not match any files"
                    % pat)
        # Save this for later
        with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
            f.write(u'\n'.join(sorted(list(files1))))
            f.write(u'\n')

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')

            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                build_file = join(source.get_dir(), 'conda_build.sh')
                with open(build_file, 'w') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if isfile(build_file):
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        assert not exists(config.info_dir)
        files2 = prefix_files()

        post_process(sorted(files2 - files1),
                     preserve_egg_dir=bool(
                         m.get_value('build/preserve_egg_dir')))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files()
        if any(config.meta_dir in join(config.build_prefix, f)
               for f in files2 - files1):
            sys.exit(
                indent(
                    """Error: Untracked file(s) %s found in conda-meta directory.  This error
usually comes from using conda in the build script.  Avoid doing this, as it
can lead to packages that include their dependencies.""" %
                    (tuple(f
                           for f in files2 - files1 if config.meta_dir in join(
                               config.build_prefix, f)), )))
        post_build(m, sorted(files2 - files1))
        create_info_files(m,
                          sorted(files2 - files1),
                          include_recipe=bool(m.path) and include_recipe)
        if m.get_value('build/noarch_python'):
            import conda_build.noarch_python as noarch_python
            noarch_python.transform(m, sorted(files2 - files1))

        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(config.build_prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 27
0
def build(m,
          config,
          post=None,
          need_source_download=True,
          need_reparse_in_env=False):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    :type keep_old_work: bool: Keep any previous work directory.
    :type need_source_download: bool: if rendering failed to download source
    (due to missing tools), retry here after build env is populated
    '''

    if m.skip():
        print_skip_message(m)
        return False

    if config.skip_existing:
        package_exists = is_package_built(m, config)
        if package_exists:
            print(m.dist(),
                  "is already built in {0}, skipping.".format(package_exists))
            return False

    if post in [False, None]:
        print("BUILD START:", m.dist())
        if m.uses_jinja and (need_source_download or need_reparse_in_env):
            print(
                "    (actual version deferred until further download or env creation)"
            )

        specs = [ms.spec for ms in m.ms_depends('build')]
        create_env(config.build_prefix, specs, config=config)
        vcs_source = m.uses_vcs_in_build
        if vcs_source and vcs_source not in specs:
            vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
            has_vcs_available = os.path.isfile(
                external.find_executable(vcs_executable, config.build_prefix)
                or "")
            if not has_vcs_available:
                if (vcs_source != "mercurial" or not any(
                        spec.startswith('python') and "3." in spec
                        for spec in specs)):
                    specs.append(vcs_source)

                    log.warn(
                        "Your recipe depends on %s at build time (for templates), "
                        "but you have not listed it as a build dependency.  Doing "
                        "so for this build.", vcs_source)

                    # Display the name only
                    # Version number could be missing due to dependency on source info.
                    create_env(config.build_prefix, specs, config=config)
                else:
                    raise ValueError(
                        "Your recipe uses mercurial in build, but mercurial"
                        " does not yet support Python 3.  Please handle all of "
                        "your mercurial actions outside of your build script.")

        if need_source_download:
            # Execute any commands fetching the source (e.g., git) in the _build environment.
            # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
            # dependencies.
            with path_prepended(config.build_prefix):
                m, need_source_download, need_reparse_in_env = parse_or_try_download(
                    m,
                    no_download_source=False,
                    force_download=True,
                    config=config)
            assert not need_source_download, "Source download failed.  Please investigate."
            if m.uses_jinja:
                print("BUILD START (revised):", m.dist())

        if need_reparse_in_env:
            reparse(m, config=config)
            print("BUILD START (revised):", m.dist())

        if m.name() in [
                i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
        ]:
            print("%s is installed as a build dependency. Removing." %
                  m.name())
            index = get_build_index(config=config, clear_cache=False)
            actions = plan.remove_actions(config.build_prefix, [m.name()],
                                          index=index)
            assert not plan.nothing_to_do(actions), actions
            plan.display_actions(actions, index)
            plan.execute_actions(actions, index)

        print("Package:", m.dist())

        with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
                                   timeout=config.timeout):
            # get_dir here might be just work, or it might be one level deeper,
            #    dependening on the source.
            src_dir = source.get_dir(config)
            if isdir(src_dir):
                print("source tree in:", src_dir)
            else:
                print("no source - creating empty work folder")
                os.makedirs(src_dir)

            rm_rf(config.info_dir)
            files1 = prefix_files(prefix=config.build_prefix)
            for pat in m.always_include_files():
                has_matches = False
                for f in set(files1):
                    if fnmatch.fnmatch(f, pat):
                        print("Including in package existing file", f)
                        files1.discard(f)
                        has_matches = True
                if not has_matches:
                    log.warn(
                        "Glob %s from always_include_files does not match any files",
                        pat)
            # Save this for later
            with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
                f.write(u'\n'.join(sorted(list(files1))))
                f.write(u'\n')

            # Use script from recipe?
            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)

            if isdir(src_dir):
                if on_win:
                    build_file = join(m.path, 'bld.bat')
                    if script:
                        build_file = join(src_dir, 'bld.bat')
                        with open(build_file, 'w') as bf:
                            bf.write(script)
                    import conda_build.windows as windows
                    windows.build(m, build_file, config=config)
                else:
                    build_file = join(m.path, 'build.sh')

                    # There is no sense in trying to run an empty build script.
                    if isfile(build_file) or script:
                        with path_prepended(config.build_prefix):
                            env = environ.get_dict(config=config, m=m)
                        env["CONDA_BUILD_STATE"] = "BUILD"
                        work_file = join(source.get_dir(config),
                                         'conda_build.sh')
                        if script:
                            with open(work_file, 'w') as bf:
                                bf.write(script)
                        if config.activate:
                            if isfile(build_file):
                                data = open(build_file).read()
                            else:
                                data = open(work_file).read()
                            with open(work_file, 'w') as bf:
                                bf.write(
                                    "source {conda_root}activate {build_prefix} &> "
                                    "/dev/null\n".format(
                                        conda_root=root_script_dir +
                                        os.path.sep,
                                        build_prefix=config.build_prefix))
                                bf.write(data)
                        else:
                            if not isfile(work_file):
                                copy_into(build_file, work_file,
                                          config.timeout)
                        os.chmod(work_file, 0o766)

                        if isfile(work_file):
                            cmd = [shell_path, '-x', '-e', work_file]
                            # this should raise if any problems occur while building
                            _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m, config=config)
        create_post_scripts(m, config=config)
        create_entry_points(m.get_value('build/entry_points'), config=config)
        files2 = prefix_files(prefix=config.build_prefix)

        post_process(sorted(files2 - files1),
                     prefix=config.build_prefix,
                     config=config,
                     preserve_egg_dir=bool(
                         m.get_value('build/preserve_egg_dir')))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files(prefix=config.build_prefix)
        if any(config.meta_dir in join(config.build_prefix, f)
               for f in files2 - files1):
            meta_files = (tuple(
                f for f in files2 - files1
                if config.meta_dir in join(config.build_prefix, f)), )
            sys.exit(
                indent(
                    """Error: Untracked file(s) %s found in conda-meta directory.
This error usually comes from using conda in the build script.  Avoid doing this, as it
can lead to packages that include their dependencies.""" % meta_files))
        post_build(m,
                   sorted(files2 - files1),
                   prefix=config.build_prefix,
                   build_python=config.build_python,
                   croot=config.croot)
        create_info_files(m,
                          sorted(files2 - files1),
                          config=config,
                          prefix=config.build_prefix)
        if m.get_value('build/noarch_python'):
            import conda_build.noarch_python as noarch_python
            noarch_python.transform(m, sorted(files2 - files1),
                                    config.build_prefix)

        files3 = prefix_files(prefix=config.build_prefix)
        fix_permissions(files3 - files1, config.build_prefix)

        path = bldpkg_path(m, config)

        # lock the output directory while we build this file
        # create the tarball in a temporary directory to minimize lock time
        with TemporaryDirectory() as tmp:
            tmp_path = os.path.join(tmp, os.path.basename(path))
            t = tarfile.open(tmp_path, 'w:bz2')

            def order(f):
                # we don't care about empty files so send them back via 100000
                fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
                # info/* records will be False == 0, others will be 1.
                info_order = int(os.path.dirname(f) != 'info')
                return info_order, fsize

            # add files in order of a) in info directory, b) increasing size so
            # we can access small manifest or json files without decompressing
            # possible large binary or data files
            for f in sorted(files3 - files1, key=order):
                t.add(join(config.build_prefix, f), f)
            t.close()

            # we're done building, perform some checks
            tarcheck.check_all(tmp_path)

            copy_into(tmp_path, path, config.timeout)
        update_index(config.bldpkgs_dir, config)

    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())

    # returning true here says package is OK to test
    return True
Exemplo n.º 28
0
def build(m,
          get_src=True,
          verbose=True,
          post=None,
          channel_urls=(),
          override_channels=False):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''
    if (m.get_value('build/detect_binary_files_with_prefix')
            or m.binary_has_prefix_files()):
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if post in [False, None]:
        print("Removing old build directory")
        rm_rf(config.short_build_prefix)
        rm_rf(config.long_build_prefix)
        print("Removing old work directory")
        rm_rf(source.WORK_DIR)

        # Display the name only
        # Version number could be missing due to dependency on source info.
        print("BUILD START:", m.dist())
        create_env(config.build_prefix,
                   [ms.spec for ms in m.ms_depends('build')],
                   verbose=verbose,
                   channel_urls=channel_urls,
                   override_channels=override_channels)

        if get_src:
            source.provide(m.path, m.get_section('source'))
            # Parse our metadata again because we did not initialize the source
            # information before.
            m.parse_again()

        print("Package:", m.dist())

        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(config.info_dir)
        files1 = prefix_files()
        for f in m.always_include_files():
            if f not in files1:
                sys.exit("Error: File %s from always_include_files not found" %
                         f)
        files1 = files1.difference(set(m.always_include_files()))
        # Save this for later
        with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
            f.write(u'\n'.join(sorted(list(files1))))
            f.write(u'\n')

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')

            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                build_file = join(source.get_dir(), 'conda_build.sh')
                with open(build_file, 'w') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if exists(build_file):
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        assert not exists(config.info_dir)
        files2 = prefix_files()

        post_process(sorted(files2 - files1),
                     preserve_egg_dir=bool(
                         m.get_value('build/preserve_egg_dir')))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files()
        post_build(m, sorted(files2 - files1))
        create_info_files(m,
                          sorted(files2 - files1),
                          include_recipe=bool(m.path))
        if m.get_value('build/noarch_python'):
            import conda_build.noarch_python as noarch_python
            noarch_python.transform(m, sorted(files2 - files1))

        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(config.build_prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 29
0
def write_build_scripts(m, script, build_file):

    with utils.path_prepended(m.config.host_prefix):
        with utils.path_prepended(m.config.build_prefix):
            env = environ.get_dict(m=m)

    sysvars = get_sys_vars_stubs(env["target_platform"])
    for s in sysvars:
        if s not in env and s in m.config.variant:
            env[s] = m.config.variant[s]

    env.update(m.build_features())

    env["CONDA_BUILD_STATE"] = "BUILD"

    # forcing shiny colors everywhere
    env["CLICOLOR_FORCE"] = 1
    env["AM_COLOR_TESTS"] = "always"
    env["MAKE_TERMOUT"] = "1"
    env["CMAKE_COLOR_MAKEFILE"] = "ON"
    env["CXXFLAGS"] = "-fdiagnostics-color=always"
    env["CFLAGS"] = "-fdiagnostics-color=always"

    # hard-code this because we never want pip's build isolation
    #    https://github.com/conda/conda-build/pull/2972#discussion_r198290241
    #
    # Note that pip env "NO" variables are inverted logic.
    #      PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
    #
    env["PIP_NO_BUILD_ISOLATION"] = "False"
    # some other env vars to have pip ignore dependencies.
    # we supply them ourselves instead.
    env["PIP_NO_DEPENDENCIES"] = True
    env["PIP_IGNORE_INSTALLED"] = True
    # pip's cache directory (PIP_NO_CACHE_DIR) should not be
    # disabled as this results in .egg-info rather than
    # .dist-info directories being created, see gh-3094

    # set PIP_CACHE_DIR to a path in the work dir that does not exist.
    env["PIP_CACHE_DIR"] = m.config.pip_cache_dir

    # tell pip to not get anything from PyPI, please.  We have everything we need
    # locally, and if we don't, it's a problem.
    env["PIP_NO_INDEX"] = True

    if m.noarch == "python":
        env["PYTHONDONTWRITEBYTECODE"] = True

    work_file = join(m.config.work_dir, "conda_build.sh")
    env_file = join(m.config.work_dir, "build_env_setup.sh")

    with open(env_file, "w") as bf:
        for k, v in env.items():
            if v != "" and v is not None:
                bf.write('export {0}="{1}"\n'.format(k, v))
                # console.print('export {0}="{1}"\n'.format(k, v))

        if m.activate_build_script:
            _write_sh_activation_text(bf, m)

        # add the feature function
        bf.write("""
function feature()
{
    if [[ $1 != "0" ]]
    then
        echo $2
    else
        echo $3
    fi
}
""")

    with open(work_file, "w") as bf:
        # bf.write('set -ex\n')
        bf.write("if [ -z ${CONDA_BUILD+x} ]; then\n")
        bf.write("    source {}\n".format(env_file))
        bf.write("fi\n")

        if isfile(build_file):
            bf.write(open(build_file).read())
        elif script:
            bf.write(script)

    os.chmod(work_file, 0o766)
    return work_file, env_file
Exemplo n.º 30
0
def test(m, move_broken=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''
    # remove from package cache
    rm_pkgs_cache(m.dist())

    tmp_dir = join(config.croot, 'test-tmp_dir')
    rm_rf(tmp_dir)
    if on_win:
        time.sleep(
            1)  # wait for rm_rf(tmp_dir) to finish before recreating tmp_dir
    os.makedirs(tmp_dir)
    create_files(tmp_dir, m)
    # Make Perl or Python-specific test files
    if m.name().startswith('perl-'):
        pl_files = create_pl_files(tmp_dir, m)
        py_files = False
        lua_files = False
    else:
        py_files = create_py_files(tmp_dir, m)
        pl_files = False
        lua_files = False
    shell_files = create_shell_files(tmp_dir, m)
    if not (py_files or shell_files or pl_files or lua_files):
        print("Nothing to test for:", m.dist())
        return

    print("TEST START:", m.dist())
    if on_win:
        if isdir(config.build_prefix):
            move_to_trash(config.build_prefix, '')
        if isdir(config.test_prefix):
            move_to_trash(config.test_prefix, '')
    else:
        rm_rf(config.build_prefix)
        rm_rf(config.test_prefix)

    get_build_metadata(m)
    specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

    # add packages listed in test/requires
    specs += m.get_value('test/requires', [])

    if py_files:
        # as the tests are run by python, ensure that python is installed.
        # (If they already provided python as a run or test requirement, this won't hurt anything.)
        specs += ['python %s*' % environ.get_py_ver()]
    if pl_files:
        # as the tests are run by perl, we need to specify it
        specs += ['perl %s*' % environ.get_perl_ver()]
    if lua_files:
        # not sure how this shakes out
        specs += ['lua %s*' % environ.get_lua_ver()]

    create_env(config.test_prefix, specs)

    env = dict(os.environ)
    env.update(environ.get_dict(m, prefix=config.test_prefix))

    # prepend bin (or Scripts) directory
    env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)

    if sys.platform == 'win32':
        env['PATH'] = config.test_prefix + os.pathsep + env['PATH']
    for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
        env[varname] = str(getattr(config, varname) or '')
    env['PREFIX'] = config.test_prefix

    # Python 2 Windows requires that envs variables be string, not unicode
    env = {str(key): str(value) for key, value in env.items()}
    if py_files:
        try:
            subprocess.check_call(
                [config.test_python, '-s',
                 join(tmp_dir, 'run_test.py')],
                env=env,
                cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    if pl_files:
        try:
            subprocess.check_call(
                [config.test_perl,
                 join(tmp_dir, 'run_test.pl')],
                env=env,
                cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    if lua_files:
        try:
            subprocess.check_call(
                [config.test_lua,
                 join(tmp_dir, 'run_test.lua')],
                env=env,
                cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if shell_files:
        if sys.platform == 'win32':
            test_file = join(tmp_dir, 'run_test.bat')
            cmd = [os.environ['COMSPEC'], '/c', 'call', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m, move_broken=move_broken)
        else:
            test_file = join(tmp_dir, 'run_test.sh')
            # TODO: Run the test/commands here instead of in run_test.py
            cmd = [shell_path, '-x', '-e', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m, move_broken=move_broken)

    print("TEST END:", m.dist())
Exemplo n.º 31
0
def build(m, get_src=True, verbose=True, post=None):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''
    if post in [False, None]:
        rm_rf(prefix)

        print("BUILD START:", m.dist())
        create_env(prefix, [ms.spec for ms in m.ms_depends('build')],
                   verbose=verbose)

        if get_src:
            source.provide(m.path, m.get_section('source'))
        assert isdir(source.WORK_DIR)
        if os.listdir(source.get_dir()):
            print("source tree in:", source.get_dir())
        else:
            print("no source")

        rm_rf(info_dir)
        files1 = prefix_files()
        if post == False:
            # Save this for later
            with open(join(source.WORK_DIR, 'prefix_files'), 'w') as f:
                json.dump(list(files1), f)

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')
            if exists(build_file):
                script = m.get_value('build/script', None)
                if script:
                    if isinstance(script, list):
                        script = '\n'.join(script)
                    with open(build_file, 'w', encoding='utf-8') as bf:
                        bf.write(script)
                    os.chmod(build_file, 0o766)
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=source.get_dir())

    if post in [True, None]:
        if post == True:
            with open(join(source.WORK_DIR, 'prefix_files')) as f:
                files1 = set(json.load(f))

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        post_process(preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

        assert not exists(info_dir)
        files2 = prefix_files()

        post_build(sorted(files2 - files1),
              binary_relocation=bool(m.get_value('build/binary_relocation', True)))
        create_info_files(m, sorted(files2 - files1), include_recipe=bool(m.path))
        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 32
0
def test(m, move_broken=True, activate=True, debug=False):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''

    with Locked(cc.root_dir):

        # remove from package cache
        rm_pkgs_cache(m.dist())

        tmp_dir = join(config.croot, 'test-tmp_dir')
        rm_rf(tmp_dir)
        if on_win:
            time.sleep(1)  # wait for rm_rf(tmp_dir) to finish before recreating tmp_dir
        os.makedirs(tmp_dir)
        create_files(tmp_dir, m)
        # Make Perl or Python-specific test files
        if m.name().startswith('perl-'):
            pl_files = create_pl_files(tmp_dir, m)
            py_files = False
            lua_files = False
        else:
            py_files = create_py_files(tmp_dir, m)
            pl_files = False
            lua_files = False
        shell_files = create_shell_files(tmp_dir, m)
        if not (py_files or shell_files or pl_files or lua_files):
            print("Nothing to test for:", m.dist())
            return

        print("TEST START:", m.dist())
        if on_win:
            if isdir(config.build_prefix):
                move_to_trash(config.build_prefix, '')
            if isdir(config.test_prefix):
                move_to_trash(config.test_prefix, '')
        else:
            rm_rf(config.build_prefix)
            rm_rf(config.test_prefix)

        get_build_metadata(m)
        specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

        # add packages listed in the run environment and test/requires
        specs.extend(ms.spec for ms in m.ms_depends('run'))
        specs += m.get_value('test/requires', [])

        if py_files:
            # as the tests are run by python, ensure that python is installed.
            # (If they already provided python as a run or test requirement,
            #  this won't hurt anything.)
            specs += ['python %s*' % environ.get_py_ver()]
        if pl_files:
            # as the tests are run by perl, we need to specify it
            specs += ['perl %s*' % environ.get_perl_ver()]
        if lua_files:
            # not sure how this shakes out
            specs += ['lua %s*' % environ.get_lua_ver()]

        create_env(config.test_prefix, specs, debug=debug)
        env = dict(os.environ.copy())
        env.update(environ.get_dict(m, prefix=config.test_prefix))

        if not activate:
            # prepend bin (or Scripts) directory
            env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)

            if on_win:
                env['PATH'] = config.test_prefix + os.pathsep + env['PATH']

        for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
            env[varname] = str(getattr(config, varname) or '')
        env['PREFIX'] = config.test_prefix

        # Python 2 Windows requires that envs variables be string, not unicode
        env = {str(key): str(value) for key, value in env.items()}
        suffix = "bat" if on_win else "sh"
        test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix))

        with open(test_script, 'w') as tf:
            if activate:
                source = "call " if on_win else "source "
                ext = ".bat" if on_win else ""
                tf.write("{source}activate{ext} {test_env}\n".format(source=source, ext=ext,
                                                                     test_env=config.test_prefix))
                tf.write("if errorlevel 1 exit 1\n") if on_win else None

            if py_files:
                tf.write("{python} -s {test_file}\n".format(
                    python=config.test_python,
                    test_file=join(tmp_dir, 'run_test.py')))
                tf.write("if errorlevel 1 exit 1\n") if on_win else None

            if pl_files:
                tf.write("{perl} {test_file}\n".format(
                    python=config.test_perl,
                    test_file=join(tmp_dir, 'run_test.pl')))
                tf.write("if errorlevel 1 exit 1\n") if on_win else None

            if lua_files:
                tf.write("{lua} {test_file}\n".format(
                    python=config.test_perl,
                    test_file=join(tmp_dir, 'run_test.lua')))
                tf.write("if errorlevel 1 exit 1\n") if on_win else None

            if shell_files:
                test_file = join(tmp_dir, 'run_test.' + suffix)
                if on_win:
                    tf.write("call {test_file}\n".format(test_file=test_file))
                    tf.write("if errorlevel 1 exit 1\n")
                else:
                    # TODO: Run the test/commands here instead of in run_test.py
                    tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path,
                                                                       test_file=test_file))
        if on_win:
            cmd = [env["COMSPEC"], "/d", "/c", test_script]
        else:
            cmd = [shell_path, '-x', '-e', test_script]
        try:
            subprocess.check_call(cmd, env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    print("TEST END:", m.dist())
Exemplo n.º 33
0
def build(m, post=None, include_recipe=True, keep_old_work=False,
          need_source_download=True, need_reparse_in_env=False,
          verbose=True, dirty=False, activate=True, debug=False):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    :type keep_old_work: bool: Keep any previous work directory.
    :type need_source_download: bool: if rendering failed to download source
    (due to missing tools), retry here after build env is populated
    '''

    if (m.get_value('build/detect_binary_files_with_prefix') or
            m.binary_has_prefix_files()) and not on_win:
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if m.skip():
        print("Skipped: The %s recipe defines build/skip for this "
              "configuration." % m.dist())
        return

    with Locked(cc.root_dir):

        # If --keep-old-work, then move the contents of source.WORK_DIR to a
        # temporary directory for the duration of the build.
        # The source unpacking procedure is too varied and complex
        # to allow this to be written cleanly (see source.get_dir() for example)
        if keep_old_work:
            old_WORK_DIR = tempfile.mkdtemp()
            old_sub_dirs = [name for name in os.listdir(source.WORK_DIR)
                            if os.path.isdir(os.path.join(source.WORK_DIR, name))]
            if len(old_sub_dirs):
                print("Keeping old work directory backup: %s => %s"
                    % (old_sub_dirs, old_WORK_DIR))
                for old_sub in old_sub_dirs:
                    shutil.move(os.path.join(source.WORK_DIR, old_sub), old_WORK_DIR)

        if post in [False, None]:
            print("Removing old build environment")
            print("BUILD START:", m.dist())
            if need_source_download or need_reparse_in_env:
                print("    (actual version deferred until further download or env creation)")
            if on_win:
                if isdir(config.short_build_prefix):
                    move_to_trash(config.short_build_prefix, '')
                if isdir(config.long_build_prefix):
                    move_to_trash(config.long_build_prefix, '')
            else:
                rm_rf(config.short_build_prefix)
                rm_rf(config.long_build_prefix)

            specs = [ms.spec for ms in m.ms_depends('build')]
            if activate:
                # If we activate the build envrionment, we need to be sure that we
                #    have the appropriate VCS available in the environment.  People
                #    are not used to explicitly listing it in recipes, though.
                #    We add it for them here, but warn them about it.
                vcs_source = m.uses_vcs_in_build()
                if vcs_source and vcs_source not in specs:
                    vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
                    has_vcs_available = os.path.isfile(external.find_executable(vcs_executable) or
                                                       "")
                    if not has_vcs_available:
                        if (vcs_source != "mercurial" or
                                not any(spec.startswith('python') and "3." in spec
                                        for spec in specs)):
                            specs.append(vcs_source)

                            log.warn("Your recipe depends on {} at build time (for templates), "
                                    "but you have not listed it as a build dependency.  Doing "
                                    "so for this build.")
                        else:
                            raise ValueError("Your recipe uses mercurial in build, but mercurial"
                                            " does not yet support Python 3.  Please handle all of "
                                            "your mercurial actions outside of your build script.")
            # Display the name only
            # Version number could be missing due to dependency on source info.
            create_env(config.build_prefix, specs, debug=debug)

            if need_source_download:
                # Execute any commands fetching the source (e.g., git) in the _build environment.
                # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
                # dependencies.
                if not activate:
                    _old_path = os.environ['PATH']
                    os.environ['PATH'] = prepend_bin_path({'PATH': _old_path},
                                                          config.build_prefix)['PATH']
                try:
                    m, need_source_download, need_reparse_in_env = parse_or_try_download(m,
                                                                    no_download_source=False,
                                                                    force_download=True,
                                                                    verbose=verbose,
                                                                    dirty=dirty)
                    assert not need_source_download, "Source download failed.  Please investigate."
                finally:
                    if not activate:
                        os.environ['PATH'] = _old_path
                print("BUILD START:", m.dist())

            if need_reparse_in_env:
                reparse(m)
                print("BUILD START:", m.dist())

            if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
                print("%s is installed as a build dependency. Removing." %
                    m.name())
                index = get_build_index(clear_cache=False)
                actions = plan.remove_actions(config.build_prefix, [m.name()], index=index)
                assert not plan.nothing_to_do(actions), actions
                plan.display_actions(actions, index)
                plan.execute_actions(actions, index)

            print("Package:", m.dist())

            src_dir = source.get_dir()
            if isdir(src_dir):
                print("source tree in:", src_dir)
            else:
                print("no source - creating empty work folder")
                os.makedirs(src_dir)

            rm_rf(config.info_dir)
            files1 = prefix_files()
            for pat in m.always_include_files():
                has_matches = False
                for f in set(files1):
                    if fnmatch.fnmatch(f, pat):
                        print("Including in package existing file", f)
                        files1.discard(f)
                        has_matches = True
                if not has_matches:
                    log.warn("Glob %s from always_include_files does not match any files" % pat)
            # Save this for later
            with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
                f.write(u'\n'.join(sorted(list(files1))))
                f.write(u'\n')

            # Use script from recipe?
            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)

            if isdir(source.WORK_DIR):
                if on_win:
                    build_file = join(m.path, 'bld.bat')
                    if script:
                        build_file = join(source.get_dir(), 'bld.bat')
                        with open(join(source.get_dir(), 'bld.bat'), 'w') as bf:
                            bf.write(script)
                    import conda_build.windows as windows
                    windows.build(m, build_file, dirty=dirty, activate=activate)
                else:
                    build_file = join(m.path, 'build.sh')

                    # There is no sense in trying to run an empty build script.
                    if isfile(build_file) or script:
                        env = environ.get_dict(m, dirty=dirty)
                        work_file = join(source.get_dir(), 'conda_build.sh')
                        if script:
                            with open(work_file, 'w') as bf:
                                bf.write(script)
                        if activate:
                            if isfile(build_file):
                                data = open(build_file).read()
                            else:
                                data = open(work_file).read()
                            with open(work_file, 'w') as bf:
                                bf.write("source activate {build_prefix}\n".format(
                                    build_prefix=config.build_prefix))
                                bf.write(data)
                        else:
                            if not isfile(work_file):
                                shutil.copy(build_file, work_file)
                        os.chmod(work_file, 0o766)

                        if isfile(work_file):
                            cmd = [shell_path, '-x', '-e', work_file]

                            _check_call(cmd, env=env, cwd=src_dir)

        if post in [True, None]:
            if post:
                with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                    files1 = set(f.read().splitlines())

            get_build_metadata(m)
            create_post_scripts(m)
            create_entry_points(m.get_value('build/entry_points'))
            assert not exists(config.info_dir)
            files2 = prefix_files()

            post_process(sorted(files2 - files1),
                         preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

            # The post processing may have deleted some files (like easy-install.pth)
            files2 = prefix_files()
            if any(config.meta_dir in join(config.build_prefix, f) for f in
                    files2 - files1):
                sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory.
    This error usually comes from using conda in the build script.  Avoid doing this, as it
    can lead to packages that include their dependencies.""" %
                    (tuple(f for f in files2 - files1 if config.meta_dir in
                        join(config.build_prefix, f)),)))
            post_build(m, sorted(files2 - files1))
            create_info_files(m, sorted(files2 - files1),
                            include_recipe=bool(m.path) and include_recipe)
            if m.get_value('build/noarch_python'):
                import conda_build.noarch_python as noarch_python
                noarch_python.transform(m, sorted(files2 - files1))

            files3 = prefix_files()
            fix_permissions(files3 - files1)

            path = bldpkg_path(m)
            t = tarfile.open(path, 'w:bz2')

            def order(f):
                # we don't care about empty files so send them back via 100000
                fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
                # info/* records will be False == 0, others will be 1.
                info_order = int(os.path.dirname(f) != 'info')
                return info_order, fsize

            # add files in order of a) in info directory, b) increasing size so
            # we can access small manifest or json files without decompressing
            # possible large binary or data files
            for f in sorted(files3 - files1, key=order):
                t.add(join(config.build_prefix, f), f)
            t.close()

            print("BUILD END:", m.dist())

            # we're done building, perform some checks
            tarcheck.check_all(path)
            update_index(config.bldpkgs_dir)
        else:
            print("STOPPING BUILD BEFORE POST:", m.dist())

        if keep_old_work and len(old_sub_dirs):
            print("Restoring old work directory backup: %s :: %s => %s"
                % (old_WORK_DIR, old_sub_dirs, source.WORK_DIR))
            for old_sub in old_sub_dirs:
                if os.path.exists(os.path.join(source.WORK_DIR, old_sub)):
                    print("Not restoring old source directory %s over new build's version" %
                          (old_sub))
                else:
                    shutil.move(os.path.join(old_WORK_DIR, old_sub), source.WORK_DIR)
            shutil.rmtree(old_WORK_DIR, ignore_errors=True)
Exemplo n.º 34
0
def context_processor():
    ctx = environ.get_dict()
    ctx.update(load_setuptools=load_setuptools,
               load_npm=load_npm,
               environ=os.environ)
    return ctx
Exemplo n.º 35
0
def test(m, move_broken=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''
    # remove from package cache
    rm_pkgs_cache(m.dist())

    tmp_dir = join(config.croot, 'test-tmp_dir')
    rm_rf(tmp_dir)
    os.makedirs(tmp_dir)
    create_files(tmp_dir, m)
    # Make Perl or Python-specific test files
    if m.name().startswith('perl-'):
        pl_files = create_pl_files(tmp_dir, m)
        py_files = False
    else:
        py_files = create_py_files(tmp_dir, m)
        pl_files = False
    shell_files = create_shell_files(tmp_dir, m)
    if not (py_files or shell_files or pl_files):
        print("Nothing to test for:", m.dist())
        return

    print("TEST START:", m.dist())
    if on_win:
        if isdir(config.build_prefix):
            move_to_trash(config.build_prefix, '')
        if isdir(config.test_prefix):
            move_to_trash(config.test_prefix, '')
    else:
        rm_rf(config.build_prefix)
        rm_rf(config.test_prefix)

    get_build_metadata(m)
    specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

    # add packages listed in test/requires
    specs += m.get_value('test/requires', [])

    if py_files:
        # as the tests are run by python, ensure that python is installed.
        # (If they already provided python as a run or test requirement, this won't hurt anything.)
        specs += ['python %s*' % environ.get_py_ver()]
    if pl_files:
        # as the tests are run by perl, we need to specify it
        specs += ['perl %s*' % environ.get_perl_ver()]

    create_env(config.test_prefix, specs)

    env = dict(os.environ)
    env.update(environ.get_dict(m, prefix=config.test_prefix))

    # prepend bin (or Scripts) directory
    env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)

    if sys.platform == 'win32':
        env['PATH'] = config.test_prefix + os.pathsep + env['PATH']
    for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL':
        env[varname] = str(getattr(config, varname) or '')
    env['PREFIX'] = config.test_prefix

    # Python 2 Windows requires that envs variables be string, not unicode
    env = {str(key): str(value) for key, value in env.items()}
    if py_files:
        try:
            subprocess.check_call([config.test_python, '-s',
                                   join(tmp_dir, 'run_test.py')],
                                  env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    if pl_files:
        try:
            subprocess.check_call([config.test_perl,
                                   join(tmp_dir, 'run_test.pl')],
                                  env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    if shell_files:
        if sys.platform == 'win32':
            test_file = join(tmp_dir, 'run_test.bat')
            cmd = [os.environ['COMSPEC'], '/c', 'call', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m, move_broken=move_broken)
        else:
            test_file = join(tmp_dir, 'run_test.sh')
            # TODO: Run the test/commands here instead of in run_test.py
            cmd = [shell_path, '-x', '-e', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m, move_broken=move_broken)

    print("TEST END:", m.dist())
Exemplo n.º 36
0
def run_test(
    recipedir_or_package_or_metadata,
    config,
    stats,
    move_broken=True,
    provision_only=False,
    solver=None,
):
    """
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    """

    # we want to know if we're dealing with package input.  If so, we can move the input on success.
    hash_input = {}

    # store this name to keep it consistent.  By changing files, we change the hash later.
    #    It matches the build hash now, so let's keep it around.
    test_package_name = (
        recipedir_or_package_or_metadata.dist()
        if hasattr(recipedir_or_package_or_metadata, "dist")
        else recipedir_or_package_or_metadata
    )

    if not provision_only:
        print("TEST START:", test_package_name)

    if hasattr(recipedir_or_package_or_metadata, "config"):
        metadata = recipedir_or_package_or_metadata
        utils.rm_rf(metadata.config.test_dir)
    else:
        metadata, hash_input = construct_metadata_for_test(
            recipedir_or_package_or_metadata, config
        )

    trace = "-x " if metadata.config.debug else ""

    # Must download *after* computing build id, or else computing build id will change
    #     folder destination
    _extract_test_files_from_package(metadata)

    # When testing a .tar.bz2 in the pkgs dir, clean_pkg_cache() will remove it.
    # Prevent this. When https://github.com/conda/conda/issues/5708 gets fixed
    # I think we can remove this call to clean_pkg_cache().
    in_pkg_cache = (
        not hasattr(recipedir_or_package_or_metadata, "config")
        and os.path.isfile(recipedir_or_package_or_metadata)
        and recipedir_or_package_or_metadata.endswith(CONDA_PACKAGE_EXTENSIONS)
        and os.path.dirname(recipedir_or_package_or_metadata) in pkgs_dirs[0]
    )
    if not in_pkg_cache:
        environ.clean_pkg_cache(metadata.dist(), metadata.config)

    copy_test_source_files(metadata, metadata.config.test_dir)
    # this is also copying tests/source_files from work_dir to testing workdir

    _, pl_files, py_files, r_files, lua_files, shell_files = create_all_test_files(
        metadata
    )

    if (
        not any([py_files, shell_files, pl_files, lua_files, r_files])
        and not metadata.config.test_run_post
    ):
        print("Nothing to test for:", test_package_name)
        return True

    if metadata.config.remove_work_dir:
        for name, prefix in (
            ("host", metadata.config.host_prefix),
            ("build", metadata.config.build_prefix),
        ):
            if os.path.isdir(prefix):
                # move host folder to force hardcoded paths to host env to break during tests
                #    (so that they can be properly addressed by recipe author)
                dest = os.path.join(
                    os.path.dirname(prefix),
                    "_".join(
                        (
                            "%s_prefix_moved" % name,
                            metadata.dist(),
                            getattr(metadata.config, "%s_subdir" % name),
                        )
                    ),
                )
                # Needs to come after create_files in case there's test/source_files
                shutil_move_more_retrying(prefix, dest, "{} prefix".format(prefix))

        # nested if so that there's no warning when we just leave the empty workdir in place
        if metadata.source_provided:
            dest = os.path.join(
                os.path.dirname(metadata.config.work_dir),
                "_".join(("work_moved", metadata.dist(), metadata.config.host_subdir)),
            )
            # Needs to come after create_files in case there's test/source_files
            shutil_move_more_retrying(config.work_dir, dest, "work")
    else:
        log.warn(
            "Not moving work directory after build.  Your package may depend on files "
            "in the work directory that are not included with your package"
        )

    # looks like a dead function to me
    # get_build_metadata(metadata)

    specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files)

    with utils.path_prepended(metadata.config.test_prefix):
        env = dict(os.environ.copy())
        env.update(environ.get_dict(m=metadata, prefix=config.test_prefix))
        env["CONDA_BUILD_STATE"] = "TEST"
        env["CONDA_BUILD"] = "1"
        if env_path_backup_var_exists:
            env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]

    if not metadata.config.activate or metadata.name() == "conda":
        # prepend bin (or Scripts) directory
        env = utils.prepend_bin_path(
            env, metadata.config.test_prefix, prepend_prefix=True
        )

    if utils.on_win:
        env["PATH"] = metadata.config.test_prefix + os.pathsep + env["PATH"]

    env["PREFIX"] = metadata.config.test_prefix
    if "BUILD_PREFIX" in env:
        del env["BUILD_PREFIX"]

    # In the future, we will need to support testing cross compiled
    #     packages on physical hardware. until then it is expected that
    #     something like QEMU or Wine will be used on the build machine,
    #     therefore, for now, we use host_subdir.

    # ensure that the test prefix isn't kept between variants
    utils.rm_rf(metadata.config.test_prefix)

    if solver is None:
        solver, pkg_cache_path = get_solver(metadata.config.host_subdir)
    else:
        pkg_cache_path = PackageCacheData.first_writable().pkgs_dir

    solver.replace_channels()
    transaction = solver.solve(specs, [pkg_cache_path])

    downloaded = transaction.fetch_extract_packages(
        pkg_cache_path, solver.repos + list(solver.local_repos.values()),
    )
    if not downloaded:
        raise RuntimeError("Did not succeed in downloading packages.")

    mkdir_p(metadata.config.test_prefix)
    transaction.execute(
        PrefixData(metadata.config.test_prefix), pkg_cache_path,
    )

    with utils.path_prepended(metadata.config.test_prefix):
        env = dict(os.environ.copy())
        env.update(environ.get_dict(m=metadata, prefix=metadata.config.test_prefix))
        env["CONDA_BUILD_STATE"] = "TEST"
        if env_path_backup_var_exists:
            env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]

    if config.test_run_post:
        from conda_build.utils import get_installed_packages

        installed = get_installed_packages(metadata.config.test_prefix)
        files = installed[metadata.meta["package"]["name"]]["files"]
        replacements = get_all_replacements(metadata.config)
        try_download(metadata, False, True)
        create_info_files(metadata, replacements, files, metadata.config.test_prefix)
        post_build(metadata, files, None, metadata.config.test_prefix, True)

    # when workdir is removed, the source files are unavailable.  There's the test/source_files
    #    entry that lets people keep these files around.  The files are copied into test_dir for
    #    intuitive relative path behavior, though, not work_dir, so we need to adjust where
    #    SRC_DIR points.  The initial CWD during tests is test_dir.
    if metadata.config.remove_work_dir:
        env["SRC_DIR"] = metadata.config.test_dir

    test_script, _ = write_test_scripts(
        metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace
    )

    if utils.on_win:
        cmd = [os.environ.get("COMSPEC", "cmd.exe"), "/d", "/c", test_script]
    else:
        cmd = (
            [shell_path]
            + (["-x"] if metadata.config.debug else [])
            + ["-o", "errexit", test_script]
        )
    try:
        test_stats = {}
        if not provision_only:
            # rewrite long paths in stdout back to their env variables
            if metadata.config.debug or metadata.config.no_rewrite_stdout_env:
                rewrite_env = None
            else:
                rewrite_env = {k: env[k] for k in ["PREFIX", "SRC_DIR"] if k in env}
                if metadata.config.verbose:
                    for k, v in rewrite_env.items():
                        print(
                            "{0} {1}={2}".format(
                                "set" if test_script.endswith(".bat") else "export",
                                k,
                                v,
                            )
                        )
            utils.check_call_env(
                cmd,
                env=env,
                cwd=metadata.config.test_dir,
                stats=test_stats,
                rewrite_stdout_env=rewrite_env,
            )
            log_stats(test_stats, "testing {}".format(metadata.name()))
            # TODO need to implement metadata.get_used_loop_vars
            # if stats is not None and metadata.config.variants:
            #     stats[
            #         stats_key(metadata, "test_{}".format(metadata.name()))
            #     ] = test_stats
            if os.path.exists(join(metadata.config.test_dir, "TEST_FAILED")):
                raise subprocess.CalledProcessError(-1, "")
            print("TEST END:", test_package_name)

    except subprocess.CalledProcessError as _:  # noqa
        tests_failed(
            metadata,
            move_broken=move_broken,
            broken_dir=metadata.config.broken_dir,
            config=metadata.config,
        )
        raise

    if config.need_cleanup and config.recipe_dir is not None and not provision_only:
        utils.rm_rf(config.recipe_dir)

    return True
Exemplo n.º 37
0
def build(m, get_src=True, verbose=True, post=None, channel_urls=(),
    override_channels=False, include_recipe=True):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''

    if (m.get_value('build/detect_binary_files_with_prefix')
        or m.binary_has_prefix_files()):
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if m.skip():
        print("Skipped: The %s recipe defines build/skip for this "
              "configuration." % m.dist())
        sys.exit(0)

    if post in [False, None]:
        print("Removing old build environment")
        if on_win:
            if isdir(config.short_build_prefix):
                move_to_trash(config.short_build_prefix, '')
            if isdir(config.long_build_prefix):
                move_to_trash(config.long_build_prefix, '')
        else:
            rm_rf(config.short_build_prefix)
            rm_rf(config.long_build_prefix)
        print("Removing old work directory")
        if on_win:
            if isdir(source.WORK_DIR):
                move_to_trash(source.WORK_DIR, '')
        else:
            rm_rf(source.WORK_DIR)

        # Display the name only
        # Version number could be missing due to dependency on source info.
        print("BUILD START:", m.dist())
        create_env(config.build_prefix,
            [ms.spec for ms in m.ms_depends('build')],
            verbose=verbose, channel_urls=channel_urls,
            override_channels=override_channels)

        if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
            print("%s is installed as a build dependency. Removing." %
                m.name())
            index = get_build_index(clear_cache=False, channel_urls=channel_urls, override_channels=override_channels)
            actions = plan.remove_actions(config.build_prefix, [m.name()], index=index)
            assert not plan.nothing_to_do(actions), actions
            plan.display_actions(actions, index)
            plan.execute_actions(actions, index)

        if get_src:
            source.provide(m.path, m.get_section('source'))
            # Parse our metadata again because we did not initialize the source
            # information before.
            m.parse_again()

        print("Package:", m.dist())

        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(config.info_dir)
        files1 = prefix_files()
        for pat in m.always_include_files():
            has_matches = False
            for f in set(files1):
                if fnmatch.fnmatch(f, pat):
                    print("Including in package existing file", f)
                    files1.discard(f)
                    has_matches = True
            if not has_matches:
                sys.exit("Error: Glob %s from always_include_files does not match any files" % pat)
        # Save this for later
        with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
            f.write(u'\n'.join(sorted(list(files1))))
            f.write(u'\n')

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')

            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                build_file = join(source.get_dir(), 'conda_build.sh')
                with open(build_file, 'w') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if isfile(build_file):
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        assert not exists(config.info_dir)
        files2 = prefix_files()

        post_process(sorted(files2 - files1), preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files()
        if any(config.meta_dir in join(config.build_prefix, f) for f in
            files2 - files1):
            sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory.  This error
usually comes from using conda in the build script.  Avoid doing this, as it
can lead to packages that include their dependencies.""" %
                (tuple(f for f in files2 - files1 if config.meta_dir in
                    join(config.build_prefix, f)),)))
        post_build(m, sorted(files2 - files1))
        create_info_files(m, sorted(files2 - files1),
                          include_recipe=bool(m.path) and include_recipe)
        if m.get_value('build/noarch_python'):
            import conda_build.noarch_python as noarch_python
            noarch_python.transform(m, sorted(files2 - files1))

        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(config.build_prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 38
0
def build(m, get_src=True, verbose=True, post=None):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''
    if post in [False, None]:
        rm_rf(prefix)

        print("BUILD START:", m.dist())
        create_env(prefix, [ms.spec for ms in m.ms_depends('build')],
                   verbose=verbose)

        if get_src:
            source.provide(m.path, m.get_section('source'))
        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(info_dir)
        files1 = prefix_files()
        if post == False:
            # Save this for later
            with open(join(source.WORK_DIR, 'prefix_files.txt'), 'w') as f:
                f.write(u'\n'.join(sorted(list(files1))))
                f.write(u'\n')

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')

            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                with open(build_file, 'w', encoding='utf-8') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if exists(build_file):
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(source.WORK_DIR, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        post_process(
            preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

        assert not exists(info_dir)
        files2 = prefix_files()

        post_build(sorted(files2 - files1),
                   binary_relocation=bool(
                       m.get_value('build/binary_relocation', True)))
        create_info_files(m,
                          sorted(files2 - files1),
                          include_recipe=bool(m.path))
        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 39
0
def build(m, bld_bat):
    with path_prepended(m.config.build_prefix):
        env = environ.get_dict(config=m.config, m=m)
    env["CONDA_BUILD_STATE"] = "BUILD"

    # set variables like CONDA_PY in the test environment
    env.update(set_language_env_vars(m.config.variant))

    for name in 'BIN', 'INC', 'LIB':
        path = env['LIBRARY_' + name]
        if not isdir(path):
            os.makedirs(path)

    src_dir = m.config.work_dir
    if os.path.isfile(bld_bat):
        with open(bld_bat) as fi:
            data = fi.read()
        with open(join(src_dir, 'bld.bat'), 'w') as fo:
            # more debuggable with echo on
            fo.write('@echo on\n')
            for key, value in env.items():
                if value:
                    fo.write('set "{key}={value}"\n'.format(key=key, value=value))
            if not m.uses_new_style_compiler_activation:
                fo.write(msvc_env_cmd(bits=bits, config=m.config,
                                    override=m.get_value('build/msvc_compiler', None)))
            # Reset echo on, because MSVC scripts might have turned it off
            fo.write('@echo on\n')
            fo.write('set "INCLUDE={};%INCLUDE%"\n'.format(env["LIBRARY_INC"]))
            fo.write('set "LIB={};%LIB%"\n'.format(env["LIBRARY_LIB"]))
            if m.config.activate:
                fo.write('call "{conda_root}\\activate.bat" "{prefix}"\n'.format(
                    conda_root=root_script_dir,
                    prefix=m.config.build_prefix))
                if m.is_cross:
                    # HACK: we need both build and host envs "active" - i.e. on PATH,
                    #     and with their activate.d scripts sourced. Conda only
                    #     lets us activate one, though. This is a
                    #     vile hack to trick conda into "stacking"
                    #     two environments.
                    #
                    # Net effect: binaries come from host first, then build
                    #
                    # Conda 4.4 may break this by reworking the activate scripts.
                    #  ^^ shouldn't be true
                    # In conda 4.4, export CONDA_MAX_SHLVL=2 to stack envs to two
                    #   levels deep.
                    # conda 4.4 does require that a conda-meta/history file
                    #   exists to identify a valid conda environment
                    history_file = join(m.config.host_prefix, 'conda-meta', 'history')
                    if not isfile(history_file):
                        if not isdir(dirname(history_file)):
                            os.makedirs(dirname(history_file))
                        open(history_file, 'a').close()
                    # removing this placeholder should make conda double-activate with conda 4.3
                    fo.write('set "PATH=%PATH:CONDA_PATH_PLACEHOLDER;=%"\n')
                    fo.write('set CONDA_MAX_SHLVL=2\n')
                    fo.write('call "{conda_root}\\activate.bat" "{prefix}"\n'.format(
                        conda_root=root_script_dir,
                        prefix=m.config.host_prefix))
            fo.write("REM ===== end generated header =====\n")
            fo.write(data)

        cmd = ['cmd.exe', '/c', 'bld.bat']
        check_call_env(cmd, cwd=src_dir)

    fix_staged_scripts(join(m.config.build_prefix, 'Scripts'))
Exemplo n.º 40
0
def test(m, config, move_broken=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''

    if not os.path.isdir(config.build_folder):
        os.makedirs(config.build_folder)

    clean_pkg_cache(m.dist(), config.timeout)

    with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
                               timeout=config.timeout):
        tmp_dir = config.test_dir
        if not isdir(tmp_dir):
            os.makedirs(tmp_dir)
        create_files(tmp_dir, m, config)
        # Make Perl or Python-specific test files
        if m.name().startswith('perl-'):
            pl_files = create_pl_files(tmp_dir, m)
            py_files = False
            lua_files = False
        else:
            py_files = create_py_files(tmp_dir, m)
            pl_files = False
            lua_files = False
        shell_files = create_shell_files(tmp_dir, m, config)
        if not (py_files or shell_files or pl_files or lua_files):
            print("Nothing to test for:", m.dist())
            return

        print("TEST START:", m.dist())

        get_build_metadata(m, config=config)
        specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

        # add packages listed in the run environment and test/requires
        specs.extend(ms.spec for ms in m.ms_depends('run'))
        specs += m.get_value('test/requires', [])

        if py_files:
            # as the tests are run by python, ensure that python is installed.
            # (If they already provided python as a run or test requirement,
            #  this won't hurt anything.)
            specs += ['python %s*' % environ.get_py_ver(config)]
        if pl_files:
            # as the tests are run by perl, we need to specify it
            specs += ['perl %s*' % environ.get_perl_ver(config)]
        if lua_files:
            # not sure how this shakes out
            specs += ['lua %s*' % environ.get_lua_ver(config)]

        create_env(config.test_prefix, specs, config=config)

        with path_prepended(config.test_prefix):
            env = dict(os.environ.copy())
            env.update(
                environ.get_dict(config=config, m=m,
                                 prefix=config.test_prefix))
            env["CONDA_BUILD_STATE"] = "TEST"

        if not config.activate:
            # prepend bin (or Scripts) directory
            env = prepend_bin_path(env,
                                   config.test_prefix,
                                   prepend_prefix=True)

            if on_win:
                env['PATH'] = config.test_prefix + os.pathsep + env['PATH']

        for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
            env[varname] = str(getattr(config, varname) or '')

        # Python 2 Windows requires that envs variables be string, not unicode
        env = {str(key): str(value) for key, value in env.items()}
        suffix = "bat" if on_win else "sh"
        test_script = join(tmp_dir,
                           "conda_test_runner.{suffix}".format(suffix=suffix))

        with open(test_script, 'w') as tf:
            if config.activate:
                ext = ".bat" if on_win else ""
                tf.write(
                    "{source} {conda_root}activate{ext} {test_env} {squelch}\n"
                    .format(conda_root=root_script_dir + os.path.sep,
                            source="call" if on_win else "source",
                            ext=ext,
                            test_env=config.test_prefix,
                            squelch=">nul 2>&1" if on_win else "&> /dev/null"))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if py_files:
                tf.write("{python} -s {test_file}\n".format(
                    python=config.test_python,
                    test_file=join(tmp_dir, 'run_test.py')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if pl_files:
                tf.write("{perl} {test_file}\n".format(perl=config.test_perl,
                                                       test_file=join(
                                                           tmp_dir,
                                                           'run_test.pl')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if lua_files:
                tf.write("{lua} {test_file}\n".format(lua=config.test_lua,
                                                      test_file=join(
                                                          tmp_dir,
                                                          'run_test.lua')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if shell_files:
                test_file = join(tmp_dir, 'run_test.' + suffix)
                if on_win:
                    tf.write("call {test_file}\n".format(test_file=test_file))
                    if on_win:
                        tf.write("if errorlevel 1 exit 1\n")
                else:
                    # TODO: Run the test/commands here instead of in run_test.py
                    tf.write("{shell_path} -x -e {test_file}\n".format(
                        shell_path=shell_path, test_file=test_file))

        if on_win:
            cmd = ['cmd.exe', "/d", "/c", test_script]
        else:
            cmd = [shell_path, '-x', '-e', test_script]
        try:
            subprocess.check_call(cmd, env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m,
                         move_broken=move_broken,
                         broken_dir=config.broken_dir,
                         config=config)

    print("TEST END:", m.dist())
Exemplo n.º 41
0
def test(m, config, move_broken=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''

    if not os.path.isdir(config.build_folder):
        os.makedirs(config.build_folder)

    clean_pkg_cache(m.dist(), config.timeout)

    with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"), timeout=config.timeout):
        tmp_dir = config.test_dir
        if not isdir(tmp_dir):
            os.makedirs(tmp_dir)
        create_files(tmp_dir, m, config)
        # Make Perl or Python-specific test files
        if m.name().startswith('perl-'):
            pl_files = create_pl_files(tmp_dir, m)
            py_files = False
            lua_files = False
        else:
            py_files = create_py_files(tmp_dir, m)
            pl_files = False
            lua_files = False
        shell_files = create_shell_files(tmp_dir, m, config)
        if not (py_files or shell_files or pl_files or lua_files):
            print("Nothing to test for:", m.dist())
            return

        print("TEST START:", m.dist())

        get_build_metadata(m, config=config)
        specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

        # add packages listed in the run environment and test/requires
        specs.extend(ms.spec for ms in m.ms_depends('run'))
        specs += m.get_value('test/requires', [])

        if py_files:
            # as the tests are run by python, ensure that python is installed.
            # (If they already provided python as a run or test requirement,
            #  this won't hurt anything.)
            specs += ['python %s*' % environ.get_py_ver(config)]
        if pl_files:
            # as the tests are run by perl, we need to specify it
            specs += ['perl %s*' % environ.get_perl_ver(config)]
        if lua_files:
            # not sure how this shakes out
            specs += ['lua %s*' % environ.get_lua_ver(config)]

        create_env(config.test_prefix, specs, config=config)

        with path_prepended(config.test_prefix):
            env = dict(os.environ.copy())
            env.update(environ.get_dict(config=config, m=m, prefix=config.test_prefix))
            env["CONDA_BUILD_STATE"] = "TEST"

        if not config.activate:
            # prepend bin (or Scripts) directory
            env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)

            if on_win:
                env['PATH'] = config.test_prefix + os.pathsep + env['PATH']

        for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
            env[varname] = str(getattr(config, varname) or '')

        # Python 2 Windows requires that envs variables be string, not unicode
        env = {str(key): str(value) for key, value in env.items()}
        suffix = "bat" if on_win else "sh"
        test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix))

        with open(test_script, 'w') as tf:
            if config.activate:
                ext = ".bat" if on_win else ""
                tf.write("{source} {conda_root}activate{ext} {test_env} {squelch}\n".format(
                    conda_root=root_script_dir + os.path.sep,
                    source="call" if on_win else "source",
                    ext=ext,
                    test_env=config.test_prefix,
                    squelch=">nul 2>&1" if on_win else "&> /dev/null"))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if py_files:
                tf.write("{python} -s {test_file}\n".format(
                    python=config.test_python,
                    test_file=join(tmp_dir, 'run_test.py')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if pl_files:
                tf.write("{perl} {test_file}\n".format(
                    perl=config.test_perl,
                    test_file=join(tmp_dir, 'run_test.pl')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if lua_files:
                tf.write("{lua} {test_file}\n".format(
                    lua=config.test_lua,
                    test_file=join(tmp_dir, 'run_test.lua')))
                if on_win:
                    tf.write("if errorlevel 1 exit 1\n")
            if shell_files:
                test_file = join(tmp_dir, 'run_test.' + suffix)
                if on_win:
                    tf.write("call {test_file}\n".format(test_file=test_file))
                    if on_win:
                        tf.write("if errorlevel 1 exit 1\n")
                else:
                    # TODO: Run the test/commands here instead of in run_test.py
                    tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path,
                                                                       test_file=test_file))

        if on_win:
            cmd = ['cmd.exe', "/d", "/c", test_script]
        else:
            cmd = [shell_path, '-x', '-e', test_script]
        try:
            subprocess.check_call(cmd, env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken, broken_dir=config.broken_dir, config=config)

    print("TEST END:", m.dist())
Exemplo n.º 42
0
def build(m, get_src=True, verbose=True, post=None, channel_urls=(), override_channels=False):
    """
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    """

    if m.get_value("build/detect_binary_files_with_prefix") or m.binary_has_prefix_files():
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if post in [False, None]:
        print("Removing old build directory")
        rm_rf(config.short_build_prefix)
        rm_rf(config.long_build_prefix)
        print("Removing old work directory")
        rm_rf(source.WORK_DIR)

        # Display the name only
        # Version number could be missing due to dependency on source info.
        print("BUILD START:", m.dist())
        create_env(
            config.build_prefix,
            [ms.spec for ms in m.ms_depends("build")],
            verbose=verbose,
            channel_urls=channel_urls,
            override_channels=override_channels,
        )

        if m.name() in [i.rsplit("-", 2)[0] for i in linked(config.build_prefix)]:
            print("%s is installed as a build dependency. Removing." % m.name())
            index = get_build_index(clear_cache=False, channel_urls=channel_urls, override_channels=override_channels)
            actions = plan.remove_actions(config.build_prefix, [m.name()], index=index)
            assert not plan.nothing_to_do(actions), actions
            plan.display_actions(actions, index)
            plan.execute_actions(actions, index)

        if get_src:
            source.provide(m.path, m.get_section("source"))
            # Parse our metadata again because we did not initialize the source
            # information before.
            m.parse_again()

        print("Package:", m.dist())

        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(config.info_dir)
        files1 = prefix_files()
        for rx in m.always_include_files():
            pat = re.compile(rx)
            has_matches = False
            for f in set(files1):
                if pat.match(f):
                    print("Including in package existing file", f)
                    files1.discard(f)
                    has_matches = True
            if not has_matches:
                sys.exit("Error: Regex %s from always_include_files does not match any files" % rx)
        # Save this for later
        with open(join(config.croot, "prefix_files.txt"), "w") as f:
            f.write(u"\n".join(sorted(list(files1))))
            f.write(u"\n")

        if sys.platform == "win32":
            import conda_build.windows as windows

            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, "build.sh")

            script = m.get_value("build/script", None)
            if script:
                if isinstance(script, list):
                    script = "\n".join(script)
                build_file = join(source.get_dir(), "conda_build.sh")
                with open(build_file, "w") as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if exists(build_file):
                cmd = ["/bin/bash", "-x", "-e", build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(config.croot, "prefix_files.txt"), "r") as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value("build/entry_points"))
        assert not exists(config.info_dir)
        files2 = prefix_files()

        post_process(sorted(files2 - files1), preserve_egg_dir=bool(m.get_value("build/preserve_egg_dir")))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files()
        assert not any(config.meta_dir in join(config.build_prefix, f) for f in files2 - files1)
        post_build(m, sorted(files2 - files1))
        create_info_files(m, sorted(files2 - files1), include_recipe=bool(m.path))
        if m.get_value("build/noarch_python"):
            import conda_build.noarch_python as noarch_python

            noarch_python.transform(m, sorted(files2 - files1))

        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, "w:bz2")
        for f in sorted(files3 - files1):
            t.add(join(config.build_prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 43
0
def build(m, config, post=None, need_source_download=True, need_reparse_in_env=False):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    :type keep_old_work: bool: Keep any previous work directory.
    :type need_source_download: bool: if rendering failed to download source
    (due to missing tools), retry here after build env is populated
    '''

    if m.skip():
        print_skip_message(m)
        return False

    if config.skip_existing:
        package_exists = is_package_built(m, config)
        if package_exists:
            print(m.dist(), "is already built in {0}, skipping.".format(package_exists))
            return False

    if post in [False, None]:
        print("BUILD START:", m.dist())
        if m.uses_jinja and (need_source_download or need_reparse_in_env):
            print("    (actual version deferred until further download or env creation)")

        specs = [ms.spec for ms in m.ms_depends('build')]
        create_env(config.build_prefix, specs, config=config)
        vcs_source = m.uses_vcs_in_build
        if vcs_source and vcs_source not in specs:
            vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
            has_vcs_available = os.path.isfile(external.find_executable(vcs_executable,
                                                                config.build_prefix) or "")
            if not has_vcs_available:
                if (vcs_source != "mercurial" or
                        not any(spec.startswith('python') and "3." in spec
                                for spec in specs)):
                    specs.append(vcs_source)

                    log.warn("Your recipe depends on %s at build time (for templates), "
                            "but you have not listed it as a build dependency.  Doing "
                                "so for this build.", vcs_source)

                    # Display the name only
                    # Version number could be missing due to dependency on source info.
                    create_env(config.build_prefix, specs, config=config)
                else:
                    raise ValueError("Your recipe uses mercurial in build, but mercurial"
                                    " does not yet support Python 3.  Please handle all of "
                                    "your mercurial actions outside of your build script.")

        if need_source_download:
            # Execute any commands fetching the source (e.g., git) in the _build environment.
            # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
            # dependencies.
            with path_prepended(config.build_prefix):
                m, need_source_download, need_reparse_in_env = parse_or_try_download(m,
                                                                no_download_source=False,
                                                                force_download=True,
                                                                config=config)
            assert not need_source_download, "Source download failed.  Please investigate."
            if m.uses_jinja:
                print("BUILD START (revised):", m.dist())

        if need_reparse_in_env:
            reparse(m, config=config)
            print("BUILD START (revised):", m.dist())

        if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
            print("%s is installed as a build dependency. Removing." %
                m.name())
            index = get_build_index(config=config, clear_cache=False)
            actions = plan.remove_actions(config.build_prefix, [m.name()], index=index)
            assert not plan.nothing_to_do(actions), actions
            plan.display_actions(actions, index)
            plan.execute_actions(actions, index)

        print("Package:", m.dist())

        with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
                                   timeout=config.timeout):
            # get_dir here might be just work, or it might be one level deeper,
            #    dependening on the source.
            src_dir = config.work_dir
            if isdir(src_dir):
                print("source tree in:", src_dir)
            else:
                print("no source - creating empty work folder")
                os.makedirs(src_dir)

            rm_rf(config.info_dir)
            files1 = prefix_files(prefix=config.build_prefix)
            for pat in m.always_include_files():
                has_matches = False
                for f in set(files1):
                    if fnmatch.fnmatch(f, pat):
                        print("Including in package existing file", f)
                        files1.discard(f)
                        has_matches = True
                if not has_matches:
                    log.warn("Glob %s from always_include_files does not match any files", pat)
            # Save this for later
            with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
                f.write(u'\n'.join(sorted(list(files1))))
                f.write(u'\n')

            # Use script from recipe?
            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)

            if isdir(src_dir):
                if on_win:
                    build_file = join(m.path, 'bld.bat')
                    if script:
                        build_file = join(src_dir, 'bld.bat')
                        with open(build_file, 'w') as bf:
                            bf.write(script)
                    import conda_build.windows as windows
                    windows.build(m, build_file, config=config)
                else:
                    build_file = join(m.path, 'build.sh')

                    # There is no sense in trying to run an empty build script.
                    if isfile(build_file) or script:
                        with path_prepended(config.build_prefix):
                            env = environ.get_dict(config=config, m=m)
                        env["CONDA_BUILD_STATE"] = "BUILD"
                        work_file = join(config.work_dir, 'conda_build.sh')
                        if script:
                            with open(work_file, 'w') as bf:
                                bf.write(script)
                        if config.activate:
                            if isfile(build_file):
                                data = open(build_file).read()
                            else:
                                data = open(work_file).read()
                            with open(work_file, 'w') as bf:
                                bf.write("source {conda_root}activate {build_prefix} &> "
                                    "/dev/null\n".format(conda_root=root_script_dir + os.path.sep,
                                                         build_prefix=config.build_prefix))
                                bf.write(data)
                        else:
                            if not isfile(work_file):
                                copy_into(build_file, work_file, config.timeout)
                        os.chmod(work_file, 0o766)

                        if isfile(work_file):
                            cmd = [shell_path, '-x', '-e', work_file]
                            # this should raise if any problems occur while building
                            _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m, config=config)
        create_post_scripts(m, config=config)

        if not is_noarch_python(m):
            create_entry_points(m.get_value('build/entry_points'), config=config)
        files2 = prefix_files(prefix=config.build_prefix)

        post_process(sorted(files2 - files1),
                     prefix=config.build_prefix,
                     config=config,
                     preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')),
                     noarch=m.get_value('build/noarch'))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files(prefix=config.build_prefix)
        if any(config.meta_dir in join(config.build_prefix, f) for f in files2 - files1):
            meta_files = (tuple(f for f in files2 - files1 if config.meta_dir in
                    join(config.build_prefix, f)),)
            sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory.
This error usually comes from using conda in the build script.  Avoid doing this, as it
can lead to packages that include their dependencies.""" % meta_files))
        post_build(m, sorted(files2 - files1),
                    prefix=config.build_prefix,
                    build_python=config.build_python,
                    croot=config.croot)

        entry_point_script_names = get_entry_point_script_names(get_entry_points(config, m))
        if is_noarch_python(m):
            pkg_files = [f for f in sorted(files2 - files1) if f not in entry_point_script_names]
        else:
            pkg_files = sorted(files2 - files1)

        create_info_files(m, pkg_files, config=config, prefix=config.build_prefix)

        if m.get_value('build/noarch_python'):
            noarch_python.transform(m, sorted(files2 - files1), config.build_prefix)
        elif is_noarch_python(m):
            noarch_python.populate_files(
                m, pkg_files, config.build_prefix, entry_point_script_names)

        files3 = prefix_files(prefix=config.build_prefix)
        fix_permissions(files3 - files1, config.build_prefix)

        path = bldpkg_path(m, config)

        # lock the output directory while we build this file
        # create the tarball in a temporary directory to minimize lock time
        with TemporaryDirectory() as tmp:
            tmp_path = os.path.join(tmp, os.path.basename(path))
            t = tarfile.open(tmp_path, 'w:bz2')

            def order(f):
                # we don't care about empty files so send them back via 100000
                fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
                # info/* records will be False == 0, others will be 1.
                info_order = int(os.path.dirname(f) != 'info')
                return info_order, fsize

            # add files in order of a) in info directory, b) increasing size so
            # we can access small manifest or json files without decompressing
            # possible large binary or data files
            for f in sorted(files3 - files1, key=order):
                t.add(join(config.build_prefix, f), f)
            t.close()

            # we're done building, perform some checks
            tarcheck.check_all(tmp_path)

            copy_into(tmp_path, path, config.timeout)
        update_index(config.bldpkgs_dir, config, could_be_mirror=False)

    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())

    # returning true here says package is OK to test
    return True
Exemplo n.º 44
0
def build(m, get_src=True, verbose=True, post=None, channel_urls=(), override_channels=False):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :param get_src: Should we download the source?
    :type get_src: bool
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    '''
    if post in [False, None]:
        print("Removing old build directory")
        rm_rf(config.short_build_prefix)
        rm_rf(config.long_build_prefix)
        print("Removing old work directory")
        rm_rf(source.WORK_DIR)

        if (m.get_value('build/detect_binary_files_with_prefix')
            or m.binary_has_prefix_files()):
            # We must use a long prefix here as the package will only be
            # installable into prefixes shorter than this one.
            config.use_long_build_prefix = True
        else:
            # In case there are multiple builds in the same process
            config.use_long_build_prefix = False

        # Display the name only
        # Version number could be missing due to dependency on source info.
        print("BUILD START:", m.dist())
        create_env(config.build_prefix,
            [ms.spec for ms in m.ms_depends('build')],
            verbose=verbose, channel_urls=channel_urls,
            override_channels=override_channels)

        if get_src:
            source.provide(m.path, m.get_section('source'))
            # Parse our metadata again because we did not initialize the source
            # information before.
            m.parse_again()

        print("Package:", m.dist())

        assert isdir(source.WORK_DIR)
        src_dir = source.get_dir()
        contents = os.listdir(src_dir)
        if contents:
            print("source tree in:", src_dir)
        else:
            print("no source")

        rm_rf(config.info_dir)
        files1 = prefix_files().difference(set(m.always_include_files()))
        # Save this for later
        with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
            f.write(u'\n'.join(sorted(list(files1))))
            f.write(u'\n')

        if sys.platform == 'win32':
            import conda_build.windows as windows
            windows.build(m)
        else:
            env = environ.get_dict(m)
            build_file = join(m.path, 'build.sh')

            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)
                build_file = join(source.get_dir(), 'conda_build.sh')
                with open(build_file, 'w') as bf:
                    bf.write(script)
                os.chmod(build_file, 0o766)

            if exists(build_file):
                cmd = ['/bin/bash', '-x', '-e', build_file]

                _check_call(cmd, env=env, cwd=src_dir)

    if post in [True, None]:
        if post == True:
            with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                files1 = set(f.read().splitlines())

        get_build_metadata(m)
        create_post_scripts(m)
        create_entry_points(m.get_value('build/entry_points'))
        assert not exists(config.info_dir)
        files2 = prefix_files()

        post_process(sorted(files2 - files1), preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))

        # The post processing may have deleted some files (like easy-install.pth)
        files2 = prefix_files()
        post_build(m, sorted(files2 - files1))
        create_info_files(m, sorted(files2 - files1),
                          include_recipe=bool(m.path))
        if m.get_value('build/noarch_python'):
            import conda_build.noarch_python as noarch_python
            noarch_python.transform(m, sorted(files2 - files1))

        files3 = prefix_files()
        fix_permissions(files3 - files1)

        path = bldpkg_path(m)
        t = tarfile.open(path, 'w:bz2')
        for f in sorted(files3 - files1):
            t.add(join(config.build_prefix, f), f)
        t.close()

        print("BUILD END:", m.dist())

        # we're done building, perform some checks
        tarcheck.check_all(path)
        update_index(config.bldpkgs_dir)
    else:
        print("STOPPING BUILD BEFORE POST:", m.dist())
Exemplo n.º 45
0
def build(m,
          post=None,
          include_recipe=True,
          keep_old_work=False,
          need_source_download=True,
          verbose=True,
          dirty=False,
          activate=True):
    '''
    Build the package with the specified metadata.

    :param m: Package metadata
    :type m: Metadata
    :type post: bool or None. None means run the whole build. True means run
    post only. False means stop just before the post.
    :type keep_old_work: bool: Keep any previous work directory.
    :type need_source_download: bool: if rendering failed to download source
    (due to missing tools), retry here after build env is populated
    '''

    if (m.get_value('build/detect_binary_files_with_prefix')
            or m.binary_has_prefix_files()) and not on_win:
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if m.skip():
        print("Skipped: The %s recipe defines build/skip for this "
              "configuration." % m.dist())
        return

    with Locked(cc.root_dir):

        # If --keep-old-work, then move the contents of source.WORK_DIR to a
        # temporary directory for the duration of the build.
        # The source unpacking procedure is too varied and complex
        # to allow this to be written cleanly (see source.get_dir() for example)
        if keep_old_work:
            old_WORK_DIR = tempfile.mkdtemp()
            old_sub_dirs = [
                name for name in os.listdir(source.WORK_DIR)
                if os.path.isdir(os.path.join(source.WORK_DIR, name))
            ]
            if len(old_sub_dirs):
                print("Keeping old work directory backup: %s => %s" %
                      (old_sub_dirs, old_WORK_DIR))
                for old_sub in old_sub_dirs:
                    shutil.move(os.path.join(source.WORK_DIR, old_sub),
                                old_WORK_DIR)

        if post in [False, None]:
            print("Removing old build environment")
            print("BUILD START:", m.dist())
            if on_win:
                if isdir(config.short_build_prefix):
                    move_to_trash(config.short_build_prefix, '')
                if isdir(config.long_build_prefix):
                    move_to_trash(config.long_build_prefix, '')
            else:
                rm_rf(config.short_build_prefix)
                rm_rf(config.long_build_prefix)

            # Display the name only
            # Version number could be missing due to dependency on source info.
            create_env(config.build_prefix,
                       [ms.spec for ms in m.ms_depends('build')])

            if need_source_download:
                # Execute any commands fetching the source (e.g., git) in the _build environment.
                # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
                # dependencies.
                m, need_source_download = parse_or_try_download(
                    m,
                    no_download_source=False,
                    force_download=True,
                    verbose=verbose,
                    dirty=dirty)
                assert not need_source_download, "Source download failed.  Please investigate."

            if m.name() in [
                    i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
            ]:
                print("%s is installed as a build dependency. Removing." %
                      m.name())
                index = get_build_index(clear_cache=False)
                actions = plan.remove_actions(config.build_prefix, [m.name()],
                                              index=index)
                assert not plan.nothing_to_do(actions), actions
                plan.display_actions(actions, index)
                plan.execute_actions(actions, index)

            print("Package:", m.dist())

            assert isdir(source.WORK_DIR)
            src_dir = source.get_dir()
            contents = os.listdir(src_dir)
            if contents:
                print("source tree in:", src_dir)
            else:
                print("no source")

            rm_rf(config.info_dir)
            files1 = prefix_files()
            for pat in m.always_include_files():
                has_matches = False
                for f in set(files1):
                    if fnmatch.fnmatch(f, pat):
                        print("Including in package existing file", f)
                        files1.discard(f)
                        has_matches = True
                if not has_matches:
                    sys.exit(
                        "Error: Glob %s from always_include_files does not match any files"
                        % pat)
            # Save this for later
            with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
                f.write(u'\n'.join(sorted(list(files1))))
                f.write(u'\n')

            # Use script from recipe?
            script = m.get_value('build/script', None)
            if script:
                if isinstance(script, list):
                    script = '\n'.join(script)

            if on_win:
                build_file = join(m.path, 'bld.bat')
                if script:
                    build_file = join(source.get_dir(), 'bld.bat')
                    with open(join(source.get_dir(), 'bld.bat'), 'w') as bf:
                        bf.write(script)
                import conda_build.windows as windows
                windows.build(m, build_file, dirty=dirty, activate=activate)
            else:
                build_file = join(m.path, 'build.sh')

                # There is no sense in trying to run an empty build script.
                if isfile(build_file) or script:
                    env = environ.get_dict(m, dirty=dirty)
                    work_file = join(source.get_dir(), 'conda_build.sh')
                    if script:
                        with open(work_file, 'w') as bf:
                            bf.write(script)
                    if activate:
                        if isfile(build_file):
                            data = open(build_file).read()
                        else:
                            data = open(work_file).read()
                        with open(work_file, 'w') as bf:
                            bf.write("source activate {build_prefix}\n".format(
                                build_prefix=config.build_prefix))
                            bf.write(data)
                    else:
                        if not isfile(work_file):
                            shutil.copy(build_file, work_file)
                    os.chmod(work_file, 0o766)

                    if isfile(work_file):
                        cmd = [shell_path, '-x', '-e', work_file]

                        _check_call(cmd, env=env, cwd=src_dir)

        if post in [True, None]:
            if post:
                with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
                    files1 = set(f.read().splitlines())

            get_build_metadata(m)
            create_post_scripts(m)
            create_entry_points(m.get_value('build/entry_points'))
            assert not exists(config.info_dir)
            files2 = prefix_files()

            post_process(sorted(files2 - files1),
                         preserve_egg_dir=bool(
                             m.get_value('build/preserve_egg_dir')))

            # The post processing may have deleted some files (like easy-install.pth)
            files2 = prefix_files()
            if any(config.meta_dir in join(config.build_prefix, f)
                   for f in files2 - files1):
                sys.exit(
                    indent(
                        """Error: Untracked file(s) %s found in conda-meta directory.
    This error usually comes from using conda in the build script.  Avoid doing this, as it
    can lead to packages that include their dependencies.""" %
                        (tuple(f for f in files2 - files1 if config.meta_dir in
                               join(config.build_prefix, f)), )))
            post_build(m, sorted(files2 - files1))
            create_info_files(m,
                              sorted(files2 - files1),
                              include_recipe=bool(m.path) and include_recipe)
            if m.get_value('build/noarch_python'):
                import conda_build.noarch_python as noarch_python
                noarch_python.transform(m, sorted(files2 - files1))

            files3 = prefix_files()
            fix_permissions(files3 - files1)

            path = bldpkg_path(m)
            t = tarfile.open(path, 'w:bz2')

            def order(f):
                # we don't care about empty files so send them back via 100000
                fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
                # info/* records will be False == 0, others will be 1.
                info_order = int(os.path.dirname(f) != 'info')
                return info_order, fsize

            # add files in order of a) in info directory, b) increasing size so
            # we can access small manifest or json files without decompressing
            # possible large binary or data files
            for f in sorted(files3 - files1, key=order):
                t.add(join(config.build_prefix, f), f)
            t.close()

            print("BUILD END:", m.dist())

            # we're done building, perform some checks
            tarcheck.check_all(path)
            update_index(config.bldpkgs_dir)
        else:
            print("STOPPING BUILD BEFORE POST:", m.dist())

        if keep_old_work and len(old_sub_dirs):
            print("Restoring old work directory backup: %s :: %s => %s" %
                  (old_WORK_DIR, old_sub_dirs, source.WORK_DIR))
            for old_sub in old_sub_dirs:
                if os.path.exists(os.path.join(source.WORK_DIR, old_sub)):
                    print(
                        "Not restoring old source directory %s over new build's version"
                        % (old_sub))
                else:
                    shutil.move(os.path.join(old_WORK_DIR, old_sub),
                                source.WORK_DIR)
            shutil.rmtree(old_WORK_DIR, ignore_errors=True)
Exemplo n.º 46
0
def build(m,
          verbose=True,
          channel_urls=(),
          override_channels=False,
          wheel_dir="./build"):
    '''
    Build the package with the specified metadata.
    :param m: Package metadata
    :type m: Metadata
    '''

    if (m.get_value('build/detect_binary_files_with_prefix')
            or m.binary_has_prefix_files()):
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if m.skip():
        print("Skipped: The %s recipe defines build/skip for this "
              "configuration." % m.dist())
        return

    print("Removing old build environment")
    if on_win:
        if isdir(config.short_build_prefix):
            move_to_trash(config.short_build_prefix, '')
        if isdir(config.long_build_prefix):
            move_to_trash(config.long_build_prefix, '')
    else:
        rm_rf(config.short_build_prefix)
        rm_rf(config.long_build_prefix)
    print("Removing old work directory")
    if on_win:
        if isdir(source.WORK_DIR):
            move_to_trash(source.WORK_DIR, '')
    else:
        rm_rf(source.WORK_DIR)

    # Display the name only
    # Version number could be missing due to dependency on source info.
    print("BUILD START:", m.dist())
    create_env(config.build_prefix, [ms.spec for ms in m.ms_depends('build')],
               verbose=verbose,
               channel_urls=channel_urls,
               override_channels=override_channels)

    if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
        print("%s is installed as a build dependency. Removing." % m.name())
        index = get_build_index(clear_cache=False,
                                channel_urls=channel_urls,
                                override_channels=override_channels)
        actions = plan.remove_actions(config.build_prefix, [m.name()],
                                      index=index)
        assert not plan.nothing_to_do(actions), actions
        plan.display_actions(actions, index)
        plan.execute_actions(actions, index)

    # downlaod source code...
    source.provide(m.path, m.get_section('source'))

    # Parse our metadata again because we did not initialize the source
    # information before.
    m.parse_again()

    print("Package:", m.dist())

    assert isdir(source.WORK_DIR)
    src_dir = source.get_dir()
    contents = os.listdir(src_dir)
    if contents:
        print("source tree in:", src_dir)
    else:
        print("no source")

    rm_rf(config.info_dir)
    files1 = prefix_files()
    for pat in m.always_include_files():
        has_matches = False
        for f in set(files1):
            if fnmatch.fnmatch(f, pat):
                print("Including in package existing file", f)
                files1.discard(f)
                has_matches = True
        if not has_matches:
            sys.exit(
                "Error: Glob %s from always_include_files does not match any files"
                % pat)
    # Save this for later
    with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
        f.write(u'\n'.join(sorted(list(files1))))
        f.write(u'\n')
    print("Source dir: %s" % src_dir)
    if sys.platform == 'win32':
        windows_build(m)
    else:
        env = environ.get_dict(m)
        build_file = join(m.path, 'build_wheel.sh')

        if not isfile(build_file):
            print("Using plain 'python setup.py bdist_wheel'  as build script")
            build_file = join(src_dir, 'build_wheel.sh')
            with open(build_file, 'w') as fo:
                fo.write('\n')
                fo.write('# Autogenerated build command:\n')
                fo.write('python setup.py bdist_wheel\n')
                fo.write('\n')

        cmd = [shell_path, '-x', '-e', build_file]
        _check_call(cmd, env=env, cwd=src_dir)

    all_wheels = glob(join(src_dir, "dist", '*.whl'))
    if len(all_wheels) == 0:
        print("No wheels produced!")
    else:
        if len(all_wheels) == 1:
            print("More than one wheel produced!")
        try:
            os.makedirs(wheel_dir)
            print("Created wheel dir: %s:" % wheel_dir)
        except OSError:
            if not isdir(wheel_dir):
                raise
        print("Copying to %s:" % wheel_dir)
        for wheel in all_wheels:
            shutil.copy(wheel, wheel_dir)
            print(" %s" % basename(wheel))
Exemplo n.º 47
0
def test(m, move_broken=True, activate=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''

    with Locked(cc.root_dir):

        # remove from package cache
        rm_pkgs_cache(m.dist())

        tmp_dir = join(config.croot, 'test-tmp_dir')
        rm_rf(tmp_dir)
        if on_win:
            time.sleep(
                1
            )  # wait for rm_rf(tmp_dir) to finish before recreating tmp_dir
        os.makedirs(tmp_dir)
        create_files(tmp_dir, m)
        # Make Perl or Python-specific test files
        if m.name().startswith('perl-'):
            pl_files = create_pl_files(tmp_dir, m)
            py_files = False
            lua_files = False
        else:
            py_files = create_py_files(tmp_dir, m)
            pl_files = False
            lua_files = False
        shell_files = create_shell_files(tmp_dir, m)
        if not (py_files or shell_files or pl_files or lua_files):
            print("Nothing to test for:", m.dist())
            return

        print("TEST START:", m.dist())
        if on_win:
            if isdir(config.build_prefix):
                move_to_trash(config.build_prefix, '')
            if isdir(config.test_prefix):
                move_to_trash(config.test_prefix, '')
        else:
            rm_rf(config.build_prefix)
            rm_rf(config.test_prefix)

        get_build_metadata(m)
        specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

        # add packages listed in the run environment and test/requires
        specs.extend(ms.spec for ms in m.ms_depends('run'))
        specs += m.get_value('test/requires', [])

        if py_files:
            # as the tests are run by python, ensure that python is installed.
            # (If they already provided python as a run or test requirement,
            #  this won't hurt anything.)
            specs += ['python %s*' % environ.get_py_ver()]
        if pl_files:
            # as the tests are run by perl, we need to specify it
            specs += ['perl %s*' % environ.get_perl_ver()]
        if lua_files:
            # not sure how this shakes out
            specs += ['lua %s*' % environ.get_lua_ver()]

        create_env(config.test_prefix, specs)

        env = dict(os.environ)
        env.update(environ.get_dict(m, prefix=config.test_prefix))

        if not activate:
            # prepend bin (or Scripts) directory
            env = prepend_bin_path(env,
                                   config.test_prefix,
                                   prepend_prefix=True)

            if on_win:
                env['PATH'] = config.test_prefix + os.pathsep + env['PATH']

        for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
            env[varname] = str(getattr(config, varname) or '')
        env['PREFIX'] = config.test_prefix

        # Python 2 Windows requires that envs variables be string, not unicode
        env = {str(key): str(value) for key, value in env.items()}
        suffix = "bat" if on_win else "sh"
        test_script = join(tmp_dir,
                           "conda_test_runner.{suffix}".format(suffix=suffix))

        with open(test_script, 'w') as tf:
            if activate:
                tf.write("{source}activate _test\n".format(
                    source="" if on_win else "source "))
            if py_files:
                tf.write("{python} -s {test_file}\n".format(
                    python=config.test_python,
                    test_file=join(tmp_dir, 'run_test.py')))

            if pl_files:
                tf.write("{perl} {test_file}\n".format(python=config.test_perl,
                                                       test_file=join(
                                                           tmp_dir,
                                                           'run_test.pl')))

            if lua_files:
                tf.write("{lua} {test_file}\n".format(python=config.test_perl,
                                                      test_file=join(
                                                          tmp_dir,
                                                          'run_test.lua')))

            if shell_files:
                test_file = join(tmp_dir, 'run_test.' + suffix)
                if on_win:
                    tf.write("call {test_file}\n".format(test_file=test_file))
                else:
                    # TODO: Run the test/commands here instead of in run_test.py
                    tf.write("{shell_path} -x -e {test_file}\n".format(
                        shell_path=shell_path, test_file=test_file))
        if on_win:
            cmd = [env["COMSPEC"], "/d", "/c", test_script]
        else:
            cmd = [shell_path, '-x', '-e', test_script]
        try:
            subprocess.check_call(cmd, env=env, cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m, move_broken=move_broken)

    print("TEST END:", m.dist())
Exemplo n.º 48
0
def test(m, verbose=True):
    '''
    Execute any test scripts for the given package.

    :param m: Package's metadata.
    :type m: Metadata
    '''
    # remove from package cache
    rm_pkgs_cache(m.dist())

    tmp_dir = join(config.croot, 'test-tmp_dir')
    rm_rf(tmp_dir)
    os.makedirs(tmp_dir)
    create_files(tmp_dir, m)
    # Make Perl or Python-specific test files
    if m.name().startswith('perl-'):
        pl_files = create_pl_files(tmp_dir, m)
        py_files = False
    else:
        py_files = create_py_files(tmp_dir, m)
        pl_files = False
    shell_files = create_shell_files(tmp_dir, m)
    if not (py_files or shell_files or pl_files):
        print("Nothing to test for:", m.dist())
        return

    print("TEST START:", m.dist())
    rm_rf(prefix)
    rm_rf(config.test_prefix)
    specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]

    if py_files:
        # as the tests are run by python, we need to specify it
        specs += ['python %s*' % environ.PY_VER]
    if pl_files:
        # as the tests are run by perl, we need to specify it
        specs += ['perl %s*' % environ.PERL_VER]
    # add packages listed in test/requires
    for spec in m.get_value('test/requires'):
        specs.append(spec)

    create_env(config.test_prefix, specs, verbose=verbose)

    env = dict(os.environ)
    # TODO: Include all the same environment variables that are used in
    # building.
    env.update(environ.get_dict(m, prefix=config.test_prefix))

    # prepend bin (or Scripts) directory
    env['PATH'] = (join(config.test_prefix, bin_dirname) + os.pathsep +
                   env['PATH'])

    for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL':
        env[varname] = str(getattr(config, varname))
    env['PREFIX'] = config.test_prefix

    # Python 2 Windows requires that envs variables be string, not unicode
    env = {str(key): str(value) for key, value in env.items()}
    if py_files:
        try:
            subprocess.check_call(
                [config.test_python,
                 join(tmp_dir, 'run_test.py')],
                env=env,
                cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if pl_files:
        try:
            subprocess.check_call(
                [config.test_perl,
                 join(tmp_dir, 'run_test.pl')],
                env=env,
                cwd=tmp_dir)
        except subprocess.CalledProcessError:
            tests_failed(m)

    if shell_files:
        if sys.platform == 'win32':
            test_file = join(tmp_dir, 'run_test.bat')
            cmd = [os.environ['COMSPEC'], '/c', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)
        else:
            test_file = join(tmp_dir, 'run_test.sh')
            # TODO: Run the test/commands here instead of in run_test.py
            cmd = ['/bin/bash', '-x', '-e', test_file]
            try:
                subprocess.check_call(cmd, env=env, cwd=tmp_dir)
            except subprocess.CalledProcessError:
                tests_failed(m)

    print("TEST END:", m.dist())
Exemplo n.º 49
0
def build(m, verbose=True,  channel_urls=(),  override_channels=False, wheel_dir="./build"):
    '''
    Build the package with the specified metadata.
    :param m: Package metadata
    :type m: Metadata
    '''

    if (m.get_value('build/detect_binary_files_with_prefix')
        or m.binary_has_prefix_files()):
        # We must use a long prefix here as the package will only be
        # installable into prefixes shorter than this one.
        config.use_long_build_prefix = True
    else:
        # In case there are multiple builds in the same process
        config.use_long_build_prefix = False

    if m.skip():
        print("Skipped: The %s recipe defines build/skip for this "
              "configuration." % m.dist())
        return

    print("Removing old build environment")
    if on_win:
        if isdir(config.short_build_prefix):
            move_to_trash(config.short_build_prefix, '')
        if isdir(config.long_build_prefix):
            move_to_trash(config.long_build_prefix, '')
    else:
        rm_rf(config.short_build_prefix)
        rm_rf(config.long_build_prefix)
    print("Removing old work directory")
    if on_win:
        if isdir(source.WORK_DIR):
            move_to_trash(source.WORK_DIR, '')
    else:
        rm_rf(source.WORK_DIR)

    # Display the name only
    # Version number could be missing due to dependency on source info.
    print("BUILD START:", m.dist())
    create_env(config.build_prefix,
        [ms.spec for ms in m.ms_depends('build')],
        verbose=verbose, channel_urls=channel_urls,
        override_channels=override_channels)

    if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
        print("%s is installed as a build dependency. Removing." %
            m.name())
        index = get_build_index(clear_cache=False, channel_urls=channel_urls, override_channels=override_channels)
        actions = plan.remove_actions(config.build_prefix, [m.name()], index=index)
        assert not plan.nothing_to_do(actions), actions
        plan.display_actions(actions, index)
        plan.execute_actions(actions, index)

    # downlaod source code...
    source.provide(m.path, m.get_section('source'))

    # Parse our metadata again because we did not initialize the source
    # information before.
    m.parse_again()

    print("Package:", m.dist())

    assert isdir(source.WORK_DIR)
    src_dir = source.get_dir()
    contents = os.listdir(src_dir)
    if contents:
        print("source tree in:", src_dir)
    else:
        print("no source")

    rm_rf(config.info_dir)
    files1 = prefix_files()
    for pat in m.always_include_files():
        has_matches = False
        for f in set(files1):
            if fnmatch.fnmatch(f, pat):
                print("Including in package existing file", f)
                files1.discard(f)
                has_matches = True
        if not has_matches:
            sys.exit("Error: Glob %s from always_include_files does not match any files" % pat)
    # Save this for later
    with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
        f.write(u'\n'.join(sorted(list(files1))))
        f.write(u'\n')
    print("Source dir: %s" % src_dir)
    if sys.platform == 'win32':
        windows_build(m)
    else:
        env = environ.get_dict(m)
        build_file = join(m.path, 'build_wheel.sh')

        if not isfile(build_file):
            print("Using plain 'python setup.py bdist_wheel'  as build script")
            build_file = join(src_dir, 'build_wheel.sh')
            with open(build_file, 'w') as fo:
                fo.write('\n')
                fo.write('# Autogenerated build command:\n')
                fo.write('python setup.py bdist_wheel\n')
                fo.write('\n')

        cmd = [shell_path, '-x', '-e', build_file]
        _check_call(cmd, env=env, cwd=src_dir)

    all_wheels = glob(join(src_dir, "dist", '*.whl'))
    if len(all_wheels) == 0:
        print("No wheels produced!")
    else:
        if len(all_wheels) == 1:
            print("More than one wheel produced!")
        try:
            os.makedirs(wheel_dir)
            print("Created wheel dir: %s:" % wheel_dir)
        except OSError:
            if not isdir(wheel_dir):
                raise
        print("Copying to %s:" % wheel_dir)
        for wheel in all_wheels:
            shutil.copy(wheel, wheel_dir)
            print(" %s" % basename(wheel))