def test_set_exe_file(self): perms = [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH] set_mode = functools.reduce(op.or_, perms) for set_exe in [True, False]: for tmp_dir, repo, pathfunc in parameterize(): filename = "test.txt" filename = os.path.join(tmp_dir, filename) with io.open( filename, "w", encoding="utf-8", newline="\n" ) as fh: fh.write("") if repo is not None: repo.index.add([filename]) fio.set_exe_file(pathfunc(filename), set_exe) file_mode = os.stat(filename).st_mode self.assertEqual(file_mode & set_mode, int(set_exe) * set_mode) if repo is not None: blob = next(repo.index.iter_blobs(BlobFilter(filename)))[1] self.assertEqual( blob.mode & set_mode, int(set_exe) * set_mode )
def test_set_exe_file(self): perms = [ stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH ] set_mode = functools.reduce(op.or_, perms) for set_exe in [True, False]: for tmp_dir, repo, pathfunc in parameterize(): filename = "test.txt" filename = os.path.join(tmp_dir, filename) with io.open(filename, "w", encoding="utf-8", newline="\n") as fh: fh.write("") if repo is not None: repo.index.add([filename]) fio.set_exe_file(pathfunc(filename), set_exe) file_mode = os.stat(filename).st_mode self.assertEqual(file_mode & set_mode, int(set_exe) * set_mode) if repo is not None: blob = next(repo.index.iter_blobs(BlobFilter(filename)))[1] self.assertEqual(blob.mode & set_mode, int(set_exe) * set_mode)
def _render_template_exe_files(forge_config, target_dir, jinja_env, template_files): for template_file in template_files: template = jinja_env.get_template(template_file) target_fname = os.path.join(target_dir, template_file[:-len('.tmpl')]) with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) # Fix permission of template shell files set_exe_file(target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform): if platform == "linux": yum_build_setup = generate_yum_requirements(forge_dir) if yum_build_setup: forge_config["yum_build_setup"] = yum_build_setup forge_config["build_setup"] = _get_build_setup_line( forge_dir, platform, forge_config) if platform == "linux": run_file_name = "run_docker_build" else: run_file_name = "run_osx_build" # TODO: Conda has a convenience for accessing nested yaml content. template_files = [ "{}.sh.tmpl".format(run_file_name), "fast_finish_ci_pr_build.sh.tmpl", ] if platform == "linux": template_files.append("build_steps.sh.tmpl") _render_template_exe_files( forge_config=forge_config, target_dir=os.path.join(forge_dir, ".circleci"), jinja_env=jinja_env, template_files=template_files, ) # Fix permission of other shell files. target_fnames = [ os.path.join(forge_dir, ".circleci", "checkout_merge_commit.sh") ] for target_fname in target_fnames: set_exe_file(target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform): # If the recipe supplies its own run_conda_forge_build_setup script_linux, # we use it instead of the global one. if platform == 'linux': cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_linux') else: cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_osx') build_setup = "" if os.path.exists(cfbs_fpath): if platform == 'linux': build_setup += textwrap.dedent("""\ # Overriding global run_conda_forge_build_setup_linux with local copy. source /home/conda/recipe_root/run_conda_forge_build_setup_linux """) else: build_setup += textwrap.dedent("""\ # Overriding global run_conda_forge_build_setup_osx with local copy. source {recipe_dir}/run_conda_forge_build_setup_osx """.format(recipe_dir=forge_config["recipe_dir"])) else: build_setup += textwrap.dedent("""\ source run_conda_forge_build_setup """) if platform == 'linux': # If there is a "yum_requirements.txt" file in the recipe, we honour it. yum_requirements_fpath = os.path.join(forge_dir, 'recipe', 'yum_requirements.txt') if os.path.exists(yum_requirements_fpath): with open(yum_requirements_fpath) as fh: requirements = [line.strip() for line in fh if line.strip() and not line.strip().startswith('#')] if not requirements: raise ValueError("No yum requirements enabled in the " "yum_requirements.txt, please remove the file " "or add some.") build_setup += textwrap.dedent("""\ # Install the yum requirements defined canonically in the # "recipe/yum_requirements.txt" file. After updating that file, # run "conda smithy rerender" and this line be updated # automatically. /usr/bin/sudo -n yum install -y {} """.format(' '.join(requirements))) forge_config['build_setup'] = build_setup if platform == 'linux': run_file_name = 'run_docker_build' else: run_file_name = 'run_osx_build' # TODO: Conda has a convenience for accessing nested yaml content. template = jinja_env.get_template('{}.tmpl'.format(run_file_name)) target_fname = os.path.join(forge_dir, '.circleci', '{}.sh'.format(run_file_name)) with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) template_name = 'fast_finish_ci_pr_build.sh.tmpl' template = jinja_env.get_template(template_name) target_fname = os.path.join(forge_dir, '.circleci', 'fast_finish_ci_pr_build.sh') with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) # Fix permissions. target_fnames = [ os.path.join(forge_dir, '.circleci', 'checkout_merge_commit.sh'), os.path.join(forge_dir, '.circleci', 'fast_finish_ci_pr_build.sh'), os.path.join(forge_dir, '.circleci', '{}.sh'.format(run_file_name)), ] for each_target_fname in target_fnames: set_exe_file(each_target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform): # If the recipe supplies its own run_conda_forge_build_setup script_linux, # we use it instead of the global one. if platform == "linux": cfbs_fpath = os.path.join(forge_dir, "recipe", "run_conda_forge_build_setup_linux") else: cfbs_fpath = os.path.join(forge_dir, "recipe", "run_conda_forge_build_setup_osx") build_setup = "" if os.path.exists(cfbs_fpath): if platform == "linux": build_setup += textwrap.dedent("""\ # Overriding global run_conda_forge_build_setup_linux with local copy. source /home/conda/recipe_root/run_conda_forge_build_setup_linux """) else: build_setup += textwrap.dedent("""\ # Overriding global run_conda_forge_build_setup_osx with local copy. source {recipe_dir}/run_conda_forge_build_setup_osx """.format(recipe_dir=forge_config["recipe_dir"])) else: build_setup += textwrap.dedent("""\ source run_conda_forge_build_setup """) if platform == "linux": yum_build_setup = generate_yum_requirements(forge_dir) if yum_build_setup: forge_config["yum_build_setup"] = yum_build_setup forge_config["build_setup"] = build_setup if platform == "linux": run_file_name = "run_docker_build" else: run_file_name = "run_osx_build" # TODO: Conda has a convenience for accessing nested yaml content. template_files = [ "{}.sh.tmpl".format(run_file_name), "fast_finish_ci_pr_build.sh.tmpl", ] if platform == "linux": template_files.append("build_steps.sh.tmpl") _render_template_exe_files( forge_config=forge_config, target_dir=os.path.join(forge_dir, ".circleci"), jinja_env=jinja_env, template_files=template_files, ) # Fix permission of other shell files. target_fnames = [ os.path.join(forge_dir, ".circleci", "checkout_merge_commit.sh") ] for target_fname in target_fnames: set_exe_file(target_fname, True)
def render_run_docker_build(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() matrix = compute_build_matrix(meta, forge_config.get('matrix')) cases_not_skipped = [] for case in matrix: pkgs, vars = split_case(case) with enable_vars(vars): if not ResolvedDistribution(meta, pkgs).skip(): cases_not_skipped.append(vars + sorted(pkgs)) matrix = sorted(cases_not_skipped, key=sort_without_target_arch) if not matrix: # There are no cases to build (not even a case without any special # dependencies), so remove the run_docker_build.sh if it exists. forge_config["circle"]["enabled"] = False target_fnames = [ os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'), os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'), ] for each_target_fname in target_fnames: remove_file(each_target_fname) else: forge_config["circle"]["enabled"] = True matrix = prepare_matrix_for_env_vars(matrix) forge_config = update_matrix(forge_config, matrix) build_setup = "" # If the recipe supplies its own conda-forge-build-setup script, # we use it instead of the global one. cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_linux') if os.path.exists(cfbs_fpath): build_setup += textwrap.dedent("""\ # Overriding global conda-forge-build-setup with local copy. source /recipe_root/run_conda_forge_build_setup_linux """) else: build_setup += textwrap.dedent("""\ source run_conda_forge_build_setup """) # If there is a "yum_requirements.txt" file in the recipe, we honour it. yum_requirements_fpath = os.path.join(forge_dir, 'recipe', 'yum_requirements.txt') if os.path.exists(yum_requirements_fpath): with open(yum_requirements_fpath) as fh: requirements = [line.strip() for line in fh if line.strip() and not line.strip().startswith('#')] if not requirements: raise ValueError("No yum requirements enabled in the " "yum_requirements.txt, please remove the file " "or add some.") build_setup += textwrap.dedent("""\ # Install the yum requirements defined canonically in the # "recipe/yum_requirements.txt" file. After updating that file, # run "conda smithy rerender" and this line be updated # automatically. yum install -y {} """.format(' '.join(requirements))) forge_config['build_setup'] = build_setup # If the recipe supplies its own conda-forge-build-setup upload script, # we use it instead of the global one. upload_fpath = os.path.join(forge_dir, 'recipe', 'upload_or_check_non_existence.py') if os.path.exists(upload_fpath): forge_config['upload_script'] = ( "/recipe_root/upload_or_check_non_existence.py" ) else: forge_config['upload_script'] = "upload_or_check_non_existence" # TODO: Conda has a convenience for accessing nested yaml content. templates = forge_config.get('templates', {}) template_name = templates.get('run_docker_build', 'run_docker_build_matrix.tmpl') template = jinja_env.get_template(template_name) target_fname = os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh') with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) # Fix permissions. target_fnames = [ os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'), os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'), ] for each_target_fname in target_fnames: set_exe_file(each_target_fname, True)
def render_circle(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() matrix = compute_build_matrix( meta, forge_config.get('matrix'), forge_config.get('channels', {}).get('sources', tuple()) ) cases_not_skipped = [] for case in matrix: pkgs, vars = split_case(case) with enable_vars(vars): if not ResolvedDistribution(meta, pkgs).skip(): cases_not_skipped.append(vars + sorted(pkgs)) matrix = sorted(cases_not_skipped, key=sort_without_target_arch) if not matrix: # There are no cases to build (not even a case without any special # dependencies), so remove the run_docker_build.sh if it exists. forge_config["circle"]["enabled"] = False target_fnames = [ os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'), os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh'), os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'), ] for each_target_fname in target_fnames: remove_file(each_target_fname) else: forge_config["circle"]["enabled"] = True matrix = prepare_matrix_for_env_vars(matrix) forge_config = update_matrix(forge_config, matrix) fast_finish = textwrap.dedent("""\ {get_fast_finish_script} | \\ python - -v --ci "circle" "${{CIRCLE_PROJECT_USERNAME}}/${{CIRCLE_PROJECT_REPONAME}}" "${{CIRCLE_BUILD_NUM}}" "${{CIRCLE_PR_NUMBER}}" """) get_fast_finish_script = "" # If the recipe supplies its own conda-forge-build-setup script, # we use it instead of the global one. cfbs_fpath = os.path.join(forge_dir, 'recipe', 'ff_ci_pr_build.py') if os.path.exists(cfbs_fpath): get_fast_finish_script += "cat {recipe_dir}/ff_ci_pr_build.py".format(recipe_dir=forge_config["recipe_dir"]) else: get_fast_finish_script += "curl https://raw.githubusercontent.com/conda-forge/conda-forge-build-setup-feedstock/master/recipe/ff_ci_pr_build.py" fast_finish = fast_finish.format( get_fast_finish_script=get_fast_finish_script ) fast_finish = fast_finish.strip() forge_config['fast_finish'] = fast_finish build_setup = "" # If the recipe supplies its own conda-forge-build-setup script, # we use it instead of the global one. cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_linux') if os.path.exists(cfbs_fpath): build_setup += textwrap.dedent("""\ # Overriding global conda-forge-build-setup with local copy. source /recipe_root/run_conda_forge_build_setup_linux """) else: build_setup += textwrap.dedent("""\ source run_conda_forge_build_setup """) # If there is a "yum_requirements.txt" file in the recipe, we honour it. yum_requirements_fpath = os.path.join(forge_dir, 'recipe', 'yum_requirements.txt') if os.path.exists(yum_requirements_fpath): with open(yum_requirements_fpath) as fh: requirements = [line.strip() for line in fh if line.strip() and not line.strip().startswith('#')] if not requirements: raise ValueError("No yum requirements enabled in the " "yum_requirements.txt, please remove the file " "or add some.") build_setup += textwrap.dedent("""\ # Install the yum requirements defined canonically in the # "recipe/yum_requirements.txt" file. After updating that file, # run "conda smithy rerender" and this line be updated # automatically. /usr/bin/sudo -n yum install -y {} """.format(' '.join(requirements))) forge_config['build_setup'] = build_setup # If the recipe supplies its own conda-forge-build-setup upload script, # we use it instead of the global one. upload_fpath = os.path.join(forge_dir, 'recipe', 'upload_or_check_non_existence.py') if os.path.exists(upload_fpath): forge_config['upload_script'] = ( "/recipe_root/upload_or_check_non_existence.py" ) else: forge_config['upload_script'] = "upload_or_check_non_existence" # TODO: Conda has a convenience for accessing nested yaml content. templates = forge_config.get('templates', {}) template_name = templates.get('run_docker_build', 'run_docker_build_matrix.tmpl') template = jinja_env.get_template(template_name) target_fname = os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh') with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) template_name = 'fast_finish_ci_pr_build.sh.tmpl' template = jinja_env.get_template(template_name) target_fname = os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh') with write_file(target_fname) as fh: fh.write(template.render(**forge_config)) # Fix permissions. target_fnames = [ os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'), os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh'), os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'), ] for each_target_fname in target_fnames: set_exe_file(each_target_fname, True) target_fname = os.path.join(forge_dir, 'circle.yml') template = jinja_env.get_template('circle.yml.tmpl') with write_file(target_fname) as fh: fh.write(template.render(**forge_config))
def main(forge_file_directory, no_check_uptodate=False, commit=False, exclusive_config_file=None, check=False): if check: index = conda_build.conda_interface.get_index( channel_urls=["conda-forge"]) r = conda_build.conda_interface.Resolve(index) # Check that conda-smithy is up-to-date check_version_uptodate(r, "conda-smithy", __version__, True) get_cfp_file_path(r, True) return True error_on_warn = False if no_check_uptodate else True index = conda_build.conda_interface.get_index(channel_urls=["conda-forge"]) r = conda_build.conda_interface.Resolve(index) # Check that conda-smithy is up-to-date check_version_uptodate(r, "conda-smithy", __version__, error_on_warn) forge_dir = os.path.abspath(forge_file_directory) if exclusive_config_file is not None: exclusive_config_file = os.path.join(forge_dir, exclusive_config_file) if not os.path.exists(exclusive_config_file): raise RuntimeError("Given exclusive-config-file not found.") cf_pinning_ver = None else: exclusive_config_file, cf_pinning_ver = get_cfp_file_path( r, error_on_warn) config = _load_forge_config(forge_dir, exclusive_config_file) for each_ci in ["travis", "circle", "appveyor"]: if config[each_ci].pop("enabled", None): warnings.warn( "It is not allowed to set the `enabled` parameter for `%s`." " All CIs are enabled by default. To disable a CI, please" " add `skip: true` to the `build` section of `meta.yaml`" " and an appropriate selector so as to disable the build." % each_ci) tmplt_dir = os.path.join(conda_forge_content, "templates") # Load templates from the feedstock in preference to the smithy's templates. env = Environment( extensions=["jinja2.ext.do"], loader=FileSystemLoader( [os.path.join(forge_dir, "templates"), tmplt_dir]), ) copy_feedstock_content(forge_dir) set_exe_file(os.path.join(forge_dir, "build-locally.py")) clear_variants(forge_dir) render_circle(env, config, forge_dir) render_travis(env, config, forge_dir) render_appveyor(env, config, forge_dir) render_azure(env, config, forge_dir) render_README(env, config, forge_dir) if os.path.isdir(os.path.join(forge_dir, ".ci_support")): with write_file(os.path.join(forge_dir, ".ci_support", "README")) as f: f.write( "This file is automatically generated by conda-smithy. To change " "any matrix elements, you should change conda-smithy's input " "conda_build_config.yaml and re-render the recipe, rather than editing " "these files directly.") commit_changes( forge_file_directory, commit, __version__, cf_pinning_ver, conda_build_version, )