def test_set_exe_file(self):
perms = [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH]
set_mode = functools.reduce(op.or_, perms)
for set_exe in [True, False]:
for tmp_dir, repo, pathfunc in parameterize():
filename = "test.txt"
filename = os.path.join(tmp_dir, filename)
with io.open(
filename, "w", encoding="utf-8", newline="\n"
) as fh:
fh.write("")
if repo is not None:
repo.index.add([filename])
fio.set_exe_file(pathfunc(filename), set_exe)
file_mode = os.stat(filename).st_mode
self.assertEqual(file_mode & set_mode, int(set_exe) * set_mode)
if repo is not None:
blob = next(repo.index.iter_blobs(BlobFilter(filename)))[1]
self.assertEqual(
blob.mode & set_mode, int(set_exe) * set_mode
)
def test_set_exe_file(self):
perms = [
stat.S_IXUSR,
stat.S_IXGRP,
stat.S_IXOTH
]
set_mode = functools.reduce(op.or_, perms)
for set_exe in [True, False]:
for tmp_dir, repo, pathfunc in parameterize():
filename = "test.txt"
filename = os.path.join(tmp_dir, filename)
with io.open(filename, "w", encoding="utf-8", newline="\n") as fh:
fh.write("")
if repo is not None:
repo.index.add([filename])
fio.set_exe_file(pathfunc(filename), set_exe)
file_mode = os.stat(filename).st_mode
self.assertEqual(file_mode & set_mode,
int(set_exe) * set_mode)
if repo is not None:
blob = next(repo.index.iter_blobs(BlobFilter(filename)))[1]
self.assertEqual(blob.mode & set_mode,
int(set_exe) * set_mode)
def _render_template_exe_files(forge_config, target_dir, jinja_env, template_files):
for template_file in template_files:
template = jinja_env.get_template(template_file)
target_fname = os.path.join(target_dir, template_file[:-len('.tmpl')])
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
# Fix permission of template shell files
set_exe_file(target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform):
if platform == "linux":
yum_build_setup = generate_yum_requirements(forge_dir)
if yum_build_setup:
forge_config["yum_build_setup"] = yum_build_setup
forge_config["build_setup"] = _get_build_setup_line(
forge_dir, platform, forge_config)
if platform == "linux":
run_file_name = "run_docker_build"
else:
run_file_name = "run_osx_build"
# TODO: Conda has a convenience for accessing nested yaml content.
template_files = [
"{}.sh.tmpl".format(run_file_name),
"fast_finish_ci_pr_build.sh.tmpl",
]
if platform == "linux":
template_files.append("build_steps.sh.tmpl")
_render_template_exe_files(
forge_config=forge_config,
target_dir=os.path.join(forge_dir, ".circleci"),
jinja_env=jinja_env,
template_files=template_files,
)
# Fix permission of other shell files.
target_fnames = [
os.path.join(forge_dir, ".circleci", "checkout_merge_commit.sh")
]
for target_fname in target_fnames:
set_exe_file(target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform):
# If the recipe supplies its own run_conda_forge_build_setup script_linux,
# we use it instead of the global one.
if platform == 'linux':
cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_linux')
else:
cfbs_fpath = os.path.join(forge_dir, 'recipe', 'run_conda_forge_build_setup_osx')
build_setup = ""
if os.path.exists(cfbs_fpath):
if platform == 'linux':
build_setup += textwrap.dedent("""\
# Overriding global run_conda_forge_build_setup_linux with local copy.
source /home/conda/recipe_root/run_conda_forge_build_setup_linux
""")
else:
build_setup += textwrap.dedent("""\
# Overriding global run_conda_forge_build_setup_osx with local copy.
source {recipe_dir}/run_conda_forge_build_setup_osx
""".format(recipe_dir=forge_config["recipe_dir"]))
else:
build_setup += textwrap.dedent("""\
source run_conda_forge_build_setup
""")
if platform == 'linux':
# If there is a "yum_requirements.txt" file in the recipe, we honour it.
yum_requirements_fpath = os.path.join(forge_dir, 'recipe',
'yum_requirements.txt')
if os.path.exists(yum_requirements_fpath):
with open(yum_requirements_fpath) as fh:
requirements = [line.strip() for line in fh
if line.strip() and not line.strip().startswith('#')]
if not requirements:
raise ValueError("No yum requirements enabled in the "
"yum_requirements.txt, please remove the file "
"or add some.")
build_setup += textwrap.dedent("""\
# Install the yum requirements defined canonically in the
# "recipe/yum_requirements.txt" file. After updating that file,
# run "conda smithy rerender" and this line be updated
# automatically.
/usr/bin/sudo -n yum install -y {}
""".format(' '.join(requirements)))
forge_config['build_setup'] = build_setup
if platform == 'linux':
run_file_name = 'run_docker_build'
else:
run_file_name = 'run_osx_build'
# TODO: Conda has a convenience for accessing nested yaml content.
template = jinja_env.get_template('{}.tmpl'.format(run_file_name))
target_fname = os.path.join(forge_dir, '.circleci', '{}.sh'.format(run_file_name))
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
template_name = 'fast_finish_ci_pr_build.sh.tmpl'
template = jinja_env.get_template(template_name)
target_fname = os.path.join(forge_dir, '.circleci', 'fast_finish_ci_pr_build.sh')
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
# Fix permissions.
target_fnames = [
os.path.join(forge_dir, '.circleci', 'checkout_merge_commit.sh'),
os.path.join(forge_dir, '.circleci', 'fast_finish_ci_pr_build.sh'),
os.path.join(forge_dir, '.circleci', '{}.sh'.format(run_file_name)),
]
for each_target_fname in target_fnames:
set_exe_file(each_target_fname, True)
def _circle_specific_setup(jinja_env, forge_config, forge_dir, platform):
# If the recipe supplies its own run_conda_forge_build_setup script_linux,
# we use it instead of the global one.
if platform == "linux":
cfbs_fpath = os.path.join(forge_dir, "recipe",
"run_conda_forge_build_setup_linux")
else:
cfbs_fpath = os.path.join(forge_dir, "recipe",
"run_conda_forge_build_setup_osx")
build_setup = ""
if os.path.exists(cfbs_fpath):
if platform == "linux":
build_setup += textwrap.dedent("""\
# Overriding global run_conda_forge_build_setup_linux with local copy.
source /home/conda/recipe_root/run_conda_forge_build_setup_linux
""")
else:
build_setup += textwrap.dedent("""\
# Overriding global run_conda_forge_build_setup_osx with local copy.
source {recipe_dir}/run_conda_forge_build_setup_osx
""".format(recipe_dir=forge_config["recipe_dir"]))
else:
build_setup += textwrap.dedent("""\
source run_conda_forge_build_setup
""")
if platform == "linux":
yum_build_setup = generate_yum_requirements(forge_dir)
if yum_build_setup:
forge_config["yum_build_setup"] = yum_build_setup
forge_config["build_setup"] = build_setup
if platform == "linux":
run_file_name = "run_docker_build"
else:
run_file_name = "run_osx_build"
# TODO: Conda has a convenience for accessing nested yaml content.
template_files = [
"{}.sh.tmpl".format(run_file_name),
"fast_finish_ci_pr_build.sh.tmpl",
]
if platform == "linux":
template_files.append("build_steps.sh.tmpl")
_render_template_exe_files(
forge_config=forge_config,
target_dir=os.path.join(forge_dir, ".circleci"),
jinja_env=jinja_env,
template_files=template_files,
)
# Fix permission of other shell files.
target_fnames = [
os.path.join(forge_dir, ".circleci", "checkout_merge_commit.sh")
]
for target_fname in target_fnames:
set_exe_file(target_fname, True)
def render_run_docker_build(jinja_env, forge_config, forge_dir):
meta = forge_config['package']
with fudge_subdir('linux-64', build_config=meta_config(meta)):
meta.parse_again()
matrix = compute_build_matrix(meta, forge_config.get('matrix'))
cases_not_skipped = []
for case in matrix:
pkgs, vars = split_case(case)
with enable_vars(vars):
if not ResolvedDistribution(meta, pkgs).skip():
cases_not_skipped.append(vars + sorted(pkgs))
matrix = sorted(cases_not_skipped, key=sort_without_target_arch)
if not matrix:
# There are no cases to build (not even a case without any special
# dependencies), so remove the run_docker_build.sh if it exists.
forge_config["circle"]["enabled"] = False
target_fnames = [
os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'),
os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'),
]
for each_target_fname in target_fnames:
remove_file(each_target_fname)
else:
forge_config["circle"]["enabled"] = True
matrix = prepare_matrix_for_env_vars(matrix)
forge_config = update_matrix(forge_config, matrix)
build_setup = ""
# If the recipe supplies its own conda-forge-build-setup script,
# we use it instead of the global one.
cfbs_fpath = os.path.join(forge_dir, 'recipe',
'run_conda_forge_build_setup_linux')
if os.path.exists(cfbs_fpath):
build_setup += textwrap.dedent("""\
# Overriding global conda-forge-build-setup with local copy.
source /recipe_root/run_conda_forge_build_setup_linux
""")
else:
build_setup += textwrap.dedent("""\
source run_conda_forge_build_setup
""")
# If there is a "yum_requirements.txt" file in the recipe, we honour it.
yum_requirements_fpath = os.path.join(forge_dir, 'recipe',
'yum_requirements.txt')
if os.path.exists(yum_requirements_fpath):
with open(yum_requirements_fpath) as fh:
requirements = [line.strip() for line in fh
if line.strip() and not line.strip().startswith('#')]
if not requirements:
raise ValueError("No yum requirements enabled in the "
"yum_requirements.txt, please remove the file "
"or add some.")
build_setup += textwrap.dedent("""\
# Install the yum requirements defined canonically in the
# "recipe/yum_requirements.txt" file. After updating that file,
# run "conda smithy rerender" and this line be updated
# automatically.
yum install -y {}
""".format(' '.join(requirements)))
forge_config['build_setup'] = build_setup
# If the recipe supplies its own conda-forge-build-setup upload script,
# we use it instead of the global one.
upload_fpath = os.path.join(forge_dir, 'recipe',
'upload_or_check_non_existence.py')
if os.path.exists(upload_fpath):
forge_config['upload_script'] = (
"/recipe_root/upload_or_check_non_existence.py"
)
else:
forge_config['upload_script'] = "upload_or_check_non_existence"
# TODO: Conda has a convenience for accessing nested yaml content.
templates = forge_config.get('templates', {})
template_name = templates.get('run_docker_build',
'run_docker_build_matrix.tmpl')
template = jinja_env.get_template(template_name)
target_fname = os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh')
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
# Fix permissions.
target_fnames = [
os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'),
os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'),
]
for each_target_fname in target_fnames:
set_exe_file(each_target_fname, True)
def render_circle(jinja_env, forge_config, forge_dir):
meta = forge_config['package']
with fudge_subdir('linux-64', build_config=meta_config(meta)):
meta.parse_again()
matrix = compute_build_matrix(
meta,
forge_config.get('matrix'),
forge_config.get('channels', {}).get('sources', tuple())
)
cases_not_skipped = []
for case in matrix:
pkgs, vars = split_case(case)
with enable_vars(vars):
if not ResolvedDistribution(meta, pkgs).skip():
cases_not_skipped.append(vars + sorted(pkgs))
matrix = sorted(cases_not_skipped, key=sort_without_target_arch)
if not matrix:
# There are no cases to build (not even a case without any special
# dependencies), so remove the run_docker_build.sh if it exists.
forge_config["circle"]["enabled"] = False
target_fnames = [
os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'),
os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh'),
os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'),
]
for each_target_fname in target_fnames:
remove_file(each_target_fname)
else:
forge_config["circle"]["enabled"] = True
matrix = prepare_matrix_for_env_vars(matrix)
forge_config = update_matrix(forge_config, matrix)
fast_finish = textwrap.dedent("""\
{get_fast_finish_script} | \\
python - -v --ci "circle" "${{CIRCLE_PROJECT_USERNAME}}/${{CIRCLE_PROJECT_REPONAME}}" "${{CIRCLE_BUILD_NUM}}" "${{CIRCLE_PR_NUMBER}}"
""")
get_fast_finish_script = ""
# If the recipe supplies its own conda-forge-build-setup script,
# we use it instead of the global one.
cfbs_fpath = os.path.join(forge_dir, 'recipe',
'ff_ci_pr_build.py')
if os.path.exists(cfbs_fpath):
get_fast_finish_script += "cat {recipe_dir}/ff_ci_pr_build.py".format(recipe_dir=forge_config["recipe_dir"])
else:
get_fast_finish_script += "curl https://raw.githubusercontent.com/conda-forge/conda-forge-build-setup-feedstock/master/recipe/ff_ci_pr_build.py"
fast_finish = fast_finish.format(
get_fast_finish_script=get_fast_finish_script
)
fast_finish = fast_finish.strip()
forge_config['fast_finish'] = fast_finish
build_setup = ""
# If the recipe supplies its own conda-forge-build-setup script,
# we use it instead of the global one.
cfbs_fpath = os.path.join(forge_dir, 'recipe',
'run_conda_forge_build_setup_linux')
if os.path.exists(cfbs_fpath):
build_setup += textwrap.dedent("""\
# Overriding global conda-forge-build-setup with local copy.
source /recipe_root/run_conda_forge_build_setup_linux
""")
else:
build_setup += textwrap.dedent("""\
source run_conda_forge_build_setup
""")
# If there is a "yum_requirements.txt" file in the recipe, we honour it.
yum_requirements_fpath = os.path.join(forge_dir, 'recipe',
'yum_requirements.txt')
if os.path.exists(yum_requirements_fpath):
with open(yum_requirements_fpath) as fh:
requirements = [line.strip() for line in fh
if line.strip() and not line.strip().startswith('#')]
if not requirements:
raise ValueError("No yum requirements enabled in the "
"yum_requirements.txt, please remove the file "
"or add some.")
build_setup += textwrap.dedent("""\
# Install the yum requirements defined canonically in the
# "recipe/yum_requirements.txt" file. After updating that file,
# run "conda smithy rerender" and this line be updated
# automatically.
/usr/bin/sudo -n yum install -y {}
""".format(' '.join(requirements)))
forge_config['build_setup'] = build_setup
# If the recipe supplies its own conda-forge-build-setup upload script,
# we use it instead of the global one.
upload_fpath = os.path.join(forge_dir, 'recipe',
'upload_or_check_non_existence.py')
if os.path.exists(upload_fpath):
forge_config['upload_script'] = (
"/recipe_root/upload_or_check_non_existence.py"
)
else:
forge_config['upload_script'] = "upload_or_check_non_existence"
# TODO: Conda has a convenience for accessing nested yaml content.
templates = forge_config.get('templates', {})
template_name = templates.get('run_docker_build',
'run_docker_build_matrix.tmpl')
template = jinja_env.get_template(template_name)
target_fname = os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh')
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
template_name = 'fast_finish_ci_pr_build.sh.tmpl'
template = jinja_env.get_template(template_name)
target_fname = os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh')
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
# Fix permissions.
target_fnames = [
os.path.join(forge_dir, 'ci_support', 'checkout_merge_commit.sh'),
os.path.join(forge_dir, 'ci_support', 'fast_finish_ci_pr_build.sh'),
os.path.join(forge_dir, 'ci_support', 'run_docker_build.sh'),
]
for each_target_fname in target_fnames:
set_exe_file(each_target_fname, True)
target_fname = os.path.join(forge_dir, 'circle.yml')
template = jinja_env.get_template('circle.yml.tmpl')
with write_file(target_fname) as fh:
fh.write(template.render(**forge_config))
def main(forge_file_directory,
no_check_uptodate=False,
commit=False,
exclusive_config_file=None,
check=False):
if check:
index = conda_build.conda_interface.get_index(
channel_urls=["conda-forge"])
r = conda_build.conda_interface.Resolve(index)
# Check that conda-smithy is up-to-date
check_version_uptodate(r, "conda-smithy", __version__, True)
get_cfp_file_path(r, True)
return True
error_on_warn = False if no_check_uptodate else True
index = conda_build.conda_interface.get_index(channel_urls=["conda-forge"])
r = conda_build.conda_interface.Resolve(index)
# Check that conda-smithy is up-to-date
check_version_uptodate(r, "conda-smithy", __version__, error_on_warn)
forge_dir = os.path.abspath(forge_file_directory)
if exclusive_config_file is not None:
exclusive_config_file = os.path.join(forge_dir, exclusive_config_file)
if not os.path.exists(exclusive_config_file):
raise RuntimeError("Given exclusive-config-file not found.")
cf_pinning_ver = None
else:
exclusive_config_file, cf_pinning_ver = get_cfp_file_path(
r, error_on_warn)
config = _load_forge_config(forge_dir, exclusive_config_file)
for each_ci in ["travis", "circle", "appveyor"]:
if config[each_ci].pop("enabled", None):
warnings.warn(
"It is not allowed to set the `enabled` parameter for `%s`."
" All CIs are enabled by default. To disable a CI, please"
" add `skip: true` to the `build` section of `meta.yaml`"
" and an appropriate selector so as to disable the build." %
each_ci)
tmplt_dir = os.path.join(conda_forge_content, "templates")
# Load templates from the feedstock in preference to the smithy's templates.
env = Environment(
extensions=["jinja2.ext.do"],
loader=FileSystemLoader(
[os.path.join(forge_dir, "templates"), tmplt_dir]),
)
copy_feedstock_content(forge_dir)
set_exe_file(os.path.join(forge_dir, "build-locally.py"))
clear_variants(forge_dir)
render_circle(env, config, forge_dir)
render_travis(env, config, forge_dir)
render_appveyor(env, config, forge_dir)
render_azure(env, config, forge_dir)
render_README(env, config, forge_dir)
if os.path.isdir(os.path.join(forge_dir, ".ci_support")):
with write_file(os.path.join(forge_dir, ".ci_support", "README")) as f:
f.write(
"This file is automatically generated by conda-smithy. To change "
"any matrix elements, you should change conda-smithy's input "
"conda_build_config.yaml and re-render the recipe, rather than editing "
"these files directly.")
commit_changes(
forge_file_directory,
commit,
__version__,
cf_pinning_ver,
conda_build_version,
)
