def test_filter_files():
# Files that should be filtered out.
files_list = ['.git/a', 'something/.git/a', '.git\\a', 'something\\.git\\a']
assert not utils.filter_files(files_list, '')
# Files that should *not* be filtered out.
# Example of valid 'x.git' directory:
# lib/python3.4/site-packages/craftr/stl/craftr.utils.git/Craftrfile
files_list = ['a', 'x.git/a', 'something/x.git/a',
'x.git\\a', 'something\\x.git\\a']
assert len(utils.filter_files(files_list, '')) == len(files_list)
def test_filter_files():
# Files that should be filtered out.
files_list = ['.git/a', 'something/.git/a', '.git\\a', 'something\\.git\\a']
assert not utils.filter_files(files_list, '')
# Files that should *not* be filtered out.
# Example of valid 'x.git' directory:
# lib/python3.4/site-packages/craftr/stl/craftr.utils.git/Craftrfile
files_list = ['a', 'x.git/a', 'something/x.git/a',
'x.git\\a', 'something\\x.git\\a']
assert len(utils.filter_files(files_list, '')) == len(files_list)
def test_filter_files():
# Files that should be filtered out.
files_list = ['.git/a', 'something/.git/a', '.git\\a', 'something\\.git\\a',
'file.la', 'something/file.la', 'python.exe.conda_trash',
'bla.dll.conda_trash_1', 'bla.dll.conda_trash.conda_trash']
assert not utils.filter_files(files_list, '')
# Files that should *not* be filtered out.
# Example of valid 'x.git' directory:
# lib/python3.4/site-packages/craftr/stl/craftr.utils.git/Craftrfile
files_list = ['a', 'x.git/a', 'something/x.git/a',
'x.git\\a', 'something\\x.git\\a', 'something/.gitmodules',
'some/template/directory/.gitignore', 'another.lab',
'miniconda_trashcan.py', 'conda_trash_avoider.py']
assert len(utils.filter_files(files_list, '')) == len(files_list)
def test_filter_files():
# Files that should be filtered out.
files_list = ['.git/a', 'something/.git/a', '.git\\a', 'something\\.git\\a',
'file.la', 'something/file.la', 'python.exe.conda_trash',
'bla.dll.conda_trash_1', 'bla.dll.conda_trash.conda_trash']
assert not utils.filter_files(files_list, '')
# Files that should *not* be filtered out.
# Example of valid 'x.git' directory:
# lib/python3.4/site-packages/craftr/stl/craftr.utils.git/Craftrfile
files_list = ['a', 'x.git/a', 'something/x.git/a',
'x.git\\a', 'something\\x.git\\a', 'something/.gitmodules',
'some/template/directory/.gitignore', 'another.lab',
'miniconda_trashcan.py', 'conda_trash_avoider.py']
assert len(utils.filter_files(files_list, '')) == len(files_list)
def _get_hash_contents(self):
sections = ['source', 'requirements', 'build']
# make a copy of values, so that no sorting occurs in place
composite = HashableDict({section: copy.copy(self.get_section(section))
for section in sections})
outputs = self.get_section('outputs')
if outputs:
outs = []
for out in outputs:
out = copy.copy(out)
# files are dynamically determined, and there's no way to match them at render time.
# we need to exclude them from the hash.
if 'files' in out:
del out['files']
outs.append(out)
composite.update({'outputs': [HashableDict(out) for out in outs]})
# filter build requirements for ones that should not be in the hash
requirements = composite.get('requirements', {})
build_reqs = requirements.get('build', [])
excludes = self.config.variant.get('exclude_from_build_hash', [])
if excludes:
exclude_pattern = re.compile('|'.join('{}[\s$]?.*'.format(exc) for exc in excludes))
build_reqs = [req for req in build_reqs if not exclude_pattern.match(req)]
requirements['build'] = build_reqs
composite['requirements'] = requirements
# remove the build number from the hash, so that we can bump it without changing the hash
if 'number' in composite['build']:
del composite['build']['number']
# remove the build string, so that hashes don't affect themselves
if 'string' in composite['build']:
del composite['build']['string']
if not composite['build']:
del composite['build']
for key in 'build', 'run':
if key in composite['requirements'] and not composite['requirements'].get(key):
del composite['requirements'][key]
trim_empty_keys(composite)
file_paths = []
if self.path:
recorded_input_files = os.path.join(self.path, '..', 'hash_input_files')
if os.path.exists(recorded_input_files):
with open(recorded_input_files) as f:
file_paths = f.read().splitlines()
else:
files = utils.rec_glob(self.path, "*")
file_paths = sorted([f.replace(self.path + os.sep, '') for f in files])
# exclude meta.yaml and meta.yaml.template, because the json dictionary captures
# their content
file_paths = [f for f in file_paths if not f.startswith('meta.yaml')]
file_paths = sorted(filter_files(file_paths, self.path))
return composite, file_paths
def _copy_output_recipe(m, dest_dir):
_copy_top_level_recipe(m.path, m.config, dest_dir, "parent")
this_output = m.get_rendered_output(m.name()) or {}
install_script = this_output.get("script")
build_inputs = []
inputs = [install_script] + build_inputs
file_paths = [script for script in inputs if script]
file_paths = utils.filter_files(file_paths, m.path)
for f in file_paths:
utils.copy_into(
join(m.path, f),
join(dest_dir, f),
timeout=m.config.timeout,
locking=m.config.locking,
clobber=True,
)
def _copy_top_level_recipe(path, config, dest_dir, destination_subdir=None):
files = utils.rec_glob(path, "*")
file_paths = sorted([f.replace(path + os.sep, "") for f in files])
# when this actually has a value, we're copying the top-level recipe into a subdirectory,
# so that we have record of what parent recipe produced subpackages.
if destination_subdir:
dest_dir = join(dest_dir, destination_subdir)
else:
# exclude recipe.yaml because the json dictionary captures its content
file_paths = [
f for f in file_paths
if not (f == "recipe.yaml" or f == "conda_build_config.yaml")
]
file_paths = utils.filter_files(file_paths, path)
for f in file_paths:
utils.copy_into(
join(path, f),
join(dest_dir, f),
timeout=config.timeout,
locking=config.locking,
clobber=True,
)
def bundle_conda(metadata, initial_files, env, files_selector=None):
files = post_process_files(metadata, initial_files)
# first filter is so that info_files does not pick up ignored files
files = utils.filter_files(files, prefix=metadata.config.host_prefix)
if files_selector:
files = select_files(files, files_selector.get("include"),
files_selector.get("exclude"))
console.print(f"\n[yellow]Adding files for {metadata.name()}[/yellow]\n")
if files:
for f in sorted(files):
console.print(f"- {f}")
else:
console.print(
f"[red]ATTENTION: No files added in target [bold]{metadata.name()}[/bold][/red]"
)
console.print("\n")
# this is also copying things like run_test.sh into info/recipe
utils.rm_rf(os.path.join(metadata.config.info_dir, "test"))
output = {}
with tmp_chdir(metadata.config.host_prefix):
output["checksums"] = create_info_files(
metadata, files, prefix=metadata.config.host_prefix)
# here we add the info files into the prefix, so we want to re-collect the files list
prefix_files = set(utils.prefix_files(metadata.config.host_prefix))
files = utils.filter_files(prefix_files - initial_files,
prefix=metadata.config.host_prefix)
if files_selector:
include_files = files_selector.get("include")
if include_files:
include_files += ["info/*"]
files = select_files(files, include_files,
files_selector.get("exclude"))
basename = metadata.dist()
tmp_archives = []
final_outputs = []
ext = ".tar.bz2"
if output.get(
"type") == "conda_v2" or metadata.config.conda_pkg_format == "2":
ext = ".conda"
with TemporaryDirectory() as tmp:
conda_package_handling.api.create(metadata.config.host_prefix,
files,
basename + ext,
out_folder=tmp)
tmp_archives = [os.path.join(tmp, basename + ext)]
# we're done building, perform some checks
for tmp_path in tmp_archives:
# if tmp_path.endswith('.tar.bz2'):
# tarcheck.check_all(tmp_path, metadata.config)
output_filename = os.path.basename(tmp_path)
# # we do the import here because we want to respect logger level context
# try:
# from conda_verify.verify import Verify
# except ImportError:
# Verify = None
# log.warn("Importing conda-verify failed. Please be sure to test your packages. "
# "conda install conda-verify to make this message go away.")
# if getattr(metadata.config, "verify", False) and Verify:
# verifier = Verify()
# checks_to_ignore = (utils.ensure_list(metadata.config.ignore_verify_codes) +
# metadata.ignore_verify_codes())
# try:
# verifier.verify_package(path_to_package=tmp_path, checks_to_ignore=checks_to_ignore,
# exit_on_error=metadata.config.exit_on_verify_error)
# except KeyError as e:
# log.warn("Package doesn't have necessary files. It might be too old to inspect."
# "Legacy noarch packages are known to fail. Full message was {}".format(e))
try:
crossed_subdir = metadata.config.target_subdir
except AttributeError:
crossed_subdir = metadata.config.host_subdir
subdir = ("noarch" if (metadata.noarch or metadata.noarch_python)
else crossed_subdir)
if metadata.config.output_folder:
output_folder = os.path.join(metadata.config.output_folder,
subdir)
else:
output_folder = os.path.join(
os.path.dirname(metadata.config.bldpkgs_dir), subdir)
final_output = os.path.join(output_folder, output_filename)
if os.path.isfile(final_output):
utils.rm_rf(final_output)
# disable locking here. It's just a temp folder getting locked.
# Having it proved a major bottleneck.
utils.copy_into(tmp_path,
final_output,
metadata.config.timeout,
locking=False)
final_outputs.append(final_output)
update_index(os.path.dirname(output_folder),
verbose=metadata.config.debug,
threads=1)
# clean out host prefix so that this output's files don't interfere with other outputs
# We have a backup of how things were before any output scripts ran. That's
# restored elsewhere.
if metadata.config.keep_old_work:
prefix = metadata.config.host_prefix
dest = os.path.join(
os.path.dirname(prefix),
"_".join(("_h_env_moved", metadata.dist(),
metadata.config.host_subdir)),
)
console.print("Renaming host env directory, ", prefix, " to ", dest)
if os.path.exists(dest):
utils.rm_rf(dest)
shutil.move(prefix, dest)
else:
utils.rm_rf(metadata.config.host_prefix)
return final_outputs
def post_process_files(m, initial_prefix_files):
get_build_metadata(m)
create_post_scripts(m)
# this is new-style noarch, with a value of 'python'
if m.noarch != "python":
utils.create_entry_points(m.get_value("build/entry_points"),
config=m.config)
current_prefix_files = utils.prefix_files(prefix=m.config.host_prefix)
python = (m.config.build_python if os.path.isfile(m.config.build_python)
else m.config.host_python)
post_process(
m.get_value("package/name"),
m.get_value("package/version"),
sorted(current_prefix_files - initial_prefix_files),
prefix=m.config.host_prefix,
config=m.config,
preserve_egg_dir=bool(m.get_value("build/preserve_egg_dir")),
noarch=m.get_value("build/noarch"),
skip_compile_pyc=m.get_value("build/skip_compile_pyc"),
)
# The post processing may have deleted some files (like easy-install.pth)
current_prefix_files = utils.prefix_files(prefix=m.config.host_prefix)
new_files = sorted(current_prefix_files - initial_prefix_files)
new_files = utils.filter_files(new_files, prefix=m.config.host_prefix)
host_prefix = m.config.host_prefix
meta_dir = m.config.meta_dir
if any(meta_dir in join(host_prefix, f) for f in new_files):
meta_files = (tuple(
f for f in new_files
if m.config.meta_dir in join(m.config.host_prefix, f)), )
sys.exit(
indent(
"""Error: Untracked file(s) %s found in conda-meta directory.
This error usually comes from using conda in the build script. Avoid doing this, as it
can lead to packages that include their dependencies.""" % meta_files))
post_build(m, new_files, build_python=python)
entry_point_script_names = get_entry_point_script_names(
m.get_value("build/entry_points"))
if m.noarch == "python":
pkg_files = [
fi for fi in new_files if fi not in entry_point_script_names
]
else:
pkg_files = new_files
# the legacy noarch
if m.get_value("build/noarch_python"):
noarch_python.transform(m, new_files, m.config.host_prefix)
# new way: build/noarch: python
elif m.noarch == "python":
noarch_python.populate_files(m, pkg_files, m.config.host_prefix,
entry_point_script_names)
current_prefix_files = utils.prefix_files(prefix=m.config.host_prefix)
new_files = current_prefix_files - initial_prefix_files
fix_permissions(new_files, m.config.host_prefix)
return new_files
