def fetch_stage_packages(
cls,
*,
package_names: List[str],
base: str,
stage_packages_path: pathlib.Path,
target_arch: str,
) -> List[str]:
logger.debug(f"Requested stage-packages: {sorted(package_names)!r}")
installed: Set[str] = set()
package_list = [DebPackage.from_unparsed(name) for name in package_names]
filtered_names = _get_filtered_stage_package_names(
base=base, package_list=package_list
)
stage_packages_path.mkdir(exist_ok=True)
with AptCache(
stage_cache=_STAGE_CACHE_DIR, stage_cache_arch=target_arch
) as apt_cache:
apt_cache.mark_packages(set(package_names))
apt_cache.unmark_packages(filtered_names)
for pkg_name, pkg_version, dl_path in apt_cache.fetch_archives(
_DEB_CACHE_DIR
):
logger.debug(f"Extracting stage package: {pkg_name}")
installed.add(f"{pkg_name}={pkg_version}")
file_utils.link_or_copy(
str(dl_path), str(stage_packages_path / dl_path.name)
)
return sorted(installed)
def setup_assets(self) -> None:
# We do _setup_from_setup first since it is legacy and let the
# declarative items take over.
self._setup_gui()
icon_path = self._finalize_icon()
if icon_path is not None:
if str(icon_path).startswith(self._prime_dir):
icon_path = icon_path.relative_to(self._prime_dir)
relative_icon_path: Optional[str] = str(icon_path)
else:
relative_icon_path = None
snap_name = self._project_config.project.info.name
for app_name, app in self._snap_meta.apps.items():
app.write_command_wrappers(prime_dir=self._prime_dir)
app.write_application_desktop_file(
snap_name=snap_name,
prime_dir=self._prime_dir,
gui_dir=self.meta_gui_dir,
icon_path=relative_icon_path,
)
app.validate_command_chain_executables(self._prime_dir)
if self._config_data.get("type", "") == "gadget":
if not os.path.exists("gadget.yaml"):
raise errors.MissingGadgetError()
file_utils.link_or_copy("gadget.yaml",
os.path.join(self.meta_dir, "gadget.yaml"))
if self._config_data.get("type", "") == "kernel":
if os.path.exists("kernel.yaml"):
file_utils.link_or_copy(
"kernel.yaml", os.path.join(self.meta_dir, "kernel.yaml"))
def _install_workspace_artifacts(self) -> None:
"""Install workspace artifacts."""
# Find artifacts in release directory.
release_dir = Path(self.builddir, "target", "release")
# Install binaries to bin/.
bins_dir = Path(self.installdir, "bin")
bins_dir.mkdir(parents=True, exist_ok=True)
# Install shared objects to usr/lib/<arch-triplet>.
# TODO: Dynamic library support needs to be properly added.
# Although we install libraries if we find them, they are most
# likely going to be missing dependencies, e.g.:
# /home/ubuntu/.cargo/toolchains/stable-x86_64-unknown-linux-gnu/lib/libstd-fae576517123aa4e.so
libs_dir = Path(self.installdir, "usr", "lib",
self.project.arch_triplet)
libs_dir.mkdir(parents=True, exist_ok=True)
# Cargo build marks binaries and shared objects executable...
# Search target directory to get these and install them to the
# correct location (*.so to lib directory, otherwise bin directory).
executables = [
i for i in release_dir.iterdir()
if os.path.isfile(i) and os.access(i, os.X_OK)
]
for exe_path in executables:
if exe_path.name.endswith(".so"):
file_utils.link_or_copy(exe_path.as_posix(),
libs_dir.as_posix())
else:
file_utils.link_or_copy(exe_path.as_posix(),
bins_dir.as_posix())
def build(self):
super().build()
package_dir = self._install(rootdir=self.builddir)
# Now move everything over to the plugin's installdir
link_or_copy_tree(package_dir, self.installdir)
# Copy in the node binary
link_or_copy(
os.path.join(self._npm_dir, "bin", "node"),
os.path.join(self.installdir, "bin", "node"),
)
# Create binary entries
package_json = self._get_package_json(rootdir=self.builddir)
_create_bins(package_json, self.installdir)
lock_file_path = os.path.join(self.installdir, "yarn.lock")
if os.path.isfile(lock_file_path):
with open(lock_file_path) as lock_file:
self._manifest["yarn-lock-contents"] = lock_file.read()
# Get the names and versions of installed packages
if self.options.nodejs_package_manager == "npm":
installed_node_packages = self._get_installed_node_packages(self.installdir)
self._manifest["node-packages"] = [
"{}={}".format(name, installed_node_packages[name])
for name in installed_node_packages
]
# Skip this step if yarn is used, as it may produce different
# dependency trees than npm
else:
self._manifest["node-packages"] = []
def test_link_file_ioerror(self):
orig_link = os.link
def link_and_ioerror(a, b, **kwargs):
orig_link(a, b)
raise IOError()
with mock.patch("os.link") as mock_link:
mock_link.side_effect = link_and_ioerror
file_utils.link_or_copy("1", "foo/1")
def _finalize_icon(self) -> Optional[pathlib.Path]:
"""Ensure sure icon is properly configured and installed.
Fetch from a remote URL, if required, and place in the meta/gui
directory.
"""
# Nothing to do if no icon is configured, search for existing icon.
icon: Optional[str] = self._config_data.get("icon")
if icon is None and self._extracted_metadata is not None:
icon = self._extracted_metadata.get_icon()
if icon is None:
return _find_icon_file()
# Extracted appstream icon paths will either:
# (1) point to a file relative to prime
# (2) point to a remote http(s) url
#
# The 'icon' specified in the snapcraft.yaml has the same
# constraint as (2) and would have already been validated
# as existing by the schema. So we can treat it the same
# at this point, regardless of the source of the icon.
parsed_url = urllib.parse.urlparse(icon)
parsed_path = pathlib.Path(parsed_url.path)
icon_ext = parsed_path.suffix[1:]
target_icon_path = pathlib.Path(self.meta_gui_dir, f"icon.{icon_ext}")
target_icon_path.parent.mkdir(parents=True, exist_ok=True)
if parsed_url.scheme in ["http", "https"]:
# Remote - fetch URL and write to target.
logger.info(f"Fetching icon from {icon!r}.")
icon_data = requests.get(icon).content
target_icon_path.write_bytes(icon_data)
elif parsed_url.scheme == "":
source_path = pathlib.Path(
self._prime_dir,
parsed_path.relative_to("/")
if parsed_path.is_absolute() else parsed_path,
)
if source_path.exists():
# Local with path relative to prime.
file_utils.link_or_copy(str(source_path),
str(target_icon_path))
elif parsed_path.exists():
# Local with path relative to project.
file_utils.link_or_copy(str(parsed_path),
str(target_icon_path))
else:
# No icon found, fall back to searching for existing icon.
return _find_icon_file()
else:
raise RuntimeError(f"Unexpected icon path: {parsed_url!r}")
return target_icon_path
def common_decorator(*, snap_filename: str) -> None:
snap_filename_common_path = (_get_review_tools_user_common() /
pathlib.Path(snap_filename).name)
# Create the directory tree for the cases where the review-tools have not run before.
snap_filename_common_path.parent.mkdir(parents=True, exist_ok=True)
file_utils.link_or_copy(snap_filename, str(snap_filename_common_path))
try:
method(snap_filename=snap_filename_common_path)
finally:
if snap_filename_common_path.exists():
snap_filename_common_path.unlink()
def build(self):
super().build()
if self._using_gradlew():
gradle_cmd = ["./gradlew"]
else:
self._gradle_tar.provision(self._gradle_dir, keep_zip=True)
gradle_cmd = ["gradle"]
self.run(
gradle_cmd + self._get_proxy_options() +
self.options.gradle_options + ["jar"],
rootdir=self.builddir,
)
src = os.path.join(self.builddir, self.options.gradle_output_dir)
jarfiles = glob(os.path.join(src, "*.jar"))
warfiles = glob(os.path.join(src, "*.war"))
if len(jarfiles) > 0:
basedir = "jar"
elif len(warfiles) > 0:
basedir = "war"
jarfiles = warfiles
else:
raise RuntimeError("Could not find any built jar files for part")
file_utils.link_or_copy_tree(
src,
os.path.join(self.installdir, basedir),
copy_function=lambda src, dst: file_utils.link_or_copy(
src, dst, self.installdir),
)
self._create_symlinks()
def build(self):
super().build()
self.run(
["shards", "build", "--without-development"] +
self.options.crystal_build_options,
self.builddir,
)
output_bin = os.path.join(self.builddir, "bin")
if not os.path.exists(output_bin):
raise errors.SnapcraftEnvironmentError(
"No binaries were built. Ensure the shards.yaml contains valid targets."
)
install_bin_path = os.path.join(self.installdir, "bin")
bin_paths = (os.path.join(output_bin, b)
for b in os.listdir(output_bin))
elf_files = (elf.ElfFile(path=b) for b in bin_paths
if elf.ElfFile.is_elf(b))
os.makedirs(install_bin_path, exist_ok=True)
for elf_file in elf_files:
shutil.copy2(
elf_file.path,
os.path.join(install_bin_path,
os.path.basename(elf_file.path)),
)
elf_dependencies_path = elf_file.load_dependencies(
root_path=self.installdir,
core_base_path=common.get_installed_snap_path(
self.project._get_build_base()),
arch_triplet=self.project.arch_triplet,
content_dirs=self.project._get_provider_content_dirs(),
)
for elf_dependency_path in elf_dependencies_path:
lib_install_path = os.path.join(self.installdir,
elf_dependency_path[1:])
os.makedirs(os.path.dirname(lib_install_path), exist_ok=True)
if not os.path.exists(lib_install_path):
file_utils.link_or_copy(elf_dependency_path,
lib_install_path,
follow_symlinks=True)
def stage_runtime_dependencies(
part_src: str,
part_install: str,
part_build: str,
arch_triplet: str,
content_dirs: str,
):
build_path = os.path.join(part_build, "bin")
install_path = os.path.join(part_install, "bin")
if not os.path.exists(build_path):
raise errors.SnapcraftEnvironmentError(
"No binaries were built. Ensure the shards.yaml contains valid targets."
)
bin_paths = (os.path.join(build_path, b) for b in os.listdir(build_path))
elf_files = (elf.ElfFile(path=b) for b in bin_paths if elf.ElfFile.is_elf(b))
os.makedirs(install_path, exist_ok=True)
# convert colon-delimited paths into a set
if content_dirs == "":
content_dirs_set = set()
else:
content_dirs_set = set(content_dirs.split(":"))
for elf_file in elf_files:
shutil.copy2(
elf_file.path, os.path.join(install_path, os.path.basename(elf_file.path)),
)
elf_dependencies_path = elf_file.load_dependencies(
root_path=part_install,
core_base_path=common.get_installed_snap_path("core20"),
arch_triplet=arch_triplet,
content_dirs=content_dirs_set,
)
for elf_dependency_path in elf_dependencies_path:
lib_install_path = os.path.join(part_install, elf_dependency_path[1:])
os.makedirs(os.path.dirname(lib_install_path), exist_ok=True)
if not os.path.exists(lib_install_path):
file_utils.link_or_copy(
elf_dependency_path, lib_install_path, follow_symlinks=True
)
def build(self):
super().build()
self._maven_tar.provision(self._maven_dir,
clean_target=False,
keep_tarball=True)
mvn_cmd = ["mvn", "package"]
if self._use_proxy():
settings_path = os.path.join(self.partdir, "m2", "settings.xml")
_create_settings(settings_path)
mvn_cmd += ["-s", settings_path]
self.run(mvn_cmd + self.options.maven_options, rootdir=self.builddir)
for f in self.options.maven_targets:
src = os.path.join(self.builddir, f, "target")
jarfiles = glob(os.path.join(src, "*.jar"))
warfiles = glob(os.path.join(src, "*.war"))
arfiles = glob(os.path.join(src, "*.[jw]ar"))
if len(arfiles) == 0:
raise EmptyBuildTargetDirectoryError(src)
if len(jarfiles) > 0 and len(f) == 0:
basedir = "jar"
elif len(warfiles) > 0 and len(f) == 0:
basedir = "war"
else:
basedir = f
targetdir = os.path.join(self.installdir, basedir)
os.makedirs(targetdir, exist_ok=True)
for src in arfiles:
# Make rebuilding easier when enabled
base = os.path.basename(src)
dst = os.path.join(targetdir, base)
if os.path.exists(dst):
os.unlink(dst)
file_utils.link_or_copy(src, dst)
self._create_symlinks()
def write_snap_directory(self) -> None:
"""Record manifest and copy assets found under $SNAPCRAFT_PROJECT_ROOT/snap.
These assets have priority over any code generated assets and include:
- hooks
- gui
"""
snap_assets_dir = self._project_config.project._get_snapcraft_assets_dir(
)
prime_snap_dir = os.path.join(self._prime_dir, "snap")
snap_dir_iter = itertools.product([prime_snap_dir], ["hooks"])
meta_dir_iter = itertools.product([self.meta_dir], ["hooks", "gui"])
for origin in itertools.chain(snap_dir_iter, meta_dir_iter):
src_dir = os.path.join(snap_assets_dir, origin[1])
dst_dir = os.path.join(origin[0], origin[1])
if os.path.isdir(src_dir):
os.makedirs(dst_dir, exist_ok=True)
for asset in os.listdir(src_dir):
source = os.path.join(src_dir, asset)
destination = os.path.join(dst_dir, asset)
with contextlib.suppress(FileNotFoundError):
os.remove(destination)
file_utils.link_or_copy(source,
destination,
follow_symlinks=True)
# Ensure that the hook is executable in meta/hooks, this is a moot
# point considering the prior link_or_copy call, but is technically
# correct and allows for this operation to take place only once.
if origin[0] == self.meta_dir and origin[1] == "hooks":
_prepare_hook(destination)
self._record_manifest_and_source_snapcraft_yaml()
def test_copy_nested_file(self):
file_utils.link_or_copy("foo/bar/baz/4", "foo2/bar/baz/4")
self.assertTrue(os.path.isfile("foo2/bar/baz/4"))
def wheel(
self,
packages,
*,
setup_py_dir: Optional[str] = None,
constraints: Optional[Set[str]] = None,
requirements: Optional[Sequence[str]] = None,
process_dependency_links: bool = False,
):
"""Build wheels of packages in the cache.
The packages should have already been downloaded via `download()`.
:param iterable packages: Packages in cache for which to build wheels.
:param str setup_py_dir: Directory containing setup.py.
:param iterable constraints: Collection of paths to constraints files.
:param iterable requirements: Collection of paths to requirements
files.
:param boolean process_dependency_links: Enable the processing of
dependency links.
:return: List of paths to each wheel that was built.
:rtype: list
"""
package_args = _process_package_args(packages=packages,
requirements=requirements,
setup_py_dir=setup_py_dir)
if not package_args:
return [] # No operation was requested
args = _process_common_args(
process_dependency_links=process_dependency_links,
constraints=constraints)
wheels = [] # type: List[str]
with tempfile.TemporaryDirectory() as temp_dir:
# Using pip with a few special parameters:
#
# --no-index: Don't hit pypi, assume the packages are already
# downloaded (i.e. by using `self.download()`)
# --find-links: Provide the directory into which the packages
# should have already been fetched
# --wheel-dir: Build wheels into a temporary working area rather
# rather than cwd. We'll copy them over. FIXME: We can
# probably get away just building them in the package
# dir. Try that once this refactor has been validated.
self._run(
[
"wheel",
"--no-index",
"--find-links",
self._python_package_dir,
"--wheel-dir",
temp_dir,
] + args + package_args,
cwd=setup_py_dir,
)
wheels = os.listdir(temp_dir)
for wheel in wheels:
file_utils.link_or_copy(
os.path.join(temp_dir, wheel),
os.path.join(self._python_package_dir, wheel),
)
return [
os.path.join(self._python_package_dir, wheel) for wheel in wheels
]