def _compile_from(self, patterns: typing.Iterable[str]):
for line_num, rule in enumerate(patterns, 1):
orig_rule = rule
rule = rule.lstrip().rstrip("\r\n")
if not rule or rule.startswith("#"):
continue
invert = False
if rule.startswith("!"):
invert = True
rule = rule.lstrip("!")
rule = _unescape_rule(rule)
only_dirs = False
if rule.endswith("/"):
only_dirs = True
rule = rule.rstrip("/")
if not rule.startswith("/"):
# A rule that doesn't start with '/' means to match any
# subdirectory
rule = "**/" + rule
regex = _rule_to_regex(rule)
m = _Matcher(
line_num=line_num,
orig_rule=orig_rule,
invert=invert,
only_dirs=only_dirs,
regex=regex,
)
self._matchers.append(m)
emit.trace(
f'Translated .jujuignore {line_num:d} "{orig_rule}" => "{regex}"'
)
def _pack_charm(self, parsed_args) -> List[pathlib.Path]:
"""Pack a charm."""
emit.progress("Packing the charm.")
# adapt arguments to use the build infrastructure
build_args = Namespace(
**{
"debug": parsed_args.debug,
"destructive_mode": parsed_args.destructive_mode,
"from": self.config.project.dirpath,
"entrypoint": parsed_args.entrypoint,
"requirement": parsed_args.requirement,
"shell": parsed_args.shell,
"shell_after": parsed_args.shell_after,
"bases_indices": parsed_args.bases_index,
"force": parsed_args.force,
})
# mimic the "build" command
validator = build.Validator(self.config)
args = validator.process(build_args)
emit.trace(f"Working arguments: {args}")
builder = build.Builder(args, self.config)
charms = builder.run(parsed_args.bases_index,
destructive_mode=build_args.destructive_mode)
emit.message("Charms packed:")
for charm in charms:
emit.message(f" {charm}")
def has_legacy_credentials(cls) -> bool:
"""Return True if legacy credentials are stored."""
if cls.CONFIG_PATH.exists():
emit.trace(f"Found legacy credentials stored in {cls.CONFIG_PATH}")
return True
return False
def run(self, parsed_args):
"""Run the command."""
validator = Validator(self.config)
args = validator.process(parsed_args)
emit.trace(f"Working arguments: {args}")
builder = Builder(args, self.config)
builder.run(destructive_mode=args["destructive_mode"])
def show_linting_results(self, linting_results):
"""Manage the linters results, show some in different conditions, decide if continue."""
attribute_results = []
lint_results_by_outcome = {}
for result in linting_results:
if result.result == linters.IGNORED:
continue
if result.check_type == linters.CheckType.attribute:
attribute_results.append(result)
else:
lint_results_by_outcome.setdefault(result.result,
[]).append(result)
# show attribute results
for result in attribute_results:
emit.trace(
f"Check result: {result.name} [{result.check_type}] {result.result} "
f"({result.text}; see more at {result.url}).", )
# show warnings (if any), then errors (if any)
template = "- {0.name}: {0.text} ({0.url})"
if linters.WARNINGS in lint_results_by_outcome:
emit.message("Lint Warnings:", intermediate=True)
for result in lint_results_by_outcome[linters.WARNINGS]:
emit.message(template.format(result), intermediate=True)
if linters.ERRORS in lint_results_by_outcome:
emit.message("Lint Errors:", intermediate=True)
for result in lint_results_by_outcome[linters.ERRORS]:
emit.message(template.format(result), intermediate=True)
if self.force_packing:
emit.message("Packing anyway as requested.", intermediate=True)
else:
raise CommandError(
"Aborting due to lint errors (use --force to override).",
retcode=2)
def _pack_charm(self, parsed_args) -> List[pathlib.Path]:
"""Pack a charm."""
emit.progress("Packing the charm.")
# adapt arguments to use the build infrastructure
build_args = Namespace(
**{
"debug": parsed_args.debug,
"destructive_mode": parsed_args.destructive_mode,
"from": self.config.project.dirpath,
"entrypoint": parsed_args.entrypoint,
"requirement": parsed_args.requirement,
"shell": parsed_args.shell,
"shell_after": parsed_args.shell_after,
"bases_indices": parsed_args.bases_index,
"force": parsed_args.force,
})
# mimic the "build" command
validator = build.Validator(self.config)
args = validator.process(build_args)
emit.trace(f"Working arguments: {args}")
builder = build.Builder(args, self.config)
charms = builder.run(parsed_args.bases_index,
destructive_mode=build_args.destructive_mode)
# avoid showing results when run inside a container (the outer charmcraft
# is responsible of the final message to the user)
if not env.is_charmcraft_running_in_managed_mode():
emit.message("Charms packed:")
for charm in charms:
emit.message(f" {charm}")
def extract_metadata(self) -> List[ExtractedMetadata]:
"""Obtain metadata information."""
if self._adopt_info is None or self._adopt_info not in self._parse_info:
return []
dirs = ProjectDirs(work_dir=self._work_dir)
part = Part(self._adopt_info, {}, project_dirs=dirs)
locations = (
part.part_src_dir,
part.part_build_dir,
part.part_install_dir,
)
metadata_list: List[ExtractedMetadata] = []
for metadata_file in self._parse_info[self._adopt_info]:
emit.trace(f"extract metadata: parse info from {metadata_file}")
for location in locations:
if pathlib.Path(location, metadata_file.lstrip("/")).is_file():
metadata = extract_metadata(metadata_file,
workdir=str(location))
if metadata:
metadata_list.append(metadata)
break
emit.progress(
f"No metadata extracted from {metadata_file}",
permanent=True)
return metadata_list
def get_os_platform(filepath=pathlib.Path("/etc/os-release")):
"""Determine a system/release combo for an OS using /etc/os-release if available."""
system = platform.system()
release = platform.release()
machine = platform.machine()
if system == "Linux":
try:
with filepath.open("rt", encoding="utf-8") as fh:
lines = fh.readlines()
except FileNotFoundError:
emit.trace(
"Unable to locate 'os-release' file, using default values")
else:
os_release = {}
for line in lines:
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.rstrip().split("=", 1)
if value[0] == value[-1] and value[0] in ('"', "'"):
value = value[1:-1]
os_release[key] = value
system = os_release.get("ID", system)
release = os_release.get("VERSION_ID", release)
return OSPlatform(system=system, release=release, machine=machine)
def read_metadata_yaml(charm_dir: pathlib.Path) -> Any:
"""Parse project's metadata.yaml.
:returns: the YAML decoded metadata.yaml content
"""
metadata_path = charm_dir / CHARM_METADATA
emit.trace(f"Reading {str(metadata_path)!r}")
with metadata_path.open("rt", encoding="utf8") as fh:
return yaml.safe_load(fh)
def load_command(self, app_config):
"""Load a command."""
self.loaded_command = self.command_class(app_config)
# load and parse the command specific options/params
parser = CustomArgumentParser(prog=self.loaded_command.name)
self.loaded_command.fill_parser(parser)
self.parsed_command_args = parser.parse_args(self.command_args)
emit.trace(f"Command parsed sysargs: {self.parsed_command_args}")
return self.loaded_command
def run(self, parsed_args):
"""Run the command."""
# this command is deprecated now (note that the whole infrastructure behind
# is ok to use, but through PackCommand)
notify_deprecation("dn06")
validator = Validator(self.config)
args = validator.process(parsed_args)
emit.trace(f"Working arguments: {args}")
builder = Builder(args, self.config)
builder.run(destructive_mode=args["destructive_mode"])
def create_symlink(self, src_path, dest_path):
"""Create a symlink in dest_path pointing relatively like src_path.
It also verifies that the linked dir or file is inside the project.
"""
resolved_path = src_path.resolve()
if self.charmdir in resolved_path.parents:
relative_link = relativise(src_path, resolved_path)
dest_path.symlink_to(relative_link)
else:
rel_path = src_path.relative_to(self.charmdir)
emit.trace(f"Ignoring symlink because targets outside the project: {str(rel_path)!r}")
def build_charm(self, bases_config: BasesConfiguration) -> str:
"""Build the charm.
:param bases_config: Bases configuration to use for build.
:returns: File name of charm.
:raises CraftError: on lifecycle exception.
:raises RuntimeError: on unexpected lifecycle exception.
"""
if env.is_charmcraft_running_in_managed_mode():
work_dir = env.get_managed_environment_home_path()
else:
work_dir = self.buildpath
emit.progress(f"Building charm in {str(work_dir)!r}")
if self._special_charm_part:
# all current deprecated arguments set charm plugin parameters
self._handle_deprecated_cli_arguments()
# add charm files to the prime filter
self._set_prime_filter()
# set source if empty or not declared in charm part
if not self._special_charm_part.get("source"):
self._special_charm_part["source"] = str(self.charmdir)
# run the parts lifecycle
emit.trace(f"Parts definition: {self._parts}")
lifecycle = parts.PartsLifecycle(
self._parts,
work_dir=work_dir,
project_dir=self.charmdir,
project_name=self.metadata.name,
ignore_local_sources=["*.charm"],
)
lifecycle.run(Step.PRIME)
# run linters and show the results
linting_results = linters.analyze(self.config, lifecycle.prime_dir)
self.show_linting_results(linting_results)
create_manifest(
lifecycle.prime_dir,
self.config.project.started_at,
bases_config,
linting_results,
)
zipname = self.handle_package(lifecycle.prime_dir, bases_config)
emit.message(f"Created '{zipname}'.", intermediate=True)
return zipname
def load_yaml(fpath):
"""Return the content of a YAML file."""
if not fpath.is_file():
emit.trace(f"Couldn't find config file {str(fpath)!r}")
return
try:
with fpath.open("rb") as fh:
content = yaml.safe_load(fh)
except (yaml.error.YAMLError, OSError) as err:
emit.trace(f"Failed to read/parse config file {str(fpath)!r}: {err!r}")
return
return content
def _authenticate(self, auth_info):
"""Get the auth token."""
headers = {}
if self.auth_encoded_credentials is not None:
headers["Authorization"] = "Basic {}".format(self.auth_encoded_credentials)
emit.trace(f"Authenticating! {auth_info}")
url = "{realm}?service={service}&scope={scope}".format_map(auth_info)
response = requests.get(url, headers=headers)
result = assert_response_ok(response)
auth_token = result["token"]
return auth_token
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata:
"""Parse project's metadata.yaml.
:returns: a CharmMetadata object.
:raises: CraftError if metadata does not exist.
"""
try:
metadata = read_metadata_yaml(charm_dir)
except OSError as exc:
raise CraftError(f"Cannot read the metadata.yaml file: {exc!r}") from exc
emit.trace("Validating metadata format")
return CharmMetadata.unmarshal(metadata)
def env_has_legacy_credentials(cls) -> bool:
"""Return True if legacy credentials are exported in the environment."""
credentials = os.getenv(constants.ENVIRONMENT_STORE_CREDENTIALS)
if credentials is None:
return False
try:
get_auth(credentials)
emit.trace(
f"Found legacy credentials exported on {constants.ENVIRONMENT_STORE_CREDENTIALS}"
)
except errors.LegacyCredentialsParseError:
return False
return True
def run_legacy(err: Optional[Exception] = None):
"""Run legacy implementation."""
# Reset the libraries to their original log level
for lib_name in _LIB_NAMES:
logger = logging.getLogger(lib_name)
logger.setLevel(_ORIGINAL_LIB_NAME_LOG_LEVEL[lib_name])
snapcraft.BasePlugin = snapcraft_legacy.BasePlugin # type: ignore
snapcraft.ProjectOptions = snapcraft_legacy.ProjectOptions # type: ignore
# Legacy does not use craft-cli
if err is not None:
emit.trace(f"run legacy implementation: {err!s}")
emit.ended_ok()
legacy.legacy_run()
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata:
"""Parse project's metadata.yaml.
:returns: a CharmMetadata object.
:raises: CommandError if metadata does not exist.
"""
metadata_path = charm_dir / CHARM_METADATA
emit.trace(f"Parsing {str(metadata_path)!r}")
if not metadata_path.exists():
raise CommandError("Missing mandatory metadata.yaml.")
with metadata_path.open("rt", encoding="utf8") as fh:
metadata = yaml.safe_load(fh)
return CharmMetadata.unmarshal(metadata)
def get_build_plan(yaml_data: Dict[str, Any],
parsed_args: "argparse.Namespace") -> List[Tuple[str, str]]:
"""Get a list of all build_on->build_for architectures from the project file.
Additionally, check for the command line argument `--build-for <architecture>`
When defined, the build plan will only contain builds where `build-for`
matches `SNAPCRAFT_BUILD_FOR`.
Note: `--build-for` defaults to the environmental variable `SNAPCRAFT_BUILD_FOR`.
:param yaml_data: The project YAML data.
:param parsed_args: snapcraft's argument namespace
:return: List of tuples of every valid build-on->build-for combination.
"""
archs = ArchitectureProject.unmarshal(yaml_data).architectures
host_arch = get_host_architecture()
build_plan: List[Tuple[str, str]] = []
# `isinstance()` calls are for mypy type checking and should not change logic
for arch in [arch for arch in archs if isinstance(arch, Architecture)]:
for build_on in arch.build_on:
if build_on in host_arch and isinstance(arch.build_for, list):
build_plan.append((host_arch, arch.build_for[0]))
else:
emit.verbose(
f"Skipping build-on: {build_on} build-for: {arch.build_for}"
f" because build-on doesn't match host arch: {host_arch}")
# filter out builds not matching argument `--build_for` or env `SNAPCRAFT_BUILD_FOR`
build_for_arg = parsed_args.build_for
if build_for_arg is not None:
build_plan = [
build for build in build_plan if build[1] == build_for_arg
]
if len(build_plan) == 0:
emit.message("Could not make build plan:"
" build-on architectures in snapcraft.yaml"
f" does not match host architecture ({host_arch}).")
else:
log_output = "Created build plan:"
for build in build_plan:
log_output += f"\n build-on: {build[0]} build-for: {build[1]}"
emit.trace(log_output)
return build_plan
def plan(
self, *, bases_indices: Optional[List[int]],
destructive_mode: bool, managed_mode: bool
) -> List[Tuple[BasesConfiguration, Base, int, int]]:
"""Determine the build plan based on user inputs and host environment.
Provide a list of bases that are buildable and scoped according to user
configuration. Provide all relevant details including the applicable
bases configuration and the indices of the entries to build for.
:returns: List of Tuples (bases_config, build_on, bases_index, build_on_index).
"""
build_plan: List[Tuple[BasesConfiguration, Base, int, int]] = []
for bases_index, bases_config in enumerate(self.config.bases):
if bases_indices and bases_index not in bases_indices:
emit.trace(
f"Skipping 'bases[{bases_index:d}]' due to --base-index usage."
)
continue
for build_on_index, build_on in enumerate(bases_config.build_on):
if managed_mode or destructive_mode:
matches, reason = check_if_base_matches_host(build_on)
else:
matches, reason = self.provider.is_base_available(build_on)
if matches:
emit.trace(
f"Building for 'bases[{bases_index:d}]' "
f"as host matches 'build-on[{build_on_index:d}]'.", )
build_plan.append(
(bases_config, build_on, bases_index, build_on_index))
break
else:
emit.progress(
f"Skipping 'bases[{bases_index:d}].build-on[{build_on_index:d}]': "
f"{reason}.", )
else:
emit.message(
"No suitable 'build-on' environment found "
f"in 'bases[{bases_index:d}]' configuration.",
intermediate=True,
)
return build_plan
def clean_project_environments(
self,
*,
charm_name: str,
project_path: pathlib.Path,
) -> List[str]:
"""Clean up any environments created for project.
:param charm_name: Name of project.
:param project_path: Directory of charm project.
:returns: List of containers deleted.
"""
deleted: List[str] = []
# Nothing to do if provider is not installed.
if not self.is_provider_available():
return deleted
inode = str(project_path.stat().st_ino)
try:
names = self.lxc.list_names(project=self.lxd_project, remote=self.lxd_remote)
except lxd.LXDError as error:
raise CraftError(str(error)) from error
for name in names:
match_regex = f"^charmcraft-{charm_name}-{inode}-.+-.+-.+$"
if re.match(match_regex, name):
emit.trace(f"Deleting container {name!r}.")
try:
self.lxc.delete(
instance_name=name,
force=True,
project=self.lxd_project,
remote=self.lxd_remote,
)
except lxd.LXDError as error:
raise CraftError(str(error)) from error
deleted.append(name)
else:
emit.trace(f"Not deleting container {name!r}.")
return deleted
def _is_item_already_uploaded(self, url):
"""Verify if a generic item is uploaded."""
response = self._hit("HEAD", url)
if response.status_code == 200:
# item is there, done!
uploaded = True
elif response.status_code == 404:
# confirmed item is NOT there
uploaded = False
else:
# something else is going on, log what we have and return False so at least
# we can continue with the upload
emit.trace(
f"Bad response when checking for uploaded {url!r}: "
f"{response.status_code!r} (headers={response.headers})",
)
uploaded = False
return uploaded
def run(self, target_step: Step) -> None:
"""Run the parts lifecycle.
:param target_step: The final step to execute.
:raises CraftError: On error during lifecycle ops.
:raises RuntimeError: On unexpected error.
"""
previous_dir = os.getcwd()
try:
os.chdir(self._project_dir)
# invalidate build if packing a charm and entrypoint changed
if "charm" in self._all_parts:
charm_part = self._all_parts["charm"]
if charm_part.get("plugin") == "charm":
entrypoint = os.path.normpath(charm_part["charm-entrypoint"])
dis_entrypoint = os.path.normpath(_get_dispatch_entrypoint(self.prime_dir))
if entrypoint != dis_entrypoint:
self._lcm.clean(Step.BUILD, part_names=["charm"])
self._lcm.reload_state()
emit.trace(f"Executing parts lifecycle in {str(self._project_dir)!r}")
actions = self._lcm.plan(target_step)
emit.trace(f"Parts actions: {actions}")
with self._lcm.action_executor() as aex:
for action in actions:
emit.progress(f"Running step {action.step.name} for part {action.part_name!r}")
with emit.open_stream("Execute action") as stream:
aex.execute([action], stdout=stream, stderr=stream)
except RuntimeError as err:
raise RuntimeError(f"Parts processing internal error: {err}") from err
except OSError as err:
msg = err.strerror
if err.filename:
msg = f"{err.filename}: {msg}"
raise CraftError(f"Parts processing error: {msg}") from err
except Exception as err:
raise CraftError(f"Parts processing error: {err}") from err
finally:
os.chdir(previous_dir)
def set_legacy_env() -> None:
"""Set constants.ENVIRONMENT_STORE_CREDENTIALS to a valid value.
Transform the configparser based environment into a value useful
for craft-store.
"""
if LegacyUbuntuOne.env_has_legacy_credentials():
emit.trace(
f"Found legacy credentials exported on {constants.ENVIRONMENT_STORE_CREDENTIALS!r}"
)
auth = get_auth(config_content=os.getenv(
constants.ENVIRONMENT_STORE_CREDENTIALS) # type: ignore
)
os.environ[constants.ENVIRONMENT_STORE_CREDENTIALS] = auth
elif LegacyUbuntuOne.has_legacy_credentials():
emit.trace(
f"Found legacy credentials stored in {LegacyUbuntuOne.CONFIG_PATH!r}"
)
config_content = LegacyUbuntuOne.CONFIG_PATH.read_text()
auth = get_auth(config_content=config_content)
os.environ[constants.ENVIRONMENT_STORE_CREDENTIALS] = auth
def capture_logs_from_instance(instance: Executor) -> None:
"""Retrieve logs from instance.
:param instance: Instance to retrieve logs from.
:returns: String of logs.
"""
# Get a temporary file path.
tmp_file = tempfile.NamedTemporaryFile(delete=False, prefix="charmcraft-")
tmp_file.close()
local_log_path = pathlib.Path(tmp_file.name)
instance_log_path = get_managed_environment_log_path()
try:
instance.pull_file(source=instance_log_path,
destination=local_log_path)
except FileNotFoundError:
emit.trace("No logs found in instance.")
return
emit.trace("Logs captured from managed instance:")
with open(local_log_path, "rt", encoding="utf8") as fh:
for line in fh:
emit.trace(f":: {line.rstrip()}")
local_log_path.unlink()
def get_image_info(self, digest: str) -> Union[dict, None]:
"""Get the info for a specific image.
Returns None to flag that the requested digest was not found by any reason.
"""
url = self.dockerd_socket_baseurl + "/images/{}/json".format(digest)
try:
response = self.session.get(url)
except requests.exceptions.ConnectionError:
emit.trace(
"Cannot connect to /var/run/docker.sock , please ensure dockerd is running.",
)
return
if response.status_code == 200:
# image is there, we're fine
return response.json()
# 404 is the standard response to "not found", if not exactly that let's log
# for proper debugging
if response.status_code != 404:
emit.trace(f"Bad response when validation local image: {response.status_code}")
def validate_environment(self,
*,
part_dependencies: Optional[List[str]] = None):
"""Ensure the environment contains dependencies needed by the plugin.
:param part_dependencies: A list of the parts this part depends on.
:raises PluginEnvironmentValidationError: If the environment is invalid.
"""
try:
output = self._execute("charm version").strip()
_, tools_version = output.split("\n")
if not tools_version.startswith("charm-tools"):
raise PluginEnvironmentValidationError(
part_name=self._part_name,
reason=f"invalid charm tools version {tools_version}",
)
emit.trace(f"found {tools_version}")
except ValueError as err:
raise PluginEnvironmentValidationError(
part_name=self._part_name,
reason="invalid charm tools installed",
) from err
except subprocess.CalledProcessError as err:
if err.returncode != plugins.validator.COMMAND_NOT_FOUND:
raise PluginEnvironmentValidationError(
part_name=self._part_name,
reason=
f"charm tools failed with error code {err.returncode}",
) from err
if part_dependencies is None or "charm-tools" not in part_dependencies:
raise PluginEnvironmentValidationError(
part_name=self._part_name,
reason=(
f"charm tool not found and part {self._part_name!r} "
f"does not depend on a part named 'charm-tools'"),
) from err
def _process_run(cmd: List[str]) -> None:
"""Run an external command logging its output.
:raises CommandError: if execution crashes or ends with return code not zero.
"""
emit.progress(f"Running external command {cmd}")
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
except Exception as err:
raise CommandError(f"Subprocess execution crashed for command {cmd}") from err
for line in proc.stdout:
emit.trace(f" :: {line.rstrip()}")
retcode = proc.wait()
if retcode:
raise CommandError(f"Subprocess command {cmd} execution failed with retcode {retcode}")
def handle_dispatcher(self, linked_entrypoint):
"""Handle modern and classic dispatch mechanisms."""
# dispatch mechanism, create one if wasn't provided by the project
dispatch_path = self.buildpath / DISPATCH_FILENAME
if not dispatch_path.exists():
emit.progress("Creating the dispatch mechanism")
dispatch_content = DISPATCH_CONTENT.format(
entrypoint_relative_path=linked_entrypoint.relative_to(self.buildpath)
)
with dispatch_path.open("wt", encoding="utf8") as fh:
fh.write(dispatch_content)
make_executable(fh)
# bunch of symlinks, to support old juju: verify that any of the already included hooks
# in the directory is not linking directly to the entrypoint, and also check all the
# mandatory ones are present
dest_hookpath = self.buildpath / HOOKS_DIR
if not dest_hookpath.exists():
dest_hookpath.mkdir()
# get those built hooks that we need to replace because they are pointing to the
# entrypoint directly and we need to fix the environment in the middle
current_hooks_to_replace = []
for node in dest_hookpath.iterdir():
if node.resolve() == linked_entrypoint:
current_hooks_to_replace.append(node)
node.unlink()
emit.trace(
f"Replacing existing hook {node.name!r} as it's a symlink to the entrypoint"
)
# include the mandatory ones and those we need to replace
hooknames = MANDATORY_HOOK_NAMES | {x.name for x in current_hooks_to_replace}
for hookname in hooknames:
emit.trace(f"Creating the {hookname!r} hook script pointing to dispatch")
dest_hook = dest_hookpath / hookname
if not dest_hook.exists():
relative_link = relativise(dest_hook, dispatch_path)
dest_hook.symlink_to(relative_link)
