def _compile_from(self, patterns: typing.Iterable[str]): for line_num, rule in enumerate(patterns, 1): orig_rule = rule rule = rule.lstrip().rstrip("\r\n") if not rule or rule.startswith("#"): continue invert = False if rule.startswith("!"): invert = True rule = rule.lstrip("!") rule = _unescape_rule(rule) only_dirs = False if rule.endswith("/"): only_dirs = True rule = rule.rstrip("/") if not rule.startswith("/"): # A rule that doesn't start with '/' means to match any # subdirectory rule = "**/" + rule regex = _rule_to_regex(rule) m = _Matcher( line_num=line_num, orig_rule=orig_rule, invert=invert, only_dirs=only_dirs, regex=regex, ) self._matchers.append(m) emit.trace( f'Translated .jujuignore {line_num:d} "{orig_rule}" => "{regex}"' )
def _pack_charm(self, parsed_args) -> List[pathlib.Path]: """Pack a charm.""" emit.progress("Packing the charm.") # adapt arguments to use the build infrastructure build_args = Namespace( **{ "debug": parsed_args.debug, "destructive_mode": parsed_args.destructive_mode, "from": self.config.project.dirpath, "entrypoint": parsed_args.entrypoint, "requirement": parsed_args.requirement, "shell": parsed_args.shell, "shell_after": parsed_args.shell_after, "bases_indices": parsed_args.bases_index, "force": parsed_args.force, }) # mimic the "build" command validator = build.Validator(self.config) args = validator.process(build_args) emit.trace(f"Working arguments: {args}") builder = build.Builder(args, self.config) charms = builder.run(parsed_args.bases_index, destructive_mode=build_args.destructive_mode) emit.message("Charms packed:") for charm in charms: emit.message(f" {charm}")
def has_legacy_credentials(cls) -> bool: """Return True if legacy credentials are stored.""" if cls.CONFIG_PATH.exists(): emit.trace(f"Found legacy credentials stored in {cls.CONFIG_PATH}") return True return False
def run(self, parsed_args): """Run the command.""" validator = Validator(self.config) args = validator.process(parsed_args) emit.trace(f"Working arguments: {args}") builder = Builder(args, self.config) builder.run(destructive_mode=args["destructive_mode"])
def show_linting_results(self, linting_results): """Manage the linters results, show some in different conditions, decide if continue.""" attribute_results = [] lint_results_by_outcome = {} for result in linting_results: if result.result == linters.IGNORED: continue if result.check_type == linters.CheckType.attribute: attribute_results.append(result) else: lint_results_by_outcome.setdefault(result.result, []).append(result) # show attribute results for result in attribute_results: emit.trace( f"Check result: {result.name} [{result.check_type}] {result.result} " f"({result.text}; see more at {result.url}).", ) # show warnings (if any), then errors (if any) template = "- {0.name}: {0.text} ({0.url})" if linters.WARNINGS in lint_results_by_outcome: emit.message("Lint Warnings:", intermediate=True) for result in lint_results_by_outcome[linters.WARNINGS]: emit.message(template.format(result), intermediate=True) if linters.ERRORS in lint_results_by_outcome: emit.message("Lint Errors:", intermediate=True) for result in lint_results_by_outcome[linters.ERRORS]: emit.message(template.format(result), intermediate=True) if self.force_packing: emit.message("Packing anyway as requested.", intermediate=True) else: raise CommandError( "Aborting due to lint errors (use --force to override).", retcode=2)
def _pack_charm(self, parsed_args) -> List[pathlib.Path]: """Pack a charm.""" emit.progress("Packing the charm.") # adapt arguments to use the build infrastructure build_args = Namespace( **{ "debug": parsed_args.debug, "destructive_mode": parsed_args.destructive_mode, "from": self.config.project.dirpath, "entrypoint": parsed_args.entrypoint, "requirement": parsed_args.requirement, "shell": parsed_args.shell, "shell_after": parsed_args.shell_after, "bases_indices": parsed_args.bases_index, "force": parsed_args.force, }) # mimic the "build" command validator = build.Validator(self.config) args = validator.process(build_args) emit.trace(f"Working arguments: {args}") builder = build.Builder(args, self.config) charms = builder.run(parsed_args.bases_index, destructive_mode=build_args.destructive_mode) # avoid showing results when run inside a container (the outer charmcraft # is responsible of the final message to the user) if not env.is_charmcraft_running_in_managed_mode(): emit.message("Charms packed:") for charm in charms: emit.message(f" {charm}")
def extract_metadata(self) -> List[ExtractedMetadata]: """Obtain metadata information.""" if self._adopt_info is None or self._adopt_info not in self._parse_info: return [] dirs = ProjectDirs(work_dir=self._work_dir) part = Part(self._adopt_info, {}, project_dirs=dirs) locations = ( part.part_src_dir, part.part_build_dir, part.part_install_dir, ) metadata_list: List[ExtractedMetadata] = [] for metadata_file in self._parse_info[self._adopt_info]: emit.trace(f"extract metadata: parse info from {metadata_file}") for location in locations: if pathlib.Path(location, metadata_file.lstrip("/")).is_file(): metadata = extract_metadata(metadata_file, workdir=str(location)) if metadata: metadata_list.append(metadata) break emit.progress( f"No metadata extracted from {metadata_file}", permanent=True) return metadata_list
def get_os_platform(filepath=pathlib.Path("/etc/os-release")): """Determine a system/release combo for an OS using /etc/os-release if available.""" system = platform.system() release = platform.release() machine = platform.machine() if system == "Linux": try: with filepath.open("rt", encoding="utf-8") as fh: lines = fh.readlines() except FileNotFoundError: emit.trace( "Unable to locate 'os-release' file, using default values") else: os_release = {} for line in lines: line = line.strip() if not line or line.startswith("#") or "=" not in line: continue key, value = line.rstrip().split("=", 1) if value[0] == value[-1] and value[0] in ('"', "'"): value = value[1:-1] os_release[key] = value system = os_release.get("ID", system) release = os_release.get("VERSION_ID", release) return OSPlatform(system=system, release=release, machine=machine)
def read_metadata_yaml(charm_dir: pathlib.Path) -> Any: """Parse project's metadata.yaml. :returns: the YAML decoded metadata.yaml content """ metadata_path = charm_dir / CHARM_METADATA emit.trace(f"Reading {str(metadata_path)!r}") with metadata_path.open("rt", encoding="utf8") as fh: return yaml.safe_load(fh)
def load_command(self, app_config): """Load a command.""" self.loaded_command = self.command_class(app_config) # load and parse the command specific options/params parser = CustomArgumentParser(prog=self.loaded_command.name) self.loaded_command.fill_parser(parser) self.parsed_command_args = parser.parse_args(self.command_args) emit.trace(f"Command parsed sysargs: {self.parsed_command_args}") return self.loaded_command
def run(self, parsed_args): """Run the command.""" # this command is deprecated now (note that the whole infrastructure behind # is ok to use, but through PackCommand) notify_deprecation("dn06") validator = Validator(self.config) args = validator.process(parsed_args) emit.trace(f"Working arguments: {args}") builder = Builder(args, self.config) builder.run(destructive_mode=args["destructive_mode"])
def create_symlink(self, src_path, dest_path): """Create a symlink in dest_path pointing relatively like src_path. It also verifies that the linked dir or file is inside the project. """ resolved_path = src_path.resolve() if self.charmdir in resolved_path.parents: relative_link = relativise(src_path, resolved_path) dest_path.symlink_to(relative_link) else: rel_path = src_path.relative_to(self.charmdir) emit.trace(f"Ignoring symlink because targets outside the project: {str(rel_path)!r}")
def build_charm(self, bases_config: BasesConfiguration) -> str: """Build the charm. :param bases_config: Bases configuration to use for build. :returns: File name of charm. :raises CraftError: on lifecycle exception. :raises RuntimeError: on unexpected lifecycle exception. """ if env.is_charmcraft_running_in_managed_mode(): work_dir = env.get_managed_environment_home_path() else: work_dir = self.buildpath emit.progress(f"Building charm in {str(work_dir)!r}") if self._special_charm_part: # all current deprecated arguments set charm plugin parameters self._handle_deprecated_cli_arguments() # add charm files to the prime filter self._set_prime_filter() # set source if empty or not declared in charm part if not self._special_charm_part.get("source"): self._special_charm_part["source"] = str(self.charmdir) # run the parts lifecycle emit.trace(f"Parts definition: {self._parts}") lifecycle = parts.PartsLifecycle( self._parts, work_dir=work_dir, project_dir=self.charmdir, project_name=self.metadata.name, ignore_local_sources=["*.charm"], ) lifecycle.run(Step.PRIME) # run linters and show the results linting_results = linters.analyze(self.config, lifecycle.prime_dir) self.show_linting_results(linting_results) create_manifest( lifecycle.prime_dir, self.config.project.started_at, bases_config, linting_results, ) zipname = self.handle_package(lifecycle.prime_dir, bases_config) emit.message(f"Created '{zipname}'.", intermediate=True) return zipname
def load_yaml(fpath): """Return the content of a YAML file.""" if not fpath.is_file(): emit.trace(f"Couldn't find config file {str(fpath)!r}") return try: with fpath.open("rb") as fh: content = yaml.safe_load(fh) except (yaml.error.YAMLError, OSError) as err: emit.trace(f"Failed to read/parse config file {str(fpath)!r}: {err!r}") return return content
def _authenticate(self, auth_info): """Get the auth token.""" headers = {} if self.auth_encoded_credentials is not None: headers["Authorization"] = "Basic {}".format(self.auth_encoded_credentials) emit.trace(f"Authenticating! {auth_info}") url = "{realm}?service={service}&scope={scope}".format_map(auth_info) response = requests.get(url, headers=headers) result = assert_response_ok(response) auth_token = result["token"] return auth_token
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata: """Parse project's metadata.yaml. :returns: a CharmMetadata object. :raises: CraftError if metadata does not exist. """ try: metadata = read_metadata_yaml(charm_dir) except OSError as exc: raise CraftError(f"Cannot read the metadata.yaml file: {exc!r}") from exc emit.trace("Validating metadata format") return CharmMetadata.unmarshal(metadata)
def env_has_legacy_credentials(cls) -> bool: """Return True if legacy credentials are exported in the environment.""" credentials = os.getenv(constants.ENVIRONMENT_STORE_CREDENTIALS) if credentials is None: return False try: get_auth(credentials) emit.trace( f"Found legacy credentials exported on {constants.ENVIRONMENT_STORE_CREDENTIALS}" ) except errors.LegacyCredentialsParseError: return False return True
def run_legacy(err: Optional[Exception] = None): """Run legacy implementation.""" # Reset the libraries to their original log level for lib_name in _LIB_NAMES: logger = logging.getLogger(lib_name) logger.setLevel(_ORIGINAL_LIB_NAME_LOG_LEVEL[lib_name]) snapcraft.BasePlugin = snapcraft_legacy.BasePlugin # type: ignore snapcraft.ProjectOptions = snapcraft_legacy.ProjectOptions # type: ignore # Legacy does not use craft-cli if err is not None: emit.trace(f"run legacy implementation: {err!s}") emit.ended_ok() legacy.legacy_run()
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata: """Parse project's metadata.yaml. :returns: a CharmMetadata object. :raises: CommandError if metadata does not exist. """ metadata_path = charm_dir / CHARM_METADATA emit.trace(f"Parsing {str(metadata_path)!r}") if not metadata_path.exists(): raise CommandError("Missing mandatory metadata.yaml.") with metadata_path.open("rt", encoding="utf8") as fh: metadata = yaml.safe_load(fh) return CharmMetadata.unmarshal(metadata)
def get_build_plan(yaml_data: Dict[str, Any], parsed_args: "argparse.Namespace") -> List[Tuple[str, str]]: """Get a list of all build_on->build_for architectures from the project file. Additionally, check for the command line argument `--build-for <architecture>` When defined, the build plan will only contain builds where `build-for` matches `SNAPCRAFT_BUILD_FOR`. Note: `--build-for` defaults to the environmental variable `SNAPCRAFT_BUILD_FOR`. :param yaml_data: The project YAML data. :param parsed_args: snapcraft's argument namespace :return: List of tuples of every valid build-on->build-for combination. """ archs = ArchitectureProject.unmarshal(yaml_data).architectures host_arch = get_host_architecture() build_plan: List[Tuple[str, str]] = [] # `isinstance()` calls are for mypy type checking and should not change logic for arch in [arch for arch in archs if isinstance(arch, Architecture)]: for build_on in arch.build_on: if build_on in host_arch and isinstance(arch.build_for, list): build_plan.append((host_arch, arch.build_for[0])) else: emit.verbose( f"Skipping build-on: {build_on} build-for: {arch.build_for}" f" because build-on doesn't match host arch: {host_arch}") # filter out builds not matching argument `--build_for` or env `SNAPCRAFT_BUILD_FOR` build_for_arg = parsed_args.build_for if build_for_arg is not None: build_plan = [ build for build in build_plan if build[1] == build_for_arg ] if len(build_plan) == 0: emit.message("Could not make build plan:" " build-on architectures in snapcraft.yaml" f" does not match host architecture ({host_arch}).") else: log_output = "Created build plan:" for build in build_plan: log_output += f"\n build-on: {build[0]} build-for: {build[1]}" emit.trace(log_output) return build_plan
def plan( self, *, bases_indices: Optional[List[int]], destructive_mode: bool, managed_mode: bool ) -> List[Tuple[BasesConfiguration, Base, int, int]]: """Determine the build plan based on user inputs and host environment. Provide a list of bases that are buildable and scoped according to user configuration. Provide all relevant details including the applicable bases configuration and the indices of the entries to build for. :returns: List of Tuples (bases_config, build_on, bases_index, build_on_index). """ build_plan: List[Tuple[BasesConfiguration, Base, int, int]] = [] for bases_index, bases_config in enumerate(self.config.bases): if bases_indices and bases_index not in bases_indices: emit.trace( f"Skipping 'bases[{bases_index:d}]' due to --base-index usage." ) continue for build_on_index, build_on in enumerate(bases_config.build_on): if managed_mode or destructive_mode: matches, reason = check_if_base_matches_host(build_on) else: matches, reason = self.provider.is_base_available(build_on) if matches: emit.trace( f"Building for 'bases[{bases_index:d}]' " f"as host matches 'build-on[{build_on_index:d}]'.", ) build_plan.append( (bases_config, build_on, bases_index, build_on_index)) break else: emit.progress( f"Skipping 'bases[{bases_index:d}].build-on[{build_on_index:d}]': " f"{reason}.", ) else: emit.message( "No suitable 'build-on' environment found " f"in 'bases[{bases_index:d}]' configuration.", intermediate=True, ) return build_plan
def clean_project_environments( self, *, charm_name: str, project_path: pathlib.Path, ) -> List[str]: """Clean up any environments created for project. :param charm_name: Name of project. :param project_path: Directory of charm project. :returns: List of containers deleted. """ deleted: List[str] = [] # Nothing to do if provider is not installed. if not self.is_provider_available(): return deleted inode = str(project_path.stat().st_ino) try: names = self.lxc.list_names(project=self.lxd_project, remote=self.lxd_remote) except lxd.LXDError as error: raise CraftError(str(error)) from error for name in names: match_regex = f"^charmcraft-{charm_name}-{inode}-.+-.+-.+$" if re.match(match_regex, name): emit.trace(f"Deleting container {name!r}.") try: self.lxc.delete( instance_name=name, force=True, project=self.lxd_project, remote=self.lxd_remote, ) except lxd.LXDError as error: raise CraftError(str(error)) from error deleted.append(name) else: emit.trace(f"Not deleting container {name!r}.") return deleted
def _is_item_already_uploaded(self, url): """Verify if a generic item is uploaded.""" response = self._hit("HEAD", url) if response.status_code == 200: # item is there, done! uploaded = True elif response.status_code == 404: # confirmed item is NOT there uploaded = False else: # something else is going on, log what we have and return False so at least # we can continue with the upload emit.trace( f"Bad response when checking for uploaded {url!r}: " f"{response.status_code!r} (headers={response.headers})", ) uploaded = False return uploaded
def run(self, target_step: Step) -> None: """Run the parts lifecycle. :param target_step: The final step to execute. :raises CraftError: On error during lifecycle ops. :raises RuntimeError: On unexpected error. """ previous_dir = os.getcwd() try: os.chdir(self._project_dir) # invalidate build if packing a charm and entrypoint changed if "charm" in self._all_parts: charm_part = self._all_parts["charm"] if charm_part.get("plugin") == "charm": entrypoint = os.path.normpath(charm_part["charm-entrypoint"]) dis_entrypoint = os.path.normpath(_get_dispatch_entrypoint(self.prime_dir)) if entrypoint != dis_entrypoint: self._lcm.clean(Step.BUILD, part_names=["charm"]) self._lcm.reload_state() emit.trace(f"Executing parts lifecycle in {str(self._project_dir)!r}") actions = self._lcm.plan(target_step) emit.trace(f"Parts actions: {actions}") with self._lcm.action_executor() as aex: for action in actions: emit.progress(f"Running step {action.step.name} for part {action.part_name!r}") with emit.open_stream("Execute action") as stream: aex.execute([action], stdout=stream, stderr=stream) except RuntimeError as err: raise RuntimeError(f"Parts processing internal error: {err}") from err except OSError as err: msg = err.strerror if err.filename: msg = f"{err.filename}: {msg}" raise CraftError(f"Parts processing error: {msg}") from err except Exception as err: raise CraftError(f"Parts processing error: {err}") from err finally: os.chdir(previous_dir)
def set_legacy_env() -> None: """Set constants.ENVIRONMENT_STORE_CREDENTIALS to a valid value. Transform the configparser based environment into a value useful for craft-store. """ if LegacyUbuntuOne.env_has_legacy_credentials(): emit.trace( f"Found legacy credentials exported on {constants.ENVIRONMENT_STORE_CREDENTIALS!r}" ) auth = get_auth(config_content=os.getenv( constants.ENVIRONMENT_STORE_CREDENTIALS) # type: ignore ) os.environ[constants.ENVIRONMENT_STORE_CREDENTIALS] = auth elif LegacyUbuntuOne.has_legacy_credentials(): emit.trace( f"Found legacy credentials stored in {LegacyUbuntuOne.CONFIG_PATH!r}" ) config_content = LegacyUbuntuOne.CONFIG_PATH.read_text() auth = get_auth(config_content=config_content) os.environ[constants.ENVIRONMENT_STORE_CREDENTIALS] = auth
def capture_logs_from_instance(instance: Executor) -> None: """Retrieve logs from instance. :param instance: Instance to retrieve logs from. :returns: String of logs. """ # Get a temporary file path. tmp_file = tempfile.NamedTemporaryFile(delete=False, prefix="charmcraft-") tmp_file.close() local_log_path = pathlib.Path(tmp_file.name) instance_log_path = get_managed_environment_log_path() try: instance.pull_file(source=instance_log_path, destination=local_log_path) except FileNotFoundError: emit.trace("No logs found in instance.") return emit.trace("Logs captured from managed instance:") with open(local_log_path, "rt", encoding="utf8") as fh: for line in fh: emit.trace(f":: {line.rstrip()}") local_log_path.unlink()
def get_image_info(self, digest: str) -> Union[dict, None]: """Get the info for a specific image. Returns None to flag that the requested digest was not found by any reason. """ url = self.dockerd_socket_baseurl + "/images/{}/json".format(digest) try: response = self.session.get(url) except requests.exceptions.ConnectionError: emit.trace( "Cannot connect to /var/run/docker.sock , please ensure dockerd is running.", ) return if response.status_code == 200: # image is there, we're fine return response.json() # 404 is the standard response to "not found", if not exactly that let's log # for proper debugging if response.status_code != 404: emit.trace(f"Bad response when validation local image: {response.status_code}")
def validate_environment(self, *, part_dependencies: Optional[List[str]] = None): """Ensure the environment contains dependencies needed by the plugin. :param part_dependencies: A list of the parts this part depends on. :raises PluginEnvironmentValidationError: If the environment is invalid. """ try: output = self._execute("charm version").strip() _, tools_version = output.split("\n") if not tools_version.startswith("charm-tools"): raise PluginEnvironmentValidationError( part_name=self._part_name, reason=f"invalid charm tools version {tools_version}", ) emit.trace(f"found {tools_version}") except ValueError as err: raise PluginEnvironmentValidationError( part_name=self._part_name, reason="invalid charm tools installed", ) from err except subprocess.CalledProcessError as err: if err.returncode != plugins.validator.COMMAND_NOT_FOUND: raise PluginEnvironmentValidationError( part_name=self._part_name, reason= f"charm tools failed with error code {err.returncode}", ) from err if part_dependencies is None or "charm-tools" not in part_dependencies: raise PluginEnvironmentValidationError( part_name=self._part_name, reason=( f"charm tool not found and part {self._part_name!r} " f"does not depend on a part named 'charm-tools'"), ) from err
def _process_run(cmd: List[str]) -> None: """Run an external command logging its output. :raises CommandError: if execution crashes or ends with return code not zero. """ emit.progress(f"Running external command {cmd}") try: proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) except Exception as err: raise CommandError(f"Subprocess execution crashed for command {cmd}") from err for line in proc.stdout: emit.trace(f" :: {line.rstrip()}") retcode = proc.wait() if retcode: raise CommandError(f"Subprocess command {cmd} execution failed with retcode {retcode}")
def handle_dispatcher(self, linked_entrypoint): """Handle modern and classic dispatch mechanisms.""" # dispatch mechanism, create one if wasn't provided by the project dispatch_path = self.buildpath / DISPATCH_FILENAME if not dispatch_path.exists(): emit.progress("Creating the dispatch mechanism") dispatch_content = DISPATCH_CONTENT.format( entrypoint_relative_path=linked_entrypoint.relative_to(self.buildpath) ) with dispatch_path.open("wt", encoding="utf8") as fh: fh.write(dispatch_content) make_executable(fh) # bunch of symlinks, to support old juju: verify that any of the already included hooks # in the directory is not linking directly to the entrypoint, and also check all the # mandatory ones are present dest_hookpath = self.buildpath / HOOKS_DIR if not dest_hookpath.exists(): dest_hookpath.mkdir() # get those built hooks that we need to replace because they are pointing to the # entrypoint directly and we need to fix the environment in the middle current_hooks_to_replace = [] for node in dest_hookpath.iterdir(): if node.resolve() == linked_entrypoint: current_hooks_to_replace.append(node) node.unlink() emit.trace( f"Replacing existing hook {node.name!r} as it's a symlink to the entrypoint" ) # include the mandatory ones and those we need to replace hooknames = MANDATORY_HOOK_NAMES | {x.name for x in current_hooks_to_replace} for hookname in hooknames: emit.trace(f"Creating the {hookname!r} hook script pointing to dispatch") dest_hook = dest_hookpath / hookname if not dest_hook.exists(): relative_link = relativise(dest_hook, dispatch_path) dest_hook.symlink_to(relative_link)