def error_decorator(self, *args, **kwargs): """Handle craft-store error situations and login scenarios.""" try: return method(self, *args, **kwargs) except craft_store.errors.CredentialsUnavailable: if os.getenv(ALTERNATE_AUTH_ENV_VAR): raise RuntimeError( "Charmcraft error: internal inconsistency detected " "(CredentialsUnavailable error while having user provided credentials)." ) if not auto_login: raise emit.progress("Credentials not found. Trying to log in...") except craft_store.errors.StoreServerError as error: if error.response.status_code == 401: if os.getenv(ALTERNATE_AUTH_ENV_VAR): raise CraftError( "Provided credentials are no longer valid for Charmhub. " "Regenerate them and try again." ) if not auto_login: raise CraftError("Existing credentials are no longer valid for Charmhub.") emit.progress("Existing credentials no longer valid. Trying to log in...") # Clear credentials before trying to login again self.logout() else: raise CraftError(str(error)) from error self.login() return method(self, *args, **kwargs)
def ensure_provider_is_available(cls) -> None: """Ensure provider is available, prompting the user to install it if required. :raises CraftError: if provider is not available. """ if not lxd.is_installed(): if confirm_with_user( "LXD is required, but not installed. Do you wish to install LXD " "and configure it with the defaults?", default=False, ): try: lxd.install() except lxd.LXDInstallationError as error: raise CraftError( "Failed to install LXD. Visit https://snapcraft.io/lxd for " "instructions on how to install the LXD snap for your distribution" ) from error else: raise CraftError( "LXD is required, but not installed. Visit https://snapcraft.io/lxd for " "instructions on how to install the LXD snap for your distribution" ) try: lxd.ensure_lxd_is_ready() except lxd.LXDError as error: raise CraftError(str(error)) from error
def ensure_provider_is_available(cls) -> None: """Ensure provider is available, prompting the user to install it if required. :raises CraftError: if provider is not available. """ if not multipass.is_installed(): if confirm_with_user( "Multipass is required, but not installed. Do you wish to install Multipass " "and configure it with the defaults?", default=False, ): try: multipass.install() except multipass.MultipassInstallationError as error: raise CraftError( "Failed to install Multipass. Visit https://multipass.run/ for " "instructions on installing Multipass for your operating system." ) from error else: raise CraftError( "Multipass is required, but not installed. Visit https://multipass.run/ for " "instructions on installing Multipass for your operating system." ) try: multipass.ensure_multipass_is_ready() except multipass.MultipassError as error: raise CraftError(str(error)) from error
def useful_filepath(filepath): """Return a valid Path with user name expansion for filepath. CraftError is raised if filepath is not a valid file or is not readable. """ filepath = pathlib.Path(filepath).expanduser() if not os.access(filepath, os.R_OK): raise CraftError("Cannot access {!r}.".format(str(filepath))) if not filepath.is_file(): raise CraftError("{!r} is not a file.".format(str(filepath))) return filepath
def show_linting_results(self, linting_results): """Manage the linters results, show some in different conditions, decide if continue.""" attribute_results = [] lint_results_by_outcome = {} for result in linting_results: if result.result == linters.IGNORED: continue if result.check_type == linters.CheckType.attribute: attribute_results.append(result) else: lint_results_by_outcome.setdefault(result.result, []).append(result) # show attribute results for result in attribute_results: emit.trace( f"Check result: {result.name} [{result.check_type}] {result.result} " f"({result.text}; see more at {result.url}).", ) # show warnings (if any), then errors (if any) template = "- {0.name}: {0.text} ({0.url})" if linters.WARNINGS in lint_results_by_outcome: emit.message("Lint Warnings:", intermediate=True) for result in lint_results_by_outcome[linters.WARNINGS]: emit.message(template.format(result), intermediate=True) if linters.ERRORS in lint_results_by_outcome: emit.message("Lint Errors:", intermediate=True) for result in lint_results_by_outcome[linters.ERRORS]: emit.message(template.format(result), intermediate=True) if self.force_packing: emit.message("Packing anyway as requested.", intermediate=True) else: raise CraftError( "Aborting due to lint errors (use --force to override).", retcode=2)
def get_provider(): """Get the configured or appropriate provider for the host OS. If platform is not Linux, use Multipass. If platform is Linux: (1) use provider specified with CHARMCRAFT_PROVIDER if running in developer mode, (2) use provider specified with snap configuration if running as snap, (3) default to platform default (LXD on Linux). :return: Provider instance. """ provider = None if is_charmcraft_running_in_developer_mode(): provider = os.getenv("CHARMCRAFT_PROVIDER") if provider is None and is_charmcraft_running_from_snap(): snap_config = get_snap_configuration() provider = snap_config.provider if snap_config else None if provider is None: provider = _get_platform_default_provider() if provider == "lxd": return LXDProvider() elif provider == "multipass": return MultipassProvider() raise CraftError(f"Unsupported provider specified {provider!r}.")
def logout(self, *args, **kwargs): """Intercept regular logout functionality to forbid it when using alternate auth.""" if os.getenv(ALTERNATE_AUTH_ENV_VAR) is not None: raise CraftError( f"Cannot logout when using alternative auth through {ALTERNATE_AUTH_ENV_VAR} " "environment variable.") return super().logout(*args, **kwargs)
def __init__( self, all_parts: Dict[str, Any], *, work_dir: pathlib.Path, project_dir: pathlib.Path, project_name: str, ignore_local_sources: List[str], ): self._all_parts = all_parts.copy() self._project_dir = project_dir # set the cache dir for parts package management cache_dir = BaseDirectory.save_cache_path("charmcraft") try: self._lcm = LifecycleManager( {"parts": all_parts}, application_name="charmcraft", work_dir=work_dir, cache_dir=cache_dir, ignore_local_sources=ignore_local_sources, project_name=project_name, ) except PartsError as err: raise CraftError(f"Error bootstrapping lifecycle manager: {err}") from err
def validate_from(self, dirpath): """Validate that the charm dir is there and yes, a directory.""" if dirpath is None: dirpath = pathlib.Path.cwd() else: dirpath = dirpath.expanduser().absolute() if not dirpath.exists(): raise CraftError("Charm directory was not found: {!r}".format( str(dirpath))) if not dirpath.is_dir(): raise CraftError( "Charm directory is not really a directory: {!r}".format( str(dirpath))) self.basedir = dirpath return dirpath
def run(self, parsed_args): """Run the command.""" # decide if this will work on a charm or a bundle if self.config.type == "charm": self._pack_charm(parsed_args) elif self.config.type == "bundle": if parsed_args.entrypoint is not None: raise CraftError( "The -e/--entry option is valid only when packing a charm") if parsed_args.requirement is not None: raise CraftError( "The -r/--requirement option is valid only when packing a charm" ) self._pack_bundle(parsed_args) else: raise CraftError("Unknown type {!r} in charmcraft.yaml".format( self.config.type))
def clean_project_environments( self, *, charm_name: str, project_path: pathlib.Path, ) -> List[str]: """Clean up any environments created for project. :param charm_name: Name of project. :param project_path: Directory of charm project. :returns: List of containers deleted. """ deleted: List[str] = [] # Nothing to do if provider is not installed. if not self.is_provider_available(): return deleted inode = str(project_path.stat().st_ino) try: names = self.lxc.list_names(project=self.lxd_project, remote=self.lxd_remote) except lxd.LXDError as error: raise CraftError(str(error)) from error for name in names: match_regex = f"^charmcraft-{charm_name}-{inode}-.+-.+-.+$" if re.match(match_regex, name): emit.trace(f"Deleting container {name!r}.") try: self.lxc.delete( instance_name=name, force=True, project=self.lxd_project, remote=self.lxd_remote, ) except lxd.LXDError as error: raise CraftError(str(error)) from error deleted.append(name) else: emit.trace(f"Not deleting container {name!r}.") return deleted
def test_main_controlled_error(base_config_present): """Work raised CraftError: message handler notified properly, use indicated return code.""" simulated_exception = CraftError("boom", retcode=33) with patch("charmcraft.main.emit") as emit_mock: with patch("charmcraft.main.Dispatcher.run") as d_mock: d_mock.side_effect = simulated_exception retcode = main(["charmcraft", "version"]) assert retcode == 33 emit_mock.error.assert_called_once_with(simulated_exception)
def validate_entrypoint(self, filepath): """Validate that the entrypoint exists and is executable.""" if filepath is None: return None filepath = filepath.expanduser().absolute() if not filepath.exists(): raise CraftError("Charm entry point was not found: {!r}".format( str(filepath))) if self.basedir not in filepath.parents: raise CraftError( "Charm entry point must be inside the project: {!r}".format( str(filepath))) if not os.access(filepath, os.X_OK): raise CraftError( "Charm entry point must be executable: {!r}".format( str(filepath))) return filepath
def validate_bases_indices(self, bases_indices): """Validate that bases index is valid.""" if not bases_indices: return for bases_index in bases_indices: if bases_index < 0: raise CraftError( f"Bases index '{bases_index}' is invalid (must be >= 0).") if not self.config.bases: raise CraftError( "No bases configuration found, required when using --bases-index.", ) if bases_index >= len(self.config.bases): raise CraftError( f"No bases configuration found for specified index '{bases_index}'." )
def test_bundle_debug_with_error(tmp_path, bundle_yaml, bundle_config, mock_parts, mock_launch_shell): mock_parts.PartsLifecycle.return_value.run.side_effect = CraftError("fail") bundle_yaml(name="testbundle") bundle_config.set(type="bundle") (tmp_path / "README.md").write_text("test readme") with pytest.raises(CraftError): PackCommand(bundle_config).run(get_namespace(debug=True)) assert mock_launch_shell.mock_calls == [mock.call()]
def unmarshal(cls, obj: Dict[str, Any]): """Unmarshal object with necessary translations and error handling. :returns: valid CharmMetadata. :raises CraftError: On failure to unmarshal object. """ try: return cls.parse_obj(obj) except pydantic.error_wrappers.ValidationError as error: raise CraftError(format_pydantic_errors(error.errors(), file_name=CHARM_METADATA))
def run(self, target_step: Step) -> None: """Run the parts lifecycle. :param target_step: The final step to execute. :raises CraftError: On error during lifecycle ops. :raises RuntimeError: On unexpected error. """ previous_dir = os.getcwd() try: os.chdir(self._project_dir) # invalidate build if packing a charm and entrypoint changed if "charm" in self._all_parts: charm_part = self._all_parts["charm"] if charm_part.get("plugin") == "charm": entrypoint = os.path.normpath(charm_part["charm-entrypoint"]) dis_entrypoint = os.path.normpath(_get_dispatch_entrypoint(self.prime_dir)) if entrypoint != dis_entrypoint: self._lcm.clean(Step.BUILD, part_names=["charm"]) self._lcm.reload_state() emit.trace(f"Executing parts lifecycle in {str(self._project_dir)!r}") actions = self._lcm.plan(target_step) emit.trace(f"Parts actions: {actions}") with self._lcm.action_executor() as aex: for action in actions: emit.progress(f"Running step {action.step.name} for part {action.part_name!r}") with emit.open_stream("Execute action") as stream: aex.execute([action], stdout=stream, stderr=stream) except RuntimeError as err: raise RuntimeError(f"Parts processing internal error: {err}") from err except OSError as err: msg = err.strerror if err.filename: msg = f"{err.filename}: {msg}" raise CraftError(f"Parts processing error: {msg}") from err except Exception as err: raise CraftError(f"Parts processing error: {err}") from err finally: os.chdir(previous_dir)
def request_urlpath_json(self, method: str, urlpath: str, *args, **kwargs) -> Dict[str, Any]: """Return .json() from a request.Response to a urlpath.""" response = super().request(method, self.api_base_url + urlpath, *args, **kwargs) try: return response.json() except JSONDecodeError as json_error: raise CraftError( f"Could not retrieve json response ({response.status_code} from request" ) from json_error
def create_manifest( basedir: pathlib.Path, started_at: datetime.datetime, bases_config: Optional[config.BasesConfiguration], linting_results: List[linters.CheckResult], ): """Create manifest.yaml in basedir for given base configuration. For packing bundles, `bases` will be skipped when bases_config is None. Charms should always include a valid bases_config. :param basedir: Directory to create Charm in. :param started_at: Build start time. :param bases_config: Relevant bases configuration, if any. :returns: Path to created manifest.yaml. """ content = { "charmcraft-version": __version__, "charmcraft-started-at": started_at.isoformat() + "Z", } # Annotate bases only if bases_config is not None. if bases_config is not None: bases = [{ "name": r.name, "channel": r.channel, "architectures": r.architectures, } for r in bases_config.run_on] content["bases"] = bases # include the linters results (only for attributes) attributes_info = [{ "name": result.name, "result": result.result } for result in linting_results if result.check_type == linters.CheckType.attribute] content["analysis"] = {"attributes": attributes_info} # include the image info, if present image_info_raw = os.environ.get(IMAGE_INFO_ENV_VAR) if image_info_raw: try: image_info = json.loads(image_info_raw) except json.decoder.JSONDecodeError as exc: msg = f"Failed to parse the content of {IMAGE_INFO_ENV_VAR} environment variable" raise CraftError(msg) from exc content["image-info"] = image_info filepath = basedir / "manifest.yaml" filepath.write_text(yaml.dump(content)) return filepath
def _process_run(cmd: List[str]) -> None: """Run an external command logging its output. :raises CraftError: if execution crashes or ends with return code not zero. """ emit.progress(f"Running external command {cmd}") try: proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) except Exception as err: raise CraftError(f"Subprocess execution crashed for command {cmd}") from err for line in proc.stdout: emit.trace(f" :: {line.rstrip()}") retcode = proc.wait() if retcode: raise CraftError(f"Subprocess command {cmd} execution failed with retcode {retcode}")
def assert_response_ok( response: requests.Response, expected_status: int = 200 ) -> Union[Dict[str, Any], None]: """Assert the response is ok.""" if response.status_code != expected_status: ct = response.headers.get("Content-Type", "") if ct.split(";")[0] in JSON_RELATED_MIMETYPES: errors = response.json().get("errors") else: errors = None raise CraftError( "Wrong status code from server " f"(expected={expected_status}, got={response.status_code})", details=f"errors={errors} headers={response.headers}", ) if response.headers.get("Content-Type") not in JSON_RELATED_MIMETYPES: return result = response.json() if "errors" in result: raise CraftError("Response with errors from server: {}".format(result["errors"])) return result
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata: """Parse project's metadata.yaml. :returns: a CharmMetadata object. :raises: CraftError if metadata does not exist. """ try: metadata = read_metadata_yaml(charm_dir) except OSError as exc: raise CraftError(f"Cannot read the metadata.yaml file: {exc!r}") from exc emit.trace("Validating metadata format") return CharmMetadata.unmarshal(metadata)
def validate_requirement(self, filepaths): """Validate that the given requirement(s) (if any) exist. If not specified, default to requirements.txt if there. """ if filepaths is None: return [] filepaths = [x.expanduser().absolute() for x in filepaths] for fpath in filepaths: if not fpath.exists(): raise CraftError( "the requirements file was not found: {!r}".format( str(fpath))) return filepaths
def _handle_deprecated_cli_arguments(self): # verify if deprecated --requirement is used and update the plugin property if self._special_charm_part.get("charm-requirements"): if self.requirement_paths: raise CraftError( "--requirement not supported when charm-requirements " "specified in charmcraft.yaml") else: if self.requirement_paths: self._special_charm_part["charm-requirements"] = [ str(p) for p in self.requirement_paths ] self.requirement_paths = None else: default_reqfile = self.charmdir / "requirements.txt" if default_reqfile.is_file(): self._special_charm_part["charm-requirements"] = [ "requirements.txt" ] else: self._special_charm_part["charm-requirements"] = [] # verify if deprecated --entrypoint is used and update the plugin property if self._special_charm_part.get("charm-entrypoint"): if self.entrypoint: raise CraftError( "--entrypoint not supported when charm-entrypoint " "specified in charmcraft.yaml") else: if self.entrypoint: rel_entrypoint = self.entrypoint.relative_to(self.charmdir) self._special_charm_part["charm-entrypoint"] = str( rel_entrypoint) self.entrypoint = None else: self._special_charm_part["charm-entrypoint"] = "src/charm.py"
def _unzip_charm(self, filepath: pathlib.Path) -> pathlib.Path: """Extract the charm content to a temp directory.""" tmpdir = pathlib.Path(tempfile.mkdtemp()) try: zf = zipfile.ZipFile(str(filepath)) zf.extractall(path=str(tmpdir)) except Exception as exc: raise CraftError( f"Cannot open charm file {str(filepath)!r}: {exc!r}.") # fix permissions as extractall does not keep them (see https://bugs.python.org/issue15795) for name in zf.namelist(): info = zf.getinfo(name) inside_zip_mode = info.external_attr >> 16 extracted_file = tmpdir / name current_mode = extracted_file.stat().st_mode if current_mode != inside_zip_mode: extracted_file.chmod(inside_zip_mode) return tmpdir
def _hit(self, method, url, headers=None, log=True, **kwargs): """Hit the specific URL, taking care of the authentication.""" if headers is None: headers = {} if self.auth_token is not None: headers["Authorization"] = "Bearer {}".format(self.auth_token) if log: emit.trace(f"Hitting the registry: {method} {url}") response = requests.request(method, url, headers=headers, **kwargs) if response.status_code == 401: # token expired or missing, let's get another one and retry try: auth_info = self._get_auth_info(response) except (ValueError, KeyError) as exc: raise CraftError( "Bad 401 response: {}; headers: {!r}".format(exc, response.headers) ) self.auth_token = self._authenticate(auth_info) headers["Authorization"] = "Bearer {}".format(self.auth_token) response = requests.request(method, url, headers=headers, **kwargs) return response
def expand_short_form_bases(cls, bases: List[Dict[str, Any]]) -> None: """Expand short-form base configuration into long-form in-place.""" for index, base in enumerate(bases): # Skip if already long-form. Account for common typos in case user # intends to use long-form, but did so incorrectly (for better # error message handling). if "run-on" in base or "run_on" in base or "build-on" in base or "build_on" in base: continue try: converted_base = Base(**base) except pydantic.error_wrappers.ValidationError as error: # Rewrite location to assist user. pydantic_errors = error.errors() for pydantic_error in pydantic_errors: pydantic_error["loc"] = ("bases", index, pydantic_error["loc"][0]) raise CraftError(format_pydantic_errors(pydantic_errors)) base.clear() base["build-on"] = [converted_base.dict()] base["run-on"] = [converted_base.dict()]
def unmarshal(cls, obj: Dict[str, Any], project: Project): """Unmarshal object with necessary translations and error handling. (1) Perform any necessary translations. (2) Standardize error reporting. :returns: valid CharmcraftConfig. :raises CraftError: On failure to unmarshal object. """ try: # Ensure short-form bases are expanded into long-form # base configurations. Doing it here rather than a Union # type will simplify user facing errors. bases = obj.get("bases") if bases is None: # "type" is accessed with get because this code happens before # pydantic actually validating that type is present if obj.get("type") == "charm": notify_deprecation("dn03") # Set default bases to Ubuntu 20.04 to match strict snap's # effective behavior. bases = [{ "name": "ubuntu", "channel": "20.04", "architectures": [get_host_architecture()], }] # Expand short-form bases if only the bases is a valid list. If it # is not a valid list, parse_obj() will properly handle the error. if isinstance(bases, list): cls.expand_short_form_bases(bases) return cls.parse_obj({"project": project, **obj}) except pydantic.error_wrappers.ValidationError as error: raise CraftError(format_pydantic_errors(error.errors()))
def push_file(self, filepath) -> str: """Push the bytes from filepath to the Storage.""" emit.progress(f"Starting to push {str(filepath)!r}") with filepath.open("rb") as fh: encoder = MultipartEncoder(fields={ "binary": (filepath.name, fh, "application/octet-stream") }) # create a monitor (so that progress can be displayed) as call the real pusher monitor = MultipartEncoderMonitor(encoder) with emit.progress_bar("Uploading...", monitor.len, delta=False) as progress: monitor.callback = lambda mon: progress.advance(mon.bytes_read) response = self._storage_push(monitor) result = response.json() if not result["successful"]: raise CraftError( "Server error while pushing file: {}".format(result)) upload_id = result["upload_id"] emit.progress(f"Uploading bytes ended, id {upload_id}") return upload_id
def _pack_bundle(self, parsed_args) -> List[pathlib.Path]: """Pack a bundle.""" emit.progress("Packing the bundle.") if parsed_args.shell: build.launch_shell() return [] project = self.config.project if self.config.parts: config_parts = self.config.parts.copy() else: # "parts" not declared, create an implicit "bundle" part config_parts = {"bundle": {"plugin": "bundle"}} # a part named "bundle" using plugin "bundle" is special and has # predefined values set automatically. bundle_part = config_parts.get("bundle") if bundle_part and bundle_part.get("plugin") == "bundle": special_bundle_part = bundle_part else: special_bundle_part = None # get the config files bundle_filepath = project.dirpath / "bundle.yaml" bundle_config = load_yaml(bundle_filepath) if bundle_config is None: raise CraftError( "Missing or invalid main bundle file: {!r}.".format( str(bundle_filepath))) bundle_name = bundle_config.get("name") if not bundle_name: raise CraftError( "Invalid bundle config; missing a 'name' field indicating the bundle's name in " "file {!r}.".format(str(bundle_filepath))) if special_bundle_part: # set prime filters for fname in MANDATORY_FILES: fpath = project.dirpath / fname if not fpath.exists(): raise CraftError("Missing mandatory file: {!r}.".format( str(fpath))) prime = special_bundle_part.setdefault("prime", []) prime.extend(MANDATORY_FILES) # set source if empty or not declared in charm part if not special_bundle_part.get("source"): special_bundle_part["source"] = str(project.dirpath) if env.is_charmcraft_running_in_managed_mode(): work_dir = env.get_managed_environment_home_path() else: work_dir = project.dirpath / build.BUILD_DIRNAME # run the parts lifecycle emit.trace(f"Parts definition: {config_parts}") lifecycle = parts.PartsLifecycle( config_parts, work_dir=work_dir, project_dir=project.dirpath, project_name=bundle_name, ignore_local_sources=[bundle_name + ".zip"], ) try: lifecycle.run(Step.PRIME) except (RuntimeError, CraftError) as error: if parsed_args.debug: emit.trace(f"Error when running PRIME step: {error}") build.launch_shell() raise # pack everything create_manifest(lifecycle.prime_dir, project.started_at, None, []) zipname = project.dirpath / (bundle_name + ".zip") build_zip(zipname, lifecycle.prime_dir) emit.message(f"Created {str(zipname)!r}.") if parsed_args.shell_after: build.launch_shell() return [zipname]