def error_decorator(self, *args, **kwargs):
"""Handle craft-store error situations and login scenarios."""
try:
return method(self, *args, **kwargs)
except craft_store.errors.CredentialsUnavailable:
if os.getenv(ALTERNATE_AUTH_ENV_VAR):
raise RuntimeError(
"Charmcraft error: internal inconsistency detected "
"(CredentialsUnavailable error while having user provided credentials)."
)
if not auto_login:
raise
emit.progress("Credentials not found. Trying to log in...")
except craft_store.errors.StoreServerError as error:
if error.response.status_code == 401:
if os.getenv(ALTERNATE_AUTH_ENV_VAR):
raise CraftError(
"Provided credentials are no longer valid for Charmhub. "
"Regenerate them and try again."
)
if not auto_login:
raise CraftError("Existing credentials are no longer valid for Charmhub.")
emit.progress("Existing credentials no longer valid. Trying to log in...")
# Clear credentials before trying to login again
self.logout()
else:
raise CraftError(str(error)) from error
self.login()
return method(self, *args, **kwargs)
def ensure_provider_is_available(cls) -> None:
"""Ensure provider is available, prompting the user to install it if required.
:raises CraftError: if provider is not available.
"""
if not lxd.is_installed():
if confirm_with_user(
"LXD is required, but not installed. Do you wish to install LXD "
"and configure it with the defaults?",
default=False,
):
try:
lxd.install()
except lxd.LXDInstallationError as error:
raise CraftError(
"Failed to install LXD. Visit https://snapcraft.io/lxd for "
"instructions on how to install the LXD snap for your distribution"
) from error
else:
raise CraftError(
"LXD is required, but not installed. Visit https://snapcraft.io/lxd for "
"instructions on how to install the LXD snap for your distribution"
)
try:
lxd.ensure_lxd_is_ready()
except lxd.LXDError as error:
raise CraftError(str(error)) from error
def ensure_provider_is_available(cls) -> None:
"""Ensure provider is available, prompting the user to install it if required.
:raises CraftError: if provider is not available.
"""
if not multipass.is_installed():
if confirm_with_user(
"Multipass is required, but not installed. Do you wish to install Multipass "
"and configure it with the defaults?",
default=False,
):
try:
multipass.install()
except multipass.MultipassInstallationError as error:
raise CraftError(
"Failed to install Multipass. Visit https://multipass.run/ for "
"instructions on installing Multipass for your operating system."
) from error
else:
raise CraftError(
"Multipass is required, but not installed. Visit https://multipass.run/ for "
"instructions on installing Multipass for your operating system."
)
try:
multipass.ensure_multipass_is_ready()
except multipass.MultipassError as error:
raise CraftError(str(error)) from error
def useful_filepath(filepath):
"""Return a valid Path with user name expansion for filepath.
CraftError is raised if filepath is not a valid file or is not readable.
"""
filepath = pathlib.Path(filepath).expanduser()
if not os.access(filepath, os.R_OK):
raise CraftError("Cannot access {!r}.".format(str(filepath)))
if not filepath.is_file():
raise CraftError("{!r} is not a file.".format(str(filepath)))
return filepath
def show_linting_results(self, linting_results):
"""Manage the linters results, show some in different conditions, decide if continue."""
attribute_results = []
lint_results_by_outcome = {}
for result in linting_results:
if result.result == linters.IGNORED:
continue
if result.check_type == linters.CheckType.attribute:
attribute_results.append(result)
else:
lint_results_by_outcome.setdefault(result.result,
[]).append(result)
# show attribute results
for result in attribute_results:
emit.trace(
f"Check result: {result.name} [{result.check_type}] {result.result} "
f"({result.text}; see more at {result.url}).", )
# show warnings (if any), then errors (if any)
template = "- {0.name}: {0.text} ({0.url})"
if linters.WARNINGS in lint_results_by_outcome:
emit.message("Lint Warnings:", intermediate=True)
for result in lint_results_by_outcome[linters.WARNINGS]:
emit.message(template.format(result), intermediate=True)
if linters.ERRORS in lint_results_by_outcome:
emit.message("Lint Errors:", intermediate=True)
for result in lint_results_by_outcome[linters.ERRORS]:
emit.message(template.format(result), intermediate=True)
if self.force_packing:
emit.message("Packing anyway as requested.", intermediate=True)
else:
raise CraftError(
"Aborting due to lint errors (use --force to override).",
retcode=2)
def get_provider():
"""Get the configured or appropriate provider for the host OS.
If platform is not Linux, use Multipass.
If platform is Linux:
(1) use provider specified with CHARMCRAFT_PROVIDER if running
in developer mode,
(2) use provider specified with snap configuration if running
as snap,
(3) default to platform default (LXD on Linux).
:return: Provider instance.
"""
provider = None
if is_charmcraft_running_in_developer_mode():
provider = os.getenv("CHARMCRAFT_PROVIDER")
if provider is None and is_charmcraft_running_from_snap():
snap_config = get_snap_configuration()
provider = snap_config.provider if snap_config else None
if provider is None:
provider = _get_platform_default_provider()
if provider == "lxd":
return LXDProvider()
elif provider == "multipass":
return MultipassProvider()
raise CraftError(f"Unsupported provider specified {provider!r}.")
def logout(self, *args, **kwargs):
"""Intercept regular logout functionality to forbid it when using alternate auth."""
if os.getenv(ALTERNATE_AUTH_ENV_VAR) is not None:
raise CraftError(
f"Cannot logout when using alternative auth through {ALTERNATE_AUTH_ENV_VAR} "
"environment variable.")
return super().logout(*args, **kwargs)
def __init__(
self,
all_parts: Dict[str, Any],
*,
work_dir: pathlib.Path,
project_dir: pathlib.Path,
project_name: str,
ignore_local_sources: List[str],
):
self._all_parts = all_parts.copy()
self._project_dir = project_dir
# set the cache dir for parts package management
cache_dir = BaseDirectory.save_cache_path("charmcraft")
try:
self._lcm = LifecycleManager(
{"parts": all_parts},
application_name="charmcraft",
work_dir=work_dir,
cache_dir=cache_dir,
ignore_local_sources=ignore_local_sources,
project_name=project_name,
)
except PartsError as err:
raise CraftError(f"Error bootstrapping lifecycle manager: {err}") from err
def validate_from(self, dirpath):
"""Validate that the charm dir is there and yes, a directory."""
if dirpath is None:
dirpath = pathlib.Path.cwd()
else:
dirpath = dirpath.expanduser().absolute()
if not dirpath.exists():
raise CraftError("Charm directory was not found: {!r}".format(
str(dirpath)))
if not dirpath.is_dir():
raise CraftError(
"Charm directory is not really a directory: {!r}".format(
str(dirpath)))
self.basedir = dirpath
return dirpath
def run(self, parsed_args):
"""Run the command."""
# decide if this will work on a charm or a bundle
if self.config.type == "charm":
self._pack_charm(parsed_args)
elif self.config.type == "bundle":
if parsed_args.entrypoint is not None:
raise CraftError(
"The -e/--entry option is valid only when packing a charm")
if parsed_args.requirement is not None:
raise CraftError(
"The -r/--requirement option is valid only when packing a charm"
)
self._pack_bundle(parsed_args)
else:
raise CraftError("Unknown type {!r} in charmcraft.yaml".format(
self.config.type))
def clean_project_environments(
self,
*,
charm_name: str,
project_path: pathlib.Path,
) -> List[str]:
"""Clean up any environments created for project.
:param charm_name: Name of project.
:param project_path: Directory of charm project.
:returns: List of containers deleted.
"""
deleted: List[str] = []
# Nothing to do if provider is not installed.
if not self.is_provider_available():
return deleted
inode = str(project_path.stat().st_ino)
try:
names = self.lxc.list_names(project=self.lxd_project, remote=self.lxd_remote)
except lxd.LXDError as error:
raise CraftError(str(error)) from error
for name in names:
match_regex = f"^charmcraft-{charm_name}-{inode}-.+-.+-.+$"
if re.match(match_regex, name):
emit.trace(f"Deleting container {name!r}.")
try:
self.lxc.delete(
instance_name=name,
force=True,
project=self.lxd_project,
remote=self.lxd_remote,
)
except lxd.LXDError as error:
raise CraftError(str(error)) from error
deleted.append(name)
else:
emit.trace(f"Not deleting container {name!r}.")
return deleted
def test_main_controlled_error(base_config_present):
"""Work raised CraftError: message handler notified properly, use indicated return code."""
simulated_exception = CraftError("boom", retcode=33)
with patch("charmcraft.main.emit") as emit_mock:
with patch("charmcraft.main.Dispatcher.run") as d_mock:
d_mock.side_effect = simulated_exception
retcode = main(["charmcraft", "version"])
assert retcode == 33
emit_mock.error.assert_called_once_with(simulated_exception)
def validate_entrypoint(self, filepath):
"""Validate that the entrypoint exists and is executable."""
if filepath is None:
return None
filepath = filepath.expanduser().absolute()
if not filepath.exists():
raise CraftError("Charm entry point was not found: {!r}".format(
str(filepath)))
if self.basedir not in filepath.parents:
raise CraftError(
"Charm entry point must be inside the project: {!r}".format(
str(filepath)))
if not os.access(filepath, os.X_OK):
raise CraftError(
"Charm entry point must be executable: {!r}".format(
str(filepath)))
return filepath
def validate_bases_indices(self, bases_indices):
"""Validate that bases index is valid."""
if not bases_indices:
return
for bases_index in bases_indices:
if bases_index < 0:
raise CraftError(
f"Bases index '{bases_index}' is invalid (must be >= 0).")
if not self.config.bases:
raise CraftError(
"No bases configuration found, required when using --bases-index.",
)
if bases_index >= len(self.config.bases):
raise CraftError(
f"No bases configuration found for specified index '{bases_index}'."
)
def test_bundle_debug_with_error(tmp_path, bundle_yaml, bundle_config,
mock_parts, mock_launch_shell):
mock_parts.PartsLifecycle.return_value.run.side_effect = CraftError("fail")
bundle_yaml(name="testbundle")
bundle_config.set(type="bundle")
(tmp_path / "README.md").write_text("test readme")
with pytest.raises(CraftError):
PackCommand(bundle_config).run(get_namespace(debug=True))
assert mock_launch_shell.mock_calls == [mock.call()]
def unmarshal(cls, obj: Dict[str, Any]):
"""Unmarshal object with necessary translations and error handling.
:returns: valid CharmMetadata.
:raises CraftError: On failure to unmarshal object.
"""
try:
return cls.parse_obj(obj)
except pydantic.error_wrappers.ValidationError as error:
raise CraftError(format_pydantic_errors(error.errors(), file_name=CHARM_METADATA))
def run(self, target_step: Step) -> None:
"""Run the parts lifecycle.
:param target_step: The final step to execute.
:raises CraftError: On error during lifecycle ops.
:raises RuntimeError: On unexpected error.
"""
previous_dir = os.getcwd()
try:
os.chdir(self._project_dir)
# invalidate build if packing a charm and entrypoint changed
if "charm" in self._all_parts:
charm_part = self._all_parts["charm"]
if charm_part.get("plugin") == "charm":
entrypoint = os.path.normpath(charm_part["charm-entrypoint"])
dis_entrypoint = os.path.normpath(_get_dispatch_entrypoint(self.prime_dir))
if entrypoint != dis_entrypoint:
self._lcm.clean(Step.BUILD, part_names=["charm"])
self._lcm.reload_state()
emit.trace(f"Executing parts lifecycle in {str(self._project_dir)!r}")
actions = self._lcm.plan(target_step)
emit.trace(f"Parts actions: {actions}")
with self._lcm.action_executor() as aex:
for action in actions:
emit.progress(f"Running step {action.step.name} for part {action.part_name!r}")
with emit.open_stream("Execute action") as stream:
aex.execute([action], stdout=stream, stderr=stream)
except RuntimeError as err:
raise RuntimeError(f"Parts processing internal error: {err}") from err
except OSError as err:
msg = err.strerror
if err.filename:
msg = f"{err.filename}: {msg}"
raise CraftError(f"Parts processing error: {msg}") from err
except Exception as err:
raise CraftError(f"Parts processing error: {err}") from err
finally:
os.chdir(previous_dir)
def request_urlpath_json(self, method: str, urlpath: str, *args,
**kwargs) -> Dict[str, Any]:
"""Return .json() from a request.Response to a urlpath."""
response = super().request(method, self.api_base_url + urlpath, *args,
**kwargs)
try:
return response.json()
except JSONDecodeError as json_error:
raise CraftError(
f"Could not retrieve json response ({response.status_code} from request"
) from json_error
def create_manifest(
basedir: pathlib.Path,
started_at: datetime.datetime,
bases_config: Optional[config.BasesConfiguration],
linting_results: List[linters.CheckResult],
):
"""Create manifest.yaml in basedir for given base configuration.
For packing bundles, `bases` will be skipped when bases_config is None.
Charms should always include a valid bases_config.
:param basedir: Directory to create Charm in.
:param started_at: Build start time.
:param bases_config: Relevant bases configuration, if any.
:returns: Path to created manifest.yaml.
"""
content = {
"charmcraft-version": __version__,
"charmcraft-started-at": started_at.isoformat() + "Z",
}
# Annotate bases only if bases_config is not None.
if bases_config is not None:
bases = [{
"name": r.name,
"channel": r.channel,
"architectures": r.architectures,
} for r in bases_config.run_on]
content["bases"] = bases
# include the linters results (only for attributes)
attributes_info = [{
"name": result.name,
"result": result.result
} for result in linting_results
if result.check_type == linters.CheckType.attribute]
content["analysis"] = {"attributes": attributes_info}
# include the image info, if present
image_info_raw = os.environ.get(IMAGE_INFO_ENV_VAR)
if image_info_raw:
try:
image_info = json.loads(image_info_raw)
except json.decoder.JSONDecodeError as exc:
msg = f"Failed to parse the content of {IMAGE_INFO_ENV_VAR} environment variable"
raise CraftError(msg) from exc
content["image-info"] = image_info
filepath = basedir / "manifest.yaml"
filepath.write_text(yaml.dump(content))
return filepath
def _process_run(cmd: List[str]) -> None:
"""Run an external command logging its output.
:raises CraftError: if execution crashes or ends with return code not zero.
"""
emit.progress(f"Running external command {cmd}")
try:
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
except Exception as err:
raise CraftError(f"Subprocess execution crashed for command {cmd}") from err
for line in proc.stdout:
emit.trace(f" :: {line.rstrip()}")
retcode = proc.wait()
if retcode:
raise CraftError(f"Subprocess command {cmd} execution failed with retcode {retcode}")
def assert_response_ok(
response: requests.Response, expected_status: int = 200
) -> Union[Dict[str, Any], None]:
"""Assert the response is ok."""
if response.status_code != expected_status:
ct = response.headers.get("Content-Type", "")
if ct.split(";")[0] in JSON_RELATED_MIMETYPES:
errors = response.json().get("errors")
else:
errors = None
raise CraftError(
"Wrong status code from server "
f"(expected={expected_status}, got={response.status_code})",
details=f"errors={errors} headers={response.headers}",
)
if response.headers.get("Content-Type") not in JSON_RELATED_MIMETYPES:
return
result = response.json()
if "errors" in result:
raise CraftError("Response with errors from server: {}".format(result["errors"]))
return result
def parse_metadata_yaml(charm_dir: pathlib.Path) -> CharmMetadata:
"""Parse project's metadata.yaml.
:returns: a CharmMetadata object.
:raises: CraftError if metadata does not exist.
"""
try:
metadata = read_metadata_yaml(charm_dir)
except OSError as exc:
raise CraftError(f"Cannot read the metadata.yaml file: {exc!r}") from exc
emit.trace("Validating metadata format")
return CharmMetadata.unmarshal(metadata)
def validate_requirement(self, filepaths):
"""Validate that the given requirement(s) (if any) exist.
If not specified, default to requirements.txt if there.
"""
if filepaths is None:
return []
filepaths = [x.expanduser().absolute() for x in filepaths]
for fpath in filepaths:
if not fpath.exists():
raise CraftError(
"the requirements file was not found: {!r}".format(
str(fpath)))
return filepaths
def _handle_deprecated_cli_arguments(self):
# verify if deprecated --requirement is used and update the plugin property
if self._special_charm_part.get("charm-requirements"):
if self.requirement_paths:
raise CraftError(
"--requirement not supported when charm-requirements "
"specified in charmcraft.yaml")
else:
if self.requirement_paths:
self._special_charm_part["charm-requirements"] = [
str(p) for p in self.requirement_paths
]
self.requirement_paths = None
else:
default_reqfile = self.charmdir / "requirements.txt"
if default_reqfile.is_file():
self._special_charm_part["charm-requirements"] = [
"requirements.txt"
]
else:
self._special_charm_part["charm-requirements"] = []
# verify if deprecated --entrypoint is used and update the plugin property
if self._special_charm_part.get("charm-entrypoint"):
if self.entrypoint:
raise CraftError(
"--entrypoint not supported when charm-entrypoint "
"specified in charmcraft.yaml")
else:
if self.entrypoint:
rel_entrypoint = self.entrypoint.relative_to(self.charmdir)
self._special_charm_part["charm-entrypoint"] = str(
rel_entrypoint)
self.entrypoint = None
else:
self._special_charm_part["charm-entrypoint"] = "src/charm.py"
def _unzip_charm(self, filepath: pathlib.Path) -> pathlib.Path:
"""Extract the charm content to a temp directory."""
tmpdir = pathlib.Path(tempfile.mkdtemp())
try:
zf = zipfile.ZipFile(str(filepath))
zf.extractall(path=str(tmpdir))
except Exception as exc:
raise CraftError(
f"Cannot open charm file {str(filepath)!r}: {exc!r}.")
# fix permissions as extractall does not keep them (see https://bugs.python.org/issue15795)
for name in zf.namelist():
info = zf.getinfo(name)
inside_zip_mode = info.external_attr >> 16
extracted_file = tmpdir / name
current_mode = extracted_file.stat().st_mode
if current_mode != inside_zip_mode:
extracted_file.chmod(inside_zip_mode)
return tmpdir
def _hit(self, method, url, headers=None, log=True, **kwargs):
"""Hit the specific URL, taking care of the authentication."""
if headers is None:
headers = {}
if self.auth_token is not None:
headers["Authorization"] = "Bearer {}".format(self.auth_token)
if log:
emit.trace(f"Hitting the registry: {method} {url}")
response = requests.request(method, url, headers=headers, **kwargs)
if response.status_code == 401:
# token expired or missing, let's get another one and retry
try:
auth_info = self._get_auth_info(response)
except (ValueError, KeyError) as exc:
raise CraftError(
"Bad 401 response: {}; headers: {!r}".format(exc, response.headers)
)
self.auth_token = self._authenticate(auth_info)
headers["Authorization"] = "Bearer {}".format(self.auth_token)
response = requests.request(method, url, headers=headers, **kwargs)
return response
def expand_short_form_bases(cls, bases: List[Dict[str, Any]]) -> None:
"""Expand short-form base configuration into long-form in-place."""
for index, base in enumerate(bases):
# Skip if already long-form. Account for common typos in case user
# intends to use long-form, but did so incorrectly (for better
# error message handling).
if "run-on" in base or "run_on" in base or "build-on" in base or "build_on" in base:
continue
try:
converted_base = Base(**base)
except pydantic.error_wrappers.ValidationError as error:
# Rewrite location to assist user.
pydantic_errors = error.errors()
for pydantic_error in pydantic_errors:
pydantic_error["loc"] = ("bases", index,
pydantic_error["loc"][0])
raise CraftError(format_pydantic_errors(pydantic_errors))
base.clear()
base["build-on"] = [converted_base.dict()]
base["run-on"] = [converted_base.dict()]
def unmarshal(cls, obj: Dict[str, Any], project: Project):
"""Unmarshal object with necessary translations and error handling.
(1) Perform any necessary translations.
(2) Standardize error reporting.
:returns: valid CharmcraftConfig.
:raises CraftError: On failure to unmarshal object.
"""
try:
# Ensure short-form bases are expanded into long-form
# base configurations. Doing it here rather than a Union
# type will simplify user facing errors.
bases = obj.get("bases")
if bases is None:
# "type" is accessed with get because this code happens before
# pydantic actually validating that type is present
if obj.get("type") == "charm":
notify_deprecation("dn03")
# Set default bases to Ubuntu 20.04 to match strict snap's
# effective behavior.
bases = [{
"name": "ubuntu",
"channel": "20.04",
"architectures": [get_host_architecture()],
}]
# Expand short-form bases if only the bases is a valid list. If it
# is not a valid list, parse_obj() will properly handle the error.
if isinstance(bases, list):
cls.expand_short_form_bases(bases)
return cls.parse_obj({"project": project, **obj})
except pydantic.error_wrappers.ValidationError as error:
raise CraftError(format_pydantic_errors(error.errors()))
def push_file(self, filepath) -> str:
"""Push the bytes from filepath to the Storage."""
emit.progress(f"Starting to push {str(filepath)!r}")
with filepath.open("rb") as fh:
encoder = MultipartEncoder(fields={
"binary": (filepath.name, fh, "application/octet-stream")
})
# create a monitor (so that progress can be displayed) as call the real pusher
monitor = MultipartEncoderMonitor(encoder)
with emit.progress_bar("Uploading...", monitor.len,
delta=False) as progress:
monitor.callback = lambda mon: progress.advance(mon.bytes_read)
response = self._storage_push(monitor)
result = response.json()
if not result["successful"]:
raise CraftError(
"Server error while pushing file: {}".format(result))
upload_id = result["upload_id"]
emit.progress(f"Uploading bytes ended, id {upload_id}")
return upload_id
def _pack_bundle(self, parsed_args) -> List[pathlib.Path]:
"""Pack a bundle."""
emit.progress("Packing the bundle.")
if parsed_args.shell:
build.launch_shell()
return []
project = self.config.project
if self.config.parts:
config_parts = self.config.parts.copy()
else:
# "parts" not declared, create an implicit "bundle" part
config_parts = {"bundle": {"plugin": "bundle"}}
# a part named "bundle" using plugin "bundle" is special and has
# predefined values set automatically.
bundle_part = config_parts.get("bundle")
if bundle_part and bundle_part.get("plugin") == "bundle":
special_bundle_part = bundle_part
else:
special_bundle_part = None
# get the config files
bundle_filepath = project.dirpath / "bundle.yaml"
bundle_config = load_yaml(bundle_filepath)
if bundle_config is None:
raise CraftError(
"Missing or invalid main bundle file: {!r}.".format(
str(bundle_filepath)))
bundle_name = bundle_config.get("name")
if not bundle_name:
raise CraftError(
"Invalid bundle config; missing a 'name' field indicating the bundle's name in "
"file {!r}.".format(str(bundle_filepath)))
if special_bundle_part:
# set prime filters
for fname in MANDATORY_FILES:
fpath = project.dirpath / fname
if not fpath.exists():
raise CraftError("Missing mandatory file: {!r}.".format(
str(fpath)))
prime = special_bundle_part.setdefault("prime", [])
prime.extend(MANDATORY_FILES)
# set source if empty or not declared in charm part
if not special_bundle_part.get("source"):
special_bundle_part["source"] = str(project.dirpath)
if env.is_charmcraft_running_in_managed_mode():
work_dir = env.get_managed_environment_home_path()
else:
work_dir = project.dirpath / build.BUILD_DIRNAME
# run the parts lifecycle
emit.trace(f"Parts definition: {config_parts}")
lifecycle = parts.PartsLifecycle(
config_parts,
work_dir=work_dir,
project_dir=project.dirpath,
project_name=bundle_name,
ignore_local_sources=[bundle_name + ".zip"],
)
try:
lifecycle.run(Step.PRIME)
except (RuntimeError, CraftError) as error:
if parsed_args.debug:
emit.trace(f"Error when running PRIME step: {error}")
build.launch_shell()
raise
# pack everything
create_manifest(lifecycle.prime_dir, project.started_at, None, [])
zipname = project.dirpath / (bundle_name + ".zip")
build_zip(zipname, lifecycle.prime_dir)
emit.message(f"Created {str(zipname)!r}.")
if parsed_args.shell_after:
build.launch_shell()
return [zipname]
