def annotate_snapcraft(data, parts_dir: str):
manifest = OrderedDict() # type: Dict[str, Any]
manifest["snapcraft-version"] = snapcraft._get_version()
release = os_release.OsRelease()
with contextlib.suppress(errors.OsReleaseIdError):
manifest["snapcraft-os-release-id"] = release.id()
with contextlib.suppress(errors.OsReleaseVersionIdError):
manifest["snapcraft-os-release-version-id"] = release.version_id()
for k, v in data.items():
manifest[k] = v
image_info = os.environ.get("SNAPCRAFT_IMAGE_INFO")
if image_info:
try:
image_info_dict = json.loads(image_info)
except json.decoder.JSONDecodeError as exception:
raise errors.InvalidContainerImageInfoError(
image_info) from exception
manifest["image-info"] = image_info_dict
for field in ("build-packages", "build-snaps"):
manifest[field] = get_global_state().assets.get(field, [])
for part in data["parts"]:
state_dir = os.path.join(parts_dir, part, "state")
pull_state = get_state(state_dir, steps.PULL)
manifest["parts"][part]["build-packages"] = pull_state.assets.get(
"build-packages", [])
manifest["parts"][part]["stage-packages"] = pull_state.assets.get(
"stage-packages", [])
source_details = pull_state.assets.get("source-details", {})
if source_details:
manifest["parts"][part].update(source_details)
build_state = get_state(state_dir, steps.BUILD)
manifest["parts"][part].update(build_state.assets)
return manifest
def annotate_snapcraft(project: "Project", data: Dict[str,
Any]) -> Dict[str, Any]:
manifest = OrderedDict() # type: Dict[str, Any]
manifest["snapcraft-version"] = snapcraft._get_version()
manifest["snapcraft-started-at"] = project._get_start_time().isoformat(
) + "Z"
release = os_release.OsRelease()
with contextlib.suppress(errors.OsReleaseIdError):
manifest["snapcraft-os-release-id"] = release.id()
with contextlib.suppress(errors.OsReleaseVersionIdError):
manifest["snapcraft-os-release-version-id"] = release.version_id()
for k, v in data.items():
manifest[k] = v
image_info = os.environ.get("SNAPCRAFT_IMAGE_INFO")
if image_info:
try:
image_info_dict = json.loads(image_info)
except json.decoder.JSONDecodeError as exception:
raise errors.InvalidContainerImageInfoError(
image_info) from exception
manifest["image-info"] = image_info_dict
global_state = GlobalState.load(
filepath=project._get_global_state_file_path())
manifest["build-packages"] = sorted(global_state.get_build_packages())
manifest["build-snaps"] = sorted(global_state.get_build_snaps())
for part in data["parts"]:
state_dir = os.path.join(project.parts_dir, part, "state")
pull_state = get_state(state_dir, steps.PULL)
manifest["parts"][part]["build-packages"] = sorted(
pull_state.assets.get("build-packages", []))
manifest["parts"][part]["stage-packages"] = sorted(
pull_state.assets.get("stage-packages", []))
source_details = pull_state.assets.get("source-details", {})
if source_details:
manifest["parts"][part].update(source_details)
build_state = get_state(state_dir, steps.BUILD)
manifest["parts"][part].update(build_state.assets)
# Assemble all primed stage packages into a single list...
primed_stage_packages: Set[str] = set()
for part in data["parts"]:
state_dir = os.path.join(project.parts_dir, part, "state")
prime_state = get_state(state_dir, steps.PRIME)
primed_stage_packages |= prime_state.primed_stage_packages
manifest["primed-stage-packages"] = sorted(primed_stage_packages)
return manifest
def get_dirty_report(self, step):
"""Return a DirtyReport class describing why step is dirty.
Returns None if step is not dirty.
"""
# Retrieve the stored state for this step (assuming it has already run)
state = states.get_state(self.plugin.statedir, step)
differing_properties = set()
differing_options = set()
with contextlib.suppress(AttributeError):
# state.properties contains the old YAML that this step cares
# about, and we're comparing it to those same keys in the current
# YAML (self._part_properties). If they've changed, then this step
# is dirty and needs to run again.
differing_properties = state.diff_properties_of_interest(
self._part_properties)
with contextlib.suppress(AttributeError):
# state.project_options contains the old project options that this
# step cares about, and we're comparing it to those same options in
# the current project. If they've changed, then this step is dirty
# and needs to run again.
differing_options = state.diff_project_options_of_interest(
self._project_options)
if differing_properties or differing_options:
return DirtyReport(differing_properties, differing_options)
return None
def get_dirty_report(self, step):
"""Return a DirtyReport class describing why step is dirty.
Returns None if step is not dirty.
"""
# Retrieve the stored state for this step (assuming it has already run)
state = states.get_state(self.plugin.statedir, step)
differing_properties = set()
differing_options = set()
with contextlib.suppress(AttributeError):
# state.properties contains the old YAML that this step cares
# about, and we're comparing it to those same keys in the current
# YAML (self._part_properties). If they've changed, then this step
# is dirty and needs to run again.
differing_properties = state.diff_properties_of_interest(
self._part_properties)
with contextlib.suppress(AttributeError):
# state.project_options contains the old project options that this
# step cares about, and we're comparing it to those same options in
# the current project. If they've changed, then this step is dirty
# and needs to run again.
differing_options = state.diff_project_options_of_interest(
self._project_options)
if differing_properties or differing_options:
return DirtyReport(differing_properties, differing_options)
return None
def get_dirty_report(self, step):
"""Return a DirtyReport class describing why step is dirty.
Returns None if step is not dirty.
"""
# Retrieve the stored state for this step (assuming it has already run)
state = states.get_state(self.plugin.statedir, step)
if state:
# state.properties contains the old YAML that this step cares
# about, and we're comparing it to those same keys in the current
# YAML (self._part_properties). If they've changed, then this step
# is dirty and needs to run again.
properties = state.diff_properties_of_interest(
self._part_properties)
# state.project_options contains the old project options that this
# step cares about, and we're comparing it to those same options in
# the current project. If they've changed, then this step is dirty
# and needs to run again.
options = state.diff_project_options_of_interest(
self._project_options)
# If dependencies have changed since this step ran, then it's dirty
# and needs to run again.
dependencies = state.changed_dependencies
if properties or options or dependencies:
return DirtyReport(properties, options, dependencies)
return None
def _annotate_snapcraft(self, data):
for field in ('build-packages', 'build-snaps'):
data[field] = get_global_state().assets.get(field, [])
for part in data['parts']:
state_dir = os.path.join(self._parts_dir, part, 'state')
pull_state = get_state(state_dir, 'pull')
data['parts'][part]['build-packages'] = (pull_state.assets.get(
'build-packages', []))
data['parts'][part]['stage-packages'] = (pull_state.assets.get(
'stage-packages', []))
source_details = pull_state.assets.get('source-details', {})
if source_details:
data['parts'][part].update(source_details)
build_state = get_state(state_dir, 'build')
data['parts'][part].update(build_state.assets)
return data
def _set_scriptlet_metadata(
self, metadata: snapcraft.extractors.ExtractedMetadata):
step = self.next_step()
# First, ensure the metadata set here doesn't conflict with metadata
# already set for this step
conflicts = metadata.overlap(self._scriptlet_metadata[step])
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(step, step,
list(conflicts))
last_step = self.last_step()
if last_step:
# Now ensure the metadata from this step doesn't conflict with
# metadata from any other step
index = common.COMMAND_ORDER.index(last_step)
for index in reversed(range(0, index + 1)):
other_step = common.COMMAND_ORDER[index]
state = states.get_state(self.plugin.statedir, other_step)
conflicts = metadata.overlap(state.scriptlet_metadata)
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(
step, other_step, list(conflicts))
self._scriptlet_metadata[step].update(metadata)
def _annotate_snapcraft(self, data):
for field in ('build-packages', 'build-snaps'):
data[field] = get_global_state().assets.get(field, [])
for part in data['parts']:
state_dir = os.path.join(self._parts_dir, part, 'state')
pull_state = get_state(state_dir, 'pull')
data['parts'][part]['build-packages'] = (
pull_state.assets.get('build-packages', []))
data['parts'][part]['stage-packages'] = (
pull_state.assets.get('stage-packages', []))
source_details = pull_state.assets.get('source-details', {})
if source_details:
data['parts'][part].update(source_details)
build_state = get_state(state_dir, 'build')
data['parts'][part].update(build_state.assets)
return data
def get_project_state(self, step):
"""Returns a dict of states for the given step of each part."""
state = {}
for part in self.parts.all_parts:
state[part.name] = states.get_state(part.plugin.statedir, step)
return state
def get_project_state(self, step: steps.Step):
"""Returns a dict of states for the given step of each part."""
state = {}
for part in self.parts.all_parts:
state[part.name] = states.get_state(part.plugin.statedir, step)
return state
def get_primed_dependency_paths(self):
dependency_paths = set()
state = states.get_state(self.plugin.statedir, 'prime')
if state:
for path in state.dependency_paths:
dependency_paths.add(
os.path.join(self.primedir, path.lstrip('/')))
return dependency_paths
def _annotate_snapcraft(self, data):
for part in data['parts']:
pull_state = get_state(
os.path.join(self._parts_dir, part, 'state'), 'pull')
data['parts'][part]['build-packages'] = (pull_state.assets.get(
'build-packages', []))
data['parts'][part]['stage-packages'] = (pull_state.assets.get(
'stage-packages', []))
return data
def get_primed_dependency_paths(self):
dependency_paths = set()
state = states.get_state(self.plugin.statedir, 'prime')
if state:
for path in state.dependency_paths:
dependency_paths.add(
os.path.join(self.primedir, path.lstrip('/')))
return dependency_paths
def clean_stage(self, project_staged_state):
if self.is_clean(steps.STAGE):
return
state = states.get_state(self.plugin.statedir, steps.STAGE)
try:
self._clean_shared_area(self.stagedir, state, project_staged_state)
except AttributeError:
raise errors.MissingStateCleanError(steps.STAGE)
self.mark_cleaned(steps.STAGE)
def clean_stage(self, project_staged_state):
if self.is_clean(steps.STAGE):
return
state = states.get_state(self.plugin.statedir, steps.STAGE)
try:
self._clean_shared_area(self.stagedir, state, project_staged_state)
except AttributeError:
raise errors.MissingStateCleanError(steps.STAGE)
self.mark_cleaned(steps.STAGE)
def annotate_snapcraft(data, parts_dir: str, global_state_path: str):
manifest = OrderedDict() # type: Dict[str, Any]
manifest["snapcraft-version"] = snapcraft._get_version()
release = os_release.OsRelease()
with contextlib.suppress(errors.OsReleaseIdError):
manifest["snapcraft-os-release-id"] = release.id()
with contextlib.suppress(errors.OsReleaseVersionIdError):
manifest["snapcraft-os-release-version-id"] = release.version_id()
for k, v in data.items():
manifest[k] = v
image_info = os.environ.get("SNAPCRAFT_IMAGE_INFO")
if image_info:
try:
image_info_dict = json.loads(image_info)
except json.decoder.JSONDecodeError as exception:
raise errors.InvalidContainerImageInfoError(image_info) from exception
manifest["image-info"] = image_info_dict
global_state = GlobalState.load(filepath=global_state_path)
manifest["build-packages"] = global_state.get_build_packages()
manifest["build-snaps"] = global_state.get_build_snaps()
for part in data["parts"]:
state_dir = os.path.join(parts_dir, part, "state")
pull_state = get_state(state_dir, steps.PULL)
manifest["parts"][part]["build-packages"] = pull_state.assets.get(
"build-packages", []
)
manifest["parts"][part]["stage-packages"] = pull_state.assets.get(
"stage-packages", []
)
source_details = pull_state.assets.get("source-details", {})
if source_details:
manifest["parts"][part].update(source_details)
build_state = get_state(state_dir, steps.BUILD)
manifest["parts"][part].update(build_state.assets)
return manifest
def _annotate_snapcraft(self, data):
data['build-packages'] = repo.Repo.get_installed_build_packages(
data.get('build-packages', []))
for part in data['parts']:
pull_state = get_state(
os.path.join(self._parts_dir, part, 'state'), 'pull')
data['parts'][part]['build-packages'] = (pull_state.assets.get(
'build-packages', []))
data['parts'][part]['stage-packages'] = (pull_state.assets.get(
'stage-packages', []))
source_details = pull_state.assets.get('source-details', {})
if source_details:
data['parts'][part].update(source_details)
return data
def annotate_snapcraft(data, parts_dir: str):
image_info = os.environ.get('SNAPCRAFT_IMAGE_INFO')
if image_info:
try:
image_info_dict = json.loads(image_info)
except json.decoder.JSONDecodeError as exception:
raise errors.InvalidContainerImageInfoError(
image_info) from exception
data['image-info'] = image_info_dict
for field in ('build-packages', 'build-snaps'):
data[field] = get_global_state().assets.get(field, [])
for part in data['parts']:
state_dir = os.path.join(parts_dir, part, 'state')
pull_state = get_state(state_dir, 'pull')
data['parts'][part]['build-packages'] = (pull_state.assets.get(
'build-packages', []))
data['parts'][part]['stage-packages'] = (pull_state.assets.get(
'stage-packages', []))
source_details = pull_state.assets.get('source-details', {})
if source_details:
data['parts'][part].update(source_details)
build_state = get_state(state_dir, 'build')
data['parts'][part].update(build_state.assets)
return data
def mark_dependency_change(self, dependency_name, changed_step):
if changed_step <= steps.STAGE:
dirty_step = steps.next_step(None)
else:
dirty_step = changed_step
state = states.get_state(self.plugin.statedir, dirty_step)
if state:
state.changed_dependencies.append({
'name': dependency_name,
'step': changed_step.name
})
self.mark_done(dirty_step, state)
return True
return False
def clean_prime(self, project_primed_state, hint=''):
if self.is_clean('prime'):
hint = '{} {}'.format(hint, '(already clean)').strip()
self.notify_part_progress('Skipping cleaning priming area for',
hint)
return
self.notify_part_progress('Cleaning priming area for', hint)
state = states.get_state(self.plugin.statedir, 'prime')
try:
self._clean_shared_area(self.primedir, state, project_primed_state)
except AttributeError:
raise errors.MissingStateCleanError('prime')
self.mark_cleaned('prime')
def clean_prime(self, project_primed_state, hint=''):
if self.is_clean('prime'):
hint = '{} {}'.format(hint, '(already clean)').strip()
self.notify_part_progress('Skipping cleaning priming area for',
hint)
return
self.notify_part_progress('Cleaning priming area for', hint)
state = states.get_state(self.plugin.statedir, 'prime')
try:
self._clean_shared_area(self.primedir, state,
project_primed_state)
except AttributeError:
raise errors.MissingStateCleanError('prime')
self.mark_cleaned('prime')
def clean_prime(self, project_primed_state, hint=''):
if self.is_clean('prime'):
hint = '{} {}'.format(hint, '(already clean)').strip()
self.notify_part_progress('Skipping cleaning priming area for',
hint)
return
self.notify_part_progress('Cleaning priming area for', hint)
state = states.get_state(self.statedir, 'prime')
try:
self._clean_shared_area(self.primedir, state, project_primed_state)
except AttributeError:
raise errors.MissingState(
"Failed to clean step 'prime': Missing necessary state. "
"This won't work until a complete clean has occurred.")
self.mark_cleaned('prime')
def _set_scriptlet_metadata(self, metadata: snapcraft.extractors.ExtractedMetadata):
step = self.next_step()
# First, ensure the metadata set here doesn't conflict with metadata
# already set for this step
conflicts = metadata.overlap(self._scriptlet_metadata[step])
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(step, step, list(conflicts))
# Now ensure the metadata from this step doesn't conflict with
# metadata from any other step (if any)
with contextlib.suppress(errors.NoLatestStepError):
latest_step = self.latest_step()
required_steps = latest_step.previous_steps() + [latest_step]
for other_step in reversed(required_steps):
state = states.get_state(self.plugin.statedir, other_step)
conflicts = metadata.overlap(state.scriptlet_metadata)
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(
step, other_step, list(conflicts)
)
self._scriptlet_metadata[step].update(metadata)
def _set_scriptlet_metadata(self, metadata: snapcraft.extractors.ExtractedMetadata):
step = self.next_step()
# First, ensure the metadata set here doesn't conflict with metadata
# already set for this step
conflicts = metadata.overlap(self._scriptlet_metadata[step])
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(step, step, list(conflicts))
# Now ensure the metadata from this step doesn't conflict with
# metadata from any other step (if any)
with contextlib.suppress(errors.NoLatestStepError):
latest_step = self.latest_step()
required_steps = latest_step.previous_steps() + [latest_step]
for other_step in reversed(required_steps):
state = states.get_state(self.plugin.statedir, other_step)
conflicts = metadata.overlap(state.scriptlet_metadata)
if len(conflicts) > 0:
raise errors.ScriptletDuplicateDataError(
step, other_step, list(conflicts)
)
self._scriptlet_metadata[step].update(metadata)
def get_dirty_report(self, step: steps.Step) -> DirtyReport:
"""Return a DirtyReport class describing why the step is dirty.
A step is considered to be dirty if either YAML properties used by it
(`stage-packages` are used by the `pull` step, for example), or project
options used by it (`--target-arch` is used by the `pull` step as well)
have changed since the step was run. This means the step needs to be
cleaned and run again. This is in contrast to an "outdated" step, which
typically doesn't need to be cleaned, just updated with files from an
earlier step in the lifecycle.
:param steps.Step step: The step to be checked.
:returns: DirtyReport if the step is dirty, None otherwise.
"""
# Retrieve the stored state for this step (assuming it has already run)
state = states.get_state(self.plugin.statedir, step)
if state:
# state.properties contains the old YAML that this step cares
# about, and we're comparing it to those same keys in the current
# YAML (self._part_properties). If they've changed, then this step
# is dirty and needs to run again.
properties = state.diff_properties_of_interest(
self._part_properties)
# state.project_options contains the old project options that this
# step cares about, and we're comparing it to those same options in
# the current project. If they've changed, then this step is dirty
# and needs to run again.
options = state.diff_project_options_of_interest(
self._project_options)
if properties or options:
return DirtyReport(dirty_properties=properties,
dirty_project_options=options)
return None
def get_dirty_report(self, step: steps.Step) -> DirtyReport:
"""Return a DirtyReport class describing why the step is dirty.
A step is considered to be dirty if either YAML properties used by it
(`stage-packages` are used by the `pull` step, for example), or project
options used by it (`--target-arch` is used by the `pull` step as well)
have changed since the step was run. This means the step needs to be
cleaned and run again. This is in contrast to an "outdated" step, which
typically doesn't need to be cleaned, just updated with files from an
earlier step in the lifecycle.
:param steps.Step step: The step to be checked.
:returns: DirtyReport if the step is dirty, None otherwise.
"""
# Retrieve the stored state for this step (assuming it has already run)
state = states.get_state(self.plugin.statedir, step)
if state:
# state.properties contains the old YAML that this step cares
# about, and we're comparing it to those same keys in the current
# YAML (self._part_properties). If they've changed, then this step
# is dirty and needs to run again.
properties = state.diff_properties_of_interest(self._part_properties)
# state.project_options contains the old project options that this
# step cares about, and we're comparing it to those same options in
# the current project. If they've changed, then this step is dirty
# and needs to run again.
options = state.diff_project_options_of_interest(self._project_options)
if properties or options:
return DirtyReport(
dirty_properties=properties, dirty_project_options=options
)
return None
def get_stage_state(self) -> states.StageState:
if not self._stage_state:
self._stage_state = states.get_state(self.plugin.statedir, steps.STAGE)
return self._stage_state
def get_prime_state(self) -> states.PrimeState:
if not self._prime_state:
self._prime_state = states.get_state(self.plugin.statedir, 'prime')
return self._prime_state
def get_stage_state(self) -> states.StageState:
if not self._stage_state:
self._stage_state = states.get_state(self.plugin.statedir, 'stage')
return self._stage_state
def get_build_state(self) -> states.BuildState:
if not self._build_state:
self._build_state = states.get_state(self.plugin.statedir, 'build')
return self._build_state
def get_prime_state(self) -> states.PrimeState:
if not self._prime_state:
self._prime_state = states.get_state(self.plugin.statedir, steps.PRIME)
return self._prime_state
def get_stage_state(self) -> states.StageState:
if not self._stage_state:
self._stage_state = states.get_state(self.plugin.statedir,
steps.STAGE)
return self._stage_state
def get_state(self, step) -> states.PartState:
return states.get_state(self.plugin.statedir, step)
def get_build_state(self) -> states.BuildState:
if not self._build_state:
self._build_state = states.get_state(self.plugin.statedir, steps.BUILD)
return self._build_state
def get_pull_state(self) -> states.PullState:
if not self._pull_state:
self._pull_state = states.get_state(self.plugin.statedir, 'pull')
return self._pull_state
def get_pull_state(self) -> states.PullState:
if not self._pull_state:
self._pull_state = states.get_state(self.plugin.statedir, steps.PULL)
return self._pull_state