def _get_manifest_list(self, image): """try to figure out manifest list""" if image in self.manifest_list_cache: return self.manifest_list_cache[image] manifest_list = get_manifest_list( image, image.registry, insecure=self.parent_registry_insecure) if '@sha256:' in str(image) and not manifest_list: # we want to adjust the tag only for manifest list fetching image = image.copy() try: config_blob = get_config_from_registry( image, image.registry, image.tag, insecure=self.parent_registry_insecure) except (HTTPError, RetryError, Timeout) as ex: self.log.warning('Unable to fetch config for %s, got error %s', image, ex.response.status_code) raise RuntimeError('Unable to fetch config for base image') release = config_blob['config']['Labels']['release'] version = config_blob['config']['Labels']['version'] docker_tag = "%s-%s" % (version, release) image.tag = docker_tag manifest_list = get_manifest_list( image, image.registry, insecure=self.parent_registry_insecure) self.manifest_list_cache[image] = manifest_list return self.manifest_list_cache[image]
def validate_platforms_in_base_image(self, base_image): expected_platforms = self.get_expected_platforms() if not expected_platforms: self.log.info('Skipping validation of available platforms ' 'because expected platforms are unknown') return if len(expected_platforms) == 1: self.log.info('Skipping validation of available platforms for base image ' 'because this is a single platform build') return if not base_image.registry: self.log.info('Cannot validate available platforms for base image ' 'because base image registry is not defined') return try: platform_to_arch = get_platform_to_goarch_mapping(self.workflow) except KeyError: self.log.info('Cannot validate available platforms for base image ' 'because platform descriptors are not defined') return if '@sha256:' in str(base_image): # we want to adjust the tag only for manifest list fetching base_image = base_image.copy() try: config_blob = get_config_from_registry(base_image, base_image.registry, base_image.tag, insecure=self.parent_registry_insecure) except (HTTPError, RetryError, Timeout) as ex: self.log.warning('Unable to fetch config for %s, got error %s', base_image, ex.response.status_code) raise RuntimeError('Unable to fetch config for base image') release = config_blob['config']['Labels']['release'] version = config_blob['config']['Labels']['version'] docker_tag = "%s-%s" % (version, release) base_image.tag = docker_tag manifest_list = get_manifest_list(base_image, base_image.registry, insecure=self.parent_registry_insecure) if not manifest_list: raise RuntimeError('Unable to fetch manifest list for base image') all_manifests = manifest_list.json()['manifests'] manifest_list_arches = set( manifest['platform']['architecture'] for manifest in all_manifests) expected_arches = set( platform_to_arch[platform] for platform in expected_platforms) self.log.info('Manifest list arches: %s, expected arches: %s', manifest_list_arches, expected_arches) assert manifest_list_arches >= expected_arches, \ 'Missing arches in manifest list for base image' self.log.info('Base image is a manifest list for all required platforms')
def run(self): pushed_images = [] if not self.workflow.tag_conf.unique_images: self.workflow.tag_conf.add_unique_image(self.workflow.image) config_manifest_digest = None config_manifest_type = None config_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError("Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry if self.need_skopeo_push(): self.push_with_skopeo(registry_image, insecure, docker_push_secret) else: self.tasker.tag_and_push_image(self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) defer_removal(self.workflow, registry_image) pushed_images.append(registry_image) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not config_manifest_digest and (digests.v2 or digests.oci): if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config_registry_image = registry_image if config_manifest_digest: push_conf_registry.config = get_config_from_registry( config_registry_image, registry, config_manifest_digest, insecure, docker_push_secret, config_manifest_type) else: self.log.info("V2 schema 2 or OCI manifest is not available to get config from") self.log.info("All images were tagged and pushed") return pushed_images
def run(self): pushed_images = [] if not self.workflow.tag_conf.unique_images: self.workflow.tag_conf.add_unique_image(self.workflow.image) first_v2_digest = None first_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError( "Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry logs = self.tasker.tag_and_push_image( self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) pushed_images.append(registry_image) defer_removal(self.workflow, registry_image) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not first_v2_digest and digests.v2: first_v2_digest = digests.v2 first_registry_image = registry_image if first_v2_digest: push_conf_registry.config = get_config_from_registry( first_registry_image, registry, first_v2_digest, insecure, docker_push_secret, 'v2') else: self.log.info("V2 schema 2 digest is not available") self.log.info("All images were tagged and pushed") return pushed_images
def run(self): pushed_images = [] if not self.workflow.tag_conf.unique_images: self.workflow.tag_conf.add_unique_image(self.workflow.image) first_v2_digest = None first_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError("Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry logs = self.tasker.tag_and_push_image(self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) pushed_images.append(registry_image) defer_removal(self.workflow, registry_image) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not first_v2_digest and digests.v2: first_v2_digest = digests.v2 first_registry_image = registry_image if first_v2_digest: push_conf_registry.config = get_config_from_registry( first_registry_image, registry, first_v2_digest, insecure, docker_push_secret, 'v2') else: self.log.info("V2 schema 2 digest is not available") self.log.info("All images were tagged and pushed") return pushed_images
def run(self): pushed_images = [] if not self.workflow.tag_conf.unique_images: self.workflow.tag_conf.add_unique_image(self.workflow.image) config_manifest_digest = None config_manifest_type = None config_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError("Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry max_retries = DOCKER_PUSH_MAX_RETRIES expect_v2s2 = False for registry in self.registries: media_types = self.registries[registry].get('expected_media_types', []) if MEDIA_TYPE_DOCKER_V2_SCHEMA2 in media_types: expect_v2s2 = True if not (self.group or expect_v2s2): max_retries = 0 for retry in range(max_retries + 1): if self.need_skopeo_push(): self.push_with_skopeo(registry_image, insecure, docker_push_secret) else: self.tasker.tag_and_push_image(self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) if (not (digests.v2 or digests.oci) and (retry < max_retries)): sleep_time = DOCKER_PUSH_BACKOFF_FACTOR * (2 ** retry) self.log.info("Retrying push because V2 schema 2 or " "OCI manifest not found in %is", sleep_time) time.sleep(sleep_time) else: if not self.need_skopeo_push(): defer_removal(self.workflow, registry_image) break pushed_images.append(registry_image) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not config_manifest_digest and (digests.v2 or digests.oci): if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config_registry_image = registry_image if config_manifest_digest: push_conf_registry.config = get_config_from_registry( config_registry_image, registry, config_manifest_digest, insecure, docker_push_secret, config_manifest_type) else: self.log.info("V2 schema 2 or OCI manifest is not available to get config from") self.log.info("All images were tagged and pushed") return pushed_images
def run(self): pushed_images = [] if not self.workflow.tag_conf.unique_images: self.workflow.tag_conf.add_unique_image(self.workflow.image) config_manifest_digest = None config_manifest_type = None config_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError( "Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry if self.need_skopeo_push(): self.push_with_skopeo(registry_image, insecure, docker_push_secret) else: self.tasker.tag_and_push_image( self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) defer_removal(self.workflow, registry_image) pushed_images.append(registry_image) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not config_manifest_digest and (digests.v2 or digests.oci): if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config_registry_image = registry_image if config_manifest_digest: push_conf_registry.config = get_config_from_registry( config_registry_image, registry, config_manifest_digest, insecure, docker_push_secret, config_manifest_type) else: self.log.info( "V2 schema 2 or OCI manifest is not available to get config from" ) self.log.info("All images were tagged and pushed") return pushed_images
def get_output(workflow: DockerBuildWorkflow, buildroot_id: str, pullspec: ImageName, platform: str, source_build: bool = False): """ Build the 'output' section of the metadata. :param buildroot_id: str, buildroot_id :param pullspec: ImageName :param platform: str, output platform :param source_build: bool, is source_build ? :param logs: list, of Output logs :return: tuple, list of Output instances, and extra Output file """ def add_buildroot_id(output: Output) -> Output: output.metadata.update({'buildroot_id': buildroot_id}) return output extra_output_file = None output_files: List[Output] = [] image_id: str if source_build: manifest = workflow.data.koji_source_manifest image_id = manifest['config']['digest'] # we are using digest from manifest, because we can't get diff_ids # unless we pull image, which would fail due because there are so many layers layer_sizes = [{ 'digest': layer['digest'], 'size': layer['size'] } for layer in manifest['layers']] platform = os.uname()[4] else: imageutil = workflow.imageutil image_id = imageutil.get_inspect_for_image(pullspec, platform=platform)['Id'] parent_id = None if not workflow.data.dockerfile_images.base_from_scratch: parent_id = imageutil.base_image_inspect(platform)['Id'] image_archive = str( workflow.build_dir.platform_dir(platform).exported_squashed_image) layer_sizes = imageutil.get_uncompressed_image_layer_sizes( image_archive) digests = get_manifest_digests(pullspec, workflow.conf.registry['uri'], workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None)) if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config = get_config_from_registry( pullspec, workflow.conf.registry['uri'], config_manifest_digest, workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None), config_manifest_type) # We don't need container_config section if config and 'container_config' in config: del config['container_config'] digest_pullspec = f"{pullspec.to_str(tag=False)}@{select_digest(digests)}" repositories = [pullspec.to_str(), digest_pullspec] typed_digests = { get_manifest_media_type(version): digest for version, digest in digests.items() if version != "v1" } tag_conf = workflow.data.tag_conf if source_build: image_type = IMAGE_TYPE_DOCKER_ARCHIVE tags = sorted(set(image.tag for image in tag_conf.images)) else: image_metadatas = workflow.data.postbuild_results[ FetchDockerArchivePlugin.key] image_type = image_metadatas[platform]["type"] tags = sorted( image.tag for image in tag_conf.get_unique_images_with_platform(platform)) metadata, output = get_image_output(image_type, image_id, platform, pullspec) metadata.update({ 'arch': platform, 'type': 'docker-image', 'components': [], 'extra': { 'image': { 'arch': platform, }, 'docker': { 'id': image_id, 'repositories': repositories, 'layer_sizes': layer_sizes, 'tags': tags, 'config': config, 'digests': typed_digests, }, }, }) if not config: del metadata['extra']['docker']['config'] if not source_build: metadata['components'] = get_image_components(workflow, platform) if not workflow.data.dockerfile_images.base_from_scratch: metadata['extra']['docker']['parent_id'] = parent_id # Add the 'docker save' image to the output image = add_buildroot_id(output) # when doing regular build, worker already uploads image, # so orchestrator needs only metadata, # but source contaiener build didn't upload that image yet, # so we want metadata, and the image to upload if source_build: output_files.append(metadata) extra_output_file = output else: output_files.append(image) if not source_build: # add operator manifests to output operator_manifests_path = (workflow.data.postbuild_results.get( PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY)) if operator_manifests_path: manifests_metadata = get_output_metadata( operator_manifests_path, OPERATOR_MANIFESTS_ARCHIVE) operator_manifests_output = Output( filename=operator_manifests_path, metadata=manifests_metadata) add_custom_type(operator_manifests_output, KOJI_BTYPE_OPERATOR_MANIFESTS) operator_manifests = add_buildroot_id(operator_manifests_output) output_files.append(operator_manifests) return output_files, extra_output_file
def run(self): pushed_images = [] source_oci_image_path = self.workflow.build_result.oci_image_path if source_oci_image_path: source_unique_image = self.source_get_unique_image() if not self.workflow.tag_conf.unique_images: if source_oci_image_path: self.workflow.tag_conf.add_unique_image(source_unique_image) else: self.workflow.tag_conf.add_unique_image(self.workflow.image) config_manifest_digest = None config_manifest_type = None config_registry_image = None for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError( "Image name must not contain registry: %r" % image.registry) registry_image = image.copy() registry_image.registry = registry max_retries = DOCKER_PUSH_MAX_RETRIES expect_v2s2 = False for registry in self.registries: media_types = self.registries[registry].get( 'expected_media_types', []) if MEDIA_TYPE_DOCKER_V2_SCHEMA2 in media_types: expect_v2s2 = True if not (self.group or expect_v2s2): max_retries = 0 for retry in range(max_retries + 1): if self.need_skopeo_push() or source_oci_image_path: self.push_with_skopeo(registry_image, insecure, docker_push_secret, source_oci_image_path) else: self.tasker.tag_and_push_image( self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) if source_oci_image_path: manifests_dict = get_all_manifests(registry_image, registry, insecure, docker_push_secret, versions=('v2', )) try: koji_source_manifest_response = manifests_dict[ 'v2'] except KeyError: raise RuntimeError( 'Unable to fetch v2 schema 2 digest for {}'. format(registry_image.to_str())) self.workflow.koji_source_manifest = koji_source_manifest_response.json( ) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) if (not (digests.v2 or digests.oci) and (retry < max_retries)): sleep_time = DOCKER_PUSH_BACKOFF_FACTOR * (2**retry) self.log.info( "Retrying push because V2 schema 2 or " "OCI manifest not found in %is", sleep_time) time.sleep(sleep_time) else: if not self.need_skopeo_push(): defer_removal(self.workflow, registry_image) break pushed_images.append(registry_image) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not config_manifest_digest and (digests.v2 or digests.oci): if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config_registry_image = registry_image if config_manifest_digest: push_conf_registry.config = get_config_from_registry( config_registry_image, registry, config_manifest_digest, insecure, docker_push_secret, config_manifest_type) else: self.log.info( "V2 schema 2 or OCI manifest is not available to get config from" ) self.log.info("All images were tagged and pushed") return pushed_images
def get_output(workflow: DockerBuildWorkflow, buildroot_id: str, pullspec: ImageName, platform: str, source_build: bool = False): """ Build the 'output' section of the metadata. :param buildroot_id: str, buildroot_id :param pullspec: ImageName :param platform: str, output platform :param source_build: bool, is source_build ? :param logs: list, of Output logs :return: tuple, list of Output instances, and extra Output file """ def add_buildroot_id(output: Output) -> Output: output.metadata.update({'buildroot_id': buildroot_id}) return output extra_output_file = None output_files: List[Output] = [] image_id: str if source_build: manifest = workflow.data.koji_source_manifest image_id = manifest['config']['digest'] # we are using digest from manifest, because we can't get diff_ids # unless we pull image, which would fail due because there are so many layers layer_sizes = [{ 'digest': layer['digest'], 'size': layer['size'] } for layer in manifest['layers']] platform = os.uname()[4] else: imageutil = workflow.imageutil image_id = imageutil.get_inspect_for_image(pullspec, platform=platform)['Id'] inspect = imageutil.base_image_inspect(platform) parent_id = inspect['Id'] if inspect else None image_archive = str( workflow.build_dir.platform_dir(platform).exported_squashed_image) layer_sizes = imageutil.get_uncompressed_image_layer_sizes( image_archive) digests = get_manifest_digests(pullspec, workflow.conf.registry['uri'], workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None)) if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config = get_config_from_registry( pullspec, workflow.conf.registry['uri'], config_manifest_digest, workflow.conf.registry['insecure'], workflow.conf.registry.get('secret', None), config_manifest_type) # We don't need container_config section if config and 'container_config' in config: del config['container_config'] digest_pullspec = f"{pullspec.to_str(tag=False)}@{select_digest(digests)}" repositories = [pullspec.to_str(), digest_pullspec] typed_digests = { get_manifest_media_type(version): digest for version, digest in digests.items() if version != "v1" } tag_conf = workflow.data.tag_conf if source_build: tags = sorted(set(image.tag for image in tag_conf.images)) else: tags = sorted( image.tag for image in tag_conf.get_unique_images_with_platform(platform)) # since we are storing oci image as v2s2 all images now have 'docker-archive' type metadata, output = get_image_output(IMAGE_TYPE_DOCKER_ARCHIVE, image_id, platform, pullspec) metadata.update({ 'arch': platform, 'type': 'docker-image', 'components': [], 'extra': { 'image': { 'arch': platform, }, 'docker': { 'id': image_id, 'repositories': repositories, 'layer_sizes': layer_sizes, 'tags': tags, 'config': config, 'digests': typed_digests, }, }, }) if not config: del metadata['extra']['docker']['config'] if not source_build: metadata['components'] = get_image_components(workflow.data, platform) if parent_id is not None: metadata['extra']['docker']['parent_id'] = parent_id # Add the 'docker save' image to the output image = add_buildroot_id(output) if source_build: output_files.append(metadata) extra_output_file = output else: output_files.append(image) return output_files, extra_output_file
def run(self): pushed_images = [] source_oci_image_path = self.workflow.build_result.oci_image_path if source_oci_image_path: source_unique_image = self.source_get_unique_image() if not self.workflow.tag_conf.unique_images: if source_oci_image_path: self.workflow.tag_conf.add_unique_image(source_unique_image) else: self.workflow.tag_conf.add_unique_image(self.workflow.image) config_manifest_digest = None config_manifest_type = None config_registry_image = None image_size_limit = get_image_size_limit(self.workflow) for registry, registry_conf in self.registries.items(): insecure = registry_conf.get('insecure', False) push_conf_registry = \ self.workflow.push_conf.add_docker_registry(registry, insecure=insecure) docker_push_secret = registry_conf.get('secret', None) self.log.info("Registry %s secret %s", registry, docker_push_secret) for image in self.workflow.tag_conf.images: if image.registry: raise RuntimeError( "Image name must not contain registry: %r" % image.registry) if not source_oci_image_path: image_size = sum(item['size'] for item in self.workflow.layer_sizes) config_image_size = image_size_limit['binary_image'] # Only handle the case when size is set > 0 in config if config_image_size and image_size > config_image_size: raise ExceedsImageSizeError( 'The size {} of image {} exceeds the limitation {} ' 'configured in reactor config.'.format( image_size, image, image_size_limit)) registry_image = image.copy() registry_image.registry = registry max_retries = DOCKER_PUSH_MAX_RETRIES for retry in range(max_retries + 1): if self.need_skopeo_push() or source_oci_image_path: self.push_with_skopeo(registry_image, insecure, docker_push_secret, source_oci_image_path) else: self.tasker.tag_and_push_image( self.workflow.builder.image_id, registry_image, insecure=insecure, force=True, dockercfg=docker_push_secret) if source_oci_image_path: manifests_dict = get_all_manifests(registry_image, registry, insecure, docker_push_secret, versions=('v2', )) try: koji_source_manifest_response = manifests_dict[ 'v2'] except KeyError as exc: raise RuntimeError( f'Unable to fetch v2 schema 2 digest for {registry_image.to_str()}' ) from exc self.workflow.koji_source_manifest = koji_source_manifest_response.json( ) digests = get_manifest_digests(registry_image, registry, insecure, docker_push_secret) if (not (digests.v2 or digests.oci) and (retry < max_retries)): sleep_time = DOCKER_PUSH_BACKOFF_FACTOR * (2**retry) self.log.info( "Retrying push because V2 schema 2 or " "OCI manifest not found in %is", sleep_time) time.sleep(sleep_time) else: if not self.need_skopeo_push(): defer_removal(self.workflow, registry_image) break pushed_images.append(registry_image) tag = registry_image.to_str(registry=False) push_conf_registry.digests[tag] = digests if not config_manifest_digest and (digests.v2 or digests.oci): if digests.v2: config_manifest_digest = digests.v2 config_manifest_type = 'v2' else: config_manifest_digest = digests.oci config_manifest_type = 'oci' config_registry_image = registry_image if config_manifest_digest: push_conf_registry.config = get_config_from_registry( config_registry_image, registry, config_manifest_digest, insecure, docker_push_secret, config_manifest_type) else: self.log.info( "V2 schema 2 or OCI manifest is not available to get config from" ) self.log.info("All images were tagged and pushed") return pushed_images