def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) if not parsed_args.yes: confirm = utils.prompt_user_for_confirmation( message=_("Are you sure you want to delete this image " "[y/N]? "), logger=self.log) if not confirm: raise oscexc.CommandError("Action not confirmed, exiting.") lock = processlock.ProcessLock() manager = image_uploader.ImageUploadManager(lock=lock) uploader = manager.uploader('python') registry_url_arg = parsed_args.registry_url if registry_url_arg is None: registry_url_arg = image_uploader.get_undercloud_registry() url = uploader._image_to_url(registry_url_arg) session = uploader.authenticate(url, parsed_args.username, parsed_args.password) try: uploader.delete(parsed_args.image_to_delete, session=session) except OSError as e: self.log.error("Unable to remove due to permissions. " "Please prefix command with sudo.") raise oscexc.CommandError(e)
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) uploader = image_uploader.ImageUploadManager(parsed_args.config_files) try: uploader.upload() except KeyboardInterrupt: # ctrl-c self.log.warning('Upload was interrupted by ctrl-c.')
def test_file_parsing(self, mock_gur, mockdocker, mockioctl, mockpath, mock_images_match, mock_is_insecure): manager = image_uploader.ImageUploadManager(self.filelist, debug=True) parsed_data = manager.upload() mockpath(self.filelist[0]) expected_data = fakes.create_parsed_upload_images() sorted_expected_data = sorted(expected_data, key=operator.itemgetter('imagename')) sorted_parsed_data = sorted(parsed_data, key=operator.itemgetter('imagename')) self.assertEqual(sorted_expected_data, sorted_parsed_data) dockerc = mockdocker.return_value dockerc.remove_image.assert_has_calls([ mock.call('192.0.2.0:8787/tripleomaster' '/centos-binary-nova-libvirt:liberty'), mock.call('docker.io/tripleomaster' '/centos-binary-nova-compute:liberty'), mock.call('docker.io/tripleomaster' '/centos-binary-nova-libvirt:liberty'), mock.call('docker.io/tripleomaster' '/heat-docker-agents-centos:latest'), mock.call('docker.io/tripleomaster' '/image-with-missing-tag:latest'), mock.call('localhost:8787/tripleomaster' '/centos-binary-nova-compute:liberty'), mock.call('localhost:8787/tripleomaster' '/heat-docker-agents-centos:latest'), mock.call('localhost:8787/tripleomaster/' 'image-with-missing-tag:latest'), ])
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) uploader = image_uploader.ImageUploadManager([]) print( uploader.discover_image_tag( image=parsed_args.image, tag_from_label=parsed_args.tag_from_label))
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) manager = image_uploader.ImageUploadManager() uploader = manager.uploader('python') source_image = parsed_args.image_to_push if parsed_args.local or source_image.startswith('containers-storage:'): storage = 'containers-storage:' if not source_image.startswith(storage): source_image = storage + source_image.replace('docker://', '') elif not parsed_args.local: self.log.warning('Assuming local container based on provided ' 'container path. (e.g. starts with ' 'containers-storage:)') source_url = parse.urlparse(source_image) image_name = source_url.geturl() image_source = None else: storage = 'docker://' if not source_image.startswith(storage): source_image = storage + source_image source_url = parse.urlparse(source_image) image_source = source_url.netloc image_name = source_url.path[1:] if len(image_name.split('/')) != 2: raise exceptions.DownloadError('Invalid container. Provided ' 'container image should be ' '<registry>/<namespace>/<name>:' '<tag>') registry_url = parsed_args.registry_url if not registry_url.startswith('docker://'): registry_url = 'docker://%s' % registry_url reg_url = parse.urlparse(registry_url) uploader.authenticate(reg_url, parsed_args.username, parsed_args.password) task = image_uploader.UploadTask( image_name=image_name, pull_source=image_source, push_destination=parsed_args.registry_url, append_tag=parsed_args.append_tag, modify_role=None, modify_vars=None, dry_run=parsed_args.dry_run, cleanup=parsed_args.cleanup, multi_arch=parsed_args.multi_arch) try: uploader.add_upload_task(task) uploader.run_tasks() except OSError as e: self.log.error("Unable to upload due to permissions. " "Please prefix command with sudo.") raise oscexc.CommandError(e)
def detect_insecure_registries(params): insecure = set() uploader = image_uploader.ImageUploadManager().uploader('docker') for image in params.values(): host = image.split('/')[0] if uploader.is_insecure_registry(host): insecure.add(host) if not insecure: return {} return {'DockerInsecureRegistryAddress': sorted(insecure)}
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) self.log.warning("[DEPRECATED] This command has been deprecated and " "replaced by the 'openstack tripleo container image " "prepare' command.") uploader = image_uploader.ImageUploadManager([]) print( uploader.discover_image_tag( image=parsed_args.image, tag_from_label=parsed_args.tag_from_label))
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) if parsed_args.cleanup not in image_uploader.CLEANUP: raise oscexc.CommandError('--cleanup must be one of: %s' % ', '.join(image_uploader.CLEANUP)) uploader = image_uploader.ImageUploadManager( parsed_args.config_files, cleanup=parsed_args.cleanup) try: uploader.upload() except KeyboardInterrupt: # ctrl-c self.log.warning('Upload was interrupted by ctrl-c.')
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) manager = image_uploader.ImageUploadManager() uploader = manager.uploader('python') url = uploader._image_to_url(parsed_args.registry_url) session = uploader.authenticate(url, parsed_args.username, parsed_args.password) results = uploader.list(url.geturl(), session=session) cliff_results = [] for r in results: cliff_results.append((r, )) return (("Image Name", ), cliff_results)
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) manager = image_uploader.ImageUploadManager() uploader = manager.uploader('python') url = uploader._image_to_url(parsed_args.registry_url) session = uploader.authenticate(url, parsed_args.username, parsed_args.password) image_inspect_result = uploader.inspect(parsed_args.image_to_inspect, session=session) return self.format_image_inspect(image_inspect_result)
def detect_insecure_registries(params): """Detect insecure registries in image parameters :param params: dict of container image parameters :returns: dict containing DockerInsecureRegistryAddress parameter to be merged into other parameters """ insecure = set() uploader = image_uploader.ImageUploadManager().uploader('python') for image in params.values(): host = image.split('/')[0] if uploader.is_insecure_registry(host): insecure.add(host) if not insecure: return {} return {'DockerInsecureRegistryAddress': sorted(insecure)}
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) lock = processlock.ProcessLock() manager = image_uploader.ImageUploadManager(lock=lock) uploader = manager.uploader('python') registry_url_arg = parsed_args.registry_url if registry_url_arg is None: registry_url_arg = image_uploader.get_undercloud_registry() url = uploader._image_to_url(registry_url_arg) session = uploader.authenticate(url, parsed_args.username, parsed_args.password) results = uploader.list(url.geturl(), session=session) cliff_results = [] for r in results: cliff_results.append((r,)) return (("Image Name",), cliff_results)
def test_get_push_destination(self, mock_interfaces, mock_addresses): mock_interfaces.return_value = ['lo', 'eth0', 'br-ctlplane'] mock_addresses.return_value = {2: [{'addr': '192.0.2.0'}]} manager = image_uploader.ImageUploadManager(self.filelist, debug=True) self.assertEqual('192.0.2.0:8787', manager.get_push_destination({})) self.assertEqual( '192.0.2.1:8787', manager.get_push_destination( {'push_destination': '192.0.2.1:8787'})) self.assertEqual( '192.0.2.0:8787', manager.get_push_destination({'push_destination': False})) self.assertEqual( '192.0.2.0:8787', manager.get_push_destination({'push_destination': True})) self.assertEqual( '192.0.2.0:8787', manager.get_push_destination({'push_destination': None}))
def container_images_prepare_multi(environment, roles_data, dry_run=False, cleanup=image_uploader.CLEANUP_FULL, lock=None): """Perform multiple container image prepares and merge result Given the full heat environment and roles data, perform multiple image prepare operations. The data to drive the multiple prepares is taken from the ContainerImagePrepare parameter in the provided environment. If push_destination is specified, uploads will be performed during the preparation. :param environment: Heat environment for deployment :param roles_data: Roles file data used to filter services :param lock: a locking object to use when handling uploads :returns: dict containing merged container image parameters from all prepare operations """ if not lock: lock = threadinglock.ThreadingLock() pd = environment.get('parameter_defaults', {}) cip = pd.get('ContainerImagePrepare') # if user does not provide a ContainerImagePrepare, use the defaults. if not cip: LOG.info( _("No ContainerImagePrepare parameter defined. Using " "the defaults.")) cip = CONTAINER_IMAGE_PREPARE_PARAM mirrors = {} mirror = pd.get('DockerRegistryMirror') if mirror: mirrors['docker.io'] = mirror creds = pd.get('ContainerImageRegistryCredentials') multi_arch = len(pd.get('AdditionalArchitectures', [])) env_params = {} service_filter = build_service_filter(environment, roles_data) for cip_entry in cip: mapping_args = cip_entry.get('set', {}) set_neutron_driver(pd, mapping_args) push_destination = cip_entry.get('push_destination') # use the configured registry IP as the discovered registry # if it is available if push_destination and isinstance(push_destination, bool): local_registry_ip = pd.get('LocalContainerRegistry') if local_registry_ip: push_destination = '%s:8787' % local_registry_ip pull_source = cip_entry.get('pull_source') modify_role = cip_entry.get('modify_role') modify_vars = cip_entry.get('modify_vars') modify_only_with_labels = cip_entry.get('modify_only_with_labels') modify_only_with_source = cip_entry.get('modify_only_with_source') modify_append_tag = cip_entry.get( 'modify_append_tag', time.strftime('-modified-%Y%m%d%H%M%S')) # do not use tag_from_label if a tag is specified in the set tag_from_label = None if not mapping_args.get('tag'): tag_from_label = cip_entry.get('tag_from_label') if multi_arch and 'multi_arch' in cip_entry: # individual entry sets multi_arch, # so set global multi_arch to False multi_arch = False prepare_data = container_images_prepare( excludes=cip_entry.get('excludes'), includes=cip_entry.get('includes'), service_filter=service_filter, pull_source=pull_source, push_destination=push_destination, mapping_args=mapping_args, output_env_file='image_params', output_images_file='upload_data', tag_from_label=tag_from_label, append_tag=modify_append_tag, modify_role=modify_role, modify_vars=modify_vars, modify_only_with_labels=modify_only_with_labels, modify_only_with_source=modify_only_with_source, mirrors=mirrors, registry_credentials=creds, multi_arch=multi_arch, lock=lock) env_params.update(prepare_data['image_params']) if not dry_run and (push_destination or pull_source or modify_role): with tempfile.NamedTemporaryFile(mode='w') as f: yaml.safe_dump( {'container_images': prepare_data['upload_data']}, f) uploader = image_uploader.ImageUploadManager( [f.name], cleanup=cleanup, mirrors=mirrors, registry_credentials=creds, multi_arch=multi_arch, lock=lock) uploader.upload() return env_params
def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, excludes=None, includes=None, service_filter=None, pull_source=None, push_destination=None, mapping_args=None, output_env_file=None, output_images_file=None, tag_from_label=None, append_tag=None, modify_role=None, modify_vars=None, modify_only_with_labels=None, mirrors=None): """Perform container image preparation :param template_file: path to Jinja2 file containing all image entries :param excludes: list of image name substrings to use for exclude filter :param includes: list of image name substrings, at least one must match. All excludes are ignored if includes is specified. :param service_filter: set of heat resource types for containerized services to filter by. Disable by passing None. :param pull_source: DEPRECATED namespace for pulling during image uploads :param push_destination: namespace for pushing during image uploads. When specified the image parameters will use this namespace too. :param mapping_args: dict containing substitutions for template file. See CONTAINER_IMAGES_DEFAULTS for expected keys. :param output_env_file: key to use for heat environment parameter data :param output_images_file: key to use for image upload data :param tag_from_label: string when set will trigger tag discovery on every image :param append_tag: string to append to the tag for the destination image :param modify_role: string of ansible role name to run during upload before the push to destination :param modify_vars: dict of variables to pass to modify_role :param modify_only_with_labels: only modify the container images with the given labels :param mirrors: dict of registry netloc values to mirror urls :returns: dict with entries for the supplied output_env_file or output_images_file """ if mapping_args is None: mapping_args = {} def ffunc(entry): imagename = entry.get('imagename', '') if service_filter is not None: # check the entry is for a service being deployed image_services = set(entry.get('services', [])) if not image_services.intersection(service_filter): return None if includes: for p in includes: if re.search(p, imagename): return entry return None if excludes: for p in excludes: if re.search(p, imagename): return None return entry builder = KollaImageBuilder([template_file]) result = builder.container_images_from_template(filter=ffunc, **mapping_args) manager = image_uploader.ImageUploadManager(mirrors=mirrors) uploader = manager.uploader('python') images = [i.get('imagename', '') for i in result] if tag_from_label: image_version_tags = uploader.discover_image_tags( images, tag_from_label) for entry in result: imagename = entry.get('imagename', '') image_no_tag = imagename.rpartition(':')[0] if image_no_tag in image_version_tags: entry['imagename'] = '%s:%s' % ( image_no_tag, image_version_tags[image_no_tag]) if modify_only_with_labels: images_with_labels = uploader.filter_images_with_labels( images, modify_only_with_labels) params = {} modify_append_tag = append_tag for entry in result: imagename = entry.get('imagename', '') append_tag = '' if modify_role and ((not modify_only_with_labels) or imagename in images_with_labels): entry['modify_role'] = modify_role if modify_append_tag: entry['modify_append_tag'] = modify_append_tag append_tag = modify_append_tag if modify_vars: entry['modify_vars'] = modify_vars if pull_source: entry['pull_source'] = pull_source if push_destination: # substitute discovered registry if push_destination is set to true if isinstance(push_destination, bool): push_destination = image_uploader.get_undercloud_registry() entry['push_destination'] = push_destination # replace the host portion of the imagename with the # push_destination, since that is where they will be uploaded to image = imagename.partition('/')[2] imagename = '/'.join((push_destination, image)) if 'params' in entry: for p in entry.pop('params'): params[p] = imagename + append_tag if 'services' in entry: del (entry['services']) params.update(detect_insecure_registries(params)) return_data = {} if output_env_file: return_data[output_env_file] = params if output_images_file: return_data[output_images_file] = result return return_data
def container_images_prepare_multi(environment, roles_data, dry_run=False, cleanup=image_uploader.CLEANUP_FULL): """Perform multiple container image prepares and merge result Given the full heat environment and roles data, perform multiple image prepare operations. The data to drive the multiple prepares is taken from the ContainerImagePrepare parameter in the provided environment. If push_destination is specified, uploads will be performed during the preparation. :param environment: Heat environment for deployment :param roles_data: Roles file data used to filter services :returns: dict containing merged container image parameters from all prepare operations """ pd = environment.get('parameter_defaults', {}) cip = pd.get('ContainerImagePrepare') if not cip: return mirrors = {} mirror = pd.get('DockerRegistryMirror') if mirror: mirrors['docker.io'] = mirror env_params = {} service_filter = build_service_filter(environment, roles_data) for cip_entry in cip: mapping_args = cip_entry.get('set', {}) set_neutron_driver(pd, mapping_args) push_destination = cip_entry.get('push_destination') # use the configured registry IP as the discovered registry # if it is available if push_destination and isinstance(push_destination, bool): local_registry_ip = pd.get('LocalContainerRegistry') if local_registry_ip: push_destination = '%s:8787' % local_registry_ip pull_source = cip_entry.get('pull_source') modify_role = cip_entry.get('modify_role') modify_vars = cip_entry.get('modify_vars') modify_only_with_labels = cip_entry.get('modify_only_with_labels') modify_append_tag = cip_entry.get( 'modify_append_tag', time.strftime('-modified-%Y%m%d%H%M%S')) prepare_data = container_images_prepare( excludes=cip_entry.get('excludes'), includes=cip_entry.get('includes'), service_filter=service_filter, pull_source=pull_source, push_destination=push_destination, mapping_args=mapping_args, output_env_file='image_params', output_images_file='upload_data', tag_from_label=cip_entry.get('tag_from_label'), append_tag=modify_append_tag, modify_role=modify_role, modify_vars=modify_vars, modify_only_with_labels=modify_only_with_labels, mirrors=mirrors) env_params.update(prepare_data['image_params']) if push_destination or pull_source or modify_role: with tempfile.NamedTemporaryFile(mode='w') as f: yaml.safe_dump( {'container_images': prepare_data['upload_data']}, f) uploader = image_uploader.ImageUploadManager([f.name], dry_run=dry_run, cleanup=cleanup, mirrors=mirrors) uploader.upload() return env_params
def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, excludes=None, service_filter=None, pull_source=None, push_destination=None, mapping_args=None, output_env_file=None, output_images_file=None, tag_from_label=None): if mapping_args is None: mapping_args = {} def ffunc(entry): imagename = entry.get('imagename', '') if excludes: for p in excludes: if re.search(p, imagename): return None if service_filter is not None: # check the entry is for a service being deployed image_services = set(entry.get('services', [])) if not image_services.intersection(service_filter): return None return entry builder = KollaImageBuilder([template_file]) result = builder.container_images_from_template(filter=ffunc, **mapping_args) if tag_from_label: uploader = image_uploader.ImageUploadManager().uploader('docker') images = [i.get('imagename', '') for i in result] image_version_tags = uploader.discover_image_tags( images, tag_from_label) for entry in result: imagename = entry.get('imagename', '') image_no_tag = imagename.rpartition(':')[0] if image_no_tag in image_version_tags: entry['imagename'] = '%s:%s' % ( image_no_tag, image_version_tags[image_no_tag]) params = {} for entry in result: imagename = entry.get('imagename', '') if pull_source: entry['pull_source'] = pull_source if push_destination: entry['push_destination'] = push_destination # replace the host portion of the imagename with the # push_destination, since that is where they will be uploaded to image = imagename.partition('/')[2] imagename = '/'.join((push_destination, image)) if 'params' in entry: for p in entry.pop('params'): params[p] = imagename if 'services' in entry: del (entry['services']) params.update(detect_insecure_registries(params)) return_data = {} if output_env_file: return_data[output_env_file] = params if output_images_file: return_data[output_images_file] = result return return_data
def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) uploader = image_uploader.ImageUploadManager(parsed_args.config_files) uploader.upload()
def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE, template_dir=DEFAULT_TEMPLATE_DIR, excludes=None, includes=None, service_filter=None, pull_source=None, push_destination=None, mapping_args=None, output_env_file=None, output_images_file=None, tag_from_label=None, append_tag=None, modify_role=None, modify_vars=None, modify_only_with_labels=None, modify_only_with_source=None, mirrors=None, registry_credentials=None, multi_arch=False, lock=None): """Perform container image preparation :param template_file: path to Jinja2 file containing all image entries :param template_dir: path to Jinja2 files included in the main template :param excludes: list of image name substrings to use for exclude filter :param includes: list of image name substrings, at least one must match. All excludes are ignored if includes is specified. :param service_filter: set of heat resource types for containerized services to filter by. Disable by passing None. :param pull_source: DEPRECATED namespace for pulling during image uploads :param push_destination: namespace for pushing during image uploads. When specified the image parameters will use this namespace too. :param mapping_args: dict containing substitutions for template file. See CONTAINER_IMAGES_DEFAULTS for expected keys. :param output_env_file: key to use for heat environment parameter data :param output_images_file: key to use for image upload data :param tag_from_label: string when set will trigger tag discovery on every image :param append_tag: string to append to the tag for the destination image :param modify_role: string of ansible role name to run during upload before the push to destination :param modify_vars: dict of variables to pass to modify_role :param modify_only_with_labels: only modify the container images with the given labels :param modify_only_with_source: only modify the container images from a image_source in the tripleo-common service to container mapping (e.g. kolla/tripleo) :param mirrors: dict of registry netloc values to mirror urls :param registry_credentials: dict of registry netloc values to authentication credentials for that registry. The value is a single-entry dict where the username is the key and the password is the value. :param multi_arch: boolean whether to prepare every architecture of each image :param lock: a locking object to use when handling uploads :returns: dict with entries for the supplied output_env_file or output_images_file """ if mapping_args is None: mapping_args = {} if not lock: lock = threadinglock.ThreadingLock() def ffunc(entry): imagename = entry.get('imagename', '') if service_filter is not None: # check the entry is for a service being deployed image_services = set(entry.get('services', [])) if not image_services.intersection(service_filter): return None if includes: for p in includes: if re.search(p, imagename): return entry return None if excludes: for p in excludes: if re.search(p, imagename): return None return entry builder = KollaImageBuilder([template_file], template_dir) result = builder.container_images_from_template(filter=ffunc, **mapping_args) manager = image_uploader.ImageUploadManager( mirrors=mirrors, registry_credentials=registry_credentials, multi_arch=multi_arch, lock=lock) uploader = manager.uploader('python') images = [i.get('imagename', '') for i in result] # set a flag to record whether the default tag is used or not. the # logic here is that if the tag key is not already in mapping then it # wil be added during the template render, so default_tag is set to # True. default_tag = 'tag' not in mapping_args if tag_from_label: image_version_tags = uploader.discover_image_tags( images, tag_from_label, default_tag) for entry in result: imagename = entry.get('imagename', '') image_no_tag = imagename.rpartition(':')[0] if image_no_tag in image_version_tags: entry['imagename'] = '%s:%s' % ( image_no_tag, image_version_tags[image_no_tag]) images_with_labels = [] if modify_only_with_labels: images_with_labels = uploader.filter_images_with_labels( images, modify_only_with_labels) images_with_source = [] if modify_only_with_source: images_with_source = [ i.get('imagename') for i in result if i.get('image_source', '') in modify_only_with_source ] params = {} modify_append_tag = append_tag for entry in result: imagename = entry.get('imagename', '') append_tag = '' if modify_role and ( (not modify_only_with_labels and not modify_only_with_source) or (imagename in images_with_labels or imagename in images_with_source)): entry['modify_role'] = modify_role if modify_append_tag: entry['modify_append_tag'] = modify_append_tag append_tag = modify_append_tag if modify_vars: entry['modify_vars'] = modify_vars if pull_source: entry['pull_source'] = pull_source if push_destination: # substitute discovered registry if push_destination is set to true if isinstance(push_destination, bool): push_destination = image_uploader.get_undercloud_registry() entry['push_destination'] = push_destination # replace the host portion of the imagename with the # push_destination, since that is where they will be uploaded to image = imagename.partition('/')[2] imagename = '/'.join((push_destination, image)) if 'params' in entry: for p in entry.pop('params'): params[p] = imagename + append_tag if 'services' in entry: del (entry['services']) params.update(detect_insecure_registries(params, lock=lock)) return_data = {} if output_env_file: return_data[output_env_file] = params if output_images_file: return_data[output_images_file] = result return return_data