def _pull_image(self, repo, tag, registry):
auth_config = None
image_ref = docker_image.Reference.parse(repo)
registry_domain, remainder = image_ref.split_hostname()
if registry and registry.username:
auth_config = {
'username': registry.username,
'password': registry.password
}
elif (registry_domain
and registry_domain == CONF.docker.default_registry
and CONF.docker.default_registry_username):
auth_config = {
'username': CONF.docker.default_registry_username,
'password': CONF.docker.default_registry_password
}
with docker_utils.docker_client() as docker:
try:
docker.pull(repo, tag=tag, auth_config=auth_config)
except errors.NotFound as e:
raise exception.ImageNotFound(message=str(e))
except errors.APIError:
LOG.exception('Error on pulling image')
message = _('Error on pulling image: %(repo)s:%(tag)s') % {
'repo': repo,
'tag': tag
}
raise exception.ZunException(message)
def get_image_by_uuid(self, context, image_uuid):
try:
res = self.client.read('/images/' + image_uuid)
image = translate_etcd_result(res, 'image')
filtered_images = self._filter_resources(
[image], self._add_tenant_filters(context, {}))
if len(filtered_images) > 0:
return filtered_images[0]
else:
raise exception.ImageNotFound(image=image_uuid)
except etcd.EtcdKeyNotFound:
raise exception.ImageNotFound(image=image_uuid)
except Exception as e:
LOG.error('Error occurred while retrieving image: %s',
six.text_type(e))
raise
def test_container_run_image_not_found(
self, mock_pull, mock_attach_volume, mock_detach_volume,
mock_list_by_container, mock_save, mock_spawn_n):
container_dict = utils.get_test_container(
image='test:latest', image_driver='docker',
image_pull_policy='ifnotpresent')
container = Container(self.context, **container_dict)
mock_pull.side_effect = exception.ImageNotFound(
message="Image Not Found")
mock_spawn_n.side_effect = lambda f, *x, **y: f(*x, **y)
networks = []
volumes = [FakeVolumeMapping()]
self.compute_manager.container_create(
self.context,
requested_networks=networks,
requested_volumes=volumes,
container=container,
limits=None, run=True)
mock_save.assert_called_with(self.context)
self.assertEqual('Error', container.status)
self.assertEqual('Image Not Found', container.status_reason)
mock_pull.assert_called_once_with(self.context, 'test', 'latest',
'ifnotpresent', 'docker')
mock_attach_volume.assert_called_once()
mock_detach_volume.assert_called_once()
self.assertEqual(0, len(FakeVolumeMapping.volumes))
def find_image(context, image_ident, image_tag):
matches = find_images(context, image_ident, exact_match=True)
LOG.debug('Found matches %s ', matches)
# added for glance image tag and name support
match = []
for i in range(len(matches)):
if matches[i]['tags']:
if len(image_tag) < len(matches[i]['tags']):
data1, data2 = image_tag, matches[i]['tags']
else:
data1, data2 = matches[i]['tags'], image_tag
if all(map(lambda x: x in data1, data2)):
match.append(matches[i])
else:
if matches[i]['tags'] == image_tag:
match.append(matches[i])
if len(match) == 0:
raise exception.ImageNotFound(image=image_ident)
if len(match) > 1:
msg = ("Multiple images exist with same name "
"%(image_ident)s. Please use the image id "
"instead.") % {
'image_ident': image_ident
}
raise exception.Conflict(msg)
return match[0]
def container_create(self,
context,
new_container,
extra_spec,
requested_networks,
requested_volumes,
run,
pci_requests=None):
host_state = None
try:
host_state = self._schedule_container(context, new_container,
extra_spec)
except Exception as exc:
new_container.status = consts.ERROR
new_container.status_reason = str(exc)
new_container.save(context)
return
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
if CONF.api.enable_image_validation:
images = self.rpcapi.image_search(context, new_container.image,
new_container.image_driver, True,
host_state['host'])
if not images:
raise exception.ImageNotFound(image=new_container.image)
self.rpcapi.container_create(context, host_state['host'],
new_container, host_state['limits'],
requested_networks, requested_volumes,
run, pci_requests)
def pull_image(self, context, repo, tag, image_pull_policy):
image_loaded = True
image = self._search_image_on_host(repo, tag)
if not utils.should_pull_image(image_pull_policy, bool(image)):
if image:
LOG.debug('Image %s present locally' % repo)
return image, image_loaded
else:
message = _(
'Image %s not present with pull policy of Never') % repo
raise exception.ImageNotFound(message)
try:
LOG.debug('Pulling image from docker %s,'
' context %s' % (repo, context))
self._pull_image(repo, tag)
return {'image': repo, 'path': None}, image_loaded
except exception.ImageNotFound:
with excutils.save_and_reraise_exception():
LOG.error('Image %s was not found in docker repo' % repo)
except exception.DockerError:
with excutils.save_and_reraise_exception():
LOG.error('Docker API error occurred during downloading\
image %s' % repo)
except Exception as e:
msg = _('Cannot download image from docker: {0}')
raise exception.ZunException(msg.format(e))
def pull_image(context,
repo,
tag,
image_pull_policy='always',
image_driver=None):
if image_driver:
image_driver_list = [image_driver.lower()]
else:
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
image, image_loaded = image_driver.pull_image(
context, repo, tag, image_pull_policy)
if image:
image['driver'] = driver.split('.')[0]
break
except exception.ImageNotFound:
image = None
except Exception as e:
LOG.exception(
'Unknown exception occurred while loading '
'image: %s', six.text_type(e))
raise exception.ZunException(six.text_type(e))
if not image:
raise exception.ImageNotFound("Image %s not found" % repo)
return image, image_loaded
def get_image_by_name_or_id(self, image_ident):
"""Get an image by name or ID."""
try:
return self.client().glance.find_image(image_ident)
except exceptions.NotFound as e:
raise exception.ImageNotFound(six.text_type(e))
except exceptions.NoUniqueMatch as e:
raise exception.Conflict(six.text_type(e))
def test_container_create_pull_image_failed_image_not_found(
self, mock_fail, mock_pull, mock_create_sandbox, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.side_effect = exception.ImageNotFound("Image Not Found")
mock_create_sandbox.return_value = mock.MagicMock()
self.compute_manager._do_container_create(self.context, container)
mock_fail.assert_called_once_with(self.context, container,
"Image Not Found")
def get_image_by_uuid(self, context, image_uuid):
query = model_query(models.Image)
query = self._add_tenant_filters(context, query)
query = query.filter_by(uuid=image_uuid)
try:
return query.one()
except NoResultFound:
raise exception.ImageNotFound(image=image_uuid)
def pull_image(self, context, repo, tag, image_pull_policy):
# TODO(shubhams): glance driver does not handle tags
# once metadata is stored in db then handle tags
image_loaded = False
image = self._search_image_on_host(context, repo)
if image:
image_path = image['path']
image_checksum = image['checksum']
md5sum = hashlib.md5()
with open(image_path, 'rb') as fd:
while True:
# read 10MB of data each time
data = fd.read(10 * 1024 * 1024)
if not data:
break
md5sum.update(data)
md5sum = md5sum.hexdigest()
if md5sum == image_checksum:
image_loaded = True
return image, image_loaded
if not common_utils.should_pull_image(image_pull_policy, bool(image)):
if image:
LOG.debug('Image %s present locally', repo)
image_loaded = True
return image, image_loaded
else:
message = _(
'Image %s not present with pull policy of Never') % repo
raise exception.ImageNotFound(message)
LOG.debug('Pulling image from glance %s', repo)
try:
image_meta = utils.find_image(context, repo)
LOG.debug('Image %s was found in glance, downloading now...', repo)
image_chunks = utils.download_image_in_chunks(
context, image_meta.id)
except exception.ImageNotFound:
LOG.error('Image %s was not found in glance', repo)
raise
except Exception as e:
msg = _('Cannot download image from glance: {0}')
raise exception.ZunException(msg.format(e))
try:
images_directory = CONF.glance.images_directory
fileutils.ensure_tree(images_directory)
out_path = os.path.join(images_directory, image_meta.id + '.tar')
with open(out_path, 'wb') as fd:
for chunk in image_chunks:
fd.write(chunk)
except Exception as e:
msg = _('Error occurred while writing image: {0}')
raise exception.ZunException(msg.format(e))
LOG.debug('Image %(repo)s was downloaded to path : %(path)s', {
'repo': repo,
'path': out_path
})
return {'image': repo, 'path': out_path}, image_loaded
def test_container_create_pull_image_failed_image_not_found(
self, mock_fail, mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.side_effect = exception.ImageNotFound("Image Not Found")
networks = []
self.compute_manager._do_container_create(self.context, container,
networks)
mock_fail.assert_called_once_with(self.context,
container, "Image Not Found")
def _pull_image(self, repo, tag):
with docker_utils.docker_client() as docker:
for line in docker.pull(repo, tag=tag, stream=True):
error = json.loads(line).get('errorDetail')
if error:
if "not found" in error['message']:
raise exception.ImageNotFound(error['message'])
else:
raise exception.DockerError(error['message'])
def test_create_container_image_not_found(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
mock_search.side_effect = exception.ImageNotFound()
params = {"name": "MyDocker", "image": "not-found"}
response = self.post_json('/containers/', params, expect_errors=True)
self.assertEqual('application/json', response.content_type)
self.assertEqual(404, response.status_int)
self.assertFalse(mock_container_create.called)
def find_image(context, image_ident):
matches = find_images(context, image_ident, exact_match=True)
LOG.debug('Found matches %s ' % matches)
if len(matches) == 0:
raise exception.ImageNotFound(image=image_ident)
if len(matches) > 1:
msg = ("Multiple images exist with same name "
"%(image_ident)s. Please use the image id "
"instead.") % {'image_ident': image_ident}
raise exception.Conflict(msg)
return matches[0]
def test_container_run_image_not_found(self, mock_fail, mock_pull,
mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.side_effect = exception.ImageNotFound(
message="Image Not Found")
self.compute_manager._do_container_run(self.context, container)
mock_save.assert_called_with(self.context)
mock_fail.assert_called_with(self.context, container,
'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
def _do_update_image(self, image_id, values):
session = get_session()
with session.begin():
query = model_query(models.Image, session=session)
query = add_identity_filter(query, image_id)
try:
ref = query.with_lockmode('update').one()
except NoResultFound:
raise exception.ImageNotFound(image=image_id)
ref.update(values)
return ref
def container_create(self, context, new_container, extra_spec,
requested_networks, requested_volumes, run,
pci_requests=None):
try:
host_state = self._schedule_container(context, new_container,
extra_spec)
except exception.NoValidHost:
new_container.status = consts.ERROR
new_container.status_reason = _(
"There are not enough hosts available.")
new_container.save(context)
return
except Exception:
new_container.status = consts.ERROR
new_container.status_reason = _("Unexpected exception occurred.")
new_container.save(context)
raise
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
if CONF.api.enable_image_validation:
try:
images = self.rpcapi.image_search(
context, new_container.image,
new_container.image_driver, True, new_container.registry,
host_state['host'])
if not images:
raise exception.ImageNotFound(image=new_container.image)
if len(images) > 1:
raise exception.Conflict('Multiple images exist with same '
'name. Please use the container '
'uuid instead.')
except exception.OperationNotSupported:
LOG.info("Skip validation since search is not supported for "
"image '%(image)s' and image driver '%(driver)s'.",
{'image': new_container.image,
'driver': new_container.image_driver})
except exception.ReferenceInvalidFormat:
raise exception.InvalidValue(_("The format of image name '%s' "
"is invalid.")
% new_container.image)
except Exception as e:
LOG.warning("Skip validation since image search failed with "
"unexpected exception: %s", str(e))
self._record_action_start(context, new_container,
container_actions.CREATE)
self.rpcapi.container_create(context, host_state['host'],
new_container, host_state['limits'],
requested_networks, requested_volumes,
run, pci_requests)
def test_pull_image_not_found(self, mock_should_pull_image, mock_search,
mock_parse_image):
mock_should_pull_image.return_value = True
mock_search.return_value = {'image': 'nginx', 'path': 'xyz'}
mock_parse_image.return_value = ('repo', 'tag')
with mock.patch.object(
self.mock_docker,
'pull',
side_effect=exception.ImageNotFound('Error')) as mock_pull:
self.assertRaises(exception.ImageNotFound, self.driver.pull_image,
None, 'repo', 'tag', 'always')
self.mock_docker.pull.assert_called_once_with('repo', tag='tag')
self.assertEqual(1, mock_pull.call_count)
def container_create(self,
context,
new_container,
extra_spec,
requested_networks,
requested_volumes,
run,
pci_requests=None):
try:
host_state = self._schedule_container(context, new_container,
extra_spec)
except exception.NoValidHost:
new_container.status = consts.ERROR
new_container.status_reason = _(
"There are not enough hosts available.")
new_container.save(context)
return
except Exception:
new_container.status = consts.ERROR
new_container.status_reason = _("Unexpected exception occurred.")
new_container.save(context)
raise
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
if CONF.api.enable_image_validation:
try:
images = self.rpcapi.image_search(context, new_container.image,
new_container.image_driver,
True, host_state['host'])
if not images:
raise exception.ImageNotFound(image=new_container.image)
if len(images) > 1:
raise exception.Conflict('Multiple images exist with same '
'name. Please use the container '
'uuid instead.')
except Exception as e:
new_container.status = consts.ERROR
new_container.status_reason = str(e)
new_container.save(context)
raise
self._record_action_start(context, new_container,
container_actions.CREATE)
self.rpcapi.container_create(context, host_state['host'],
new_container, host_state['limits'],
requested_networks, requested_volumes,
run, pci_requests)
def test_container_run_image_not_found(self, mock_fail,
mock_pull, mock_save):
container_dict = utils.get_test_container(
image='test:latest', image_driver='docker',
image_pull_policy='ifnotpresent')
container = Container(self.context, **container_dict)
mock_pull.side_effect = exception.ImageNotFound(
message="Image Not Found")
networks = []
self.compute_manager._do_container_run(self.context,
container,
networks)
mock_save.assert_called_with(self.context)
mock_fail.assert_called_with(self.context,
container, 'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'test', 'latest',
'ifnotpresent', 'docker')
def pull_image(self, context, repo, tag, image_pull_policy, registry):
image_loaded = False
image = self._search_image_on_host(context, repo, tag)
if not common_utils.should_pull_image(image_pull_policy, bool(image)):
if image:
if self._verify_md5sum_for_image(image):
image_loaded = True
return image, image_loaded
else:
message = _(
'Image %s not present with pull policy of Never') % repo
raise exception.ImageNotFound(message)
LOG.debug('Pulling image from glance %s', repo)
try:
image_meta = utils.find_image(context, repo, tag)
LOG.debug('Image %s was found in glance, downloading now...', repo)
image_chunks = utils.download_image_in_chunks(
context, image_meta.id)
except exception.ImageNotFound:
LOG.error('Image %s was not found in glance', repo)
raise
except Exception as e:
msg = _('Cannot download image from glance: {0}')
raise exception.ZunException(msg.format(e))
try:
images_directory = CONF.glance.images_directory
fileutils.ensure_tree(images_directory)
out_path = os.path.join(images_directory, image_meta.id + '.tar')
with open(out_path, 'wb') as fd:
for chunk in image_chunks:
fd.write(chunk)
except Exception as e:
msg = _('Error occurred while writing image: {0}')
raise exception.ZunException(msg.format(e))
LOG.debug('Image %(repo)s was downloaded to path : %(path)s', {
'repo': repo,
'path': out_path
})
image = {
'image': image_meta.name,
'tags': image_meta.tags,
'path': out_path
}
return image, image_loaded
def update_image(self, image_uuid, values):
if 'uuid' in values:
msg = _('Cannot overwrite UUID for an existing image.')
raise exception.InvalidParameterValue(err=msg)
try:
target = self.client.read('/images/' + image_uuid)
target_value = json.loads(target.value)
target_value.update(values)
target.value = json.dump_as_bytes(target_value)
self.client.update(target)
except etcd.EtcdKeyNotFound:
raise exception.ImageNotFound(image=image_uuid)
except Exception as e:
LOG.error('Error occurred while updating image: %s',
six.text_type(e))
raise
return translate_etcd_result(target, 'image')
def pull_image(context, repo, tag, image_pull_policy):
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
image = image_driver.pull_image(context, repo, tag,
image_pull_policy)
if image:
break
except exception.ImageNotFound:
image = None
except Exception as e:
LOG.exception(
_LE('Unknown exception occurred while loading '
'image: %s'), six.text_type(e))
raise exception.ZunException(six.text_type(e))
if not image:
raise exception.ImageNotFound("Image %s not found" % repo)
return image
def post(self, run=False, **container_dict):
"""Create a new container.
:param run: if true, starts the container
:param container: a container within the request body.
"""
context = pecan.request.context
compute_api = pecan.request.compute_api
policy.enforce(context, "container:create", action="container:create")
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
images = compute_api.image_search(context, container_dict['image'],
True)
if not images:
raise exception.ImageNotFound(container_dict['image'])
container_dict['project_id'] = context.project_id
container_dict['user_id'] = context.user_id
name = container_dict.get('name') or \
self._generate_name_for_container()
container_dict['name'] = name
if container_dict.get('memory'):
container_dict['memory'] = \
str(container_dict['memory']) + 'M'
container_dict['status'] = fields.ContainerStatus.CREATING
new_container = objects.Container(context, **container_dict)
new_container.create(context)
try:
run = strutils.bool_from_string(run, strict=True)
except ValueError:
msg = _('Valid run values are true, false, 0, 1, yes and no')
raise exception.InvalidValue(msg)
if run:
compute_api.container_run(context, new_container)
else:
compute_api.container_create(context, new_container)
# Set the HTTP Location Header
pecan.response.location = link.build_url('containers',
new_container.uuid)
pecan.response.status = 202
return view.format_container(pecan.request.host_url, new_container)
def pull_image(self, context, repo, tag, image_pull_policy):
# TODO(shubhams): glance driver does not handle tags
# once metadata is stored in db then handle tags
image = self._search_image_on_host(context, repo)
if not common_utils.should_pull_image(image_pull_policy, bool(image)):
if image:
LOG.debug('Image %s present locally' % repo)
return image
else:
message = _(
'Image %s not present with pull policy of Never') % repo
raise exception.ImageNotFound(message)
LOG.debug('Pulling image from glance %s' % repo)
try:
glance = utils.create_glanceclient(context)
image_meta = utils.find_image(context, repo)
LOG.debug('Image %s was found in glance, downloading now...' %
repo)
image_chunks = glance.images.data(image_meta.id)
except exception.ImageNotFound:
LOG.error('Image %s was not found in glance' % repo)
raise
except Exception as e:
msg = _('Cannot download image from glance: {0}')
raise exception.ZunException(msg.format(e))
try:
images_directory = CONF.glance.images_directory
fileutils.ensure_tree(images_directory)
out_path = os.path.join(images_directory, image_meta.id + '.tar')
with open(out_path, 'wb') as fd:
for chunk in image_chunks:
fd.write(chunk)
except Exception as e:
msg = _('Error occurred while writing image: {0}')
raise exception.ZunException(msg.format(e))
LOG.debug('Image %s was downloaded to path : %s' % (repo, out_path))
return {'image': repo, 'path': out_path}
def post(self, run=False, **container_dict):
"""Create a new container.
:param run: if true, starts the container
:param container_dict: a container within the request body.
"""
context = pecan.request.context
compute_api = pecan.request.compute_api
policy.enforce(context, "container:create",
action="container:create")
# remove duplicate security_groups from list
if container_dict.get('security_groups'):
container_dict['security_groups'] = list(
set(container_dict.get('security_groups')))
try:
run = strutils.bool_from_string(run, strict=True)
container_dict['interactive'] = strutils.bool_from_string(
container_dict.get('interactive', False), strict=True)
except ValueError:
msg = _('Valid run or interactive value is ''true'', '
'"false", True, False, "True" and "False"')
raise exception.InvalidValue(msg)
requested_networks = container_dict.get('nets', [])
# Valiadtion accepts 'None' so need to convert it to None
if container_dict.get('image_driver'):
container_dict['image_driver'] = api_utils.string_or_none(
container_dict.get('image_driver'))
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
images = compute_api.image_search(context, container_dict['image'],
container_dict.get('image_driver'),
True)
if not images:
raise exception.ImageNotFound(image=container_dict['image'])
container_dict['project_id'] = context.project_id
container_dict['user_id'] = context.user_id
name = container_dict.get('name') or \
self._generate_name_for_container()
container_dict['name'] = name
if container_dict.get('memory'):
container_dict['memory'] = \
str(container_dict['memory']) + 'M'
if container_dict.get('restart_policy'):
self._check_for_restart_policy(container_dict)
auto_remove = container_dict.pop('auto_remove', None)
if auto_remove is not None:
req_version = pecan.request.version
min_version = versions.Version('', '', '', '1.3')
if req_version >= min_version:
try:
container_dict['auto_remove'] = strutils.bool_from_string(
auto_remove, strict=True)
except ValueError:
msg = _('Auto_remove value are true or false')
raise exception.InvalidValue(msg)
else:
msg = _('Invalid param auto_remove because current request '
'version is %(req_version)s. Auto_remove is only '
'supported from version %(min_version)s') % \
{'req_version': req_version,
'min_version': min_version}
raise exception.InvalidParam(msg)
container_dict['status'] = consts.CREATING
extra_spec = container_dict.get('hints', None)
new_container = objects.Container(context, **container_dict)
new_container.create(context)
if run:
compute_api.container_run(context, new_container, extra_spec,
requested_networks)
else:
compute_api.container_create(context, new_container, extra_spec,
requested_networks)
# Set the HTTP Location Header
pecan.response.location = link.build_url('containers',
new_container.uuid)
pecan.response.status = 202
return view.format_container(pecan.request.host_url, new_container)
def _pull_image(self, repo, tag):
with docker_utils.docker_client() as docker:
try:
docker.pull(repo, tag=tag)
except errors.NotFound as e:
raise exception.ImageNotFound(message=six.text_type(e))