def test_glance(self, glance):
context = mock.NonCallableMock(session=mock.sentinel.session)
res = clients.glance(context)
self.assertEqual(glance.return_value, res)
glance.assert_called_with('1',
service_type='image',
session=mock.sentinel.session)
def get_os_items(self):
os_images = list(clients.glance(self.context).images.list())
self.ec2_created_os_images = {
os_image.properties['ec2_id']: os_image
for os_image in os_images
if (os_image.properties.get('ec2_id') and
self.context.project_id == os_image.owner)}
return os_images
def get_os_items(self):
os_images = list(clients.glance(self.context).images.list())
self.ec2_created_os_images = {
os_image.properties['ec2_id']: os_image
for os_image in os_images
if (os_image.properties.get('ec2_id')
and self.context.project_id == os_image.owner)
}
return os_images
def test_glance(self, glance):
context = mock.NonCallableMock(
auth_token='fake_token',
service_catalog=[{'type': 'image',
'endpoints': [{'publicURL': 'glance_url'}]}])
res = clients.glance(context)
self.assertEqual(glance.return_value, res)
glance.assert_called_with(
'1', auth_url='keystone_url', service_type='image',
token='fake_token', endpoint='glance_url')
def get_os_image(context, ec2_image_id):
kind = get_ec2_id_kind(ec2_image_id)
images = db_api.get_public_items(context, kind, (ec2_image_id,))
image = (images[0] if len(images) else
get_db_item(context, ec2_image_id))
glance = clients.glance(context)
try:
return glance.images.get(image['os_id'])
except glance_exception.HTTPNotFound:
raise exception.InvalidAMIIDNotFound(id=ec2_image_id)
def get_os_image(context, ec2_image_id):
kind = get_ec2_id_kind(ec2_image_id)
ids = db_api.get_items_ids(context, kind, item_ids=(ec2_image_id,))
if not ids:
raise exception.InvalidAMIIDNotFound(id=ec2_image_id)
_id, os_id = ids[0]
glance = clients.glance(context)
try:
return glance.images.get(os_id)
except glance_exception.HTTPNotFound:
raise exception.InvalidAMIIDNotFound(id=ec2_image_id)
def get_os_image(context, ec2_image_id):
kind = get_ec2_id_kind(ec2_image_id)
ids = db_api.get_items_ids(context, kind, item_ids=(ec2_image_id, ))
if not ids:
raise exception.InvalidAMIIDNotFound(id=ec2_image_id)
_id, os_id = ids[0]
glance = clients.glance(context)
try:
return glance.images.get(os_id)
except glance_exception.HTTPNotFound:
raise exception.InvalidAMIIDNotFound(id=ec2_image_id)
def deregister_image(context, image_id):
os_image = ec2utils.get_os_image(context, image_id)
_check_owner(context, os_image)
glance = clients.glance(context)
try:
glance.images.delete(os_image.id)
except glance_exception.HTTPNotFound:
pass
db_api.delete_item(context, image_id)
return True
def test_glance(self, glance):
context = mock.Mock(
auth_token='fake_token',
service_catalog=[{'type': 'image',
'endpoints': [{'publicURL': 'glance_url'}]}])
res = clients.glance(context)
self.assertEqual(glance.return_value, res)
glance.assert_called_with(
'1', auth_url='keystone_url', service_type='image',
token='fake_token', cacert=None, endpoint='glance_url',
insecure=False)
def deregister_image(context, image_id):
os_image = ec2utils.get_os_image(context, image_id)
if not os_image:
image = db_api.get_item_by_id(context, image_id)
if image.get('state') != 'failed':
# TODO(ft): figure out corresponding AWS error
raise exception.IncorrectState(
reason='Image is still being created')
else:
_check_owner(context, os_image)
glance = clients.glance(context)
try:
glance.images.delete(os_image.id)
except glance_exception.HTTPNotFound:
pass
db_api.delete_item(context, image_id)
return True
def deregister_image(context, image_id):
os_image = ec2utils.get_os_image(context, image_id)
if not os_image:
image = db_api.get_item_by_id(context, image_id)
if image.get('state') != 'failed':
# TODO(ft): figure out corresponding AWS error
raise exception.IncorrectState(
reason='Image is still being created')
else:
_check_owner(context, os_image)
glance = clients.glance(context)
try:
glance.images.delete(os_image.id)
except glance_exception.HTTPNotFound:
pass
db_api.delete_item(context, image_id)
return True
def test_glance(self, glance):
context = mock.NonCallableMock(session=mock.sentinel.session)
res = clients.glance(context)
self.assertEqual(glance.return_value, res)
glance.assert_called_with("1", service_type="image", session=mock.sentinel.session)
def create_image(context,
instance_id,
name=None,
description=None,
no_reboot=False,
block_device_mapping=None):
instance = ec2utils.get_db_item(context, instance_id)
if not instance_api._is_ebs_instance(context, instance['os_id']):
msg = _('Instance does not have a volume attached at root (null).')
raise exception.InvalidParameterValue(value=instance_id,
parameter='InstanceId',
reason=msg)
nova = clients.nova(context)
os_instance = nova.servers.get(instance['os_id'])
restart_instance = False
if not no_reboot and os_instance.status != 'SHUTOFF':
if os_instance.status != 'ACTIVE':
# TODO(ft): Change the error code and message with the real AWS
# ones
msg = _('Instance must be run or stopped')
raise exception.IncorrectState(reason=msg)
restart_instance = True
# meaningful image name
name_map = dict(instance=instance['os_id'], now=timeutils.isotime())
name = name or _('image of %(instance)s at %(now)s') % name_map
def delayed_create(context, image, name, os_instance):
try:
os_instance.stop()
# wait instance for really stopped
start_time = time.time()
while os_instance.status != 'SHUTOFF':
time.sleep(1)
os_instance.get()
# NOTE(yamahata): timeout and error. 1 hour for now for safety.
# Is it too short/long?
# Or is there any better way?
timeout = 1 * 60 * 60
if time.time() > start_time + timeout:
err = (_("Couldn't stop instance within %d sec") % timeout)
raise exception.EC2Exception(message=err)
# NOTE(ft): create an image with ec2_id metadata to let other code
# link os and db objects in race conditions
os_image_id = os_instance.create_image(
name, metadata={'ec2_id': image['id']})
image['os_id'] = os_image_id
db_api.update_item(context, image)
except Exception:
LOG.exception(_LE('Failed to complete image %s creation'),
image.id)
try:
image['state'] = 'failed'
db_api.update_item(context, image)
except Exception:
LOG.warning(_LW("Couldn't set 'failed' state for db image %s"),
image.id,
exc_info=True)
try:
os_instance.start()
except Exception:
LOG.warning(_LW('Failed to start instance %(i_id)s after '
'completed creation of image %(image_id)s'), {
'i_id': instance['id'],
'image_id': image['id']
},
exc_info=True)
image = {'is_public': False, 'description': description}
if restart_instance:
# NOTE(ft): image type is hardcoded, because we don't know it now,
# but cannot change it later. But Nova doesn't specify container format
# for snapshots of volume backed instances, so that it is 'ami' in fact
image = db_api.add_item(context, 'ami', image)
eventlet.spawn_n(delayed_create, context, image, name, os_instance)
else:
glance = clients.glance(context)
with common.OnCrashCleaner() as cleaner:
os_image_id = os_instance.create_image(name)
cleaner.addCleanup(glance.images.delete, os_image_id)
# TODO(andrey-mp): snapshot and volume also must be deleted in case
# of error
os_image = glance.images.get(os_image_id)
image['os_id'] = os_image_id
image = db_api.add_item(context, _get_os_image_kind(os_image),
image)
return {'imageId': image['id']}
def create_image(context,
instance_id,
name=None,
description=None,
no_reboot=False,
block_device_mapping=None):
instance = ec2utils.get_db_item(context, instance_id)
if not instance_api._is_ebs_instance(context, instance['os_id']):
msg = _('Instance does not have a volume attached at root (null).')
raise exception.InvalidParameterValue(value=instance_id,
parameter='InstanceId',
reason=msg)
nova = clients.nova(context)
os_instance = nova.servers.get(instance['os_id'])
restart_instance = False
if not no_reboot and os_instance.status != 'SHUTOFF':
if os_instance.status != 'ACTIVE':
# TODO(ft): Change the error code and message with the real AWS
# ones
msg = _('Instance must be run or stopped')
raise exception.IncorrectState(reason=msg)
restart_instance = True
os_instance.stop()
# wait instance for really stopped
start_time = time.time()
while os_instance.status != 'SHUTOFF':
time.sleep(1)
os_instance.get()
# NOTE(yamahata): timeout and error. 1 hour for now for safety.
# Is it too short/long?
# Or is there any better way?
timeout = 1 * 60 * 60
if time.time() > start_time + timeout:
err = _("Couldn't stop instance within %d sec") % timeout
raise exception.EC2Exception(message=err)
# meaningful image name
name_map = dict(instance=instance['os_id'], now=timeutils.isotime())
name = name or _('image of %(instance)s at %(now)s') % name_map
glance = clients.glance(context)
with common.OnCrashCleaner() as cleaner:
os_image_id = os_instance.create_image(name)
cleaner.addCleanup(glance.images.delete, os_image_id)
# TODO(andrey-mp): snapshot and volume also must be deleted in case
# of error
os_image = glance.images.get(os_image_id)
image = db_api.add_item(context, _get_os_image_kind(os_image), {
'os_id': os_image_id,
'is_public': False,
'description': description
})
if restart_instance:
os_instance.start()
return {'imageId': image['id']}
def _s3_create(context, metadata):
"""Gets a manifest from s3 and makes an image."""
image_location = metadata['properties']['image_location'].lstrip('/')
bucket_name = image_location.split('/')[0]
manifest_path = image_location[len(bucket_name) + 1:]
bucket = _s3_conn(context).get_bucket(bucket_name)
key = bucket.get_key(manifest_path)
manifest = key.get_contents_as_string()
(image_metadata, image_parts, encrypted_key,
encrypted_iv) = _s3_parse_manifest(context, manifest)
properties = metadata['properties']
properties.update(image_metadata['properties'])
properties['image_state'] = 'pending'
metadata.update(image_metadata)
metadata.update({'properties': properties, 'is_public': False})
# TODO(bcwaldon): right now, this removes user-defined ids
# We need to re-enable this.
metadata.pop('id', None)
glance = clients.glance(context)
image = glance.images.create(**metadata)
def _update_image_state(image_state):
image.update(properties={'image_state': image_state})
def delayed_create():
"""This handles the fetching and decrypting of the part files."""
context.update_store()
try:
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
log_vars = {
'image_location': image_location,
'image_path': image_path
}
_update_image_state('downloading')
try:
parts = []
for part_name in image_parts:
part = _s3_download_file(bucket, part_name, image_path)
parts.append(part)
# NOTE(vish): this may be suboptimal, should we use cat?
enc_filename = os.path.join(image_path, 'image.encrypted')
with open(enc_filename, 'w') as combined:
for filename in parts:
with open(filename) as part:
shutil.copyfileobj(part, combined)
except Exception:
LOG.exception(
_LE('Failed to download %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_download')
return
_update_image_state('decrypting')
try:
dec_filename = os.path.join(image_path, 'image.tar.gz')
_s3_decrypt_image(context, enc_filename, encrypted_key,
encrypted_iv, dec_filename)
except Exception:
LOG.exception(
_LE('Failed to decrypt %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_decrypt')
return
_update_image_state('untarring')
try:
unz_filename = _s3_untarzip_image(image_path, dec_filename)
except Exception:
LOG.exception(
_LE('Failed to untar %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_untar')
return
_update_image_state('uploading')
try:
with open(unz_filename) as image_file:
image.update(data=image_file)
except Exception:
LOG.exception(
_LE('Failed to upload %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_upload')
return
_update_image_state('available')
shutil.rmtree(image_path)
except glance_exception.HTTPNotFound:
LOG.info(_LI('Image %swas deleted underneath us'), image.id)
except Exception:
LOG.exception(_LE('Failed to complete image %s creation'),
image.id)
eventlet.spawn_n(delayed_create)
return image
def get_os_items(self):
return clients.glance(self.context).images.list()
def register_image(context,
name=None,
image_location=None,
description=None,
architecture=None,
root_device_name=None,
block_device_mapping=None,
virtualization_type=None,
kernel_id=None,
ramdisk_id=None,
sriov_net_support=None):
if not image_location and not root_device_name:
# NOTE(ft): for backward compatibility with a hypothetical code
# which uses name as image_location
image_location = name
if not image_location and not root_device_name:
msg = _("Either imageLocation or rootDeviceName must be set.")
raise exception.InvalidParameterCombination(msg)
if not image_location and not name:
msg = _('The request must contain the parameter name')
raise exception.MissingParameter(msg)
# TODO(ft): check parameters
properties = {}
metadata = {'properties': properties}
if name:
# TODO(ft): check the name is unique (at least for EBS image case)
metadata['name'] = name
if image_location:
properties['image_location'] = image_location
if 'name' not in metadata:
# NOTE(ft): it's needed for backward compatibility
metadata['name'] = image_location
if root_device_name:
properties['root_device_name'] = root_device_name
if block_device_mapping:
mappings = [
instance_api._cloud_parse_block_device_mapping(context, bdm)
for bdm in block_device_mapping
]
properties['block_device_mapping'] = json.dumps(mappings)
if architecture is not None:
properties['architecture'] = architecture
if kernel_id:
properties['kernel_id'] = ec2utils.get_os_image(context, kernel_id).id
if ramdisk_id:
properties['ramdisk_id'] = ec2utils.get_os_image(context,
ramdisk_id).id
with common.OnCrashCleaner() as cleaner:
if 'image_location' in properties:
os_image = _s3_create(context, metadata)
else:
metadata.update({'size': 0, 'is_public': False})
# TODO(ft): set default values of image properties
glance = clients.glance(context)
os_image = glance.images.create(**metadata)
cleaner.addCleanup(os_image.delete)
kind = _get_os_image_kind(os_image)
image = db_api.add_item(context, kind, {
'os_id': os_image.id,
'is_public': False,
'description': description
})
return {'imageId': image['id']}
def _s3_create(context, metadata):
"""Gets a manifest from s3 and makes an image."""
image_location = metadata['properties']['image_location'].lstrip('/')
bucket_name = image_location.split('/')[0]
manifest_path = image_location[len(bucket_name) + 1:]
bucket = _s3_conn(context).get_bucket(bucket_name)
key = bucket.get_key(manifest_path)
manifest = key.get_contents_as_string()
(image_metadata, image_parts,
encrypted_key, encrypted_iv) = _s3_parse_manifest(context, manifest)
properties = metadata['properties']
properties.update(image_metadata['properties'])
properties['image_state'] = 'pending'
metadata.update(image_metadata)
metadata.update({'properties': properties,
'is_public': False})
# TODO(bcwaldon): right now, this removes user-defined ids
# We need to re-enable this.
metadata.pop('id', None)
glance = clients.glance(context)
image = glance.images.create(**metadata)
def _update_image_state(image_state):
image.update(properties={'image_state': image_state})
def delayed_create():
"""This handles the fetching and decrypting of the part files."""
context.update_store()
try:
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
log_vars = {'image_location': image_location,
'image_path': image_path}
_update_image_state('downloading')
try:
parts = []
for part_name in image_parts:
part = _s3_download_file(bucket, part_name, image_path)
parts.append(part)
# NOTE(vish): this may be suboptimal, should we use cat?
enc_filename = os.path.join(image_path, 'image.encrypted')
with open(enc_filename, 'w') as combined:
for filename in parts:
with open(filename) as part:
shutil.copyfileobj(part, combined)
except Exception:
LOG.exception(_LE('Failed to download %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_download')
return
_update_image_state('decrypting')
try:
dec_filename = os.path.join(image_path, 'image.tar.gz')
_s3_decrypt_image(context, enc_filename, encrypted_key,
encrypted_iv, dec_filename)
except Exception:
LOG.exception(_LE('Failed to decrypt %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_decrypt')
return
_update_image_state('untarring')
try:
unz_filename = _s3_untarzip_image(image_path, dec_filename)
except Exception:
LOG.exception(_LE('Failed to untar %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_untar')
return
_update_image_state('uploading')
try:
with open(unz_filename) as image_file:
image.update(data=image_file)
except Exception:
LOG.exception(_LE('Failed to upload %(image_location)s '
'to %(image_path)s'), log_vars)
_update_image_state('failed_upload')
return
_update_image_state('available')
shutil.rmtree(image_path)
except glance_exception.HTTPNotFound:
LOG.info(_LI('Image %swas deleted underneath us'), image.id)
except Exception:
LOG.exception(_LE('Failed to complete image %s creation'),
image.id)
eventlet.spawn_n(delayed_create)
return image
def create_image(context, instance_id, name=None, description=None,
no_reboot=False, block_device_mapping=None):
instance = ec2utils.get_db_item(context, instance_id)
if not instance_api._is_ebs_instance(context, instance['os_id']):
msg = _('Instance does not have a volume attached at root (null).')
raise exception.InvalidParameterValue(value=instance_id,
parameter='InstanceId',
reason=msg)
nova = clients.nova(context)
os_instance = nova.servers.get(instance['os_id'])
restart_instance = False
if not no_reboot and os_instance.status != 'SHUTOFF':
if os_instance.status != 'ACTIVE':
# TODO(ft): Change the error code and message with the real AWS
# ones
msg = _('Instance must be run or stopped')
raise exception.IncorrectState(reason=msg)
restart_instance = True
# meaningful image name
name_map = dict(instance=instance['os_id'], now=timeutils.isotime())
name = name or _('image of %(instance)s at %(now)s') % name_map
def delayed_create(context, image, name, os_instance):
try:
os_instance.stop()
# wait instance for really stopped
start_time = time.time()
while os_instance.status != 'SHUTOFF':
time.sleep(1)
os_instance.get()
# NOTE(yamahata): timeout and error. 1 hour for now for safety.
# Is it too short/long?
# Or is there any better way?
timeout = 1 * 60 * 60
if time.time() > start_time + timeout:
err = (_("Couldn't stop instance within %d sec") % timeout)
raise exception.EC2Exception(message=err)
# NOTE(ft): create an image with ec2_id metadata to let other code
# link os and db objects in race conditions
os_image_id = os_instance.create_image(
name, metadata={'ec2_id': image['id']})
image['os_id'] = os_image_id
db_api.update_item(context, image)
except Exception:
LOG.exception(_LE('Failed to complete image %s creation'),
image.id)
try:
image['state'] = 'failed'
db_api.update_item(context, image)
except Exception:
LOG.warning(_LW("Couldn't set 'failed' state for db image %s"),
image.id, exc_info=True)
try:
os_instance.start()
except Exception:
LOG.warning(_LW('Failed to start instance %(i_id)s after '
'completed creation of image %(image_id)s'),
{'i_id': instance['id'],
'image_id': image['id']},
exc_info=True)
image = {'is_public': False,
'description': description}
if restart_instance:
# NOTE(ft): image type is hardcoded, because we don't know it now,
# but cannot change it later. But Nova doesn't specify container format
# for snapshots of volume backed instances, so that it is 'ami' in fact
image = db_api.add_item(context, 'ami', image)
eventlet.spawn_n(delayed_create, context, image, name, os_instance)
else:
glance = clients.glance(context)
with common.OnCrashCleaner() as cleaner:
os_image_id = os_instance.create_image(name)
cleaner.addCleanup(glance.images.delete, os_image_id)
# TODO(andrey-mp): snapshot and volume also must be deleted in case
# of error
os_image = glance.images.get(os_image_id)
image['os_id'] = os_image_id
image = db_api.add_item(context, _get_os_image_kind(os_image),
image)
return {'imageId': image['id']}
def register_image(context, name=None, image_location=None,
description=None, architecture=None,
root_device_name=None, block_device_mapping=None,
virtualization_type=None, kernel_id=None,
ramdisk_id=None, sriov_net_support=None):
if not image_location and not root_device_name:
# NOTE(ft): for backward compatibility with a hypothetical code
# which uses name as image_location
image_location = name
if not image_location and not root_device_name:
msg = _("Either imageLocation or rootDeviceName must be set.")
raise exception.InvalidParameterCombination(msg)
if not image_location and not name:
msg = _('The request must contain the parameter name')
raise exception.MissingParameter(msg)
# TODO(ft): check parameters
properties = {}
metadata = {'properties': properties}
if name:
# TODO(ft): check the name is unique (at least for EBS image case)
metadata['name'] = name
if image_location:
properties['image_location'] = image_location
if 'name' not in metadata:
# NOTE(ft): it's needed for backward compatibility
metadata['name'] = image_location
if root_device_name:
properties['root_device_name'] = root_device_name
cinder = clients.cinder(context)
if block_device_mapping:
mappings = instance_api._parse_block_device_mapping(
context, block_device_mapping)
# TODO(ft): merge with image manifets's virtual device mappings
short_root_device_name = (
ec2utils.block_device_strip_dev(root_device_name))
for bdm in mappings:
instance_api._populate_parsed_bdm_parameter(
bdm, short_root_device_name)
if 'volume_size' in bdm:
continue
try:
if bdm['source_type'] == 'snapshot':
snapshot = cinder.volume_snapshots.get(bdm['snapshot_id'])
bdm['volume_size'] = snapshot.size
elif bdm['source_type'] == 'volume':
volume = cinder.volumes.get(bdm['volume_id'])
bdm['volume_size'] = volume.size
except cinder_exception.NotFound:
pass
properties['bdm_v2'] = True
properties['block_device_mapping'] = json.dumps(mappings)
if architecture is not None:
properties['architecture'] = architecture
if kernel_id:
properties['kernel_id'] = ec2utils.get_os_image(context,
kernel_id).id
if ramdisk_id:
properties['ramdisk_id'] = ec2utils.get_os_image(context,
ramdisk_id).id
with common.OnCrashCleaner() as cleaner:
if 'image_location' in properties:
os_image = _s3_create(context, metadata)
else:
metadata.update({'size': 0,
'is_public': False})
# TODO(ft): set default values of image properties
glance = clients.glance(context)
os_image = glance.images.create(**metadata)
cleaner.addCleanup(os_image.delete)
kind = _get_os_image_kind(os_image)
image = db_api.add_item(context, kind, {'os_id': os_image.id,
'is_public': False,
'description': description})
return {'imageId': image['id']}
