def handle_unpaired_item(self, item): if item['os_id']: return super(ImageDescriber, self).handle_unpaired_item(item) if 'is_public' not in item: return None # NOTE(ft): process creating images, ignoring ids mapping # NOTE(ft): the image is being creating, Glance had created # image, but creating thread doesn't yet update db item os_image = self.ec2_created_os_images.get(item['id']) if os_image: item['os_id'] = os_image.id item['is_public'] = os_image.is_public db_api.update_item(self.context, item) image = self.format(item, os_image) else: # NOTE(ft): Glance image is not yet created, but DB item # exists. So that we adds EC2 image to output results # with all data we have. # TODO(ft): check if euca2ools can process such result image = {'imageId': item['id'], 'imageOwnerId': self.context.project_id, 'imageType': IMAGE_TYPES[ ec2utils.get_ec2_id_kind(item['id'])], 'isPublic': item['is_public']} if 'description' in item: image['description'] = item['description'] image['imageState'] = item.get('state', 'pending') return image
def auto_update_db(self, image, os_image): if not image: kind = _get_os_image_kind(os_image) if self.context.project_id == os_image.owner: image = ec2utils.get_db_item_by_os_id(self.context, kind, os_image.id, self.items_dict, os_image=os_image) else: image_id = ec2utils.os_id_to_ec2_id( self.context, kind, os_image.id, items_by_os_id=self.items_dict, ids_by_os_id=self.ids_dict) image = {'id': image_id, 'os_id': os_image.id} elif (self.context.project_id == os_image.owner and image.get('is_public') != os_image.is_public): image['is_public'] = os_image.is_public if image['id'] in self.local_images_os_ids: db_api.update_item(self.context, image) else: # TODO(ft): currently update_item can not update id mapping, # because its project_id is None. Instead of upgrade db_api, # we use add_item. But its execution leads to a needless # DB call. This should be reworked in the future. kind = ec2utils.get_ec2_id_kind(image['id']) db_api.add_item(self.context, kind, image) return image
def handle_unpaired_item(self, item): if item['os_id']: return super(ImageDescriber, self).handle_unpaired_item(item) if 'is_public' not in item: return None # NOTE(ft): process creating images, ignoring ids mapping # NOTE(ft): the image is being creating, Glance had created # image, but creating thread doesn't yet update db item os_image = self.ec2_created_os_images.get(item['id']) if os_image: item['os_id'] = os_image.id item['is_public'] = os_image.visibility == 'public' db_api.update_item(self.context, item) image = self.format(item, os_image) else: # NOTE(ft): Glance image is not yet created, but DB item # exists. So that we adds EC2 image to output results # with all data we have. # TODO(ft): check if euca2ools can process such result image = { 'imageId': item['id'], 'imageOwnerId': self.context.project_id, 'imageType': IMAGE_TYPES[ec2utils.get_ec2_id_kind(item['id'])], 'isPublic': item['is_public'] } if 'description' in item: image['description'] = item['description'] image['imageState'] = item.get('state', 'pending') return image
def auto_update_db(self, image, os_image): if not image: kind = _get_os_image_kind(os_image) if self.context.project_id == os_image.owner: if os_image.properties.get('ec2_id') in self.pending_images: # NOTE(ft): the image is being creating, Glance had created # image, but creating thread doesn't yet update db item image = self.pending_images[os_image.properties['ec2_id']] image['os_id'] = os_image.id image['is_public'] = os_image.is_public db_api.update_item(self.context, image) else: image = ec2utils.get_db_item_by_os_id( self.context, kind, os_image.id, self.items_dict, os_image=os_image) else: image_id = ec2utils.os_id_to_ec2_id( self.context, kind, os_image.id, items_by_os_id=self.items_dict, ids_by_os_id=self.ids_dict) image = {'id': image_id, 'os_id': os_image.id} elif (self.context.project_id == os_image.owner and image.get('is_public') != os_image.is_public): image['is_public'] = os_image.is_public if image['id'] in self.local_images_os_ids: db_api.update_item(self.context, image) else: # TODO(ft): currently update_item can not update id mapping, # because its project_id is None. Instead of upgrade db_api, # we use add_item. But its execution leads to a needless # DB call. This should be reworked in the future. kind = ec2utils.get_ec2_id_kind(image['id']) db_api.add_item(self.context, kind, image) return image
def _format_tag(tag): kind = ec2utils.get_ec2_id_kind(tag['item_id']) return { 'resourceType': RESOURCE_TYPES.get(kind, kind), 'resourceId': tag['item_id'], 'key': tag['key'], 'value': tag['value'], }
def check_tag_support(self, operation, resultset_key, sample_item_id, id_key, item_kinds=[]): self.db_api.get_tags = tools.CopyingMock( return_value=[{ 'item_id': sample_item_id, 'key': 'fake_key', 'value': 'fake_value' }]) ec2_tags = [{'key': 'fake_key', 'value': 'fake_value'}] resp = self.execute(operation, {}) tag_found = False if type(resultset_key) is list: resp_items = itertools.chain(*(r[resultset_key[1]] for r in resp[resultset_key[0]])) else: resp_items = resp[resultset_key] resultset_key = [resultset_key] for resp_item in resp_items: if resp_item.get(id_key) == sample_item_id: self.assertIn('tagSet', resp_item) self.assertThat(resp_item['tagSet'], matchers.ListMatches(ec2_tags)) tag_found = True else: self.assertTrue('tagSet' not in resp_item or resp_item['tagSet'] == []) self.assertTrue(tag_found) if not item_kinds: item_kinds = (ec2utils.get_ec2_id_kind(sample_item_id), ) self.assertTrue(self.db_api.get_tags.call_count == 1 and (self.db_api.get_tags.mock_calls[0] in (mock.call( mock.ANY, item_kinds, set()), mock.call(mock.ANY, item_kinds, None)))) self.db_api.reset_mock() id_param = '%s%s.1' % (id_key[0].capitalize(), id_key[1:]) resp = self.execute(operation, {id_param: sample_item_id}) self.assertTrue( self.db_api.get_tags.call_count == 1 and (self.db_api.get_tags.mock_calls[0] in (mock.call(mock.ANY, item_kinds, set([sample_item_id])), mock.call(mock.ANY, item_kinds, [sample_item_id])))) self.check_filtering(operation, resultset_key[0], [('tag-key', 'fake_key'), ('tag-value', 'fake_value'), ('tag:fake_key', 'fake_value')])
def check_tag_support(self, operation, resultset_key, sample_item_id, id_key, item_kinds=[]): self.db_api.get_tags = tools.CopyingMock( return_value=[{'item_id': sample_item_id, 'key': 'fake_key', 'value': 'fake_value'}]) ec2_tags = [{'key': 'fake_key', 'value': 'fake_value'}] resp = self.execute(operation, {}) tag_found = False if type(resultset_key) is list: resp_items = itertools.chain(*(r[resultset_key[1]] for r in resp[resultset_key[0]])) else: resp_items = resp[resultset_key] resultset_key = [resultset_key] for resp_item in resp_items: if resp_item.get(id_key) == sample_item_id: self.assertIn('tagSet', resp_item) self.assertThat(resp_item['tagSet'], matchers.ListMatches(ec2_tags)) tag_found = True else: self.assertTrue('tagSet' not in resp_item or resp_item['tagSet'] == []) self.assertTrue(tag_found) if not item_kinds: item_kinds = (ec2utils.get_ec2_id_kind(sample_item_id),) self.assertTrue(self.db_api.get_tags.call_count == 1 and (self.db_api.get_tags.mock_calls[0] in (mock.call(mock.ANY, item_kinds, set()), mock.call(mock.ANY, item_kinds, None)))) self.db_api.reset_mock() id_param = '%s%s.1' % (id_key[0].capitalize(), id_key[1:]) resp = self.execute(operation, {id_param: sample_item_id}) self.assertTrue( self.db_api.get_tags.call_count == 1 and (self.db_api.get_tags.mock_calls[0] in (mock.call(mock.ANY, item_kinds, set([sample_item_id])), mock.call(mock.ANY, item_kinds, [sample_item_id])))) self.check_filtering( operation, resultset_key[0], [('tag-key', 'fake_key'), ('tag-value', 'fake_value'), ('tag:fake_key', 'fake_value')])
def _get_active_route_destinations(context, route_table): vpn_connections = {vpn['vpn_gateway_id']: vpn for vpn in db_api.get_items(context, 'vpn')} dst_ids = [route[id_key] for route in route_table['routes'] for id_key in ('gateway_id', 'network_interface_id') if route.get(id_key) is not None] dst_ids.extend(route_table.get('propagating_gateways', [])) destinations = {item['id']: item for item in db_api.get_items_by_ids(context, dst_ids) if (item['vpc_id'] == route_table['vpc_id'] and (ec2utils.get_ec2_id_kind(item['id']) != 'vgw' or item['id'] in vpn_connections))} for vpn in six.itervalues(vpn_connections): if vpn['vpn_gateway_id'] in destinations: destinations[vpn['vpn_gateway_id']]['vpn_connection'] = vpn return destinations
def auto_update_db(self, image, os_image): if not image: kind = _get_os_image_kind(os_image) if self.context.project_id == os_image.owner: if getattr(os_image, 'ec2_id', None) in self.pending_images: # NOTE(ft): the image is being creating, Glance had created # image, but creating thread doesn't yet update db item image = self.pending_images[os_image.ec2_id] image['os_id'] = os_image.id image['is_public'] = os_image.visibility == 'public' db_api.update_item(self.context, image) else: image = ec2utils.get_db_item_by_os_id(self.context, kind, os_image.id, self.items_dict, os_image=os_image) else: image_id = ec2utils.os_id_to_ec2_id( self.context, kind, os_image.id, items_by_os_id=self.items_dict, ids_by_os_id=self.ids_dict) image = {'id': image_id, 'os_id': os_image.id} elif (self.context.project_id == os_image.owner and image.get('is_public') != os_image.visibility == 'public'): image['is_public'] = os_image.visibility == 'public' if image['id'] in self.local_images_os_ids: db_api.update_item(self.context, image) else: # TODO(ft): currently update_item can not update id mapping, # because its project_id is None. Instead of upgrade db_api, # we use add_item. But its execution leads to a needless # DB call. This should be reworked in the future. kind = ec2utils.get_ec2_id_kind(image['id']) db_api.add_item(self.context, kind, image) return image
def create_tags(context, resource_id, tag): reason = None for tag_pair in tag: if not tag_pair.get('key'): reason = _('Not empty key must be present') elif len(tag_pair['key']) > 127: reason = _('Tag key exceeds the maximum length of 127 characters') elif tag_pair['key'].startswith('aws:'): reason = _("Tag keys starting with 'aws:' are reserved for " "internal use") elif 'value' not in tag_pair: reason = _('Value must be present') elif len(tag_pair['value']) > 255: reason = _('Tag value exceeds the maximum length of 255 ' 'characters') if reason: raise exception.InvalidParameterValue( parameter='Tag', value=str(tag_pair), reason=reason) for item_id in resource_id: kind = ec2utils.get_ec2_id_kind(item_id) if kind not in RESOURCE_TYPES: raise exception.InvalidID(id=item_id) # NOTE(ft): check items exist (excluding images because AWS allows to # create a tag with any image id) if kind not in ('ami', 'ari', 'aki'): ec2utils.get_db_item(context, item_id) tags = [dict(item_id=item_id, key=tag_pair['key'], value=tag_pair['value']) for item_id in resource_id for tag_pair in tag] db_api.add_tags(context, tags) return True
def create_tags(context, resource_id, tag): reason = None for tag_pair in tag: if not tag_pair.get('key'): reason = _('Not empty key must be present') elif len(tag_pair['key']) > 127: reason = _('Tag key exceeds the maximum length of 127 characters') elif tag_pair['key'].startswith('aws:'): reason = _("Tag keys starting with 'aws:' are reserved for " "internal use") elif 'value' not in tag_pair: reason = _('Value must be present') elif len(tag_pair['value']) > 255: reason = _('Tag value exceeds the maximum length of 255 ' 'characters') if reason: raise exception.InvalidParameterValue(parameter='Tag', value=str(tag_pair), reason=reason) for item_id in resource_id: kind = ec2utils.get_ec2_id_kind(item_id) if kind not in RESOURCE_TYPES: raise exception.InvalidID(id=item_id) # NOTE(ft): check items exist (excluding images because AWS allows to # create a tag with any image id) if kind not in ('jmi', 'jri', 'jki'): ec2utils.get_db_item(context, item_id) tags = [ dict(item_id=item_id, key=tag_pair['key'], value=tag_pair['value']) for item_id in resource_id for tag_pair in tag ] db_api.add_tags(context, tags) return True
def _format_image(context, image, os_image, images_dict, ids_dict, snapshot_ids=None): ec2_image = { 'imageId': image['id'], 'imageOwnerId': os_image.owner, 'imageType': IMAGE_TYPES[ec2utils.get_ec2_id_kind(image['id'])], 'isPublic': os_image.is_public, 'architecture': os_image.properties.get('architecture'), 'creationDate': os_image.created_at } if 'description' in image: ec2_image['description'] = image['description'] if 'state' in image: state = image['state'] else: state = GLANCE_STATUS_TO_EC2.get(os_image.status, 'error') if state in ('available', 'pending'): state = _s3_image_state_map.get(os_image.properties.get('image_state'), state) ec2_image['imageState'] = state kernel_id = os_image.properties.get('kernel_id') if kernel_id: ec2_image['kernelId'] = ec2utils.os_id_to_ec2_id( context, 'aki', kernel_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) ramdisk_id = os_image.properties.get('ramdisk_id') if ramdisk_id: ec2_image['ramdiskId'] = ec2utils.os_id_to_ec2_id( context, 'ari', ramdisk_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) name = os_image.name img_loc = os_image.properties.get('image_location') if img_loc: ec2_image['imageLocation'] = img_loc else: ec2_image['imageLocation'] = "%s (%s)" % (img_loc, name) if not name and img_loc: # This should only occur for images registered with ec2 api # prior to that api populating the glance name ec2_image['name'] = img_loc else: ec2_image['name'] = name properties = ec2utils.deserialize_os_image_properties(os_image) root_device_name = ( ec2utils.block_device_properties_root_device_name(properties)) mappings = _format_mappings(context, properties, root_device_name, snapshot_ids, os_image.owner) if mappings: ec2_image['blockDeviceMapping'] = mappings root_device_type = 'instance-store' if root_device_name: ec2_image['rootDeviceName'] = root_device_name short_root_device_name = ec2utils.block_device_strip_dev( root_device_name) if any((short_root_device_name == ec2utils.block_device_strip_dev( bdm.get('deviceName'))) for bdm in mappings): root_device_type = 'ebs' ec2_image['rootDeviceType'] = root_device_type return ec2_image
def _set_route(context, route_table_id, destination_cidr_block, gateway_id, instance_id, network_interface_id, vpc_peering_connection_id, do_replace): route_table = ec2utils.get_db_item(context, route_table_id) vpc = db_api.get_item_by_id(context, route_table['vpc_id']) vpc_ipnet = netaddr.IPNetwork(vpc['cidr_block']) route_ipnet = netaddr.IPNetwork(destination_cidr_block) if route_ipnet in vpc_ipnet: msg = _('Cannot create a more specific route for ' '%(destination_cidr_block)s than local route ' '%(vpc_cidr_block)s in route table %(rtb_id)s') msg = msg % {'rtb_id': route_table_id, 'destination_cidr_block': destination_cidr_block, 'vpc_cidr_block': vpc['cidr_block']} raise exception.InvalidParameterValue(msg) obj_param_count = len([p for p in (gateway_id, network_interface_id, instance_id, vpc_peering_connection_id) if p is not None]) if obj_param_count != 1: msg = _('The request must contain exactly one of gatewayId, ' 'networkInterfaceId, vpcPeeringConnectionId or instanceId') if obj_param_count == 0: raise exception.MissingParameter(msg) else: raise exception.InvalidParameterCombination(msg) rollabck_route_table_state = copy.deepcopy(route_table) if do_replace: route_index, old_route = next( ((i, r) for i, r in enumerate(route_table['routes']) if r['destination_cidr_block'] == destination_cidr_block), (None, None)) if route_index is None: msg = _("There is no route defined for " "'%(destination_cidr_block)s' in the route table. " "Use CreateRoute instead.") msg = msg % {'destination_cidr_block': destination_cidr_block} raise exception.InvalidParameterValue(msg) else: del route_table['routes'][route_index] if gateway_id: gateway = ec2utils.get_db_item(context, gateway_id) if gateway.get('vpc_id') != route_table['vpc_id']: if ec2utils.get_ec2_id_kind(gateway_id) == 'vgw': raise exception.InvalidGatewayIDNotFound(id=gateway['id']) else: # igw raise exception.InvalidParameterValue( _('Route table %(rtb_id)s and network gateway %(igw_id)s ' 'belong to different networks') % {'rtb_id': route_table_id, 'igw_id': gateway_id}) route = {'gateway_id': gateway['id']} elif network_interface_id: network_interface = ec2utils.get_db_item(context, network_interface_id) if network_interface['vpc_id'] != route_table['vpc_id']: msg = _('Route table %(rtb_id)s and interface %(eni_id)s ' 'belong to different networks') msg = msg % {'rtb_id': route_table_id, 'eni_id': network_interface_id} raise exception.InvalidParameterValue(msg) route = {'network_interface_id': network_interface['id']} elif instance_id: # TODO(ft): implement search in DB layer network_interfaces = [eni for eni in db_api.get_items(context, 'eni') if eni.get('instance_id') == instance_id] if len(network_interfaces) == 0: msg = _("Invalid value '%(i_id)s' for instance ID. " "Instance is not in a VPC.") msg = msg % {'i_id': instance_id} raise exception.InvalidParameterValue(msg) elif len(network_interfaces) > 1: raise exception.InvalidInstanceId(instance_id=instance_id) network_interface = network_interfaces[0] if network_interface['vpc_id'] != route_table['vpc_id']: msg = _('Route table %(rtb_id)s and interface %(eni_id)s ' 'belong to different networks') msg = msg % {'rtb_id': route_table_id, 'eni_id': network_interface['id']} raise exception.InvalidParameterValue(msg) route = {'network_interface_id': network_interface['id']} else: raise exception.InvalidRequest('Parameter VpcPeeringConnectionId is ' 'not supported by this implementation') route['destination_cidr_block'] = destination_cidr_block update_target = _get_route_target(route) if do_replace: idempotent_call = False old_target = _get_route_target(old_route) if old_target != update_target: update_target = None else: old_route = next((r for r in route_table['routes'] if r['destination_cidr_block'] == destination_cidr_block), None) idempotent_call = old_route == route if old_route and not idempotent_call: raise exception.RouteAlreadyExists( destination_cidr_block=destination_cidr_block) if not idempotent_call: route_table['routes'].append(route) with common.OnCrashCleaner() as cleaner: db_api.update_item(context, route_table) cleaner.addCleanup(db_api.update_item, context, rollabck_route_table_state) _update_routes_in_associated_subnets(context, cleaner, route_table, update_target=update_target) return True
def _format_image(context, image, os_image, images_dict, ids_dict, snapshot_ids=None): ec2_image = { 'imageId': image['id'], 'imageType': IMAGE_TYPES[ec2utils.get_ec2_id_kind(image['id'])], 'isPublic': os_image.is_public, 'architecture': os_image.properties.get('architecture'), } if 'description' in image: ec2_image['description'] = image['description'] state = os_image.status # NOTE(vish): fallback status if image_state isn't set if state == 'active': state = 'available' ec2_image['imageState'] = os_image.properties.get('image_state', state) kernel_id = os_image.properties.get('kernel_id') if kernel_id: ec2_image['kernelId'] = ec2utils.os_id_to_ec2_id( context, 'jki', kernel_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) ramdisk_id = os_image.properties.get('ramdisk_id') if ramdisk_id: ec2_image['ramdiskId'] = ec2utils.os_id_to_ec2_id( context, 'jri', ramdisk_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) name = os_image.name # Commenting out the image location code as it is not required # in the first release # img_loc = os_image.properties.get('image_location') # if img_loc: # ec2_image['imageLocation'] = img_loc # else: # ec2_image['imageLocation'] = "%s (%s)" % (img_loc, name) # Glance allows image names to be empty if name: ec2_image['name'] = name else: # Corner case: no image name which is allowed in glance ec2_image['name'] = "" _prepare_mappings(os_image) properties = os_image.properties root_bdm = _prepare_root_bdm_for_image(os_image) ec2_image['blockDeviceMapping'] = root_bdm # Commenting the root device name and root device type from the response # Will remove any dead code in next iteration root_device_name = _block_device_properties_root_device_name(properties) if root_device_name: # ec2_image['rootDeviceName'] = root_device_name root_device_type = 'instance-store' short_root_device_name = instance_api._block_device_strip_dev( root_device_name) for bdm in properties.get('block_device_mapping', []): if (('snapshot_id' in bdm or 'volume_id' in bdm) and not bdm.get('no_device') and (bdm.get('boot_index') == 0 or short_root_device_name == instance_api._block_device_strip_dev( bdm.get('device_name')))): root_device_type = 'ebs' break # ec2_image['rootDeviceType'] = root_device_type _cloud_format_mappings(context, properties, ec2_image, root_device_name, snapshot_ids, os_image.owner) return ec2_image
def db_api_get_items_ids(context, kind, item_ids=None, item_os_ids=None): return [(item['id'], item['os_id']) for item in items if (ec2utils.get_ec2_id_kind(item['id']) == kind and ( not item_ids or item['id'] in item_ids) and ( not item_os_ids or item['os_id'] in item_os_ids))]
def db_api_get_items(context, kind): return [ copy.deepcopy(item) for item in items if ec2utils.get_ec2_id_kind(item['id']) == kind ]
def _get_route_target(route): if ec2utils.get_ec2_id_kind(route.get('gateway_id') or '') == 'vgw': return VPN_TARGET else: return HOST_TARGET
def db_api_get_items(context, kind): return [copy.deepcopy(item) for item in items if ec2utils.get_ec2_id_kind(item['id']) == kind]
def db_api_get_items_ids(context, kind, item_ids=None, item_os_ids=None): return [(item['id'], item['os_id']) for item in items if (ec2utils.get_ec2_id_kind(item['id']) == kind and (not item_ids or item['id'] in item_ids) and (not item_os_ids or item['os_id'] in item_os_ids))]
def _format_image(context, image, os_image, images_dict, ids_dict, snapshot_ids=None): ec2_image = {'imageId': image['id'], 'imageOwnerId': os_image.owner, 'imageType': IMAGE_TYPES[ ec2utils.get_ec2_id_kind(image['id'])], 'isPublic': os_image.is_public, 'architecture': os_image.properties.get('architecture'), 'creationDate': os_image.created_at } if 'description' in image: ec2_image['description'] = image['description'] if 'state' in image: state = image['state'] else: state = GLANCE_STATUS_TO_EC2.get(os_image.status, 'error') if state in ('available', 'pending'): state = _s3_image_state_map.get(os_image.properties.get('image_state'), state) ec2_image['imageState'] = state kernel_id = os_image.properties.get('kernel_id') if kernel_id: ec2_image['kernelId'] = ec2utils.os_id_to_ec2_id( context, 'aki', kernel_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) ramdisk_id = os_image.properties.get('ramdisk_id') if ramdisk_id: ec2_image['ramdiskId'] = ec2utils.os_id_to_ec2_id( context, 'ari', ramdisk_id, items_by_os_id=images_dict, ids_by_os_id=ids_dict) name = os_image.name img_loc = os_image.properties.get('image_location') if img_loc: ec2_image['imageLocation'] = img_loc else: ec2_image['imageLocation'] = "%s (%s)" % (img_loc, name) if not name and img_loc: # This should only occur for images registered with ec2 api # prior to that api populating the glance name ec2_image['name'] = img_loc else: ec2_image['name'] = name properties = ec2utils.deserialize_os_image_properties(os_image) root_device_name = ( ec2utils.block_device_properties_root_device_name(properties)) mappings = _format_mappings(context, properties, root_device_name, snapshot_ids, os_image.owner) if mappings: ec2_image['blockDeviceMapping'] = mappings root_device_type = 'instance-store' if root_device_name: ec2_image['rootDeviceName'] = root_device_name short_root_device_name = ec2utils.block_device_strip_dev( root_device_name) if any((short_root_device_name == ec2utils.block_device_strip_dev(bdm.get('deviceName'))) for bdm in mappings): root_device_type = 'ebs' ec2_image['rootDeviceType'] = root_device_type return ec2_image