def _post_downgrade_012(self, engine):
images = get_table(engine, "images")
image_members = get_table(engine, "image_members")
image_properties = get_table(engine, "image_properties")
# Find kernel, ramdisk and normal images. Make sure id has been
# changed back to an integer
ids = {}
for name in ("kernel", "ramdisk", "normal"):
image_name = "%s migration 012 test" % name
rows = images.select().where(images.c.name == image_name).execute().fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertFalse(utils.is_uuid_like(row["id"]))
ids[name] = row["id"]
# Find all image_members to ensure image_id has been updated
results = image_members.select().where(image_members.c.image_id == ids["normal"]).execute().fetchall()
self.assertEqual(len(results), 1)
# Find all image_properties to ensure image_id has been updated
# as well as ensure kernel_id and ramdisk_id values have been
# updated too
results = image_properties.select().where(image_properties.c.image_id == ids["normal"]).execute().fetchall()
self.assertEqual(len(results), 2)
for row in results:
self.assertIn(row["name"], ("kernel_id", "ramdisk_id"))
if row["name"] == "kernel_id":
self.assertEqual(row["value"], str(ids["kernel"]))
if row["name"] == "ramdisk_id":
self.assertEqual(row["value"], str(ids["ramdisk"]))
def parse_uri(self, uri):
valid_schema = 'sheepdog://'
if not uri.startswith(valid_schema):
raise exception.BadStoreUri(_("URI must start with %s://") %
valid_schema)
self.image = uri[len(valid_schema):]
if not utils.is_uuid_like(self.image):
raise exception.BadStoreUri(_("URI must contains well-formated "
"image id"))
def _get_marker(self, req):
"""Parse a marker query param into something usable."""
marker = req.params.get('marker', None)
if marker and not utils.is_uuid_like(marker):
msg = _('Invalid marker format')
raise exc.HTTPBadRequest(explanation=msg)
return marker
def parse_uri(self, uri):
if not uri.startswith("cinder://"):
reason = _("URI must start with cinder://")
LOG.error(reason)
raise exception.BadStoreUri(uri, reason)
self.scheme = "cinder"
self.volume_id = uri[9:]
if not utils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID: %s") % self.volume_id
LOG.error(reason)
raise exception.BadStoreUri(uri, reason)
def parse_uri(self, uri):
if not uri.startswith('cinder://'):
reason = _("URI must start with 'cinder://'")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
self.scheme = 'cinder'
self.volume_id = uri[9:]
if not utils.is_uuid_like(self.volume_id):
reason = _("URI contains invalid volume ID")
LOG.info(reason)
raise exception.BadStoreUri(message=reason)
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:retval Returns the newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not utils.is_uuid_like(image_id):
msg = _LI("Rejecting image creation request for invalid image "
"id '%(bad_id)s'") % {'bad_id': image_id}
LOG.info(msg)
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if 'location' in image_data:
image_data['locations'] = [image_data.pop('location')]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
image_data = dict(image=make_image_dict(image_data))
msg = (_LI("Successfully created image %(id)s") %
image_data['image'])
LOG.info(msg)
return image_data
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.warn(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = (_("Failed to add image metadata. "
"Got error: %s") % utils.exception_to_str(e))
LOG.error(msg)
return exc.HTTPBadRequest(msg)
except Exception:
LOG.exception(_LE("Unable to create image %s"), image_id)
raise
def _walk_all_locations(self, remove=False):
"""Returns a list of image id and location tuple from scrub queue.
:param remove: Whether remove location from queue or not after walk
:retval a list of image id, location id and uri tuple from scrub queue
"""
if not os.path.exists(self.scrubber_datadir):
LOG.warn(_LW("%s directory does not exist.") %
self.scrubber_datadir)
return []
ret = []
for root, dirs, files in os.walk(self.scrubber_datadir):
for image_id in files:
if not utils.is_uuid_like(image_id):
continue
with lockutils.lock("scrubber-%s" % image_id,
lock_file_prefix='glance-', external=True):
file_path = os.path.join(self.scrubber_datadir, image_id)
records = self._read_queue_file(file_path)
loc_ids, uris, delete_times = records
remove_record_idxs = []
skipped = False
for (record_idx, delete_time) in enumerate(delete_times):
if delete_time > time.time():
skipped = True
continue
else:
ret.append((image_id,
loc_ids[record_idx],
uris[record_idx]))
remove_record_idxs.append(record_idx)
if remove:
if skipped:
# NOTE(zhiyan): remove location records from
# the queue file.
self._update_queue_file(file_path,
remove_record_idxs)
else:
utils.safe_remove(file_path)
return ret
def _check_012(self, engine, test_data):
images = get_table(engine, 'images')
image_members = get_table(engine, 'image_members')
image_properties = get_table(engine, 'image_properties')
# Find kernel, ramdisk and normal images. Make sure id has been
# changed to a uuid
uuids = {}
for name in ('kernel', 'ramdisk', 'normal'):
image_name = '%s migration 012 test' % name
rows = images.select()\
.where(images.c.name == image_name)\
.execute().fetchall()
self.assertEqual(len(rows), 1)
row = rows[0]
self.assertTrue(utils.is_uuid_like(row['id']))
uuids[name] = row['id']
# Find all image_members to ensure image_id has been updated
results = image_members.select()\
.where(image_members.c.image_id ==
uuids['normal'])\
.execute().fetchall()
self.assertEqual(len(results), 1)
# Find all image_properties to ensure image_id has been updated
# as well as ensure kernel_id and ramdisk_id values have been
# updated too
results = image_properties.select()\
.where(image_properties.c.image_id ==
uuids['normal'])\
.execute().fetchall()
self.assertEqual(len(results), 2)
for row in results:
self.assertIn(row['name'], ('kernel_id', 'ramdisk_id'))
if row['name'] == 'kernel_id':
self.assertEqual(row['value'], uuids['kernel'])
if row['name'] == 'ramdisk_id':
self.assertEqual(row['value'], uuids['ramdisk'])
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:retval Returns the newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body["image"]
# Ensure the image has a status set
image_data.setdefault("status", "active")
# Set up the image owner
if not req.context.is_admin or "owner" not in image_data:
image_data["owner"] = req.context.owner
image_id = image_data.get("id")
if image_id and not utils.is_uuid_like(image_id):
msg = _("Rejecting image creation request for invalid image " "id '%(bad_id)s'")
LOG.info(msg % {"bad_id": image_id})
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if "location" in image_data:
image_data["locations"] = [image_data.pop("location")]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
msg = _("Successfully created image %(id)s") % {"id": image_id}
LOG.info(msg)
return dict(image=make_image_dict(image_data))
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.error(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = _("Failed to add image metadata. " "Got error: %(e)s") % {"e": e}
LOG.error(msg)
return exc.HTTPBadRequest(msg)
def create(self, req, body):
"""
Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:retval Returns the newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
if req.context.read_only:
raise exc.HTTPForbidden()
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not utils.is_uuid_like(image_id):
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
try:
image_data = db_api.image_create(req.context, image_data)
return dict(image=make_image_dict(image_data))
except exception.Duplicate:
msg = (_("Image with identifier %s already exists!") % id)
logger.error(msg)
return exc.HTTPConflict(msg)
except exception.Invalid, e:
msg = (_("Failed to add image metadata. "
"Got error: %(e)s") % locals())
logger.error(msg)
return exc.HTTPBadRequest(msg)
def _post_downgrade_012(self, engine):
images = db_utils.get_table(engine, 'images')
image_members = db_utils.get_table(engine, 'image_members')
image_properties = db_utils.get_table(engine, 'image_properties')
# Find kernel, ramdisk and normal images. Make sure id has been
# changed back to an integer
ids = {}
for name in ('kernel', 'ramdisk', 'normal'):
image_name = '%s migration 012 test' % name
rows = images.select().where(
images.c.name == image_name).execute().fetchall()
self.assertEqual(1, len(rows))
row = rows[0]
self.assertFalse(utils.is_uuid_like(row['id']))
ids[name] = row['id']
# Find all image_members to ensure image_id has been updated
results = image_members.select().where(
image_members.c.image_id == ids['normal']).execute().fetchall()
self.assertEqual(1, len(results))
# Find all image_properties to ensure image_id has been updated
# as well as ensure kernel_id and ramdisk_id values have been
# updated too
results = image_properties.select().where(
image_properties.c.image_id == ids['normal']).execute().fetchall()
self.assertEqual(2, len(results))
for row in results:
self.assertIn(row['name'], ('kernel_id', 'ramdisk_id'))
if row['name'] == 'kernel_id':
self.assertEqual(row['value'], str(ids['kernel']))
if row['name'] == 'ramdisk_id':
self.assertEqual(row['value'], str(ids['ramdisk']))
def replication_load(options, args):
"""%(prog)s load <server:port> <path>
Load the contents of a local directory into glance.
server:port: the location of the glance instance.
path: a directory on disk containing the data.
"""
# Make sure server and path are provided
if len(args) < 2:
raise TypeError(_("Too few arguments."))
path = args.pop()
server, port = utils.parse_valid_host_port(args.pop())
imageservice = get_image_service()
client = imageservice(httplib.HTTPConnection(server, port),
options.slavetoken)
updated = []
for ent in os.listdir(path):
if utils.is_uuid_like(ent):
image_uuid = ent
LOG.info(_LI('Considering: %s') % image_uuid)
meta_file_name = os.path.join(path, image_uuid)
with open(meta_file_name) as meta_file:
meta = jsonutils.loads(meta_file.read())
# Remove keys which don't make sense for replication
for key in options.dontreplicate.split(' '):
if key in meta:
LOG.debug('Stripping %(header)s from saved '
'metadata', {'header': key})
del meta[key]
if _image_present(client, image_uuid):
# NOTE(mikal): Perhaps we just need to update the metadata?
# Note that we don't attempt to change an image file once it
# has been uploaded.
LOG.debug('Image %s already present', image_uuid)
headers = client.get_image_meta(image_uuid)
for key in options.dontreplicate.split(' '):
if key in headers:
LOG.debug('Stripping %(header)s from slave '
'metadata', {'header': key})
del headers[key]
if _dict_diff(meta, headers):
LOG.info(_LI('... metadata has changed'))
headers, body = client.add_image_meta(meta)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
else:
if not os.path.exists(os.path.join(path, image_uuid + '.img')):
LOG.info(_LI('... dump is missing image data, skipping'))
continue
# Upload the image itself
with open(os.path.join(path, image_uuid + '.img')) as img_file:
try:
headers, body = client.add_image(meta, img_file)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
except ImageAlreadyPresentException:
LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE)
% image_uuid)
return updated
def test_is_uuid_like_fails(self):
fixture = 'pants'
self.assertFalse(utils.is_uuid_like(fixture))
def test_is_uuid_like_success(self):
fixture = 'b694bf02-6b01-4905-a50e-fcf7bce7e4d2'
self.assertTrue(utils.is_uuid_like(fixture))
def test_id_is_uuid_like(self):
self.assertFalse(utils.is_uuid_like(1234567))
def _validate_marker(self, marker):
if marker and not utils.is_uuid_like(marker):
msg = _('Invalid marker format')
raise webob.exc.HTTPBadRequest(explanation=msg)
return marker
def test_is_uuid_like_fails(self):
fixture = 'pants'
self.assertFalse(utils.is_uuid_like(fixture))
def test_is_uuid_like_success(self):
fixture = 'b694bf02-6b01-4905-a50e-fcf7bce7e4d2'
self.assertTrue(utils.is_uuid_like(fixture))
def test_is_uuid_like(self):
self.assertTrue(utils.is_uuid_like(str(uuid.uuid4())))
def replication_load(options, args):
"""%(prog)s load <server:port> <path>
Load the contents of a local directory into glance.
server:port: the location of the glance instance.
path: a directory on disk containing the data.
"""
# Make sure server and path are provided
if len(args) < 2:
raise TypeError(_("Too few arguments."))
path = args.pop()
server, port = utils.parse_valid_host_port(args.pop())
imageservice = get_image_service()
client = imageservice(http_client.HTTPConnection(server, port),
options.slavetoken)
updated = []
for ent in os.listdir(path):
if utils.is_uuid_like(ent):
image_uuid = ent
LOG.info(_LI('Considering: %s'), image_uuid)
meta_file_name = os.path.join(path, image_uuid)
with open(meta_file_name) as meta_file:
meta = jsonutils.loads(meta_file.read())
# Remove keys which don't make sense for replication
for key in options.dontreplicate.split(' '):
if key in meta:
LOG.debug('Stripping %(header)s from saved '
'metadata', {'header': key})
del meta[key]
if _image_present(client, image_uuid):
# NOTE(mikal): Perhaps we just need to update the metadata?
# Note that we don't attempt to change an image file once it
# has been uploaded.
LOG.debug('Image %s already present', image_uuid)
headers = client.get_image_meta(image_uuid)
for key in options.dontreplicate.split(' '):
if key in headers:
LOG.debug('Stripping %(header)s from slave '
'metadata', {'header': key})
del headers[key]
if _dict_diff(meta, headers):
LOG.info(_LI('Image %s metadata has changed'), image_uuid)
headers, body = client.add_image_meta(meta)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
else:
if not os.path.exists(os.path.join(path, image_uuid + '.img')):
LOG.debug('%s dump is missing image data, skipping' %
image_uuid)
continue
# Upload the image itself
with open(os.path.join(path, image_uuid + '.img')) as img_file:
try:
headers, body = client.add_image(meta, img_file)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
except exc.HTTPConflict:
LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE)
% image_uuid) # noqa
return updated
def test_name_is_uuid_like(self):
self.assertFalse(utils.is_uuid_like('zhongyueluo'))
def _validate_marker(self, marker):
if marker and not utils.is_uuid_like(marker):
msg = _('Invalid marker format')
raise webob.exc.HTTPBadRequest(explanation=msg)
return marker
