def replication_compare(options, args): """%(prog)s compare <fromserver:port> <toserver:port> Compare the contents of fromserver with those of toserver. fromserver:port: the location of the master glance instance. toserver:port: the location of the slave glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() slave_server, slave_port = utils.parse_valid_host_port(args.pop()) slave_conn = httplib.HTTPConnection(slave_server, slave_port) slave_client = imageservice(slave_conn, options.slavetoken) master_server, master_port = utils.parse_valid_host_port(args.pop()) master_conn = httplib.HTTPConnection(master_server, master_port) master_client = imageservice(master_conn, options.mastertoken) differences = {} for image in master_client.get_images(): if _image_present(slave_client, image['id']): headers = slave_client.get_image_meta(image['id']) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from master metadata', {'header': key}) del image[key] if key in headers: LOG.debug('Stripping %(header)s from slave metadata', {'header': key}) del headers[key] for key in image: if image[key] != headers.get(key, None): LOG.info( _LI('%(image_id)s: field %(key)s differs ' '(source is %(master_value)s, destination ' 'is %(slave_value)s)') % { 'image_id': image['id'], 'key': key, 'master_value': image[key], 'slave_value': headers.get(key, 'undefined') }) differences[image['id']] = 'diff' else: LOG.debug('%(image_id)s is identical' % {'image_id': image['id']}) elif image['status'] == 'active': LOG.info( _LI('%s: entirely missing from the destination') % image['id']) differences[image['id']] = 'missing' return differences
def replication_compare(options, args): """%(prog)s compare <fromserver:port> <toserver:port> Compare the contents of fromserver with those of toserver. fromserver:port: the location of the master glance instance. toserver:port: the location of the slave glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() slave_server, slave_port = utils.parse_valid_host_port(args.pop()) slave_conn = http.HTTPConnection(slave_server, slave_port) slave_client = imageservice(slave_conn, options.slavetoken) master_server, master_port = utils.parse_valid_host_port(args.pop()) master_conn = http.HTTPConnection(master_server, master_port) master_client = imageservice(master_conn, options.mastertoken) differences = {} for image in master_client.get_images(): if _image_present(slave_client, image['id']): headers = slave_client.get_image_meta(image['id']) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from master metadata', {'header': key}) del image[key] if key in headers: LOG.debug('Stripping %(header)s from slave metadata', {'header': key}) del headers[key] for key in image: if image[key] != headers.get(key, None): LOG.warn(_LW('%(image_id)s: field %(key)s differs ' '(source is %(master_value)s, destination ' 'is %(slave_value)s)') % {'image_id': image['id'], 'key': key, 'master_value': image[key], 'slave_value': headers.get(key, 'undefined')}) differences[image['id']] = 'diff' else: LOG.debug('%(image_id)s is identical', {'image_id': image['id']}) elif image['status'] == 'active': LOG.warn(_LW('Image %(image_id)s ("%(image_name)s") ' 'entirely missing from the destination') % {'image_id': image['id'], 'image_name': image.get('name', '--unnamed')}) differences[image['id']] = 'missing' return differences
def replication_size(options, args): """%(prog)s size <server:port> Determine the size of a glance instance if dumped to disk. server:port: the location of the glance instance. """ # Make sure server info is provided if args is None or len(args) < 1: raise TypeError(_("Too few arguments.")) server, port = utils.parse_valid_host_port(args.pop()) total_size = 0 count = 0 imageservice = get_image_service() client = imageservice(http_client.HTTPConnection(server, port), options.slavetoken) for image in client.get_images(): LOG.debug('Considering image: %(image)s', {'image': image}) if image['status'] == 'active': total_size += int(image['size']) count += 1 print( _('Total size is %(size)d bytes (%(human_size)s) across ' '%(img_count)d images') % { 'size': total_size, 'human_size': _human_readable_size(total_size), 'img_count': count })
def replication_size(options, args): """%(prog)s size <server:port> Determine the size of a glance instance if dumped to disk. server:port: the location of the glance instance. """ # Make sure server info is provided if len(args) < 1: raise TypeError(_("Too few arguments.")) server, port = utils.parse_valid_host_port(args.pop()) total_size = 0 count = 0 imageservice = get_image_service() client = imageservice(httplib.HTTPConnection(server, port), options.slavetoken) for image in client.get_images(): LOG.debug('Considering image: %(image)s' % {'image': image}) if image['status'] == 'active': total_size += int(image['size']) count += 1 print(_('Total size is %(size)d bytes across %(img_count)d images') % {'size': total_size, 'img_count': count})
def replication_dump(options, args): """%(prog)s dump <server:port> <path> Dump the contents of a glance instance to local disk. server:port: the location of the glance instance. path: a directory on disk to contain the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(http.HTTPConnection(server, port), options.mastertoken) for image in client.get_images(): LOG.debug('Considering: %(image_id)s (%(image_name)s) ' '(%(image_size)d bytes)', {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size']}) data_path = os.path.join(path, image['id']) data_filename = data_path + '.img' if not os.path.exists(data_path): LOG.info(_LI('Storing: %(image_id)s (%(image_name)s)' ' (%(image_size)d bytes) in %(data_filename)s'), {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size'], 'data_filename': data_filename}) # Dump glance information if six.PY3: f = open(data_path, 'w', encoding='utf-8') else: f = open(data_path, 'w') with f: f.write(jsonutils.dumps(image)) if image['status'] == 'active' and not options.metaonly: # Now fetch the image. The metadata returned in headers here # is the same as that which we got from the detailed images # request earlier, so we can ignore it here. Note that we also # only dump active images. LOG.debug('Image %s is active', image['id']) image_response = client.get_image(image['id']) with open(data_filename, 'wb') as f: while True: chunk = image_response.read(options.chunksize) if not chunk: break f.write(chunk)
def replication_dump(options, args): """%(prog)s dump <server:port> <path> Dump the contents of a glance instance to local disk. server:port: the location of the glance instance. path: a directory on disk to contain the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(http.HTTPConnection(server, port), options.sourcetoken) for image in client.get_images(): LOG.debug('Considering: %(image_id)s (%(image_name)s) ' '(%(image_size)d bytes)', {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size']}) data_path = os.path.join(path, image['id']) data_filename = data_path + '.img' if not os.path.exists(data_path): LOG.info(_LI('Storing: %(image_id)s (%(image_name)s)' ' (%(image_size)d bytes) in %(data_filename)s'), {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size'], 'data_filename': data_filename}) # Dump glance information if six.PY3: f = open(data_path, 'w', encoding='utf-8') else: f = open(data_path, 'w') with f: f.write(jsonutils.dumps(image)) if image['status'] == 'active' and not options.metaonly: # Now fetch the image. The metadata returned in headers here # is the same as that which we got from the detailed images # request earlier, so we can ignore it here. Note that we also # only dump active images. LOG.debug('Image %s is active', image['id']) image_response = client.get_image(image['id']) with open(data_filename, 'wb') as f: while True: chunk = image_response.read(options.chunksize) if not chunk: break f.write(chunk)
def replication_dump(options, args): """%(prog)s dump <server:port> <path> Dump the contents of a glance instance to local disk. server:port: the location of the glance instance. path: a directory on disk to contain the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(httplib.HTTPConnection(server, port), options.mastertoken) for image in client.get_images(): LOG.info(_LI('Considering: %s') % image['id']) data_path = os.path.join(path, image['id']) if not os.path.exists(data_path): LOG.info(_LI('... storing')) # Dump glance information with open(data_path, 'w') as f: f.write(jsonutils.dumps(image)) if image['status'] == 'active' and not options.metaonly: # Now fetch the image. The metadata returned in headers here # is the same as that which we got from the detailed images # request earlier, so we can ignore it here. Note that we also # only dump active images. LOG.info(_LI('... image is active')) image_response = client.get_image(image['id']) with open(data_path + '.img', 'wb') as f: while True: chunk = image_response.read(options.chunksize) if not chunk: break f.write(chunk)
def replication_dump(options, args): """%(prog)s dump <server:port> <path> Dump the contents of a glance instance to local disk. server:port: the location of the glance instance. path: a directory on disk to contain the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(httplib.HTTPConnection(server, port), options.mastertoken) for image in client.get_images(): logging.info(_('Considering: %s') % image['id']) data_path = os.path.join(path, image['id']) if not os.path.exists(data_path): logging.info(_('... storing')) # Dump glance information with open(data_path, 'w') as f: f.write(jsonutils.dumps(image)) if image['status'] == 'active' and not options.metaonly: # Now fetch the image. The metadata returned in headers here # is the same as that which we got from the detailed images # request earlier, so we can ignore it here. Note that we also # only dump active images. logging.info(_('... image is active')) image_response = client.get_image(image['id']) with open(data_path + '.img', 'wb') as f: while True: chunk = image_response.read(options.chunksize) if not chunk: break f.write(chunk)
def test_valid_host_port_string(self): valid_pairs = [ '10.11.12.13:80', '172.17.17.1:65535', '[fe80::a:b:c:d]:9990', 'localhost:9990', 'localhost.localdomain:9990', 'glance02.stack42.local:1234', 'glance04-a.stack47.local:1234', 'img83.glance.xn--penstack-r74e.org:13080' ] for pair_str in valid_pairs: host, port = utils.parse_valid_host_port(pair_str) escaped = pair_str.startswith('[') expected_host = '%s%s%s' % ('[' if escaped else '', host, ']' if escaped else '') self.assertTrue(pair_str.startswith(expected_host)) self.assertGreater(port, 0) expected_pair = '%s:%d' % (expected_host, port) self.assertEqual(expected_pair, pair_str)
def test_valid_host_port_string(self): valid_pairs = ['10.11.12.13:80', '172.17.17.1:65535', '[fe80::a:b:c:d]:9990', 'localhost:9990', 'localhost.localdomain:9990', 'glance02.stack42.local:1234', 'glance04-a.stack47.local:1234', 'img83.glance.xn--penstack-r74e.org:13080'] for pair_str in valid_pairs: host, port = utils.parse_valid_host_port(pair_str) escaped = pair_str.startswith('[') expected_host = '%s%s%s' % ('[' if escaped else '', host, ']' if escaped else '') self.assertTrue(pair_str.startswith(expected_host)) self.assertTrue(port > 0) expected_pair = '%s:%d' % (expected_host, port) self.assertEqual(expected_pair, pair_str)
def replication_livecopy(options, args): """%(prog)s livecopy <fromserver:port> <toserver:port> Load the contents of one glance instance into another. fromserver:port: the location of the master glance instance. toserver:port: the location of the slave glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() slave_server, slave_port = utils.parse_valid_host_port(args.pop()) slave_conn = http_client.HTTPConnection(slave_server, slave_port) slave_client = imageservice(slave_conn, options.slavetoken) master_server, master_port = utils.parse_valid_host_port(args.pop()) master_conn = http_client.HTTPConnection(master_server, master_port) master_client = imageservice(master_conn, options.mastertoken) updated = [] for image in master_client.get_images(): LOG.debug('Considering %(id)s', {'id': image['id']}) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from master metadata', {'header': key}) del image[key] if _image_present(slave_client, image['id']): # NOTE(mikal): Perhaps we just need to update the metadata? # Note that we don't attempt to change an image file once it # has been uploaded. headers = slave_client.get_image_meta(image['id']) if headers['status'] == 'active': for key in options.dontreplicate.split(' '): if key in image: LOG.debug( 'Stripping %(header)s from master ' 'metadata', {'header': key}) del image[key] if key in headers: LOG.debug( 'Stripping %(header)s from slave ' 'metadata', {'header': key}) del headers[key] if _dict_diff(image, headers): LOG.info( _LI('Image %(image_id)s (%(image_name)s) ' 'metadata has changed'), { 'image_id': image['id'], 'image_name': image.get('name', '--unnamed--') }) headers, body = slave_client.add_image_meta(image) _check_upload_response_headers(headers, body) updated.append(image['id']) elif image['status'] == 'active': LOG.info( _LI('Image %(image_id)s (%(image_name)s) ' '(%(image_size)d bytes) ' 'is being synced'), { 'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size'] }) if not options.metaonly: image_response = master_client.get_image(image['id']) try: headers, body = slave_client.add_image( image, image_response) _check_upload_response_headers(headers, body) updated.append(image['id']) except exc.HTTPConflict: LOG.error( _LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image['id']) # noqa return updated
def replication_load(options, args): """%(prog)s load <server:port> <path> Load the contents of a local directory into glance. server:port: the location of the glance instance. path: a directory on disk containing the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(http_client.HTTPConnection(server, port), options.slavetoken) updated = [] for ent in os.listdir(path): if uuidutils.is_uuid_like(ent): image_uuid = ent LOG.info(_LI('Considering: %s'), image_uuid) meta_file_name = os.path.join(path, image_uuid) with open(meta_file_name) as meta_file: meta = jsonutils.loads(meta_file.read()) # Remove keys which don't make sense for replication for key in options.dontreplicate.split(' '): if key in meta: LOG.debug('Stripping %(header)s from saved ' 'metadata', {'header': key}) del meta[key] if _image_present(client, image_uuid): # NOTE(mikal): Perhaps we just need to update the metadata? # Note that we don't attempt to change an image file once it # has been uploaded. LOG.debug('Image %s already present', image_uuid) headers = client.get_image_meta(image_uuid) for key in options.dontreplicate.split(' '): if key in headers: LOG.debug( 'Stripping %(header)s from slave ' 'metadata', {'header': key}) del headers[key] if _dict_diff(meta, headers): LOG.info(_LI('Image %s metadata has changed'), image_uuid) headers, body = client.add_image_meta(meta) _check_upload_response_headers(headers, body) updated.append(meta['id']) else: if not os.path.exists(os.path.join(path, image_uuid + '.img')): LOG.debug('%s dump is missing image data, skipping', image_uuid) continue # Upload the image itself with open(os.path.join(path, image_uuid + '.img')) as img_file: try: headers, body = client.add_image(meta, img_file) _check_upload_response_headers(headers, body) updated.append(meta['id']) except exc.HTTPConflict: LOG.error( _LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image_uuid) # noqa return updated
def replication_livecopy(options, args): """%(prog)s livecopy <fromserver:port> <toserver:port> Load the contents of one glance instance into another. fromserver:port: the location of the master glance instance. toserver:port: the location of the slave glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() slave_server, slave_port = utils.parse_valid_host_port(args.pop()) slave_conn = httplib.HTTPConnection(slave_server, slave_port) slave_client = imageservice(slave_conn, options.slavetoken) master_server, master_port = utils.parse_valid_host_port(args.pop()) master_conn = httplib.HTTPConnection(master_server, master_port) master_client = imageservice(master_conn, options.mastertoken) updated = [] for image in master_client.get_images(): LOG.info(_LI('Considering %(id)s') % {'id': image['id']}) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from master metadata', {'header': key}) del image[key] if _image_present(slave_client, image['id']): # NOTE(mikal): Perhaps we just need to update the metadata? # Note that we don't attempt to change an image file once it # has been uploaded. headers = slave_client.get_image_meta(image['id']) if headers['status'] == 'active': for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from master ' 'metadata', {'header': key}) del image[key] if key in headers: LOG.debug('Stripping %(header)s from slave ' 'metadata', {'header': key}) del headers[key] if _dict_diff(image, headers): LOG.info(_LI('... metadata has changed')) headers, body = slave_client.add_image_meta(image) _check_upload_response_headers(headers, body) updated.append(image['id']) elif image['status'] == 'active': LOG.info(_LI('%s is being synced') % image['id']) if not options.metaonly: image_response = master_client.get_image(image['id']) try: headers, body = slave_client.add_image(image, image_response) _check_upload_response_headers(headers, body) updated.append(image['id']) except ImageAlreadyPresentException: LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image['id']) return updated
def replication_load(options, args): """%(prog)s load <server:port> <path> Load the contents of a local directory into glance. server:port: the location of the glance instance. path: a directory on disk containing the data. """ # Make sure server and path are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) path = args.pop() server, port = utils.parse_valid_host_port(args.pop()) imageservice = get_image_service() client = imageservice(httplib.HTTPConnection(server, port), options.slavetoken) updated = [] for ent in os.listdir(path): if utils.is_uuid_like(ent): image_uuid = ent LOG.info(_LI('Considering: %s') % image_uuid) meta_file_name = os.path.join(path, image_uuid) with open(meta_file_name) as meta_file: meta = jsonutils.loads(meta_file.read()) # Remove keys which don't make sense for replication for key in options.dontreplicate.split(' '): if key in meta: LOG.debug('Stripping %(header)s from saved ' 'metadata', {'header': key}) del meta[key] if _image_present(client, image_uuid): # NOTE(mikal): Perhaps we just need to update the metadata? # Note that we don't attempt to change an image file once it # has been uploaded. LOG.debug('Image %s already present', image_uuid) headers = client.get_image_meta(image_uuid) for key in options.dontreplicate.split(' '): if key in headers: LOG.debug('Stripping %(header)s from slave ' 'metadata', {'header': key}) del headers[key] if _dict_diff(meta, headers): LOG.info(_LI('... metadata has changed')) headers, body = client.add_image_meta(meta) _check_upload_response_headers(headers, body) updated.append(meta['id']) else: if not os.path.exists(os.path.join(path, image_uuid + '.img')): LOG.info(_LI('... dump is missing image data, skipping')) continue # Upload the image itself with open(os.path.join(path, image_uuid + '.img')) as img_file: try: headers, body = client.add_image(meta, img_file) _check_upload_response_headers(headers, body) updated.append(meta['id']) except ImageAlreadyPresentException: LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image_uuid) return updated
def replication_compare(options, args): """%(prog)s compare <fromserver:port> <toserver:port> Compare the contents of fromserver with those of toserver. fromserver:port: the location of the source glance instance. toserver:port: the location of the target glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() target_server, target_port = utils.parse_valid_host_port(args.pop()) target_conn = http.HTTPConnection(target_server, target_port) target_client = imageservice(target_conn, options.targettoken) source_server, source_port = utils.parse_valid_host_port(args.pop()) source_conn = http.HTTPConnection(source_server, source_port) source_client = imageservice(source_conn, options.sourcetoken) differences = {} for image in source_client.get_images(): if _image_present(target_client, image['id']): headers = target_client.get_image_meta(image['id']) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from source metadata', {'header': key}) del image[key] if key in headers: LOG.debug('Stripping %(header)s from target metadata', {'header': key}) del headers[key] for key in image: if image[key] != headers.get(key): LOG.warn( _LW('%(image_id)s: field %(key)s differs ' '(source is %(source_value)s, destination ' 'is %(target_value)s)') % { 'image_id': image['id'], 'key': key, 'source_value': image[key], 'target_value': headers.get(key, 'undefined') }) differences[image['id']] = 'diff' else: LOG.debug('%(image_id)s is identical', {'image_id': image['id']}) elif image['status'] == 'active': LOG.warn( _LW('Image %(image_id)s ("%(image_name)s") ' 'entirely missing from the destination') % { 'image_id': image['id'], 'image_name': image.get('name', '--unnamed') }) differences[image['id']] = 'missing' return differences
def replication_livecopy(options, args): """%(prog)s livecopy <fromserver:port> <toserver:port> Load the contents of one glance instance into another. fromserver:port: the location of the source glance instance. toserver:port: the location of the target glance instance. """ # Make sure from-server and to-server are provided if len(args) < 2: raise TypeError(_("Too few arguments.")) imageservice = get_image_service() target_server, target_port = utils.parse_valid_host_port(args.pop()) target_conn = http.HTTPConnection(target_server, target_port) target_client = imageservice(target_conn, options.targettoken) source_server, source_port = utils.parse_valid_host_port(args.pop()) source_conn = http.HTTPConnection(source_server, source_port) source_client = imageservice(source_conn, options.sourcetoken) updated = [] for image in source_client.get_images(): LOG.debug('Considering %(id)s', {'id': image['id']}) for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from source metadata', {'header': key}) del image[key] if _image_present(target_client, image['id']): # NOTE(mikal): Perhaps we just need to update the metadata? # Note that we don't attempt to change an image file once it # has been uploaded. headers = target_client.get_image_meta(image['id']) if headers['status'] == 'active': for key in options.dontreplicate.split(' '): if key in image: LOG.debug('Stripping %(header)s from source ' 'metadata', {'header': key}) del image[key] if key in headers: LOG.debug('Stripping %(header)s from target ' 'metadata', {'header': key}) del headers[key] if _dict_diff(image, headers): LOG.info(_LI('Image %(image_id)s (%(image_name)s) ' 'metadata has changed'), {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--')}) headers, body = target_client.add_image_meta(image) _check_upload_response_headers(headers, body) updated.append(image['id']) elif image['status'] == 'active': LOG.info(_LI('Image %(image_id)s (%(image_name)s) ' '(%(image_size)d bytes) ' 'is being synced'), {'image_id': image['id'], 'image_name': image.get('name', '--unnamed--'), 'image_size': image['size']}) if not options.metaonly: image_response = source_client.get_image(image['id']) try: headers, body = target_client.add_image(image, image_response) _check_upload_response_headers(headers, body) updated.append(image['id']) except exc.HTTPConflict: LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image['id']) # noqa return updated