def _test_format_at_block_size(self, format_name, img, block_size):
fmt = format_inspector.get_inspector(format_name)()
self.assertIsNotNone(
fmt, 'Did not get format inspector for %s' % (format_name))
wrapper = format_inspector.InfoWrapper(open(img, 'rb'), fmt)
while True:
chunk = wrapper.read(block_size)
if not chunk:
break
wrapper.close()
return fmt
def _test_format_with_invalid_data(self, format_name):
fmt = format_inspector.get_inspector(format_name)()
wrapper = format_inspector.InfoWrapper(open(__file__, 'rb'), fmt)
while True:
chunk = wrapper.read(32)
if not chunk:
break
wrapper.close()
self.assertFalse(fmt.format_match)
self.assertEqual(0, fmt.virtual_size)
memory = sum(fmt.context_info.values())
self.assertLess(
memory, 512 * units.Ki,
'Format used more than 512KiB of memory: %s' % (fmt.context_info))
def set_data(self, data, size=None, backend=None, set_active=True):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
# Create the verifier for signature verification (if correct properties
# are present)
extra_props = self.image.extra_properties
verifier = None
if signature_utils.should_create_verifier(extra_props):
# NOTE(bpoulos): if creating verifier fails, exception will be
# raised
img_signature = extra_props[signature_utils.SIGNATURE]
hash_method = extra_props[signature_utils.HASH_METHOD]
key_type = extra_props[signature_utils.KEY_TYPE]
cert_uuid = extra_props[signature_utils.CERT_UUID]
verifier = signature_utils.get_verifier(
context=self.context,
img_signature_certificate_uuid=cert_uuid,
img_signature_hash_method=hash_method,
img_signature=img_signature,
img_signature_key_type=key_type)
if not self.image.virtual_size:
inspector = format_inspector.get_inspector(self.image.disk_format)
else:
# No need to do this again
inspector = None
if inspector and self.image.container_format == 'bare':
fmt = inspector()
data = format_inspector.InfoWrapper(data, fmt)
LOG.debug('Enabling in-flight format inspection for %s', fmt)
else:
fmt = None
self._upload_to_store(data, verifier, backend, size)
if fmt and fmt.format_match and fmt.virtual_size:
self.image.virtual_size = fmt.virtual_size
LOG.info('Image format matched and virtual size computed: %i',
self.image.virtual_size)
elif fmt:
LOG.warning(
'Image format %s did not match; '
'unable to calculate virtual size', self.image.disk_format)
if set_active and self.image.status != 'active':
self.image.status = 'active'
def test_info_wrapper_iter_like_eats_error(self):
fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw'))
wrapper = format_inspector.InfoWrapper(iter([b'123', b'456']),
fake_fmt)
fake_fmt.eat_chunk.side_effect = Exception('fail')
data = b''
for chunk in wrapper:
data += chunk
# Make sure we got all the data despite the error
self.assertEqual(b'123456', data)
# Make sure we only called this once and never again after
# the error was raised
fake_fmt.eat_chunk.assert_called_once_with(b'123')
def test_get_inspector(self):
self.assertEqual(format_inspector.QcowInspector,
format_inspector.get_inspector('qcow2'))
self.assertIsNone(format_inspector.get_inspector('foo'))
def _get_wrapper(self, data):
source = io.BytesIO(data)
fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw'))
return format_inspector.InfoWrapper(source, fake_fmt)
def main():
formats = ['raw', 'qcow2', 'vhd', 'vhdx', 'vmdk', 'vdi']
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--debug', action='store_true')
parser.add_argument('-f',
'--format',
default='raw',
help='Format (%s)' % ','.join(sorted(formats)))
parser.add_argument('-b',
'--block-size',
default=65536,
type=int,
help='Block read size')
parser.add_argument('--context-limit',
default=(1 * 1024),
type=int,
help='Maximum memory footprint (KiB)')
parser.add_argument('-i',
'--input',
default=None,
help='Input file. Defaults to stdin')
parser.add_argument('-v',
'--verify',
action='store_true',
help=('Verify our number with qemu-img '
'(requires --input)'))
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
fmt = format_inspector.get_inspector(args.format)(tracing=args.debug)
if args.input:
input_stream = open(args.input, 'rb')
else:
input_stream = sys.stdin.buffer
stream = format_inspector.InfoWrapper(input_stream, fmt)
count = 0
found_size = False
while True:
chunk = stream.read(int(args.block_size))
# This could stream to an output destination or stdin for testing
# sys.stdout.write(chunk)
if not chunk:
break
count += len(chunk)
if args.format != 'raw' and not found_size and fmt.virtual_size != 0:
# Print the point at which we've seen enough of the file to
# know what the virtual size is. This is almost always less
# than the raw_size
print('Determined virtual size at byte %i' % count)
found_size = True
if fmt.format_match:
print('Source was %s file, virtual size %i MiB (%i bytes)' %
(fmt, fmt.virtual_size / units.Mi, fmt.virtual_size))
else:
print('*** Format inspector did not detect file as %s' % args.format)
print('Raw size %i MiB (%i bytes)' %
(fmt.actual_size / units.Mi, fmt.actual_size))
print('Required contexts: %s' % str(fmt.context_info))
mem_total = sum(fmt.context_info.values())
print('Total memory footprint: %i bytes' % mem_total)
# To make sure we're not storing the whole image, complain if the
# format inspector stored more than context_limit data
if mem_total > args.context_limit * 1024:
print('*** ERROR: Memory footprint exceeded!')
if args.verify and args.input:
size = test_format_inspector.get_size_from_qemu_img(args.input)
if size != fmt.virtual_size:
print('*** QEMU disagrees with our size of %i: %i' %
(fmt.virtual_size, size))
else:
print('Confirmed size with qemu-img')