def _test_format_at_block_size(self, format_name, img, block_size): fmt = format_inspector.get_inspector(format_name)() self.assertIsNotNone( fmt, 'Did not get format inspector for %s' % (format_name)) wrapper = format_inspector.InfoWrapper(open(img, 'rb'), fmt) while True: chunk = wrapper.read(block_size) if not chunk: break wrapper.close() return fmt
def _test_format_with_invalid_data(self, format_name): fmt = format_inspector.get_inspector(format_name)() wrapper = format_inspector.InfoWrapper(open(__file__, 'rb'), fmt) while True: chunk = wrapper.read(32) if not chunk: break wrapper.close() self.assertFalse(fmt.format_match) self.assertEqual(0, fmt.virtual_size) memory = sum(fmt.context_info.values()) self.assertLess( memory, 512 * units.Ki, 'Format used more than 512KiB of memory: %s' % (fmt.context_info))
def set_data(self, data, size=None, backend=None, set_active=True): if size is None: size = 0 # NOTE(markwash): zero -> unknown size # Create the verifier for signature verification (if correct properties # are present) extra_props = self.image.extra_properties verifier = None if signature_utils.should_create_verifier(extra_props): # NOTE(bpoulos): if creating verifier fails, exception will be # raised img_signature = extra_props[signature_utils.SIGNATURE] hash_method = extra_props[signature_utils.HASH_METHOD] key_type = extra_props[signature_utils.KEY_TYPE] cert_uuid = extra_props[signature_utils.CERT_UUID] verifier = signature_utils.get_verifier( context=self.context, img_signature_certificate_uuid=cert_uuid, img_signature_hash_method=hash_method, img_signature=img_signature, img_signature_key_type=key_type) if not self.image.virtual_size: inspector = format_inspector.get_inspector(self.image.disk_format) else: # No need to do this again inspector = None if inspector and self.image.container_format == 'bare': fmt = inspector() data = format_inspector.InfoWrapper(data, fmt) LOG.debug('Enabling in-flight format inspection for %s', fmt) else: fmt = None self._upload_to_store(data, verifier, backend, size) if fmt and fmt.format_match and fmt.virtual_size: self.image.virtual_size = fmt.virtual_size LOG.info('Image format matched and virtual size computed: %i', self.image.virtual_size) elif fmt: LOG.warning( 'Image format %s did not match; ' 'unable to calculate virtual size', self.image.disk_format) if set_active and self.image.status != 'active': self.image.status = 'active'
def test_info_wrapper_iter_like_eats_error(self): fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw')) wrapper = format_inspector.InfoWrapper(iter([b'123', b'456']), fake_fmt) fake_fmt.eat_chunk.side_effect = Exception('fail') data = b'' for chunk in wrapper: data += chunk # Make sure we got all the data despite the error self.assertEqual(b'123456', data) # Make sure we only called this once and never again after # the error was raised fake_fmt.eat_chunk.assert_called_once_with(b'123')
def test_get_inspector(self): self.assertEqual(format_inspector.QcowInspector, format_inspector.get_inspector('qcow2')) self.assertIsNone(format_inspector.get_inspector('foo'))
def _get_wrapper(self, data): source = io.BytesIO(data) fake_fmt = mock.create_autospec(format_inspector.get_inspector('raw')) return format_inspector.InfoWrapper(source, fake_fmt)
def main(): formats = ['raw', 'qcow2', 'vhd', 'vhdx', 'vmdk', 'vdi'] parser = argparse.ArgumentParser() parser.add_argument('-d', '--debug', action='store_true') parser.add_argument('-f', '--format', default='raw', help='Format (%s)' % ','.join(sorted(formats))) parser.add_argument('-b', '--block-size', default=65536, type=int, help='Block read size') parser.add_argument('--context-limit', default=(1 * 1024), type=int, help='Maximum memory footprint (KiB)') parser.add_argument('-i', '--input', default=None, help='Input file. Defaults to stdin') parser.add_argument('-v', '--verify', action='store_true', help=('Verify our number with qemu-img ' '(requires --input)')) args = parser.parse_args() if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) fmt = format_inspector.get_inspector(args.format)(tracing=args.debug) if args.input: input_stream = open(args.input, 'rb') else: input_stream = sys.stdin.buffer stream = format_inspector.InfoWrapper(input_stream, fmt) count = 0 found_size = False while True: chunk = stream.read(int(args.block_size)) # This could stream to an output destination or stdin for testing # sys.stdout.write(chunk) if not chunk: break count += len(chunk) if args.format != 'raw' and not found_size and fmt.virtual_size != 0: # Print the point at which we've seen enough of the file to # know what the virtual size is. This is almost always less # than the raw_size print('Determined virtual size at byte %i' % count) found_size = True if fmt.format_match: print('Source was %s file, virtual size %i MiB (%i bytes)' % (fmt, fmt.virtual_size / units.Mi, fmt.virtual_size)) else: print('*** Format inspector did not detect file as %s' % args.format) print('Raw size %i MiB (%i bytes)' % (fmt.actual_size / units.Mi, fmt.actual_size)) print('Required contexts: %s' % str(fmt.context_info)) mem_total = sum(fmt.context_info.values()) print('Total memory footprint: %i bytes' % mem_total) # To make sure we're not storing the whole image, complain if the # format inspector stored more than context_limit data if mem_total > args.context_limit * 1024: print('*** ERROR: Memory footprint exceeded!') if args.verify and args.input: size = test_format_inspector.get_size_from_qemu_img(args.input) if size != fmt.virtual_size: print('*** QEMU disagrees with our size of %i: %i' % (fmt.virtual_size, size)) else: print('Confirmed size with qemu-img')