def get_image_sizes(image_filepath: str, required_paths: list = None, min_size: int = None) -> typing.Dict[str, int]: """Extracts an image to a temporary directory and calls sub_paths_size_iter. Args: image_filepath: The filepath of the image to analyze. required_paths: get sizes for only these paths min_size: if required_paths not set will filter to only return paths where size of path is at least min_size Returns: A dictionary of path -> size. """ sizes = {} with osutils.TempDir(prefix=TEMPFILE_PREFIX) as temp_dir, \ image_lib.LoopbackPartitions(image_filepath, temp_dir) as image: root_path = image.Mount([constants.PART_ROOT_A])[0] for path, size in sub_paths_size_iter(root_path): if required_paths: if path not in required_paths: continue elif size < min_size: continue sizes[path] = size return sizes
def PrepareImage(path, content, domain=None): """Prepares a recovery image for OOBE autoconfiguration. Args: path: Path to the recovery image. content: The content of the OOBE autoconfiguration. domain: Which domain to enroll to. """ with osutils.TempDir() as tmp, \ image_lib.LoopbackPartitions(path, tmp) as image: stateful_mnt = image.Mount((constants.CROS_PART_STATEFUL, ), mount_opts=('rw', ))[0] # /stateful/unencrypted may not exist at this point in time on the # recovery image, so create it root-owned here. unencrypted = os.path.join(stateful_mnt, 'unencrypted') osutils.SafeMakedirs(unencrypted, mode=0o755, sudo=True) # The OOBE autoconfig directory must be owned by the chronos user so # that we can delete the config file from it from Chrome. oobe_autoconf = os.path.join(unencrypted, _OOBE_DIRECTORY) osutils.SafeMakedirsNonRoot(oobe_autoconf, user='******') # Create the config file to be owned by the chronos user, and write the # given data into it. config = os.path.join(oobe_autoconf, _CONFIG_PATH) osutils.WriteFile(config, content, sudo=True) cros_build_lib.sudo_run(['chown', 'chronos:chronos', config]) # If we have a plaintext domain name, write it. if domain: domain_path = os.path.join(oobe_autoconf, _DOMAIN_PATH) osutils.WriteFile(domain_path, SanitizeDomain(domain), sudo=True) cros_build_lib.sudo_run(['chown', 'chronos:chronos', domain_path])
def test_GetMountPointAndSymlink(self): """Test _GetMountPointAndSymlink().""" lb = image_lib.LoopbackPartitions(FAKE_PATH, destination=self.tempdir) for part in LOOP_PARTITION_INFO: expected = [os.path.join(lb.destination, 'dir-%s' % n) for n in (part.number, part.name)] self.assertEqual(expected, list(lb._GetMountPointAndSymlink(part)))
def testMountUnmount(self): """Test Mount() and Unmount() entry points.""" lb = image_lib.LoopbackPartitions(FAKE_PATH, destination=self.tempdir) # Mount four partitions. lb.Mount((1, 3, 'ROOT-B', 'ROOT-C')) for p in (1, 3, 5, 7): self.mount_mock.assert_any_call( '%sp%d' % (LOOP_DEV, p), '%s/dir-%d' % (self.tempdir, p), makedirs=True, skip_mtab=False, sudo=True, mount_opts=('ro',)) linkname = '%s/dir-%s' % (self.tempdir, LOOP_PARTITION_INFO[p - 1].name) self.assertTrue(stat.S_ISLNK(os.lstat(linkname).st_mode)) self.assertEqual(4, self.mount_mock.call_count) self.umount_mock.assert_not_called() # Unmount half of them, confirm that they were unmounted. lb.Unmount((1, 'ROOT-B')) for p in (1, 5): self.umount_mock.assert_any_call('%s/dir-%d' % (self.tempdir, p), cleanup=False) self.assertEqual(2, self.umount_mock.call_count) self.umount_mock.reset_mock() # Close the object, so that we unmount the other half of them. lb.close() for p in (3, 7): self.umount_mock.assert_any_call('%s/dir-%d' % (self.tempdir, p), cleanup=False) self.assertEqual(2, self.umount_mock.call_count) # Verify that the directories were cleaned up. for p in (1, 3): self.retry_mock.assert_any_call( cros_build_lib.RunCommandError, 60, osutils.RmDir, '%s/dir-%d' % (self.tempdir, p), sudo=True, sleep=1)
def testContextManagerWithMounts(self): """Test using the loopback class as a context manager with mounts.""" syml = self.PatchObject(osutils, 'SafeSymlink') part_ids = (1, 'ROOT-A') with image_lib.LoopbackPartitions(FAKE_PATH, part_ids=part_ids, mount_opts=('ro', )) as lb: expected_mounts = set() expected_calls = [] for part_id in part_ids: for part in LOOP_PARTITION_INFO: if part.name == part_id or part.number == part_id: expected_mounts.add(part) expected_calls.append( mock.call( 'dir-%d' % part.number, os.path.join(lb.destination, 'dir-%s' % part.name))) break self.rc_mock.assertCommandContains( ['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains( ['losetup', '--detach', LOOP_DEV], expected=False) self.assertEqual(lb.parts, LOOP_PARTS_DICT) self.assertEqual(lb._gpt_table, LOOP_PARTITION_INFO) self.assertEqual(expected_calls, syml.call_args_list) self.assertEqual(expected_mounts, lb._mounted) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV])
def testMountingMountedPartReturnsName(self): """Test that Mount returns the directory name even when already mounted.""" lb = image_lib.LoopbackPartitions(FAKE_PATH, destination=self.tempdir) dirname = '%s/dir-%d' % (self.tempdir, lb._gpt_table[0].number) # First make sure we get the directory name when we actually mount. self.assertEqual(dirname, lb._Mount(lb._gpt_table[0], ('ro',))) # Then make sure we get it when we call it again. self.assertEqual(dirname, lb._Mount(lb._gpt_table[0], ('ro',)))
def testGetPartitionDevName(self): """Test GetPartitionDevName().""" lb = image_lib.LoopbackPartitions(FAKE_PATH) for part in LOOP_PARTITION_INFO: self.assertEqual('%sp%d' % (LOOP_DEV, part.number), lb.GetPartitionDevName(part.number)) if part.name != 'reserved': self.assertEqual('%sp%d' % (LOOP_DEV, part.number), lb.GetPartitionDevName(part.name))
def gcFunc(self): """This function isolates a local variable so it'll be garbage collected.""" self.rc_mock.AddCmdResult(partial_mock.In('--show'), output=LOOP_DEV) lb = image_lib.LoopbackPartitions(FAKE_PATH) self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEquals(lb.parts, LOOP_PARTS_DICT)
def gcFunc(self): """This function isolates a local variable so it'll be garbage collected.""" lb = image_lib.LoopbackPartitions(FAKE_PATH) self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEqual(lb.parts, LOOP_PARTS_DICT) self.assertEqual(lb._gpt_table, LOOP_PARTITION_INFO)
def main(args): opts = ParseArgs(args) # Build up test suites. loader = unittest.TestLoader() loader.suiteClass = image_test_lib.ImageTestSuite # We use a different prefix here so that unittest DO NOT pick up the # image tests automatically because they depend on a proper environment. loader.testMethodPrefix = 'Test' tests_namespace = 'chromite.cros.test.image_test' if opts.tests: tests = ['%s.%s' % (tests_namespace, x) for x in opts.tests] else: tests = (tests_namespace, ) all_tests = loader.loadTestsFromNames(tests) # If they just want to see the lists of tests, show them now. if opts.list: def _WalkSuite(suite): for test in suite: if isinstance(test, unittest.BaseTestSuite): for result in _WalkSuite(test): yield result else: yield (test.id()[len(tests_namespace) + 1:], test.shortDescription() or '') test_list = list(_WalkSuite(all_tests)) maxlen = max(len(x[0]) for x in test_list) for name, desc in test_list: print('%-*s %s' % (maxlen, name, desc)) return # Run them in the image directory. runner = image_test_lib.ImageTestRunner() runner.SetBoard(opts.board) runner.SetResultDir(opts.test_results_root) image_file = FindImage(opts.image) tmp_in_chroot = path_util.FromChrootPath('/tmp') with osutils.TempDir(base_dir=tmp_in_chroot) as temp_dir: with image_lib.LoopbackPartitions(image_file, temp_dir) as image: # Due to the lack of mount context, we mount the partitions # but do not reference directly. This will be removed with the # submission of http://crrev/c/1795578 _ = image.Mount((constants.PART_ROOT_A, ))[0] _ = image.Mount((constants.PART_STATE, ))[0] with osutils.ChdirContext(temp_dir): result = runner.run(all_tests) if result and not result.wasSuccessful(): return 1 return 0
def testContextManager(self): """Test using the loopback class as a context manager.""" self.rc_mock.AddCmdResult(partial_mock.In('--show'), output=LOOP_DEV) with image_lib.LoopbackPartitions(FAKE_PATH) as lb: self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEquals(lb.parts, LOOP_PARTS_DICT) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV])
def testContextManager(self): """Test using the loopback class as a context manager.""" with image_lib.LoopbackPartitions(FAKE_PATH) as lb: self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEqual(lb.parts, LOOP_PARTS_DICT) self.assertEqual(lb._gpt_table, LOOP_PARTITION_INFO) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV])
def testManual(self): """Test using the loopback class closed manually.""" lb = image_lib.LoopbackPartitions(FAKE_PATH) self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEqual(lb.parts, LOOP_PARTS_DICT) self.assertEqual(lb._gpt_table, LOOP_PARTITION_INFO) lb.close() self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV])
def testManual(self): """Test using the loopback class closed manually.""" self.rc_mock.AddCmdResult(partial_mock.In('--show'), output=LOOP_DEV) lb = image_lib.LoopbackPartitions(FAKE_PATH) self.rc_mock.assertCommandContains(['losetup', '--show', '-f', FAKE_PATH]) self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['partx', '-a', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV], expected=False) self.assertEquals(lb.parts, LOOP_PARTS_DICT) lb.close() self.rc_mock.assertCommandContains(['partx', '-d', LOOP_DEV]) self.rc_mock.assertCommandContains(['losetup', '--detach', LOOP_DEV])
def setUp(self): """Create a small test disk image for testing.""" self.image = os.path.join(self.tempdir, 'image.bin') state = os.path.join(self.tempdir, 'state.bin') # Allocate space for the disk image and stateful partition. osutils.AllocateFile(self.image, _IMAGE_SIZE) osutils.AllocateFile(state, _STATEFUL_SIZE) commands = ( # Format the stateful image as ext4. ['/sbin/mkfs.ext4', state], # Create the GPT headers and entry for the stateful partition. ['cgpt', 'create', self.image], ['cgpt', 'boot', '-p', self.image], [ 'cgpt', 'add', self.image, '-t', 'data', '-l', str(constants.CROS_PART_STATEFUL), '-b', str(_STATEFUL_OFFSET // _SECTOR_SIZE), '-s', str(_STATEFUL_SIZE // _SECTOR_SIZE), '-i', '1' ], # Copy the stateful partition into the GPT image. [ 'dd', 'if=%s' % state, 'of=%s' % self.image, 'conv=notrunc', 'bs=4K', 'seek=%d' % (_STATEFUL_OFFSET // _BLOCK_SIZE), 'count=%s' % (_STATEFUL_SIZE // _BLOCK_SIZE) ], ['sync']) for cmd in commands: cros_build_lib.run(cmd, quiet=True) # Run the preparation script on the image. cros_oobe_autoconfig.main([self.image] + list(_TEST_CLI_PARAMETERS)[1:]) # Mount the image's stateful partition for inspection. self.mount_tmp = os.path.join(self.tempdir, 'mount') osutils.SafeMakedirs(self.mount_tmp) self.mount_ctx = image_lib.LoopbackPartitions(self.image, self.mount_tmp) self.mount = os.path.join(self.mount_tmp, 'dir-%s' % constants.CROS_PART_STATEFUL) self.oobe_autoconf_path = os.path.join(self.mount, 'unencrypted', 'oobe_auto_config') self.config_path = os.path.join(self.oobe_autoconf_path, 'config.json') self.domain_path = os.path.join(self.oobe_autoconf_path, 'enrollment_domain')
def testRemountCallsMount(self): """Test that Mount returns the directory name even when already mounted.""" lb = image_lib.LoopbackPartitions(FAKE_PATH, destination=self.tempdir) devname = '%sp%d' % (LOOP_DEV, lb._gpt_table[0].number) dirname = '%s/dir-%d' % (self.tempdir, lb._gpt_table[0].number) # First make sure we get the directory name when we actually mount. self.assertEqual(dirname, lb._Mount(lb._gpt_table[0], ('ro',))) self.mount_mock.assert_called_once_with( devname, dirname, makedirs=True, skip_mtab=False, sudo=True, mount_opts=('ro',)) # Then make sure we get it when we call it again. self.assertEqual(dirname, lb._Mount(lb._gpt_table[0], ('remount', 'rw'))) self.assertEqual( mock.call(devname, dirname, makedirs=True, skip_mtab=False, sudo=True, mount_opts=('remount', 'rw')), self.mount_mock.call_args)
def DumpConfig(image_file): """Dump kernel config for both kernels. This implements the necessary logic for bin/dump_config, which is intended primarily for debugging of images. Args: image_file: path to the image file from which to dump kernel configs. """ with image_lib.LoopbackPartitions(image_file) as image: for kernel_part in ('KERN-A', 'KERN-B'): loop_kern = image.GetPartitionDevName(kernel_part) config = GetKernelConfig(loop_kern, check=False) if config: logging.info('Partition %s', kernel_part) logging.info(config) else: logging.info('Partition %s has no configuration.', kernel_part)
def testIsExt2OnVarious(self): """Test _IsExt2 works with the various partition types.""" FS_PARTITIONS = (1, 3, 8) # STATE, ROOT-A, and OEM generally have ext2 filesystems. for x in FS_PARTITIONS: self.rc_mock.AddCmdResult( partial_mock.In('if=%sp%d' % (LOOP_DEV, x)), output=b'\x53\xef') # Throw errors on all of the partitions that are < 1000 bytes. for part in LOOP_PARTITION_INFO: if part.size < 1000: self.rc_mock.AddCmdResult( partial_mock.In('if=%sp%d' % (LOOP_DEV, part.number)), returncode=1, error='Seek failed\n') lb = image_lib.LoopbackPartitions(FAKE_PATH, destination=self.tempdir) # We expect that only the partitions in FS_PARTITIONS are ext2. self.assertEqual( [part.number in FS_PARTITIONS for part in LOOP_PARTITION_INFO], [lb._IsExt2(part.name) for part in LOOP_PARTITION_INFO])
def MoblabVmTest(input_proto, _output_proto, _config): """Run Moblab VM tests.""" chroot = controller_util.ParseChroot(input_proto.chroot) image_payload_dir = input_proto.image_payload.path.path cache_payload_dirs = [cp.path.path for cp in input_proto.cache_payloads] # Autotest and Moblab depend on the builder path, so we must read it from # the image. image_file = os.path.join(image_payload_dir, constants.TEST_IMAGE_BIN) with osutils.TempDir() as mount_dir: with image_lib.LoopbackPartitions(image_file, destination=mount_dir) as lp: # The file we want is /etc/lsb-release, which lives in the ROOT-A # disk partition. partition_paths = lp.Mount([constants.PART_ROOT_A]) assert len(partition_paths) == 1, ( 'expected one partition path, got: %r' % partition_paths) partition_path = partition_paths[0] lsb_release_file = os.path.join( partition_path, constants.LSB_RELEASE_PATH.strip('/')) lsb_release_kvs = key_value_store.LoadFile(lsb_release_file) builder = lsb_release_kvs.get( cros_set_lsb_release.LSB_KEY_BUILDER_PATH) if not builder: cros_build_lib.Die('Image did not contain key %s in %s', cros_set_lsb_release.LSB_KEY_BUILDER_PATH, constants.LSB_RELEASE_PATH) # Now we can run the tests. with chroot.tempdir() as workspace_dir, chroot.tempdir() as results_dir: # Convert the results directory to an absolute chroot directory. chroot_results_dir = '/%s' % os.path.relpath(results_dir, chroot.path) vms = test.CreateMoblabVm(workspace_dir, chroot.path, image_payload_dir) cache_dir = test.PrepareMoblabVmImageCache(vms, builder, cache_payload_dirs) test.RunMoblabVmTest(chroot, vms, builder, cache_dir, chroot_results_dir) test.ValidateMoblabVmTest(results_dir)
def GenerateStatefulPayload(image_path, output_directory): """Generates a stateful update payload given a full path to an image. Args: image_path: Full path to the image. output_directory: Path to the directory to leave the resulting output. Returns: str: The full path to the generated file. """ logging.info('Generating stateful update file.') output_gz = os.path.join(output_directory, _STATEFUL_FILE) # Mount the image to pull out the important directories. with osutils.TempDir() as stateful_mnt, \ image_lib.LoopbackPartitions(image_path, stateful_mnt) as image: rootfs_dir = image.Mount((constants.PART_STATE,))[0] try: logging.info('Tarring up /usr/local and /var!') cros_build_lib.sudo_run([ 'tar', '-czf', output_gz, '--directory=%s' % rootfs_dir, '--hard-dereference', '--transform=s,^dev_image,dev_image_new,', '--transform=s,^var_overlay,var_new,', 'dev_image', 'var_overlay', ]) except: logging.error('Failed to create stateful update file') raise logging.info('Successfully generated %s', output_gz) return output_gz
def _GetPlatformImageParams(self, image): """Returns parameters related to target or source platform images. Since this function is mounting a GPT image, if the mount (for reasons like a bug, etc), changes the bits on the image, then the image cannot be trusted after this call. Args: image: The input image. Returns: The release APPID of the image. """ # Mount the ROOT-A partition of the image. The reason we don't mount the # extracted partition directly is that if by mistake/bug the mount changes # the bits on the partition, then we will create a payload for a changed # partition which is not equivalent to the original partition. So just mount # the partition of the GPT image and even if it changes, then who cares. # # TODO(crbug.com/925203): Replace this with image_lib.LoopbackPartition() # once the mentioned bug is resolved. with osutils.TempDir(base_dir=self.work_dir) as mount_point: with image_lib.LoopbackPartitions( image, destination=mount_point, part_ids=(constants.PART_ROOT_A, )): sysroot_dir = os.path.join(mount_point, 'dir-%s' % constants.PART_ROOT_A) lsb_release = utils.ReadLsbRelease(sysroot_dir) app_id = lsb_release.get( cros_set_lsb_release.LSB_KEY_APPID_RELEASE) if app_id is None: board = lsb_release.get( cros_set_lsb_release.LSB_KEY_APPID_BOARD) logging.error( 'APPID is missing in board %s. In some boards that do ' 'not do auto updates, like amd64-generic, this is ' 'expected, otherwise this is an error.', board) return app_id
def SignImage(image_type, input_file, output_file, kernel_part_id, keydir, keyA_prefix='', vboot_path=None): """Sign the image file. A Chromium OS image file (INPUT) always contains 2 partitions (kernel A & B). This function will rebuild hash data by DM_PARTNO, resign kernel partitions by their KEYBLOCK and PRIVKEY files, and then write to OUTPUT file. Note some special images (specified by IMAGE_TYPE, like 'recovery' or 'factory_install') may have additional steps (ex, tweaking verity hash or not stripping files) when generating output file. Args: image_type: Type of image (e.g., 'factory', 'recovery'). input_file: Image to sign. (read-only: copied to output_file). output_file: Signed image. (file is created here) kernel_part_id: partition number (or name) for the kernel (usually 2, 4 on recovery media.) keydir: Path of keyset dir to use. keyA_prefix: Prefix for kernA key (e.g., 'recovery_'). vboot_path: Vboot_reference/scripts/image_signing dir path. Raises SignImageException """ extra_env = _PathForVbootSigningScripts(vboot_path) logging.info('Preparing %s image...', image_type) cros_build_lib.run(['cp', '--sparse=always', input_file, output_file]) keyset = keys.Keyset(keydir) firmware.ResignImageFirmware(output_file, keyset) with osutils.TempDir() as dest_dir: with image_lib.LoopbackPartitions(output_file, dest_dir) as image: rootfs_dir = image.Mount(('ROOT-A', ))[0] SignAndroidImage(rootfs_dir, keyset, vboot_path=vboot_path) SignUefiBinaries(image, rootfs_dir, keyset, vboot_path=vboot_path) image.Unmount(('ROOT-A', )) # TODO(lamontjones): From this point on, all we really want at the moment # is the loopback devicefile names, but that may change as we implement # more of the shell functions. # # We do not actually want to have any filesystems mounted at this point. loop_kernA = image.GetPartitionDevName('KERN-A') loop_rootfs = image.GetPartitionDevName('ROOT-A') loop_kern = image.GetPartitionDevName(kernel_part_id) kernA_cmd = _GetKernelCmdLine(loop_kernA) if (image_type != 'factory_install' and not kernA_cmd.GetKernelParameter('cros_legacy') and not kernA_cmd.GetKernelParameter('cros_efi')): cros_build_lib.run( ['strip_boot_from_image.sh', '--image', loop_rootfs], extra_env=extra_env) ClearResignFlag(image) UpdateRootfsHash(image, loop_kern, keyset, keyA_prefix) UpdateStatefulPartitionVblock(image, keyset) if image_type == 'recovery': UpdateRecoveryKernelHash(image, keyset) UpdateLegacyBootloader(image, loop_kern) logging.info('Signed %s image written to %s', image_type, output_file)