def test_deserialization(self):
self.maxDiff = None
aib = AssemblyInputBundle()
aib.base.update(["package1", "package2"])
aib.cache.update(["package3", "package4"])
aib.system.add("package0")
aib.kernel.path = "path/to/kernel"
aib.kernel.args.update(["arg1", "arg2"])
aib.kernel.clock_backstop = 1234
aib.boot_args.update(["arg3", "arg4"])
aib.bootfs_files.add(FileEntry("path/to/source", "path/to/destination"))
aib.config_data["package1"] = set(
[FileEntry("path/to/source.json", "config.json")])
parsed_aib = AssemblyInputBundle.json_loads(
raw_assembly_input_bundle_json)
def assert_field_equal(parsed, expected, field_name):
self.assertEqual(
getattr(parsed, field_name), getattr(expected, field_name))
assert_field_equal(parsed_aib, aib, "base")
assert_field_equal(parsed_aib, aib, "cache")
assert_field_equal(parsed_aib, aib, "system")
assert_field_equal(parsed_aib, aib, "kernel")
assert_field_equal(parsed_aib, aib, "boot_args")
assert_field_equal(parsed_aib, aib, "bootfs_files")
assert_field_equal(parsed_aib, aib, "config_data")
self.assertEqual(parsed_aib, aib)
def test_serialization(self):
self.maxDiff = None
aib = AssemblyInputBundle()
aib.base.update(["package1", "package2"])
aib.cache.update(["package3", "package4"])
aib.system.add("package0")
aib.kernel.path = "path/to/kernel"
aib.kernel.args.update(["arg1", "arg2"])
aib.kernel.clock_backstop = 1234
aib.boot_args.update(["arg3", "arg4"])
aib.bootfs_files.add(FileEntry("path/to/source", "path/to/destination"))
aib.config_data["package1"] = set(
[FileEntry("path/to/source.json", "config.json")])
self.assertEqual(
aib.json_dumps(indent=2), raw_assembly_input_bundle_json)
def main():
parser = argparse.ArgumentParser(
description=
"Create an image assembly configuration that is what remains after removing the configs to 'subtract'"
)
parser.add_argument("--image-assembly-config",
type=argparse.FileType('r'),
required=True)
parser.add_argument("--config-data-entries", type=argparse.FileType('r'))
parser.add_argument("--subtract",
default=[],
nargs="*",
type=argparse.FileType('r'))
parser.add_argument("--outdir", required=True)
parser.add_argument("--depfile", type=argparse.FileType('w'))
parser.add_argument("--export-manifest", type=argparse.FileType('w'))
args = parser.parse_args()
# Read in the legacy config and the others to subtract from it
legacy: ImageAssemblyConfig = ImageAssemblyConfig.json_load(
args.image_assembly_config)
subtract = [
ImageAssemblyConfig.json_load(other) for other in args.subtract
]
# Subtract each from the legacy config, in the order given in args.
for other in subtract:
legacy = legacy.difference(other)
# Read in the config_data entries if available.
if args.config_data_entries:
config_data_entries = [
FileEntry.from_dict(entry)
for entry in json.load(args.config_data_entries)
]
else:
config_data_entries = []
# Create an Assembly Input Bundle from the remaining contents
(assembly_input_bundle, assembly_config_manifest_path,
deps) = copy_to_assembly_input_bundle(legacy, config_data_entries,
args.outdir)
# Write out a fini manifest of the files that have been copied, to create a
# package or archive that contains all of the files in the bundle.
if args.export_manifest:
assembly_input_bundle.write_fini_manifest(args.export_manifest,
base_dir=args.outdir)
# Write out a depfile.
if args.depfile:
dep_file = DepFile(assembly_config_manifest_path)
dep_file.update(deps)
dep_file.write_to(args.depfile)
def test_aib_creator_file_copy_and_package_manifest_relative_paths(self):
"""This tests that the AIBCreator will correctly copy the blobs/* files
and create the package manifests in the correct location within the
AIB structure, with package-manifest relative paths to the blobs.
It also tests that the fini manifest and dep-files contain the
correct paths.
"""
# Diffs can be very large for some of these lists, so show the whole
# thing.
self.maxDiff = None
# Mock out the copy routine so it doesn't fail, but we can see the ops
# it would have made.
(_, copies) = mock_fast_copy_in(assembly.assembly_input_bundle)
outdir = tempfile.TemporaryDirectory()
# save off the current cwd so that it can be restored later.
curr_cwd = os.getcwd()
try:
# Switch into the tempdir to simulate a build environment.
os.chdir(outdir.name)
assembly_dir = "my_assembly"
inputs_dir = "inputs"
os.mkdir(assembly_dir)
os.mkdir(inputs_dir)
# Create two package manifests to use as base packages. These will
# be used to validate that blobs are copied to the right places, and
# that the manifests are re-written correctly in a portable manner.
some_package_manifest_path = "inputs/some_package_manifest.json"
some_package_manifest = PackageManifestBuilder(
"some_package"
).manifest_path(some_package_manifest_path).blob(
path="meta/",
merkle=
"0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789ABCDEF",
source="some/meta.far").fake_blob(11).fake_blob(12).fake_blob(
13).build()
another_package_manifest_path = "inputs/another_package_manifest.json"
another_package_manifest = PackageManifestBuilder(
"another_package"
).manifest_path(another_package_manifest_path).blob(
path="meta/",
merkle=
"123456789abcdef0123456789ABCDEF0123456789abcdef0123456789ABCDEF0",
source="another/meta.far").fake_blob(26).fake_blob(27).build()
created_manifests = [
some_package_manifest, another_package_manifest
]
# These are the files, and their sources, that we expect to find
# copied into the AIB. The destination paths here are the paths
# used within the AIB, as that can then be used to compute the path
# as seen from the cwd (which is what needs to be in the copy and
# fini manifest operations).
expected_files = [
# some_package files
FileEntry(
"some/meta.far",
"blobs/0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789ABCDEF"
),
FileEntry(
"source/path/for/input_11", f"blobs/{format_merkle(11)}"),
FileEntry(
"source/path/for/input_12", f"blobs/{format_merkle(12)}"),
FileEntry(
"source/path/for/input_13", f"blobs/{format_merkle(13)}"),
# another_package files
FileEntry(
"another/meta.far",
"blobs/123456789abcdef0123456789ABCDEF0123456789abcdef0123456789ABCDEF0"
),
FileEntry(
"source/path/for/input_26", f"blobs/{format_merkle(26)}"),
FileEntry(
"source/path/for/input_27", f"blobs/{format_merkle(27)}")
]
source_package_manifests = [
some_package_manifest_path, another_package_manifest_path
]
expected_package_manifests = [
"packages/base/some_package", "packages/base/another_package"
]
# Create the AIBCreator and perform the operation that's under test.
aib_creator = AIBCreator(assembly_dir)
aib_creator.base.update(
[some_package_manifest_path, another_package_manifest_path])
bundle, bundle_path, deps = aib_creator.build()
# Verify that the bundle was written to the correct location (and it
# matches the returned bundle).
self.assertEqual(
bundle_path, os.path.join(assembly_dir, "assembly_config.json"))
with open(bundle_path) as bundle_file:
parsed_bundle = AssemblyInputBundle.json_load(bundle_file)
self.assertEqual(parsed_bundle, bundle)
# Verify that resultant AIB contains the correct base packages.
self.assertEqual(bundle.base, set(expected_package_manifests))
# Verify that the package manfiests have been rewritten to use file-
# relative blob paths into the correct directory.
def validate_rewritten_package_manifest(
path: FilePath, expected: PackageManifest):
"""Parses the PackageManifest at the given path, and compares it
with the `expected` one.
"""
with open(path) as package_manifest_file:
parsed_manifest = serialization.json_load(
PackageManifest, package_manifest_file)
self.assertEqual(parsed_manifest.package, expected.package)
self.assertEqual(parsed_manifest.blob_sources_relative, "file")
# The expected blobs are the passed-in set, but with the source
# path set to be by merkle in the blobs/ dir of the AIB.
expected_blobs = [
BlobEntry(
blob.path, blob.merkle, blob.size,
f"../../blobs/{blob.merkle}") for blob in expected.blobs
]
self.assertEqual(parsed_manifest.blobs, expected_blobs)
for (path, manifest) in zip(expected_package_manifests,
created_manifests):
validate_rewritten_package_manifest(
os.path.join(assembly_dir, path), manifest)
# Verify that the copies that were performed by the mocked
# `fast_copy()` fn are correct.
#
# The expected copies are the the expected_files, but the
# destination path is rebased to the cwd (prepending `assembly_dir`)
#
expected_copies = [
rebase_destination(entry, assembly_dir)
for entry in expected_files
]
self.assertEqual(sorted(copies), sorted(expected_copies))
# Verify that the deps are correctly reported.
#
# All the source paths that files were copied from should be listed,
# as well as the package manifest paths that were read to create the
# in-AIB version of the manifests.
expected_deps = [entry.source for entry in expected_files]
expected_deps.extend(source_package_manifests)
self.assertEqual(
sorted(deps), sorted(expected_deps)) # type: ignore
# Verify that the fini manifest created (used to create archives of
# the AIB is correct).
#
# The fini manifest contains all the destination paths from
# expected_files, for both source and destination, but assembly_dir
# is prepended to all source paths to make them relative to the cwd.
# Verify that all_file_paths() returns the correct (AIB-relative)
# files.
expected_paths = [entry.destination for entry in expected_files]
expected_paths.extend(expected_package_manifests)
self.assertEqual(
sorted(bundle.all_file_paths()),
sorted(expected_paths)) # type: ignore
# Created the expected entries from the expected_paths by pre-
# pending the assembly_dir to create the source path.
expected_paths.append("assembly_config.json")
expected_fini_contents = sorted(
[f"{path}={assembly_dir}/{path}" for path in expected_paths])
# Write the fini manifest to a string buffer
fini_file = io.StringIO()
bundle.write_fini_manifest(fini_file, base_dir=assembly_dir)
# Parse the written buffer into lines to compare with the expected
# entries.
fini_entries = sorted(fini_file.getvalue().splitlines())
self.assertEqual(fini_entries, expected_fini_contents)
finally:
os.chdir(curr_cwd)
# Clean up the tempdir
outdir.cleanup()
def rebase_destination(entry: FileEntry, path: str) -> FileEntry:
return FileEntry(entry.source, os.path.join(path, entry.destination))
def test_make_legacy_config(self):
self.maxDiff = None
# Patch in a mock for the fast_copy() fn
fast_copy_mock_fn, copies = mock_fast_copy_in(
assembly.assembly_input_bundle)
with tempfile.TemporaryDirectory() as temp_dir_path:
os.chdir(temp_dir_path)
source_dir = "source"
os.mkdir(source_dir)
# Create an ImageAssembly configuration
image_assembly = ImageAssemblyConfig()
# Write out package manifests which are part of the package.
for package_set in ["base", "cache", "system"]:
for suffix in ["a", "b"]:
package_name = f"{package_set}_{suffix}"
manifest = PackageManifest(PackageMetaData(package_name),
[])
# Create a few blob entries (that don't need to fully exist)
for blob_suffix in ["1", "2", "3"]:
blob_name = f"internal/path/file_{suffix}_{blob_suffix}"
entry = BlobEntry(
blob_name, make_merkle(package_name + blob_name),
None,
os.path.join(source_dir, package_name, blob_name))
manifest.blobs.append(entry)
# Write the manifest out to the temp dir
manifest_path = os.path.join(source_dir,
f"{package_name}.json")
with open(manifest_path, 'w') as manifest_file:
serialization.json_dump(manifest,
manifest_file,
indent=2)
# Add to the ImageAssembly in the correct package set.
getattr(image_assembly, package_set).add(manifest_path)
# Add the rest of the fields we expect to see in an image_assembly
# config.
image_assembly.boot_args.update(["boot-arg-1", "boot-arg-2"])
image_assembly.kernel.path = os.path.join(source_dir, "kernel.bin")
image_assembly.kernel.args.update(["arg1", "arg2"])
image_assembly.kernel.clock_backstop = 123456
image_assembly.bootfs_files.update([
FileEntry(os.path.join(source_dir, "some/file"), "some/file"),
FileEntry(os.path.join(source_dir, "another/file"),
"another/file"),
])
# Create the outdir path, and perform the "copying" into the
# AssemblyInputBundle.
outdir = "outdir"
aib, assembly_config, deps = make_legacy_config.copy_to_assembly_input_bundle(
image_assembly, [], outdir)
# Validate the contents of the AssemblyInputBundle itself
self.assertEqual(
aib.base, set(["packages/base/base_a",
"packages/base/base_b"]))
self.assertEqual(
aib.cache,
set(["packages/cache/cache_a", "packages/cache/cache_b"]))
self.assertEqual(
aib.system,
set(["packages/system/system_a", "packages/system/system_b"]))
self.assertEqual(aib.boot_args, set(["boot-arg-1", "boot-arg-2"]))
self.assertEqual(aib.kernel.path, "kernel/kernel.bin")
self.assertEqual(aib.kernel.args, set(["arg1", "arg2"]))
self.assertEqual(aib.kernel.clock_backstop, 123456)
self.assertEqual(
aib.bootfs_files,
set([
FileEntry(source="bootfs/some/file",
destination="some/file"),
FileEntry(source="bootfs/another/file",
destination="another/file"),
]))
# Make sure all the manifests were created in the correct location.
for package_set in ["base", "cache", "system"]:
for suffix in ["a", "b"]:
package_name = f"{package_set}_{suffix}"
with open(f"outdir/packages/{package_set}/{package_name}"
) as manifest_file:
manifest = serialization.json_load(
PackageManifest, manifest_file)
self.assertEqual(manifest.package.name, package_name)
self.assertEqual(
set(manifest.blobs_by_path().keys()),
set([
f'internal/path/file_{suffix}_1',
f'internal/path/file_{suffix}_2',
f'internal/path/file_{suffix}_3',
]))
# Spot-check one of the manifests, that it contains the correct
# source paths to the blobs.
with open("outdir/packages/base/base_a") as manifest_file:
manifest = serialization.json_load(PackageManifest,
manifest_file)
self.assertEqual(manifest.package.name, "base_a")
self.assertEqual(len(manifest.blobs), 3)
blobs = manifest.blobs_by_path()
self.assertEqual(
blobs['internal/path/file_a_1'].source_path,
'../../blobs/efac096092f7cf879c72ac51d23d9f142e97405dec7dd9c69aeee81de083f794'
)
self.assertEqual(
blobs['internal/path/file_a_1'].merkle,
'efac096092f7cf879c72ac51d23d9f142e97405dec7dd9c69aeee81de083f794'
)
self.assertEqual(
blobs['internal/path/file_a_2'].source_path,
'../../blobs/bf0c3ae1356b5863258f73a37d555cf878007b8bfe4fd780d74466ec62fe062d'
)
self.assertEqual(
blobs['internal/path/file_a_2'].merkle,
'bf0c3ae1356b5863258f73a37d555cf878007b8bfe4fd780d74466ec62fe062d'
)
self.assertEqual(
blobs['internal/path/file_a_3'].source_path,
'../../blobs/a2e574ccd55c815f0a87c4f27e7a3115fe8e46d41a2e0caf2a91096a41421f78'
)
self.assertEqual(
blobs['internal/path/file_a_3'].merkle,
'a2e574ccd55c815f0a87c4f27e7a3115fe8e46d41a2e0caf2a91096a41421f78'
)
# Validate that the deps were correctly identified (all package
# manifest paths, the blob source paths, the bootfs source paths,
# and the kernel source path)
self.assertEqual(
deps,
set([
'source/base_a.json',
'source/base_a/internal/path/file_a_1',
'source/base_a/internal/path/file_a_2',
'source/base_a/internal/path/file_a_3',
'source/base_b.json',
'source/base_b/internal/path/file_b_1',
'source/base_b/internal/path/file_b_2',
'source/base_b/internal/path/file_b_3',
'source/cache_a.json',
'source/cache_a/internal/path/file_a_1',
'source/cache_a/internal/path/file_a_2',
'source/cache_a/internal/path/file_a_3',
'source/cache_b.json',
'source/cache_b/internal/path/file_b_1',
'source/cache_b/internal/path/file_b_2',
'source/cache_b/internal/path/file_b_3',
'source/system_a.json',
'source/system_a/internal/path/file_a_1',
'source/system_a/internal/path/file_a_2',
'source/system_a/internal/path/file_a_3',
'source/system_b.json',
'source/system_b/internal/path/file_b_1',
'source/system_b/internal/path/file_b_2',
'source/system_b/internal/path/file_b_3',
'source/kernel.bin', 'source/some/file',
'source/another/file'
]))
# Validate that all the files were correctly copied to the
# correct paths in the AIB.
self.assertEqual(
set(copies),
set([
FileEntry(
source='source/base_a/internal/path/file_a_1',
destination=
'outdir/blobs/efac096092f7cf879c72ac51d23d9f142e97405dec7dd9c69aeee81de083f794'
),
FileEntry(
source='source/base_a/internal/path/file_a_1',
destination=
'outdir/blobs/efac096092f7cf879c72ac51d23d9f142e97405dec7dd9c69aeee81de083f794'
),
FileEntry(
source='source/base_a/internal/path/file_a_2',
destination=
'outdir/blobs/bf0c3ae1356b5863258f73a37d555cf878007b8bfe4fd780d74466ec62fe062d'
),
FileEntry(
source='source/base_a/internal/path/file_a_3',
destination=
'outdir/blobs/a2e574ccd55c815f0a87c4f27e7a3115fe8e46d41a2e0caf2a91096a41421f78'
),
FileEntry(
source='source/base_b/internal/path/file_b_1',
destination=
'outdir/blobs/ae9fd81e1c2fd1b084ec2c362737e812c5ef9b3aa8cb0538ec8e2269ea7fbe1a'
),
FileEntry(
source='source/base_b/internal/path/file_b_2',
destination=
'outdir/blobs/d3cd38c4881c3bc31f1e2e397a548d431a6430299785446f28be10cc5b76d92b'
),
FileEntry(
source='source/base_b/internal/path/file_b_3',
destination=
'outdir/blobs/6468d9d6761c8afcc97744dfd9e066f29bb697a9a0c8248b5e6eec989134a048'
),
FileEntry(
source='source/cache_a/internal/path/file_a_1',
destination=
'outdir/blobs/f0601d51be1ec8c11d825b756841937706eb2805ce9b924b67b4b0dc14caba29'
),
FileEntry(
source='source/cache_a/internal/path/file_a_2',
destination=
'outdir/blobs/1834109a42a5ff6501fbe05216475b2b0acc44e0d9c94924469a485d6f45dc86'
),
FileEntry(
source='source/cache_a/internal/path/file_a_3',
destination=
'outdir/blobs/0f32059964674afd810001c76c2a5d783a2ce012c41303685ec1adfdb83290fd'
),
FileEntry(
source='source/cache_b/internal/path/file_b_1',
destination=
'outdir/blobs/301e8584305e63f0b764daf52dcf312eecb6378b201663fcc77d7ad68aab1f23'
),
FileEntry(
source='source/cache_b/internal/path/file_b_2',
destination=
'outdir/blobs/8135016519df51d386efaea9b02f50cb454b6c7afe69c77895c1d4d844c3584d'
),
FileEntry(
source='source/cache_b/internal/path/file_b_3',
destination=
'outdir/blobs/b548948fd2dc40574775308a92a8330e5c5d84ddf31513d1fe69964b458479e7'
),
FileEntry(
source='source/system_a/internal/path/file_a_1',
destination=
'outdir/blobs/8ca898b1389c58b6cd9a6a777e320f2756ab3437b402c61d774dd2758ad9cf06'
),
FileEntry(
source='source/system_a/internal/path/file_a_2',
destination=
'outdir/blobs/ef84c6711eaba482164fe4eb08a6c45f18fe62d493e5a31a631c32937bf7229d'
),
FileEntry(
source='source/system_a/internal/path/file_a_3',
destination=
'outdir/blobs/d66cb673257e25393a319fb2c3e9745ef6e0f1cfa4fb89c5576df73cd3eba586'
),
FileEntry(
source='source/system_b/internal/path/file_b_1',
destination=
'outdir/blobs/fd0891d15ce65d7682f7437e441e917b8ed4bde4db07a11dc100104f25056051'
),
FileEntry(
source='source/system_b/internal/path/file_b_2',
destination=
'outdir/blobs/c244c7c6ebf40a9a4c9d59e7b08a1cf54ae3d60404d1cecb417a7b55cc308d91'
),
FileEntry(
source='source/system_b/internal/path/file_b_3',
destination=
'outdir/blobs/0cdbf3e4f1246ce7522e78c21bcf1c3aef2d41ac2b4de3f0ee98fc6273f62eb9'
),
FileEntry(source='source/kernel.bin',
destination='outdir/kernel/kernel.bin'),
FileEntry(source='source/some/file',
destination='outdir/bootfs/some/file'),
FileEntry(source='source/another/file',
destination='outdir/bootfs/another/file'),
]))