def test_MergeManifests_missing_files():
"""Test that files that only appear in one manifest are not modified."""
d1 = dpack_pb2.DataPackage()
f1 = d1.file.add()
f1.relative_path = 'a'
f1.comment = 'abc'
d2 = dpack_pb2.DataPackage()
f2 = d2.file.add()
f2.relative_path = 'b'
f2.comment = 'def'
dpack.MergeManifests(d1, d2)
assert d1.file[0].comment == 'abc'
assert d2.file[0].comment == 'def'
def InitManifest(package_dir: pathlib.Path,
contents: typing.List[pathlib.Path], update: bool) -> None:
"""Write the MANIFEST.pbtxt file for a package."""
manifest = CreatePackageManifest(package_dir, contents)
manifest_path = package_dir / 'MANIFEST.pbtxt'
if update and pbutil.ProtoIsReadable(manifest_path,
dpack_pb2.DataPackage()):
old = pbutil.FromFile(manifest_path, dpack_pb2.DataPackage())
MergeManifests(manifest, old)
elif manifest_path.is_file():
raise OSError('Refusing to overwrite MANIFEST.pbtxt file.')
pbutil.ToFile(manifest, manifest_path)
logging.info('Wrote %s', manifest_path.absolute())
def test_MergeManifests_comments():
"""Test that comments from old manifests are copied to the new ones."""
d1 = dpack_pb2.DataPackage()
f1 = d1.file.add()
f1.relative_path = 'a'
d2 = dpack_pb2.DataPackage()
d2.comment = 'abc'
f2 = d2.file.add()
f2.comment = 'def'
f2.relative_path = 'a'
dpack.MergeManifests(d1, d2)
assert d1.comment == d2.comment
assert d1.file[0].comment == d2.file[0].comment
def CreatePackageArchiveSidecar(archive_path: pathlib.Path,
manifest: dpack_pb2.DataPackage,
sidecar_path: pathlib.Path) -> None:
"""Create a sidecar manifest to accompany an archive.
Args:
archive_path: The path of the archive tarball.
manifest: A DataPackage manifest instance.
sidecar_path: The path of the sidecar to create
Raises:
OSError: If sidecar_path already exists, or archive_path does not.
"""
if sidecar_path.exists():
raise OSError(f'Refusing to overwrite {sidecar_path}.')
if not archive_path.is_file():
raise OSError(f'Archive {archive_path} does not exist')
sidecar = dpack_pb2.DataPackage()
sidecar.CopyFrom(manifest)
# Clear the file attributes. Only the file names and comments are stored in the sidecar.
for f in sidecar.file:
if not f.comment:
f.ClearField("comment")
f.ClearField("size_in_bytes")
f.ClearField("checksum_hash")
f.ClearField("checksum")
sidecar.checksum_hash = dpack_pb2.SHA256
sidecar.checksum = crypto.sha256_file(archive_path)
pbutil.ToFile(sidecar, sidecar_path)
logging.info('Wrote %s', sidecar_path.absolute())
def test_MergeManifests_file_attributes():
"""Test that file attributes are not merged."""
d1 = dpack_pb2.DataPackage()
f1 = d1.file.add()
f1.relative_path = 'a'
f1.size_in_bytes = 1
f1.checksum_hash = dpack_pb2.SHA1
f1.checksum = 'abc'
d2 = dpack_pb2.DataPackage()
f2 = d2.file.add()
f2.relative_path = 'a'
f2.size_in_bytes = 2
f2.checksum_hash = dpack_pb2.MD5
f2.checksum = 'def'
dpack.MergeManifests(d1, d2)
assert d1.file[0].size_in_bytes == 1
assert d1.file[0].checksum_hash == dpack_pb2.SHA1
assert d1.file[0].checksum == 'abc'
def PackDataPackage(package_dir: pathlib.Path) -> None:
"""Create an archive and sidecar of a package."""
manifest = pbutil.FromFile(package_dir / 'MANIFEST.pbtxt',
dpack_pb2.DataPackage())
PackageManifestIsValid(package_dir, manifest)
archive_path = (package_dir /
f'../{package_dir.name}.dpack.tar.bz2').resolve()
sidecar_path = (package_dir /
f'../{package_dir.name}.dpack.pbtxt').resolve()
CreatePackageArchive(package_dir, manifest, archive_path)
CreatePackageArchiveSidecar(archive_path, manifest, sidecar_path)
def VerifyManifest(package_dir: pathlib.Path) -> bool:
"""Verify that the MANIFEST.pbtext file matches the contents."""
if not (package_dir / 'MANIFEST.pbtxt').is_file():
logging.info('%s/MANIFEST.pbtxt missing, nothing to do.', package_dir)
return False
manifest = pbutil.FromFile(package_dir / 'MANIFEST.pbtxt',
dpack_pb2.DataPackage())
if not PackageManifestIsValid(package_dir, manifest):
logging.error('Package %s contains errors.', package_dir)
return False
logging.info('%s verified. No changes to files in the manifest.',
package_dir)
return True
def VerifyManifest(package_dir: pathlib.Path) -> bool:
"""Verify that the MANIFEST.pbtext file matches the contents."""
if not (package_dir / "MANIFEST.pbtxt").is_file():
app.Log(1, "%s/MANIFEST.pbtxt missing, nothing to do.", package_dir)
return False
manifest = pbutil.FromFile(package_dir / "MANIFEST.pbtxt",
dpack_pb2.DataPackage())
if not PackageManifestIsValid(package_dir, manifest):
app.Error("Package %s contains errors.", package_dir)
return False
app.Log(1, "%s verified. No changes to files in the manifest.",
package_dir)
return True
def SidecarIsValid(archive: pathlib.Path, sidecar: pathlib.Path) -> None:
"""Check the archive matches the attributes in the sidecar."""
sidecar_manifest = pbutil.FromFile(sidecar, dpack_pb2.DataPackage())
hash_fn = dpack_pb2.ChecksumHash.Name(
sidecar_manifest.checksum_hash).lower()
try:
checksum_fn = getattr(crypto, hash_fn + "_file")
except AttributeError:
app.Warning("unknown value for field checksum_hash in manifest")
return False
checksum = checksum_fn(archive)
if sidecar_manifest.checksum != checksum:
app.Warning("the contents of '%s' have changed", archive.absolute())
return False
app.Log(1, "Package verified using the sidecar.")
return True
def CreatePackageManifest(
package_root: pathlib.Path,
contents: typing.List[pathlib.Path]) -> dpack_pb2.DataPackage:
"""Create a DataPackage message for the contents of a package.
Args:
package_root: The root of the package.
contents: A list of relative paths to files to include.
Returns:
A DataPackage instance with attributes set.
"""
manifest = dpack_pb2.DataPackage()
manifest.comment = ''
manifest.utc_epoch_ms_packaged = labdate.MillisecondsTimestamp(
labdate.GetUtcMillisecondsNow())
for path in contents:
f = manifest.file.add()
SetDataPackageFileAttributes(package_root, path, f)
f.comment = f.comment or ''
return manifest
def _IsManifest(path: pathlib.Path) -> bool:
"""Check if a path contains a DataPackafe file."""
return pbutil.ProtoIsReadable(path, dpack_pb2.DataPackage())