def test_MergeManifests_missing_files(): """Test that files that only appear in one manifest are not modified.""" d1 = dpack_pb2.DataPackage() f1 = d1.file.add() f1.relative_path = 'a' f1.comment = 'abc' d2 = dpack_pb2.DataPackage() f2 = d2.file.add() f2.relative_path = 'b' f2.comment = 'def' dpack.MergeManifests(d1, d2) assert d1.file[0].comment == 'abc' assert d2.file[0].comment == 'def'
def InitManifest(package_dir: pathlib.Path, contents: typing.List[pathlib.Path], update: bool) -> None: """Write the MANIFEST.pbtxt file for a package.""" manifest = CreatePackageManifest(package_dir, contents) manifest_path = package_dir / 'MANIFEST.pbtxt' if update and pbutil.ProtoIsReadable(manifest_path, dpack_pb2.DataPackage()): old = pbutil.FromFile(manifest_path, dpack_pb2.DataPackage()) MergeManifests(manifest, old) elif manifest_path.is_file(): raise OSError('Refusing to overwrite MANIFEST.pbtxt file.') pbutil.ToFile(manifest, manifest_path) logging.info('Wrote %s', manifest_path.absolute())
def test_MergeManifests_comments(): """Test that comments from old manifests are copied to the new ones.""" d1 = dpack_pb2.DataPackage() f1 = d1.file.add() f1.relative_path = 'a' d2 = dpack_pb2.DataPackage() d2.comment = 'abc' f2 = d2.file.add() f2.comment = 'def' f2.relative_path = 'a' dpack.MergeManifests(d1, d2) assert d1.comment == d2.comment assert d1.file[0].comment == d2.file[0].comment
def CreatePackageArchiveSidecar(archive_path: pathlib.Path, manifest: dpack_pb2.DataPackage, sidecar_path: pathlib.Path) -> None: """Create a sidecar manifest to accompany an archive. Args: archive_path: The path of the archive tarball. manifest: A DataPackage manifest instance. sidecar_path: The path of the sidecar to create Raises: OSError: If sidecar_path already exists, or archive_path does not. """ if sidecar_path.exists(): raise OSError(f'Refusing to overwrite {sidecar_path}.') if not archive_path.is_file(): raise OSError(f'Archive {archive_path} does not exist') sidecar = dpack_pb2.DataPackage() sidecar.CopyFrom(manifest) # Clear the file attributes. Only the file names and comments are stored in the sidecar. for f in sidecar.file: if not f.comment: f.ClearField("comment") f.ClearField("size_in_bytes") f.ClearField("checksum_hash") f.ClearField("checksum") sidecar.checksum_hash = dpack_pb2.SHA256 sidecar.checksum = crypto.sha256_file(archive_path) pbutil.ToFile(sidecar, sidecar_path) logging.info('Wrote %s', sidecar_path.absolute())
def test_MergeManifests_file_attributes(): """Test that file attributes are not merged.""" d1 = dpack_pb2.DataPackage() f1 = d1.file.add() f1.relative_path = 'a' f1.size_in_bytes = 1 f1.checksum_hash = dpack_pb2.SHA1 f1.checksum = 'abc' d2 = dpack_pb2.DataPackage() f2 = d2.file.add() f2.relative_path = 'a' f2.size_in_bytes = 2 f2.checksum_hash = dpack_pb2.MD5 f2.checksum = 'def' dpack.MergeManifests(d1, d2) assert d1.file[0].size_in_bytes == 1 assert d1.file[0].checksum_hash == dpack_pb2.SHA1 assert d1.file[0].checksum == 'abc'
def PackDataPackage(package_dir: pathlib.Path) -> None: """Create an archive and sidecar of a package.""" manifest = pbutil.FromFile(package_dir / 'MANIFEST.pbtxt', dpack_pb2.DataPackage()) PackageManifestIsValid(package_dir, manifest) archive_path = (package_dir / f'../{package_dir.name}.dpack.tar.bz2').resolve() sidecar_path = (package_dir / f'../{package_dir.name}.dpack.pbtxt').resolve() CreatePackageArchive(package_dir, manifest, archive_path) CreatePackageArchiveSidecar(archive_path, manifest, sidecar_path)
def VerifyManifest(package_dir: pathlib.Path) -> bool: """Verify that the MANIFEST.pbtext file matches the contents.""" if not (package_dir / 'MANIFEST.pbtxt').is_file(): logging.info('%s/MANIFEST.pbtxt missing, nothing to do.', package_dir) return False manifest = pbutil.FromFile(package_dir / 'MANIFEST.pbtxt', dpack_pb2.DataPackage()) if not PackageManifestIsValid(package_dir, manifest): logging.error('Package %s contains errors.', package_dir) return False logging.info('%s verified. No changes to files in the manifest.', package_dir) return True
def VerifyManifest(package_dir: pathlib.Path) -> bool: """Verify that the MANIFEST.pbtext file matches the contents.""" if not (package_dir / "MANIFEST.pbtxt").is_file(): app.Log(1, "%s/MANIFEST.pbtxt missing, nothing to do.", package_dir) return False manifest = pbutil.FromFile(package_dir / "MANIFEST.pbtxt", dpack_pb2.DataPackage()) if not PackageManifestIsValid(package_dir, manifest): app.Error("Package %s contains errors.", package_dir) return False app.Log(1, "%s verified. No changes to files in the manifest.", package_dir) return True
def SidecarIsValid(archive: pathlib.Path, sidecar: pathlib.Path) -> None: """Check the archive matches the attributes in the sidecar.""" sidecar_manifest = pbutil.FromFile(sidecar, dpack_pb2.DataPackage()) hash_fn = dpack_pb2.ChecksumHash.Name( sidecar_manifest.checksum_hash).lower() try: checksum_fn = getattr(crypto, hash_fn + "_file") except AttributeError: app.Warning("unknown value for field checksum_hash in manifest") return False checksum = checksum_fn(archive) if sidecar_manifest.checksum != checksum: app.Warning("the contents of '%s' have changed", archive.absolute()) return False app.Log(1, "Package verified using the sidecar.") return True
def CreatePackageManifest( package_root: pathlib.Path, contents: typing.List[pathlib.Path]) -> dpack_pb2.DataPackage: """Create a DataPackage message for the contents of a package. Args: package_root: The root of the package. contents: A list of relative paths to files to include. Returns: A DataPackage instance with attributes set. """ manifest = dpack_pb2.DataPackage() manifest.comment = '' manifest.utc_epoch_ms_packaged = labdate.MillisecondsTimestamp( labdate.GetUtcMillisecondsNow()) for path in contents: f = manifest.file.add() SetDataPackageFileAttributes(package_root, path, f) f.comment = f.comment or '' return manifest
def _IsManifest(path: pathlib.Path) -> bool: """Check if a path contains a DataPackafe file.""" return pbutil.ProtoIsReadable(path, dpack_pb2.DataPackage())