def verify_cloudfront(config, root, files): ret = True tree = SimpleTree(config) md5sums = ChecksumFile( config, os.path.join(tree.directory, ".pool"), "MD5SUMS", None) md5sums.read() opener = build_opener(HTTPHeadRedirectHandler()) if not root.endswith("/"): root += "/" for f in files: if f not in md5sums.entries: # There are lots of miscellaneous boring files with no local # checksums. Silently ignore these for convenience. continue url = urljoin(root, f.replace("+", "%2B")) try: response = opener.open(HeadRequest(url)) except HTTPError as e: print("%s: %s" % (url, e), file=sys.stderr) continue try: etag = response.info()["ETag"].strip('"') if md5sums.entries[f] == etag: print("%s matches %s" % (f, url)) else: print("%s DOES NOT MATCH %s" % (f, url), file=sys.stderr) print(" Local: %s" % md5sums.entries[f], file=sys.stderr) print(" Remote: %s" % etag, file=sys.stderr) ret = False except KeyError: print("No remote ETag for %s; skipping." % url, file=sys.stderr) return ret
def verify_cloudfront(config, root, files): ret = True tree = SimpleReleaseTree(config) md5sums = ChecksumFile(config, os.path.join(tree.directory, ".pool"), "MD5SUMS", None) md5sums.read() opener = build_opener(HTTPHeadRedirectHandler()) if not root.endswith("/"): root += "/" for f in files: if f not in md5sums.entries: # There are lots of miscellaneous boring files with no local # checksums. Silently ignore these for convenience. continue url = urljoin(root, f.replace("+", "%2B")) try: response = opener.open(HeadRequest(url)) except HTTPError as e: print("%s: %s" % (url, e), file=sys.stderr) continue try: etag = response.info()["ETag"].strip('"') if md5sums.entries[f] == etag: print("%s matches %s" % (f, url)) else: print("%s DOES NOT MATCH %s" % (f, url), file=sys.stderr) print(" Local: %s" % md5sums.entries[f], file=sys.stderr) print(" Remote: %s" % etag, file=sys.stderr) ret = False except KeyError: print("No remote ETag for %s; skipping." % url, file=sys.stderr) return ret
def test_checksum_large_file(self): entry_path = os.path.join(self.temp_dir, "entry") data = b"a" * 1048576 with mkfile(entry_path, mode="wb") as entry: entry.write(data) checksum_file = ChecksumFile(self.config, self.temp_dir, "SHA1SUMS", hashlib.sha1) self.assertEqual( hashlib.sha1(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_small_file(self): entry_path = os.path.join(self.temp_dir, "entry") data = b"test\n" with mkfile(entry_path, mode="wb") as entry: entry.write(data) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) self.assertEqual( hashlib.md5(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_small_file(self): entry_path = os.path.join(self.temp_dir, "entry") data = "test\n" with open(entry_path, "w") as entry: print(data, end="", file=entry) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) self.assertEqual( hashlib.md5(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_large_file(self): entry_path = os.path.join(self.temp_dir, "entry") data = "a" * 1048576 with open(entry_path, "w") as entry: print(data, end="", file=entry) checksum_file = ChecksumFile( self.config, self.temp_dir, "SHA1SUMS", hashlib.sha1) self.assertEqual( hashlib.sha1(data).hexdigest(), checksum_file.checksum(entry_path))
def test_merge_takes_other_names(self): old_dir = os.path.join(self.temp_dir, "old") touch(os.path.join(self.temp_dir, "entry")) with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums: print("checksum *other-entry", file=old_md5sums) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.merge([old_dir], "entry", ["other-entry"]) self.assertEqual({"entry": "checksum"}, checksum_file.entries)
def test_add(self): entry_path = os.path.join(self.temp_dir, "entry") data = "test\n" with open(entry_path, "w") as entry: print(data, end="", file=entry) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.add("entry") self.assertEqual( {"entry": hashlib.md5(data).hexdigest()}, checksum_file.entries)
def test_merge_takes_other_names(self): old_dir = os.path.join(self.temp_dir, "old") os.mkdir(old_dir) touch(os.path.join(self.temp_dir, "entry")) with open(os.path.join(old_dir, "MD5SUMS"), "w") as old_md5sums: print("checksum *other-entry", file=old_md5sums) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.merge([old_dir], "entry", ["other-entry"]) self.assertEqual({"entry": "checksum"}, checksum_file.entries)
def test_add(self): entry_path = os.path.join(self.temp_dir, "entry") data = b"test\n" with mkfile(entry_path, mode="wb") as entry: entry.write(data) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.add("entry") self.assertEqual({"entry": hashlib.md5(data).hexdigest()}, checksum_file.entries)
def test_read(self): with open(os.path.join(self.temp_dir, "MD5SUMS"), "w") as md5sums: print(dedent("""\ checksum one-path checksum *another-path """), file=md5sums) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.read() self.assertEqual( {"one-path": "checksum", "another-path": "checksum"}, checksum_file.entries)
def test_add_existing(self): # Attempting to add an existing file has no effect. (Use .remove() # first to overwrite an existing checksum.) entry_path = os.path.join(self.temp_dir, "entry") data = "test\n" with open(entry_path, "w") as entry: print(data, end="", file=entry) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.entries["entry"] = "" checksum_file.add("entry") self.assertEqual("", checksum_file.entries["entry"])
def test_merge_ignores_stale_checksums(self): old_dir = os.path.join(self.temp_dir, "old") with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums: print("checksum *entry", file=old_md5sums) entry_path = os.path.join(self.temp_dir, "entry") touch(entry_path) next_minute = time.time() + 60 os.utime(entry_path, (next_minute, next_minute)) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.merge([old_dir], "entry", ["entry"]) self.assertEqual({}, checksum_file.entries)
def test_merge_ignores_stale_checksums(self): old_dir = os.path.join(self.temp_dir, "old") os.mkdir(old_dir) with open(os.path.join(old_dir, "MD5SUMS"), "w") as old_md5sums: print("checksum *entry", file=old_md5sums) entry_path = os.path.join(self.temp_dir, "entry") touch(entry_path) next_minute = time.time() + 60 os.utime(entry_path, (next_minute, next_minute)) checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.merge([old_dir], "entry", ["entry"]) self.assertEqual({}, checksum_file.entries)
def test_add_existing(self): # Attempting to add an existing file that is not newer than the # checksums file has no effect. (Use .remove() first to overwrite # an existing checksum.) entry_path = os.path.join(self.temp_dir, "entry") data = "test\n" with mkfile(entry_path) as entry: entry.write(data) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.entries["entry"] = "" checksum_file.add("entry") self.assertEqual("", checksum_file.entries["entry"])
def test_read(self): with mkfile(os.path.join(self.temp_dir, "MD5SUMS")) as md5sums: print(dedent("""\ checksum one-path checksum *another-path """), file=md5sums) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.read() self.assertEqual({ "one-path": "checksum", "another-path": "checksum" }, checksum_file.entries)
def test_merge_handles_symlinks(self): old_dir = os.path.join(self.temp_dir, "old") touch(os.path.join(old_dir, "entry")) with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums: print("correct-checksum *entry", file=old_md5sums) new_dir = os.path.join(self.temp_dir, "new") os.mkdir(new_dir) os.symlink(os.path.join(os.pardir, "old", "entry"), os.path.join(new_dir, "entry")) with mkfile(os.path.join(new_dir, "MD5SUMS")) as new_md5sums: print("wrong-checksum *entry", file=new_md5sums) checksum_file = ChecksumFile(self.config, new_dir, "MD5SUMS", hashlib.md5) checksum_file.merge([new_dir, old_dir], "entry", ["entry"]) self.assertEqual({"entry": "correct-checksum"}, checksum_file.entries)
def test_write(self): checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False) for name in "1", "2": entry_path = os.path.join(self.temp_dir, name) with open(entry_path, "w") as entry: print(name, end="", file=entry) checksum_file.add(name) checksum_file.write() with open(checksum_file.path) as md5sums: self.assertEqual(dedent("""\ %s *1 %s *2 """) % (hashlib.md5("1").hexdigest(), hashlib.md5("2").hexdigest()), md5sums.read()) self.assertEqual( 0, subprocess.call( ["md5sum", "-c", "--status", "MD5SUMS"], cwd=self.temp_dir))
def test_add_updated_mtime(self): # Adding an existing file with an mtime newer than that of the # checksums file causes its checksum to be updated. path = os.path.join(self.temp_dir, "entry") with mkfile(path) as entry: pass checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False) checksum_file.add("entry") checksum_file.write() self.rewind_mtime(checksum_file.path) with mkfile(path) as entry: print("mtime", end="", file=entry) checksum_file.add("entry") self.assertEqual( hashlib.md5(b"mtime").hexdigest(), checksum_file.entries["entry"])
def test_add_updated_ctime(self): # Adding an existing file with a ctime newer than that of the # checksums file causes its checksum to be updated. path = os.path.join(self.temp_dir, "entry") with mkfile(path) as entry: print("ctime", end="", file=entry) checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False) checksum_file.entries["entry"] = "" checksum_file.changed = True checksum_file.write() # We can simulate a ctime change by rewinding the mtime of both # entry and the checksums file. self.rewind_mtime(checksum_file.path) self.rewind_mtime(path) checksum_file.add("entry") self.assertEqual( hashlib.md5(b"ctime").hexdigest(), checksum_file.entries["entry"])
def test_context_manager(self): for name in "1", "2": entry_path = os.path.join(self.temp_dir, name) with mkfile(entry_path) as entry: print(name, end="", file=entry) md5sums_path = os.path.join(self.temp_dir, "MD5SUMS") with mkfile(md5sums_path) as md5sums: subprocess.call(["md5sum", "-b", "1", "2"], stdout=md5sums, cwd=self.temp_dir) with ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False) as checksum_file: self.assertCountEqual(["1", "2"], checksum_file.entries) checksum_file.remove("1") with open(md5sums_path) as md5sums: self.assertEqual("%s *2\n" % hashlib.md5(b"2").hexdigest(), md5sums.read())
def test_write(self): checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False) for name in "1", "2": entry_path = os.path.join(self.temp_dir, name) with mkfile(entry_path) as entry: print(name, end="", file=entry) checksum_file.add(name) checksum_file.write() with open(checksum_file.path) as md5sums: expected = dedent("""\ %s *1 %s *2 """) % (hashlib.md5(b"1").hexdigest(), hashlib.md5(b"2").hexdigest()) self.assertEqual(expected, md5sums.read()) self.assertEqual( 0, subprocess.call(["md5sum", "-c", "--status", "MD5SUMS"], cwd=self.temp_dir))
def test_read_missing(self): checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.read() self.assertEqual({}, checksum_file.entries)
def test_remove(self): checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.entries["entry"] = "checksum" checksum_file.remove("entry") self.assertEqual({}, checksum_file.entries)
def test_read_missing(self): checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.read() self.assertEqual({}, checksum_file.entries)
def test_remove(self): checksum_file = ChecksumFile( self.config, self.temp_dir, "MD5SUMS", hashlib.md5) checksum_file.entries["entry"] = "checksum" checksum_file.remove("entry") self.assertEqual({}, checksum_file.entries)