def verify_cloudfront(config, root, files):
ret = True
tree = SimpleTree(config)
md5sums = ChecksumFile(
config, os.path.join(tree.directory, ".pool"), "MD5SUMS", None)
md5sums.read()
opener = build_opener(HTTPHeadRedirectHandler())
if not root.endswith("/"):
root += "/"
for f in files:
if f not in md5sums.entries:
# There are lots of miscellaneous boring files with no local
# checksums. Silently ignore these for convenience.
continue
url = urljoin(root, f.replace("+", "%2B"))
try:
response = opener.open(HeadRequest(url))
except HTTPError as e:
print("%s: %s" % (url, e), file=sys.stderr)
continue
try:
etag = response.info()["ETag"].strip('"')
if md5sums.entries[f] == etag:
print("%s matches %s" % (f, url))
else:
print("%s DOES NOT MATCH %s" % (f, url), file=sys.stderr)
print(" Local: %s" % md5sums.entries[f], file=sys.stderr)
print(" Remote: %s" % etag, file=sys.stderr)
ret = False
except KeyError:
print("No remote ETag for %s; skipping." % url, file=sys.stderr)
return ret
def verify_cloudfront(config, root, files):
ret = True
tree = SimpleReleaseTree(config)
md5sums = ChecksumFile(config, os.path.join(tree.directory, ".pool"),
"MD5SUMS", None)
md5sums.read()
opener = build_opener(HTTPHeadRedirectHandler())
if not root.endswith("/"):
root += "/"
for f in files:
if f not in md5sums.entries:
# There are lots of miscellaneous boring files with no local
# checksums. Silently ignore these for convenience.
continue
url = urljoin(root, f.replace("+", "%2B"))
try:
response = opener.open(HeadRequest(url))
except HTTPError as e:
print("%s: %s" % (url, e), file=sys.stderr)
continue
try:
etag = response.info()["ETag"].strip('"')
if md5sums.entries[f] == etag:
print("%s matches %s" % (f, url))
else:
print("%s DOES NOT MATCH %s" % (f, url), file=sys.stderr)
print(" Local: %s" % md5sums.entries[f], file=sys.stderr)
print(" Remote: %s" % etag, file=sys.stderr)
ret = False
except KeyError:
print("No remote ETag for %s; skipping." % url, file=sys.stderr)
return ret
def test_checksum_large_file(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = b"a" * 1048576
with mkfile(entry_path, mode="wb") as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "SHA1SUMS",
hashlib.sha1)
self.assertEqual(
hashlib.sha1(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_small_file(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = b"test\n"
with mkfile(entry_path, mode="wb") as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
self.assertEqual(
hashlib.md5(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_small_file(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with open(entry_path, "w") as entry:
print(data, end="", file=entry)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
self.assertEqual(
hashlib.md5(data).hexdigest(), checksum_file.checksum(entry_path))
def test_checksum_large_file(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = "a" * 1048576
with open(entry_path, "w") as entry:
print(data, end="", file=entry)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "SHA1SUMS", hashlib.sha1)
self.assertEqual(
hashlib.sha1(data).hexdigest(), checksum_file.checksum(entry_path))
def test_merge_takes_other_names(self):
old_dir = os.path.join(self.temp_dir, "old")
touch(os.path.join(self.temp_dir, "entry"))
with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums:
print("checksum *other-entry", file=old_md5sums)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.merge([old_dir], "entry", ["other-entry"])
self.assertEqual({"entry": "checksum"}, checksum_file.entries)
def test_add(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with open(entry_path, "w") as entry:
print(data, end="", file=entry)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.add("entry")
self.assertEqual(
{"entry": hashlib.md5(data).hexdigest()}, checksum_file.entries)
def test_merge_takes_other_names(self):
old_dir = os.path.join(self.temp_dir, "old")
os.mkdir(old_dir)
touch(os.path.join(self.temp_dir, "entry"))
with open(os.path.join(old_dir, "MD5SUMS"), "w") as old_md5sums:
print("checksum *other-entry", file=old_md5sums)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.merge([old_dir], "entry", ["other-entry"])
self.assertEqual({"entry": "checksum"}, checksum_file.entries)
def test_add(self):
entry_path = os.path.join(self.temp_dir, "entry")
data = b"test\n"
with mkfile(entry_path, mode="wb") as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.add("entry")
self.assertEqual({"entry": hashlib.md5(data).hexdigest()},
checksum_file.entries)
def test_read(self):
with open(os.path.join(self.temp_dir, "MD5SUMS"), "w") as md5sums:
print(dedent("""\
checksum one-path
checksum *another-path
"""), file=md5sums)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.read()
self.assertEqual(
{"one-path": "checksum", "another-path": "checksum"},
checksum_file.entries)
def test_add_existing(self):
# Attempting to add an existing file has no effect. (Use .remove()
# first to overwrite an existing checksum.)
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with open(entry_path, "w") as entry:
print(data, end="", file=entry)
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.entries["entry"] = ""
checksum_file.add("entry")
self.assertEqual("", checksum_file.entries["entry"])
def test_merge_ignores_stale_checksums(self):
old_dir = os.path.join(self.temp_dir, "old")
with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums:
print("checksum *entry", file=old_md5sums)
entry_path = os.path.join(self.temp_dir, "entry")
touch(entry_path)
next_minute = time.time() + 60
os.utime(entry_path, (next_minute, next_minute))
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.merge([old_dir], "entry", ["entry"])
self.assertEqual({}, checksum_file.entries)
def test_merge_ignores_stale_checksums(self):
old_dir = os.path.join(self.temp_dir, "old")
os.mkdir(old_dir)
with open(os.path.join(old_dir, "MD5SUMS"), "w") as old_md5sums:
print("checksum *entry", file=old_md5sums)
entry_path = os.path.join(self.temp_dir, "entry")
touch(entry_path)
next_minute = time.time() + 60
os.utime(entry_path, (next_minute, next_minute))
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.merge([old_dir], "entry", ["entry"])
self.assertEqual({}, checksum_file.entries)
def test_add_existing(self):
# Attempting to add an existing file that is not newer than the
# checksums file has no effect. (Use .remove() first to overwrite
# an existing checksum.)
entry_path = os.path.join(self.temp_dir, "entry")
data = "test\n"
with mkfile(entry_path) as entry:
entry.write(data)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.entries["entry"] = ""
checksum_file.add("entry")
self.assertEqual("", checksum_file.entries["entry"])
def test_read(self):
with mkfile(os.path.join(self.temp_dir, "MD5SUMS")) as md5sums:
print(dedent("""\
checksum one-path
checksum *another-path
"""),
file=md5sums)
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.read()
self.assertEqual({
"one-path": "checksum",
"another-path": "checksum"
}, checksum_file.entries)
def test_merge_handles_symlinks(self):
old_dir = os.path.join(self.temp_dir, "old")
touch(os.path.join(old_dir, "entry"))
with mkfile(os.path.join(old_dir, "MD5SUMS")) as old_md5sums:
print("correct-checksum *entry", file=old_md5sums)
new_dir = os.path.join(self.temp_dir, "new")
os.mkdir(new_dir)
os.symlink(os.path.join(os.pardir, "old", "entry"),
os.path.join(new_dir, "entry"))
with mkfile(os.path.join(new_dir, "MD5SUMS")) as new_md5sums:
print("wrong-checksum *entry", file=new_md5sums)
checksum_file = ChecksumFile(self.config, new_dir, "MD5SUMS",
hashlib.md5)
checksum_file.merge([new_dir, old_dir], "entry", ["entry"])
self.assertEqual({"entry": "correct-checksum"}, checksum_file.entries)
def test_write(self):
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5, sign=False)
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with open(entry_path, "w") as entry:
print(name, end="", file=entry)
checksum_file.add(name)
checksum_file.write()
with open(checksum_file.path) as md5sums:
self.assertEqual(dedent("""\
%s *1
%s *2
""") %
(hashlib.md5("1").hexdigest(), hashlib.md5("2").hexdigest()),
md5sums.read())
self.assertEqual(
0,
subprocess.call(
["md5sum", "-c", "--status", "MD5SUMS"], cwd=self.temp_dir))
def test_add_updated_mtime(self):
# Adding an existing file with an mtime newer than that of the
# checksums file causes its checksum to be updated.
path = os.path.join(self.temp_dir, "entry")
with mkfile(path) as entry:
pass
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
checksum_file.add("entry")
checksum_file.write()
self.rewind_mtime(checksum_file.path)
with mkfile(path) as entry:
print("mtime", end="", file=entry)
checksum_file.add("entry")
self.assertEqual(
hashlib.md5(b"mtime").hexdigest(), checksum_file.entries["entry"])
def test_add_updated_ctime(self):
# Adding an existing file with a ctime newer than that of the
# checksums file causes its checksum to be updated.
path = os.path.join(self.temp_dir, "entry")
with mkfile(path) as entry:
print("ctime", end="", file=entry)
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
checksum_file.entries["entry"] = ""
checksum_file.changed = True
checksum_file.write()
# We can simulate a ctime change by rewinding the mtime of both
# entry and the checksums file.
self.rewind_mtime(checksum_file.path)
self.rewind_mtime(path)
checksum_file.add("entry")
self.assertEqual(
hashlib.md5(b"ctime").hexdigest(), checksum_file.entries["entry"])
def test_context_manager(self):
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
md5sums_path = os.path.join(self.temp_dir, "MD5SUMS")
with mkfile(md5sums_path) as md5sums:
subprocess.call(["md5sum", "-b", "1", "2"],
stdout=md5sums,
cwd=self.temp_dir)
with ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False) as checksum_file:
self.assertCountEqual(["1", "2"], checksum_file.entries)
checksum_file.remove("1")
with open(md5sums_path) as md5sums:
self.assertEqual("%s *2\n" % hashlib.md5(b"2").hexdigest(),
md5sums.read())
def test_write(self):
checksum_file = ChecksumFile(self.config,
self.temp_dir,
"MD5SUMS",
hashlib.md5,
sign=False)
for name in "1", "2":
entry_path = os.path.join(self.temp_dir, name)
with mkfile(entry_path) as entry:
print(name, end="", file=entry)
checksum_file.add(name)
checksum_file.write()
with open(checksum_file.path) as md5sums:
expected = dedent("""\
%s *1
%s *2
""") % (hashlib.md5(b"1").hexdigest(),
hashlib.md5(b"2").hexdigest())
self.assertEqual(expected, md5sums.read())
self.assertEqual(
0,
subprocess.call(["md5sum", "-c", "--status", "MD5SUMS"],
cwd=self.temp_dir))
def test_read_missing(self):
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.read()
self.assertEqual({}, checksum_file.entries)
def test_remove(self):
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.entries["entry"] = "checksum"
checksum_file.remove("entry")
self.assertEqual({}, checksum_file.entries)
def test_read_missing(self):
checksum_file = ChecksumFile(self.config, self.temp_dir, "MD5SUMS",
hashlib.md5)
checksum_file.read()
self.assertEqual({}, checksum_file.entries)
def test_remove(self):
checksum_file = ChecksumFile(
self.config, self.temp_dir, "MD5SUMS", hashlib.md5)
checksum_file.entries["entry"] = "checksum"
checksum_file.remove("entry")
self.assertEqual({}, checksum_file.entries)
