def sizehashes(f):
size = os.fstat(f.fileno())[6]
f.seek(0)
sha1sum = apt_pkg.sha1sum(f)
f.seek(0)
sha256sum = apt_pkg.sha256sum(f)
return (size, sha1sum, sha256sum)
def index_file(self, suite, fname, check=True):
'''
Retrieve a package list (index) file from the repository.
The file will be downloaded if necessary:
Returns: A file path to the index file.
'''
if type(suite) is ArchiveSuite:
suite_name = suite.name
else:
suite_name = suite
ird = self._read_repo_information(suite_name)
index_fname = self._fetch_repo_file_internal(os.path.join('dists', suite_name, fname))
if not index_fname:
return None
# validate the file
with open(index_fname, 'rb') as f:
index_sha256sum = sha256sum(f)
valid = False
for af in ird.files:
if af.fname == fname:
if index_sha256sum != af.sha256sum:
raise Exception('Checksum validation of "{}" failed ({} != {})'.format(fname, index_sha256sum, af.sha256sum))
valid = True
if not valid and check:
raise Exception('Unable to validate "{}": File not mentioned in InRelease.'.format(fname))
return index_fname
def _verifyRelease(self, signatures):
" verify the signatues and hashes "
for sig in signatures:
basepath = os.path.split(sig)[0]
# do gpg checking
releasef = os.path.splitext(sig)[0]
verify_env = os.environ.copy()
cmd = ["apt-key", "--quiet", "verify", sig, releasef]
with tempfile.NamedTemporaryFile() as fp:
fp.write(apt_pkg.config.dump())
verify_env["APT_CONFIG"] = fp.name
ret = subprocess.call(cmd, env=verify_env)
if not (ret == 0):
return False
# now do the hash sum checks
with open(releasef) as f:
t = apt_pkg.TagFile(f)
t.step()
sha256_section = t.section["SHA256"]
for entry in sha256_section.split("\n"):
(hash, size, name) = entry.split()
f = os.path.join(basepath, name)
if not os.path.exists(f):
logging.info("ignoring missing '%s'" % f)
continue
with open(f) as fp:
sum = apt_pkg.sha256sum(fp)
if not (sum == hash):
logging.error(
"hash sum mismatch expected %s but got %s" % (
hash, sum))
return False
return True
def check(self, directory):
"""Validate hashes
Check if size and hashes match the expected value.
@type directory: str
@param directory: directory the file is located in
@raise InvalidHashException: hash mismatch
"""
path = os.path.join(directory, self.filename)
fh = open(path, 'r')
size = os.stat(path).st_size
if size != self.size:
raise InvalidHashException(self.filename, 'size', self.size, size)
md5sum = apt_pkg.md5sum(fh)
if md5sum != self.md5sum:
raise InvalidHashException(self.filename, 'md5sum', self.md5sum, md5sum)
fh.seek(0)
sha1sum = apt_pkg.sha1sum(fh)
if sha1sum != self.sha1sum:
raise InvalidHashException(self.filename, 'sha1sum', self.sha1sum, sha1sum)
fh.seek(0)
sha256sum = apt_pkg.sha256sum(fh)
if sha256sum != self.sha256sum:
raise InvalidHashException(self.filename, 'sha256sum', self.sha256sum, sha256sum)
def _file_is_same(path, size, sha256):
# type: (str, int, str) -> bool
"""Return ``True`` if the file is the same."""
if os.path.exists(path) and os.path.getsize(path) == size:
with open(path) as fobj:
return apt_pkg.sha256sum(fobj) == sha256
return False
def _verifyRelease(self, signatures):
" verify the signatues and hashes "
gpgv = apt_pkg.config.find("Dir::Bin::gpg", "/usr/bin/gpgv")
keyring = apt_pkg.config.find("Apt::GPGV::TrustedKeyring",
"/etc/apt/trusted.gpg")
for sig in signatures:
basepath = os.path.split(sig)[0]
# do gpg checking
releasef = os.path.splitext(sig)[0]
cmd = [
gpgv, "--keyring", keyring, "--ignore-time-conflict", sig,
releasef
]
ret = subprocess.call(cmd)
if not (ret == 0):
return False
# now do the hash sum checks
t = apt_pkg.TagFile(open(releasef))
t.step()
for entry in t.section["SHA256"].split("\n"):
(hash, size, name) = entry.split()
f = os.path.join(basepath, name)
if not os.path.exists(f):
logging.info("ignoring missing '%s'" % f)
continue
sum = apt_pkg.sha256sum(open(f))
if not (sum == hash):
logging.error("hash sum mismatch expected %s but got %s" %
(hash, sum))
return False
return True
def sizehashes(f):
size = os.fstat(f.fileno())[6]
f.seek(0)
sha1sum = apt_pkg.sha1sum(f)
f.seek(0)
sha256sum = apt_pkg.sha256sum(f)
return (size, sha1sum, sha256sum)
def _verifyRelease(self, signatures):
" verify the signatues and hashes "
gpgv = apt_pkg.config.find("Dir::Bin::gpg", "/usr/bin/gpgv")
keyring = apt_pkg.config.find("Apt::GPGV::TrustedKeyring",
"/etc/apt/trusted.gpg")
for sig in signatures:
basepath = os.path.split(sig)[0]
# do gpg checking
releasef = os.path.splitext(sig)[0]
cmd = [gpgv, "--keyring", keyring,
"--ignore-time-conflict",
sig, releasef]
ret = subprocess.call(cmd)
if not (ret == 0):
return False
# now do the hash sum checks
with open(releasef) as f:
t = apt_pkg.TagFile(f)
t.step()
sha256_section = t.section["SHA256"]
for entry in sha256_section.split("\n"):
(hash, size, name) = entry.split()
f = os.path.join(basepath, name)
if not os.path.exists(f):
logging.info("ignoring missing '%s'" % f)
continue
with open(f) as fp:
sum = apt_pkg.sha256sum(open(fp))
if not (sum == hash):
logging.error(
"hash sum mismatch expected %s but got %s" % (
hash, sum))
return False
return True
def testSHA256(self):
# simple
s = "foo"
s_hash = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
res = apt_pkg.sha256sum(s)
self.assert_(res == s_hash)
# file
res = apt_pkg.sha256sum(open("hashsum_test.data"))
self.assert_(res == s_hash)
# with zero (\0) in the string
s = "foo\0bar"
s_hash = "d6b681bfce7155d44721afb79c296ef4f0fa80a9dd6b43c5cf74dd0f64c85512"
res = apt_pkg.sha256sum(s)
self.assert_(res == s_hash)
# file
res = apt_pkg.sha256sum(open("hashsum_test_with_zero.data"))
self.assert_(res == s_hash)
def testSHA256(self):
# simple
s = "foo"
s_hash = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
res = apt_pkg.sha256sum(s)
self.assert_(res == s_hash)
# file
res = apt_pkg.sha256sum(open("hashsum_test.data"))
self.assert_(res == s_hash)
# with zero (\0) in the string
s = "foo\0bar"
s_hash = "d6b681bfce7155d44721afb79c296ef4f0fa80a9dd6b43c5cf74dd0f64c85512"
res = apt_pkg.sha256sum(s)
self.assert_(res == s_hash)
# file
res = apt_pkg.sha256sum(open("hashsum_test_with_zero.data"))
self.assert_(res == s_hash)
def testSHA256(self):
# simple
s = b"foo"
s_hash = \
"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
res = apt_pkg.sha256sum(s)
self.assertEqual(res, s_hash)
# file
with open(self.DATA_PATH) as fobj:
self.assertEqual(apt_pkg.sha256sum(fobj), s_hash)
# with zero (\0) in the string
s = b"foo\0bar"
s_hash = \
"d6b681bfce7155d44721afb79c296ef4f0fa80a9dd6b43c5cf74dd0f64c85512"
res = apt_pkg.sha256sum(s)
self.assertEqual(res, s_hash)
# file
with open(self.DATA_WITH_ZERO_PATH) as fobj:
self.assertEqual(apt_pkg.sha256sum(fobj), s_hash)
def testSHA256(self):
# simple
s = b"foo"
s_hash = \
"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
res = apt_pkg.sha256sum(s)
self.assertEqual(res, s_hash)
# file
with open(self.DATA_PATH) as fobj:
self.assertEqual(apt_pkg.sha256sum(fobj), s_hash)
# with zero (\0) in the string
s = b"foo\0bar"
s_hash = \
"d6b681bfce7155d44721afb79c296ef4f0fa80a9dd6b43c5cf74dd0f64c85512"
res = apt_pkg.sha256sum(s)
self.assertEqual(res, s_hash)
# file
with open(self.DATA_WITH_ZERO_PATH) as fobj:
self.assertEqual(apt_pkg.sha256sum(fobj), s_hash)
def get_file(self, afile, check=True) -> str:
'''
Get a file from the repository.
Returns: An absolute path to the repository file.
'''
assert type(afile) is ArchiveFile
fname = self._fetch_repo_file_internal(afile.fname, check=True)
if check:
with open(fname, 'rb') as f:
sha256h = sha256sum(f)
if sha256h != afile.sha256sum:
raise Exception('Checksum validation of "{}" failed ({} != {}).'.format(fname, sha256h, afile.sha256sum))
return fname
def fix_checksums():
"""
Update missing checksums
"""
print "Getting file information from database..."
session = DBConn().session()
q = session.query(PoolFile)
print "Checking file checksums & sizes..."
for f in q:
filename = f.fullpath
try:
fi = utils.open_file(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
size = os.stat(filename)[stat.ST_SIZE]
if size != f.filesize:
utils.warn(
"**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, size, f.filesize))
md5sum = apt_pkg.md5sum(fi)
if md5sum != f.md5sum:
utils.warn(
"**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, md5sum, f.md5sum))
continue
fi.seek(0)
sha1sum = apt_pkg.sha1sum(fi)
if f.sha1sum is None:
f.sha1sum = sha1sum
print "Added missing sha1 checksum for {0}".format(f.filename)
fi.seek(0)
sha256sum = apt_pkg.sha256sum(fi)
if f.sha256sum is None:
f.sha256sum = sha256sum
print "Added missing sha256 checksum for {0}".format(f.filename)
session.commit()
print "Done."
def check_checksums():
"""
Validate all files
"""
print("Getting file information from database...")
q = DBConn().session().query(PoolFile)
print("Checking file checksums & sizes...")
for f in q:
filename = f.fullpath
try:
fi = open(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
size = os.stat(filename)[stat.ST_SIZE]
if size != f.filesize:
utils.warn(
"**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, size, f.filesize))
md5sum = apt_pkg.md5sum(fi)
if md5sum != f.md5sum:
utils.warn(
"**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, md5sum, f.md5sum))
fi.seek(0)
sha1sum = apt_pkg.sha1sum(fi)
if sha1sum != f.sha1sum:
utils.warn(
"**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, sha1sum, f.sha1sum))
fi.seek(0)
sha256sum = apt_pkg.sha256sum(fi)
if sha256sum != f.sha256sum:
utils.warn(
"**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])."
% (filename, sha256sum, f.sha256sum))
fi.close()
print("Done.")
def fetch_binary(package: apt.package.Version,
destdir: pathlib.Path) -> pathlib.Path:
"""Download the DEB file corresponding to the given package."""
filename = pathlib.Path(package.filename).name
destfile = destdir / filename
print('Downloading package {}'.format(filename))
with urllib.request.urlopen(package.uri) as response:
destfile.write_bytes(response.read())
# Check package size.
if destfile.stat().st_size != package.size:
raise Exception(
'Failed to download package {}: size mismatch'.format(filename))
# Check package checksum.
with open(destfile) as fp:
if apt_pkg.sha256sum(fp) != package.sha256:
raise Exception('Failed to download package {}: checksum failed'\
.format(filename))
return destfile
def fix_checksums():
"""
Update missing checksums
"""
print "Getting file information from database..."
session = DBConn().session();
q = session.query(PoolFile)
print "Checking file checksums & sizes..."
for f in q:
filename = f.fullpath
try:
fi = utils.open_file(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
size = os.stat(filename)[stat.ST_SIZE]
if size != f.filesize:
utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, f.filesize))
md5sum = apt_pkg.md5sum(fi)
if md5sum != f.md5sum:
utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, f.md5sum))
continue;
fi.seek(0)
sha1sum = apt_pkg.sha1sum(fi)
if f.sha1sum is None:
f.sha1sum = sha1sum
print "Added missing sha1 checksum for {0}".format(f.filename)
fi.seek(0)
sha256sum = apt_pkg.sha256sum(fi)
if f.sha256sum is None:
f.sha256sum = sha256sum
print "Added missing sha256 checksum for {0}".format(f.filename)
session.commit()
print "Done."
def check_checksums():
"""
Validate all files
"""
print "Getting file information from database..."
q = DBConn().session().query(PoolFile)
print "Checking file checksums & sizes..."
for f in q:
filename = f.fullpath
try:
fi = utils.open_file(filename)
except:
utils.warn("can't open '%s'." % (filename))
continue
size = os.stat(filename)[stat.ST_SIZE]
if size != f.filesize:
utils.warn("**WARNING** size mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, size, f.filesize))
md5sum = apt_pkg.md5sum(fi)
if md5sum != f.md5sum:
utils.warn("**WARNING** md5sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, md5sum, f.md5sum))
fi.seek(0)
sha1sum = apt_pkg.sha1sum(fi)
if sha1sum != f.sha1sum:
utils.warn("**WARNING** sha1sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha1sum, f.sha1sum))
fi.seek(0)
sha256sum = apt_pkg.sha256sum(fi)
if sha256sum != f.sha256sum:
utils.warn("**WARNING** sha256sum mismatch for '%s' ('%s' [current] vs. '%s' [db])." % (filename, sha256sum, f.sha256sum))
print "Done."
def test_sha256sum(self):
"""hashes: Test apt_pkg.sha256sum()"""
self.assertEqual(apt_pkg.sha256sum(self.value), self.sha256)
self.assertEqual(apt_pkg.sha256sum(self.file), self.sha256)
def __init__(self, session=None):
cnf = Config()
try:
newq = get_policy_queue('new', session)
for changes_fn in glob.glob(newq.path + "/*.changes"):
changes_bn = os.path.basename(changes_fn)
chg = get_dbchange(changes_bn, session)
u = Upload()
success = u.load_changes(changes_fn)
u.pkg.changes_file = changes_bn
u.check_hashes()
if not chg:
chg = u.pkg.add_known_changes(newq.path, newq.policy_queue_id, session)
session.add(chg)
if not success:
log.critical("failed to load %s" % changes_fn)
sys.exit(1)
else:
log.critical("ACCLAIM: %s" % changes_fn)
files=[]
for chg_fn in u.pkg.files.keys():
try:
f = open(os.path.join(newq.path, chg_fn))
cpf = ChangePendingFile()
cpf.filename = chg_fn
cpf.size = u.pkg.files[chg_fn]['size']
cpf.md5sum = u.pkg.files[chg_fn]['md5sum']
if u.pkg.files[chg_fn].has_key('sha1sum'):
cpf.sha1sum = u.pkg.files[chg_fn]['sha1sum']
else:
log.warning("Having to generate sha1sum for %s" % chg_fn)
f.seek(0)
cpf.sha1sum = apt_pkg.sha1sum(f)
if u.pkg.files[chg_fn].has_key('sha256sum'):
cpf.sha256sum = u.pkg.files[chg_fn]['sha256sum']
else:
log.warning("Having to generate sha256sum for %s" % chg_fn)
f.seek(0)
cpf.sha256sum = apt_pkg.sha256sum(f)
session.add(cpf)
files.append(cpf)
f.close()
except IOError:
# Can't find the file, try to look it up in the pool
poolname = poolify(u.pkg.changes["source"], u.pkg.files[chg_fn]["component"])
l = get_location(cnf["Dir::Pool"], u.pkg.files[chg_fn]["component"], session=session)
if not l:
log.critical("ERROR: Can't find location for %s (component %s)" % (chg_fn, u.pkg.files[chg_fn]["component"]))
found, poolfile = check_poolfile(os.path.join(poolname, chg_fn),
u.pkg.files[chg_fn]['size'],
u.pkg.files[chg_fn]["md5sum"],
l.location_id,
session=session)
if found is None:
log.critical("ERROR: Found multiple files for %s in pool" % chg_fn)
sys.exit(1)
elif found is False and poolfile is not None:
log.critical("ERROR: md5sum / size mismatch for %s in pool" % chg_fn)
sys.exit(1)
else:
if poolfile is None:
log.critical("ERROR: Could not find %s in pool" % chg_fn)
sys.exit(1)
else:
chg.poolfiles.append(poolfile)
chg.files = files
session.commit()
except KeyboardInterrupt:
print("Caught C-c; terminating.")
utils.warn("Caught C-c; terminating.")
self.plsDie()
def test_sha256sum(self):
"""hashes: Test apt_pkg.sha256sum()"""
self.assertEqual(apt_pkg.sha256sum(self.value), self.sha256)
self.assertEqual(apt_pkg.sha256sum(self.file), self.sha256)
