def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
filename = "%s/dists/%s/%s/source/Sources" % (Cnf["Dir::Root"], suite,
component)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print("W: %s missing." % (filename))
else:
pool_location = utils.poolify(source)
pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"],
pool_location, name)
if not os.path.exists(pool_filename):
print("E: %s missing (%s)." %
(filename, pool_filename))
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename,
Cnf["Dir::Root"])
print("Symlinking: %s -> %s" % (filename, src))
def _install_file(self, directory, hashed_file, archive, component, source_name):
"""Install a file
Will not give an error when the file is already present.
@rtype: L{daklib.dbconn.PoolFile}
@return: database object for the new file
"""
session = self.session
poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
try:
poolfile = self.get_file(hashed_file, source_name)
except KeyError:
poolfile = PoolFile(filename=poolname, filesize=hashed_file.size)
poolfile.md5sum = hashed_file.md5sum
poolfile.sha1sum = hashed_file.sha1sum
poolfile.sha256sum = hashed_file.sha256sum
session.add(poolfile)
session.flush()
try:
session.query(ArchiveFile).filter_by(archive=archive, component=component, file=poolfile).one()
except NoResultFound:
archive_file = ArchiveFile(archive, component, poolfile)
session.add(archive_file)
session.flush()
path = os.path.join(archive.path, 'pool', component.component_name, poolname)
hashed_file_path = os.path.join(directory, hashed_file.filename)
self.fs.copy(hashed_file_path, path, link=False, mode=archive.mode)
return poolfile
def get_file(self, hashed_file, source_name, check_hashes=True):
"""Look for file C{hashed_file} in database
@type hashed_file: L{daklib.upload.HashedFile}
@param hashed_file: file to look for in the database
@type source_name: str
@param source_name: source package name
@type check_hashes: bool
@param check_hashes: check size and hashes match
@raise KeyError: file was not found in the database
@raise HashMismatchException: hash mismatch
@rtype: L{daklib.dbconn.PoolFile}
@return: database entry for the file
"""
poolname = os.path.join(utils.poolify(source_name), hashed_file.filename)
try:
poolfile = self.session.query(PoolFile).filter_by(filename=poolname).one()
if check_hashes and (poolfile.filesize != hashed_file.size
or poolfile.md5sum != hashed_file.md5sum
or poolfile.sha1sum != hashed_file.sha1sum
or poolfile.sha256sum != hashed_file.sha256sum):
raise HashMismatchException('{0}: Does not match file already existing in the pool.'.format(hashed_file.filename))
return poolfile
except NoResultFound:
raise KeyError('{0} not found in database.'.format(poolname))
def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
filename = "%s/dists/%s/%s/source/Sources" % (Cnf["Dir::Root"], suite, component)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print("W: %s missing." % (filename))
else:
pool_location = utils.poolify(source)
pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"], pool_location, name)
if not os.path.exists(pool_filename):
print("E: %s missing (%s)." % (filename, pool_filename))
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename, Cnf["Dir::Root"])
print("Symlinking: %s -> %s" % (filename, src))
def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
cnf = Config()
filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite,
component)
print "Processing %s..." % (filename)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
(filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print "W: %s missing." % (filename)
else:
pool_location = utils.poolify(source, component)
pool_filename = "%s/%s/%s" % (cnf["Dir::Pool"],
pool_location, name)
if not os.path.exists(pool_filename):
print "E: %s missing (%s)." % (filename, pool_filename)
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename,
cnf["Dir::Root"])
print "Symlinking: %s -> %s" % (filename, src)
#os.symlink(src, filename)
sources.close()
os.unlink(temp_filename)
def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
cnf = Config()
filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite, component)
print "Processing %s..." % (filename)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print "W: %s missing." % (filename)
else:
pool_location = utils.poolify (source, component)
pool_filename = "%s/%s/%s" % (cnf["Dir::Pool"], pool_location, name)
if not os.path.exists(pool_filename):
print "E: %s missing (%s)." % (filename, pool_filename)
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename, cnf["Dir::Root"])
print "Symlinking: %s -> %s" % (filename, src)
#os.symlink(src, filename)
sources.close()
os.unlink(temp_filename)
def __init__(self, session=None):
cnf = Config()
try:
newq = get_policy_queue('new', session)
for changes_fn in glob.glob(newq.path + "/*.changes"):
changes_bn = os.path.basename(changes_fn)
chg = get_dbchange(changes_bn, session)
u = Upload()
success = u.load_changes(changes_fn)
u.pkg.changes_file = changes_bn
u.check_hashes()
if not chg:
chg = u.pkg.add_known_changes(newq.path, newq.policy_queue_id, session)
session.add(chg)
if not success:
log.critical("failed to load %s" % changes_fn)
sys.exit(1)
else:
log.critical("ACCLAIM: %s" % changes_fn)
files=[]
for chg_fn in u.pkg.files.keys():
try:
f = open(os.path.join(newq.path, chg_fn))
cpf = ChangePendingFile()
cpf.filename = chg_fn
cpf.size = u.pkg.files[chg_fn]['size']
cpf.md5sum = u.pkg.files[chg_fn]['md5sum']
if u.pkg.files[chg_fn].has_key('sha1sum'):
cpf.sha1sum = u.pkg.files[chg_fn]['sha1sum']
else:
log.warning("Having to generate sha1sum for %s" % chg_fn)
f.seek(0)
cpf.sha1sum = apt_pkg.sha1sum(f)
if u.pkg.files[chg_fn].has_key('sha256sum'):
cpf.sha256sum = u.pkg.files[chg_fn]['sha256sum']
else:
log.warning("Having to generate sha256sum for %s" % chg_fn)
f.seek(0)
cpf.sha256sum = apt_pkg.sha256sum(f)
session.add(cpf)
files.append(cpf)
f.close()
except IOError:
# Can't find the file, try to look it up in the pool
poolname = poolify(u.pkg.changes["source"], u.pkg.files[chg_fn]["component"])
l = get_location(cnf["Dir::Pool"], u.pkg.files[chg_fn]["component"], session=session)
if not l:
log.critical("ERROR: Can't find location for %s (component %s)" % (chg_fn, u.pkg.files[chg_fn]["component"]))
found, poolfile = check_poolfile(os.path.join(poolname, chg_fn),
u.pkg.files[chg_fn]['size'],
u.pkg.files[chg_fn]["md5sum"],
l.location_id,
session=session)
if found is None:
log.critical("ERROR: Found multiple files for %s in pool" % chg_fn)
sys.exit(1)
elif found is False and poolfile is not None:
log.critical("ERROR: md5sum / size mismatch for %s in pool" % chg_fn)
sys.exit(1)
else:
if poolfile is None:
log.critical("ERROR: Could not find %s in pool" % chg_fn)
sys.exit(1)
else:
chg.poolfiles.append(poolfile)
chg.files = files
session.commit()
except KeyboardInterrupt:
print("Caught C-c; terminating.")
utils.warn("Caught C-c; terminating.")
self.plsDie()
