def do_update(self):
"""
Add missing PrimaryMirror field to archive table
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute("ALTER TABLE archive ADD COLUMN primary_mirror TEXT")
c.execute("SELECT id, name FROM archive")
query = "UPDATE archive SET primary_mirror = %s WHERE id = %s"
for a_id, a_name in c.fetchall():
if cnf.has_key('Archive::%s::PrimaryMirror' % a_name):
primloc = cnf['Archive::%s::PrimaryMirror' % a_name]
print "Setting archive %s PrimaryMirror to %s" % (a_name,
primloc)
c.execute(query, [primloc, a_id])
c.execute("UPDATE config SET value = '63' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 63, rollback issued. Error message : %s'
% (str(msg)))
def do_update(self):
"""
Make lastused in queuefiles/policyqueuefiles default now()
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute(
"ALTER TABLE build_queue_files ALTER COLUMN lastused SET DEFAULT now()"
)
c.execute(
"ALTER TABLE build_queue_files ALTER COLUMN lastused SET NOT NULL")
c.execute(
"ALTER TABLE build_queue_policy_files ALTER COLUMN lastused SET DEFAULT now()"
)
c.execute(
"ALTER TABLE build_queue_policy_files ALTER COLUMN lastused SET NOT NULL"
)
c.execute("UPDATE config SET value = '71' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 71, rollback issued. Error message : %s'
% (str(msg)))
def do_update(self):
print(__doc__)
try:
cnf = Config()
c = self.db.cursor()
c.execute(
"GRANT SELECT ON files_archive_map, policy_queue_byhand_file, policy_queue_upload, policy_queue_upload_binaries_map TO PUBLIC"
)
c.execute(
"GRANT ALL ON files_archive_map, policy_queue_byhand_file, policy_queue_upload, policy_queue_upload_binaries_map TO ftpmaster"
)
c.execute("ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO PUBLIC")
c.execute("ALTER DEFAULT PRIVILEGES GRANT ALL ON TABLES TO ftpmaster")
c.execute(
"ALTER DEFAULT PRIVILEGES GRANT SELECT ON SEQUENCES TO PUBLIC")
c.execute(
"ALTER DEFAULT PRIVILEGES GRANT ALL ON SEQUENCES TO ftpmaster")
c.execute(
"ALTER DEFAULT PRIVILEGES GRANT ALL ON FUNCTIONS TO ftpmaster")
c.execute("UPDATE config SET value = '78' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 78, rollback issued. Error message: {0}'
.format(msg))
def action_dm_remove(self, fingerprint, section, session):
self._action_dm_admin_common(fingerprint, section, session)
cnf = Config()
acl_name = cnf.get('Command::DM::ACL', 'dm')
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash = section['Fingerprint'].translate(None, ' ')
fpr = session.query(Fingerprint).filter_by(
fingerprint=fpr_hash).first()
if fpr is None:
self.result.append(
'Unknown fingerprint: {0}\nNo action taken.'.format(fpr_hash))
return
self.log.log(['dm-remove', fpr.fingerprint])
count = 0
for entry in session.query(ACLPerSource).filter_by(acl=acl,
fingerprint=fpr):
self.log.log([
'dm-remove', fpr.fingerprint, 'source={0}'.format(entry.source)
])
count += 1
session.delete(entry)
self.result.append('Removed: {0}.\n{1} acl entries removed.'.format(
fpr.fingerprint, count))
session.commit()
def announce_new(upload):
cnf = Config()
subst = _subst_for_upload(upload)
whitelists = _whitelists(upload)
message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.new'))
send_mail(message, whitelists=whitelists)
def main():
cnf = Config()
Arguments = [('h', "help", "External-Overrides::Options::Help"),
('f', 'force', 'External-Overrides::Options::Force')]
args = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("External-Overrides::Options")
except KeyError:
Options = {}
if "Help" in Options:
usage()
force = False
if "Force" in Options and Options["Force"]:
force = True
logger = daklog.Logger('external-overrides')
command = args[0]
if command in ('import', 'i'):
external_overrides_import(args[1], args[2], args[3], sys.stdin, force)
elif command in ('copy', 'c'):
external_overrides_copy(args[1], args[2], force)
else:
print "E: Unknown commands."
def do_bxa_notification(new, upload, session):
cnf = Config()
new = set([o['package'] for o in new if o['type'] == 'deb'])
if len(new) == 0:
return
key = session.query(MetadataKey).filter_by(key='Description').one()
summary = ""
for binary in upload.binaries:
if binary.package not in new:
continue
description = session.query(BinaryMetadata).filter_by(
binary=binary, key=key).one().value
summary += "\n"
summary += "Package: {0}\n".format(binary.package)
summary += "Description: {0}\n".format(description)
subst = {
'__DISTRO__': cnf['Dinstall::MyDistribution'],
'__BCC__': 'X-DAK: dak process-new',
'__BINARY_DESCRIPTIONS__': summary,
}
bxa_mail = utils.TemplateSubst(
subst,
os.path.join(cnf["Dir::Templates"], "process-new.bxa_notification"))
utils.send_mail(bxa_mail)
def lock_package(package):
"""
Lock C{package} so that noone else jumps in processing it.
@type package: string
@param package: source package name to lock
"""
cnf = Config()
path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']),
package)
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY)
except OSError as e:
if e.errno == errno.EEXIST or e.errno == errno.EACCES:
user = pwd.getpwuid(
os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '')
raise AlreadyLockedError(user)
try:
yield fd
finally:
os.unlink(path)
def main():
cnf = Config()
Arguments = [
('h', "help", "Copy-Installer::Options::Help"),
('s', "source", "Copy-Installer::Options::Source", "HasArg"),
('d', "destination", "Copy-Installer::Options::Destination", "HasArg"),
('n', "no-action", "Copy-Installer::Options::No-Action"),
]
for option in ["help", "source", "destination", "no-action"]:
if not cnf.has_key("Copy-Installer::Options::%s" % (option)):
cnf["Copy-Installer::Options::%s" % (option)] = ""
extra_arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Copy-Installer::Options")
if Options["Help"]:
usage()
if len(extra_arguments) != 1:
usage(1)
initializer = {"version": extra_arguments[0]}
if Options["Source"] != "":
initializer["source"] = Options["Source"]
if Options["Destination"] != "":
initializer["dest"] = Options["Destination"]
copier = InstallerCopier(**initializer)
print copier.get_message()
if Options["No-Action"]:
print 'Do nothing because --no-action has been set.'
else:
copier.do_copy()
print 'Installer has been copied successfully.'
def sign_release_dir(suite, dirname):
cnf = Config()
if 'Dinstall::SigningKeyring' in cnf or 'Dinstall::SigningHomedir' in cnf:
args = {
'keyids': suite.signingkeys or [],
'pubring': cnf.get('Dinstall::SigningPubKeyring') or None,
'secring': cnf.get('Dinstall::SigningKeyring') or None,
'homedir': cnf.get('Dinstall::SigningHomedir') or None,
'passphrase_file': cnf.get('Dinstall::SigningPassphraseFile')
or None,
}
relname = os.path.join(dirname, 'Release')
dest = os.path.join(dirname, 'Release.gpg')
if os.path.exists(dest):
os.unlink(dest)
inlinedest = os.path.join(dirname, 'InRelease')
if os.path.exists(inlinedest):
os.unlink(inlinedest)
with open(relname, 'r') as stdin:
with open(dest, 'w') as stdout:
daklib.gpg.sign(stdin, stdout, inline=False, **args)
stdin.seek(0)
with open(inlinedest, 'w') as stdout:
daklib.gpg.sign(stdin, stdout, inline=True, **args)
def do_update(self):
print(__doc__)
try:
cnf = Config()
c = self.db.cursor()
c.execute(
"UPDATE policy_queue set queue_name='stable-new' where queue_name='proposedupdates'"
)
c.execute(
"UPDATE policy_queue set queue_name='oldstable-new' where queue_name='oldproposedupdates'"
)
c.execute(
"UPDATE suite set suite_name='stable-new' where suite_name='proposedupdates'"
)
c.execute(
"UPDATE suite set suite_name='oldstable-new' where suite_name='oldproposedupdates'"
)
c.execute("UPDATE config SET value = '79' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 79, rollback issued. Error message: {0}'
.format(msg))
def sign_release_dir(suite, dirname):
cnf = Config()
if cnf.has_key("Dinstall::SigningKeyring"):
keyring = "--secret-keyring \"%s\"" % cnf["Dinstall::SigningKeyring"]
if cnf.has_key("Dinstall::SigningPubKeyring"):
keyring += " --keyring \"%s\"" % cnf["Dinstall::SigningPubKeyring"]
arguments = "--no-options --batch --no-tty --armour --personal-digest-preferences=SHA256"
relname = os.path.join(dirname, 'Release')
dest = os.path.join(dirname, 'Release.gpg')
if os.path.exists(dest):
os.unlink(dest)
inlinedest = os.path.join(dirname, 'InRelease')
if os.path.exists(inlinedest):
os.unlink(inlinedest)
defkeyid = ""
for keyid in suite.signingkeys or []:
defkeyid += "--local-user %s " % keyid
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments, relname, dest))
os.system("gpg %s %s %s --clearsign <%s >>%s" %
(keyring, defkeyid, arguments, relname, inlinedest))
def do_pkg(upload_id):
cnf = Config()
session = DBConn().session()
upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one()
queue = upload.policy_queue
changes = upload.changes
origchanges = os.path.join(queue.path, changes.changesname)
print origchanges
htmlname = "{0}_{1}.html".format(changes.source, changes.version)
htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname)
# Have we already processed this?
if os.path.exists(htmlfile) and \
os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()):
with open(htmlfile, "r") as fd:
if fd.read() != timeout_str:
sources.append(htmlname)
return (PROC_STATUS_SUCCESS,
'%s already up-to-date' % htmlfile)
# Go, process it... Now!
htmlfiles_to_process.append(htmlfile)
sources.append(htmlname)
group = cnf.get('Dinstall::UnprivGroup') or None
with open(htmlfile, 'w') as outfile:
with policy.UploadCopy(upload, group=group) as upload_copy:
handler = policy.PolicyQueueUploadHandler(upload, session)
missing = [(o['type'], o['package'])
for o in handler.missing_overrides()]
distribution = changes.distribution
print >> outfile, html_header(changes.source, missing)
print >> outfile, examine_package.display_changes(
distribution, origchanges)
if upload.source is not None and ('dsc',
upload.source.source) in missing:
fn = os.path.join(upload_copy.directory,
upload.source.poolfile.basename)
print >> outfile, examine_package.check_dsc(
distribution, fn, session)
for binary in upload.binaries:
if (binary.binarytype, binary.package) not in missing:
continue
fn = os.path.join(upload_copy.directory,
binary.poolfile.basename)
print >> outfile, examine_package.check_deb(
distribution, fn, session)
print >> outfile, html_footer()
session.close()
htmlfiles_to_process.remove(htmlfile)
return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile))
def check_files():
"""
Prepare the dictionary of existing filenames, then walk through the archive
pool/ directory to compare it.
"""
cnf = Config()
session = DBConn().session()
query = """
SELECT archive.name, suite.suite_name, f.filename
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin
JOIN suite ON ba.suite = suite.id
JOIN archive ON suite.archive_id = archive.id
JOIN files f ON b.file = f.id
WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af
WHERE af.archive_id = suite.archive_id
AND af.file_id = b.file)
ORDER BY archive.name, suite.suite_name, f.filename
"""
for row in session.execute(query):
print "MISSING-ARCHIVE-FILE {0} {1} {2}".vformat(row)
query = """
SELECT archive.name, suite.suite_name, f.filename
FROM source s
JOIN src_associations sa ON s.id = sa.source
JOIN suite ON sa.suite = suite.id
JOIN archive ON suite.archive_id = archive.id
JOIN dsc_files df ON s.id = df.source
JOIN files f ON df.file = f.id
WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af
WHERE af.archive_id = suite.archive_id
AND af.file_id = df.file)
ORDER BY archive.name, suite.suite_name, f.filename
"""
for row in session.execute(query):
print "MISSING-ARCHIVE-FILE {0} {1} {2}".vformat(row)
archive_files = session.query(ArchiveFile) \
.join(ArchiveFile.archive).join(ArchiveFile.file) \
.order_by(Archive.archive_name, PoolFile.filename)
expected_files = set()
for af in archive_files:
path = af.path
expected_files.add(af.path)
if not os.path.exists(path):
print "MISSING-FILE {0} {1} {2}".format(af.archive.archive_name, af.file.filename, path)
archives = session.query(Archive).order_by(Archive.archive_name)
for a in archives:
top = os.path.join(a.path, 'pool')
for dirpath, dirnames, filenames in os.walk(top):
for fn in filenames:
path = os.path.join(dirpath, fn)
if path in expected_files:
continue
print "UNEXPECTED-FILE {0} {1}".format(a.archive_name, path)
def get_header(self):
'''
Returns the header for the Contents files as a string.
'''
filename = os.path.join(Config()['Dir::Templates'], 'contents')
with open(filename) as header_file:
return header_file.read()
def main(argv=None):
if argv is None:
argv = sys.argv
arguments = [('h', 'help', 'Export::Options::Help'),
('a', 'all', 'Export::Options::All'),
('c', 'copy', 'Export::Options::Copy'),
('d', 'directory', 'Export::Options::Directory', 'HasArg'),
('q', 'queue', 'Export::Options::Queue', 'HasArg')]
cnf = Config()
source_names = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
options = cnf.subtree('Export::Options')
if 'Help' in options or 'Queue' not in options:
usage()
sys.exit(0)
session = DBConn().session()
queue = session.query(PolicyQueue).filter_by(queue_name=options['Queue']).first()
if queue is None:
print "Unknown queue '{0}'".format(options['Queue'])
sys.exit(1)
uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=queue)
if 'All' not in options:
uploads = uploads.filter(DBChange.source.in_(source_names))
directory = options.get('Directory', '.')
symlink = 'Copy' not in options
for u in uploads:
UploadCopy(u).export(directory, symlink=symlink, ignore_existing=True)
def init(self):
cnf = Config()
arguments = [('h', "help", "Update-DB::Options::Help")]
for i in ["help"]:
if not cnf.has_key("Update-DB::Options::%s" % (i)):
cnf["Update-DB::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, arguments, sys.argv)
options = cnf.subtree("Update-DB::Options")
if options["Help"]:
self.usage()
elif arguments:
utils.warn("dak update-db takes no arguments.")
self.usage(exit_code=1)
try:
if os.path.isdir(cnf["Dir::Lock"]):
lock_fd = os.open(
os.path.join(cnf["Dir::Lock"], 'dinstall.lock'),
os.O_RDWR | os.O_CREAT)
fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
else:
utils.warn("Lock directory doesn't exist yet - not locking")
except IOError as e:
if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[
e.errno] == 'EAGAIN':
utils.fubar(
"Couldn't obtain lock; assuming another 'dak process-unchecked' is already running."
)
self.update_db()
def do_update(self):
print __doc__
try:
cnf = Config()
c = self.db.cursor()
for stmt in statements:
c.execute(stmt)
for component in ('main', 'contrib', 'non-free'):
c.execute(
"UPDATE component SET ordering = nextval('component_ordering_seq') WHERE name = '{0}'"
.format(component))
c.execute(
"UPDATE component SET ordering = nextval('component_ordering_seq') WHERE ordering IS NULL"
)
c.execute(
"""ALTER TABLE component ALTER COLUMN ordering SET NOT NULL""")
c.execute(
"""ALTER TABLE component ALTER COLUMN ordering SET DEFAULT nextval('component_ordering_seq')"""
)
c.execute("UPDATE config SET value = '99' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError(
'Unable to apply sick update 99, rollback issued. Error message: {0}'
.format(msg))
def get_propup_suites(suite, session):
propup_suites = []
for rule in Config().value_list("SuiteMappings"):
fields = rule.split()
if fields[0] == "propup-version" and fields[1] == suite.suite_name:
propup_suites.append(session.query(Suite).filter_by(suite_name=fields[2]).one())
return propup_suites
def main():
global Options, Logger, Sections, Priorities
cnf = Config()
session = DBConn().session()
Arguments = [('a', "automatic", "Process-New::Options::Automatic"),
('b', "no-binaries", "Process-New::Options::No-Binaries"),
('c', "comments", "Process-New::Options::Comments"),
('h', "help", "Process-New::Options::Help"),
('m', "manual-reject", "Process-New::Options::Manual-Reject",
"HasArg"), ('t', "trainee", "Process-New::Options::Trainee"),
('q', 'queue', 'Process-New::Options::Queue', 'HasArg'),
('n', "no-action", "Process-New::Options::No-Action")]
changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
for i in [
"automatic", "no-binaries", "comments", "help", "manual-reject",
"no-action", "version", "trainee"
]:
if not cnf.has_key("Process-New::Options::%s" % (i)):
cnf["Process-New::Options::%s" % (i)] = ""
queue_name = cnf.get('Process-New::Options::Queue', 'new')
new_queue = session.query(PolicyQueue).filter_by(
queue_name=queue_name).one()
if len(changes_files) == 0:
uploads = new_queue.uploads
else:
uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \
.join(DBChange).filter(DBChange.changesname.in_(changes_files)).all()
Options = cnf.subtree("Process-New::Options")
if Options["Help"]:
usage()
if not Options["No-Action"]:
try:
Logger = daklog.Logger("process-new")
except CantOpenError as e:
Options["Trainee"] = "True"
Sections = Section_Completer(session)
Priorities = Priority_Completer(session)
readline.parse_and_bind("tab: complete")
if len(uploads) > 1:
sys.stderr.write("Sorting changes...\n")
uploads = sort_uploads(new_queue, uploads, session,
Options["No-Binaries"])
if Options["Comments"]:
show_new_comments(uploads, session)
else:
for upload in uploads:
do_pkg(upload, session)
end()
def init(session):
global cnf, Options
cnf = Config()
Arguments = [('h', "help", "Show-New::Options::Help"),
("p", "html-path", "Show-New::HTMLPath", "HasArg"),
('q', 'queue', 'Show-New::Options::Queue', 'HasArg')]
for i in ["help"]:
if not cnf.has_key("Show-New::Options::%s" % (i)):
cnf["Show-New::Options::%s" % (i)] = ""
changesnames = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Show-New::Options")
if Options["help"]:
usage()
queue_names = Options.find('Queue', 'new').split(',')
uploads = session.query(PolicyQueueUpload) \
.join(PolicyQueueUpload.policy_queue).filter(PolicyQueue.queue_name.in_(queue_names)) \
.join(PolicyQueueUpload.changes).order_by(DBChange.source)
if len(changesnames) > 0:
uploads = uploads.filter(DBChange.changesname.in_(changesnames))
return uploads
def expire_dep11_data_cache(session, suitename):
'''
Clears the stale cache items per suite.
'''
# dic that has pkg name as key and bin_ids as values in a list,
# these are not to be deleted
keep = list()
dir_list = []
print("Clearing stale cached data...")
# select all the binids with a package-name
# (select all package-name from binaries)
sql = """select bd.binary_id,b.package
from bin_dep11 bd, binaries b
where b.id = bd.binary_id"""
q = session.execute(sql)
result = q.fetchall()
for r in result:
keep.append("%s-%s" % (r[1], r[0]))
glob_tmpl = "%s/*/*" % (os.path.join(Config()["Dir::MetaInfo"], suitename))
for fname in glob.glob(glob_tmpl):
if not os.path.basename(fname) in keep:
print("Removing DEP-11 cache directory: %s" % (fname))
rmtree(fname)
print("Cache pruned.")
def validate_packages(suite, component, architecture):
"""
Ensure files mentioned in Packages exist
"""
cnf = Config()
filename = "%s/dists/%s/%s/binary-%s/Packages.gz" \
% (cnf["Dir::Root"], suite, component, architecture)
print "Processing %s..." % (filename)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
(filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
packages = utils.open_file(temp_filename)
Packages = apt_pkg.TagFile(packages)
while Packages.step():
filename = "%s/%s" % (cnf["Dir::Root"],
Packages.section.find('Filename'))
if not os.path.exists(filename):
print "W: %s missing." % (filename)
packages.close()
os.unlink(temp_filename)
def parse_nfu(architecture):
cnf = Config()
# utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date]
r = re.compile("^\w+/([^_]+)_.*: Not-For-Us")
ret = set()
filename = "%s/%s-all.txt" % (
cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture)
# Not all architectures may have a wanna-build dump, so we want to ignore missin
# files
if os.path.exists(filename):
f = utils.open_file(filename)
for line in f:
if line[0] == ' ':
continue
m = r.match(line)
if m:
ret.add(m.group(1))
f.close()
else:
utils.warn("No wanna-build dump file for architecture %s" %
architecture)
return ret
def initialize(self):
cnf = Config()
if cnf["DB::Name"] in ('backports', 'obscurity', 'projectb'):
self.fail("You have configured an invalid database name: '%s'." % \
cnf["DB::Name"])
if cnf["DB::Host"]:
# TCP/IP
connstr = "postgres://%s" % cnf["DB::Host"]
if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
connstr += ":%s" % cnf["DB::Port"]
connstr += "/%s" % cnf["DB::Name"]
else:
# Unix Socket
connstr = "postgres:///%s" % cnf["DB::Name"]
if cnf["DB::Port"] and cnf["DB::Port"] != "-1":
connstr += "?port=%s" % cnf["DB::Port"]
pickle_filename = 'db-metadata-%s.pkl' % __version__
pickle_file = open(fixture(pickle_filename), 'r')
DBDakTestCase.metadata = pickle.load(pickle_file)
self.metadata.ddl_listeners = pickle.load(pickle_file)
pickle_file.close()
self.metadata.bind = create_engine(connstr)
self.metadata.create_all()
self.create_all_triggers()
def clean_byhash(now_date, session):
cnf = Config()
suite_suffix = cnf.find("Dinstall::SuiteSuffix", "")
Logger.log(["Cleaning out unused by-hash files..."])
q = session.execute("""
DELETE FROM hashfile h
USING suite s, archive a
WHERE s.id = h.suite_id
AND a.id = s.archive_id
AND h.unreferenced + a.stayofexecution < CURRENT_TIMESTAMP
RETURNING a.path, s.suite_name, h.path""")
count = q.rowcount
if not Options["No-Action"]:
for base, suite, path in q:
filename = os.path.join(base, 'dists', suite, suite_suffix, path)
try:
os.unlink(filename)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
Logger.log(['database referred to non-existing file', filename])
else:
Logger.log(['delete hashfile', suite, path])
session.commit()
if count > 0:
Logger.log(["total", count])
def check(self, upload):
allow_source_untrusted_sig_keys = Config().value_list(
'Dinstall::AllowSourceUntrustedSigKeys')
changes = upload.changes
if not changes.valid_signature:
raise Reject("Signature for .changes not valid.")
self.check_replay(upload)
self._check_hashes(upload, changes.filename, changes.files.values())
source = None
try:
source = changes.source
except Exception as e:
raise Reject("Invalid dsc file: {0}".format(e))
if source is not None:
if changes.primary_fingerprint not in allow_source_untrusted_sig_keys:
if not source.valid_signature:
raise Reject("Signature for .dsc not valid.")
if source.primary_fingerprint != changes.primary_fingerprint:
raise Reject(
".changes and .dsc not signed by the same key.")
self._check_hashes(upload, source.filename, source.files.values())
if upload.fingerprint is None or upload.fingerprint.uid is None:
raise Reject(".changes signed by unknown key.")
def real_comment_reject(upload, srcqueue, comments, transaction, notify=True, manual=False):
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
queuedir = upload.policy_queue.path
rejectdir = cnf['Dir::Reject']
### Copy files to reject/
poolfiles = [b.poolfile for b in upload.binaries]
if upload.source is not None:
poolfiles.extend([df.poolfile for df in upload.source.srcfiles])
# Not beautiful...
files = [ af.path for af in session.query(ArchiveFile) \
.filter_by(archive=upload.policy_queue.suite.archive) \
.join(ArchiveFile.file) \
.filter(PoolFile.file_id.in_([ f.file_id for f in poolfiles ])) ]
for byhand in upload.byhand:
path = os.path.join(queuedir, byhand.filename)
if os.path.exists(path):
files.append(path)
files.append(os.path.join(queuedir, changesname))
for fn in files:
dst = utils.find_next_free(os.path.join(rejectdir, os.path.basename(fn)))
fs.copy(fn, dst, link=True)
### Write reason
dst = utils.find_next_free(os.path.join(rejectdir, '{0}.reason'.format(changesname)))
fh = fs.create(dst)
fh.write(comments)
fh.close()
### Send mail notification
if notify:
rejected_by = None
reason = comments
# Try to use From: from comment file if there is one.
# This is not very elegant...
match = re.match(r"\AFrom: ([^\n]+)\n\n", comments)
if match:
rejected_by = match.group(1)
reason = '\n'.join(comments.splitlines()[2:])
pu = get_processed_upload(upload)
daklib.announce.announce_reject(pu, reason, rejected_by)
print " REJECT"
if not Options["No-Action"]:
Logger.log(["Policy Queue REJECT", srcqueue.queue_name, upload.changes.changesname])
changes = upload.changes
remove_upload(upload, transaction)
session.delete(changes)
def main(argv=None):
if argv is None:
argv = sys.argv
arguments = [('h', 'help', 'Export::Options::Help'),
('c', 'copy', 'Export::Options::Copy'),
('d', 'directory', 'Export::Options::Directory', 'HasArg'),
('r', 'relative', 'Export::Options::Relative'),
('s', 'suite', 'Export::Options::Suite', 'HasArg')]
cnf = Config()
apt_pkg.parse_commandline(cnf.Cnf, arguments, argv)
options = cnf.subtree('Export::Options')
if 'Help' in options or 'Suite' not in options:
usage()
sys.exit(0)
session = DBConn().session()
suite = session.query(Suite).filter_by(suite_name=options['Suite']).first()
if suite is None:
print("Unknown suite '{0}'".format(options['Suite']))
sys.exit(1)
directory = options.get('Directory')
if not directory:
print("No target directory.")
sys.exit(1)
symlink = 'Copy' not in options
relative = 'Relative' in options
if relative and not symlink:
print("E: --relative and --copy cannot be used together.")
sys.exit(1)
binaries = suite.binaries
sources = suite.sources
files = []
files.extend([b.poolfile for b in binaries])
for s in sources:
files.extend([ds.poolfile for ds in s.srcfiles])
with FilesystemTransaction() as fs:
for f in files:
af = session.query(ArchiveFile) \
.join(ArchiveFile.component).join(ArchiveFile.file) \
.filter(ArchiveFile.archive == suite.archive) \
.filter(ArchiveFile.file == f).first()
src = af.path
if relative:
src = os.path.relpath(src, directory)
dst = os.path.join(directory, f.basename)
if not os.path.exists(dst):
fs.copy(src, dst, symlink=symlink)
fs.commit()
def check(self, upload):
changes = upload.changes
# Only check sourceful uploads.
if changes.source is None:
return True
# Only check uploads to unstable or experimental.
if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions:
return True
cnf = Config()
if 'Dinstall::LintianTags' not in cnf:
return True
tagfile = cnf['Dinstall::LintianTags']
with open(tagfile, 'r') as sourcefile:
sourcecontent = sourcefile.read()
try:
lintiantags = yaml.safe_load(sourcecontent)['lintian']
except yaml.YAMLError as msg:
raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg))
fd, temp_filename = utils.temp_filename(mode=0o644)
temptagfile = os.fdopen(fd, 'w')
for tags in lintiantags.itervalues():
for tag in tags:
print >>temptagfile, tag
temptagfile.close()
changespath = os.path.join(upload.directory, changes.filename)
try:
cmd = []
result = 0
user = cnf.get('Dinstall::UnprivUser') or None
if user is not None:
cmd.extend(['sudo', '-H', '-u', user])
cmd.extend(['/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath])
output = daklib.daksubprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
result = e.returncode
output = e.output
finally:
os.unlink(temp_filename)
if result == 2:
utils.warn("lintian failed for %s [return code: %s]." % \
(changespath, result))
utils.warn(utils.prefix_multi_line_string(output, \
" [possible output:] "))
parsed_tags = lintian.parse_lintian_output(output)
rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags))
if len(rejects) != 0:
raise Reject('\n'.join(rejects))
return True
