def check_upload_for_external_signature_request(session, target_suite, suite,
binary):
if 'External-Signature-Requests' not in Config():
return
config = Config().subtree('External-Signature-Requests')
config_sources = config.subtree('Sources')
source = binary.source
if source.source not in config_sources:
return
src_config = config_sources.subtree(source.source)
if binary.package not in src_config.value_list('Packages'):
return
suites = config.value_list('Default-Suites')
if 'Suites' in src_config:
suites = src_config.value_list('Suites')
if target_suite.suite_name not in suites:
return
archs = config.value_list('Default-Architectures')
if 'Architectures' in src_config:
archs = src_config.value_list('Architectures')
if binary.architecture.arch_string not in archs:
return
add_external_signature_request(session, target_suite, suite, binary)
def check_upload_for_external_signature_request(session, target_suite, suite, binary):
if 'External-Signature-Requests' not in Config():
return
config = Config().subtree('External-Signature-Requests')
config_sources = config.subtree('Sources')
source = binary.source
if source.source not in config_sources:
return
src_config = config_sources.subtree(source.source)
if binary.package not in src_config.value_list('Packages'):
return
suites = config.value_list('Default-Suites')
if 'Suites' in src_config:
suites = src_config.value_list('Suites')
if target_suite.suite_name not in suites:
return
archs = config.value_list('Default-Architectures')
if 'Architectures' in src_config:
archs = src_config.value_list('Architectures')
if binary.architecture.arch_string not in archs:
return
add_external_signature_request(session, target_suite, suite, binary)
def action_dm_migrate(self, fingerprint, section, session):
self._action_dm_admin_common(fingerprint, section, session)
cnf = Config()
acl_name = cnf.get('Command::DM::ACL', 'dm')
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash_from = section['From'].translate(None, ' ')
fpr_from = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_from).first()
if fpr_from is None:
self.result.append('Unknown fingerprint (From): {0}\nNo action taken.'.format(fpr_hash_from))
return
fpr_hash_to = section['To'].translate(None, ' ')
fpr_to = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_to).first()
if fpr_to is None:
self.result.append('Unknown fingerprint (To): {0}\nNo action taken.'.format(fpr_hash_to))
return
if fpr_to.keyring is None or fpr_to.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
self.result.append('Key (To) {0} is not in DM keyring.\nNo action taken.'.format(fpr_to.fingerprint))
return
self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to)])
sources = []
for entry in session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr_from):
self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to), 'source={0}'.format(entry.source)])
entry.fingerprint = fpr_to
sources.append(entry.source)
self.result.append('Migrated {0} to {1}.\n{2} acl entries changed: {3}'.format(fpr_hash_from, fpr_hash_to, len(sources), ", ".join(sources)))
session.commit()
def process_suite(session, suite, logger, force=False):
'''
Extract new metadata for a given suite.
'''
path = Config()["Dir::Pool"]
if suite.untouchable and not force:
import daklib.utils
daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % suite.suite_name)
return
for component in [ c.component_name for c in suite.components ]:
mif = MetaInfoFinder(session)
pkglist = mif.find_meta_files(component=component, suitename=suite.suite_name)
values = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component,
}
pool = DakProcessPool()
dpool = MetadataPool(values)
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
# we abuse the message return value here...
logger.log([msg['message']])
dpool.append_cptdata(msg['arch'], msg['cpts'])
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess recieved signal ', msg])
else:
logger.log(['E: ', msg])
cnf = Config()
iconf = IconFinder(suite.suite_name, component)
mde = MetadataExtractor(suite.suite_name, component,
cnf["Dir::MetaInfo"],
cnf["DEP11::Url"],
cnf.value_list('DEP11::IconSizes'),
iconf)
for pkgname, pkg in pkglist.items():
for arch, data in pkg.items():
package_fname = os.path.join (path, data['filename'])
if not os.path.exists(package_fname):
print('Package not found: %s' % (package_fname))
continue
pool.apply_async(extract_metadata,
(mde, suite.suite_name, pkgname, data['files'], data['binid'], package_fname, arch), callback=parse_results)
pool.close()
pool.join()
# save new metadata to the database
dpool.export(session)
make_icon_tar(suite.suite_name, component)
logger.log(["Completed metadata extraction for suite %s/%s" % (suite.suite_name, component)])
def action_dm_migrate(self, fingerprint, section, session):
self._action_dm_admin_common(fingerprint, section, session)
cnf = Config()
acl_name = cnf.get('Command::DM::ACL', 'dm')
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash_from = section['From'].translate(None, ' ')
fpr_from = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_from).first()
if fpr_from is None:
self.result.append('Unknown fingerprint (From): {0}\nNo action taken.'.format(fpr_hash_from))
return
fpr_hash_to = section['To'].translate(None, ' ')
fpr_to = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_to).first()
if fpr_to is None:
self.result.append('Unknown fingerprint (To): {0}\nNo action taken.'.format(fpr_hash_to))
return
if fpr_to.keyring is None or fpr_to.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
self.result.append('Key (To) {0} is not in DM keyring.\nNo action taken.'.format(fpr_to.fingerprint))
return
self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to)])
sources = []
for entry in session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr_from):
self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to), 'source={0}'.format(entry.source)])
entry.fingerprint = fpr_to
sources.append(entry.source)
self.result.append('Migrated {0} to {1}.\n{2} acl entries changed: {3}'.format(fpr_hash_from, fpr_hash_to, len(sources), ", ".join(sources)))
session.commit()
def action_sync_package(self, fingerprint, section, session):
cnf = Config()
allowed_keyrings = cnf.value_list('Command::Sync::AdminKeyrings')
if fingerprint.keyring.keyring_name not in allowed_keyrings:
raise CommandError('Key {0} is not allowed to sync Debian packages.'.format(fingerprint.fingerprint))
if 'Packages' not in section or 'Suite' not in section or 'Component' not in section:
raise CommandError('Invalid commands: Section is missing.')
packages_str = section['Packages']
suite = section['Suite']
component = section['Component']
if " " in packages_str:
packages = packages_str.split(" ")
else:
packages = [packages_str]
for pkg in packages:
p = subprocess.Popen(["sync-debian-package", "-i", suite, "staging", component, pkg], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()
if p.returncode is not 0:
self.result.append("Failed syncing: {0} from {1} ({2})".format(pkg, suite, component))
out_str = ""
if output[0] != None:
out_str = output[0]
if output[1] != None:
out_str += output[1]
self.result.append(" - Error: {0}".format(out_str))
else:
self.result.append("Synced package: {0} from {1} ({2})".format(pkg, suite, component))
def __init__(self, suitename, component):
self._suite_name = suitename
self._component = component
cnf = Config()
self._icon_theme_packages = cnf.value_list('DEP11::IconThemePackages')
self._pool_dir = cnf["Dir::Pool"]
self._allowed_exts = (".png")
def __init__(self, suitename, component):
self._suite_name = suitename
self._component = component
cnf = Config()
self._icon_theme_packages = cnf.value_list('DEP11::IconThemePackages')
self._pool_dir = cnf["Dir::Pool"]
self._allowed_exts = (".png")
def _action_dm_admin_common(self, fingerprint, section, session):
cnf = Config()
if "Command::DM-Admin::AdminFingerprints" not in cnf or "Command::DM::ACL" not in cnf:
raise CommandError("DM admin command is not configured for this archive.")
allowed_fingerprints = cnf.value_list("Command::DM-Admin::AdminFingerprints")
if fingerprint.fingerprint not in allowed_fingerprints:
raise CommandError("Key {0} is not allowed to admin DM".format(fingerprint.fingerprint))
def _action_dm_admin_common(self, fingerprint, section, session):
cnf = Config()
if 'Command::DM-Admin::AdminFingerprints' not in cnf \
or 'Command::DM::ACL' not in cnf:
raise CommandError('DM admin command is not configured for this archive.')
allowed_fingerprints = cnf.value_list('Command::DM-Admin::AdminFingerprints')
if fingerprint.fingerprint not in allowed_fingerprints:
raise CommandError('Key {0} is not allowed to admin DM'.format(fingerprint.fingerprint))
def _action_dm_admin_common(self, fingerprint, section, session):
cnf = Config()
if 'Command::DM-Admin::AdminFingerprints' not in cnf \
or 'Command::DM::ACL' not in cnf:
raise CommandError('DM admin command is not configured for this archive.')
allowed_fingerprints = cnf.value_list('Command::DM-Admin::AdminFingerprints')
if fingerprint.fingerprint not in allowed_fingerprints:
raise CommandError('Key {0} is not allowed to admin DM'.format(fingerprint.fingerprint))
def make_icon_tar(suitename, component):
'''
icons-%(component)_%(size).tar.gz of each Component.
'''
cnf = Config()
sizes = cnf.value_list('DEP11::IconSizes')
for size in sizes:
icon_location_glob = os.path.join (cnf["Dir::MetaInfo"], suitename, component, "*", "icons", size, "*.*")
tar_location = os.path.join (cnf["Dir::Root"], "dists", suitename, component)
icon_tar_fname = os.path.join(tar_location, "icons-%s_%s.tar.gz" % (component, size))
tar = tarfile.open(icon_tar_fname, "w:gz")
for filename in glob.glob(icon_location_glob):
icon_name = os.path.basename (filename)
tar.add(filename,arcname=icon_name)
tar.close()
def action_sync_package(self, fingerprint, section, session):
cnf = Config()
allowed_keyrings = cnf.value_list('Command::Sync::AdminKeyrings')
if fingerprint.keyring.keyring_name not in allowed_keyrings:
raise CommandError(
'Key {0} is not allowed to sync Debian packages.'.format(
fingerprint.fingerprint))
if 'Packages' not in section or 'Suite' not in section or 'Component' not in section:
raise CommandError('Invalid commands: Section is missing.')
packages_str = section['Packages']
suite = section['Suite']
component = section['Component']
if " " in packages_str:
packages = packages_str.split(" ")
else:
packages = [packages_str]
# we always sync to the staging suite of Tanglu
sync_cmd = ["synchrotron", "-i", suite, "staging", component]
# add to-be-synced packages to the parameter list
sync_cmd.extend(packages)
p = subprocess.Popen(sync_cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()
if p.returncode is not 0:
self.result.append("Failed syncing: {0} from {1} ({2})".format(
packages_str, suite, component))
out_str = ""
if output[0] != None:
out_str = output[0]
if output[1] != None:
out_str += output[1]
self.result.append(" - Error: {0}".format(out_str))
else:
self.result.append("Synced package(s): {0} from {1} ({2})".format(
packages_str, suite, component))
def accept(directory, upload):
cnf = Config()
Logger.log(['ACCEPT', upload.changes.filename])
print("ACCEPT")
upload.install()
utils.process_buildinfos(upload.directory, upload.changes.buildinfo_files,
upload.transaction.fs, Logger)
accepted_to_real_suite = any(suite.policy_queue is None
for suite in upload.final_suites)
sourceful_upload = upload.changes.sourceful
control = upload.changes.changes
if sourceful_upload and not Options['No-Action']:
urgency = control.get('Urgency')
# As per policy 5.6.17, the urgency can be followed by a space and a
# comment. Extract only the urgency from the string.
if ' ' in urgency:
urgency, comment = urgency.split(' ', 1)
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(control['Source'], control['Version'], urgency)
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# Move .changes to done, but only for uploads that were accepted to a
# real suite. process-policy will handle this for uploads to queues.
if accepted_to_real_suite:
src = os.path.join(upload.directory, upload.changes.filename)
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
dst = os.path.join(donedir, upload.changes.filename)
dst = utils.find_next_free(dst)
upload.transaction.fs.copy(src, dst, mode=0o644)
SummaryStats().accept_count += 1
SummaryStats().accept_bytes += upload.changes.bytes
def do_update(self):
"""
Add table for version checks.
"""
print __doc__
try:
cnf = Config()
c = self.db.cursor()
c.execute(
"""
CREATE TABLE version_check (
suite INTEGER NOT NULL REFERENCES suite(id),
"check" TEXT NOT NULL CHECK ("check" IN ('Enhances', 'MustBeNewerThan', 'MustBeOlderThan')),
reference INTEGER NOT NULL REFERENCES suite(id),
PRIMARY KEY(suite, "check", reference)
)"""
)
c.execute("SELECT suite_name, id FROM suite")
suites = c.fetchall()
suite_id_map = {}
for suite_name, suite_id in suites:
suite_id_map[suite_name] = suite_id
for check in ["Enhances", "MustBeNewerThan", "MustBeOlderThan"]:
for suite_name in suite_id_map.keys():
for reference_name in [
s.lower() for s in cnf.value_list("Suite::%s::VersionChecks::%s" % (suite_name, check))
]:
c.execute(
"""INSERT INTO version_check (suite, "check", reference) VALUES (%s, %s, %s)""",
(suite_id_map[suite_name], check, suite_id_map[reference_name]),
)
c.execute("UPDATE config SET value = '52' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.ProgrammingError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply sick update 52, rollback issued. Error message : %s" % (str(msg)))
def accept(directory, upload):
cnf = Config()
Logger.log(['ACCEPT', upload.changes.filename])
print("ACCEPT")
upload.install()
process_buildinfos(upload)
accepted_to_real_suite = any(suite.policy_queue is None for suite in upload.final_suites)
sourceful_upload = 'source' in upload.changes.architectures
control = upload.changes.changes
if sourceful_upload and not Options['No-Action']:
urgency = control.get('Urgency')
# As per policy 5.6.17, the urgency can be followed by a space and a
# comment. Extract only the urgency from the string.
if ' ' in urgency:
urgency, comment = urgency.split(' ', 1)
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(control['Source'], control['Version'], urgency)
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# Move .changes to done, but only for uploads that were accepted to a
# real suite. process-policy will handle this for uploads to queues.
if accepted_to_real_suite:
src = os.path.join(upload.directory, upload.changes.filename)
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
dst = os.path.join(donedir, upload.changes.filename)
dst = utils.find_next_free(dst)
upload.transaction.fs.copy(src, dst, mode=0o644)
SummaryStats().accept_count += 1
SummaryStats().accept_bytes += upload.changes.bytes
def accept(directory, upload):
cnf = Config()
Logger.log(['ACCEPT', upload.changes.filename])
print "ACCEPT"
upload.install()
process_buildinfos(upload)
accepted_to_real_suite = False
for suite in upload.final_suites:
accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None
sourceful_upload = 'source' in upload.changes.architectures
control = upload.changes.changes
if sourceful_upload and not Options['No-Action']:
urgency = control.get('Urgency')
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(control['Source'], control['Version'], urgency)
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# Move .changes to done, but only for uploads that were accepted to a
# real suite. process-policy will handle this for uploads to queues.
if accepted_to_real_suite:
src = os.path.join(upload.directory, upload.changes.filename)
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
dst = os.path.join(donedir, upload.changes.filename)
dst = utils.find_next_free(dst)
upload.transaction.fs.copy(src, dst, mode=0o644)
SummaryStats().accept_count += 1
SummaryStats().accept_bytes += upload.changes.bytes
def comment_accept(upload, srcqueue, comments, transaction):
for byhand in upload.byhand:
path = os.path.join(srcqueue.path, byhand.filename)
if os.path.exists(path):
raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
allow_tainted = srcqueue.suite.archive.tainted
# We need overrides to get the target component
overridesuite = upload.target_suite
if overridesuite.overridesuite is not None:
overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
def binary_component_func(db_binary):
section = db_binary.proxy['Section']
component_name = 'main'
if section.find('/') != -1:
component_name = section.split('/', 1)[0]
return get_mapped_component(component_name, session=session)
def is_debug_binary(db_binary):
return daklib.utils.is_in_debug_section(db_binary.proxy)
def has_debug_binaries(upload):
return any((is_debug_binary(x) for x in upload.binaries))
def source_component_func(db_source):
package_list = PackageList(db_source.proxy)
component = source_component_from_package_list(package_list, upload.target_suite)
if component is not None:
return get_mapped_component(component.component_name, session=session)
# Fallback for packages without Package-List field
query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
.join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
.join(Component)
return query.one().component
all_target_suites = [upload.target_suite]
all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
for suite in all_target_suites:
debug_suite = suite.debug_suite
if upload.source is not None:
# If we have Source in this upload, let's include it into
# upload suite.
transaction.copy_source(
upload.source,
suite,
source_component_func(upload.source),
allow_tainted=allow_tainted,
)
if debug_suite is not None and has_debug_binaries(upload):
# If we're handing a debug package, we also need to include the
# source in the debug suite as well.
transaction.copy_source(
upload.source,
debug_suite,
source_component_func(upload.source),
allow_tainted=allow_tainted,
)
for db_binary in upload.binaries:
# Now, let's work out where to copy this guy to -- if it's
# a debug binary, and the suite has a debug suite, let's go
# ahead and target the debug suite rather then the stock
# suite.
copy_to_suite = suite
if debug_suite is not None and is_debug_binary(db_binary):
copy_to_suite = debug_suite
# build queues and debug suites may miss the source package
# if this is a binary-only upload.
if copy_to_suite != upload.target_suite:
transaction.copy_source(
db_binary.source,
copy_to_suite,
source_component_func(db_binary.source),
allow_tainted=allow_tainted,
)
transaction.copy_binary(
db_binary,
copy_to_suite,
binary_component_func(db_binary),
allow_tainted=allow_tainted,
extra_archives=[upload.target_suite.archive],
)
check_upload_for_external_signature_request(session, suite, copy_to_suite, db_binary)
suite.update_last_changed()
# Copy .changes if needed
if upload.target_suite.copychanges:
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
fs.copy(src, dst, mode=upload.target_suite.archive.mode)
# List of files in the queue directory
queue_files = [changesname]
chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False)
queue_files.extend(f.filename for f in chg.buildinfo_files)
# Copy upload to Process-Policy::CopyDir
# Used on security.d.o to sync accepted packages to ftp-master, but this
# should eventually be replaced by something else.
copydir = cnf.get('Process-Policy::CopyDir') or None
if copydir is not None:
mode = upload.target_suite.archive.mode
if upload.source is not None:
for f in [ df.poolfile for df in upload.source.srcfiles ]:
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for db_binary in upload.binaries:
f = db_binary.poolfile
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for fn in queue_files:
src = os.path.join(upload.policy_queue.path, fn)
dst = os.path.join(copydir, fn)
# We check for `src` to exist as old uploads in policy queues
# might still miss the `.buildinfo` files.
if os.path.exists(src) and not os.path.exists(dst):
fs.copy(src, dst, mode=mode)
if upload.source is not None and not Options['No-Action']:
urgency = upload.changes.urgency
# As per policy 5.6.17, the urgency can be followed by a space and a
# comment. Extract only the urgency from the string.
if ' ' in urgency:
(urgency, comment) = urgency.split(' ', 1)
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(upload.source.source, upload.source.version, urgency)
print " ACCEPT"
if not Options['No-Action']:
Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# TODO: code duplication. Similar code is in process-upload.
# Move .changes to done
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
for fn in queue_files:
src = os.path.join(upload.policy_queue.path, fn)
if os.path.exists(src):
dst = os.path.join(donedir, fn)
dst = utils.find_next_free(dst)
fs.copy(src, dst, mode=0o644)
remove_upload(upload, transaction)
def do_update(self):
print "Updating queue_build table"
try:
c = self.db.cursor()
cnf = Config()
print "Adding copy_files field to queue table"
c.execute("ALTER TABLE queue ADD copy_pool_files BOOL NOT NULL DEFAULT FALSE")
print "Adding queue_files table"
c.execute("""CREATE TABLE queue_files (
id SERIAL PRIMARY KEY,
queueid INT4 NOT NULL REFERENCES queue(id) ON DELETE RESTRICT,
insertdate TIMESTAMP NOT NULL DEFAULT now(),
lastused TIMESTAMP DEFAULT NULL,
filename TEXT NOT NULL,
fileid INT4 REFERENCES files(id) ON DELETE CASCADE)""")
c.execute("""SELECT queue_build.filename, queue_build.last_used, queue_build.queue
FROM queue_build""")
for r in c.fetchall():
print r[0]
filename = r[0]
last_used = r[1]
queue = r[2]
try:
endlink = os.readlink(filename)
c.execute("SELECT files.id FROM files WHERE filename LIKE '%%%s'" % endlink[endlink.rindex('/')+1:])
f = c.fetchone()
c.execute("""INSERT INTO queue_files (queueid, lastused, filename, fileid) VALUES
(%s, now(), %s, %s)""", (queue, filename[filename.rindex('/')+1:], f[0]))
except OSError as e:
print "Can't find file %s (%s)" % (filename, e)
print "Dropping old queue_build table"
c.execute("DROP TABLE queue_build")
print "Adding changes_pending_files table"
c.execute("""CREATE TABLE changes_pending_files (
id SERIAL PRIMARY KEY,
changeid INT4 NOT NULL REFERENCES known_changes(id) ON DELETE CASCADE,
filename TEXT NOT NULL,
source BOOL NOT NULL DEFAULT FALSE,
filesize BIGINT NOT NULL,
md5sum TEXT NOT NULL,
sha1sum TEXT NOT NULL,
sha256sum TEXT NOT NULL)""")
print "Adding changes_pool_files table"
c.execute("""CREATE TABLE changes_pool_files (
changeid INT4 NOT NULL REFERENCES known_changes(id) ON DELETE CASCADE,
fileid INT4 NOT NULL REFERENCES files(id) ON DELETE RESTRICT,
PRIMARY KEY (changeid, fileid))""")
print "Adding suite_queue_copy table"
c.execute("""CREATE TABLE suite_queue_copy (
suite INT4 NOT NULL REFERENCES suite(id),
queue INT4 NOT NULL REFERENCES queue(id),
PRIMARY KEY (suite, queue))""")
# Link all suites from accepted
c.execute("""SELECT suite.id FROM suite""")
for s in c.fetchall():
c.execute("""INSERT INTO suite_queue_copy (suite, queue) VALUES (%s, (SELECT id FROM queue WHERE queue_name = 'accepted'))""", s)
# Parse the config and add any buildd stuff
cnf = Config()
c.execute("""INSERT INTO queue (queue_name, path) VALUES ('buildd', '%s')""" % cnf["Dir::QueueBuild"].rstrip('/'))
for s in cnf.value_list("Dinstall::QueueBuildSuites"):
c.execute("""INSERT INTO suite_queue_copy (suite, queue)
VALUES ( (SELECT id FROM suite WHERE suite_name = '%s'),
(SELECT id FROM queue WHERE queue_name = 'buildd'))""" % s.lower())
print "Committing"
c.execute("UPDATE config SET value = '21' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.InternalError as msg:
self.db.rollback()
raise DBUpdateError("Unable to apply queue_build 21, rollback issued. Error message : %s" % (str(msg)))
def comment_accept(upload, srcqueue, comments, transaction):
for byhand in upload.byhand:
path = os.path.join(srcqueue.path, byhand.filename)
if os.path.exists(path):
raise Exception("E: cannot ACCEPT upload with unprocessed byhand file {0}".format(byhand.filename))
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
allow_tainted = srcqueue.suite.archive.tainted
# We need overrides to get the target component
overridesuite = upload.target_suite
if overridesuite.overridesuite is not None:
overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
def binary_component_func(db_binary):
section = db_binary.proxy["Section"]
component_name = "main"
if section.find("/") != -1:
component_name = section.split("/", 1)[0]
return get_mapped_component(component_name, session=session)
def is_debug_binary(db_binary):
return daklib.utils.is_in_debug_section(db_binary.proxy)
def has_debug_binaries(upload):
return any((is_debug_binary(x) for x in upload.binaries))
def source_component_func(db_source):
package_list = PackageList(db_source.proxy)
component = source_component_from_package_list(package_list, upload.target_suite)
if component is not None:
return get_mapped_component(component.component_name, session=session)
# Fallback for packages without Package-List field
query = (
session.query(Override)
.filter_by(suite=overridesuite, package=db_source.source)
.join(OverrideType)
.filter(OverrideType.overridetype == "dsc")
.join(Component)
)
return query.one().component
all_target_suites = [upload.target_suite]
all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
for suite in all_target_suites:
debug_suite = suite.debug_suite
if upload.source is not None:
# If we have Source in this upload, let's include it into
# upload suite.
transaction.copy_source(
upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted
)
if debug_suite is not None and has_debug_binaries(upload):
# If we're handing a debug package, we also need to include the
# source in the debug suite as well.
transaction.copy_source(
upload.source, debug_suite, source_component_func(upload.source), allow_tainted=allow_tainted
)
for db_binary in upload.binaries:
# Now, let's work out where to copy this guy to -- if it's
# a debug binary, and the suite has a debug suite, let's go
# ahead and target the debug suite rather then the stock
# suite.
copy_to_suite = suite
if debug_suite is not None and is_debug_binary(db_binary):
copy_to_suite = debug_suite
# build queues may miss the source package if this is a
# binary-only upload.
if suite != upload.target_suite:
transaction.copy_source(
db_binary.source,
copy_to_suite,
source_component_func(db_binary.source),
allow_tainted=allow_tainted,
)
transaction.copy_binary(
db_binary,
copy_to_suite,
binary_component_func(db_binary),
allow_tainted=allow_tainted,
extra_archives=[upload.target_suite.archive],
)
# Copy .changes if needed
if upload.target_suite.copychanges:
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
fs.copy(src, dst, mode=upload.target_suite.archive.mode)
# Copy upload to Process-Policy::CopyDir
# Used on security.d.o to sync accepted packages to ftp-master, but this
# should eventually be replaced by something else.
copydir = cnf.get("Process-Policy::CopyDir") or None
if copydir is not None:
mode = upload.target_suite.archive.mode
if upload.source is not None:
for f in [df.poolfile for df in upload.source.srcfiles]:
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for db_binary in upload.binaries:
f = db_binary.poolfile
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(copydir, upload.changes.changesname)
if not os.path.exists(dst):
fs.copy(src, dst, mode=mode)
if upload.source is not None and not Options["No-Action"]:
urgency = upload.changes.urgency
if urgency not in cnf.value_list("Urgency::Valid"):
urgency = cnf["Urgency::Default"]
UrgencyLog().log(upload.source.source, upload.source.version, urgency)
print " ACCEPT"
if not Options["No-Action"]:
Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# TODO: code duplication. Similar code is in process-upload.
# Move .changes to done
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
now = datetime.datetime.now()
donedir = os.path.join(cnf["Dir::Done"], now.strftime("%Y/%m/%d"))
dst = os.path.join(donedir, upload.changes.changesname)
dst = utils.find_next_free(dst)
fs.copy(src, dst, mode=0o644)
remove_upload(upload, transaction)
def comment_accept(upload, srcqueue, comments, transaction):
for byhand in upload.byhand:
path = os.path.join(srcqueue.path, byhand.filename)
if os.path.exists(path):
raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
allow_tainted = srcqueue.suite.archive.tainted
# We need overrides to get the target component
overridesuite = upload.target_suite
if overridesuite.overridesuite is not None:
overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
def binary_component_func(db_binary):
section = db_binary.proxy['Section']
component_name = 'main'
if section.find('/') != -1:
component_name = section.split('/', 1)[0]
return get_mapped_component(component_name, session=session)
def is_debug_binary(db_binary):
return daklib.utils.is_in_debug_section(db_binary.proxy)
def has_debug_binaries(upload):
return any((is_debug_binary(x) for x in upload.binaries))
def source_component_func(db_source):
package_list = PackageList(db_source.proxy)
component = source_component_from_package_list(package_list, upload.target_suite)
if component is not None:
return get_mapped_component(component.component_name, session=session)
# Fallback for packages without Package-List field
query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
.join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
.join(Component)
return query.one().component
all_target_suites = [upload.target_suite]
all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
for suite in all_target_suites:
debug_suite = suite.debug_suite
if upload.source is not None:
# If we have Source in this upload, let's include it into
# upload suite.
transaction.copy_source(
upload.source,
suite,
source_component_func(upload.source),
allow_tainted=allow_tainted,
)
if debug_suite is not None and has_debug_binaries(upload):
# If we're handing a debug package, we also need to include the
# source in the debug suite as well.
transaction.copy_source(
upload.source,
debug_suite,
source_component_func(upload.source),
allow_tainted=allow_tainted,
)
for db_binary in upload.binaries:
# Now, let's work out where to copy this guy to -- if it's
# a debug binary, and the suite has a debug suite, let's go
# ahead and target the debug suite rather then the stock
# suite.
copy_to_suite = suite
if debug_suite is not None and is_debug_binary(db_binary):
copy_to_suite = debug_suite
# build queues and debug suites may miss the source package
# if this is a binary-only upload.
if copy_to_suite != upload.target_suite:
transaction.copy_source(
db_binary.source,
copy_to_suite,
source_component_func(db_binary.source),
allow_tainted=allow_tainted,
)
transaction.copy_binary(
db_binary,
copy_to_suite,
binary_component_func(db_binary),
allow_tainted=allow_tainted,
extra_archives=[upload.target_suite.archive],
)
check_upload_for_external_signature_request(session, suite, copy_to_suite, db_binary)
suite.update_last_changed()
# Copy .changes if needed
if upload.target_suite.copychanges:
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
fs.copy(src, dst, mode=upload.target_suite.archive.mode)
# List of files in the queue directory
queue_files = [changesname]
chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False)
queue_files.extend(f.filename for f in chg.buildinfo_files)
# Copy upload to Process-Policy::CopyDir
# Used on security.d.o to sync accepted packages to ftp-master, but this
# should eventually be replaced by something else.
copydir = cnf.get('Process-Policy::CopyDir') or None
if copydir is not None:
mode = upload.target_suite.archive.mode
if upload.source is not None:
for f in [df.poolfile for df in upload.source.srcfiles]:
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for db_binary in upload.binaries:
f = db_binary.poolfile
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for fn in queue_files:
src = os.path.join(upload.policy_queue.path, fn)
dst = os.path.join(copydir, fn)
# We check for `src` to exist as old uploads in policy queues
# might still miss the `.buildinfo` files.
if os.path.exists(src) and not os.path.exists(dst):
fs.copy(src, dst, mode=mode)
if upload.source is not None and not Options['No-Action']:
urgency = upload.changes.urgency
# As per policy 5.6.17, the urgency can be followed by a space and a
# comment. Extract only the urgency from the string.
if ' ' in urgency:
urgency, comment = urgency.split(' ', 1)
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(upload.source.source, upload.source.version, urgency)
print(" ACCEPT")
if not Options['No-Action']:
Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# TODO: code duplication. Similar code is in process-upload.
# Move .changes to done
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
for fn in queue_files:
src = os.path.join(upload.policy_queue.path, fn)
if os.path.exists(src):
dst = os.path.join(donedir, fn)
dst = utils.find_next_free(dst)
fs.copy(src, dst, mode=0o644)
remove_upload(upload, transaction)
def action_dm(self, fingerprint, section, session):
cnf = Config()
if 'Command::DM::AdminKeyrings' not in cnf \
or 'Command::DM::ACL' not in cnf \
or 'Command::DM::Keyrings' not in cnf:
raise CommandError('DM command is not configured for this archive.')
allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings')
if fingerprint.keyring.keyring_name not in allowed_keyrings:
raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint))
acl_name = cnf.get('Command::DM::ACL', 'dm')
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash = section['Fingerprint'].translate(None, ' ')
fpr = session.query(Fingerprint).filter_by(fingerprint=fpr_hash).first()
if fpr is None:
raise CommandError('Unknown fingerprint {0}'.format(fpr_hash))
if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'):
raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint))
addresses = gpg_get_key_addresses(fpr.fingerprint)
if len(addresses) > 0:
self.cc.append(addresses[0])
self.log.log(['dm', 'fingerprint', fpr.fingerprint])
self.result.append('Fingerprint: {0}'.format(fpr.fingerprint))
if len(addresses) > 0:
self.log.log(['dm', 'uid', addresses[0]])
self.result.append('Uid: {0}'.format(addresses[0]))
for source in self._split_packages(section.get('Allow', '')):
# Check for existance of source package to catch typos
if session.query(DBSource).filter_by(source=source).first() is None:
raise CommandError('Tried to grant permissions for unknown source package: {0}'.format(source))
if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None:
aps = ACLPerSource()
aps.acl = acl
aps.fingerprint = fpr
aps.source = source
aps.created_by = fingerprint
aps.reason = section.get('Reason')
session.add(aps)
self.log.log(['dm', 'allow', fpr.fingerprint, source])
self.result.append('Allowed: {0}'.format(source))
else:
self.result.append('Already-Allowed: {0}'.format(source))
session.flush()
for source in self._split_packages(section.get('Deny', '')):
count = session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete()
if count == 0:
raise CommandError('Tried to remove upload permissions for package {0}, '
'but no upload permissions were granted before.'.format(source))
self.log.log(['dm', 'deny', fpr.fingerprint, source])
self.result.append('Denied: {0}'.format(source))
session.commit()
def action_dm(self, fingerprint, section, session):
cnf = Config()
if 'Command::DM::AdminKeyrings' not in cnf \
or 'Command::DM::ACL' not in cnf \
or 'Command::DM::Keyrings' not in cnf:
raise CommandError(
'DM command is not configured for this archive.')
allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings')
if fingerprint.keyring.keyring_name not in allowed_keyrings:
raise CommandError('Key {0} is not allowed to set DM'.format(
fingerprint.fingerprint))
acl_name = cnf.get('Command::DM::ACL', 'dm')
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash = section['Fingerprint'].translate(None, ' ')
fpr = session.query(Fingerprint).filter_by(
fingerprint=fpr_hash).first()
if fpr is None:
raise CommandError('Unknown fingerprint {0}'.format(fpr_hash))
if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list(
'Command::DM::Keyrings'):
raise CommandError('Key {0} is not in DM keyring.'.format(
fpr.fingerprint))
addresses = gpg_get_key_addresses(fpr.fingerprint)
if len(addresses) > 0:
self.cc.append(addresses[0])
self.log.log(['dm', 'fingerprint', fpr.fingerprint])
self.result.append('Fingerprint: {0}'.format(fpr.fingerprint))
if len(addresses) > 0:
self.log.log(['dm', 'uid', addresses[0]])
self.result.append('Uid: {0}'.format(addresses[0]))
for source in self._split_packages(section.get('Allow', '')):
# Check for existance of source package to catch typos
if session.query(DBSource).filter_by(
source=source).first() is None:
raise CommandError(
'Tried to grant permissions for unknown source package: {0}'
.format(source))
if session.query(ACLPerSource).filter_by(
acl=acl, fingerprint=fpr, source=source).first() is None:
aps = ACLPerSource()
aps.acl = acl
aps.fingerprint = fpr
aps.source = source
aps.created_by = fingerprint
aps.reason = section.get('Reason')
session.add(aps)
self.log.log(['dm', 'allow', fpr.fingerprint, source])
self.result.append('Allowed: {0}'.format(source))
else:
self.result.append('Already-Allowed: {0}'.format(source))
session.flush()
for source in self._split_packages(section.get('Deny', '')):
count = session.query(ACLPerSource).filter_by(
acl=acl, fingerprint=fpr, source=source).delete()
if count == 0:
raise CommandError(
'Tried to remove upload permissions for package {0}, '
'but no upload permissions were granted before.'.format(
source))
self.log.log(['dm', 'deny', fpr.fingerprint, source])
self.result.append('Denied: {0}'.format(source))
session.commit()
def comment_accept(upload, srcqueue, comments, transaction):
for byhand in upload.byhand:
path = os.path.join(srcqueue.path, byhand.filename)
if os.path.exists(path):
raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename))
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
allow_tainted = srcqueue.suite.archive.tainted
# We need overrides to get the target component
overridesuite = upload.target_suite
if overridesuite.overridesuite is not None:
overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one()
def binary_component_func(db_binary):
section = db_binary.proxy['Section']
component_name = 'main'
if section.find('/') != -1:
component_name = section.split('/', 1)[0]
return get_mapped_component(component_name, session=session)
def source_component_func(db_source):
package_list = PackageList(db_source.proxy)
component = source_component_from_package_list(package_list, upload.target_suite)
if component is not None:
return get_mapped_component(component.component_name, session=session)
# Fallback for packages without Package-List field
query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
.join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
.join(Component)
return query.one().component
all_target_suites = [upload.target_suite]
all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues])
for suite in all_target_suites:
if upload.source is not None:
transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted)
for db_binary in upload.binaries:
# build queues may miss the source package if this is a binary-only upload
if suite != upload.target_suite:
transaction.copy_source(db_binary.source, suite, source_component_func(db_binary.source), allow_tainted=allow_tainted)
transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive])
# Copy .changes if needed
if upload.target_suite.copychanges:
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(upload.target_suite.path, upload.changes.changesname)
fs.copy(src, dst, mode=upload.target_suite.archive.mode)
# Copy upload to Process-Policy::CopyDir
# Used on security.d.o to sync accepted packages to ftp-master, but this
# should eventually be replaced by something else.
copydir = cnf.get('Process-Policy::CopyDir') or None
if copydir is not None:
mode = upload.target_suite.archive.mode
if upload.source is not None:
for f in [ df.poolfile for df in upload.source.srcfiles ]:
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for db_binary in upload.binaries:
f = db_binary.poolfile
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
dst = os.path.join(copydir, upload.changes.changesname)
if not os.path.exists(dst):
fs.copy(src, dst, mode=mode)
if upload.source is not None and not Options['No-Action']:
urgency = upload.changes.urgency
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(upload.source.source, upload.source.version, urgency)
print " ACCEPT"
if not Options['No-Action']:
Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# TODO: code duplication. Similar code is in process-upload.
# Move .changes to done
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
dst = os.path.join(donedir, upload.changes.changesname)
dst = utils.find_next_free(dst)
fs.copy(src, dst, mode=0o644)
remove_upload(upload, transaction)
def comment_accept(upload, srcqueue, comments, transaction):
for byhand in upload.byhand:
path = os.path.join(srcqueue.path, byhand.filename)
if os.path.exists(path):
raise Exception(
'E: cannot ACCEPT upload with unprocessed byhand file {0}'.
format(byhand.filename))
cnf = Config()
fs = transaction.fs
session = transaction.session
changesname = upload.changes.changesname
allow_tainted = srcqueue.suite.archive.tainted
# We need overrides to get the target component
overridesuite = upload.target_suite
if overridesuite.overridesuite is not None:
overridesuite = session.query(Suite).filter_by(
suite_name=overridesuite.overridesuite).one()
def binary_component_func(db_binary):
override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \
.join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \
.join(Component).one()
return override.component
def source_component_func(db_source):
override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \
.join(OverrideType).filter(OverrideType.overridetype == 'dsc') \
.join(Component).one()
return override.component
all_target_suites = [upload.target_suite]
all_target_suites.extend(
[q.suite for q in upload.target_suite.copy_queues])
for suite in all_target_suites:
if upload.source is not None:
transaction.copy_source(upload.source,
suite,
source_component_func(upload.source),
allow_tainted=allow_tainted)
for db_binary in upload.binaries:
# build queues may miss the source package if this is a binary-only upload
if suite != upload.target_suite:
transaction.copy_source(db_binary.source,
suite,
source_component_func(
db_binary.source),
allow_tainted=allow_tainted)
transaction.copy_binary(
db_binary,
suite,
binary_component_func(db_binary),
allow_tainted=allow_tainted,
extra_archives=[upload.target_suite.archive])
# Copy .changes if needed
if upload.target_suite.copychanges:
src = os.path.join(upload.policy_queue.path,
upload.changes.changesname)
dst = os.path.join(upload.target_suite.path,
upload.changes.changesname)
fs.copy(src, dst, mode=upload.target_suite.archive.mode)
# Copy upload to Process-Policy::CopyDir
# Used on security.d.o to sync accepted packages to ftp-master, but this
# should eventually be replaced by something else.
copydir = cnf.get('Process-Policy::CopyDir') or None
if copydir is not None:
mode = upload.target_suite.archive.mode
if upload.source is not None:
for f in [df.poolfile for df in upload.source.srcfiles]:
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
for db_binary in upload.binaries:
f = db_binary.poolfile
dst = os.path.join(copydir, f.basename)
if not os.path.exists(dst):
fs.copy(f.fullpath, dst, mode=mode)
src = os.path.join(upload.policy_queue.path,
upload.changes.changesname)
dst = os.path.join(copydir, upload.changes.changesname)
if not os.path.exists(dst):
fs.copy(src, dst, mode=mode)
if upload.source is not None and not Options['No-Action']:
urgency = upload.changes.urgency
if urgency not in cnf.value_list('Urgency::Valid'):
urgency = cnf['Urgency::Default']
UrgencyLog().log(upload.source.source, upload.source.version, urgency)
print " ACCEPT"
if not Options['No-Action']:
Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname])
pu = get_processed_upload(upload)
daklib.announce.announce_accept(pu)
# TODO: code duplication. Similar code is in process-upload.
# Move .changes to done
src = os.path.join(upload.policy_queue.path, upload.changes.changesname)
now = datetime.datetime.now()
donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d'))
dst = os.path.join(donedir, upload.changes.changesname)
dst = utils.find_next_free(dst)
fs.copy(src, dst, mode=0o644)
remove_upload(upload, transaction)
def do_update(self):
print "Updating queue_build table"
try:
c = self.db.cursor()
cnf = Config()
print "Adding copy_files field to queue table"
c.execute(
"ALTER TABLE queue ADD copy_pool_files BOOL NOT NULL DEFAULT FALSE"
)
print "Adding queue_files table"
c.execute("""CREATE TABLE queue_files (
id SERIAL PRIMARY KEY,
queueid INT4 NOT NULL REFERENCES queue(id) ON DELETE RESTRICT,
insertdate TIMESTAMP NOT NULL DEFAULT now(),
lastused TIMESTAMP DEFAULT NULL,
filename TEXT NOT NULL,
fileid INT4 REFERENCES files(id) ON DELETE CASCADE)""")
c.execute(
"""SELECT queue_build.filename, queue_build.last_used, queue_build.queue
FROM queue_build""")
for r in c.fetchall():
print r[0]
filename = r[0]
last_used = r[1]
queue = r[2]
try:
endlink = os.readlink(filename)
c.execute(
"SELECT files.id FROM files WHERE filename LIKE '%%%s'" %
endlink[endlink.rindex('/') + 1:])
f = c.fetchone()
c.execute(
"""INSERT INTO queue_files (queueid, lastused, filename, fileid) VALUES
(%s, now(), %s, %s)""",
(queue, filename[filename.rindex('/') + 1:], f[0]))
except OSError as e:
print "Can't find file %s (%s)" % (filename, e)
print "Dropping old queue_build table"
c.execute("DROP TABLE queue_build")
print "Adding changes_pending_files table"
c.execute("""CREATE TABLE changes_pending_files (
id SERIAL PRIMARY KEY,
changeid INT4 NOT NULL REFERENCES known_changes(id) ON DELETE CASCADE,
filename TEXT NOT NULL,
source BOOL NOT NULL DEFAULT FALSE,
filesize BIGINT NOT NULL,
md5sum TEXT NOT NULL,
sha1sum TEXT NOT NULL,
sha256sum TEXT NOT NULL)""")
print "Adding changes_pool_files table"
c.execute("""CREATE TABLE changes_pool_files (
changeid INT4 NOT NULL REFERENCES known_changes(id) ON DELETE CASCADE,
fileid INT4 NOT NULL REFERENCES files(id) ON DELETE RESTRICT,
PRIMARY KEY (changeid, fileid))""")
print "Adding suite_queue_copy table"
c.execute("""CREATE TABLE suite_queue_copy (
suite INT4 NOT NULL REFERENCES suite(id),
queue INT4 NOT NULL REFERENCES queue(id),
PRIMARY KEY (suite, queue))""")
# Link all suites from accepted
c.execute("""SELECT suite.id FROM suite""")
for s in c.fetchall():
c.execute(
"""INSERT INTO suite_queue_copy (suite, queue) VALUES (%s, (SELECT id FROM queue WHERE queue_name = 'accepted'))""",
s)
# Parse the config and add any buildd stuff
cnf = Config()
c.execute(
"""INSERT INTO queue (queue_name, path) VALUES ('buildd', '%s')"""
% cnf["Dir::QueueBuild"].rstrip('/'))
for s in cnf.value_list("Dinstall::QueueBuildSuites"):
c.execute("""INSERT INTO suite_queue_copy (suite, queue)
VALUES ( (SELECT id FROM suite WHERE suite_name = '%s'),
(SELECT id FROM queue WHERE queue_name = 'buildd'))"""
% s.lower())
print "Committing"
c.execute("UPDATE config SET value = '21' WHERE name = 'db_revision'")
self.db.commit()
except psycopg2.InternalError as msg:
self.db.rollback()
raise DBUpdateError(
"Unable to apply queue_build 21, rollback issued. Error message : %s"
% (str(msg)))
def main():
cnf = Config()
Arguments = [('h',"help","Import-LDAP-Fingerprints::Options::Help")]
for i in [ "help" ]:
if not cnf.has_key("Import-LDAP-Fingerprints::Options::%s" % (i)):
cnf["Import-LDAP-Fingerprints::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Import-LDAP-Fingerprints::Options")
if Options["Help"]:
usage()
session = DBConn().session()
LDAPDn = cnf["Import-LDAP-Fingerprints::LDAPDn"]
LDAPServer = cnf["Import-LDAP-Fingerprints::LDAPServer"]
l = ldap.open(LDAPServer)
l.simple_bind_s("","")
Attrs = l.search_s(LDAPDn, ldap.SCOPE_ONELEVEL,
"(&(keyfingerprint=*)(gidnumber=%s))" % (cnf["Import-Users-From-Passwd::ValidGID"]),
["uid", "keyfingerprint", "cn", "mn", "sn"])
# Our database session is already in a transaction
# Sync LDAP with DB
db_fin_uid = {}
db_uid_name = {}
ldap_fin_uid_id = {}
q = session.execute("""
SELECT f.fingerprint, f.id, u.uid FROM fingerprint f, uid u WHERE f.uid = u.id
UNION SELECT f.fingerprint, f.id, null FROM fingerprint f where f.uid is null""")
for i in q.fetchall():
(fingerprint, fingerprint_id, uid) = i
db_fin_uid[fingerprint] = (uid, fingerprint_id)
q = session.execute("SELECT id, name FROM uid")
for i in q.fetchall():
(uid, name) = i
db_uid_name[uid] = name
for i in Attrs:
entry = i[1]
fingerprints = entry["keyFingerPrint"]
uid_name = entry["uid"][0]
name = get_ldap_name(entry)
uid = get_or_set_uid(uid_name, session)
uid_id = uid.uid_id
if not db_uid_name.has_key(uid_id) or db_uid_name[uid_id] != name:
session.execute("UPDATE uid SET name = :name WHERE id = :uidid", {'name': name, 'uidid': uid_id})
print "Assigning name of %s as %s" % (uid_name, name)
for fingerprint in fingerprints:
ldap_fin_uid_id[fingerprint] = (uid_name, uid_id)
if db_fin_uid.has_key(fingerprint):
(existing_uid, fingerprint_id) = db_fin_uid[fingerprint]
if not existing_uid:
session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
{'uidid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid_name, fingerprint)
elif existing_uid == uid_name:
pass
elif '@' not in existing_uid:
session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
{'uidid': uid_id, 'fprid': fingerprint_id})
print "Promoting DM %s to DD %s with keyid 0x%s." % (existing_uid, uid_name, fingerprint)
else:
utils.warn("%s has %s in LDAP, but database says it should be %s." % \
(uid_name, fingerprint, existing_uid))
# Try to update people who sign with non-primary key
q = session.execute("SELECT fingerprint, id FROM fingerprint WHERE uid is null")
for i in q.fetchall():
(fingerprint, fingerprint_id) = i
cmd = "gpg --no-default-keyring %s --fingerprint %s" \
% (utils.gpg_keyring_args(), fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result == 0:
m = re_gpg_fingerprint.search(output)
if not m:
print output
utils.fubar("0x%s: No fingerprint found in gpg output but it returned 0?\n%s" % \
(fingerprint, utils.prefix_multi_line_string(output, " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ","")
if not ldap_fin_uid_id.has_key(primary_key):
utils.warn("0x%s (from 0x%s): no UID found in LDAP" % (primary_key, fingerprint))
else:
(uid, uid_id) = ldap_fin_uid_id[primary_key]
session.execute("UPDATE fingerprint SET uid = :uid WHERE id = :fprid",
{'uid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
extra_keyrings = ""
for keyring in cnf.value_list("Import-LDAP-Fingerprints::ExtraKeyrings"):
extra_keyrings += " --keyring=%s" % (keyring)
cmd = "gpg %s %s --list-key %s" \
% (utils.gpg_keyring_args(), extra_keyrings, fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
cmd = "gpg --keyserver=%s --allow-non-selfsigned-uid --recv-key %s" % (cnf["Import-LDAP-Fingerprints::KeyServer"], fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
print "0x%s: NOT found on keyserver." % (fingerprint)
print cmd
print result
print output
continue
else:
cmd = "gpg --list-key %s" % (fingerprint)
(result, output) = commands.getstatusoutput(cmd)
if result != 0:
print "0x%s: --list-key returned error after --recv-key didn't." % (fingerprint)
print cmd
print result
print output
continue
m = re_debian_address.search(output)
if m:
guess_uid = m.group(1)
else:
guess_uid = "???"
name = " ".join(output.split('\n')[0].split()[3:])
print "0x%s -> %s -> %s" % (fingerprint, name, guess_uid)
# FIXME: make me optionally non-interactive
# FIXME: default to the guessed ID
uid = None
while not uid:
uid = utils.our_raw_input("Map to which UID ? ")
Attrs = l.search_s(LDAPDn,ldap.SCOPE_ONELEVEL,"(uid=%s)" % (uid), ["cn","mn","sn"])
if not Attrs:
print "That UID doesn't exist in LDAP!"
uid = None
else:
entry = Attrs[0][1]
name = get_ldap_name(entry)
prompt = "Map to %s - %s (y/N) ? " % (uid, name.replace(" "," "))
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
uid_o = get_or_set_uid(uid, session=session)
uid_id = uid_o.uid_id
session.execute("UPDATE fingerprint SET uid = :uidid WHERE id = :fprid",
{'uidid': uid_id, 'fprid': fingerprint_id})
print "Assigning %s to 0x%s." % (uid, fingerprint)
else:
uid = None
# Commit it all
session.commit()
def action_dm(self, fingerprint, section, session):
cnf = Config()
if (
"Command::DM::AdminKeyrings" not in cnf
or "Command::DM::ACL" not in cnf
or "Command::DM::Keyrings" not in cnf
):
raise CommandError("DM command is not configured for this archive.")
allowed_keyrings = cnf.value_list("Command::DM::AdminKeyrings")
if fingerprint.keyring.keyring_name not in allowed_keyrings:
raise CommandError("Key {0} is not allowed to set DM".format(fingerprint.fingerprint))
acl_name = cnf.get("Command::DM::ACL", "dm")
acl = session.query(ACL).filter_by(name=acl_name).one()
fpr_hash = section["Fingerprint"].translate(None, " ")
fpr = session.query(Fingerprint).filter_by(fingerprint=fpr_hash).first()
if fpr is None:
raise CommandError("Unknown fingerprint {0}".format(fpr_hash))
if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list("Command::DM::Keyrings"):
raise CommandError("Key {0} is not in DM keyring.".format(fpr.fingerprint))
addresses = gpg_get_key_addresses(fpr.fingerprint)
if len(addresses) > 0:
self.cc.append(addresses[0])
self.log.log(["dm", "fingerprint", fpr.fingerprint])
self.result.append("Fingerprint: {0}".format(fpr.fingerprint))
if len(addresses) > 0:
self.log.log(["dm", "uid", addresses[0]])
self.result.append("Uid: {0}".format(addresses[0]))
for source in self._split_packages(section.get("Allow", "")):
# Check for existance of source package to catch typos
if session.query(DBSource).filter_by(source=source).first() is None:
raise CommandError("Tried to grant permissions for unknown source package: {0}".format(source))
if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None:
aps = ACLPerSource()
aps.acl = acl
aps.fingerprint = fpr
aps.source = source
aps.created_by = fingerprint
aps.reason = section.get("Reason")
session.add(aps)
self.log.log(["dm", "allow", fpr.fingerprint, source])
self.result.append("Allowed: {0}".format(source))
else:
self.result.append("Already-Allowed: {0}".format(source))
session.flush()
for source in self._split_packages(section.get("Deny", "")):
count = session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete()
if count == 0:
raise CommandError(
"Tried to remove upload permissions for package {0}, "
"but no upload permissions were granted before.".format(source)
)
self.log.log(["dm", "deny", fpr.fingerprint, source])
self.result.append("Denied: {0}".format(source))
session.commit()
