def source_by_metadata(key=None):
"""
Finds all Debian source packages which have the specified metadata set.
E.g., to find out the Maintainer of all source packages, query
/source/by_metadata/Maintainer.
@type key: string
@param key: Metadata key to search for.
@rtype: dictionary
@return: A list of dictionaries of
- source
- metadata value
"""
if not key:
return bottle.HTTPError(503, 'Metadata key not specified.')
s = DBConn().session()
q = s.query(DBSource.source, SourceMetadata.value)
q = q.join(SourceMetadata).join(MetadataKey)
q = q.filter(MetadataKey.key == key)
ret = []
for p in q:
ret.append({'source': p.source, 'metadata_value': p.value})
s.close()
return json.dumps(ret)
def source_by_metadata(key=None):
"""
Finds all Debian source packages which have the specified metadata set.
E.g., to find out the Maintainer of all source packages, query
/source/by_metadata/Maintainer.
@type key: string
@param key: Metadata key to search for.
@rtype: dictionary
@return: A list of dictionaries of
- source
- metadata value
"""
if not key:
return bottle.HTTPError(503, 'Metadata key not specified.')
s = DBConn().session()
q = s.query(DBSource.source, SourceMetadata.value)
q = q.join(SourceMetadata).join(MetadataKey)
q = q.filter(MetadataKey.key == key)
ret = []
for p in q:
ret.append({'source': p.source,
'metadata_value': p.value})
s.close()
return json.dumps(ret)
def binary_by_metadata(key=None):
"""
Finds all Debian binary packages which have the specified metadata set.
E.g., to find out the Go import paths of all Debian Go packages, query
/binary/by_metadata/Go-Import-Path.
@type key: string
@param key: Metadata key to search for.
@rtype: dictionary
@return: A list of dictionaries of
- binary
- source
- metadata value
"""
if not key:
return bottle.HTTPError(503, 'Metadata key not specified.')
s = DBConn().session()
q = s.query(DBBinary.package, DBSource.source, SourceMetadata.value)
q = q.join(DBSource).join(SourceMetadata).join(MetadataKey)
q = q.filter(MetadataKey.key == key)
q = q.group_by(DBBinary.package, DBSource.source, SourceMetadata.value)
ret = []
for p in q:
ret.append({'binary': p.package,
'source': p.source,
'metadata_value': p.value})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def sources_in_suite(suite=None):
"""
Returns all source packages and their versions in a given suite.
@since: December 2014
@type suite: string
@param suite: Name of the suite.
@see: L{I{suites}<dakweb.queries.suite.suites>} on how to receive a list of valid suites.
@rtype: list of dictionaries
@return: Dictionaries made out of
- source
- version
"""
if suite is None:
return bottle.HTTPError(503, 'Suite not specified.')
s = DBConn().session()
q = s.query(DBSource).join(Suite, DBSource.suites)
q = q.filter(or_(Suite.suite_name == suite, Suite.codename == suite))
ret = []
for p in q:
ret.append({'source': p.source,
'version': p.version})
s.close()
return json.dumps(ret)
def suites():
"""
Give information about all known suites.
@maps: name maps to Suite: in the release file
@maps: codename maps to Codename: in the release file.
@maps: dakname is an internal name and should not be relied upon.
@rtype: list of dictionaries
@return: Dictionaries made out of
- name
- codename
- dakname
- archive
- architectures
- components
"""
s = DBConn().session()
q = s.query(Suite)
q = q.order_by(Suite.suite_name)
ret = []
for p in q:
ret.append({'name': p.release_suite_output,
'codename': p.codename,
'dakname': p.suite_name,
'archive': p.archive.archive_name,
'architectures': [x.arch_string for x in p.architectures],
'components': [x.component_name for x in p.components]})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def sources_in_suite(suite=None):
"""
Returns all source packages and their versions in a given suite.
@since: December 2014
@type suite: string
@param suite: Name of the suite.
@see: L{I{suites}<dakweb.queries.suite.suites>} on how to receive a list of valid suites.
@rtype: list of dictionaries
@return: Dictionaries made out of
- source
- version
"""
if suite is None:
return bottle.HTTPError(503, 'Suite not specified.')
s = DBConn().session()
q = s.query(DBSource).join(Suite, DBSource.suites)
q = q.filter(or_(Suite.suite_name == suite, Suite.codename == suite))
ret = []
for p in q:
ret.append({'source': p.source,
'version': p.version})
s.close()
return json.dumps(ret)
def sha256sum_in_archive(sha256sum=None):
"""
Check if files with matching sha256sums are known to the archive.
@since: June 2018
@type sha256sum: string
@param sha256sum: SHA256 sum of the file.
@rtype: list of dictionaries
@return: Dictionaries made out of
- filename
- sha256sum
- component
"""
if sha256sum is None:
return bottle.HTTPError(503, 'sha256sum not specified.')
s = DBConn().session()
q = s.query(PoolFile)
q = q.filter(PoolFile.sha256sum == sha256sum)
ret = []
for p in q:
ret.append({'filename': p.filename,
'component': p.component.component_name,
'sha256sum': p.sha256sum})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def read_number():
session = DBConn().session()
result = session.query('foo').from_statement(
sql.text('select 7 as foo')).scalar()
sleep(0.1)
session.close()
return result
def export_external_signature_requests(session, path):
tbl_arch = DBConn().tbl_architecture
tbl_ba = DBConn().tbl_bin_associations
tbl_bin = DBConn().tbl_binaries
tbl_esr = DBConn().tbl_external_signature_requests
tbl_suite = DBConn().tbl_suite
query = sql.select([tbl_bin.c.package, tbl_suite.c.suite_name, tbl_suite.c.codename, tbl_arch.c.arch_string, sql.func.max(tbl_bin.c.version)]) \
.select_from(tbl_esr.join(tbl_suite).join(tbl_ba, tbl_ba.c.id == tbl_esr.c.association_id).join(tbl_bin).join(tbl_arch)) \
.group_by(tbl_bin.c.package, tbl_suite.c.suite_name, tbl_suite.c.codename, tbl_arch.c.arch_string)
requests = session.execute(query)
data = {
'packages': [
{
'package': row[0],
'suite': row[1],
'codename': row[2],
'architecture': row[3],
'version': row[4],
}
for row in requests],
}
with open(path, 'w') as fh:
json.dump(data, fh, indent=2)
def sha256sum_in_archive(sha256sum=None):
"""
Check if files with matching sha256sums are known to the archive.
@since: June 2018
@type sha256sum: string
@param sha256sum: SHA256 sum of the file.
@rtype: list of dictionaries
@return: Dictionaries made out of
- filename
- sha256sum
- component
"""
if sha256sum is None:
return bottle.HTTPError(503, 'sha256sum not specified.')
s = DBConn().session()
q = s.query(PoolFile)
q = q.filter(PoolFile.sha256sum == sha256sum)
ret = []
for p in q:
ret.append({'filename': p.filename,
'component': p.component.component_name,
'sha256sum': p.sha256sum})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def binary_by_metadata(key=None):
"""
Finds all Debian binary packages which have the specified metadata set.
E.g., to find out the Go import paths of all Debian Go packages, query
/binary/by_metadata/Go-Import-Path.
@type key: string
@param key: Metadata key to search for.
@rtype: dictionary
@return: A list of dictionaries of
- binary
- source
- metadata value
"""
if not key:
return bottle.HTTPError(503, 'Metadata key not specified.')
s = DBConn().session()
q = s.query(DBBinary.package, DBSource.source, SourceMetadata.value)
q = q.join(DBSource).join(SourceMetadata).join(MetadataKey)
q = q.filter(MetadataKey.key == key)
q = q.group_by(DBBinary.package, DBSource.source, SourceMetadata.value)
ret = []
for p in q:
ret.append({'binary': p.package,
'source': p.source,
'metadata_value': p.value})
s.close()
return json.dumps(ret)
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h',"help","Queue-Report::Options::Help"),
('n',"new","Queue-Report::Options::New"),
('8','822',"Queue-Report::Options::822"),
('s',"sort","Queue-Report::Options::Sort", "HasArg"),
('a',"age","Queue-Report::Options::Age", "HasArg"),
('r',"rrd","Queue-Report::Options::Rrd", "HasArg"),
('d',"directories","Queue-Report::Options::Directories", "HasArg")]
for i in [ "help" ]:
if not Cnf.has_key("Queue-Report::Options::%s" % (i)):
Cnf["Queue-Report::Options::%s" % (i)] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if Cnf.has_key("Queue-Report::Options::New"):
header()
queue_names = []
if Cnf.has_key("Queue-Report::Options::Directories"):
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif Cnf.has_key("Queue-Report::Directories"):
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = [ "byhand", "new" ]
if Cnf.has_key("Queue-Report::Options::Rrd"):
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif Cnf.has_key("Dir::Rrd"):
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if Cnf.has_key("Queue-Report::Options::822"):
# Open the report file
f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w")
session = DBConn().session()
for queue_name in queue_names:
queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first()
if queue is not None:
process_queue(queue, f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if Cnf.has_key("Queue-Report::Options::822"):
f.close()
if Cnf.has_key("Queue-Report::Options::New"):
footer()
def file_in_archive(filepattern=None):
"""
Check if a file pattern is known to the archive. Note that the
patterns are matched against the location of the files in the
pool, so for %tmux_2.3-1.dsc it will return t/tmux/tmux_2.3-1.dsc
as filename.
@since: October 2016
@type filepattern: string
@param filepattern: Pattern of the filenames to match. SQL LIKE
statement wildcard matches are supported, that
is % for zero, one or more characters, _ for a
single character match.
@rtype: Dictionary, empty if nothing matched.
@return: A dictionary of
- filename
- sha256sum
"""
if filepattern is None:
return bottle.HTTPError(503, 'Filepattern not specified.')
s = DBConn().session()
q = s.query(PoolFile)
q = q.filter(PoolFile.filename.like(filepattern))
ret = []
for p in q:
ret.append({'filename': p.filename, 'sha256sum': p.sha256sum})
s.close()
return json.dumps(ret)
def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
query = '''SELECT source
FROM source_suite
WHERE source = :source
AND suite_name IN ('unstable', 'experimental')'''
if not session.execute(query, {'source': source}).rowcount:
trclass += " sourceNEW"
session.commit()
if row_number % 2 != 0:
print("<tr class=\"%s even\">" % (trclass))
else:
print("<tr class=\"%s odd\">" % (trclass))
if "sourceNEW" in trclass:
print("<td class=\"package\">%s</td>" % (source))
else:
print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
print("<td class=\"version\">")
for vers in version.split():
print("<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers)))
print("</td>")
print("<td class=\"arch\">%s</td>" % (arch))
print("<td class=\"distribution\">")
for dist in distribution:
print("%s<br/>" % (dist))
print("</td>")
print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
time_pp(last_mod),
))
(name, mail) = maint.split(":", 1)
print("<td class=\"upload-data\">")
print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
(name, mail) = changedby.split(":", 1)
print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
if sponsor:
print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(sponsor), utils.html_escape(sponsor)))
print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
print("</td>")
print("<td class=\"closes\">")
for close in closes:
print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close)))
print("</td></tr>")
row_number += 1
def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
query = '''SELECT source
FROM source_suite
WHERE source = :source
AND suite_name IN ('unstable', 'experimental')'''
if not session.execute(query, {'source': source}).rowcount:
trclass += " sourceNEW"
session.commit()
if row_number % 2 != 0:
print("<tr class=\"%s even\">" % (trclass))
else:
print("<tr class=\"%s odd\">" % (trclass))
if "sourceNEW" in trclass:
print("<td class=\"package\">%s</td>" % (source))
else:
print("<td class=\"package\"><a href=\"https://tracker.debian.org/pkg/%(source)s\">%(source)s</a></td>" % {'source': source})
print("<td class=\"version\">")
for vers in version.split():
print("<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers)))
print("</td>")
print("<td class=\"arch\">%s</td>" % (arch))
print("<td class=\"distribution\">")
for dist in distribution:
print("%s<br/>" % (dist))
print("</td>")
print("<td class=\"age\"><abbr title=\"%s\">%s</abbr></td>" % (
datetime.datetime.utcfromtimestamp(int(time.time()) - last_mod).strftime('%a, %d %b %Y %T UTC'),
time_pp(last_mod),
))
(name, mail) = maint.split(":", 1)
print("<td class=\"upload-data\">")
print("<span class=\"maintainer\">Maintainer: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
(name, mail) = changedby.split(":", 1)
print("<span class=\"changed-by\">Changed-By: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name)))
if sponsor:
print("<span class=\"sponsor\">Sponsor: <a href=\"https://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(sponsor), utils.html_escape(sponsor)))
print("<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint))
print("</td>")
print("<td class=\"closes\">")
for close in closes:
print("<a href=\"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close)))
print("</td></tr>")
row_number += 1
def table_row(source, version, arch, last_mod, maint, distribution, closes, fingerprint, sponsor, changedby):
global row_number
trclass = "sid"
session = DBConn().session()
for dist in distribution:
if dist == "experimental":
trclass = "exp"
if not len(session.query(DBSource).filter_by(source = source).all()):
trclass += " binNEW"
session.commit()
if row_number % 2 != 0:
print "<tr class=\"%s even\">" % (trclass)
else:
print "<tr class=\"%s odd\">" % (trclass)
if "binNEW" in trclass:
print "<td class=\"package\">%s</td>" % (source)
else:
print "<td class=\"package\"><a href=\"http://packages.qa.debian.org/%(source)s\">%(source)s</a></td>" % {'source': source}
print "<td class=\"version\">"
for vers in version.split():
print "<a href=\"new/%s_%s.html\">%s</a><br/>" % (source, utils.html_escape(vers), utils.html_escape(vers))
print "</td>"
print "<td class=\"arch\">%s</td>" % (arch)
print "<td class=\"distribution\">"
for dist in distribution:
print "%s<br/>" % (dist)
print "</td>"
print "<td class=\"age\">%s</td>" % (last_mod)
(name, mail) = maint.split(":", 1)
print "<td class=\"upload-data\">"
print "<span class=\"maintainer\">Maintainer: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
(name, mail) = changedby.split(":", 1)
print "<span class=\"changed-by\">Changed-By: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a></span><br/>" % (utils.html_escape(mail), utils.html_escape(name))
if sponsor:
try:
(login, domain) = sponsor.split("@", 1)
print "<span class=\"sponsor\">Sponsor: <a href=\"http://qa.debian.org/developer.php?login=%s\">%s</a>@debian.org</span><br/>" % (utils.html_escape(login), utils.html_escape(login))
except Exception as e:
pass
print "<span class=\"signature\">Fingerprint: %s</span>" % (fingerprint)
print "</td>"
print "<td class=\"closes\">"
for close in closes:
print "<a href=\"http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s\">#%s</a><br/>" % (utils.html_escape(close), utils.html_escape(close))
print "</td></tr>"
row_number+=1
def per_arch_space_use():
session = DBConn().session()
q = session.execute("""
SELECT a.arch_string as Architecture, sum(f.size) AS sum
FROM files f, binaries b, architecture a
WHERE a.id=b.architecture AND f.id=b.file
GROUP BY a.arch_string ORDER BY sum""").fetchall()
for j in q:
print "%-15.15s %s" % (j[0], j[1])
print
q = session.execute("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'").fetchall()
print "%-15.15s %s" % ("Source", q[0][0])
def per_arch_space_use():
session = DBConn().session()
q = session.execute("""
SELECT a.arch_string as Architecture, sum(f.size) AS sum
FROM files f, binaries b, architecture a
WHERE a.id=b.architecture AND f.id=b.file
GROUP BY a.arch_string ORDER BY sum""").fetchall()
for j in q:
print("%-15.15s %s" % (j[0], j[1]))
print()
q = session.execute("SELECT sum(size) FROM files WHERE filename ~ '.(diff.gz|tar.gz|dsc)$'").fetchall()
print("%-15.15s %s" % ("Source", q[0][0]))
def generate_packages(suite_id, component_id, architecture_id, type_name):
global _packages_query
from daklib.filewriter import PackagesFileWriter
from daklib.dbconn import Architecture, Component, DBConn, OverrideType, Suite
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
arch_all_id = session.query(Architecture).filter_by(arch_string='all').one().arch_id
type_id = session.query(OverrideType).filter_by(overridetype=type_name).one().overridetype_id
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
architecture = session.query(Architecture).get(architecture_id)
overridesuite_id = suite.get_overridesuite().suite_id
include_long_description = suite.include_long_description
# We currently filter out the "Tag" line. They are set by external
# overrides and NOT by the maintainer. And actually having it set by
# maintainer means we output it twice at the moment -> which breaks
# dselect.
metadata_skip = ["Section", "Priority", "Tag"]
if include_long_description:
metadata_skip.append("Description-md5")
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name,
'architecture': architecture.arch_string,
'debtype': type_name
}
if suite.indices_compression is not None:
writer_args['compression'] = suite.indices_compression
writer = PackagesFileWriter(**writer_args)
output = writer.open()
r = session.execute(_packages_query, {"archive_id": suite.archive.archive_id,
"suite": suite_id, "component": component_id, 'component_name': component.component_name,
"arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id,
"overridesuite": overridesuite_id, "metadata_skip": metadata_skip,
"include_long_description": 'true' if include_long_description else 'false'})
for (stanza,) in r:
print(stanza, file=output)
print("", file=output)
writer.close()
message = ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
def acl_export_per_source(acl_name):
session = DBConn().session()
acl = session.query(ACL).filter_by(name=acl_name).one()
query = r"""
SELECT
f.fingerprint,
(SELECT COALESCE(u.name, '') || ' <' || u.uid || '>'
FROM uid u
JOIN fingerprint f2 ON u.id = f2.uid
WHERE f2.id = f.id) AS name,
STRING_AGG(
a.source
|| COALESCE(' (' || (SELECT fingerprint FROM fingerprint WHERE id = a.created_by_id) || ')', ''),
E',\n ' ORDER BY a.source)
FROM acl_per_source a
JOIN fingerprint f ON a.fingerprint_id = f.id
LEFT JOIN uid u ON f.uid = u.id
WHERE a.acl_id = :acl_id
GROUP BY f.id, f.fingerprint
ORDER BY name
"""
for row in session.execute(query, {'acl_id': acl.id}):
print("Fingerprint:", row[0])
print("Uid:", row[1])
print("Allow:", row[2])
print()
session.rollback()
session.close()
def generate_sources(suite_id, component_id):
global _sources_query
from daklib.filewriter import SourcesFileWriter
from daklib.dbconn import Component, DBConn, OverrideType, Suite
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
dsc_type = session.query(OverrideType).filter_by(overridetype='dsc').one().overridetype_id
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
overridesuite_id = suite.get_overridesuite().suite_id
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name
}
if suite.indices_compression is not None:
writer_args['compression'] = suite.indices_compression
writer = SourcesFileWriter(**writer_args)
output = writer.open()
# run query and write Sources
r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "component_name": component.component_name, "dsc_type": dsc_type, "overridesuite": overridesuite_id})
for (stanza,) in r:
print(stanza, file=output)
print("", file=output)
writer.close()
message = ["generate sources", suite.suite_name, component.component_name]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
def main():
global Options, Logger
cnf = Config()
session = DBConn().session()
Arguments = [('h', "help", "Archive-Dedup-Pool::Options::Help")]
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
for i in ["help"]:
key = "Archive-Dedup-Pool::Options::%s" % i
if key not in cnf:
cnf[key] = ""
Options = cnf.subtree("Archive-Dedup-Pool::Options")
if Options["Help"]:
usage()
Logger = daklog.Logger("archive-dedup-pool")
dedup(session)
Logger.close()
def acl_deny(acl_name, fingerprint, sources):
tbl = DBConn().tbl_acl_per_source
session = DBConn().session()
acl_id = session.query(ACL).filter_by(name=acl_name).one().id
fingerprint_id = session.query(Fingerprint).filter_by(fingerprint=fingerprint).one().fingerprint_id
# TODO: check that fpr is in ACL
for source in sources:
result = session.execute(tbl.delete().where(tbl.c.acl_id == acl_id).where(tbl.c.fingerprint_id == fingerprint_id).where(tbl.c.source == source))
if result.rowcount < 1:
print("W: Tried to deny uploads of '{}', but was not allowed before.".format(source))
session.commit()
def binary_metadata_keys():
"""
List all possible metadata keys
@rtype: dictionary
@return: A list of metadata keys
"""
s = DBConn().session()
q = s.query(MetadataKey)
ret = []
for p in q:
ret.append( p.key)
s.close()
return json.dumps(ret)
def test_clone(self):
'''
Tests the ORMObject.clone() method.
'''
uid1 = Uid(uid='foobar')
# no session yet
self.assertRaises(RuntimeError, uid1.clone)
self.session.add(uid1)
# object not persistent yet
self.assertRaises(RuntimeError, uid1.clone)
self.session.commit()
# test without session parameter
uid2 = uid1.clone()
self.assertTrue(uid1 is not uid2)
self.assertEqual(uid1.uid, uid2.uid)
self.assertTrue(uid2 not in uid1.session())
self.assertTrue(uid1 not in uid2.session())
# test with explicit session parameter
new_session = DBConn().session()
uid3 = uid1.clone(session=new_session)
self.assertEqual(uid1.uid, uid3.uid)
self.assertTrue(uid3 in new_session)
# test for ressource leaks with mass cloning
for _ in xrange(1, 1000):
uid1.clone()
def binary_metadata_keys():
"""
List all possible metadata keys
@rtype: dictionary
@return: A list of metadata keys
"""
s = DBConn().session()
q = s.query(MetadataKey)
ret = []
for p in q:
ret.append(p.key)
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def dsc_in_suite(suite=None, source=None):
"""
Find all dsc files for a given source package name in a given suite.
@since: December 2014
@type suite: string
@param suite: Name of the suite.
@see: L{I{suites}<dakweb.queries.suite.suites>} on how to receive a list of valid suites.
@type source: string
@param source: Source package to query for.
@rtype: list of dictionaries
@return: Dictionaries made out of
- version
- component
- filename
- filesize
- sha256sum
"""
if suite is None:
return bottle.HTTPError(503, 'Suite not specified.')
if source is None:
return bottle.HTTPError(503, 'Source package not specified.')
s = DBConn().session()
q = s.query(DSCFile).join(PoolFile)
q = q.join(DBSource).join(Suite, DBSource.suites)
q = q.filter(or_(Suite.suite_name == suite, Suite.codename == suite))
q = q.filter(DBSource.source == source)
q = q.filter(PoolFile.filename.endswith('.dsc'))
ret = []
for p in q:
ret.append({
'version': p.source.version,
'component': p.poolfile.component.component_name,
'filename': p.poolfile.filename,
'filesize': p.poolfile.filesize,
'sha256sum': p.poolfile.sha256sum
})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def generate_packages(suite_id, component_id, architecture_id, type_name):
global _packages_query
from daklib.filewriter import PackagesFileWriter
from daklib.dbconn import Architecture, Component, DBConn, OverrideType, Suite
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
arch_all_id = session.query(Architecture).filter_by(arch_string='all').one().arch_id
type_id = session.query(OverrideType).filter_by(overridetype=type_name).one().overridetype_id
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
architecture = session.query(Architecture).get(architecture_id)
overridesuite_id = suite.get_overridesuite().suite_id
include_long_description = suite.include_long_description
# We currently filter out the "Tag" line. They are set by external
# overrides and NOT by the maintainer. And actually having it set by
# maintainer means we output it twice at the moment -> which breaks
# dselect.
metadata_skip = ["Section", "Priority", "Tag"]
if include_long_description:
metadata_skip.append("Description-md5")
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name,
'architecture': architecture.arch_string,
'debtype': type_name
}
if suite.indices_compression is not None:
writer_args['compression'] = suite.indices_compression
writer = PackagesFileWriter(**writer_args)
output = writer.open()
r = session.execute(_packages_query, {"archive_id": suite.archive.archive_id,
"suite": suite_id, "component": component_id, 'component_name': component.component_name,
"arch": architecture_id, "type_id": type_id, "type_name": type_name, "arch_all": arch_all_id,
"overridesuite": overridesuite_id, "metadata_skip": metadata_skip,
"include_long_description": 'true' if include_long_description else 'false'})
for (stanza,) in r:
print >>output, stanza
print >>output, ""
writer.close()
message = ["generate-packages", suite.suite_name, component.component_name, architecture.arch_string]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
def binary_metadata_keys():
"""
List all possible metadata keys
@rtype: dictionary
@return: A list of metadata keys
"""
s = DBConn().session()
q = s.query(MetadataKey)
ret = []
for p in q:
ret.append(p.key)
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def acl_set_fingerprints(acl_name, entries):
session = DBConn().session()
acl = session.query(ACL).filter_by(name=acl_name).one()
acl.fingerprints.clear()
for entry in entries:
entry = entry.strip()
if entry.startswith('#') or len(entry) == 0:
continue
fps = get_fingerprint(entry, session)
if len(fps) == 0:
print("Unknown key for '{0}'".format(entry))
else:
acl.fingerprints.update(fps)
session.commit()
def generate_translations(suite_id, component_id):
global _translations_query
from daklib.filewriter import TranslationFileWriter
from daklib.dbconn import DBConn, Suite, Component
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name,
'language': 'en',
}
if suite.i18n_compression is not None:
writer_args['compression'] = suite.i18n_compression
writer = TranslationFileWriter(**writer_args)
output = writer.open()
r = session.execute(_translations_query, {
"suite": suite_id,
"component": component_id
})
for (stanza, ) in r:
print >> output, stanza
writer.close()
message = [
"generate-translations", suite.suite_name, component.component_name
]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
def generate_sources(suite_id, component_id):
global _sources_query
from daklib.filewriter import SourcesFileWriter
from daklib.dbconn import Component, DBConn, OverrideType, Suite
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
dsc_type = session.query(OverrideType).filter_by(overridetype='dsc').one().overridetype_id
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
overridesuite_id = suite.get_overridesuite().suite_id
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name
}
if suite.indices_compression is not None:
writer_args['compression'] = suite.indices_compression
writer = SourcesFileWriter(**writer_args)
output = writer.open()
# run query and write Sources
r = session.execute(_sources_query, {"suite": suite_id, "component": component_id, "component_name": component.component_name, "dsc_type": dsc_type, "overridesuite": overridesuite_id})
for (stanza,) in r:
print >>output, stanza
print >>output, ""
writer.close()
message = ["generate sources", suite.suite_name, component.component_name]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
class ImportThread(threading.Thread):
def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.queue = queue
self.session = DBConn().session()
self.parent = parent
self.die = False
def plsDie(self):
self.die = True
def run(self):
while True:
try:
if self.die:
return
to_import = self.queue.dequeue()
if not to_import:
return
print( "Directory %s, file %7d, (%s)" % (to_import.dirpath[-10:], to_import.count, to_import.changesfile) )
changes = Changes()
changes.changes_file = to_import.changesfile
changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
changes.changes = parse_changes(changesfile, signing_rules=-1)
changes.changes["fingerprint"] = check_signature(changesfile)
changes.add_known_changes(to_import.dirpath, session=self.session)
self.session.commit()
except InvalidDscError as line:
warn("syntax error in .dsc file '%s', line %s." % (f, line))
except ChangesUnicodeError:
warn("found invalid changes file, not properly utf-8 encoded")
except KeyboardInterrupt:
print("Caught C-c; on ImportThread. terminating.")
self.parent.plsDie()
sys.exit(1)
except:
self.parent.plsDie()
sys.exit(1)
def dsc_in_suite(suite=None, source=None):
"""
Find all dsc files for a given source package name in a given suite.
@since: December 2014
@type suite: string
@param suite: Name of the suite.
@see: L{I{suites}<dakweb.queries.suite.suites>} on how to receive a list of valid suites.
@type source: string
@param source: Source package to query for.
@rtype: list of dictionaries
@return: Dictionaries made out of
- version
- component
- filename
- filesize
- sha256sum
"""
if suite is None:
return bottle.HTTPError(503, 'Suite not specified.')
if source is None:
return bottle.HTTPError(503, 'Source package not specified.')
s = DBConn().session()
q = s.query(DSCFile).join(PoolFile)
q = q.join(DBSource).join(Suite, DBSource.suites)
q = q.filter(or_(Suite.suite_name == suite, Suite.codename == suite))
q = q.filter(DBSource.source == source)
q = q.filter(PoolFile.filename.endswith('.dsc'))
ret = []
for p in q:
ret.append({'version': p.source.version,
'component': p.poolfile.component.component_name,
'filename': p.poolfile.filename,
'filesize': p.poolfile.filesize,
'sha256sum': p.poolfile.sha256sum})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def main():
cnf = Config()
Arguments = [('h',"help","DEP11::Options::Help"),
('s',"suite","DEP11::Options::Suite", "HasArg"),
('e',"expire","DEP11::Options::ExpireCache"),
('h',"write-hints","DEP11::Options::WriteHints"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
sys.exit(1)
# check if we have some important config options set
if not cnf.has_key("Dir::MetaInfo"):
print("You need to specify a metadata export directory (Dir::MetaInfo)")
sys.exit(1)
if not cnf.has_key("DEP11::Url"):
print("You need to specify a metadata public web URL (DEP11::Url)")
sys.exit(1)
if not cnf.has_key("DEP11::IconSizes"):
print("You need to specify a list of allowed icon-sizes (DEP11::IconSizes)")
sys.exit(1)
if Options["WriteHints"] and not cnf.has_key("Dir::MetaInfoHints"):
print("You need to specify an export directory for DEP-11 hints files (Dir::MetaInfoHints)")
sys.exit(1)
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename, logger)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(session, suite, logger)
if Options["WriteHints"]:
write_hints_files(session, suite, logger)
# we're done
logger.close()
def archives():
"""
Give information about all known archives (sets of suites)
@rtype: dict
return: list of dictionaries
"""
s = DBConn().session()
q = s.query(Archive)
q = q.order_by(Archive.archive_name)
ret = []
for a in q:
ret.append({'name': a.archive_name,
'suites': [x.suite_name for x in a.suites]})
s.close()
return json.dumps(ret)
def add_external_signature_request(session, target_suite, suite, binary):
tbl_ba = DBConn().tbl_bin_associations
tbl_esr = DBConn().tbl_external_signature_requests
# TODO [sqlalchemy >= 1.1]: use `ON CONFLICT DO NOTHING`
#select = sql.select([tbl_ba.c.id, target_suite.suite_id]).where((tbl_ba.c.suite == suite.suite_id) & (tbl_ba.c.bin == binary.binary_id))
#insert = pgsql.insert(tbl_esr).from_select([tbl_esr.c.association_id, tbl_esr.c.suite_id], select).on_conflict_do_nothing()
#session.execute(insert)
ba_id = session.execute(
sql.select([tbl_ba.c.id
]).where((tbl_ba.c.suite == suite.suite_id)
& (tbl_ba.c.bin == binary.binary_id))).scalar()
exists = session.execute(
sql.select([tbl_esr]).where(tbl_esr.c.association_id == ba_id).where(
tbl_esr.c.suite_id == target_suite.suite_id)).first()
if exists is None:
insert = sql.insert(tbl_esr).values(association_id=ba_id,
suite_id=target_suite.suite_id)
session.execute(insert)
def all_sources():
"""
Returns all source packages and their versions known to the archive
(this includes NEW).
@rtype: list of dictionaries
@return: Dictionaries made out of
- source
- version
"""
s = DBConn().session()
q = s.query(DBSource)
ret = []
for p in q:
ret.append({'source': p.source, 'version': p.version})
s.close()
return json.dumps(ret)
def archives():
"""
Give information about all known archives (sets of suites)
@rtype: dict
return: list of dictionaries
"""
s = DBConn().session()
q = s.query(Archive)
q = q.order_by(Archive.archive_name)
ret = []
for a in q:
ret.append({
'name': a.archive_name,
'suites': [x.suite_name for x in a.suites]
})
s.close()
return json.dumps(ret)
def all_sources():
"""
Returns all source packages and their versions known to the archive
(this includes NEW).
@rtype: list of dictionaries
@return: Dictionaries made out of
- source
- version
"""
s = DBConn().session()
q = s.query(DBSource)
ret = []
for p in q:
ret.append({'source': p.source,
'version': p.version})
s.close()
return json.dumps(ret)
def get_db_value(name, default=None, rettype=None):
from daklib.dbconn import DBConfig, DBConn, NoResultFound
try:
res = DBConn().session().query(DBConfig).filter(
DBConfig.name == name).one()
except NoResultFound:
return default
if rettype:
return rettype(res.value)
else:
return res.value
def generate_translations(suite_id, component_id):
global _translations_query
from daklib.filewriter import TranslationFileWriter
from daklib.dbconn import DBConn, Suite, Component
from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
session = DBConn().session()
suite = session.query(Suite).get(suite_id)
component = session.query(Component).get(component_id)
writer_args = {
'archive': suite.archive.path,
'suite': suite.suite_name,
'component': component.component_name,
'language': 'en',
}
if suite.i18n_compression is not None:
writer_args['compression'] = suite.i18n_compression
writer = TranslationFileWriter(**writer_args)
output = writer.open()
r = session.execute(_translations_query, {"suite": suite_id, "component": component_id})
for (stanza,) in r:
print >>output, stanza
writer.close()
message = ["generate-translations", suite.suite_name, component.component_name]
session.rollback()
return (PROC_STATUS_SUCCESS, message)
def file_in_archive(filepattern=None):
"""
Check if a file pattern is known to the archive. Note that the
patterns are matched against the location of the files in the
pool, so for %tmux_2.3-1.dsc it will return t/tmux/tmux_2.3-1.dsc
as filename.
@since: October 2016
@type filepattern: string
@param filepattern: Pattern of the filenames to match. SQL LIKE
statement wildcard matches are supported, that
is % for zero, one or more characters, _ for a
single character match.
@rtype: list of dictionaries
@return: Dictionaries made out of
- filename
- sha256sum
- component
"""
if filepattern is None:
return bottle.HTTPError(503, 'Filepattern not specified.')
s = DBConn().session()
q = s.query(PoolFile)
q = q.filter(PoolFile.filename.like(filepattern))
ret = []
for p in q:
ret.append({'filename': p.filename,
'component': p.component.component_name,
'sha256sum': p.sha256sum})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def get_provides(suite):
provides = set()
session = DBConn().session()
query = '''SELECT DISTINCT value
FROM binaries_metadata m
JOIN bin_associations b
ON b.bin = m.bin_id
WHERE key_id = (
SELECT key_id
FROM metadata_keys
WHERE key = 'Provides' )
AND b.suite = (
SELECT id
FROM suite
WHERE suite_name = '%(suite)s'
OR codename = '%(suite)s')''' % \
{'suite': suite}
for p in session.execute(query):
for e in p:
for i in e.split(','):
provides.add(i.strip())
session.close()
return provides
def suites():
"""
Give information about all known suites.
@maps: name maps to Suite: in the release file
@maps: codename maps to Codename: in the release file.
@maps: dakname is an internal name and should not be relied upon.
@rtype: list of dictionaries
@return: Dictionaries made out of
- name
- codename
- dakname
- archive
- architectures
- components
"""
s = DBConn().session()
q = s.query(Suite)
q = q.order_by(Suite.suite_name)
ret = []
for p in q:
ret.append({
'name': p.release_suite_output,
'codename': p.codename,
'dakname': p.suite_name,
'archive': p.archive.archive_name,
'architectures': [x.arch_string for x in p.architectures],
'components': [x.component_name for x in p.components]
})
s.close()
bottle.response.content_type = 'application/json; charset=UTF-8'
return json.dumps(ret)
def acl_allow(acl_name, fingerprint, sources):
tbl = DBConn().tbl_acl_per_source
session = DBConn().session()
acl_id = session.query(ACL).filter_by(name=acl_name).one().id
fingerprint_id = session.query(Fingerprint).filter_by(fingerprint=fingerprint).one().fingerprint_id
# TODO: check that fpr is in ACL
data = [
{
'acl_id': acl_id,
'fingerprint_id': fingerprint_id,
'source': source,
'reason': 'set by {} via CLI'.format(os.environ.get('USER', '(unknown)')),
}
for source in sources
]
session.execute(tbl.insert(), data)
session.commit()
def main():
cnf = Config()
Arguments = [
('h', "help", "DEP11::Options::Help"),
('e', "expire", "DEP11::Options::ExpireCache"),
('s', "suite", "DEP11::Options::Suite", "HasArg"),
]
for i in ["help", "suite", "ExpireCache"]:
if not cnf.has_key("DEP11::Options::%s" % (i)):
cnf["DEP11::Options::%s" % (i)] = ""
arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("DEP11::Options")
if Options["Help"]:
usage()
return
suitename = Options["Suite"]
if not suitename:
print("You need to specify a suite!")
return
logger = daklog.Logger('generate-metadata')
from daklib.dbconn import Component, DBConn, get_suite, Suite
session = DBConn().session()
suite = get_suite(suitename.lower(), session)
if Options["ExpireCache"]:
expire_dep11_data_cache(session, suitename)
process_suite(session, suite, logger)
# export database content as Components-<arch>.xz YAML documents
write_component_files(suite)
# we're done
logger.close()
def export_external_signature_requests(session, path):
tbl_arch = DBConn().tbl_architecture
tbl_ba = DBConn().tbl_bin_associations
tbl_bin = DBConn().tbl_binaries
tbl_esr = DBConn().tbl_external_signature_requests
tbl_suite = DBConn().tbl_suite
query = sql.select([tbl_bin.c.package, tbl_suite.c.suite_name, tbl_arch.c.arch_string, sql.func.max(tbl_bin.c.version)]) \
.select_from(tbl_esr.join(tbl_suite).join(tbl_ba, tbl_ba.c.id == tbl_esr.c.association_id).join(tbl_bin).join(tbl_arch)) \
.group_by(tbl_bin.c.package, tbl_suite.c.suite_name, tbl_arch.c.arch_string)
requests = session.execute(query)
data = {
'packages': [{
'package': row[0],
'suite': row[1],
'architecture': row[2],
'version': row[3],
} for row in requests],
}
with open(path, 'w') as fh:
json.dump(data, fh, indent=2)
def list_packages(packages, suites=None, components=None, architectures=None, binary_types=None,
source_and_binary=False, regex=False,
format=None, highest=None):
session = DBConn().session()
try:
t = DBConn().view_package_list
comparison_operator = "~" if regex else "="
where = sql.false()
for package in packages:
where = where | t.c.package.op(comparison_operator)(package)
if source_and_binary:
where = where | t.c.source.op(comparison_operator)(package)
if suites is not None:
where = where & (t.c.suite.in_(suites) | t.c.codename.in_(suites))
if components is not None:
where = where & t.c.component.in_(components)
if architectures is not None:
where = where & t.c.architecture.in_(architectures)
if binary_types is not None:
where = where & t.c.type.in_(binary_types)
if format is None:
c_architectures = daksql.string_agg(t.c.architecture, ', ', order_by=[t.c.architecture_is_source.desc(), t.c.architecture])
query = sql.select([t.c.package, t.c.version, t.c.display_suite, c_architectures]) \
.where(where) \
.group_by(t.c.package, t.c.version, t.c.display_suite) \
.order_by(t.c.package, t.c.version, t.c.display_suite)
result = session.execute(query).fetchall()
if len(result) == 0:
raise StopIteration
lengths = {
'package': max(10, max(len(row[t.c.package]) for row in result)),
'version': max(13, max(len(row[t.c.version]) for row in result)),
'suite': max(10, max(len(row[t.c.display_suite]) for row in result))
}
format = "{0:{lengths[package]}} | {1:{lengths[version]}} | {2:{lengths[suite]}} | {3}"
for row in result:
yield format.format(row[t.c.package], row[t.c.version], row[t.c.display_suite], row[c_architectures], lengths=lengths)
elif format in ('control-suite', 'heidi'):
query = sql.select([t.c.package, t.c.version, t.c.architecture]).where(where)
result = session.execute(query)
for row in result:
yield "{0} {1} {2}".format(row[t.c.package], row[t.c.version], row[t.c.architecture])
elif format == "python":
c_architectures = daksql.string_agg(t.c.architecture, ',', order_by=[t.c.architecture_is_source.desc(), t.c.architecture])
query = sql.select([t.c.package,
t.c.version,
t.c.display_suite,
c_architectures,
t.c.source,
t.c.source_version,
t.c.component]) \
.where(where) \
.group_by(t.c.package,
t.c.version,
t.c.display_suite,
t.c.source,
t.c.component,
t.c.source_version)
result = session.execute(query).fetchall()
if len(result) == 0:
raise StopIteration
val = lambda: defaultdict(val)
ret = val()
for row in result:
ret[row[t.c.package]] \
[row[t.c.display_suite]] \
[row[t.c.version]]={'component': row[t.c.component],
'architectures': row[c_architectures].split(','),
'source': row[t.c.source],
'source_version': row[t.c.source_version]
}
yield ret
return
else:
raise ValueError("Unknown output format requested.")
if highest is not None:
query = sql.select([t.c.package, sql.func.max(t.c.version)]).where(where) \
.group_by(t.c.package).order_by(t.c.package)
result = session.execute(query)
yield ""
for row in result:
yield "{0} ({1} {2})".format(row[0], highest, row[1])
finally:
session.close()
def main():
from daklib.config import Config
from daklib import daklog
cnf = Config()
Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
('s',"suite","Generate-Packages-Sources::Options::Suite"),
('f',"force","Generate-Packages-Sources::Options::Force"),
('o','option','','ArbItem')]
suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
try:
Options = cnf.subtree("Generate-Packages-Sources::Options")
except KeyError:
Options = {}
if Options.has_key("Help"):
usage()
from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
pool = DakProcessPool()
logger = daklog.Logger('generate-packages-sources2')
from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
session = DBConn().session()
session.execute("SELECT add_missing_description_md5()")
session.commit()
if Options.has_key("Suite"):
suites = []
for s in suite_names:
suite = get_suite(s.lower(), session)
if suite:
suites.append(suite)
else:
print "I: Cannot find suite %s" % s
logger.log(['Cannot find suite %s' % s])
else:
query = session.query(Suite).filter(Suite.untouchable == False)
if 'Archive' in Options:
query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
suites = query.all()
force = Options.has_key("Force") and Options["Force"]
def parse_results(message):
# Split out into (code, msg)
code, msg = message
if code == PROC_STATUS_SUCCESS:
logger.log([msg])
elif code == PROC_STATUS_SIGNALRAISED:
logger.log(['E: Subprocess recieved signal ', msg])
else:
logger.log(['E: ', msg])
for s in suites:
component_ids = [ c.component_id for c in s.components ]
if s.untouchable and not force:
import daklib.utils
daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
for c in component_ids:
pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
if not s.include_long_description:
pool.apply_async(generate_translations, [s.suite_id, c], callback=parse_results)
for a in s.architectures:
if a == 'source':
continue
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)
pool.close()
pool.join()
# this script doesn't change the database
session.close()
logger.close()
sys.exit(pool.overall_status())
def main():
global Cnf
Cnf = utils.get_conf()
Arguments = [('h', "help", "Queue-Report::Options::Help"),
('n', "new", "Queue-Report::Options::New"),
('8', '822', "Queue-Report::Options::822"),
('s', "sort", "Queue-Report::Options::Sort", "HasArg"),
('a', "age", "Queue-Report::Options::Age", "HasArg"),
('r', "rrd", "Queue-Report::Options::Rrd", "HasArg"),
('d', "directories", "Queue-Report::Options::Directories",
"HasArg")]
for i in ["help"]:
key = "Queue-Report::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Queue-Report::Options")
if Options["Help"]:
usage()
if "Queue-Report::Options::New" in Cnf:
header()
queue_names = []
if "Queue-Report::Options::Directories" in Cnf:
for i in Cnf["Queue-Report::Options::Directories"].split(","):
queue_names.append(i)
elif "Queue-Report::Directories" in Cnf:
queue_names = Cnf.value_list("Queue-Report::Directories")
else:
queue_names = ["byhand", "new"]
if "Queue-Report::Options::Rrd" in Cnf:
rrd_dir = Cnf["Queue-Report::Options::Rrd"]
elif "Dir::Rrd" in Cnf:
rrd_dir = Cnf["Dir::Rrd"]
else:
rrd_dir = None
f = None
if "Queue-Report::Options::822" in Cnf:
# Open the report file
f = sys.stdout
filename822 = Cnf.get("Queue-Report::ReportLocations::822Location")
if filename822:
f = open(filename822, "w")
session = DBConn().session()
for queue_name in queue_names:
queue = session.query(PolicyQueue).filter_by(
queue_name=queue_name).first()
if queue is not None:
process_queue(queue, f, rrd_dir)
else:
utils.warn("Cannot find queue %s" % queue_name)
if "Queue-Report::Options::822" in Cnf:
f.close()
if "Queue-Report::Options::New" in Cnf:
footer()
def process_queue(queue, log, rrd_dir):
msg = ""
type = queue.queue_name
session = DBConn().session()
# Divide the .changes into per-source groups
per_source = {}
total_pending = 0
for upload in queue.uploads:
source = upload.changes.source
if source not in per_source:
per_source[source] = {}
per_source[source]["list"] = []
per_source[source]["processed"] = ""
handler = PolicyQueueUploadHandler(upload, session)
if handler.get_action():
per_source[source][
"processed"] = "PENDING %s" % handler.get_action()
total_pending += 1
per_source[source]["list"].append(upload)
per_source[source]["list"].sort(key=lambda x: x.changes.created,
reverse=True)
# Determine oldest time and have note status for each source group
for source in per_source.keys():
source_list = per_source[source]["list"]
first = source_list[0]
oldest = time.mktime(first.changes.created.timetuple())
have_note = 0
for d in per_source[source]["list"]:
mtime = time.mktime(d.changes.created.timetuple())
if "Queue-Report::Options::New" in Cnf:
if mtime > oldest:
oldest = mtime
else:
if mtime < oldest:
oldest = mtime
have_note += has_new_comment(d.policy_queue, d.changes.source,
d.changes.version)
per_source[source]["oldest"] = oldest
if not have_note:
per_source[source]["note_state"] = 0 # none
elif have_note < len(source_list):
per_source[source]["note_state"] = 1 # some
else:
per_source[source]["note_state"] = 2 # all
per_source_items = per_source.items()
per_source_items.sort(key=functools.cmp_to_key(sg_compare))
update_graph_database(rrd_dir, type, len(per_source_items),
len(queue.uploads))
entries = []
max_source_len = 0
max_version_len = 0
max_arch_len = 0
try:
logins = get_logins_from_ldap()
except:
logins = dict()
for i in per_source_items:
maintainer = {}
maint = ""
distribution = ""
closes = ""
fingerprint = ""
changeby = {}
changedby = ""
sponsor = ""
filename = i[1]["list"][0].changes.changesname
last_modified = time.time() - i[1]["oldest"]
source = i[1]["list"][0].changes.source
if len(source) > max_source_len:
max_source_len = len(source)
binary_list = i[1]["list"][0].binaries
binary = ', '.join([b.package for b in binary_list])
arches = set()
versions = set()
for j in i[1]["list"]:
dbc = j.changes
changesbase = dbc.changesname
if "Queue-Report::Options::New" in Cnf or "Queue-Report::Options::822" in Cnf:
try:
(maintainer["maintainer822"], maintainer["maintainer2047"],
maintainer["maintainername"], maintainer["maintaineremail"]) = \
fix_maintainer(dbc.maintainer)
except ParseMaintError as msg:
print("Problems while parsing maintainer address\n")
maintainer["maintainername"] = "Unknown"
maintainer["maintaineremail"] = "Unknown"
maint = "%s:%s" % (maintainer["maintainername"],
maintainer["maintaineremail"])
# ...likewise for the Changed-By: field if it exists.
try:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
fix_maintainer(dbc.changedby)
except ParseMaintError as msg:
(changeby["changedby822"], changeby["changedby2047"],
changeby["changedbyname"], changeby["changedbyemail"]) = \
("", "", "", "")
changedby = "%s:%s" % (changeby["changedbyname"],
changeby["changedbyemail"])
distribution = dbc.distribution.split()
closes = dbc.closes
fingerprint = dbc.fingerprint
sponsor_name = get_uid_from_fingerprint(fingerprint).name
sponsor_login = get_uid_from_fingerprint(fingerprint).uid
if '@' in sponsor_login:
if fingerprint in logins:
sponsor_login = logins[fingerprint]
if (sponsor_name != maintainer["maintainername"]
and sponsor_name != changeby["changedbyname"]
and sponsor_login + '@debian.org' !=
maintainer["maintaineremail"]
and sponsor_name != changeby["changedbyemail"]):
sponsor = sponsor_login
for arch in dbc.architecture.split():
arches.add(arch)
versions.add(dbc.version)
arches_list = sorted(arches, key=utils.ArchKey)
arch_list = " ".join(arches_list)
version_list = " ".join(sorted(versions, reverse=True))
if len(version_list) > max_version_len:
max_version_len = len(version_list)
if len(arch_list) > max_arch_len:
max_arch_len = len(arch_list)
if i[1]["note_state"]:
note = " | [N]"
else:
note = ""
entries.append([
source, binary, version_list, arch_list,
per_source[source]["processed"], note, last_modified, maint,
distribution, closes, fingerprint, sponsor, changedby, filename
])
# direction entry consists of "Which field, which direction, time-consider" where
# time-consider says how we should treat last_modified. Thats all.
# Look for the options for sort and then do the sort.
age = "h"
if "Queue-Report::Options::Age" in Cnf:
age = Cnf["Queue-Report::Options::Age"]
if "Queue-Report::Options::New" in Cnf:
# If we produce html we always have oldest first.
direction.append([6, -1, "ao"])
else:
if "Queue-Report::Options::Sort" in Cnf:
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
direction.append([6, -1, age])
elif i == "an":
# Age, newest first.
direction.append([6, 1, age])
elif i == "na":
# Name, Ascending.
direction.append([0, 1, 0])
elif i == "nd":
# Name, Descending.
direction.append([0, -1, 0])
elif i == "nl":
# Notes last.
direction.append([5, 1, 0])
elif i == "nf":
# Notes first.
direction.append([5, -1, 0])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
# have with it. (If you combine options it will simply take the last one at the moment).
# Will be enhanced in the future.
if "Queue-Report::Options::822" in Cnf:
# print stuff out in 822 format
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint, sponsor,
changedby, changes_file) = entry
# We'll always have Source, Version, Arch, Mantainer, and Dist
# For the rest, check to see if we have them, then print them out
log.write("Source: " + source + "\n")
log.write("Binary: " + binary + "\n")
log.write("Version: " + version_list + "\n")
log.write("Architectures: ")
log.write((", ".join(arch_list.split(" "))) + "\n")
log.write("Age: " + time_pp(last_modified) + "\n")
log.write("Last-Modified: " +
str(int(time.time()) - int(last_modified)) + "\n")
log.write("Queue: " + type + "\n")
(name, mail) = maint.split(":", 1)
log.write("Maintainer: " + name + " <" + mail + ">" + "\n")
if changedby:
(name, mail) = changedby.split(":", 1)
log.write("Changed-By: " + name + " <" + mail + ">" + "\n")
if sponsor:
log.write("Sponsored-By: %[email protected]\n" % sponsor)
log.write("Distribution:")
for dist in distribution:
log.write(" " + dist)
log.write("\n")
log.write("Fingerprint: " + fingerprint + "\n")
if closes:
bug_string = ""
for bugs in closes:
bug_string += "#" + bugs + ", "
log.write("Closes: " + bug_string[:-2] + "\n")
log.write("Changes-File: " + os.path.basename(changes_file) + "\n")
log.write("\n")
total_count = len(queue.uploads)
source_count = len(per_source_items)
if "Queue-Report::Options::New" in Cnf:
direction.append([6, 1, "ao"])
entries.sort(key=functools.cmp_to_key(sortfunc))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
if len(entries) > 0:
table_header(type.upper(), source_count, total_count)
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, maint, distribution, closes, fingerprint,
sponsor, changedby, _) = entry
table_row(source, version_list, arch_list, last_modified,
maint, distribution, closes, fingerprint, sponsor,
changedby)
table_footer(type.upper())
elif "Queue-Report::Options::822" not in Cnf:
# The "normal" output without any formatting.
msg = ""
for entry in entries:
(source, binary, version_list, arch_list, processed, note,
last_modified, _, _, _, _, _, _, _) = entry
if processed:
format = "%%-%ds | %%-%ds | %%-%ds | %%s\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, processed)
else:
format = "%%-%ds | %%-%ds | %%-%ds%%s | %%s old\n" % (
max_source_len, max_version_len, max_arch_len)
msg += format % (source, version_list, arch_list, note,
time_pp(last_modified))
if msg:
print(type.upper())
print("-" * len(type))
print()
print(msg)
print((
"%s %s source package%s / %s %s package%s in total / %s %s package%s to be processed."
% (source_count, type, plural(source_count), total_count, type,
plural(total_count), total_pending, type,
plural(total_pending))))
print()
def main():
global Cnf
keyrings = None
Cnf = utils.get_conf()
Arguments = [('h', "help", "Add-User::Options::Help"),
('k', "key", "Add-User::Options::Key", "HasArg"),
('u', "user", "Add-User::Options::User", "HasArg"),
]
for i in ["help"]:
key = "Add-User::Options::%s" % i
if key not in Cnf:
Cnf[key] = ""
apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
Options = Cnf.subtree("Add-User::Options")
if Options["help"]:
usage()
session = DBConn().session()
if not keyrings:
keyrings = get_active_keyring_paths()
cmd = ["gpg", "--with-colons", "--no-secmem-warning",
"--no-auto-check-trustdb", "--with-fingerprint",
"--no-default-keyring"]
cmd.extend(utils.gpg_keyring_args(keyrings).split())
cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]])
output = subprocess.check_output(cmd).rstrip()
m = re_gpg_fingerprint_colon.search(output)
if not m:
print(output)
utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output,
" [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ", "")
uid = ""
if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]:
uid = Cnf["Add-User::Options::User"]
name = Cnf["Add-User::Options::User"]
else:
u = re_user_address.search(output)
if not u:
print(output)
utils.fubar("0x%s: (2) No userid found in gpg output but it returned 0?\n%s"
% (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] ")))
uid = u.group(1)
n = re_user_name.search(output)
name = n.group(1)
# Look for all email addresses on the key.
emails = []
for line in output.split('\n'):
e = re_user_mails.search(line)
if not e:
continue
emails.append(e.group(2))
print("0x%s -> %s <%s> -> %s -> %s" % (Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key))
prompt = "Add user %s with above data (y/N) ? " % (uid)
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
# Create an account for the user?
summary = ""
# Now add user to the database.
# Note that we provide a session, so we're responsible for committing
uidobj = get_or_set_uid(uid, session=session)
uid_id = uidobj.uid_id
session.commit()
# Lets add user to the email-whitelist file if its configured.
if "Dinstall::MailWhiteList" in Cnf and Cnf["Dinstall::MailWhiteList"] != "":
f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
for mail in emails:
f.write(mail + '\n')
f.close()
print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id,
name, primary_key))
# Should we send mail to the newly added user?
if Cnf.find_b("Add-User::SendEmail"):
mail = name + "<" + emails[0] + ">"
Subst = {}
Subst["__NEW_MAINTAINER__"] = mail
Subst["__UID__"] = uid
Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
Subst["__PRIMARY_KEY__"] = primary_key
Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
Subst["__SUMMARY__"] = summary
new_add_message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"] + "/add-user.added")
utils.send_mail(new_add_message)
else:
uid = None
def number_of_packages():
arches = {}
arch_ids = {}
suites = {}
suite_ids = {}
d = {}
session = DBConn().session()
# Build up suite mapping
for i in session.query(Suite).all():
suites[i.suite_id] = i.suite_name
suite_ids[i.suite_name] = i.suite_id
# Build up architecture mapping
for i in session.query(Architecture).all():
arches[i.arch_id] = i.arch_string
arch_ids[i.arch_string] = i.arch_id
# Pre-create the dictionary
for suite_id in suites.keys():
d[suite_id] = {}
for arch_id in arches.keys():
d[suite_id][arch_id] = 0
# Get the raw data for binaries
# Simultate 'GROUP by suite, architecture' with a dictionary
# XXX: Why don't we just get the DB to do this?
for i in session.execute("""SELECT suite, architecture, COUNT(suite)
FROM bin_associations
LEFT JOIN binaries ON bin = binaries.id
GROUP BY suite, architecture""").fetchall():
d[i[0]][i[1]] = i[2]
# Get the raw data for source
arch_id = arch_ids["source"]
for i in session.execute('SELECT suite, COUNT(suite) FROM src_associations GROUP BY suite').fetchall():
(suite_id, count) = i
d[suite_id][arch_id] = d[suite_id][arch_id] + count
## Print the results
# Setup
suite_list = suites.values()
suite_id_list = []
suite_arches = {}
for suite in suite_list:
suite_id = suite_ids[suite]
suite_arches[suite_id] = {}
for arch in get_suite_architectures(suite):
suite_arches[suite_id][arch.arch_string] = ""
suite_id_list.append(suite_id)
output_list = [output_format(i) for i in suite_list]
longest_suite = longest(output_list)
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
# Header
output = (" " * longest_arch) + " |"
for suite in output_list:
output = output + suite.center(longest_suite) + " |"
output = output + "\n" + (len(output) * "-") + "\n"
# per-arch data
arch_list = arches.values()
arch_list.sort()
longest_arch = longest(arch_list)
for arch in arch_list:
arch_id = arch_ids[arch]
output = output + arch.center(longest_arch) + " |"
for suite_id in suite_id_list:
if arch in suite_arches[suite_id]:
count = "%d" % d[suite_id][arch_id]
else:
count = "-"
output = output + count.rjust(longest_suite) + " |"
output = output + "\n"
print(output)
def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.queue = queue
self.session = DBConn().session()
self.parent = parent
self.die = False
def main():
global Cnf, Options, Logger
os.umask(0o002)
Cnf = utils.get_conf()
Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
]
suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
Options = Cnf.subtree("Generate-Index-Diffs::Options")
if Options.has_key("Help"): usage()
maxdiffs = Options.get("MaxDiffs::Default", "56")
maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
maxsources = Options.get("MaxDiffs::Sources", maxdiffs)
if not Options.has_key("PatchName"):
format = "%Y-%m-%d-%H%M.%S"
Options["PatchName"] = time.strftime( format )
session = DBConn().session()
if not suites:
query = session.query(Suite.suite_name)
if Options.get('Archive'):
query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
suites = [ s.suite_name for s in query ]
for suitename in suites:
print "Processing: " + suitename
suiteobj = get_suite(suitename.lower(), session=session)
# Use the canonical version of the suite name
suite = suiteobj.suite_name
if suiteobj.untouchable:
print "Skipping: " + suite + " (untouchable)"
continue
architectures = get_suite_architectures(suite, skipall=True, session=session)
components = [ c.component_name for c in session.query(Component.component_name) ]
suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
if components and suite_suffix:
longsuite = suite + "/" + suite_suffix
else:
longsuite = suite
tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)
# See if there are Translations which might need a new pdiff
cwd = os.getcwd()
for component in components:
#print "DEBUG: Working on %s" % (component)
workpath=os.path.join(tree, component, "i18n")
if os.path.isdir(workpath):
os.chdir(workpath)
for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
for entry in filenames:
if not re_includeinpdiff.match(entry):
#print "EXCLUDING %s" % (entry)
continue
(fname, fext) = os.path.splitext(entry)
processfile=os.path.join(workpath, fname)
#print "Working: %s" % (processfile)
storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
#print "Storefile: %s" % (storename)
genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
os.chdir(cwd)
for archobj in architectures:
architecture = archobj.arch_string
for component in components:
if architecture == "source":
longarch = architecture
packages = "Sources"
maxsuite = maxsources
else:
longarch = "binary-%s"% (architecture)
packages = "Packages"
maxsuite = maxpackages
# Process Contents
file = "%s/%s/Contents-%s" % (tree, component, architecture)
storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxcontents)
file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
genchanges(Options, file + ".diff", storename, file, maxsuite)
