def update_stats(suite, arch, stats, pkgset_name):
result = query_db("""
SELECT datum, meta_pkg, suite
FROM stats_meta_pkg_state
WHERE datum = '{date}' AND suite = '{suite}'
AND architecture = '{arch}' AND meta_pkg = '{name}'
""".format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name))
# if there is not a result for this day, add one
if not result:
insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \
"'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \
"'{count_bad}', '{count_ugly}', '{count_rest}')"
query_db(
insert.format(date=YESTERDAY,
suite=suite,
arch=arch,
pkgset_name=pkgset_name,
count_good=stats['count_good'],
count_bad=stats['count_bad'],
count_ugly=stats['count_ugly'],
count_rest=stats['count_rest']))
log.info("Updating db entry for meta pkgset %s in %s/%s on %s.",
pkgset_name, suite, arch, YESTERDAY)
else:
log.debug(
"Not updating db entry for meta pkgset %s in %s/%s on %s as one exists already.",
pkgset_name, suite, arch, YESTERDAY)
def update_stats_breakages(diffoscope_timeouts, diffoscope_crashes):
# we only do stats up until yesterday
YESTERDAY = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
result = query_db("""
SELECT datum, diffoscope_timeouts, diffoscope_crashes
FROM stats_breakages
WHERE datum = '{date}'
""".format(date=YESTERDAY))
# if there is not a result for this day, add one
if not result:
insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \
"'{diffoscope_timeouts}', '{diffoscope_crashes}')"
query_db(
insert.format(date=YESTERDAY,
diffoscope_timeouts=diffoscope_timeouts,
diffoscope_crashes=diffoscope_crashes))
log.info(
"Updating db table stats_breakages on %s with %s timeouts and %s crashes.",
YESTERDAY, diffoscope_timeouts, diffoscope_crashes)
else:
log.debug(
"Not updating db table stats_breakages as it already has data for %s.",
YESTERDAY)
def db_update():
"""
Update the database schema.
Get a list of queries to perform from schema_updates.
The need for an update is detected by checking the biggest value in the
rb_schema table against the biggest value in the schema_updates dictionary.
"""
current = query_db('SELECT MAX(version) FROM rb_schema')[0][0]
if not current:
log.warning('This is probably a new database, there are no ' +
'previous updates noted')
current = 0
last = max(schema_updates.keys())
if current == last:
return False
if current > last:
print_critiacal_message('The active database schema is higher than' +
' the last update available.\nPlease check!')
sys.exit(1)
log.info('Found schema updates.')
for update in range(current + 1, last + 1):
log.info('Applying database update #' + str(update) + '. Queries:')
startTime = datetime.now()
for query in schema_updates[update]:
log.info('\t' + query)
query_db(query)
log.info(
str(len(schema_updates[update])) + ' queries executed in ' +
str(datetime.now() - startTime))
return True
def gather_meta_stats(suite, arch, pkgset_name):
pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite,
pkgset_name + '.pkgset')
try:
with open(pkgset_file) as f:
pkgset_list = [s.strip() for s in f.readlines()]
except FileNotFoundError:
log.warning('No meta package set information exists at ' + pkgset_file)
return {}
if not pkgset_list:
log.warning('No packages listed for package set: ' + pkgset_name)
return {}
package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')"
root_query = """
SELECT s.name
FROM results AS r
JOIN sources AS s ON r.package_id=s.id
WHERE s.suite='{suite}'
AND s.architecture='{arch}'
AND date(r.build_date)<='{date}'
AND {package_where}
""".format(suite=suite,
arch=arch,
date=YESTERDAY,
package_where=package_where)
stats = {}
good = query_db(root_query + "AND r.status = 'reproducible' " +
"ORDER BY s.name;")
stats['good'] = [t[0] for t in good]
stats['count_good'] = len(stats['good'])
bad = query_db(root_query + "AND r.status = 'FTBR'" +
"ORDER BY r.build_date;")
stats['bad'] = [t[0] for t in bad]
stats['count_bad'] = len(stats['bad'])
ugly = query_db(root_query + "AND r.status = 'FTBFS'" +
"ORDER BY r.build_date;")
stats['ugly'] = [t[0] for t in ugly]
stats['count_ugly'] = len(stats['ugly'])
rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " +
"r.status != 'FTBR' AND " +
"r.status != 'reproducible') ORDER BY r.build_date;")
stats['rest'] = [t[0] for t in rest]
stats['count_rest'] = len(stats['rest'])
stats['count_all'] = (stats['count_good'] + stats['count_bad'] +
stats['count_ugly'] + stats['count_rest'])
stats['count_all'] = stats['count_all'] if stats['count_all'] else 1
stats['percent_good'] = percent(stats['count_good'], stats['count_all'])
stats['percent_bad'] = percent(stats['count_bad'], stats['count_all'])
stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all'])
stats['percent_rest'] = percent(stats['count_rest'], stats['count_all'])
return stats
def db_create_tables():
"""
Check whether all tables are present, and create them if not.
The check is done against sqlite_master, a reserved sqlite table
containing all database metadata.
"""
changed = False
for table in db_schema:
if not table_exists(table['name']):
log.warning(table['name'] + ' does not exists. Creating...')
for query in table['query']:
log.info('\t' + re.sub(' +', ' ', query.replace('\n', ' ')))
query_db(query)
changed = True
return changed
def pbuilder_dep_fail():
log.info('running pbuilder_dep_fail check...')
bad_pkgs = []
# we only care about these failures in the !unstable !experimental suites
# as they happen all the time in there, as packages are buggy
# and specific versions also come and go
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status = 'FTBFS'
AND s.suite NOT IN ('unstable', 'experimental')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.rbuild.log'
if os.access(rbuild, os.R_OK):
log.debug('\tlooking at ' + rbuild)
with open(rbuild, "br") as fd:
for line in fd:
if re.search(b'E: pbuilder-satisfydepends failed.', line):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' +
version +
') failed to satisfy its dependencies.')
return bad_pkgs
def unrep_with_dbd_issues():
log.info('running unrep_with_dbd_issues check...')
without_dbd = []
bad_dbd = []
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if not os.access(dbd, os.R_OK):
without_dbd.append((pkg, version, suite, arch))
sources_without_dbd.add(pkg)
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') is '
'unreproducible without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
if b'<' not in data:
bad_dbd.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has '
'diffoscope output, but it does not seem to '
'be an HTML page.')
sources_without_dbd.add(pkg)
return without_dbd, bad_dbd, sources_without_dbd
def packages_matching_criteria(arch, suite, criteria):
"Return a list of packages in (SUITE, ARCH) matching the given CRITERIA."
# TODO: Rewrite this function to query all suites/archs in one go
issue, status, built_after, built_before = criteria
del criteria
formatter = dict(suite=suite, arch=arch, notes_table='')
log.info('Querying packages with given issues/status...')
query = "SELECT s.name " + \
"FROM sources AS s, {notes_table} results AS r " + \
"WHERE r.package_id=s.id " + \
"AND s.architecture= '{arch}' " + \
"AND s.suite = '{suite}' AND r.status != 'blacklisted' "
if issue:
query += "AND n.package_id=s.id AND n.issues LIKE '%%{issue}%%' "
formatter['issue'] = issue
formatter['notes_table'] = "notes AS n,"
if status:
query += "AND r.status = '{status}'"
formatter['status'] = status
if built_after:
query += "AND r.build_date > '{built_after}' "
formatter['built_after'] = built_after
if built_before:
query += "AND r.build_date < '{built_before}' "
formatter['built_before'] = built_before
results = query_db(query.format_map(formatter))
results = [x for (x, ) in results]
log.info('Selected packages: ' + ' '.join(results))
return results
def create_pkgset_graph(png_file, suite, arch, pkgset_name):
table = "stats_meta_pkg_state"
columns = ["datum", "reproducible", "FTBR", "FTBFS", "other"]
where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \
(suite, arch, pkgset_name)
if arch == 'i386':
# i386 only has pkg sets since later to make nicer graphs
# (date added in commit 7f2525f7)
where += " AND datum >= '2016-05-06'"
query = "SELECT {fields} FROM {table} {where} ORDER BY datum".format(
fields=", ".join(columns), table=table, where=where)
result = query_db(query)
result_rearranged = [dict(zip(columns, row)) for row in result]
with create_temp_file(mode='w') as f:
csv_tmp_file = f.name
csv_writer = csv.DictWriter(f, columns)
csv_writer.writeheader()
csv_writer.writerows(result_rearranged)
f.flush()
graph_command = os.path.join(BIN_PATH, "make_graph.py")
main_label = "Reproducibility status for packages in " + suite + \
" from " + pkgset_name
y_label = "Amount (" + pkgset_name + " packages)"
log.info("Creating graph for meta pkgset %s in %s/%s.", pkgset_name,
suite, arch)
check_call([
graph_command, csv_tmp_file, png_file, '4', main_label, y_label,
'1920', '960'
])
def alien_rbpkg():
log.info('running alien_rbpkg check...')
query = '''SELECT s.name
FROM sources AS s
WHERE s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(RB_PKG_PATH):
if not files:
continue
# Extract the "suite" and "arch" from the directory structure
if os.path.split(root)[1] == 'diffoscope-results':
# We are presently inspecting package pages in the
# RB_PKG_PATH/{{suite}}/{{arch}}/diffoscope-results directory
suite, arch = os.path.split(root)[0].rsplit('/', 2)[1:]
else:
# We are presently inspecting package pages in the
# RB_PKG_PATH/{{suite}}/{{arch}}/ directory
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
pkg = file.rsplit('.', 1)[0]
if not query_db(query.format(pkg=pkg, suite=suite, arch=arch)):
bad_files.append('/'.join([root, file]))
log.warning('/'.join([root, file]) + ' should not be there')
return bad_files
def gen_all_rb_pkg_pages(no_clean=False):
query = 'SELECT DISTINCT name FROM sources WHERE suite = ANY(:s)'
rows = query_db(sqlalchemy.text(query), s=SUITES)
pkgs = [Package(str(i[0]), no_notes=True) for i in rows]
log.info('Processing all %s package from all suites/architectures',
len(pkgs))
gen_packages_html(pkgs, no_clean=True) # we clean at the end
purge_old_pages()
def generate_live_status_table(arch):
averagesql = select([
func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
]).where(
and_(
stats_build.c.status.in_(('reproducible', 'FTBR')),
stats_build.c.name == sources.c.name,
stats_build.c.suite == sources.c.suite,
stats_build.c.architecture == sources.c.architecture,
)).as_scalar()
query = select([
sources.c.id,
sources.c.suite,
sources.c.architecture,
sources.c.name,
sources.c.version,
schedule.c.date_build_started,
results.c.status,
results.c.build_duration,
averagesql,
schedule.c.job,
]).select_from(sources.join(schedule).join(results, isouter=True)).where(
and_(
schedule.c.date_build_started != None,
sources.c.architecture == bindparam('arch'),
)).order_by(schedule.c.date_scheduled)
html = ''
rows = query_db(query.params({'arch': arch}))
html += '<p><table class="scheduled">\n' + tab
html += '<tr><th class="center">#</th><th class="center">src pkg id</th><th class="center">suite</th><th class="center">arch</th>'
html += '<th class=\"center\">source package</th><th class=\"center\">version</th></th>'
html += '<th class=\"center\">build started</th><th class=\"center\">previous build status</th>'
html += '<th class=\"center\">previous build duration</th><th class=\"center\">average build duration</th><th class=\"center\">builder job</th>'
html += '</tr>\n'
counter = 0
for row in rows:
counter += 1
suite = row[1]
arch = row[2]
pkg = row[3]
duration = convert_into_hms_string(row[7])
avg_duration = convert_into_hms_string(row[8])
html += tab + '<tr><td> </td><td>' + str(row[0]) + '</td>'
html += '<td>' + suite + '</td><td>' + arch + '</td>'
html += '<td><code>' + Package(pkg).html_link(
suite, arch, bugs=False) + '</code></td>'
html += '<td>' + str(row[4]) + '</td><td>' + str(row[5]) + '</td>'
html += '<td>' + convert_into_status_html(
str(row[6])
) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td>'
html += '<td><a href="https://tests.reproducible-builds.org/cgi-bin/nph-logwatch?' + str(
row[9]) + '">' + str(row[9]) + '</a></td>'
html += '</tr>\n'
html += '</table></p>\n'
return html
def alien_log(directory=None):
if directory is None:
bad_files = []
for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH:
bad_files.extend(alien_log(path))
return bad_files
log.info('running alien_log check over ' + directory + '...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(directory):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
# different file have differnt name patterns and different splitting needs
if file.endswith('.diff.gz'):
rsplit_level = 2
elif file.endswith('.gz'):
rsplit_level = 3
else:
rsplit_level = 2
try:
pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1)
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def purge_old_pages():
for suite in SUITES:
for arch in ARCHS:
log.info('Removing old pages from ' + suite + '/' + arch + '.')
try:
presents = sorted(
os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch))
except OSError as e:
if e.errno != errno.ENOENT: # that's 'No such file or
raise # directory' error (errno 17)
presents = []
log.debug('page presents: ' + str(presents))
# get the existing packages
query = "SELECT name, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
cur_pkgs = set([(p.name, p.suite, p.architecture)
for p in query_db(query)])
for page in presents:
# When diffoscope results exist for a package, we create a page
# that displays the diffoscope results by default in the main iframe
# in this subdirectory. Ignore this directory.
if page == 'diffoscope-results':
continue
pkg = page.rsplit('.', 1)[0]
if (pkg, suite, arch) not in cur_pkgs:
log.info('There is no package named ' + pkg + ' from ' +
suite + '/' + arch + ' in the database. ' +
'Removing old page.')
os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
page)
# Additionally clean up the diffoscope results default pages
log.info('Removing old pages from ' + suite + '/' + arch +
'/diffoscope-results/.')
try:
presents = sorted(
os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch +
'/diffoscope-results'))
except OSError as e:
if e.errno != errno.ENOENT: # that's 'No such file or
raise # directory' error (errno 17)
presents = []
log.debug('diffoscope page presents: ' + str(presents))
for page in presents:
pkg = page.rsplit('.', 1)[0]
if (pkg, suite, arch) not in cur_pkgs:
log.info('There is no package named ' + pkg + ' from ' +
suite + '/' + arch + '/diffoscope-results in ' +
'the database. Removing old page.')
os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
'diffoscope-results/' + page)
def alien_history():
log.info('running alien_history check...')
result = query_db('SELECT DISTINCT name FROM sources')
actual_packages = [x[0] for x in result]
bad_files = []
for f in sorted(os.listdir(HISTORY_PATH)):
full_path = os.path.join(HISTORY_PATH, f)
if f.rsplit('.', 1)[0] not in actual_packages and not os.path.isdir(full_path):
bad_files.append(full_path)
os.remove(full_path)
log.warning('%s should not be there so it has been removed.', full_path)
return bad_files
def load_notes():
"""
format:
{ 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} }
"""
with open(NOTES) as fd:
possible_notes = yaml.load(fd)
log.debug("notes loaded. There are " + str(len(possible_notes)) +
" package listed")
notes = copy.copy(possible_notes)
for package in possible_notes: # check if every package listed on the notes
try: # actually have been tested
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != ''"
query = query.format(pkg=package)
query_db(query)[0] # just discard this result, we only care of its success
except IndexError:
log.warning("This query produces no results: " + query)
log.warning("This means there is no tested package with the name " + package + ".")
del notes[package]
log.debug("notes checked. There are " + str(len(notes)) +
" package listed")
return notes
def query_untested_packages(suite, arch, limit):
criteria = 'not tested before, randomly sorted'
query = """SELECT DISTINCT *
FROM (
SELECT sources.id, sources.name FROM sources
WHERE sources.suite='{suite}' AND sources.architecture='{arch}'
AND sources.id NOT IN
(SELECT schedule.package_id FROM schedule)
AND sources.id NOT IN
(SELECT results.package_id FROM results)
ORDER BY random()
) AS tmp
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
packages = query_db(query)
print_schedule_result(suite, arch, criteria, packages)
return packages
def query_old_depwait_versions(suite, arch, limit):
criteria = 'status depwait, no bug filed, tested at least 2 days ago, ' + \
'no new version available, sorted by last build date'
date = (datetime.now()-timedelta(days=2)).strftime('%Y-%m-%d %H:%M')
query = """SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status='depwait'
AND r.build_date < '{date}'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
date=date)
packages = query_db(query)
print_schedule_result(suite, arch, criteria, packages)
return packages
def query_e404_versions(suite, arch, limit):
criteria = """tested at least 12h ago, status E404,
sorted by last build date"""
date = (datetime.now()-timedelta(days=0.5)).strftime('%Y-%m-%d %H:%M')
query = """SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status = 'E404'
AND r.build_date < '{date}'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
date=date)
packages = query_db(query)
print_schedule_result(suite, arch, criteria, packages)
return packages
def lack_rbuild():
log.info('running lack_rbuild check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN ('blacklisted', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
rbuild = os.path.join(RBUILD_PATH, suite, arch) + \
'/{}_{}.rbuild.log.gz'.format(pkg, strip_epoch(version))
if not os.access(rbuild, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'built, but a buildlog is missing.')
return bad_pkgs
def alien_buildinfo():
log.info('running alien_buildinfo check...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
AND r.status IN ('reproducible', 'unreproducible')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(BUILDINFO_PATH):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
try:
pkg, version = file.rsplit('.', 1)[0].split('_')[:2]
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def query_old_versions(suite, arch, limit):
criteria = """tested at least {minimum_age} days ago, no new version available,
sorted by last build date""".format(minimum_age=MINIMUM_AGE[arch])
date = (datetime.now()-timedelta(days=MINIMUM_AGE[arch]))\
.strftime('%Y-%m-%d %H:%M')
query = """SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status != 'blacklisted'
AND r.build_date < '{date}'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch,
date=date, limit=limit)
packages = query_db(query)
print_schedule_result(suite, arch, criteria, packages)
return packages
def not_unrep_with_dbd_file():
log.info('running not_unrep_with_dbd_file check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if os.access(dbd, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg +
' (' + version + ')'
' is not unreproducible.')
return bad_pkgs
def lack_buildinfo():
log.info('running lack_buildinfo check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN
('blacklisted', 'not for us', 'FTBFS', 'depwait', '404', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \
'_' + eversion + '_' + arch + '.buildinfo'
if not os.access(buildinfo, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'successfully built, but a .buildinfo is missing')
return bad_pkgs
def query_new_versions(suite, arch, limit):
criteria = 'tested before, new version available, sorted by last build date'
query = """SELECT s.id, s.name, s.version, r.version, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND s.version != r.version
AND r.status != 'blacklisted'
AND s.id IN (SELECT package_id FROM results)
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name, s.version, r.version
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
pkgs = query_db(query)
# the next line avoids constant rescheduling of packages:
# packages in our repository != official repo,
# so they will always be selected by the query above
# so we only accept them if there version is greater than the already tested one
packages = [(x[0], x[1]) for x in pkgs if apt_pkg.version_compare(x[2], x[3]) > 0]
print_schedule_result(suite, arch, criteria, packages)
return packages
def create_breakages_graph(png_file, main_label):
png_fullpath = os.path.join(DISTRO_BASE, png_file)
table = "stats_breakages"
columns = ["datum", "diffoscope_timeouts", "diffoscope_crashes"]
query = "SELECT {fields} FROM {table} ORDER BY datum".format(
fields=", ".join(columns), table=table)
result = query_db(query)
result_rearranged = [dict(zip(columns, row)) for row in result]
with create_temp_file(mode='w') as f:
csv_tmp_file = f.name
csv_writer = csv.DictWriter(f, columns)
csv_writer.writeheader()
csv_writer.writerows(result_rearranged)
f.flush()
graph_command = os.path.join(BIN_PATH, "make_graph.py")
y_label = "Amount (packages)"
log.info("Creating graph for stats_breakges.")
check_call([graph_command, csv_tmp_file, png_fullpath, '2', main_label,
y_label, '1920', '960'])
def process_pkg(package, deactivate):
if deactivate:
_good('Deactivating notification for package ' + str(package))
flag = 0
else:
_good('Activating notification for package ' + str(package))
flag = 1
sources_table = db_table('sources')
update_query = sources_table.update().\
where(sources_table.c.name == package).\
values(notify_maintainer=flag)
rows = conn_db.execute(update_query).rowcount
if rows == 0:
log.error(bcolors.FAIL + str(package) + ' does not exists')
sys.exit(1)
if DEBUG:
log.debug('Double check the change:')
query = 'SELECT * FROM sources WHERE name="{}"'.format(package)
log.debug(query_db(query))
def build_page_section(page, section, suite, arch):
try:
if pages[page].get('global') and pages[page]['global']:
suite = defaultsuite
arch = defaultarch
if pages[page].get('notes') and pages[page]['notes']:
db_status = section['status'].value.name
query = queries[section['query']].params({
'status': db_status,
'suite': suite,
'arch': arch
})
section['icon_status'] = section['status'].value.icon
else:
query = queries[section['query']].params({
'suite': suite,
'arch': arch
})
rows = query_db(query)
except:
print_critical_message('A query failed: %s' % query)
raise
html = ''
footnote = True if rows else False
if not rows: # there are no package in this set, do not output anything
log.debug('empty query: %s' %
query.compile(compile_kwargs={"literal_binds": True}))
return (html, footnote)
html += build_leading_text_section(section, rows, suite, arch)
html += '<p>\n' + tab + '<code>\n'
for row in rows:
pkg = row[0]
html += tab * 2 + Package(pkg).html_link(suite, arch)
else:
html += tab + '</code>\n'
html += '</p>'
if section.get('bottom'):
html += section['bottom']
html = (tab * 2).join(html.splitlines(True))
return (html, footnote)
def generate_oldies(arch):
log.info('Building the oldies page for ' + arch + '...')
title = 'Oldest results on ' + arch
html = ''
for suite in SUITES:
query = select([
sources.c.suite, sources.c.architecture, sources.c.name,
results.c.status, results.c.build_date
]).select_from(results.join(sources)).where(
and_(sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch'),
results.c.status != 'blacklisted')).order_by(
results.c.build_date).limit(15)
text = Template('Oldest results on $suite/$arch:')
rows = query_db(query.params({'arch': arch, 'suite': suite}))
html += build_leading_text_section({'text': text}, rows, suite, arch)
html += '<p><table class="scheduled">\n' + tab
html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>'
html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n'
for row in rows:
# 0: suite, 1: arch, 2: pkg name 3: status 4: build date
pkg = row[2]
html += tab + '<tr><td> </td><td>' + row[0] + '</td>'
html += '<td>' + row[1] + '</td><td><code>'
html += Package(pkg).html_link(row[0], row[1])
html += '</code></td><td>' + convert_into_status_html(str(
row[3])) + '</td><td>' + row[4] + '</td></tr>\n'
html += '</table></p>\n'
destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html'
desturl = DISTRO_URL + '/index_' + arch + '_oldies.html'
left_nav_html = create_main_navigation(arch=arch)
write_html_page(title=title,
body=html,
destfile=destfile,
style_note=True,
refresh_every=60,
left_nav_html=left_nav_html)
log.info("Page generated at " + desturl)
def purge_old_notes(notes):
removed_pages = []
to_rebuild = []
presents = sorted(os.listdir(NOTES_PATH))
for page in presents:
pkg = page.rsplit('_', 1)[0]
log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed')
if pkg not in notes:
log.info('There are no notes for ' + pkg + '. Removing old page.')
os.remove(NOTES_PATH + '/' + page)
removed_pages.append(pkg)
for pkg in removed_pages:
for suite in SUITES:
try:
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != '' AND s.suite='{suite}'"
query = query.format(pkg=pkg, suite=suite)
to_rebuild.append(query_db(query)[0][0])
except IndexError: # the package is not tested. this can happen if
pass # a package got removed from the archive
if to_rebuild:
gen_packages_html([Package(x) for x in to_rebuild])
