def unrep_with_dbd_issues():
log.info('running unrep_with_dbd_issues check...')
without_dbd = []
bad_dbd = []
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if not os.access(dbd, os.R_OK):
without_dbd.append((pkg, version, suite, arch))
sources_without_dbd.add(pkg)
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') is '
'unreproducible without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
if b'<' not in data:
bad_dbd.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has '
'diffoscope output, but it does not seem to '
'be an HTML page.')
sources_without_dbd.add(pkg)
return without_dbd, bad_dbd, sources_without_dbd
def pbuilder_dep_fail():
log.info('running pbuilder_dep_fail check...')
bad_pkgs = []
# we only care about these failures in the !unstable !experimental suites
# as they happen all the time in there, as packages are buggy
# and specific versions also come and go
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status = 'FTBFS'
AND s.suite NOT IN ('unstable', 'experimental')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.rbuild.log'
if os.access(rbuild, os.R_OK):
log.debug('\tlooking at ' + rbuild)
with open(rbuild, "br") as fd:
for line in fd:
if re.search(b'E: pbuilder-satisfydepends failed.', line):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' +
version +
') failed to satisfy its dependencies.')
return bad_pkgs
def alien_log(directory=None):
if directory is None:
bad_files = []
for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH:
bad_files.extend(alien_log(path))
return bad_files
log.info('running alien_log check over ' + directory + '...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(directory):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
# different file have differnt name patterns and different splitting needs
if file.endswith('.diff.gz'):
rsplit_level = 2
elif file.endswith('.gz'):
rsplit_level = 3
else:
rsplit_level = 2
try:
pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1)
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def lack_rbuild():
log.info('running lack_rbuild check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN ('blacklisted', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
rbuild = os.path.join(RBUILD_PATH, suite, arch) + \
'/{}_{}.rbuild.log.gz'.format(pkg, strip_epoch(version))
if not os.access(rbuild, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'built, but a buildlog is missing.')
return bad_pkgs
def alien_buildinfo():
log.info('running alien_buildinfo check...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
AND r.status IN ('reproducible', 'unreproducible')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(BUILDINFO_PATH):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
try:
pkg, version = file.rsplit('.', 1)[0].split('_')[:2]
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def not_unrep_with_dbd_file():
log.info('running not_unrep_with_dbd_file check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if os.access(dbd, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg +
' (' + version + ')'
' is not unreproducible.')
return bad_pkgs
def lack_buildinfo():
log.info('running lack_buildinfo check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN
('blacklisted', 'not for us', 'FTBFS', 'depwait', '404', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \
'_' + eversion + '_' + arch + '.buildinfo'
if not os.access(buildinfo, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'successfully built, but a .buildinfo is missing')
return bad_pkgs
def not_unrep_with_dbd_file():
log.info('running not_unrep_with_dbd_file check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'FTBR'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
for prefix, extension in ((
(DBD_PATH, 'html'),
(DBDTXT_PATH, 'txt.gz'),
(DBDJSON_PATH, 'json.gz'),
)):
filename = '{}/{}/{}/{}_{}.diffoscope.{}.gz'.format(
prefix, suite, arch, pkg, eversion, extension)
if not os.access(filename, os.R_OK):
continue
bad_pkgs.append((pkg, version, suite, arch))
log.warning(filename + ' exists but ' + suite + '/' + arch + '/' + pkg + ' (' + version + ')'
' is not FTBR.')
return bad_pkgs
def gen_packages_html(packages, no_clean=False):
"""
generate the /rb-pkg/package.HTML pages.
packages should be a list of Package objects.
"""
total = len(packages)
log.info('Generating the pages of ' + str(total) + ' package(s)')
for package in sorted(packages, key=lambda x: x.name):
assert isinstance(package, Package)
gen_history_page(package)
for arch in ARCHS:
gen_history_page(package, arch)
pkg = package.name
notes_uri = ''
notes_file = NOTES_PATH + '/' + pkg + '_note.html'
if os.access(notes_file, os.R_OK):
notes_uri = NOTES_URI + '/' + pkg + '_note.html'
for suite in SUITES:
for arch in ARCHS:
status = package.builds[suite][arch].status
version = package.builds[suite][arch].version
build_date = package.builds[suite][arch].build_date
if status is None: # the package is not in the checked suite
continue
log.debug('Generating the page of %s/%s/%s @ %s built at %s',
pkg, suite, arch, version, build_date)
suitearch_section_html, default_view, reproducible = \
gen_suitearch_section(package, suite, arch)
history_uri = '{}/{}.html'.format(HISTORY_URI, pkg)
history_archs = []
for a in ARCHS:
history_archs.append({
'history_arch':
a,
'history_arch_uri':
'{}/{}/{}.html'.format(HISTORY_URI, a, pkg)
})
project_links = renderer.render(project_links_template)
desturl = '{}{}/{}/{}/{}.html'.format(
REPRODUCIBLE_URL,
RB_PKG_URI,
suite,
arch,
pkg,
)
navigation_html = renderer.render(
package_navigation_template, {
'package': pkg,
'suite': suite,
'arch': arch,
'version': version,
'history_uri': history_uri,
'history_archs': history_archs,
'notes_uri': notes_uri,
'notify_maintainer': package.notify_maint,
'suitearch_section_html': suitearch_section_html,
'project_links_html': project_links,
'reproducible': reproducible,
'dashboard_url': DISTRO_URL,
'desturl': desturl,
})
body_html = renderer.render(package_page_template, {
'default_view': default_view,
})
destfile = os.path.join(RB_PKG_PATH, suite, arch,
pkg + '.html')
title = pkg + ' - reproducible builds result'
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package page generated at " + desturl)
# Optionally generate a page in which the main iframe shows the
# diffoscope results by default. Needed for navigation between
# diffoscope pages for different suites/archs
eversion = strip_epoch(version)
dbd_links = get_and_clean_dbd_links(pkg, eversion, suite, arch,
status)
# only generate the diffoscope page if diffoscope results exist
if 'dbd_uri' in dbd_links:
body_html = renderer.render(
package_page_template, {
'default_view': dbd_links['dbd_uri'],
})
destfile = dbd_links['dbd_page_file']
desturl = REPRODUCIBLE_URL + "/" + dbd_links['dbd_page_uri']
title = "{} ({}) diffoscope results in {}/{}".format(
pkg, version, suite, arch)
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package diffoscope page generated at " +
desturl)
if not no_clean:
purge_old_pages() # housekeep is always good
def gen_suitearch_section(package, current_suite, current_arch):
# keep track of whether the package is entirely reproducible
final_version = ''
final_status = ''
default_view = ''
context = {}
context['architectures'] = []
for a in ARCHS:
suites = []
for s in SUITES:
status = package.builds[s][a].status
if not status: # The package is not available in that suite/arch
continue
version = package.builds[s][a].version
if not final_version or not final_status:
final_version = version
final_status = status
else:
final_status, final_version = determine_reproducibility(
final_status, final_version, status, version)
build_date = package.builds[s][a].build_date
status = Status.get(status)
if not (build_date and status != 'blacklisted'):
build_date = ''
li_classes = ['suite']
if s == current_suite and a == current_arch:
li_classes.append('active')
package_uri = ('{}/{}/{}/{}.html').format(RB_PKG_URI, s, a,
package.name)
suitearch_details_html = ''
if (s == current_suite and a == current_arch):
suitearch_details_html, default_view = gen_suitearch_details(
package.name, version, s, a, status.value.name,
status.value.spokenstatus, build_date)
dbd_links = get_dbd_links(package.name, strip_epoch(version), s, a)
dbd_page_uri = dbd_links.get('dbd_page_uri', '')
suites.append({
'package':
package.name,
'package_quote_plus':
urllib.parse.quote_plus(package.name),
'status':
status.value.name,
'version':
version,
'build_date':
build_date,
'icon':
status.value.icon,
'spokenstatus':
status.value.spokenstatus,
'li_classes':
' '.join(li_classes),
'arch':
a,
'suite':
s,
'untested':
status == Status.UNTESTED,
'current_suitearch':
s == current_suite and a == current_arch,
'package_uri':
package_uri,
'suitearch_details_html':
suitearch_details_html,
'dbd_page_uri':
dbd_page_uri
})
if len(suites):
context['architectures'].append({
'arch': a,
'suites': suites,
})
html = renderer.render(suitearch_section_template, context)
reproducible = True if final_status == 'reproducible' else False
return html, default_view, reproducible
def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
build_date):
eversion = strip_epoch(version) # epoch_free_version is too long
pkg = Package(package)
build = pkg.builds[suite][arch]
context = {}
default_view = ''
# Make notes the default default view
notes_file = NOTES_PATH + '/' + package + '_note.html'
notes_uri = NOTES_URI + '/' + package + '_note.html'
if os.access(notes_file, os.R_OK):
default_view = notes_uri
# Get summary context
context['status_html'] = gen_status_link_icon(status, spokenstatus, None,
suite, arch)
context['build_date'] = build_date
# Get diffoscope differences context
dbd_links = get_dbd_links(package, eversion, suite, arch)
dbd_uri = dbd_links.get('dbd_uri', '')
if dbd_uri:
context['dbd'] = {
'dbd_page_uri': dbd_links['dbd_page_uri'],
'dbdtxt_uri': dbd_links.get('dbdtxt_uri', ''),
'dbdjson_uri': dbd_links.get('dbdjson_uri', ''),
}
default_view = default_view if default_view else dbd_uri
# Get buildinfo context
if build.buildinfo:
context['buildinfo_uri'] = build.buildinfo.url
default_view = default_view if default_view else build.buildinfo.url
elif not args.ignore_missing_files and status not in \
('untested', 'blacklisted', 'FTBFS', 'NFU', 'depwait', '404'):
log.critical('buildinfo not detected at ' + build.buildinfo.path)
# Get rbuild, build2 and build diffs context
if build.rbuild:
context['rbuild_uri'] = build.rbuild.url
context['rbuild_size'] = sizeof_fmt(build.rbuild.size)
default_view = default_view if default_view else build.rbuild.url
context['buildlogs'] = {}
if build.build2 and build.logdiff:
context['buildlogs']['build2_uri'] = build.build2.url
context['buildlogs']['build2_size'] = build.build2.size
context['buildlogs']['diff_uri'] = build.logdiff.url
else:
log.error('Either {} or {} is missing'.format(
build.build2.path, build.logdiff.path))
elif status not in ('untested', 'blacklisted') and \
not args.ignore_missing_files:
log.critical(
DISTRO_URL + '/' + suite + '/' + arch + '/' + package +
' didn\'t produce a buildlog, even though it has been built.')
context['has_buildloginfo'] = 'buildinfo_uri' in context or \
'buildlogs' in context or \
'rbuild_uri' in context
default_view = '/untested.html' if not default_view else default_view
suitearch_details_html = renderer.render(suitearch_details_template,
context)
return (suitearch_details_html, default_view)