def unrep_with_dbd_issues():
log.info('running unrep_with_dbd_issues check...')
without_dbd = []
bad_dbd = []
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if not os.access(dbd, os.R_OK):
without_dbd.append((pkg, version, suite, arch))
sources_without_dbd.add(pkg)
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') is '
'unreproducible without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
if b'<' not in data:
bad_dbd.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has '
'diffoscope output, but it does not seem to '
'be an HTML page.')
sources_without_dbd.add(pkg)
return without_dbd, bad_dbd, sources_without_dbd
def db_update():
"""
Update the database schema.
Get a list of queries to perform from schema_updates.
The need for an update is detected by checking the biggest value in the
rb_schema table against the biggest value in the schema_updates dictionary.
"""
current = query_db('SELECT MAX(version) FROM rb_schema')[0][0]
if not current:
log.warning('This is probably a new database, there are no ' +
'previous updates noted')
current = 0
last = max(schema_updates.keys())
if current == last:
return False
if current > last:
print_critiacal_message('The active database schema is higher than' +
' the last update available.\nPlease check!')
sys.exit(1)
log.info('Found schema updates.')
for update in range(current + 1, last + 1):
log.info('Applying database update #' + str(update) + '. Queries:')
startTime = datetime.now()
for query in schema_updates[update]:
log.info('\t' + query)
query_db(query)
log.info(
str(len(schema_updates[update])) + ' queries executed in ' +
str(datetime.now() - startTime))
return True
def alien_rbpkg():
log.info('running alien_rbpkg check...')
query = '''SELECT s.name
FROM sources AS s
WHERE s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(RB_PKG_PATH):
if not files:
continue
# Extract the "suite" and "arch" from the directory structure
if os.path.split(root)[1] == 'diffoscope-results':
# We are presently inspecting package pages in the
# RB_PKG_PATH/{{suite}}/{{arch}}/diffoscope-results directory
suite, arch = os.path.split(root)[0].rsplit('/', 2)[1:]
else:
# We are presently inspecting package pages in the
# RB_PKG_PATH/{{suite}}/{{arch}}/ directory
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
pkg = file.rsplit('.', 1)[0]
if not query_db(query.format(pkg=pkg, suite=suite, arch=arch)):
bad_files.append('/'.join([root, file]))
log.warning('/'.join([root, file]) + ' should not be there')
return bad_files
def pbuilder_dep_fail():
log.info('running pbuilder_dep_fail check...')
bad_pkgs = []
# we only care about these failures in the !unstable !experimental suites
# as they happen all the time in there, as packages are buggy
# and specific versions also come and go
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status = 'FTBFS'
AND s.suite NOT IN ('unstable', 'experimental')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.rbuild.log'
if os.access(rbuild, os.R_OK):
log.debug('\tlooking at ' + rbuild)
with open(rbuild, "br") as fd:
for line in fd:
if re.search(b'E: pbuilder-satisfydepends failed.', line):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' +
version +
') failed to satisfy its dependencies.')
return bad_pkgs
def gather_meta_stats(suite, arch, pkgset_name):
pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite,
pkgset_name + '.pkgset')
try:
with open(pkgset_file) as f:
pkgset_list = [s.strip() for s in f.readlines()]
except FileNotFoundError:
log.warning('No meta package set information exists at ' + pkgset_file)
return {}
if not pkgset_list:
log.warning('No packages listed for package set: ' + pkgset_name)
return {}
package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')"
root_query = """
SELECT s.name
FROM results AS r
JOIN sources AS s ON r.package_id=s.id
WHERE s.suite='{suite}'
AND s.architecture='{arch}'
AND date(r.build_date)<='{date}'
AND {package_where}
""".format(suite=suite,
arch=arch,
date=YESTERDAY,
package_where=package_where)
stats = {}
good = query_db(root_query + "AND r.status = 'reproducible' " +
"ORDER BY s.name;")
stats['good'] = [t[0] for t in good]
stats['count_good'] = len(stats['good'])
bad = query_db(root_query + "AND r.status = 'FTBR'" +
"ORDER BY r.build_date;")
stats['bad'] = [t[0] for t in bad]
stats['count_bad'] = len(stats['bad'])
ugly = query_db(root_query + "AND r.status = 'FTBFS'" +
"ORDER BY r.build_date;")
stats['ugly'] = [t[0] for t in ugly]
stats['count_ugly'] = len(stats['ugly'])
rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " +
"r.status != 'FTBR' AND " +
"r.status != 'reproducible') ORDER BY r.build_date;")
stats['rest'] = [t[0] for t in rest]
stats['count_rest'] = len(stats['rest'])
stats['count_all'] = (stats['count_good'] + stats['count_bad'] +
stats['count_ugly'] + stats['count_rest'])
stats['count_all'] = stats['count_all'] if stats['count_all'] else 1
stats['percent_good'] = percent(stats['count_good'], stats['count_all'])
stats['percent_bad'] = percent(stats['count_bad'], stats['count_all'])
stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all'])
stats['percent_rest'] = percent(stats['count_rest'], stats['count_all'])
return stats
def alien_log(directory=None):
if directory is None:
bad_files = []
for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH:
bad_files.extend(alien_log(path))
return bad_files
log.info('running alien_log check over ' + directory + '...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(directory):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
# different file have differnt name patterns and different splitting needs
if file.endswith('.diff.gz'):
rsplit_level = 2
elif file.endswith('.gz'):
rsplit_level = 3
else:
rsplit_level = 2
try:
pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1)
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def fill_issue_in_note(issue):
details = issues[issue]
html = ''
if 'url' in details:
html += note_issue_html_url.substitute(url=details['url'])
if 'description' in details:
desc = details['description'].replace('\n', '<br />')
html += note_issue_html_desc.substitute(description=desc)
else:
log.warning("The issue " + issue + " misses a description")
return note_issue_html.substitute(issue=issue, issue_info=html)
def alien_history():
log.info('running alien_history check...')
result = query_db('SELECT DISTINCT name FROM sources')
actual_packages = [x[0] for x in result]
bad_files = []
for f in sorted(os.listdir(HISTORY_PATH)):
full_path = os.path.join(HISTORY_PATH, f)
if f.rsplit('.', 1)[0] not in actual_packages and not os.path.isdir(full_path):
bad_files.append(full_path)
os.remove(full_path)
log.warning('%s should not be there so it has been removed.', full_path)
return bad_files
def purge_old_issues(issues):
for root, dirs, files in os.walk(ISSUES_PATH):
if not files:
continue
for file in files:
try:
issue = file.rsplit('_', 1)[0]
except ValueError:
log.critical('/'.join([root, file]) + ' does not seems like '
+ 'a file that should be there')
sys.exit(1)
if issue not in issues:
log.warning('removing ' + '/'.join([root, file]) + '...')
os.remove('/'.join([root, file]))
def gen_html_note(package, note):
"""
Given a note as input (as a dict:
{"package_name": {"version": "0.0.0", "comments": "blablabla",
"bugs": [111, 222], "issues": ["issue1", "issue2"]}}
) it returns the html body
"""
infos = ''
# check for issues:
if 'issues' in note:
tmp = ''
for issue in note['issues']:
tmp += fill_issue_in_note(issue)
issues_count.setdefault(issue, []).append(note['package'])
infos += note_issues_html.substitute(issues=tmp)
# check for bugs:
if 'bugs' in note:
bugurls = ''
for bug in note['bugs']:
try:
bug_title = ': "%s"' % bugs[package][bug]['title']
except KeyError:
bug_title = ''
bugurls += '<a href="https://bugs.debian.org/' + str(bug) + \
'" target="_parent">' + str(bug) + '</a>' + \
get_trailing_bug_icon(bug, bugs, package) + \
bug_title + '<br />'
infos += note_bugs_html.substitute(bugs=bugurls)
# check for comments:
if 'comments' in note:
comment = note['comments']
comment = url2html.sub(r'<a href="\1">\1</a>', comment)
comment = comment.replace('\n', '<br />')
infos += note_comments_html.substitute(comments=comment)
try:
version = str(note['version'])
return renderer.render(notes_body_template, {
'version': version,
'infos': infos,
'notesgit_description': NOTESGIT_DESCRIPTION
})
except KeyError:
log.warning('You should really include a version in the ' +
str(note['package']) + ' note')
return renderer.render(notes_body_template, {
'version': 'N/A',
'infos': infos,
'notesgit_description': NOTESGIT_DESCRIPTION
})
def db_create_tables():
"""
Check whether all tables are present, and create them if not.
The check is done against sqlite_master, a reserved sqlite table
containing all database metadata.
"""
changed = False
for table in db_schema:
if not table_exists(table['name']):
log.warning(table['name'] + ' does not exists. Creating...')
for query in table['query']:
log.info('\t' + re.sub(' +', ' ', query.replace('\n', ' ')))
query_db(query)
changed = True
return changed
def lack_rbuild():
log.info('running lack_rbuild check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN ('blacklisted', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
rbuild = os.path.join(RBUILD_PATH, suite, arch) + \
'/{}_{}.rbuild.log.gz'.format(pkg, strip_epoch(version))
if not os.access(rbuild, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'built, but a buildlog is missing.')
return bad_pkgs
def alien_buildinfo():
log.info('running alien_buildinfo check...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
AND r.status IN ('reproducible', 'unreproducible')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(BUILDINFO_PATH):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
try:
pkg, version = file.rsplit('.', 1)[0].split('_')[:2]
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def not_unrep_with_dbd_file():
log.info('running not_unrep_with_dbd_file check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if os.access(dbd, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg +
' (' + version + ')'
' is not unreproducible.')
return bad_pkgs
def lack_buildinfo():
log.info('running lack_buildinfo check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status NOT IN
('blacklisted', 'not for us', 'FTBFS', 'depwait', '404', '')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \
'_' + eversion + '_' + arch + '.buildinfo'
if not os.access(buildinfo, os.R_OK):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has been '
'successfully built, but a .buildinfo is missing')
return bad_pkgs
def not_unrep_with_dbd_file():
log.info('running not_unrep_with_dbd_file check...')
bad_pkgs = []
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != 'FTBR'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
for prefix, extension in ((
(DBD_PATH, 'html'),
(DBDTXT_PATH, 'txt.gz'),
(DBDJSON_PATH, 'json.gz'),
)):
filename = '{}/{}/{}/{}_{}.diffoscope.{}.gz'.format(
prefix, suite, arch, pkg, eversion, extension)
if not os.access(filename, os.R_OK):
continue
bad_pkgs.append((pkg, version, suite, arch))
log.warning(filename + ' exists but ' + suite + '/' + arch + '/' + pkg + ' (' + version + ')'
' is not FTBR.')
return bad_pkgs
def db_update():
"""
Update the database schema.
Get a list of queries to perform from schema_updates.
The need for an update is detected by checking the biggest value in the
rb_schema table against the biggest value in the schema_updates dictionary.
"""
current = query_db('SELECT MAX(version) FROM rb_schema')[0][0]
if not current:
log.warning('This is probably a new database, there are no ' +
'previous updates noted')
current = 0
last = max(schema_updates.keys())
if current == last:
return False
if current > last:
print_critical_message('The active database schema is higher than' +
' the last update available.\nPlease check!')
sys.exit(1)
log.info('Found schema updates.')
Session = sessionmaker(bind=DB_ENGINE, autocommit=True)
session = Session()
for update in range(current + 1, last + 1):
log.info('Applying database update #' + str(update) + '. Queries:')
startTime = datetime.now()
with session.begin():
for query in schema_updates[update]:
log.info('\t' + query)
session.execute(query)
session.execute(
"INSERT INTO rb_schema (version, date) "
"VALUES (:ver, CURRENT_TIMESTAMP)", {'ver': update})
log.info(
str(len(schema_updates[update])) + ' queries executed in ' +
str(datetime.now() - startTime))
return True
def load_notes():
"""
format:
{ 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} }
"""
with open(NOTES) as fd:
possible_notes = yaml.load(fd)
log.debug("notes loaded. There are " + str(len(possible_notes)) +
" package listed")
notes = copy.copy(possible_notes)
for package in possible_notes: # check if every package listed on the notes
try: # actually have been tested
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != ''"
query = query.format(pkg=package)
query_db(query)[0] # just discard this result, we only care of its success
except IndexError:
log.warning("This query produces no results: " + query)
log.warning("This means there is no tested package with the name " + package + ".")
del notes[package]
log.debug("notes checked. There are " + str(len(notes)) +
" package listed")
return notes
def rest(scheduling_args, requester, local, suite, arch):
"Actually schedule a package for a single suite on a single arch."
# Shorter names
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.info("Scheduling packages in %s/%s", arch, suite)
ids = []
pkgs = []
query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}'
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
WHERE p.package_id='{id}'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
# tests whether the package is already building
try:
result2 = query_db(query2.format(id=result[0][0]))
except IndexError:
log.error('%sThe package %s is not available in %s/%s%s',
bcolors.FAIL, pkg, suite, arch, bcolors.ENDC)
continue
try:
if not result2[0][0]:
ids.append(result[0][0])
pkgs.append(pkg)
else:
log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' +
'already building, not scheduling it.' +
bcolors.ENDC)
except IndexError:
# it's not in the schedule
ids.append(result[0][0])
pkgs.append(pkg)
def compose_irc_message():
"One-shot closure to limit scope of the following local variables."
blablabla = '✂…' if len(' '.join(pkgs)) > 257 else ''
packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else ''
trailing = ' - artifacts will be preserved' if artifacts else ''
trailing += ' - with irc notification' if notify else ''
trailing += ' - notify on start too' if notify_on_start else ''
message = requester + ' scheduled ' + packages_txt + \
'in ' + suite + '/' + arch
if reason:
message += ', reason: \'' + reason + '\''
message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing
return message
info_msg = compose_irc_message()
del compose_irc_message
# these packages are manually scheduled, so should have high priority,
# so schedule them in the past, so they are picked earlier :)
# the current date is subtracted twice, so it sorts before early scheduling
# schedule on the full hour so we can recognize them easily
epoch = int(time.time())
now = datetime.now()
days = int(now.strftime('%j')) * 2
hours = int(now.strftime('%H')) * 2
minutes = int(now.strftime('%M'))
time_delta = timedelta(days=days, hours=hours, minutes=minutes)
date = (now - time_delta).strftime('%Y-%m-%d %H:%M')
log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta))
# a single person can't schedule more than 500 packages in the same day; this
# is actually easy to bypass, but let's give some trust to the Debian people
query = """SELECT count(*) FROM manual_scheduler
WHERE requester = '{}' AND date_request > '{}'"""
try:
amount = int(
query_db(query.format(requester, int(time.time() - 86400)))[0][0])
except IndexError:
amount = 0
log.debug(requester + ' already scheduled ' + str(amount) +
' packages today')
if amount + len(ids) > 500 and not local:
log.error(
bcolors.FAIL + 'You have exceeded the maximum number of manual ' +
'reschedulings allowed for a day. Please ask in ' +
'#debian-reproducible if you need to schedule more packages.' +
bcolors.ENDC)
sys.exit(1)
# do the actual scheduling
add_to_schedule = []
update_schedule = []
save_schedule = []
artifacts_value = 1 if artifacts else 0
if notify_on_start:
do_notify = 2
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
schedule_table = db_table('schedule')
if ids:
existing_pkg_ids = dict(
query_db(
sql.select([
schedule_table.c.package_id,
schedule_table.c.id,
]).where(schedule_table.c.package_id.in_(ids))))
for id in ids:
if id in existing_pkg_ids:
update_schedule.append({
'update_id': existing_pkg_ids[id],
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
else:
add_to_schedule.append({
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
save_schedule.append({
'package_id': id,
'requester': requester,
'date_request': epoch,
})
log.debug('Packages about to be scheduled: ' + str(add_to_schedule) +
str(update_schedule))
update_schedule_query = schedule_table.update().\
where(schedule_table.c.id == sql.bindparam('update_id'))
insert_schedule_query = schedule_table.insert()
insert_manual_query = db_table('manual_scheduler').insert()
if not dry_run:
transaction = conn_db.begin()
if add_to_schedule:
conn_db.execute(insert_schedule_query, add_to_schedule)
if update_schedule:
conn_db.execute(update_schedule_query, update_schedule)
if save_schedule:
conn_db.execute(insert_manual_query, save_schedule)
transaction.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + info_msg + bcolors.ENDC)
if not (local
and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
# Always announce on -changes
irc_msg(info_msg, 'debian-reproducible-changes')
# Announce some messages on main channel
if notify_on_start or artifacts:
irc_msg(info_msg)
def build_leading_text_section(section, rows, suite, arch):
html = '<p>\n' + tab
total = len(rows)
count_total = int(
query_db(queries['count_total'].params({
'suite': suite,
'arch': arch
}))[0][0])
try:
percent = round(((total / count_total) * 100), 1)
except ZeroDivisionError:
log.error('Looks like there are either no tested package or no ' +
'packages available at all. Maybe it\'s a new database?')
percent = 0.0
try:
html += '<a href="' + section['icon_link'] + '" target="_parent">'
no_icon_link = False
except KeyError:
no_icon_link = True # to avoid closing the </a> tag below
if section.get('icon_status'):
html += '<img src="/static/' + section['icon_status']
html += '" alt="reproducible icon" />'
if not no_icon_link:
html += '</a>'
html += '\n' + tab
if section.get('text') and section.get('timespan'):
count = len(
query_db(queries[section['query2']].params({
'suite': suite,
'arch': arch
})))
percent = round(((count / count_total) * 100), 1)
timespan = section['timespan']
timespan_date = timespan_date_map[timespan]
timespan_count = int(
query_db(queries['count_timespan'].params({
'suite':
suite,
'arch':
arch,
'timespan_date':
timespan_date
}))[0][0])
try:
timespan_percent = round(((total / timespan_count) * 100), 1)
except ZeroDivisionError:
log.error('Looks like there are either no tested package or no ' +
'packages available at all. Maybe it\'s a new database?')
timespan_percent = 0
html += section['text'].substitute(tot=total,
percent=percent,
timespan_percent=timespan_percent,
timespan_count=timespan_count,
count_total=count_total,
count=count,
suite=suite,
arch=arch)
elif section.get('text'):
html += section['text'].substitute(tot=total,
percent=percent,
suite=suite,
arch=arch)
else:
log.warning('There is no text for this section')
html += '\n</p>\n'
return html
def gen_html_issue(issue, suite):
"""
Given a issue as input (as a dict:
{"issue_identifier": {"description": "blablabla", "url": "blabla"}}
) it returns the html body
"""
# links to the issue in other suites
suite_links = ''
for i in SUITES:
if suite_links != '':
suite_links += ' / '
if i != suite:
suite_links += '<a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/' + i + '/' + issue + '_issue.html">' + i + '</a>'
else:
suite_links += '<em>' + i + '</em>'
# check for url:
if 'url' in issues[issue]:
url = issue_html_url.substitute(url=issues[issue]['url'])
else:
url = ''
# add affected packages:
affected = ''
results = db_table('results')
sources = db_table('sources')
sql = select(
[sources.c.name]
).select_from(
results.join(sources)
).where(
and_(
sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch'),
results.c.status == bindparam('status'),
)
).order_by(
sources.c.name
)
try:
arch = 'amd64'
for status in Status:
status = status.value
pkgs = query_db(sql.where(sources.c.name.in_(issues_count[issue]))\
.params({'suite': suite, 'arch': arch, 'status': status.name}))
pkgs = [p[0] for p in pkgs]
if not pkgs:
continue
affected += tab*4 + '<p>\n'
affected += tab*5 + '<img src="/static/{}"'.format(status.icon)
affected += ' alt="' + status.name + ' icon" />\n'
affected += tab*5 + str(len(pkgs)) + ' ' + status.spokenstatus
affected += ' packages in ' + suite + '/' + arch +':\n'
affected += tab*5 + '<code>\n'
pkgs_popcon = issues_popcon_annotate(pkgs)
try:
for pkg, popc_num, is_popular in sorted(pkgs_popcon, key=lambda x: x[0] in bugs):
affected += tab*6 + Package(pkg).html_link(suite, arch, bugs, popc_num, is_popular)
except ValueError:
pass
affected += tab*5 + '</code>\n'
affected += tab*4 + '</p>\n'
except KeyError: # The note is not listed in any package, that is
affected = '<i>None</i>'
# check for description:
try:
desc = issues[issue]['description']
except KeyError:
log.warning('You should really include a description in the ' +
issue + ' issue')
desc = 'N/A'
desc = url2html.sub(r'<a href="\1">\1</a>', desc)
desc = desc.replace('\n', '<br />')
return issue_html.substitute(issue=issue, urls=url, description=desc,
affected_pkgs=affected,
suite=suite, suite_links=suite_links,
notesgit_description=NOTESGIT_DESCRIPTION)
