def update_stats_breakages(diffoscope_timeouts, diffoscope_crashes): # we only do stats up until yesterday YESTERDAY = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d') result = query_db(""" SELECT datum, diffoscope_timeouts, diffoscope_crashes FROM stats_breakages WHERE datum = '{date}' """.format(date=YESTERDAY)) # if there is not a result for this day, add one if not result: insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \ "'{diffoscope_timeouts}', '{diffoscope_crashes}')" query_db( insert.format(date=YESTERDAY, diffoscope_timeouts=diffoscope_timeouts, diffoscope_crashes=diffoscope_crashes)) log.info( "Updating db table stats_breakages on %s with %s timeouts and %s crashes.", YESTERDAY, diffoscope_timeouts, diffoscope_crashes) else: log.debug( "Not updating db table stats_breakages as it already has data for %s.", YESTERDAY)
def unrep_with_dbd_issues(): log.info('running unrep_with_dbd_issues check...') without_dbd = [] bad_dbd = [] sources_without_dbd = set() query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.diffoscope.html' if not os.access(dbd, os.R_OK): without_dbd.append((pkg, version, suite, arch)) sources_without_dbd.add(pkg) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') is ' 'unreproducible without diffoscope file.') else: log.debug(dbd + ' found.') data = open(dbd, 'br').read(3) if b'<' not in data: bad_dbd.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has ' 'diffoscope output, but it does not seem to ' 'be an HTML page.') sources_without_dbd.add(pkg) return without_dbd, bad_dbd, sources_without_dbd
def update_stats(suite, arch, stats, pkgset_name): result = query_db(""" SELECT datum, meta_pkg, suite FROM stats_meta_pkg_state WHERE datum = '{date}' AND suite = '{suite}' AND architecture = '{arch}' AND meta_pkg = '{name}' """.format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name)) # if there is not a result for this day, add one if not result: insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \ "'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \ "'{count_bad}', '{count_ugly}', '{count_rest}')" query_db( insert.format(date=YESTERDAY, suite=suite, arch=arch, pkgset_name=pkgset_name, count_good=stats['count_good'], count_bad=stats['count_bad'], count_ugly=stats['count_ugly'], count_rest=stats['count_rest'])) log.info("Updating db entry for meta pkgset %s in %s/%s on %s.", pkgset_name, suite, arch, YESTERDAY) else: log.debug( "Not updating db entry for meta pkgset %s in %s/%s on %s as one exists already.", pkgset_name, suite, arch, YESTERDAY)
def pbuilder_dep_fail(): log.info('running pbuilder_dep_fail check...') bad_pkgs = [] # we only care about these failures in the !unstable !experimental suites # as they happen all the time in there, as packages are buggy # and specific versions also come and go query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status = 'FTBFS' AND s.suite NOT IN ('unstable', 'experimental') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.rbuild.log' if os.access(rbuild, os.R_OK): log.debug('\tlooking at ' + rbuild) with open(rbuild, "br") as fd: for line in fd: if re.search(b'E: pbuilder-satisfydepends failed.', line): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') failed to satisfy its dependencies.') return bad_pkgs
def schedule_packages(packages): pkgs = [{ 'package_id': x, 'date_scheduled': packages[x] } for x in packages.keys()] log.debug('IDs about to be scheduled: %s', packages.keys()) if pkgs: conn_db.execute(db_table('schedule').insert(), pkgs)
def load_issues(): """ format: { 'issue_name': {'description': 'blabla', 'url': 'blabla'} } """ with open(ISSUES) as fd: issues = yaml.load(fd) log.debug("issues loaded. There are " + str(len(issues)) + " issues listed") return issues
def purge_old_pages(): for suite in SUITES: for arch in ARCHS: log.info('Removing old pages from ' + suite + '/' + arch + '.') try: presents = sorted( os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch)) except OSError as e: if e.errno != errno.ENOENT: # that's 'No such file or raise # directory' error (errno 17) presents = [] log.debug('page presents: ' + str(presents)) # get the existing packages query = "SELECT name, suite, architecture FROM sources " + \ "WHERE suite='{}' AND architecture='{}'".format(suite, arch) cur_pkgs = set([(p.name, p.suite, p.architecture) for p in query_db(query)]) for page in presents: # When diffoscope results exist for a package, we create a page # that displays the diffoscope results by default in the main iframe # in this subdirectory. Ignore this directory. if page == 'diffoscope-results': continue pkg = page.rsplit('.', 1)[0] if (pkg, suite, arch) not in cur_pkgs: log.info('There is no package named ' + pkg + ' from ' + suite + '/' + arch + ' in the database. ' + 'Removing old page.') os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' + page) # Additionally clean up the diffoscope results default pages log.info('Removing old pages from ' + suite + '/' + arch + '/diffoscope-results/.') try: presents = sorted( os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch + '/diffoscope-results')) except OSError as e: if e.errno != errno.ENOENT: # that's 'No such file or raise # directory' error (errno 17) presents = [] log.debug('diffoscope page presents: ' + str(presents)) for page in presents: pkg = page.rsplit('.', 1)[0] if (pkg, suite, arch) not in cur_pkgs: log.info('There is no package named ' + pkg + ' from ' + suite + '/' + arch + '/diffoscope-results in ' + 'the database. Removing old page.') os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' + 'diffoscope-results/' + page)
def update_sources(suite): # download the sources file for this suite mirror = 'http://deb.debian.org/debian' remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz' log.info('Downloading sources file for %s: %s', suite, remotefile) sources = lzma.decompress(urlopen(remotefile).read()).decode('utf8') log.debug('\tdownloaded') for arch in ARCHS: log.info('Updating sources db for %s/%s...', suite, arch) update_sources_db(suite, arch, sources) log.info('DB update done for %s/%s done at %s.', suite, arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def process_pkg(package, deactivate): if deactivate: _good('Deactivating notification for package ' + str(package)) flag = 0 else: _good('Activating notification for package ' + str(package)) flag = 1 sources_table = db_table('sources') update_query = sources_table.update().\ where(sources_table.c.name == package).\ values(notify_maintainer=flag) rows = conn_db.execute(update_query).rowcount if rows == 0: log.error(bcolors.FAIL + str(package) + ' does not exists') sys.exit(1) if DEBUG: log.debug('Double check the change:') query = 'SELECT * FROM sources WHERE name="{}"'.format(package) log.debug(query_db(query))
def build_page_section(page, section, suite, arch): try: if pages[page].get('global') and pages[page]['global']: suite = defaultsuite arch = defaultarch if pages[page].get('notes') and pages[page]['notes']: db_status = section['status'].value.name query = queries[section['query']].params({ 'status': db_status, 'suite': suite, 'arch': arch }) section['icon_status'] = section['status'].value.icon else: query = queries[section['query']].params({ 'suite': suite, 'arch': arch }) rows = query_db(query) except: print_critical_message('A query failed: %s' % query) raise html = '' footnote = True if rows else False if not rows: # there are no package in this set, do not output anything log.debug('empty query: %s' % query.compile(compile_kwargs={"literal_binds": True})) return (html, footnote) html += build_leading_text_section(section, rows, suite, arch) html += '<p>\n' + tab + '<code>\n' for row in rows: pkg = row[0] html += tab * 2 + Package(pkg).html_link(suite, arch) else: html += tab + '</code>\n' html += '</p>' if section.get('bottom'): html += section['bottom'] html = (tab * 2).join(html.splitlines(True)) return (html, footnote)
def store_notes(): log.debug('Removing all notes') notes_table = db_table('notes') conn_db.execute(notes_table.delete()) to_insert = [] for entry in [x for y in sorted(notes) for x in notes[y]]: pkg_id = entry['id'] pkg_version = entry['version'] pkg_issues = json.dumps(entry['issues']) pkg_bugs = json.dumps(entry['bugs']) pkg_comments = entry['comments'] to_insert.append({ 'package_id': pkg_id, 'version': pkg_version, 'issues': pkg_issues, 'bugs': pkg_bugs, 'comments': pkg_comments }) if (len(to_insert)): conn_db.execute(notes_table.insert(), to_insert) log.info('Saved ' + str(len(to_insert)) + ' notes in the database')
def purge_old_notes(notes): removed_pages = [] to_rebuild = [] presents = sorted(os.listdir(NOTES_PATH)) for page in presents: pkg = page.rsplit('_', 1)[0] log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed') if pkg not in notes: log.info('There are no notes for ' + pkg + '. Removing old page.') os.remove(NOTES_PATH + '/' + page) removed_pages.append(pkg) for pkg in removed_pages: for suite in SUITES: try: query = "SELECT s.name " + \ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \ "WHERE s.name='{pkg}' AND r.status != '' AND s.suite='{suite}'" query = query.format(pkg=pkg, suite=suite) to_rebuild.append(query_db(query)[0][0]) except IndexError: # the package is not tested. this can happen if pass # a package got removed from the archive if to_rebuild: gen_packages_html([Package(x) for x in to_rebuild])
def store_issues(): issues_table = db_table('issues') # Get existing issues results = conn_db.execute(sql.select([issues_table.c.name])) existing_issues = set([row[0] for row in results]) to_insert = [] to_update = [] for name in issues: url = issues[name]['url'] if 'url' in issues[name] else '' desc = issues[name]['description'] if name in existing_issues: to_update.append({ 'issuename': name, 'url': url, 'description': desc }) # remove this package from the set, to know who to delete later existing_issues.remove(name) else: to_insert.append({'name': name, 'url': url, 'description': desc}) if to_update: update_query = issues_table.update().\ where(issues_table.c.name == sql.bindparam('issuename')) conn_db.execute(update_query, to_update) log.debug('Issues updated in the database') if to_insert: conn_db.execute(issues_table.insert(), to_insert) log.debug('Issues added to the database') # if there are any existing issues left, delete them. if existing_issues: to_delete = [{'issuename': name} for name in existing_issues] delete_query = issues_table.delete().\ where(issues_table.c.name == sql.bindparam('issuename')) conn_db.execute(delete_query, to_delete) log.info("Removed the following issues: " + str(existing_issues))
def load_notes(): """ format: { 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} } """ with open(NOTES) as fd: possible_notes = yaml.load(fd) log.debug("notes loaded. There are " + str(len(possible_notes)) + " package listed") notes = copy.copy(possible_notes) for package in possible_notes: # check if every package listed on the notes try: # actually have been tested query = "SELECT s.name " + \ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \ "WHERE s.name='{pkg}' AND r.status != ''" query = query.format(pkg=package) query_db(query)[0] # just discard this result, we only care of its success except IndexError: log.warning("This query produces no results: " + query) log.warning("This means there is no tested package with the name " + package + ".") del notes[package] log.debug("notes checked. There are " + str(len(notes)) + " package listed") return notes
def iterate_over_notes(notes): num_notes = str(len(notes)) i = 0 for package in sorted(notes): log.debug('iterating over notes... ' + str(i) + '/' + num_notes) note = notes[package] note['package'] = package log.debug('\t' + str(note)) html = gen_html_note(package, note) title = 'Notes for ' + package + ' - reproducible builds result' destfile = NOTES_PATH + '/' + package + '_note.html' write_html_page(title=title, body=html, destfile=destfile) desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.debug("Note created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' note pages.')
def iterate_over_issues(issues): num_issues = str(len(issues)) for suite in SUITES: i = 0 for issue in sorted(issues): log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues) log.debug('\t' + str(issue)) html = gen_html_issue(issue, suite) title = 'Notes about issue ' + issue + ' in ' + suite destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html' left_nav_html = create_main_navigation(displayed_page='issues') write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' log.debug("Issue created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' issue pages for ' + suite)
def build_page(page, suite=None, arch=None): gpage = False if pages[page].get('global') and pages[page]['global']: gpage = True suite = defaultsuite arch = defaultarch if not gpage and suite and not arch: print_critical_message('The architecture was not specified while ' + 'building a suite-specific page.') sys.exit(1) if gpage: log.debug('Building the ' + page + ' global index page...') title = pages[page]['title'] else: log.debug('Building the ' + page + ' index page for ' + suite + '/' + arch + '...') title = pages[page]['title'].format(suite=suite, arch=arch) page_sections = pages[page]['body'] html = '' footnote = False if pages[page].get('header'): if pages[page].get('notes_hint') and pages[page][ 'notes_hint'] and suite == defaultsuite: hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.' else: hint = '' if pages[page].get('header_query'): html += pages[page]['header'].format(tot=query_db( pages[page]['header_query'].format(suite=suite, arch=arch))[0][0], suite=suite, arch=arch, hint=hint) else: html += pages[page].get('header') for section in page_sections: if gpage: if section.get('nosuite') and section['nosuite']: # only defaults html += build_page_section(page, section, suite, arch)[0] else: for suite in SUITES: for arch in ARCHS: log.debug('global page §' + section['status'].name + ' in ' + page + ' for ' + suite + '/' + arch) html += build_page_section(page, section, suite, arch)[0] footnote = True else: html1, footnote1 = build_page_section(page, section, suite, arch) html += html1 footnote = True if footnote1 else footnote suite_arch_nav_template = None if gpage: destfile = DISTRO_BASE + '/index_' + page + '.html' desturl = DISTRO_URL + '/index_' + page + '.html' suite = defaultsuite # used for the links in create_main_navigation else: destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \ page + '.html' desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \ page + '.html' suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \ page + '.html' left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page=page, suite_arch_nav_template=suite_arch_nav_template, ) write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) log.info('"' + title + '" now available at ' + desturl)
def update_sources_db(suite, arch, sources): # extract relevant info (package name and version) from the sources file new_pkgs = set() newest_version = {} for src in deb822.Sources.iter_paragraphs(sources.split('\n')): pkg = (src['Package'], src['Version'], suite, arch) if 'Extra-Source-Only' in src and src['Extra-Source-Only'] == 'yes': log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg)) continue # only keep the most recent version of a src for each package/suite/arch key = src['Package'] + suite + arch if key in newest_version: oldversion = newest_version[key] oldpackage = (src['Package'], oldversion, suite, arch) new_pkgs.remove(oldpackage) newest_version[key] = src['Version'] new_pkgs.add(pkg) # get the current packages in the database query = "SELECT name, version, suite, architecture FROM sources " + \ "WHERE suite='{}' AND architecture='{}'".format(suite, arch) cur_pkgs = set([(p.name, p.version, p.suite, p.architecture) for p in query_db(query)]) pkgs_to_add = [] updated_pkgs = [] different_pkgs = [x for x in new_pkgs if x not in cur_pkgs] log.debug('Packages different in the archive and in the db: %s', different_pkgs) for pkg in different_pkgs: # pkg: (name, version, suite, arch) query = "SELECT id, version, notify_maintainer FROM sources " + \ "WHERE name='{}' AND suite='{}' AND architecture='{}'" query = query.format(pkg[0], pkg[2], pkg[3]) try: result = query_db(query)[0] except IndexError: # new package pkgs_to_add.append({ 'name': pkg[0], 'version': pkg[1], 'suite': pkg[2], 'architecture': pkg[3], }) continue pkg_id = result[0] old_version = result[1] notify_maint = int(result[2]) if apt_pkg.version_compare(pkg[1], old_version) > 0: log.debug('New version: ' + str(pkg) + ' (we had ' + old_version + ')') updated_pkgs.append({ 'update_id': pkg_id, 'name': pkg[0], 'version': pkg[1], 'suite': pkg[2], 'architecture': pkg[3], 'notify_maintainer': notify_maint, }) # Now actually update the database: sources_table = db_table('sources') # updated packages log.info('Pushing ' + str(len(updated_pkgs)) + ' updated packages to the database...') if updated_pkgs: transaction = conn_db.begin() update_query = sources_table.update().\ where(sources_table.c.id == sql.bindparam('update_id')) conn_db.execute(update_query, updated_pkgs) transaction.commit() # new packages if pkgs_to_add: log.info('Now inserting %i new sources in the database: %s', len(pkgs_to_add), pkgs_to_add) transaction = conn_db.begin() conn_db.execute(sources_table.insert(), pkgs_to_add) transaction.commit() # RM'ed packages cur_pkgs_name = [x[0] for x in cur_pkgs] new_pkgs_name = [x[0] for x in new_pkgs] rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name] log.info('Now deleting %i removed packages: %s', len(rmed_pkgs), rmed_pkgs) rmed_pkgs_id = [] pkgs_to_rm = [] query = "SELECT id FROM sources WHERE name='{}' AND suite='{}' " + \ "AND architecture='{}'" for pkg in rmed_pkgs: result = query_db(query.format(pkg, suite, arch)) rmed_pkgs_id.append({'deleteid': result[0][0]}) pkgs_to_rm.append({'name': pkg, 'suite': suite, 'architecture': arch}) log.debug('removed packages ID: %s', [str(x['deleteid']) for x in rmed_pkgs_id]) log.debug('removed packages: %s', pkgs_to_rm) if rmed_pkgs_id: transaction = conn_db.begin() results_table = db_table('results') schedule_table = db_table('schedule') notes_table = db_table('notes') removed_packages_table = db_table('removed_packages') delete_results_query = results_table.delete().\ where(results_table.c.package_id == sql.bindparam('deleteid')) delete_schedule_query = schedule_table.delete().\ where(schedule_table.c.package_id == sql.bindparam('deleteid')) delete_notes_query = notes_table.delete().\ where(notes_table.c.package_id == sql.bindparam('deleteid')) delete_sources_query = sources_table.delete().\ where(sources_table.c.id == sql.bindparam('deleteid')) conn_db.execute(delete_results_query, rmed_pkgs_id) conn_db.execute(delete_schedule_query, rmed_pkgs_id) conn_db.execute(delete_notes_query, rmed_pkgs_id) conn_db.execute(delete_sources_query, rmed_pkgs_id) conn_db.execute(removed_packages_table.insert(), pkgs_to_rm) transaction.commit() # finally check whether the db has the correct number of packages query = "SELECT count(*) FROM sources WHERE suite='{}' " + \ "AND architecture='{}'" pkgs_end = query_db(query.format(suite, arch)) count_new_pkgs = len(set([x[0] for x in new_pkgs])) if int(pkgs_end[0][0]) != count_new_pkgs: print_critical_message('AH! The number of source in the Sources file' + ' is different than the one in the DB!') log.critical('source in the debian archive for the %s suite: %s', suite, str(count_new_pkgs)) log.critical('source in the reproducible db for the %s suite: %s', suite, str(pkgs_end[0][0])) sys.exit(1) if pkgs_to_add: log.info('Building pages for the new packages') gen_packages_html([Package(x['name']) for x in pkgs_to_add], no_clean=True)
else: log.error('Failed to get the ' + suite + 'sources') continue query = "SELECT s.name " + \ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \ "WHERE r.status='unreproducible' AND s.suite='{suite}'" try: pkgs = [x[0] for x in query_db(query.format(suite=suite))] except IndexError: log.error('Looks like there are no unreproducible packages...') p = Popen(('dd-list --stdin --sources ' + sources.name).split(), stdout=PIPE, stdin=PIPE, stderr=PIPE) out, err = p.communicate(input=('\n'.join(pkgs)).encode()) if err: log.error('dd-list printed some errors:\n' + err.decode()) log.debug('dd-list output:\n' + out.decode()) html = '<p>The following maintainers and uploaders are listed ' html += 'for packages in ' + suite + ' which have built ' html += 'unreproducibly. Please note that the while the link ' html += 'always points to the amd64 version, it\'s possible that' html += 'the unreproducibility is only present in another architecture(s).</p>\n<p><pre>' out = out.decode().splitlines() get_mail = re.compile('<(.*)>') for line in out: if line[0:3] == ' ': line = line.strip().split(None, 1) html += ' ' # the final strip() is to avoid a newline html += Package(line[0]).html_link(suite, arch).strip() try:
def rest(scheduling_args, requester, local, suite, arch): "Actually schedule a package for a single suite on a single arch." # Shorter names reason = scheduling_args.message issue = scheduling_args.issue status = scheduling_args.status built_after = scheduling_args.after built_before = scheduling_args.before packages = scheduling_args.packages artifacts = scheduling_args.keep_artifacts notify = scheduling_args.notify notify_on_start = scheduling_args.notify_on_start dry_run = scheduling_args.dry_run log.info("Scheduling packages in %s/%s", arch, suite) ids = [] pkgs = [] query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}' AND architecture='{arch}'""" query2 = """SELECT p.date_build_started FROM sources AS s JOIN schedule as p ON p.package_id=s.id WHERE p.package_id='{id}'""" for pkg in set(packages): # test whether the package actually exists result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch)) # tests whether the package is already building try: result2 = query_db(query2.format(id=result[0][0])) except IndexError: log.error('%sThe package %s is not available in %s/%s%s', bcolors.FAIL, pkg, suite, arch, bcolors.ENDC) continue try: if not result2[0][0]: ids.append(result[0][0]) pkgs.append(pkg) else: log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' + 'already building, not scheduling it.' + bcolors.ENDC) except IndexError: # it's not in the schedule ids.append(result[0][0]) pkgs.append(pkg) def compose_irc_message(): "One-shot closure to limit scope of the following local variables." blablabla = '✂…' if len(' '.join(pkgs)) > 257 else '' packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else '' trailing = ' - artifacts will be preserved' if artifacts else '' trailing += ' - with irc notification' if notify else '' trailing += ' - notify on start too' if notify_on_start else '' message = requester + ' scheduled ' + packages_txt + \ 'in ' + suite + '/' + arch if reason: message += ', reason: \'' + reason + '\'' message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing return message info_msg = compose_irc_message() del compose_irc_message # these packages are manually scheduled, so should have high priority, # so schedule them in the past, so they are picked earlier :) # the current date is subtracted twice, so it sorts before early scheduling # schedule on the full hour so we can recognize them easily epoch = int(time.time()) now = datetime.now() days = int(now.strftime('%j')) * 2 hours = int(now.strftime('%H')) * 2 minutes = int(now.strftime('%M')) time_delta = timedelta(days=days, hours=hours, minutes=minutes) date = (now - time_delta).strftime('%Y-%m-%d %H:%M') log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta)) # a single person can't schedule more than 500 packages in the same day; this # is actually easy to bypass, but let's give some trust to the Debian people query = """SELECT count(*) FROM manual_scheduler WHERE requester = '{}' AND date_request > '{}'""" try: amount = int( query_db(query.format(requester, int(time.time() - 86400)))[0][0]) except IndexError: amount = 0 log.debug(requester + ' already scheduled ' + str(amount) + ' packages today') if amount + len(ids) > 500 and not local: log.error( bcolors.FAIL + 'You have exceeded the maximum number of manual ' + 'reschedulings allowed for a day. Please ask in ' + '#debian-reproducible if you need to schedule more packages.' + bcolors.ENDC) sys.exit(1) # do the actual scheduling add_to_schedule = [] update_schedule = [] save_schedule = [] artifacts_value = 1 if artifacts else 0 if notify_on_start: do_notify = 2 elif notify or artifacts: do_notify = 1 else: do_notify = 0 schedule_table = db_table('schedule') if ids: existing_pkg_ids = dict( query_db( sql.select([ schedule_table.c.package_id, schedule_table.c.id, ]).where(schedule_table.c.package_id.in_(ids)))) for id in ids: if id in existing_pkg_ids: update_schedule.append({ 'update_id': existing_pkg_ids[id], 'package_id': id, 'date_scheduled': date, 'save_artifacts': artifacts_value, 'notify': str(do_notify), 'scheduler': requester, }) else: add_to_schedule.append({ 'package_id': id, 'date_scheduled': date, 'save_artifacts': artifacts_value, 'notify': str(do_notify), 'scheduler': requester, }) save_schedule.append({ 'package_id': id, 'requester': requester, 'date_request': epoch, }) log.debug('Packages about to be scheduled: ' + str(add_to_schedule) + str(update_schedule)) update_schedule_query = schedule_table.update().\ where(schedule_table.c.id == sql.bindparam('update_id')) insert_schedule_query = schedule_table.insert() insert_manual_query = db_table('manual_scheduler').insert() if not dry_run: transaction = conn_db.begin() if add_to_schedule: conn_db.execute(insert_schedule_query, add_to_schedule) if update_schedule: conn_db.execute(update_schedule_query, update_schedule) if save_schedule: conn_db.execute(insert_manual_query, save_schedule) transaction.commit() else: log.info('Ran with --dry-run, scheduled nothing') log.info(bcolors.GOOD + info_msg + bcolors.ENDC) if not (local and requester == "jenkins maintenance job") and len(ids) != 0: if not dry_run: # Always announce on -changes irc_msg(info_msg, 'debian-reproducible-changes') # Announce some messages on main channel if notify_on_start or artifacts: irc_msg(info_msg)
result = sorted(query_db(query)) log.info('\tprocessing ' + str(len(result))) keys = ['package', 'version', 'suite', 'architecture', 'status', 'build_date'] crossarchkeys = ['package', 'version', 'suite', 'status'] archdetailkeys = ['architecture', 'version', 'status', 'build_date'] # crossarch is a dictionary of all packages used to build a summary of the # package's test results across all archs (for suite=unstable only) crossarch = {} crossarchversions = {} for row in result: pkg = dict(zip(keys, row)) log.debug(pkg) output.append(pkg) # tracker.d.o should only care about results in testing if pkg['suite'] == 'buster': package = pkg['package'] if package in crossarch: # compare statuses to get cross-arch package status status1 = crossarch[package]['status'] status2 = pkg['status'] newstatus = '' # compare the versions (only keep most up to date!) version1 = crossarch[package]['version'] version2 = pkg['version']
def parse_args(): parser = argparse.ArgumentParser( description='Reschedule packages to re-test their reproducibility', epilog='The build results will be announced on the #debian-reproducible' ' IRC channel if -n is provided. Specifying two or more filters' ' (namely two or more -r/-i/-t/-b) means "all packages with that' ' issue AND that status AND that date". Blacklisted package ' "can't be selected by a filter, but needs to be explitely listed" ' in the package list.') parser.add_argument('--dry-run', action='store_true') parser.add_argument('--null', action='store_true', help='The arguments are ' 'considered null-separated and coming from stdin.') parser.add_argument('-k', '--keep-artifacts', action='store_true', help='Save artifacts (for further offline study).') parser.add_argument('-n', '--notify', action='store_true', help='Notify the channel when the build finishes.') parser.add_argument( '--notify-on-start', action='store_true', help='Also ' 'notify when the build starts, linking to the build url.') parser.add_argument( '-m', '--message', default='', help='A text to be sent to the IRC channel when notifying' + ' about the scheduling.') parser.add_argument('-r', '--status', required=False, help='Schedule all package with this status.') parser.add_argument('-i', '--issue', required=False, help='Schedule all packages with this issue.') parser.add_argument('-t', '--after', required=False, help='Schedule all packages built after this date.') parser.add_argument('-b', '--before', required=False, help='Schedule all packages built before this date.') parser.add_argument('-a', '--architecture', required=False, default='amd64', help='Specify the architectures to schedule in ' + '(space or comma separated).' + "Default: 'amd64'.") parser.add_argument( '-s', '--suite', required=False, default='unstable', help= "Specify the suites to schedule in (space or comma separated). Default: 'unstable'." ) parser.add_argument('packages', metavar='package', nargs='*', help='Space seperated list of packages to reschedule.') # only consider "unknown_args", i.e. the arguments that the common.py script # doesn't know about and therefor ignored. scheduling_args = parser.parse_known_args(unknown_args)[0] if scheduling_args.null: scheduling_args = parser.parse_known_args( sys.stdin.read().split('\0'))[0] scheduling_args.packages = [x for x in scheduling_args.packages if x] if scheduling_args.notify_on_start: scheduling_args.notify = True # this variable is expected to come from the remote host try: requester = os.environ['LC_USER'] except KeyError: log.critical( bcolors.FAIL + 'You should use the provided script to ' 'schedule packages. Ask in #debian-reproducible if you have ' 'trouble with that.' + bcolors.ENDC) sys.exit(1) # this variable is set by reproducible scripts and so it only available in calls made on the local host (=main node) try: local = True if os.environ['LOCAL_CALL'] == 'true' else False except KeyError: local = False # Shorter names suites = [ x.strip() for x in re.compile(r'[, \t]').split(scheduling_args.suite or "") ] suites = [x for x in suites if x] archs = [ x.strip() for x in re.compile(r'[, \t]').split( scheduling_args.architecture or "") ] archs = [x for x in archs if x] reason = scheduling_args.message issue = scheduling_args.issue status = scheduling_args.status built_after = scheduling_args.after built_before = scheduling_args.before packages = scheduling_args.packages artifacts = scheduling_args.keep_artifacts notify = scheduling_args.notify notify_on_start = scheduling_args.notify_on_start dry_run = scheduling_args.dry_run log.debug('Requester: ' + requester) log.debug('Dry run: ' + str(dry_run)) log.debug('Local call: ' + str(local)) log.debug('Reason: ' + reason) log.debug('Artifacts: ' + str(artifacts)) log.debug('Notify: ' + str(notify)) log.debug('Debug url: ' + str(notify_on_start)) log.debug('Issue: ' + issue if issue else str(None)) log.debug('Status: ' + status if status else str(None)) log.debug('Date: after ' + built_after if built_after else str(None) + ' before ' + built_before if built_before else str(None)) log.debug('Suites: ' + repr(suites)) log.debug('Architectures: ' + repr(archs)) log.debug('Packages: ' + ' '.join(packages)) if not suites[0]: log.critical('You need to specify the suite name') sys.exit(1) if set(suites) - set(SUITES): # Some command-line suites don't exist. log.critical('Some of the specified suites %r are not being tested.', suites) log.critical('Please choose among ' + ', '.join(SUITES) + '.') sys.exit(1) if set(archs) - set(ARCHS): # Some command-line archs don't exist. log.critical('Some of the specified archs %r are not being tested.', archs) log.critical('Please choose among' + ', '.join(ARCHS) + '.') sys.exit(1) if issue or status or built_after or built_before: # Note: this .extend() operation modifies scheduling_args.packages, which # is used by rest() for suite in suites: for arch in archs: packages.extend( packages_matching_criteria( arch, suite, (issue, status, built_after, built_before), )) del arch del suite if len(packages) > 50 and notify: log.critical(bcolors.RED + bcolors.BOLD) subprocess.run(('figlet', 'No.')) log.critical(bcolors.FAIL + 'Do not reschedule more than 50 packages ', 'with notification.\nIf you think you need to do this, ', 'please discuss this with the IRC channel first.', bcolors.ENDC) sys.exit(1) if artifacts: log.info('The artifacts of the build(s) will be saved to the location ' 'mentioned at the end of the build log(s).') if notify_on_start: log.info('The channel will be notified when the build starts') return scheduling_args, requester, local, suites, archs
def load_notes(): """ format: { 'package_name': [ {'suite': 'unstable', 'version': '0.0', 'comments': None, 'bugs': [1234, 5678], 'issues': ['blalba','auauau']}, {'suite': 'stretch', 'version': None, 'comments': 'strstr', 'bugs': [], 'issues': []}], 'package_name':<etc> } """ with open(NOTES) as fd: original = yaml.load(fd) log.info("notes loaded. There are " + str(len(original)) + " packages listed") notes = {} for pkg in sorted(original): assert isinstance(pkg, str) try: assert 'version' in original[pkg] except AssertionError: print_critical_message(pkg + ' does not include a version') irc_msg('The note for ' + pkg + ' does not include a version.') query = """SELECT s.id, s.version, s.suite FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.name='{pkg}' AND r.status != ''""" # AND s.architecture='amd64'""" query = query.format(pkg=pkg) result = query_db(query) if not result: log.info('Warning: This query produces no results: ' + query + '\nThis means there is no tested ' + 'package with the name ' + pkg) try: irc_msg( "There is problem with the note for {} (it may " "have been removed from the archive). Please check {} and {}" .format(pkg, os.environ['BUILD_URL'], "https://tracker.debian.org/pkg/" + pkg)) except KeyError: log.error( 'There is a problem with the note for %s - please ' 'check.', pkg) else: notes[pkg] = [] for suite in result: pkg_details = {} # https://image-store.slidesharecdn.com/c2c44a06-5e28-4296-8d87-419529750f6b-original.jpeg try: if apt_pkg.version_compare(str(original[pkg]['version']), str(suite[1])) > 0: continue except KeyError: pass pkg_details['suite'] = suite[2] try: pkg_details['version'] = original[pkg]['version'] except KeyError: pkg_details['version'] = '' pkg_details['comments'] = original[pkg]['comments'] if \ 'comments' in original[pkg] else None pkg_details['bugs'] = original[pkg]['bugs'] if \ 'bugs' in original[pkg] else [] pkg_details['issues'] = original[pkg]['issues'] if \ 'issues' in original[pkg] else [] pkg_details['id'] = int(suite[0]) log.debug('adding %s => %s', pkg, pkg_details) notes[pkg].append(pkg_details) log.info("notes checked. There are " + str(len(notes)) + " packages listed") return notes
def gen_packages_html(packages, no_clean=False): """ generate the /rb-pkg/package.HTML pages. packages should be a list of Package objects. """ total = len(packages) log.info('Generating the pages of ' + str(total) + ' package(s)') for package in sorted(packages, key=lambda x: x.name): assert isinstance(package, Package) gen_history_page(package) for arch in ARCHS: gen_history_page(package, arch) pkg = package.name notes_uri = '' notes_file = NOTES_PATH + '/' + pkg + '_note.html' if os.access(notes_file, os.R_OK): notes_uri = NOTES_URI + '/' + pkg + '_note.html' for suite in SUITES: for arch in ARCHS: status = package.builds[suite][arch].status version = package.builds[suite][arch].version build_date = package.builds[suite][arch].build_date if status is None: # the package is not in the checked suite continue log.debug('Generating the page of %s/%s/%s @ %s built at %s', pkg, suite, arch, version, build_date) suitearch_section_html, default_view, reproducible = \ gen_suitearch_section(package, suite, arch) history_uri = '{}/{}.html'.format(HISTORY_URI, pkg) history_archs = [] for a in ARCHS: history_archs.append({ 'history_arch': a, 'history_arch_uri': '{}/{}/{}.html'.format(HISTORY_URI, a, pkg) }) project_links = renderer.render(project_links_template) desturl = '{}{}/{}/{}/{}.html'.format( REPRODUCIBLE_URL, RB_PKG_URI, suite, arch, pkg, ) navigation_html = renderer.render( package_navigation_template, { 'package': pkg, 'suite': suite, 'arch': arch, 'version': version, 'history_uri': history_uri, 'history_archs': history_archs, 'notes_uri': notes_uri, 'notify_maintainer': package.notify_maint, 'suitearch_section_html': suitearch_section_html, 'project_links_html': project_links, 'reproducible': reproducible, 'dashboard_url': DISTRO_URL, 'desturl': desturl, }) body_html = renderer.render(package_page_template, { 'default_view': default_view, }) destfile = os.path.join(RB_PKG_PATH, suite, arch, pkg + '.html') title = pkg + ' - reproducible builds result' write_html_page(title=title, body=body_html, destfile=destfile, no_header=True, noendpage=True, left_nav_html=navigation_html) log.debug("Package page generated at " + desturl) # Optionally generate a page in which the main iframe shows the # diffoscope results by default. Needed for navigation between # diffoscope pages for different suites/archs eversion = strip_epoch(version) dbd_links = get_and_clean_dbd_links(pkg, eversion, suite, arch, status) # only generate the diffoscope page if diffoscope results exist if 'dbd_uri' in dbd_links: body_html = renderer.render( package_page_template, { 'default_view': dbd_links['dbd_uri'], }) destfile = dbd_links['dbd_page_file'] desturl = REPRODUCIBLE_URL + "/" + dbd_links['dbd_page_uri'] title = "{} ({}) diffoscope results in {}/{}".format( pkg, version, suite, arch) write_html_page(title=title, body=body_html, destfile=destfile, no_header=True, noendpage=True, left_nav_html=navigation_html) log.debug("Package diffoscope page generated at " + desturl) if not no_clean: purge_old_pages() # housekeep is always good