def update_stats(suite, arch, stats, pkgset_name): result = query_db(""" SELECT datum, meta_pkg, suite FROM stats_meta_pkg_state WHERE datum = '{date}' AND suite = '{suite}' AND architecture = '{arch}' AND meta_pkg = '{name}' """.format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name)) # if there is not a result for this day, add one if not result: insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \ "'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \ "'{count_bad}', '{count_ugly}', '{count_rest}')" query_db( insert.format(date=YESTERDAY, suite=suite, arch=arch, pkgset_name=pkgset_name, count_good=stats['count_good'], count_bad=stats['count_bad'], count_ugly=stats['count_ugly'], count_rest=stats['count_rest'])) log.info("Updating db entry for meta pkgset %s in %s/%s on %s.", pkgset_name, suite, arch, YESTERDAY) else: log.debug( "Not updating db entry for meta pkgset %s in %s/%s on %s as one exists already.", pkgset_name, suite, arch, YESTERDAY)
def update_stats_breakages(diffoscope_timeouts, diffoscope_crashes): # we only do stats up until yesterday YESTERDAY = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d') result = query_db(""" SELECT datum, diffoscope_timeouts, diffoscope_crashes FROM stats_breakages WHERE datum = '{date}' """.format(date=YESTERDAY)) # if there is not a result for this day, add one if not result: insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \ "'{diffoscope_timeouts}', '{diffoscope_crashes}')" query_db( insert.format(date=YESTERDAY, diffoscope_timeouts=diffoscope_timeouts, diffoscope_crashes=diffoscope_crashes)) log.info( "Updating db table stats_breakages on %s with %s timeouts and %s crashes.", YESTERDAY, diffoscope_timeouts, diffoscope_crashes) else: log.debug( "Not updating db table stats_breakages as it already has data for %s.", YESTERDAY)
def db_update(): """ Update the database schema. Get a list of queries to perform from schema_updates. The need for an update is detected by checking the biggest value in the rb_schema table against the biggest value in the schema_updates dictionary. """ current = query_db('SELECT MAX(version) FROM rb_schema')[0][0] if not current: log.warning('This is probably a new database, there are no ' + 'previous updates noted') current = 0 last = max(schema_updates.keys()) if current == last: return False if current > last: print_critiacal_message('The active database schema is higher than' + ' the last update available.\nPlease check!') sys.exit(1) log.info('Found schema updates.') for update in range(current + 1, last + 1): log.info('Applying database update #' + str(update) + '. Queries:') startTime = datetime.now() for query in schema_updates[update]: log.info('\t' + query) query_db(query) log.info( str(len(schema_updates[update])) + ' queries executed in ' + str(datetime.now() - startTime)) return True
def gather_meta_stats(suite, arch, pkgset_name): pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite, pkgset_name + '.pkgset') try: with open(pkgset_file) as f: pkgset_list = [s.strip() for s in f.readlines()] except FileNotFoundError: log.warning('No meta package set information exists at ' + pkgset_file) return {} if not pkgset_list: log.warning('No packages listed for package set: ' + pkgset_name) return {} package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')" root_query = """ SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND date(r.build_date)<='{date}' AND {package_where} """.format(suite=suite, arch=arch, date=YESTERDAY, package_where=package_where) stats = {} good = query_db(root_query + "AND r.status = 'reproducible' " + "ORDER BY s.name;") stats['good'] = [t[0] for t in good] stats['count_good'] = len(stats['good']) bad = query_db(root_query + "AND r.status = 'FTBR'" + "ORDER BY r.build_date;") stats['bad'] = [t[0] for t in bad] stats['count_bad'] = len(stats['bad']) ugly = query_db(root_query + "AND r.status = 'FTBFS'" + "ORDER BY r.build_date;") stats['ugly'] = [t[0] for t in ugly] stats['count_ugly'] = len(stats['ugly']) rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " + "r.status != 'FTBR' AND " + "r.status != 'reproducible') ORDER BY r.build_date;") stats['rest'] = [t[0] for t in rest] stats['count_rest'] = len(stats['rest']) stats['count_all'] = (stats['count_good'] + stats['count_bad'] + stats['count_ugly'] + stats['count_rest']) stats['count_all'] = stats['count_all'] if stats['count_all'] else 1 stats['percent_good'] = percent(stats['count_good'], stats['count_all']) stats['percent_bad'] = percent(stats['count_bad'], stats['count_all']) stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all']) stats['percent_rest'] = percent(stats['count_rest'], stats['count_all']) return stats
def db_create_tables(): """ Check whether all tables are present, and create them if not. The check is done against sqlite_master, a reserved sqlite table containing all database metadata. """ changed = False for table in db_schema: if not table_exists(table['name']): log.warning(table['name'] + ' does not exists. Creating...') for query in table['query']: log.info('\t' + re.sub(' +', ' ', query.replace('\n', ' '))) query_db(query) changed = True return changed
def pbuilder_dep_fail(): log.info('running pbuilder_dep_fail check...') bad_pkgs = [] # we only care about these failures in the !unstable !experimental suites # as they happen all the time in there, as packages are buggy # and specific versions also come and go query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status = 'FTBFS' AND s.suite NOT IN ('unstable', 'experimental') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.rbuild.log' if os.access(rbuild, os.R_OK): log.debug('\tlooking at ' + rbuild) with open(rbuild, "br") as fd: for line in fd: if re.search(b'E: pbuilder-satisfydepends failed.', line): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') failed to satisfy its dependencies.') return bad_pkgs
def unrep_with_dbd_issues(): log.info('running unrep_with_dbd_issues check...') without_dbd = [] bad_dbd = [] sources_without_dbd = set() query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.diffoscope.html' if not os.access(dbd, os.R_OK): without_dbd.append((pkg, version, suite, arch)) sources_without_dbd.add(pkg) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') is ' 'unreproducible without diffoscope file.') else: log.debug(dbd + ' found.') data = open(dbd, 'br').read(3) if b'<' not in data: bad_dbd.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has ' 'diffoscope output, but it does not seem to ' 'be an HTML page.') sources_without_dbd.add(pkg) return without_dbd, bad_dbd, sources_without_dbd
def packages_matching_criteria(arch, suite, criteria): "Return a list of packages in (SUITE, ARCH) matching the given CRITERIA." # TODO: Rewrite this function to query all suites/archs in one go issue, status, built_after, built_before = criteria del criteria formatter = dict(suite=suite, arch=arch, notes_table='') log.info('Querying packages with given issues/status...') query = "SELECT s.name " + \ "FROM sources AS s, {notes_table} results AS r " + \ "WHERE r.package_id=s.id " + \ "AND s.architecture= '{arch}' " + \ "AND s.suite = '{suite}' AND r.status != 'blacklisted' " if issue: query += "AND n.package_id=s.id AND n.issues LIKE '%%{issue}%%' " formatter['issue'] = issue formatter['notes_table'] = "notes AS n," if status: query += "AND r.status = '{status}'" formatter['status'] = status if built_after: query += "AND r.build_date > '{built_after}' " formatter['built_after'] = built_after if built_before: query += "AND r.build_date < '{built_before}' " formatter['built_before'] = built_before results = query_db(query.format_map(formatter)) results = [x for (x, ) in results] log.info('Selected packages: ' + ' '.join(results)) return results
def create_pkgset_graph(png_file, suite, arch, pkgset_name): table = "stats_meta_pkg_state" columns = ["datum", "reproducible", "FTBR", "FTBFS", "other"] where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \ (suite, arch, pkgset_name) if arch == 'i386': # i386 only has pkg sets since later to make nicer graphs # (date added in commit 7f2525f7) where += " AND datum >= '2016-05-06'" query = "SELECT {fields} FROM {table} {where} ORDER BY datum".format( fields=", ".join(columns), table=table, where=where) result = query_db(query) result_rearranged = [dict(zip(columns, row)) for row in result] with create_temp_file(mode='w') as f: csv_tmp_file = f.name csv_writer = csv.DictWriter(f, columns) csv_writer.writeheader() csv_writer.writerows(result_rearranged) f.flush() graph_command = os.path.join(BIN_PATH, "make_graph.py") main_label = "Reproducibility status for packages in " + suite + \ " from " + pkgset_name y_label = "Amount (" + pkgset_name + " packages)" log.info("Creating graph for meta pkgset %s in %s/%s.", pkgset_name, suite, arch) check_call([ graph_command, csv_tmp_file, png_file, '4', main_label, y_label, '1920', '960' ])
def alien_rbpkg(): log.info('running alien_rbpkg check...') query = '''SELECT s.name FROM sources AS s WHERE s.name='{pkg}' AND s.suite='{suite}' AND s.architecture='{arch}' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(RB_PKG_PATH): if not files: continue # Extract the "suite" and "arch" from the directory structure if os.path.split(root)[1] == 'diffoscope-results': # We are presently inspecting package pages in the # RB_PKG_PATH/{{suite}}/{{arch}}/diffoscope-results directory suite, arch = os.path.split(root)[0].rsplit('/', 2)[1:] else: # We are presently inspecting package pages in the # RB_PKG_PATH/{{suite}}/{{arch}}/ directory suite, arch = root.rsplit('/', 2)[1:] for file in files: pkg = file.rsplit('.', 1)[0] if not query_db(query.format(pkg=pkg, suite=suite, arch=arch)): bad_files.append('/'.join([root, file])) log.warning('/'.join([root, file]) + ' should not be there') return bad_files
def gen_all_rb_pkg_pages(no_clean=False): query = 'SELECT DISTINCT name FROM sources WHERE suite = ANY(:s)' rows = query_db(sqlalchemy.text(query), s=SUITES) pkgs = [Package(str(i[0]), no_notes=True) for i in rows] log.info('Processing all %s package from all suites/architectures', len(pkgs)) gen_packages_html(pkgs, no_clean=True) # we clean at the end purge_old_pages()
def generate_live_status_table(arch): averagesql = select([ func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0) ]).where( and_( stats_build.c.status.in_(('reproducible', 'FTBR')), stats_build.c.name == sources.c.name, stats_build.c.suite == sources.c.suite, stats_build.c.architecture == sources.c.architecture, )).as_scalar() query = select([ sources.c.id, sources.c.suite, sources.c.architecture, sources.c.name, sources.c.version, schedule.c.date_build_started, results.c.status, results.c.build_duration, averagesql, schedule.c.job, ]).select_from(sources.join(schedule).join(results, isouter=True)).where( and_( schedule.c.date_build_started != None, sources.c.architecture == bindparam('arch'), )).order_by(schedule.c.date_scheduled) html = '' rows = query_db(query.params({'arch': arch})) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">src pkg id</th><th class="center">suite</th><th class="center">arch</th>' html += '<th class=\"center\">source package</th><th class=\"center\">version</th></th>' html += '<th class=\"center\">build started</th><th class=\"center\">previous build status</th>' html += '<th class=\"center\">previous build duration</th><th class=\"center\">average build duration</th><th class=\"center\">builder job</th>' html += '</tr>\n' counter = 0 for row in rows: counter += 1 suite = row[1] arch = row[2] pkg = row[3] duration = convert_into_hms_string(row[7]) avg_duration = convert_into_hms_string(row[8]) html += tab + '<tr><td> </td><td>' + str(row[0]) + '</td>' html += '<td>' + suite + '</td><td>' + arch + '</td>' html += '<td><code>' + Package(pkg).html_link( suite, arch, bugs=False) + '</code></td>' html += '<td>' + str(row[4]) + '</td><td>' + str(row[5]) + '</td>' html += '<td>' + convert_into_status_html( str(row[6]) ) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td>' html += '<td><a href="https://tests.reproducible-builds.org/cgi-bin/nph-logwatch?' + str( row[9]) + '">' + str(row[9]) + '</a></td>' html += '</tr>\n' html += '</table></p>\n' return html
def alien_log(directory=None): if directory is None: bad_files = [] for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH: bad_files.extend(alien_log(path)) return bad_files log.info('running alien_log check over ' + directory + '...') query = '''SELECT r.version FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}' AND s.architecture='{arch}' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(directory): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: # different file have differnt name patterns and different splitting needs if file.endswith('.diff.gz'): rsplit_level = 2 elif file.endswith('.gz'): rsplit_level = 3 else: rsplit_level = 2 try: pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1) except ValueError: log.critical( bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) continue try: rversion = query_db( query.format(pkg=pkg, suite=suite, arch=arch))[0][0] except IndexError: # that package is not known (or not yet tested) rversion = '' # continue towards the "bad file" path if strip_epoch(rversion) != version: try: if os.path.getmtime('/'.join([root, file ])) < time.time() - 86400: os.remove('/'.join([root, file])) log.warning( '/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.' ) else: bad_files.append('/'.join([root, file])) log.info( '/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.' ) except FileNotFoundError: pass # that bad file is already gone. return bad_files
def purge_old_pages(): for suite in SUITES: for arch in ARCHS: log.info('Removing old pages from ' + suite + '/' + arch + '.') try: presents = sorted( os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch)) except OSError as e: if e.errno != errno.ENOENT: # that's 'No such file or raise # directory' error (errno 17) presents = [] log.debug('page presents: ' + str(presents)) # get the existing packages query = "SELECT name, suite, architecture FROM sources " + \ "WHERE suite='{}' AND architecture='{}'".format(suite, arch) cur_pkgs = set([(p.name, p.suite, p.architecture) for p in query_db(query)]) for page in presents: # When diffoscope results exist for a package, we create a page # that displays the diffoscope results by default in the main iframe # in this subdirectory. Ignore this directory. if page == 'diffoscope-results': continue pkg = page.rsplit('.', 1)[0] if (pkg, suite, arch) not in cur_pkgs: log.info('There is no package named ' + pkg + ' from ' + suite + '/' + arch + ' in the database. ' + 'Removing old page.') os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' + page) # Additionally clean up the diffoscope results default pages log.info('Removing old pages from ' + suite + '/' + arch + '/diffoscope-results/.') try: presents = sorted( os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch + '/diffoscope-results')) except OSError as e: if e.errno != errno.ENOENT: # that's 'No such file or raise # directory' error (errno 17) presents = [] log.debug('diffoscope page presents: ' + str(presents)) for page in presents: pkg = page.rsplit('.', 1)[0] if (pkg, suite, arch) not in cur_pkgs: log.info('There is no package named ' + pkg + ' from ' + suite + '/' + arch + '/diffoscope-results in ' + 'the database. Removing old page.') os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' + 'diffoscope-results/' + page)
def alien_history(): log.info('running alien_history check...') result = query_db('SELECT DISTINCT name FROM sources') actual_packages = [x[0] for x in result] bad_files = [] for f in sorted(os.listdir(HISTORY_PATH)): full_path = os.path.join(HISTORY_PATH, f) if f.rsplit('.', 1)[0] not in actual_packages and not os.path.isdir(full_path): bad_files.append(full_path) os.remove(full_path) log.warning('%s should not be there so it has been removed.', full_path) return bad_files
def load_notes(): """ format: { 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} } """ with open(NOTES) as fd: possible_notes = yaml.load(fd) log.debug("notes loaded. There are " + str(len(possible_notes)) + " package listed") notes = copy.copy(possible_notes) for package in possible_notes: # check if every package listed on the notes try: # actually have been tested query = "SELECT s.name " + \ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \ "WHERE s.name='{pkg}' AND r.status != ''" query = query.format(pkg=package) query_db(query)[0] # just discard this result, we only care of its success except IndexError: log.warning("This query produces no results: " + query) log.warning("This means there is no tested package with the name " + package + ".") del notes[package] log.debug("notes checked. There are " + str(len(notes)) + " package listed") return notes
def query_untested_packages(suite, arch, limit): criteria = 'not tested before, randomly sorted' query = """SELECT DISTINCT * FROM ( SELECT sources.id, sources.name FROM sources WHERE sources.suite='{suite}' AND sources.architecture='{arch}' AND sources.id NOT IN (SELECT schedule.package_id FROM schedule) AND sources.id NOT IN (SELECT results.package_id FROM results) ORDER BY random() ) AS tmp LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit) packages = query_db(query) print_schedule_result(suite, arch, criteria, packages) return packages
def query_old_depwait_versions(suite, arch, limit): criteria = 'status depwait, no bug filed, tested at least 2 days ago, ' + \ 'no new version available, sorted by last build date' date = (datetime.now()-timedelta(days=2)).strftime('%Y-%m-%d %H:%M') query = """SELECT s.id, s.name, max(r.build_date) max_date FROM sources AS s JOIN results AS r ON s.id = r.package_id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND r.status='depwait' AND r.build_date < '{date}' AND s.id NOT IN (SELECT schedule.package_id FROM schedule) GROUP BY s.id, s.name ORDER BY max_date LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit, date=date) packages = query_db(query) print_schedule_result(suite, arch, criteria, packages) return packages
def query_e404_versions(suite, arch, limit): criteria = """tested at least 12h ago, status E404, sorted by last build date""" date = (datetime.now()-timedelta(days=0.5)).strftime('%Y-%m-%d %H:%M') query = """SELECT s.id, s.name, max(r.build_date) max_date FROM sources AS s JOIN results AS r ON s.id = r.package_id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND r.status = 'E404' AND r.build_date < '{date}' AND s.id NOT IN (SELECT schedule.package_id FROM schedule) GROUP BY s.id, s.name ORDER BY max_date LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit, date=date) packages = query_db(query) print_schedule_result(suite, arch, criteria, packages) return packages
def lack_rbuild(): log.info('running lack_rbuild check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ('blacklisted', '') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: rbuild = os.path.join(RBUILD_PATH, suite, arch) + \ '/{}_{}.rbuild.log.gz'.format(pkg, strip_epoch(version)) if not os.access(rbuild, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has been ' 'built, but a buildlog is missing.') return bad_pkgs
def alien_buildinfo(): log.info('running alien_buildinfo check...') query = '''SELECT r.version FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}' AND s.architecture='{arch}' AND r.status IN ('reproducible', 'unreproducible') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(BUILDINFO_PATH): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: try: pkg, version = file.rsplit('.', 1)[0].split('_')[:2] except ValueError: log.critical( bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) continue try: rversion = query_db( query.format(pkg=pkg, suite=suite, arch=arch))[0][0] except IndexError: # that package is not known (or not yet tested) rversion = '' # continue towards the "bad file" path if strip_epoch(rversion) != version: try: if os.path.getmtime('/'.join([root, file ])) < time.time() - 86400: os.remove('/'.join([root, file])) log.warning( '/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.' ) else: bad_files.append('/'.join([root, file])) log.info( '/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone.' ) except FileNotFoundError: pass # that bad file is already gone. return bad_files
def query_old_versions(suite, arch, limit): criteria = """tested at least {minimum_age} days ago, no new version available, sorted by last build date""".format(minimum_age=MINIMUM_AGE[arch]) date = (datetime.now()-timedelta(days=MINIMUM_AGE[arch]))\ .strftime('%Y-%m-%d %H:%M') query = """SELECT s.id, s.name, max(r.build_date) max_date FROM sources AS s JOIN results AS r ON s.id = r.package_id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND r.status != 'blacklisted' AND r.build_date < '{date}' AND s.id NOT IN (SELECT schedule.package_id FROM schedule) GROUP BY s.id, s.name ORDER BY max_date LIMIT {limit}""".format(suite=suite, arch=arch, date=date, limit=limit) packages = query_db(query) print_schedule_result(suite, arch, criteria, packages) return packages
def not_unrep_with_dbd_file(): log.info('running not_unrep_with_dbd_file check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != 'unreproducible' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.diffoscope.html' if os.access(dbd, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg + ' (' + version + ')' ' is not unreproducible.') return bad_pkgs
def lack_buildinfo(): log.info('running lack_buildinfo check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ('blacklisted', 'not for us', 'FTBFS', 'depwait', '404', '') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \ '_' + eversion + '_' + arch + '.buildinfo' if not os.access(buildinfo, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has been ' 'successfully built, but a .buildinfo is missing') return bad_pkgs
def query_new_versions(suite, arch, limit): criteria = 'tested before, new version available, sorted by last build date' query = """SELECT s.id, s.name, s.version, r.version, max(r.build_date) max_date FROM sources AS s JOIN results AS r ON s.id = r.package_id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND s.version != r.version AND r.status != 'blacklisted' AND s.id IN (SELECT package_id FROM results) AND s.id NOT IN (SELECT schedule.package_id FROM schedule) GROUP BY s.id, s.name, s.version, r.version ORDER BY max_date LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit) pkgs = query_db(query) # the next line avoids constant rescheduling of packages: # packages in our repository != official repo, # so they will always be selected by the query above # so we only accept them if there version is greater than the already tested one packages = [(x[0], x[1]) for x in pkgs if apt_pkg.version_compare(x[2], x[3]) > 0] print_schedule_result(suite, arch, criteria, packages) return packages
def create_breakages_graph(png_file, main_label): png_fullpath = os.path.join(DISTRO_BASE, png_file) table = "stats_breakages" columns = ["datum", "diffoscope_timeouts", "diffoscope_crashes"] query = "SELECT {fields} FROM {table} ORDER BY datum".format( fields=", ".join(columns), table=table) result = query_db(query) result_rearranged = [dict(zip(columns, row)) for row in result] with create_temp_file(mode='w') as f: csv_tmp_file = f.name csv_writer = csv.DictWriter(f, columns) csv_writer.writeheader() csv_writer.writerows(result_rearranged) f.flush() graph_command = os.path.join(BIN_PATH, "make_graph.py") y_label = "Amount (packages)" log.info("Creating graph for stats_breakges.") check_call([graph_command, csv_tmp_file, png_fullpath, '2', main_label, y_label, '1920', '960'])
def process_pkg(package, deactivate): if deactivate: _good('Deactivating notification for package ' + str(package)) flag = 0 else: _good('Activating notification for package ' + str(package)) flag = 1 sources_table = db_table('sources') update_query = sources_table.update().\ where(sources_table.c.name == package).\ values(notify_maintainer=flag) rows = conn_db.execute(update_query).rowcount if rows == 0: log.error(bcolors.FAIL + str(package) + ' does not exists') sys.exit(1) if DEBUG: log.debug('Double check the change:') query = 'SELECT * FROM sources WHERE name="{}"'.format(package) log.debug(query_db(query))
def build_page_section(page, section, suite, arch): try: if pages[page].get('global') and pages[page]['global']: suite = defaultsuite arch = defaultarch if pages[page].get('notes') and pages[page]['notes']: db_status = section['status'].value.name query = queries[section['query']].params({ 'status': db_status, 'suite': suite, 'arch': arch }) section['icon_status'] = section['status'].value.icon else: query = queries[section['query']].params({ 'suite': suite, 'arch': arch }) rows = query_db(query) except: print_critical_message('A query failed: %s' % query) raise html = '' footnote = True if rows else False if not rows: # there are no package in this set, do not output anything log.debug('empty query: %s' % query.compile(compile_kwargs={"literal_binds": True})) return (html, footnote) html += build_leading_text_section(section, rows, suite, arch) html += '<p>\n' + tab + '<code>\n' for row in rows: pkg = row[0] html += tab * 2 + Package(pkg).html_link(suite, arch) else: html += tab + '</code>\n' html += '</p>' if section.get('bottom'): html += section['bottom'] html = (tab * 2).join(html.splitlines(True)) return (html, footnote)
def generate_oldies(arch): log.info('Building the oldies page for ' + arch + '...') title = 'Oldest results on ' + arch html = '' for suite in SUITES: query = select([ sources.c.suite, sources.c.architecture, sources.c.name, results.c.status, results.c.build_date ]).select_from(results.join(sources)).where( and_(sources.c.suite == bindparam('suite'), sources.c.architecture == bindparam('arch'), results.c.status != 'blacklisted')).order_by( results.c.build_date).limit(15) text = Template('Oldest results on $suite/$arch:') rows = query_db(query.params({'arch': arch, 'suite': suite})) html += build_leading_text_section({'text': text}, rows, suite, arch) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>' html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n' for row in rows: # 0: suite, 1: arch, 2: pkg name 3: status 4: build date pkg = row[2] html += tab + '<tr><td> </td><td>' + row[0] + '</td>' html += '<td>' + row[1] + '</td><td><code>' html += Package(pkg).html_link(row[0], row[1]) html += '</code></td><td>' + convert_into_status_html(str( row[3])) + '</td><td>' + row[4] + '</td></tr>\n' html += '</table></p>\n' destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html' desturl = DISTRO_URL + '/index_' + arch + '_oldies.html' left_nav_html = create_main_navigation(arch=arch) write_html_page(title=title, body=html, destfile=destfile, style_note=True, refresh_every=60, left_nav_html=left_nav_html) log.info("Page generated at " + desturl)
def purge_old_notes(notes): removed_pages = [] to_rebuild = [] presents = sorted(os.listdir(NOTES_PATH)) for page in presents: pkg = page.rsplit('_', 1)[0] log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed') if pkg not in notes: log.info('There are no notes for ' + pkg + '. Removing old page.') os.remove(NOTES_PATH + '/' + page) removed_pages.append(pkg) for pkg in removed_pages: for suite in SUITES: try: query = "SELECT s.name " + \ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \ "WHERE s.name='{pkg}' AND r.status != '' AND s.suite='{suite}'" query = query.format(pkg=pkg, suite=suite) to_rebuild.append(query_db(query)[0][0]) except IndexError: # the package is not tested. this can happen if pass # a package got removed from the archive if to_rebuild: gen_packages_html([Package(x) for x in to_rebuild])