def alien_log(directory=None): if directory is None: bad_files = [] for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH: bad_files.extend(alien_log(path)) return bad_files log.info('running alien_log check over ' + directory + '...') query = '''SELECT r.version FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}' AND s.architecture='{arch}' ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(directory): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: # different file have differnt name patterns and different splitting needs if file.endswith('.diff.gz'): rsplit_level = 2 elif file.endswith('.gz'): rsplit_level = 3 else: rsplit_level = 2 try: pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1) except ValueError: log.critical( bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) continue try: rversion = query_db( query.format(pkg=pkg, suite=suite, arch=arch))[0][0] except IndexError: # that package is not known (or not yet tested) rversion = '' # continue towards the "bad file" path if strip_epoch(rversion) != version: try: if os.path.getmtime('/'.join([root, file ])) < time.time() - 86400: os.remove('/'.join([root, file])) log.warning( '/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.' ) else: bad_files.append('/'.join([root, file])) log.info( '/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.' ) except FileNotFoundError: pass # that bad file is already gone. return bad_files
def get_level(self, stage): try: return int(LIMITS[self.queue][self.arch][self.suite][stage][0]) except KeyError: log.error('No limit defined for the %s queue on %s/%s stage %s. ' 'Returning 1', self.queue, self.suite, self.arch, stage) return 1 except IndexError: log.critical('The limit is not in the format "(level, limit)". ' 'I can\'t guess what you want, giving up') sys.exit(1)
def purge_old_issues(issues): for root, dirs, files in os.walk(ISSUES_PATH): if not files: continue for file in files: try: issue = file.rsplit('_', 1)[0] except ValueError: log.critical('/'.join([root, file]) + ' does not seems like ' + 'a file that should be there') sys.exit(1) if issue not in issues: log.warning('removing ' + '/'.join([root, file]) + '...') os.remove('/'.join([root, file]))
def alien_buildinfo(): log.info('running alien_buildinfo check...') query = '''SELECT r.version FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}' AND s.architecture='{arch}' AND r.status IN ('reproducible', 'unreproducible') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(BUILDINFO_PATH): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: try: pkg, version = file.rsplit('.', 1)[0].split('_')[:2] except ValueError: log.critical( bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) continue try: rversion = query_db( query.format(pkg=pkg, suite=suite, arch=arch))[0][0] except IndexError: # that package is not known (or not yet tested) rversion = '' # continue towards the "bad file" path if strip_epoch(rversion) != version: try: if os.path.getmtime('/'.join([root, file ])) < time.time() - 86400: os.remove('/'.join([root, file])) log.warning( '/'.join([root, file]) + ' should not be there and and was older than a day so it was removed.' ) else: bad_files.append('/'.join([root, file])) log.info( '/'.join([root, file]) + ' should not be there, but is also less than 24h old and will probably soon be gone.' ) except FileNotFoundError: pass # that bad file is already gone. return bad_files
def get_limit(self, stage): try: limit = LIMITS[self.queue][self.arch][self.suite][stage] limit = limit[1] except KeyError: log.error('No limit defined for the %s queue on %s/%s stage %s. ' 'Returning 1', self.queue, self.suite, self.arch, stage) return 1 except IndexError: log.critical('The limit is not in the format "(level, limit)". ' 'I can\'t guess what you want, giving up') sys.exit(1) except TypeError: # this is the case of the default target if isinstance(limit, int): pass else: raise return int(limit)
def get_and_clean_dbd_links(package, eversion, suite, arch, status): links = get_dbd_links(package, eversion, suite, arch) dbd_links = {} if 'dbd_uri' in links: dbd_links = { 'dbd_page_file': links['dbd_page_file'], 'dbd_page_uri': links['dbd_page_uri'], 'dbd_uri': links['dbd_uri'], } else: if status == 'FTBR' and not args.ignore_missing_files: log.critical(DISTRO_URL + '/' + suite + '/' + arch + '/' + package + ' is unreproducible, but without diffoscope output.') # if there are no diffoscope results, we want to remove the old package # page used to display diffoscope results if os.access(links['dbd_page_file'], os.R_OK): os.remove(links['dbd_page_file']) return dbd_links
def parse_args(): parser = argparse.ArgumentParser( description='Reschedule packages to re-test their reproducibility', epilog='The build results will be announced on the #debian-reproducible' ' IRC channel if -n is provided. Specifying two or more filters' ' (namely two or more -r/-i/-t/-b) means "all packages with that' ' issue AND that status AND that date". Blacklisted package ' "can't be selected by a filter, but needs to be explitely listed" ' in the package list.') parser.add_argument('--dry-run', action='store_true') parser.add_argument('--null', action='store_true', help='The arguments are ' 'considered null-separated and coming from stdin.') parser.add_argument('-k', '--keep-artifacts', action='store_true', help='Save artifacts (for further offline study).') parser.add_argument('-n', '--notify', action='store_true', help='Notify the channel when the build finishes.') parser.add_argument( '--notify-on-start', action='store_true', help='Also ' 'notify when the build starts, linking to the build url.') parser.add_argument( '-m', '--message', default='', help='A text to be sent to the IRC channel when notifying' + ' about the scheduling.') parser.add_argument('-r', '--status', required=False, help='Schedule all package with this status.') parser.add_argument('-i', '--issue', required=False, help='Schedule all packages with this issue.') parser.add_argument('-t', '--after', required=False, help='Schedule all packages built after this date.') parser.add_argument('-b', '--before', required=False, help='Schedule all packages built before this date.') parser.add_argument('-a', '--architecture', required=False, default='amd64', help='Specify the architectures to schedule in ' + '(space or comma separated).' + "Default: 'amd64'.") parser.add_argument( '-s', '--suite', required=False, default='unstable', help= "Specify the suites to schedule in (space or comma separated). Default: 'unstable'." ) parser.add_argument('packages', metavar='package', nargs='*', help='Space seperated list of packages to reschedule.') # only consider "unknown_args", i.e. the arguments that the common.py script # doesn't know about and therefor ignored. scheduling_args = parser.parse_known_args(unknown_args)[0] if scheduling_args.null: scheduling_args = parser.parse_known_args( sys.stdin.read().split('\0'))[0] scheduling_args.packages = [x for x in scheduling_args.packages if x] if scheduling_args.notify_on_start: scheduling_args.notify = True # this variable is expected to come from the remote host try: requester = os.environ['LC_USER'] except KeyError: log.critical( bcolors.FAIL + 'You should use the provided script to ' 'schedule packages. Ask in #debian-reproducible if you have ' 'trouble with that.' + bcolors.ENDC) sys.exit(1) # this variable is set by reproducible scripts and so it only available in calls made on the local host (=main node) try: local = True if os.environ['LOCAL_CALL'] == 'true' else False except KeyError: local = False # Shorter names suites = [ x.strip() for x in re.compile(r'[, \t]').split(scheduling_args.suite or "") ] suites = [x for x in suites if x] archs = [ x.strip() for x in re.compile(r'[, \t]').split( scheduling_args.architecture or "") ] archs = [x for x in archs if x] reason = scheduling_args.message issue = scheduling_args.issue status = scheduling_args.status built_after = scheduling_args.after built_before = scheduling_args.before packages = scheduling_args.packages artifacts = scheduling_args.keep_artifacts notify = scheduling_args.notify notify_on_start = scheduling_args.notify_on_start dry_run = scheduling_args.dry_run log.debug('Requester: ' + requester) log.debug('Dry run: ' + str(dry_run)) log.debug('Local call: ' + str(local)) log.debug('Reason: ' + reason) log.debug('Artifacts: ' + str(artifacts)) log.debug('Notify: ' + str(notify)) log.debug('Debug url: ' + str(notify_on_start)) log.debug('Issue: ' + issue if issue else str(None)) log.debug('Status: ' + status if status else str(None)) log.debug('Date: after ' + built_after if built_after else str(None) + ' before ' + built_before if built_before else str(None)) log.debug('Suites: ' + repr(suites)) log.debug('Architectures: ' + repr(archs)) log.debug('Packages: ' + ' '.join(packages)) if not suites[0]: log.critical('You need to specify the suite name') sys.exit(1) if set(suites) - set(SUITES): # Some command-line suites don't exist. log.critical('Some of the specified suites %r are not being tested.', suites) log.critical('Please choose among ' + ', '.join(SUITES) + '.') sys.exit(1) if set(archs) - set(ARCHS): # Some command-line archs don't exist. log.critical('Some of the specified archs %r are not being tested.', archs) log.critical('Please choose among' + ', '.join(ARCHS) + '.') sys.exit(1) if issue or status or built_after or built_before: # Note: this .extend() operation modifies scheduling_args.packages, which # is used by rest() for suite in suites: for arch in archs: packages.extend( packages_matching_criteria( arch, suite, (issue, status, built_after, built_before), )) del arch del suite if len(packages) > 50 and notify: log.critical(bcolors.RED + bcolors.BOLD) subprocess.run(('figlet', 'No.')) log.critical(bcolors.FAIL + 'Do not reschedule more than 50 packages ', 'with notification.\nIf you think you need to do this, ', 'please discuss this with the IRC channel first.', bcolors.ENDC) sys.exit(1) if artifacts: log.info('The artifacts of the build(s) will be saved to the location ' 'mentioned at the end of the build log(s).') if notify_on_start: log.info('The channel will be notified when the build starts') return scheduling_args, requester, local, suites, archs
def update_sources_db(suite, arch, sources): # extract relevant info (package name and version) from the sources file new_pkgs = set() newest_version = {} for src in deb822.Sources.iter_paragraphs(sources.split('\n')): pkg = (src['Package'], src['Version'], suite, arch) if 'Extra-Source-Only' in src and src['Extra-Source-Only'] == 'yes': log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg)) continue # only keep the most recent version of a src for each package/suite/arch key = src['Package'] + suite + arch if key in newest_version: oldversion = newest_version[key] oldpackage = (src['Package'], oldversion, suite, arch) new_pkgs.remove(oldpackage) newest_version[key] = src['Version'] new_pkgs.add(pkg) # get the current packages in the database query = "SELECT name, version, suite, architecture FROM sources " + \ "WHERE suite='{}' AND architecture='{}'".format(suite, arch) cur_pkgs = set([(p.name, p.version, p.suite, p.architecture) for p in query_db(query)]) pkgs_to_add = [] updated_pkgs = [] different_pkgs = [x for x in new_pkgs if x not in cur_pkgs] log.debug('Packages different in the archive and in the db: %s', different_pkgs) for pkg in different_pkgs: # pkg: (name, version, suite, arch) query = "SELECT id, version, notify_maintainer FROM sources " + \ "WHERE name='{}' AND suite='{}' AND architecture='{}'" query = query.format(pkg[0], pkg[2], pkg[3]) try: result = query_db(query)[0] except IndexError: # new package pkgs_to_add.append({ 'name': pkg[0], 'version': pkg[1], 'suite': pkg[2], 'architecture': pkg[3], }) continue pkg_id = result[0] old_version = result[1] notify_maint = int(result[2]) if apt_pkg.version_compare(pkg[1], old_version) > 0: log.debug('New version: ' + str(pkg) + ' (we had ' + old_version + ')') updated_pkgs.append({ 'update_id': pkg_id, 'name': pkg[0], 'version': pkg[1], 'suite': pkg[2], 'architecture': pkg[3], 'notify_maintainer': notify_maint, }) # Now actually update the database: sources_table = db_table('sources') # updated packages log.info('Pushing ' + str(len(updated_pkgs)) + ' updated packages to the database...') if updated_pkgs: transaction = conn_db.begin() update_query = sources_table.update().\ where(sources_table.c.id == sql.bindparam('update_id')) conn_db.execute(update_query, updated_pkgs) transaction.commit() # new packages if pkgs_to_add: log.info('Now inserting %i new sources in the database: %s', len(pkgs_to_add), pkgs_to_add) transaction = conn_db.begin() conn_db.execute(sources_table.insert(), pkgs_to_add) transaction.commit() # RM'ed packages cur_pkgs_name = [x[0] for x in cur_pkgs] new_pkgs_name = [x[0] for x in new_pkgs] rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name] log.info('Now deleting %i removed packages: %s', len(rmed_pkgs), rmed_pkgs) rmed_pkgs_id = [] pkgs_to_rm = [] query = "SELECT id FROM sources WHERE name='{}' AND suite='{}' " + \ "AND architecture='{}'" for pkg in rmed_pkgs: result = query_db(query.format(pkg, suite, arch)) rmed_pkgs_id.append({'deleteid': result[0][0]}) pkgs_to_rm.append({'name': pkg, 'suite': suite, 'architecture': arch}) log.debug('removed packages ID: %s', [str(x['deleteid']) for x in rmed_pkgs_id]) log.debug('removed packages: %s', pkgs_to_rm) if rmed_pkgs_id: transaction = conn_db.begin() results_table = db_table('results') schedule_table = db_table('schedule') notes_table = db_table('notes') removed_packages_table = db_table('removed_packages') delete_results_query = results_table.delete().\ where(results_table.c.package_id == sql.bindparam('deleteid')) delete_schedule_query = schedule_table.delete().\ where(schedule_table.c.package_id == sql.bindparam('deleteid')) delete_notes_query = notes_table.delete().\ where(notes_table.c.package_id == sql.bindparam('deleteid')) delete_sources_query = sources_table.delete().\ where(sources_table.c.id == sql.bindparam('deleteid')) conn_db.execute(delete_results_query, rmed_pkgs_id) conn_db.execute(delete_schedule_query, rmed_pkgs_id) conn_db.execute(delete_notes_query, rmed_pkgs_id) conn_db.execute(delete_sources_query, rmed_pkgs_id) conn_db.execute(removed_packages_table.insert(), pkgs_to_rm) transaction.commit() # finally check whether the db has the correct number of packages query = "SELECT count(*) FROM sources WHERE suite='{}' " + \ "AND architecture='{}'" pkgs_end = query_db(query.format(suite, arch)) count_new_pkgs = len(set([x[0] for x in new_pkgs])) if int(pkgs_end[0][0]) != count_new_pkgs: print_critical_message('AH! The number of source in the Sources file' + ' is different than the one in the DB!') log.critical('source in the debian archive for the %s suite: %s', suite, str(count_new_pkgs)) log.critical('source in the reproducible db for the %s suite: %s', suite, str(pkgs_end[0][0])) sys.exit(1) if pkgs_to_add: log.info('Building pages for the new packages') gen_packages_html([Package(x['name']) for x in pkgs_to_add], no_clean=True)
# these are here as an hack to be able to parse the command line from rblib import query_db, db_table from rblib.confparse import log, DEBUG from rblib.const import conn_db from rblib.models import Package from rblib.utils import bcolors from rblib.bugs import Udd from reproducible_html_packages import gen_packages_html from reproducible_html_indexes import build_page packages = local_args.packages if local_args.packages else [] maintainer = local_args.maintainer if not packages and not maintainer: log.critical(bcolors.FAIL + 'You have to specify at least a package ' + 'or a maintainer.' + bcolors.ENDC) def _good(text): log.info(bcolors.GOOD + str(text) + bcolors.ENDC) def process_pkg(package, deactivate): if deactivate: _good('Deactivating notification for package ' + str(package)) flag = 0 else: _good('Activating notification for package ' + str(package)) flag = 1 sources_table = db_table('sources')
def gen_suitearch_details(package, version, suite, arch, status, spokenstatus, build_date): eversion = strip_epoch(version) # epoch_free_version is too long pkg = Package(package) build = pkg.builds[suite][arch] context = {} default_view = '' # Make notes the default default view notes_file = NOTES_PATH + '/' + package + '_note.html' notes_uri = NOTES_URI + '/' + package + '_note.html' if os.access(notes_file, os.R_OK): default_view = notes_uri # Get summary context context['status_html'] = gen_status_link_icon(status, spokenstatus, None, suite, arch) context['build_date'] = build_date # Get diffoscope differences context dbd_links = get_dbd_links(package, eversion, suite, arch) dbd_uri = dbd_links.get('dbd_uri', '') if dbd_uri: context['dbd'] = { 'dbd_page_uri': dbd_links['dbd_page_uri'], 'dbdtxt_uri': dbd_links.get('dbdtxt_uri', ''), 'dbdjson_uri': dbd_links.get('dbdjson_uri', ''), } default_view = default_view if default_view else dbd_uri # Get buildinfo context if build.buildinfo: context['buildinfo_uri'] = build.buildinfo.url default_view = default_view if default_view else build.buildinfo.url elif not args.ignore_missing_files and status not in \ ('untested', 'blacklisted', 'FTBFS', 'NFU', 'depwait', '404'): log.critical('buildinfo not detected at ' + build.buildinfo.path) # Get rbuild, build2 and build diffs context if build.rbuild: context['rbuild_uri'] = build.rbuild.url context['rbuild_size'] = sizeof_fmt(build.rbuild.size) default_view = default_view if default_view else build.rbuild.url context['buildlogs'] = {} if build.build2 and build.logdiff: context['buildlogs']['build2_uri'] = build.build2.url context['buildlogs']['build2_size'] = build.build2.size context['buildlogs']['diff_uri'] = build.logdiff.url else: log.error('Either {} or {} is missing'.format( build.build2.path, build.logdiff.path)) elif status not in ('untested', 'blacklisted') and \ not args.ignore_missing_files: log.critical( DISTRO_URL + '/' + suite + '/' + arch + '/' + package + ' didn\'t produce a buildlog, even though it has been built.') context['has_buildloginfo'] = 'buildinfo_uri' in context or \ 'buildlogs' in context or \ 'rbuild_uri' in context default_view = '/untested.html' if not default_view else default_view suitearch_details_html = renderer.render(suitearch_details_template, context) return (suitearch_details_html, default_view)