def alien_log(directory=None):
if directory is None:
bad_files = []
for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH:
bad_files.extend(alien_log(path))
return bad_files
log.info('running alien_log check over ' + directory + '...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(directory):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
# different file have differnt name patterns and different splitting needs
if file.endswith('.diff.gz'):
rsplit_level = 2
elif file.endswith('.gz'):
rsplit_level = 3
else:
rsplit_level = 2
try:
pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1)
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone. Probably diffoscope is running on that package right now.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def get_level(self, stage):
try:
return int(LIMITS[self.queue][self.arch][self.suite][stage][0])
except KeyError:
log.error('No limit defined for the %s queue on %s/%s stage %s. '
'Returning 1', self.queue, self.suite, self.arch, stage)
return 1
except IndexError:
log.critical('The limit is not in the format "(level, limit)". '
'I can\'t guess what you want, giving up')
sys.exit(1)
def purge_old_issues(issues):
for root, dirs, files in os.walk(ISSUES_PATH):
if not files:
continue
for file in files:
try:
issue = file.rsplit('_', 1)[0]
except ValueError:
log.critical('/'.join([root, file]) + ' does not seems like '
+ 'a file that should be there')
sys.exit(1)
if issue not in issues:
log.warning('removing ' + '/'.join([root, file]) + '...')
os.remove('/'.join([root, file]))
def alien_buildinfo():
log.info('running alien_buildinfo check...')
query = '''SELECT r.version
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status != '' AND s.name='{pkg}' AND s.suite='{suite}'
AND s.architecture='{arch}'
AND r.status IN ('reproducible', 'unreproducible')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
bad_files = []
for root, dirs, files in os.walk(BUILDINFO_PATH):
if not files:
continue
suite, arch = root.rsplit('/', 2)[1:]
for file in files:
try:
pkg, version = file.rsplit('.', 1)[0].split('_')[:2]
except ValueError:
log.critical(
bcolors.FAIL + '/'.join([root, file]) +
' does not seem to be a file that should be there' +
bcolors.ENDC)
continue
try:
rversion = query_db(
query.format(pkg=pkg, suite=suite, arch=arch))[0][0]
except IndexError: # that package is not known (or not yet tested)
rversion = '' # continue towards the "bad file" path
if strip_epoch(rversion) != version:
try:
if os.path.getmtime('/'.join([root, file
])) < time.time() - 86400:
os.remove('/'.join([root, file]))
log.warning(
'/'.join([root, file]) +
' should not be there and and was older than a day so it was removed.'
)
else:
bad_files.append('/'.join([root, file]))
log.info(
'/'.join([root, file]) +
' should not be there, but is also less than 24h old and will probably soon be gone.'
)
except FileNotFoundError:
pass # that bad file is already gone.
return bad_files
def get_limit(self, stage):
try:
limit = LIMITS[self.queue][self.arch][self.suite][stage]
limit = limit[1]
except KeyError:
log.error('No limit defined for the %s queue on %s/%s stage %s. '
'Returning 1', self.queue, self.suite, self.arch, stage)
return 1
except IndexError:
log.critical('The limit is not in the format "(level, limit)". '
'I can\'t guess what you want, giving up')
sys.exit(1)
except TypeError:
# this is the case of the default target
if isinstance(limit, int):
pass
else:
raise
return int(limit)
def get_and_clean_dbd_links(package, eversion, suite, arch, status):
links = get_dbd_links(package, eversion, suite, arch)
dbd_links = {}
if 'dbd_uri' in links:
dbd_links = {
'dbd_page_file': links['dbd_page_file'],
'dbd_page_uri': links['dbd_page_uri'],
'dbd_uri': links['dbd_uri'],
}
else:
if status == 'FTBR' and not args.ignore_missing_files:
log.critical(DISTRO_URL + '/' + suite + '/' + arch + '/' +
package +
' is unreproducible, but without diffoscope output.')
# if there are no diffoscope results, we want to remove the old package
# page used to display diffoscope results
if os.access(links['dbd_page_file'], os.R_OK):
os.remove(links['dbd_page_file'])
return dbd_links
def parse_args():
parser = argparse.ArgumentParser(
description='Reschedule packages to re-test their reproducibility',
epilog='The build results will be announced on the #debian-reproducible'
' IRC channel if -n is provided. Specifying two or more filters'
' (namely two or more -r/-i/-t/-b) means "all packages with that'
' issue AND that status AND that date". Blacklisted package '
"can't be selected by a filter, but needs to be explitely listed"
' in the package list.')
parser.add_argument('--dry-run', action='store_true')
parser.add_argument('--null',
action='store_true',
help='The arguments are '
'considered null-separated and coming from stdin.')
parser.add_argument('-k',
'--keep-artifacts',
action='store_true',
help='Save artifacts (for further offline study).')
parser.add_argument('-n',
'--notify',
action='store_true',
help='Notify the channel when the build finishes.')
parser.add_argument(
'--notify-on-start',
action='store_true',
help='Also '
'notify when the build starts, linking to the build url.')
parser.add_argument(
'-m',
'--message',
default='',
help='A text to be sent to the IRC channel when notifying' +
' about the scheduling.')
parser.add_argument('-r',
'--status',
required=False,
help='Schedule all package with this status.')
parser.add_argument('-i',
'--issue',
required=False,
help='Schedule all packages with this issue.')
parser.add_argument('-t',
'--after',
required=False,
help='Schedule all packages built after this date.')
parser.add_argument('-b',
'--before',
required=False,
help='Schedule all packages built before this date.')
parser.add_argument('-a',
'--architecture',
required=False,
default='amd64',
help='Specify the architectures to schedule in ' +
'(space or comma separated).' + "Default: 'amd64'.")
parser.add_argument(
'-s',
'--suite',
required=False,
default='unstable',
help=
"Specify the suites to schedule in (space or comma separated). Default: 'unstable'."
)
parser.add_argument('packages',
metavar='package',
nargs='*',
help='Space seperated list of packages to reschedule.')
# only consider "unknown_args", i.e. the arguments that the common.py script
# doesn't know about and therefor ignored.
scheduling_args = parser.parse_known_args(unknown_args)[0]
if scheduling_args.null:
scheduling_args = parser.parse_known_args(
sys.stdin.read().split('\0'))[0]
scheduling_args.packages = [x for x in scheduling_args.packages if x]
if scheduling_args.notify_on_start:
scheduling_args.notify = True
# this variable is expected to come from the remote host
try:
requester = os.environ['LC_USER']
except KeyError:
log.critical(
bcolors.FAIL + 'You should use the provided script to '
'schedule packages. Ask in #debian-reproducible if you have '
'trouble with that.' + bcolors.ENDC)
sys.exit(1)
# this variable is set by reproducible scripts and so it only available in calls made on the local host (=main node)
try:
local = True if os.environ['LOCAL_CALL'] == 'true' else False
except KeyError:
local = False
# Shorter names
suites = [
x.strip()
for x in re.compile(r'[, \t]').split(scheduling_args.suite or "")
]
suites = [x for x in suites if x]
archs = [
x.strip() for x in re.compile(r'[, \t]').split(
scheduling_args.architecture or "")
]
archs = [x for x in archs if x]
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.debug('Requester: ' + requester)
log.debug('Dry run: ' + str(dry_run))
log.debug('Local call: ' + str(local))
log.debug('Reason: ' + reason)
log.debug('Artifacts: ' + str(artifacts))
log.debug('Notify: ' + str(notify))
log.debug('Debug url: ' + str(notify_on_start))
log.debug('Issue: ' + issue if issue else str(None))
log.debug('Status: ' + status if status else str(None))
log.debug('Date: after ' + built_after if built_after else str(None) +
' before ' + built_before if built_before else str(None))
log.debug('Suites: ' + repr(suites))
log.debug('Architectures: ' + repr(archs))
log.debug('Packages: ' + ' '.join(packages))
if not suites[0]:
log.critical('You need to specify the suite name')
sys.exit(1)
if set(suites) - set(SUITES): # Some command-line suites don't exist.
log.critical('Some of the specified suites %r are not being tested.',
suites)
log.critical('Please choose among ' + ', '.join(SUITES) + '.')
sys.exit(1)
if set(archs) - set(ARCHS): # Some command-line archs don't exist.
log.critical('Some of the specified archs %r are not being tested.',
archs)
log.critical('Please choose among' + ', '.join(ARCHS) + '.')
sys.exit(1)
if issue or status or built_after or built_before:
# Note: this .extend() operation modifies scheduling_args.packages, which
# is used by rest()
for suite in suites:
for arch in archs:
packages.extend(
packages_matching_criteria(
arch,
suite,
(issue, status, built_after, built_before),
))
del arch
del suite
if len(packages) > 50 and notify:
log.critical(bcolors.RED + bcolors.BOLD)
subprocess.run(('figlet', 'No.'))
log.critical(bcolors.FAIL + 'Do not reschedule more than 50 packages ',
'with notification.\nIf you think you need to do this, ',
'please discuss this with the IRC channel first.',
bcolors.ENDC)
sys.exit(1)
if artifacts:
log.info('The artifacts of the build(s) will be saved to the location '
'mentioned at the end of the build log(s).')
if notify_on_start:
log.info('The channel will be notified when the build starts')
return scheduling_args, requester, local, suites, archs
def update_sources_db(suite, arch, sources):
# extract relevant info (package name and version) from the sources file
new_pkgs = set()
newest_version = {}
for src in deb822.Sources.iter_paragraphs(sources.split('\n')):
pkg = (src['Package'], src['Version'], suite, arch)
if 'Extra-Source-Only' in src and src['Extra-Source-Only'] == 'yes':
log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg))
continue
# only keep the most recent version of a src for each package/suite/arch
key = src['Package'] + suite + arch
if key in newest_version:
oldversion = newest_version[key]
oldpackage = (src['Package'], oldversion, suite, arch)
new_pkgs.remove(oldpackage)
newest_version[key] = src['Version']
new_pkgs.add(pkg)
# get the current packages in the database
query = "SELECT name, version, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
cur_pkgs = set([(p.name, p.version, p.suite, p.architecture)
for p in query_db(query)])
pkgs_to_add = []
updated_pkgs = []
different_pkgs = [x for x in new_pkgs if x not in cur_pkgs]
log.debug('Packages different in the archive and in the db: %s',
different_pkgs)
for pkg in different_pkgs:
# pkg: (name, version, suite, arch)
query = "SELECT id, version, notify_maintainer FROM sources " + \
"WHERE name='{}' AND suite='{}' AND architecture='{}'"
query = query.format(pkg[0], pkg[2], pkg[3])
try:
result = query_db(query)[0]
except IndexError: # new package
pkgs_to_add.append({
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
})
continue
pkg_id = result[0]
old_version = result[1]
notify_maint = int(result[2])
if apt_pkg.version_compare(pkg[1], old_version) > 0:
log.debug('New version: ' + str(pkg) + ' (we had ' + old_version +
')')
updated_pkgs.append({
'update_id': pkg_id,
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
'notify_maintainer': notify_maint,
})
# Now actually update the database:
sources_table = db_table('sources')
# updated packages
log.info('Pushing ' + str(len(updated_pkgs)) +
' updated packages to the database...')
if updated_pkgs:
transaction = conn_db.begin()
update_query = sources_table.update().\
where(sources_table.c.id == sql.bindparam('update_id'))
conn_db.execute(update_query, updated_pkgs)
transaction.commit()
# new packages
if pkgs_to_add:
log.info('Now inserting %i new sources in the database: %s',
len(pkgs_to_add), pkgs_to_add)
transaction = conn_db.begin()
conn_db.execute(sources_table.insert(), pkgs_to_add)
transaction.commit()
# RM'ed packages
cur_pkgs_name = [x[0] for x in cur_pkgs]
new_pkgs_name = [x[0] for x in new_pkgs]
rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
log.info('Now deleting %i removed packages: %s', len(rmed_pkgs), rmed_pkgs)
rmed_pkgs_id = []
pkgs_to_rm = []
query = "SELECT id FROM sources WHERE name='{}' AND suite='{}' " + \
"AND architecture='{}'"
for pkg in rmed_pkgs:
result = query_db(query.format(pkg, suite, arch))
rmed_pkgs_id.append({'deleteid': result[0][0]})
pkgs_to_rm.append({'name': pkg, 'suite': suite, 'architecture': arch})
log.debug('removed packages ID: %s',
[str(x['deleteid']) for x in rmed_pkgs_id])
log.debug('removed packages: %s', pkgs_to_rm)
if rmed_pkgs_id:
transaction = conn_db.begin()
results_table = db_table('results')
schedule_table = db_table('schedule')
notes_table = db_table('notes')
removed_packages_table = db_table('removed_packages')
delete_results_query = results_table.delete().\
where(results_table.c.package_id == sql.bindparam('deleteid'))
delete_schedule_query = schedule_table.delete().\
where(schedule_table.c.package_id == sql.bindparam('deleteid'))
delete_notes_query = notes_table.delete().\
where(notes_table.c.package_id == sql.bindparam('deleteid'))
delete_sources_query = sources_table.delete().\
where(sources_table.c.id == sql.bindparam('deleteid'))
conn_db.execute(delete_results_query, rmed_pkgs_id)
conn_db.execute(delete_schedule_query, rmed_pkgs_id)
conn_db.execute(delete_notes_query, rmed_pkgs_id)
conn_db.execute(delete_sources_query, rmed_pkgs_id)
conn_db.execute(removed_packages_table.insert(), pkgs_to_rm)
transaction.commit()
# finally check whether the db has the correct number of packages
query = "SELECT count(*) FROM sources WHERE suite='{}' " + \
"AND architecture='{}'"
pkgs_end = query_db(query.format(suite, arch))
count_new_pkgs = len(set([x[0] for x in new_pkgs]))
if int(pkgs_end[0][0]) != count_new_pkgs:
print_critical_message('AH! The number of source in the Sources file' +
' is different than the one in the DB!')
log.critical('source in the debian archive for the %s suite: %s',
suite, str(count_new_pkgs))
log.critical('source in the reproducible db for the %s suite: %s',
suite, str(pkgs_end[0][0]))
sys.exit(1)
if pkgs_to_add:
log.info('Building pages for the new packages')
gen_packages_html([Package(x['name']) for x in pkgs_to_add],
no_clean=True)
# these are here as an hack to be able to parse the command line
from rblib import query_db, db_table
from rblib.confparse import log, DEBUG
from rblib.const import conn_db
from rblib.models import Package
from rblib.utils import bcolors
from rblib.bugs import Udd
from reproducible_html_packages import gen_packages_html
from reproducible_html_indexes import build_page
packages = local_args.packages if local_args.packages else []
maintainer = local_args.maintainer
if not packages and not maintainer:
log.critical(bcolors.FAIL + 'You have to specify at least a package ' +
'or a maintainer.' + bcolors.ENDC)
def _good(text):
log.info(bcolors.GOOD + str(text) + bcolors.ENDC)
def process_pkg(package, deactivate):
if deactivate:
_good('Deactivating notification for package ' + str(package))
flag = 0
else:
_good('Activating notification for package ' + str(package))
flag = 1
sources_table = db_table('sources')
def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
build_date):
eversion = strip_epoch(version) # epoch_free_version is too long
pkg = Package(package)
build = pkg.builds[suite][arch]
context = {}
default_view = ''
# Make notes the default default view
notes_file = NOTES_PATH + '/' + package + '_note.html'
notes_uri = NOTES_URI + '/' + package + '_note.html'
if os.access(notes_file, os.R_OK):
default_view = notes_uri
# Get summary context
context['status_html'] = gen_status_link_icon(status, spokenstatus, None,
suite, arch)
context['build_date'] = build_date
# Get diffoscope differences context
dbd_links = get_dbd_links(package, eversion, suite, arch)
dbd_uri = dbd_links.get('dbd_uri', '')
if dbd_uri:
context['dbd'] = {
'dbd_page_uri': dbd_links['dbd_page_uri'],
'dbdtxt_uri': dbd_links.get('dbdtxt_uri', ''),
'dbdjson_uri': dbd_links.get('dbdjson_uri', ''),
}
default_view = default_view if default_view else dbd_uri
# Get buildinfo context
if build.buildinfo:
context['buildinfo_uri'] = build.buildinfo.url
default_view = default_view if default_view else build.buildinfo.url
elif not args.ignore_missing_files and status not in \
('untested', 'blacklisted', 'FTBFS', 'NFU', 'depwait', '404'):
log.critical('buildinfo not detected at ' + build.buildinfo.path)
# Get rbuild, build2 and build diffs context
if build.rbuild:
context['rbuild_uri'] = build.rbuild.url
context['rbuild_size'] = sizeof_fmt(build.rbuild.size)
default_view = default_view if default_view else build.rbuild.url
context['buildlogs'] = {}
if build.build2 and build.logdiff:
context['buildlogs']['build2_uri'] = build.build2.url
context['buildlogs']['build2_size'] = build.build2.size
context['buildlogs']['diff_uri'] = build.logdiff.url
else:
log.error('Either {} or {} is missing'.format(
build.build2.path, build.logdiff.path))
elif status not in ('untested', 'blacklisted') and \
not args.ignore_missing_files:
log.critical(
DISTRO_URL + '/' + suite + '/' + arch + '/' + package +
' didn\'t produce a buildlog, even though it has been built.')
context['has_buildloginfo'] = 'buildinfo_uri' in context or \
'buildlogs' in context or \
'rbuild_uri' in context
default_view = '/untested.html' if not default_view else default_view
suitearch_details_html = renderer.render(suitearch_details_template,
context)
return (suitearch_details_html, default_view)
