def sizeof_fmt(num):
for unit in ['B', 'KB', 'MB', 'GB']:
if abs(num) < 1024.0:
if unit == 'GB':
log.error('The size of this file is bigger than 1 GB!')
log.error('Please check')
return str(int(round(float("%3f" % num), 0))) + "%s" % (unit)
num /= 1024.0
return str(int(round(float("%f" % num), 0))) + "%s" % ('Yi')
def get_level(self, stage):
try:
return int(LIMITS[self.queue][self.arch][self.suite][stage][0])
except KeyError:
log.error('No limit defined for the %s queue on %s/%s stage %s. '
'Returning 1', self.queue, self.suite, self.arch, stage)
return 1
except IndexError:
log.critical('The limit is not in the format "(level, limit)". '
'I can\'t guess what you want, giving up')
sys.exit(1)
def get_limit(self, stage):
try:
limit = LIMITS[self.queue][self.arch][self.suite][stage]
limit = limit[1]
except KeyError:
log.error('No limit defined for the %s queue on %s/%s stage %s. '
'Returning 1', self.queue, self.suite, self.arch, stage)
return 1
except IndexError:
log.critical('The limit is not in the format "(level, limit)". '
'I can\'t guess what you want, giving up')
sys.exit(1)
except TypeError:
# this is the case of the default target
if isinstance(limit, int):
pass
else:
raise
return int(limit)
def process_pkg(package, deactivate):
if deactivate:
_good('Deactivating notification for package ' + str(package))
flag = 0
else:
_good('Activating notification for package ' + str(package))
flag = 1
sources_table = db_table('sources')
update_query = sources_table.update().\
where(sources_table.c.name == package).\
values(notify_maintainer=flag)
rows = conn_db.execute(update_query).rowcount
if rows == 0:
log.error(bcolors.FAIL + str(package) + ' does not exists')
sys.exit(1)
if DEBUG:
log.debug('Double check the change:')
query = 'SELECT * FROM sources WHERE name="{}"'.format(package)
log.debug(query_db(query))
def load_notes():
"""
format:
{ 'package_name': [
{'suite': 'unstable', 'version': '0.0', 'comments': None,
'bugs': [1234, 5678], 'issues': ['blalba','auauau']},
{'suite': 'stretch', 'version': None, 'comments': 'strstr',
'bugs': [], 'issues': []}],
'package_name':<etc> }
"""
with open(NOTES) as fd:
original = yaml.load(fd)
log.info("notes loaded. There are " + str(len(original)) +
" packages listed")
notes = {}
for pkg in sorted(original):
assert isinstance(pkg, str)
try:
assert 'version' in original[pkg]
except AssertionError:
print_critical_message(pkg + ' does not include a version')
irc_msg('The note for ' + pkg + ' does not include a version.')
query = """SELECT s.id, s.version, s.suite
FROM results AS r JOIN sources AS s ON r.package_id=s.id
WHERE s.name='{pkg}' AND r.status != ''"""
# AND s.architecture='amd64'"""
query = query.format(pkg=pkg)
result = query_db(query)
if not result:
log.info('Warning: This query produces no results: ' + query +
'\nThis means there is no tested ' +
'package with the name ' + pkg)
try:
irc_msg(
"There is problem with the note for {} (it may "
"have been removed from the archive). Please check {} and {}"
.format(pkg, os.environ['BUILD_URL'],
"https://tracker.debian.org/pkg/" + pkg))
except KeyError:
log.error(
'There is a problem with the note for %s - please '
'check.', pkg)
else:
notes[pkg] = []
for suite in result:
pkg_details = {}
# https://image-store.slidesharecdn.com/c2c44a06-5e28-4296-8d87-419529750f6b-original.jpeg
try:
if apt_pkg.version_compare(str(original[pkg]['version']),
str(suite[1])) > 0:
continue
except KeyError:
pass
pkg_details['suite'] = suite[2]
try:
pkg_details['version'] = original[pkg]['version']
except KeyError:
pkg_details['version'] = ''
pkg_details['comments'] = original[pkg]['comments'] if \
'comments' in original[pkg] else None
pkg_details['bugs'] = original[pkg]['bugs'] if \
'bugs' in original[pkg] else []
pkg_details['issues'] = original[pkg]['issues'] if \
'issues' in original[pkg] else []
pkg_details['id'] = int(suite[0])
log.debug('adding %s => %s', pkg, pkg_details)
notes[pkg].append(pkg_details)
log.info("notes checked. There are " + str(len(notes)) +
" packages listed")
return notes
log.info('Found schema updates.')
for update in range(current + 1, last + 1):
log.info('Applying database update #' + str(update) + '. Queries:')
startTime = datetime.now()
for query in schema_updates[update]:
log.info('\t' + query)
query_db(query)
log.info(
str(len(schema_updates[update])) + ' queries executed in ' +
str(datetime.now() - startTime))
return True
if __name__ == '__main__':
changed_created = False
if table_exists('rb_schema'):
if not query_db('SELECT * FROM rb_schema'):
# table exists but there is nothing in it
changed_create = db_create_tables()
else:
log.error('There is no rb_schema table in the database.')
log.error('Will run a full db_create_tables().')
changed_created = db_create_tables()
changed = db_update()
if changed or changed_created:
log.info('Total execution time: ' +
str(datetime.now() -
datetime.strptime(now, "%Y-%m-%d-%H-%M-%S")))
else:
log.info('No pending updates.')
def build_leading_text_section(section, rows, suite, arch):
html = '<p>\n' + tab
total = len(rows)
count_total = int(
query_db(queries['count_total'].params({
'suite': suite,
'arch': arch
}))[0][0])
try:
percent = round(((total / count_total) * 100), 1)
except ZeroDivisionError:
log.error('Looks like there are either no tested package or no ' +
'packages available at all. Maybe it\'s a new database?')
percent = 0.0
try:
html += '<a href="' + section['icon_link'] + '" target="_parent">'
no_icon_link = False
except KeyError:
no_icon_link = True # to avoid closing the </a> tag below
if section.get('icon_status'):
html += '<img src="/static/' + section['icon_status']
html += '" alt="reproducible icon" />'
if not no_icon_link:
html += '</a>'
html += '\n' + tab
if section.get('text') and section.get('timespan'):
count = len(
query_db(queries[section['query2']].params({
'suite': suite,
'arch': arch
})))
percent = round(((count / count_total) * 100), 1)
timespan = section['timespan']
timespan_date = timespan_date_map[timespan]
timespan_count = int(
query_db(queries['count_timespan'].params({
'suite':
suite,
'arch':
arch,
'timespan_date':
timespan_date
}))[0][0])
try:
timespan_percent = round(((total / timespan_count) * 100), 1)
except ZeroDivisionError:
log.error('Looks like there are either no tested package or no ' +
'packages available at all. Maybe it\'s a new database?')
timespan_percent = 0
html += section['text'].substitute(tot=total,
percent=percent,
timespan_percent=timespan_percent,
timespan_count=timespan_count,
count_total=count_total,
count=count,
suite=suite,
arch=arch)
elif section.get('text'):
html += section['text'].substitute(tot=total,
percent=percent,
suite=suite,
arch=arch)
else:
log.warning('There is no text for this section')
html += '\n</p>\n'
return html
def rest(scheduling_args, requester, local, suite, arch):
"Actually schedule a package for a single suite on a single arch."
# Shorter names
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.info("Scheduling packages in %s/%s", arch, suite)
ids = []
pkgs = []
query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}'
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
WHERE p.package_id='{id}'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
# tests whether the package is already building
try:
result2 = query_db(query2.format(id=result[0][0]))
except IndexError:
log.error('%sThe package %s is not available in %s/%s%s',
bcolors.FAIL, pkg, suite, arch, bcolors.ENDC)
continue
try:
if not result2[0][0]:
ids.append(result[0][0])
pkgs.append(pkg)
else:
log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' +
'already building, not scheduling it.' +
bcolors.ENDC)
except IndexError:
# it's not in the schedule
ids.append(result[0][0])
pkgs.append(pkg)
def compose_irc_message():
"One-shot closure to limit scope of the following local variables."
blablabla = '✂…' if len(' '.join(pkgs)) > 257 else ''
packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else ''
trailing = ' - artifacts will be preserved' if artifacts else ''
trailing += ' - with irc notification' if notify else ''
trailing += ' - notify on start too' if notify_on_start else ''
message = requester + ' scheduled ' + packages_txt + \
'in ' + suite + '/' + arch
if reason:
message += ', reason: \'' + reason + '\''
message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing
return message
info_msg = compose_irc_message()
del compose_irc_message
# these packages are manually scheduled, so should have high priority,
# so schedule them in the past, so they are picked earlier :)
# the current date is subtracted twice, so it sorts before early scheduling
# schedule on the full hour so we can recognize them easily
epoch = int(time.time())
now = datetime.now()
days = int(now.strftime('%j')) * 2
hours = int(now.strftime('%H')) * 2
minutes = int(now.strftime('%M'))
time_delta = timedelta(days=days, hours=hours, minutes=minutes)
date = (now - time_delta).strftime('%Y-%m-%d %H:%M')
log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta))
# a single person can't schedule more than 500 packages in the same day; this
# is actually easy to bypass, but let's give some trust to the Debian people
query = """SELECT count(*) FROM manual_scheduler
WHERE requester = '{}' AND date_request > '{}'"""
try:
amount = int(
query_db(query.format(requester, int(time.time() - 86400)))[0][0])
except IndexError:
amount = 0
log.debug(requester + ' already scheduled ' + str(amount) +
' packages today')
if amount + len(ids) > 500 and not local:
log.error(
bcolors.FAIL + 'You have exceeded the maximum number of manual ' +
'reschedulings allowed for a day. Please ask in ' +
'#debian-reproducible if you need to schedule more packages.' +
bcolors.ENDC)
sys.exit(1)
# do the actual scheduling
add_to_schedule = []
update_schedule = []
save_schedule = []
artifacts_value = 1 if artifacts else 0
if notify_on_start:
do_notify = 2
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
schedule_table = db_table('schedule')
if ids:
existing_pkg_ids = dict(
query_db(
sql.select([
schedule_table.c.package_id,
schedule_table.c.id,
]).where(schedule_table.c.package_id.in_(ids))))
for id in ids:
if id in existing_pkg_ids:
update_schedule.append({
'update_id': existing_pkg_ids[id],
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
else:
add_to_schedule.append({
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
save_schedule.append({
'package_id': id,
'requester': requester,
'date_request': epoch,
})
log.debug('Packages about to be scheduled: ' + str(add_to_schedule) +
str(update_schedule))
update_schedule_query = schedule_table.update().\
where(schedule_table.c.id == sql.bindparam('update_id'))
insert_schedule_query = schedule_table.insert()
insert_manual_query = db_table('manual_scheduler').insert()
if not dry_run:
transaction = conn_db.begin()
if add_to_schedule:
conn_db.execute(insert_schedule_query, add_to_schedule)
if update_schedule:
conn_db.execute(update_schedule_query, update_schedule)
if save_schedule:
conn_db.execute(insert_manual_query, save_schedule)
transaction.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + info_msg + bcolors.ENDC)
if not (local
and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
# Always announce on -changes
irc_msg(info_msg, 'debian-reproducible-changes')
# Announce some messages on main channel
if notify_on_start or artifacts:
irc_msg(info_msg)
from rblib.const import PGDATABASE
parser = argparse.ArgumentParser(
description='Create new Postgres database (reproducibledb) from backup.',
epilog='This script creates a database and populates it with the result'
' of a "pg_dump". It will not run if the database already exists.'
' Database name and database user are defined in'
' reproducible_common.py .')
parser.add_argument("-f",
"--backup-file",
required=True,
help='result of a "pg_dump"')
args, unknown_args = parser.parse_known_args()
BACKUP_FILE = args.backup_file
if not os.access(BACKUP_FILE, os.R_OK):
log.error("Backup file does not exist.")
sys.exit(1)
# We skip the database connection because the database
# may not exist yet, but we would like to use the constants
# available in reproducible_common.py
sys.argv.append('--skip-database-connection')
from rblib.utils import print_critical_message
# Get database defined in reproducible_common.py
# Note: this script will ONLY run on a completely new DB. The backup
# file will be used to re-creates the schema and populate tables. If
# run on a database with existing information, it will error.
DB_NAME = PGDATABASE
DB_USER = '******'
from rblib.html import create_main_navigation, write_html_page
arch = 'amd64' # the arch is only relevant for link targets here
mirror = 'http://deb.debian.org/debian'
for suite in SUITES:
remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz'
os.makedirs('/tmp/reproducible', exist_ok=True)
with NamedTemporaryFile(dir='/tmp/reproducible') as sources:
log.info('Downloading sources file for ' + suite + ': ' + remotefile)
xfile = lzma.decompress(urlopen(remotefile).read())
if xfile:
sources.write(xfile)
else:
log.error('Failed to get the ' + suite + 'sources')
continue
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE r.status='unreproducible' AND s.suite='{suite}'"
try:
pkgs = [x[0] for x in query_db(query.format(suite=suite))]
except IndexError:
log.error('Looks like there are no unreproducible packages...')
p = Popen(('dd-list --stdin --sources ' + sources.name).split(),
stdout=PIPE, stdin=PIPE, stderr=PIPE)
out, err = p.communicate(input=('\n'.join(pkgs)).encode())
if err:
log.error('dd-list printed some errors:\n' + err.decode())
log.debug('dd-list output:\n' + out.decode())
def gen_suitearch_details(package, version, suite, arch, status, spokenstatus,
build_date):
eversion = strip_epoch(version) # epoch_free_version is too long
pkg = Package(package)
build = pkg.builds[suite][arch]
context = {}
default_view = ''
# Make notes the default default view
notes_file = NOTES_PATH + '/' + package + '_note.html'
notes_uri = NOTES_URI + '/' + package + '_note.html'
if os.access(notes_file, os.R_OK):
default_view = notes_uri
# Get summary context
context['status_html'] = gen_status_link_icon(status, spokenstatus, None,
suite, arch)
context['build_date'] = build_date
# Get diffoscope differences context
dbd_links = get_dbd_links(package, eversion, suite, arch)
dbd_uri = dbd_links.get('dbd_uri', '')
if dbd_uri:
context['dbd'] = {
'dbd_page_uri': dbd_links['dbd_page_uri'],
'dbdtxt_uri': dbd_links.get('dbdtxt_uri', ''),
'dbdjson_uri': dbd_links.get('dbdjson_uri', ''),
}
default_view = default_view if default_view else dbd_uri
# Get buildinfo context
if build.buildinfo:
context['buildinfo_uri'] = build.buildinfo.url
default_view = default_view if default_view else build.buildinfo.url
elif not args.ignore_missing_files and status not in \
('untested', 'blacklisted', 'FTBFS', 'NFU', 'depwait', '404'):
log.critical('buildinfo not detected at ' + build.buildinfo.path)
# Get rbuild, build2 and build diffs context
if build.rbuild:
context['rbuild_uri'] = build.rbuild.url
context['rbuild_size'] = sizeof_fmt(build.rbuild.size)
default_view = default_view if default_view else build.rbuild.url
context['buildlogs'] = {}
if build.build2 and build.logdiff:
context['buildlogs']['build2_uri'] = build.build2.url
context['buildlogs']['build2_size'] = build.build2.size
context['buildlogs']['diff_uri'] = build.logdiff.url
else:
log.error('Either {} or {} is missing'.format(
build.build2.path, build.logdiff.path))
elif status not in ('untested', 'blacklisted') and \
not args.ignore_missing_files:
log.critical(
DISTRO_URL + '/' + suite + '/' + arch + '/' + package +
' didn\'t produce a buildlog, even though it has been built.')
context['has_buildloginfo'] = 'buildinfo_uri' in context or \
'buildlogs' in context or \
'rbuild_uri' in context
default_view = '/untested.html' if not default_view else default_view
suitearch_details_html = renderer.render(suitearch_details_template,
context)
return (suitearch_details_html, default_view)
