def on_push(data):
"""Handle a webhook from github.
"""
print('Got webhook request:')
pprint(data)
if data['ref'] == 'refs/heads/master' and data['repository'][
'full_name'] in [
'qmk/qmk_firmware', 'qmk/chibios', 'qmk/chibios-contrib'
]:
print('Triggering update.')
num_commits = len(data['commits'])
name = data['pusher']['name']
repo = data['repository']['full_name']
old_hash = data['before']
new_hash = data['after']
commit_url = data['head_commit']['url']
forced = 'force ' if data['forced'] else ''
print(
update_needed.delay(repo=repo,
old_hash=old_hash,
new_hash=new_hash))
discord_msg(
'info',
'%s has %spushed %s commits to %s. Head is now %s. Changes: %s' %
(name, forced, num_commits, repo, new_hash, data['compare']))
def update_qmk_firmware():
"""Update the API from qmk_firmware after a push.
"""
if qmk_redis.get('qmk_needs_update'):
print('***', strftime(TIME_FORMAT))
print('Beginning qmk_firmware update.')
job = qmk_redis.enqueue(update_kb_redis, timeout=QMK_UPDATE_TIMEOUT)
print(
'Successfully enqueued job id %s at %s, polling every 2 seconds...'
% (job.id, strftime(TIME_FORMAT)))
start_time = time()
# Wait for the job to start running
if not wait_for_job_start(job):
print('QMK update queued for %s seconds! Giving up at %s!' %
(S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'warning',
'S3 cleanup queue longer than %s seconds! Queue length %s!'
% (S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs)))
return False
# Monitor the job while it runs
while job.get_status() == 'started':
if time() - start_time > QMK_UPDATE_TIMEOUT + 5:
print(
'QMK update took longer than %s seconds! Cancelling at %s!'
% (QMK_UPDATE_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'error', 'QMK update took longer than %s seconds!' %
(QMK_UPDATE_TIMEOUT, ))
break
sleep(2)
# Check over the results
if job.result:
hash = qmk_redis.get('qmk_api_last_updated')['git_hash']
message = 'QMK update completed successfully! API is current to ' + hash
print(message)
discord_msg('info', message)
if ERROR_LOG_NAG:
error_log = qmk_redis.get('qmk_api_update_error_log')
if len(error_log) > 5:
discord_msg(
'error',
'There are %s errors from the update process. View them here: %s'
% (len(error_log), ERROR_LOG_URL))
else:
print('Could not update qmk_firmware!')
print(job)
print(job.result)
if MSG_ON_QMK_FAIL:
discord_msg('warning',
'QMK update did not complete successfully!')
def s3_cleanup():
"""Clean up old compile jobs on S3.
"""
global last_s3_cleanup
if time() - last_s3_cleanup > S3_CLEANUP_PERIOD:
print('***', strftime(TIME_FORMAT))
print('Beginning S3 storage cleanup.')
job = qmk_redis.enqueue(cleanup_storage, timeout=S3_CLEANUP_TIMEOUT)
print(
'Successfully enqueued job id %s at %s, polling every 2 seconds...'
% (job.id, strftime(TIME_FORMAT)))
start_time = time()
run_time = None
# Wait for the job to start running
if not wait_for_job_start(job):
print('S3 cleanup queued for %s seconds! Giving up at %s!' %
(QUEUE_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_S3_FAIL:
discord_msg(
'warning',
'S3 cleanup queue longer than %s seconds! Queue length %s!'
% (QUEUE_TIMEOUT, len(qmk_redis.rq.jobs)))
return False
# Monitor the job while it runs
while job.get_status() == 'started':
if time() - start_time > S3_CLEANUP_TIMEOUT + 5:
print(
'S3 cleanup took longer than %s seconds! Cancelling at %s!'
% (S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_S3_FAIL:
discord_msg(
'warning', 'S3 cleanup took longer than %s seconds!' %
(S3_CLEANUP_TIMEOUT, ))
break
sleep(2)
# Check over the S3 cleanup results
if job.result:
print('Cleanup job completed successfully!')
if MSG_ON_S3_SUCCESS:
discord_msg('info', 'S3 cleanup completed successfully.')
else:
print('Could not clean S3!')
print(job)
print(job.result)
if MSG_ON_S3_FAIL:
discord_msg('error',
'S3 cleanup did not complete successfully!')
last_s3_cleanup = time()
def run(self):
status['current'] = 'good'
last_compile = 0
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list = qmk_redis.get('qmk_api_keyboards')
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
global job_queue_last_compile
global job_queue_last_warning
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last_warning > JOB_QUEUE_TOO_LONG:
job_queue_last_warning = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last_compile.isoformat())
discord_msg(level, message)
# Cycle through each layout for this keyboard and build it
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard))
if not metadata['layouts']:
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
keyboards_tested[
keyboard +
'/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used
failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = {
'severity': 'error',
'message': '%s: No layouts defined.' % keyboard
} # FIXME: Remove when no longer used
configurator_build_status[keyboard +
'/[NO_LAYOUTS]'] = {
'works': False,
'last_tested':
int(time()),
'message':
'%s: No Layouts defined.'
% keyboard
}
continue
for layout_macro in list(metadata['layouts']):
# Skip KEYMAP as it's deprecated
if 'KEYMAP' in layout_macro:
continue
# Build a layout
keyboard_layout_name = '/'.join(
(keyboard, layout_macro))
layers = [
list(
map(
lambda x: 'KC_NO', metadata['layouts']
[layout_macro]['layout']))
]
layers.append(list(map(lambda x: 'KC_TRNS',
layers[0])))
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, layout_macro))
args = (keyboard, 'qmk_api_tasks_test_compile',
layout_macro, layers)
job = qmk_redis.enqueue(compile_firmware,
COMPILE_TIMEOUT, *args)
print(
'Successfully enqueued, polling every 2 seconds...'
)
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s/%s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard, layout_macro,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord_msg(
'warning',
'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, layout_macro,
S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard_layout_name] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard_layout_name] = True # FIXME: Remove this when it's no longer used
if keyboard_layout_name in failed_keyboards:
del failed_keyboards[
keyboard_layout_name] # FIXME: Remove this when it's no longer used
layout_results[keyboard_layout_name] = {
'result': True,
'reason':
layout_macro + ' works in configurator.'
}
else:
if job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, layout_macro,
job.result['returncode']))
print(output)
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
layout_macro
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard_layout_name] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard_layout_name] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard_layout_name] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested',
keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed',
failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord_msg(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord_msg(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard_layout_name in list(configurator_build_status):
if time() - configurator_build_status[keyboard_layout_name][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard_layout_name,
configurator_build_status[keyboard_layout_name]
['last_tested']))
del configurator_build_status[keyboard_layout_name]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboard/layout combinations.
Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations.
Check out the details here: <%s>"""
discord_msg(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord_msg(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
def run(self):
status['current'] = 'good'
last_compile = 0
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json'
keyboard_list = fetch_json(keyboard_list_url).get('keyboards')
if not keyboard_list:
print(
'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...'
% (KEYBOARD_LIST_URL, COMPILE_TIMEOUT))
periodic_tasks()
sleep(COMPILE_TIMEOUT)
continue
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
global job_queue_last_compile
global job_queue_last_warning
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last_warning > JOB_QUEUE_TOO_LONG:
job_queue_last_warning = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last_compile.isoformat())
discord_msg(level, message)
# Find or generate a default keymap for this keyboard.
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json'
metadata = fetch_json(metadata_url).get('keyboards',
{}).get(keyboard)
if not metadata:
print(f'No metadata, skipping {keyboard}.')
if metadata.get(
'keymaps') and 'default' in metadata['keymaps']:
keymap = fetch_json(
metadata['keymaps']['default']['url'])
keymap['keymap'] = 'default'
else:
keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json'
keymap = fetch_json(keymap_url)
if keymap:
keymap['keymap'] = 'default_configurator'
else:
# Fall back to building an empty keymap
if metadata.get('layouts'):
layout_macro = random.choice(
list(metadata['layouts']))
layout_len = len(metadata['layouts']
[layout_macro]['layout'])
keymap = {
'keyboard':
keyboard,
'keymap':
'generated',
'layout':
layout_macro,
'layers':
[['KC_NO' for i in range(layout_len)],
['KC_TRNS' for i in range(layout_len)]]
}
else:
output = f'No layouts for {keyboard}! Skipping!'
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
print(output)
continue
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, keymap['layout']))
job = qmk_redis.enqueue(compile_json,
COMPILE_TIMEOUT,
keymap,
send_metrics=False,
public_firmware=True)
print('Successfully enqueued, polling every 2 seconds...')
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord_msg(
'warning',
'Keyboard %s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard] = True # FIXME: Remove this when it's no longer used
if keyboard in failed_keyboards:
del failed_keyboards[
keyboard] # FIXME: Remove this when it's no longer used
layout_results[keyboard] = {
'result': True,
'reason': keyboard + ' works in configurator.'
}
else:
if job.result and job.result['returncode'] == -3:
output = f'Exception while compiling {keyboard}: {job.result["exception"]}'
print(output)
print(job.result['stacktrace'])
layout_results[keyboard] = {
'result':
False,
'reason':
f'{job.result["exception"]}: {job.result["stacktrace"]}'
}
elif job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, keymap['layout'],
job.result['returncode']))
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
keymap['layout']
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord_msg(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord_msg(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard in list(configurator_build_status):
if time() - configurator_build_status[keyboard][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard,
configurator_build_status[keyboard]['last_tested']))
del configurator_build_status[keyboard]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboards.
Non-working: %s the last round, for a total of %s non-working keyboards.
Check out the details here: <%s>"""
discord_msg(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord_msg(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'