def update_qmk_firmware():
"""Update the API from qmk_firmware after a push.
"""
if qmk_redis.get('qmk_needs_update'):
print('***', strftime(TIME_FORMAT))
print('Beginning qmk_firmware update.')
job = qmk_redis.enqueue(update_kb_redis, timeout=QMK_UPDATE_TIMEOUT)
print(
'Successfully enqueued job id %s at %s, polling every 2 seconds...'
% (job.id, strftime(TIME_FORMAT)))
start_time = time()
# Wait for the job to start running
if not wait_for_job_start(job):
print('QMK update queued for %s seconds! Giving up at %s!' %
(S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'warning',
'S3 cleanup queue longer than %s seconds! Queue length %s!'
% (S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs)))
return False
# Monitor the job while it runs
while job.get_status() == 'started':
if time() - start_time > QMK_UPDATE_TIMEOUT + 5:
print(
'QMK update took longer than %s seconds! Cancelling at %s!'
% (QMK_UPDATE_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'error', 'QMK update took longer than %s seconds!' %
(QMK_UPDATE_TIMEOUT, ))
break
sleep(2)
# Check over the results
if job.result:
hash = qmk_redis.get('qmk_api_last_updated')['git_hash']
message = 'QMK update completed successfully! API is current to ' + hash
print(message)
discord_msg('info', message)
if ERROR_LOG_NAG:
error_log = qmk_redis.get('qmk_api_update_error_log')
if len(error_log) > 5:
discord_msg(
'error',
'There are %s errors from the update process. View them here: %s'
% (len(error_log), ERROR_LOG_URL))
else:
print('Could not update qmk_firmware!')
print(job)
print(job.result)
if MSG_ON_QMK_FAIL:
discord_msg('warning',
'QMK update did not complete successfully!')
def s3_cleanup():
"""Clean up old compile jobs on S3.
"""
global last_s3_cleanup
if time() - last_s3_cleanup > S3_CLEANUP_PERIOD:
print('***', strftime(TIME_FORMAT))
print('Beginning S3 storage cleanup.')
job = qmk_redis.enqueue(cleanup_storage, timeout=S3_CLEANUP_TIMEOUT)
print(
'Successfully enqueued job id %s at %s, polling every 2 seconds...'
% (job.id, strftime(TIME_FORMAT)))
start_time = time()
run_time = None
# Wait for the job to start running
if not wait_for_job_start(job):
print('S3 cleanup queued for %s seconds! Giving up at %s!' %
(QUEUE_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_S3_FAIL:
discord_msg(
'warning',
'S3 cleanup queue longer than %s seconds! Queue length %s!'
% (QUEUE_TIMEOUT, len(qmk_redis.rq.jobs)))
return False
# Monitor the job while it runs
while job.get_status() == 'started':
if time() - start_time > S3_CLEANUP_TIMEOUT + 5:
print(
'S3 cleanup took longer than %s seconds! Cancelling at %s!'
% (S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_S3_FAIL:
discord_msg(
'warning', 'S3 cleanup took longer than %s seconds!' %
(S3_CLEANUP_TIMEOUT, ))
break
sleep(2)
# Check over the S3 cleanup results
if job.result:
print('Cleanup job completed successfully!')
if MSG_ON_S3_SUCCESS:
discord_msg('info', 'S3 cleanup completed successfully.')
else:
print('Could not clean S3!')
print(job)
print(job.result)
if MSG_ON_S3_FAIL:
discord_msg('error',
'S3 cleanup did not complete successfully!')
last_s3_cleanup = time()
def run(self):
status['current'] = 'good'
last_compile = 0
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list = qmk_redis.get('qmk_api_keyboards')
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
global job_queue_last_compile
global job_queue_last_warning
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last_warning > JOB_QUEUE_TOO_LONG:
job_queue_last_warning = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last_compile.isoformat())
discord_msg(level, message)
# Cycle through each layout for this keyboard and build it
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard))
if not metadata['layouts']:
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
keyboards_tested[
keyboard +
'/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used
failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = {
'severity': 'error',
'message': '%s: No layouts defined.' % keyboard
} # FIXME: Remove when no longer used
configurator_build_status[keyboard +
'/[NO_LAYOUTS]'] = {
'works': False,
'last_tested':
int(time()),
'message':
'%s: No Layouts defined.'
% keyboard
}
continue
for layout_macro in list(metadata['layouts']):
# Skip KEYMAP as it's deprecated
if 'KEYMAP' in layout_macro:
continue
# Build a layout
keyboard_layout_name = '/'.join(
(keyboard, layout_macro))
layers = [
list(
map(
lambda x: 'KC_NO', metadata['layouts']
[layout_macro]['layout']))
]
layers.append(list(map(lambda x: 'KC_TRNS',
layers[0])))
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, layout_macro))
args = (keyboard, 'qmk_api_tasks_test_compile',
layout_macro, layers)
job = qmk_redis.enqueue(compile_firmware,
COMPILE_TIMEOUT, *args)
print(
'Successfully enqueued, polling every 2 seconds...'
)
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s/%s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard, layout_macro,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord_msg(
'warning',
'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, layout_macro,
S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard_layout_name] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard_layout_name] = True # FIXME: Remove this when it's no longer used
if keyboard_layout_name in failed_keyboards:
del failed_keyboards[
keyboard_layout_name] # FIXME: Remove this when it's no longer used
layout_results[keyboard_layout_name] = {
'result': True,
'reason':
layout_macro + ' works in configurator.'
}
else:
if job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, layout_macro,
job.result['returncode']))
print(output)
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
layout_macro
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard_layout_name] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard_layout_name] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard_layout_name] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested',
keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed',
failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord_msg(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord_msg(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard_layout_name in list(configurator_build_status):
if time() - configurator_build_status[keyboard_layout_name][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard_layout_name,
configurator_build_status[keyboard_layout_name]
['last_tested']))
del configurator_build_status[keyboard_layout_name]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboard/layout combinations.
Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations.
Check out the details here: <%s>"""
discord_msg(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord_msg(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
def run(self):
status['current'] = 'good'
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json'
keyboard_list = fetch_json(keyboard_list_url).get('keyboards')
if not keyboard_list:
print(
'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...'
% (keyboard_list_url, COMPILE_TIMEOUT))
periodic_tasks()
sleep(COMPILE_TIMEOUT)
continue
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last['warning'] > JOB_QUEUE_TOO_LONG:
job_queue_last['warning'] = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last['compile'].isoformat())
discord.message(level, message)
# Find or generate a default keymap for this keyboard.
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json'
metadata = fetch_json(metadata_url).get('keyboards',
{}).get(keyboard)
if not metadata:
print(
'*** Sleeping for 60 seconds then continuing to the next keyboard...'
)
sleep(60)
continue
if metadata.get(
'keymaps') and 'default' in metadata['keymaps']:
keymap = fetch_json(
metadata['keymaps']['default']['url'])
keymap['keymap'] = 'default'
else:
keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json'
keymap = fetch_json(keymap_url)
if keymap:
keymap['keymap'] = 'default_configurator'
else:
# Fall back to building an empty keymap
if metadata.get('layouts'):
layout_macro = random.choice(
list(metadata['layouts']))
layout_len = len(metadata['layouts']
[layout_macro]['layout'])
keymap = {
'keyboard':
keyboard,
'keymap':
'generated',
'layout':
layout_macro,
'layers':
[['KC_NO' for i in range(layout_len)],
['KC_TRNS' for i in range(layout_len)]]
}
else:
output = f'No layouts for {keyboard}! Skipping!'
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
print(output)
continue
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, keymap['layout']))
job = qmk_redis.enqueue(compile_json,
COMPILE_TIMEOUT,
keymap,
send_metrics=False,
public_firmware=True)
print('Successfully enqueued, polling every 2 seconds...')
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord.message(
'warning',
'Keyboard %s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard] = True # FIXME: Remove this when it's no longer used
if keyboard in failed_keyboards:
del failed_keyboards[
keyboard] # FIXME: Remove this when it's no longer used
layout_results[keyboard] = {
'result': True,
'reason': keyboard + ' works in configurator.'
}
else:
if job.result and job.result['returncode'] == -3:
output = f'Exception while compiling {keyboard}: {job.result["exception"]}'
print(output)
print(job.result['stacktrace'])
layout_results[keyboard] = {
'result':
False,
'reason':
f'{job.result["exception"]}: {job.result["stacktrace"]}'
}
elif job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, keymap['layout'],
job.result['returncode']))
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
keymap['layout']
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord.message(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord.message(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard in list(configurator_build_status):
if time() - configurator_build_status[keyboard][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard,
configurator_build_status[keyboard]['last_tested']))
del configurator_build_status[keyboard]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboards.
Non-working: %s the last round, for a total of %s non-working keyboards.
Check out the details here: <%s>"""
discord.message(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord.message(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'