def GET_v1_keyboards_keyboard_keymaps_keymap(keyboard, keymap):
"""Return JSON showing data about a keyboard's keymap
"""
keyboards = qmk_redis.get('qmk_api_last_updated')
keyboards['keyboards'] = {}
for kb in keyboard.split(','):
kb_data = qmk_redis.get('qmk_api_kb_'+kb)
if kb_data:
keymaps = {}
keymap_list = kb_data['keymaps'] if keymap == 'all' else keymap.split(',')
for km in keymap_list:
keymaps[km] = qmk_redis.get('qmk_api_kb_%s_keymap_%s' % (kb, km))
if not keymaps:
return error('No such keymap: ' + keymap, 404)
kb_data['keymaps'] = keymaps
keyboards['keyboards'][kb] = kb_data
if not keyboards['keyboards']:
return error('No such keyboard: ' + keyboard, 404)
return jsonify(keyboards)
def update_qmk_firmware():
"""Update the API from qmk_firmware after a push.
"""
if qmk_redis.get('qmk_needs_update'):
print('***', strftime(TIME_FORMAT))
print('Beginning qmk_firmware update.')
job = qmk_redis.enqueue(update_kb_redis, timeout=QMK_UPDATE_TIMEOUT)
print(
'Successfully enqueued job id %s at %s, polling every 2 seconds...'
% (job.id, strftime(TIME_FORMAT)))
start_time = time()
# Wait for the job to start running
if not wait_for_job_start(job):
print('QMK update queued for %s seconds! Giving up at %s!' %
(S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'warning',
'S3 cleanup queue longer than %s seconds! Queue length %s!'
% (S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs)))
return False
# Monitor the job while it runs
while job.get_status() == 'started':
if time() - start_time > QMK_UPDATE_TIMEOUT + 5:
print(
'QMK update took longer than %s seconds! Cancelling at %s!'
% (QMK_UPDATE_TIMEOUT, strftime(TIME_FORMAT)))
if MSG_ON_QMK_FAIL:
discord_msg(
'error', 'QMK update took longer than %s seconds!' %
(QMK_UPDATE_TIMEOUT, ))
break
sleep(2)
# Check over the results
if job.result:
hash = qmk_redis.get('qmk_api_last_updated')['git_hash']
message = 'QMK update completed successfully! API is current to ' + hash
print(message)
discord_msg('info', message)
if ERROR_LOG_NAG:
error_log = qmk_redis.get('qmk_api_update_error_log')
if len(error_log) > 5:
discord_msg(
'error',
'There are %s errors from the update process. View them here: %s'
% (len(error_log), ERROR_LOG_URL))
else:
print('Could not update qmk_firmware!')
print(job)
print(job.result)
if MSG_ON_QMK_FAIL:
discord_msg('warning',
'QMK update did not complete successfully!')
def GET_v1_keyboards_keyboard(keyboard):
"""Return JSON showing data about a keyboard
"""
keyboards = qmk_redis.get('qmk_api_last_updated')
keyboards['keyboards'] = {}
for kb in keyboard.split(','):
kb_data = qmk_redis.get('qmk_api_kb_' + kb)
if kb_data:
keyboards['keyboards'][kb] = kb_data
if not keyboards['keyboards']:
return error('No such keyboard: ' + keyboard, 404)
return jsonify(keyboards)
def GET_v1_keyboards_all():
"""Return JSON showing all available keyboards and their layouts.
"""
allkb = qmk_redis.get('qmk_api_kb_all')
if allkb:
return jsonify(allkb)
return error('An unknown error occured', 500)
def GET_v1_keyboards_all():
"""Return JSON showing all available keyboards and their layouts.
"""
allkb = qmk_redis.get('qmk_api_kb_all')
if allkb:
return jsonify(allkb)
return error('An unknown error occured', 500)
def GET_v1_keyboards_keyboard(keyboard):
"""Return JSON showing data about a keyboard
"""
keyboards = {
'last_updated': qmk_redis.get('qmk_api_last_updated'),
'keyboards': {}
}
for kb in keyboard.split(','):
kb_data = qmk_redis.get('qmk_api_kb_'+kb)
if kb_data:
keyboards['keyboards'][kb] = kb_data
if not keyboards['keyboards']:
return error('No such keyboard: ' + keyboard, 404)
return jsonify(keyboards)
def GET_v1_keyboards_build_log():
"""Returns a dictionary of keyboard/layout pairs. Each entry is a dictionary with the following keys:
* `works`: Boolean indicating whether the compile was successful
* `message`: The compile output for failed builds
"""
json_data = qmk_redis.get('qmk_api_configurator_status')
return jsonify(json_data)
def GET_v1_keyboards_keyboard_readme(keyboard):
"""Returns the readme for a keyboard.
"""
readme = qmk_redis.get('qmk_api_kb_%s_readme' % (keyboard))
response = make_response(readme)
response.mimetype = 'text/markdown'
return response
def GET_v1():
"""Return the API's status.
"""
return jsonify({
'children': ['compile', 'converters', 'keyboards'],
'last_ping': qmk_redis.get('qmk_api_last_ping'),
'queue_length': len(rq),
'status': 'running',
'version': __VERSION__
})
def GET_v1_update():
"""Triggers an update of the API.
"""
result = {
'result': UPDATE_API,
'last_ping': qmk_redis.get('qmk_api_last_ping'),
'queue_length': len(rq),
'queued_job_ids': rq.job_ids,
'status': 'running',
'version': __VERSION__
}
if UPDATE_API:
rq.enqueue(update_kb_redis)
return jsonify(result)
def GET_v1_healthcheck():
"""Checks over the health of the API.
Note: This is used for operational purposes. Please don't hit it on the
live api.qmk.fm site without talking to us first. Most of this
information is available at the /v1 endpoint as well.
"""
rq.enqueue(ping, at_front=True)
return jsonify({
'last_ping': qmk_redis.get('qmk_api_last_ping'),
'queue_length': len(rq),
'queued_job_ids': rq.job_ids,
'status': 'running',
'version': __VERSION__
})
def check_pings():
"""Check the ping values and update api_status with them.
"""
api_status['queue_length'] = len(rq)
for redis_key in ('qmk_api_last_ping', 'qmk_api_tasks_ping'):
key = redis_key.replace('qmk_api_', '')
value = qmk_redis.get(redis_key)
api_status[key] = value
if value:
if time() - float(value) > CHECK_TIMEOUT:
api_status['status'] = 'degraded'
api_status['status_%s' % key] = 'degraded'
else:
api_status['status'] = 'running'
api_status['status_%s' % key] = 'good'
else:
api_status['status'] = 'degraded'
api_status['status_%s' % key] = 'degraded'
def GET_v1_usb():
"""Returns the list of USB device identifiers used in QMK.
"""
json_blob = qmk_redis.get('qmk_api_usb_list')
return jsonify(json_blob)
def GET_v1_keyboards():
"""Return a list of keyboards
"""
json_blob = qmk_redis.get('qmk_api_keyboards')
return jsonify(json_blob)
def GET_v1_keyboards_error_log():
"""Return the error log from the last run.
"""
json_blob = qmk_redis.get('qmk_api_update_error_log')
return jsonify(json_blob)
def GET_v1_keyboards_error_log():
"""Return the error log from the last run.
"""
json_blob = qmk_redis.get('qmk_api_update_error_log')
return jsonify(json_blob)
def run(self):
status['current'] = 'good'
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json'
keyboard_list = fetch_json(keyboard_list_url).get('keyboards')
if not keyboard_list:
print(
'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...'
% (keyboard_list_url, COMPILE_TIMEOUT))
periodic_tasks()
sleep(COMPILE_TIMEOUT)
continue
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last['warning'] > JOB_QUEUE_TOO_LONG:
job_queue_last['warning'] = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last['compile'].isoformat())
discord.message(level, message)
# Find or generate a default keymap for this keyboard.
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json'
metadata = fetch_json(metadata_url).get('keyboards',
{}).get(keyboard)
if not metadata:
print(
'*** Sleeping for 60 seconds then continuing to the next keyboard...'
)
sleep(60)
continue
if metadata.get(
'keymaps') and 'default' in metadata['keymaps']:
keymap = fetch_json(
metadata['keymaps']['default']['url'])
keymap['keymap'] = 'default'
else:
keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json'
keymap = fetch_json(keymap_url)
if keymap:
keymap['keymap'] = 'default_configurator'
else:
# Fall back to building an empty keymap
if metadata.get('layouts'):
layout_macro = random.choice(
list(metadata['layouts']))
layout_len = len(metadata['layouts']
[layout_macro]['layout'])
keymap = {
'keyboard':
keyboard,
'keymap':
'generated',
'layout':
layout_macro,
'layers':
[['KC_NO' for i in range(layout_len)],
['KC_TRNS' for i in range(layout_len)]]
}
else:
output = f'No layouts for {keyboard}! Skipping!'
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
print(output)
continue
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, keymap['layout']))
job = qmk_redis.enqueue(compile_json,
COMPILE_TIMEOUT,
keymap,
send_metrics=False,
public_firmware=True)
print('Successfully enqueued, polling every 2 seconds...')
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord.message(
'warning',
'Keyboard %s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard] = True # FIXME: Remove this when it's no longer used
if keyboard in failed_keyboards:
del failed_keyboards[
keyboard] # FIXME: Remove this when it's no longer used
layout_results[keyboard] = {
'result': True,
'reason': keyboard + ' works in configurator.'
}
else:
if job.result and job.result['returncode'] == -3:
output = f'Exception while compiling {keyboard}: {job.result["exception"]}'
print(output)
print(job.result['stacktrace'])
layout_results[keyboard] = {
'result':
False,
'reason':
f'{job.result["exception"]}: {job.result["stacktrace"]}'
}
elif job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, keymap['layout'],
job.result['returncode']))
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
keymap['layout']
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord.message(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord.message(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard in list(configurator_build_status):
if time() - configurator_build_status[keyboard][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard,
configurator_build_status[keyboard]['last_tested']))
del configurator_build_status[keyboard]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboards.
Non-working: %s the last round, for a total of %s non-working keyboards.
Check out the details here: <%s>"""
discord.message(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord.message(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
def GET_v1_keyboards_build_status():
"""Returns a dictionary of keyboard/layout pairs. Each entry is True if the keyboard works in configurator and
false if it doesn't.
"""
json_blob = qmk_redis.get('qmk_api_keyboards_tested')
return jsonify(json_blob)
def GET_v1_keyboards():
"""Return a list of keyboards
"""
json_blob = qmk_redis.get('qmk_api_keyboards')
return jsonify(json_blob)
def run(self):
status['current'] = 'good'
last_compile = 0
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list = qmk_redis.get('qmk_api_keyboards')
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
global job_queue_last_compile
global job_queue_last_warning
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last_warning > JOB_QUEUE_TOO_LONG:
job_queue_last_warning = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last_compile.isoformat())
discord_msg(level, message)
# Cycle through each layout for this keyboard and build it
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard))
if not metadata['layouts']:
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
keyboards_tested[
keyboard +
'/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used
failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = {
'severity': 'error',
'message': '%s: No layouts defined.' % keyboard
} # FIXME: Remove when no longer used
configurator_build_status[keyboard +
'/[NO_LAYOUTS]'] = {
'works': False,
'last_tested':
int(time()),
'message':
'%s: No Layouts defined.'
% keyboard
}
continue
for layout_macro in list(metadata['layouts']):
# Skip KEYMAP as it's deprecated
if 'KEYMAP' in layout_macro:
continue
# Build a layout
keyboard_layout_name = '/'.join(
(keyboard, layout_macro))
layers = [
list(
map(
lambda x: 'KC_NO', metadata['layouts']
[layout_macro]['layout']))
]
layers.append(list(map(lambda x: 'KC_TRNS',
layers[0])))
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, layout_macro))
args = (keyboard, 'qmk_api_tasks_test_compile',
layout_macro, layers)
job = qmk_redis.enqueue(compile_firmware,
COMPILE_TIMEOUT, *args)
print(
'Successfully enqueued, polling every 2 seconds...'
)
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s/%s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard, layout_macro,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord_msg(
'warning',
'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, layout_macro,
S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard_layout_name] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard_layout_name] = True # FIXME: Remove this when it's no longer used
if keyboard_layout_name in failed_keyboards:
del failed_keyboards[
keyboard_layout_name] # FIXME: Remove this when it's no longer used
layout_results[keyboard_layout_name] = {
'result': True,
'reason':
layout_macro + ' works in configurator.'
}
else:
if job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, layout_macro,
job.result['returncode']))
print(output)
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
layout_macro
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard_layout_name] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard_layout_name] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard_layout_name] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested',
keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed',
failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord_msg(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord_msg(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard_layout_name in list(configurator_build_status):
if time() - configurator_build_status[keyboard_layout_name][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard_layout_name,
configurator_build_status[keyboard_layout_name]
['last_tested']))
del configurator_build_status[keyboard_layout_name]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboard/layout combinations.
Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations.
Check out the details here: <%s>"""
discord_msg(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord_msg(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
return float(obj)
except TypeError:
pass
return JSONEncoder.default(self, obj)
# Useful objects
app = Flask(__name__)
app.json_encoder = CustomJSONEncoder
app.config['JSON_SORT_KEYS'] = False
cache_dir = 'kle_cache'
gist_url = 'https://api.github.com/gists/%s'
cors = CORS(app, resources={'/v*/*': {'origins': '*'}})
rq = Queue(connection=redis)
api_status = {
'last_ping': qmk_redis.get('qmk_api_last_ping'),
'queue_length': len(rq),
'status': 'starting',
'version': __VERSION__,
}
## Helper functions
def check_pings():
"""Check the ping values and update api_status with them.
"""
api_status['queue_length'] = len(rq)
for redis_key in ('qmk_api_last_ping', 'qmk_api_tasks_ping'):
key = redis_key.replace('qmk_api_', '')
value = qmk_redis.get(redis_key)
api_status[key] = value
def update_kb_redis():
"""Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data.
"""
# Check to see if we need to update
if exists('qmk_firmware'):
last_update = qmk_redis.get('qmk_api_last_updated')
if not debug and isinstance(
last_update, dict) and last_update['git_hash'] == git_hash():
# We are already up to date
logging.warning(
'update_kb_redis(): Already up to date, skipping...')
return False
# Clean up the environment and fetch the latest source
del (error_log[:])
if exists('qmk_firmware'):
rmtree('qmk_firmware')
if exists('qmk_firmware.zip'):
remove('qmk_firmware.zip')
qmk_storage.delete('cache/qmk_firmware.zip')
checkout_qmk()
# Update redis with the latest data
kb_list = []
cached_json = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
keyboard_info = {
'keyboard_name': keyboard,
'keyboard_folder': keyboard,
'keymaps': [],
'layouts': {},
'maintainer': 'qmk',
}
for layout_name, layout_json in find_all_layouts(keyboard).items():
if not layout_name.startswith('LAYOUT_kc'):
keyboard_info['layouts'][layout_name] = layout_json
for info_json_filename in find_info_json(keyboard):
# Iterate through all the possible info.json files to build the final keyboard JSON.
try:
with open(info_json_filename) as info_file:
keyboard_info = merge_info_json(info_file, keyboard_info)
except Exception as e:
error_msg = 'Error encountered processing %s! %s: %s' % (
keyboard, e.__class__.__name__, e)
error_log.append({
'severity': 'error',
'message': 'Error: ' + error_msg
})
logging.error(error_msg)
logging.exception(e)
# Iterate through all the possible keymaps to build keymap jsons.
for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps(
keyboard):
keyboard_info['keymaps'].append(keymap_name)
keymap_blob = {
'keyboard_name': keyboard,
'keymap_name': keymap_name,
'keymap_folder': keymap_folder,
'layers': keymap,
'layout_macro': layout_macro
}
# Write the keymap to redis
qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name),
keymap_blob)
readme = '%s/%s/readme.md' % (keymap_folder, keymap_name)
if exists(readme):
with open(readme, 'rb') as readme_fd:
readme_text = readme_fd.read()
readme_text = UnicodeDammit(readme_text)
readme_text = readme_text.unicode_markup
else:
readme_text = '%s does not exist.' % readme
qmk_redis.set(
'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name),
readme_text)
# Pull some keyboard information from existing rules.mk and config.h files
config_h = parse_config_h(keyboard)
rules_mk = parse_rules_mk(keyboard)
for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER',
'DESCRIPTION'):
if key in config_h:
if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'):
config_h[key].replace('0x', '')
config_h[key] = config_h[key].upper()
keyboard_info[key.lower()] = config_h[key]
if 'ARMV' in rules_mk:
keyboard_info['processor_type'] = 'arm'
# ARM processors
if 'MCU' in rules_mk:
keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT']
if 'MCU_LDSCRIPT' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
if 'STM32' in keyboard_info['processor']:
keyboard_info['bootloader'] = 'stm32-dfu'
elif keyboard_info.get('manufacturer') == 'Input Club':
keyboard_info['bootloader'] = 'kiibohd-dfu'
else:
keyboard_info['processor_type'] = 'avr'
# AVR processors
if 'ARCH' in rules_mk:
keyboard_info['platform'] = rules_mk['ARCH']
if 'MCU' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
keyboard_info['bootloader'] = 'atmel-dfu'
keyboard_info['identifier'] = ':'.join(
(keyboard_info.get('vendor_id', 'unknown'),
keyboard_info.get('product_id', 'unknown'),
keyboard_info.get('device_ver', 'unknown')))
# Store the keyboard's readme in redis
readme_filename = None
readme_path = ''
for dir in keyboard.split('/'):
readme_path = '/'.join((readme_path, dir))
new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path))
if new_name:
readme_filename = new_name # Last one wins
if readme_filename:
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard),
open(readme_filename).read())
keyboard_info['readme'] = True
else:
error_msg = '%s does not have a readme.md.' % keyboard
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg)
error_log.append({
'severity': 'warning',
'message': 'Warning: ' + error_msg
})
logging.warning(error_msg)
keyboard_info['readme'] = False
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
kb_list.append(keyboard)
cached_json['keyboards'][keyboard] = keyboard_info
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', cached_json)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
return True
def compile_json(keyboard_keymap_data, source_ip=None):
"""Compile a keymap.
Arguments:
keyboard_keymap_data
A configurator export file that's been deserialized
source_ip
The IP that submitted the compile job
"""
result = {
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
'output': 'Unknown error',
}
try:
for key in ('keyboard', 'layout', 'keymap'):
result[key] = keyboard_keymap_data[key]
result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace(
'/', '-'), result['keymap'].replace('/', '-'))
result['keymap_json'] = json.dumps(keyboard_keymap_data)
result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard'])
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + result['keyboard']):
print('Unknown keyboard: %s' % (result['keyboard'], ))
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
if kb_data.get('protocol') == 'V-USB':
checkout_vusb()
# Write the keymap file
with open(path.join('qmk_firmware', result['keymap_archive']),
'w') as fd:
fd.write(result['keymap_json'] + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
return result
def compile_firmware(keyboard, keymap, layout, layers):
"""Compile a firmware.
"""
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'command': ['make', 'COLOR=false', ':'.join((keyboard, keymap))],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': ''
}
try:
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
for pathname in ('qmk_firmware/keyboards/%s/keymaps/%s' %
(keyboard, keymap),
'qmk_firmware/keyboards/%s/../keymaps/%s' %
(keyboard, keymap)):
if path.exists(pathname):
logging.error(
'Name collision! %s already exists! This should not happen!',
pathname)
return {
'returncode':
-1,
'command':
'',
'output':
'Keymap name collision! %s already exists!' % (pathname),
'firmware':
None
}
# If this keyboard needs chibios check it out
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
if kb_data['processor_type'] == 'arm':
checkout_chibios()
# Build the keyboard firmware
create_keymap(result, layers)
store_firmware_source(result)
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
"""Compile a firmware.
"""
keyboard_safe_chars = keyboard.replace('/', '-')
keymap_safe_chars = keymap.replace('/', '-')
keymap_json_file = '%s-%s.json' % (keyboard_safe_chars, keymap_safe_chars)
keymap_json = json.dumps({
'keyboard':
keyboard,
'keymap':
keymap,
'layout':
layout,
'layers':
layers,
'author':
'',
'notes':
'',
'version':
1,
'source_ip':
source_ip,
'documentation':
'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`'
% (keymap_json_file, )
})
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'keymap_archive': keymap_json_file,
'command': ['bin/qmk', 'compile', keymap_json_file],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
}
try:
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
# Write the keymap file
with open(path.join('qmk_firmware', keymap_json_file), 'w') as fd:
fd.write(keymap_json + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
