def process_keyboard(keyboard, usb_list, kb_list, kb_entries):
"""Parse all the files associated with a specific keyboard to build an API object for it.
"""
keyboard_info = build_keyboard_info(keyboard)
for layout_name, layout_json in find_all_layouts(keyboard).items():
if not layout_name.startswith('LAYOUT_kc'):
keyboard_info['layouts'][layout_name] = layout_json
for info_json_filename in find_info_json(keyboard):
# Merge info.json files into one.
keyboard_info = merge_info_json(info_json_filename, keyboard_info)
# Iterate through all the possible keymaps to build keymap jsons.
for keymap_name, keymap_folder, layout_macro, layers in find_keymaps(keyboard):
keyboard_info['keymaps'].append(keymap_name)
write_keymap_redis(keyboard, keymap_name, keymap_folder, layers, layout_macro)
# Pull some keyboard information from existing rules.mk and config.h files
config_h = parse_config_h(keyboard)
rules_mk = parse_rules_mk(keyboard)
usb_entry = build_usb_entry(keyboard_info, config_h, usb_list)
usb_list[usb_entry['vendor_id']][usb_entry['product_id']][keyboard] = usb_entry
# Setup platform specific keys
mcu = rules_mk.get('MCU')
if mcu in ARM_PROCESSORS:
arm_processor_rules(keyboard_info, rules_mk)
elif mcu in AVR_PROCESSORS:
avr_processor_rules(keyboard_info, rules_mk)
else:
log_warning("%s: Unknown MCU: %s" % (keyboard, mcu))
unknown_processor_rules(keyboard_info, rules_mk)
# Used to identify keyboards in the redis key qmk_api_usb_list.
keyboard_info['identifier'] = ':'.join((
keyboard_info.get('vendor_id', 'unknown'),
keyboard_info.get('product_id', 'unknown'),
keyboard_info.get('device_ver', 'unknown'),
))
# Store the keyboard's readme in redis
store_keyboard_readme(keyboard_info)
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
kb_list.append(keyboard)
kb_entries['keyboards'][keyboard] = keyboard_info
def write_keymap_redis(keyboard, keymap_name, keymap_folder, layers, layout_macro):
"""Write a keymap to redis.
"""
keymap_blob = {
'keyboard_name': keyboard,
'keymap_name': keymap_name,
'keymap_folder': keymap_folder,
'layers': layers,
'layout_macro': layout_macro,
}
readme = '%s/%s/readme.md' % (keymap_folder, keymap_name)
readme_text = unicode_text(readme) if exists(readme) else '%s does not exist.' % readme
qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name), keymap_blob)
qmk_redis.set('qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name), readme_text)
def store_keyboard_readme(keyboard_info):
"""Write a keyboard's readme file to redis.
"""
keyboard = keyboard_info['keyboard_folder']
readme_filename = None
readme_path = ''
for dir in keyboard.split('/'):
readme_path = '/'.join((readme_path, dir))
new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path))
if new_name:
readme_filename = new_name # Last one wins
if readme_filename:
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), unicode_text(readme_filename))
keyboard_info['readme'] = True
else:
log_warning('%s does not have a readme.md.' % keyboard)
def update_kb_redis():
"""Called to update qmk_firmware.
"""
# Clean up the environment and fetch the latest source
del error_log[:]
if not debug:
if exists('update_kb_redis'):
rmtree('update_kb_redis')
mkdir('update_kb_redis')
chdir('update_kb_redis')
qmk_redis.set('qmk_needs_update', False)
if not debug or not exists('qmk_firmware'):
checkout_qmk(skip_cache=True)
# Update redis with the latest data
kb_list = []
usb_list = {
} # Structure: VENDOR_ID: {PRODUCT_ID: {KEYBOARD_FOLDER: {'vendor_id': VENDOR_ID, 'product_id': PRODUCT_ID, 'device_ver': DEVICE_VER, 'manufacturer': MANUFACTURER, 'product': PRODUCT, 'keyboard': KEYBOARD_FOLDER}
kb_all = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
try:
process_keyboard(keyboard, usb_list, kb_list, kb_all)
except Exception as e:
# Uncaught exception handler. Ideally this is never hit.
log_error(
'Uncaught exception while processing keyboard %s! %s: %s' %
(keyboard, e.__class__.__name__, str(e)))
logging.exception(e)
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', kb_all)
qmk_redis.set('qmk_api_usb_list', usb_list)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
logging.info('*** All keys successfully written to redis! Total size:',
len(json.dumps(kb_all)))
chdir('..')
return True
def update_needed(**update_info):
"""Called when updates happen to QMK Firmware.
"""
qmk_redis.set('qmk_needs_update', True)
import qmk_redis
qmk_redis.set('qmk_needs_update', True)
def update_kb_redis():
"""Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data.
"""
# Check to see if we need to update
if exists('qmk_firmware'):
last_update = qmk_redis.get('qmk_api_last_updated')
if not debug and isinstance(
last_update, dict) and last_update['git_hash'] == git_hash():
# We are already up to date
logging.warning(
'update_kb_redis(): Already up to date, skipping...')
return False
# Clean up the environment and fetch the latest source
del (error_log[:])
if exists('qmk_firmware'):
rmtree('qmk_firmware')
if exists('qmk_firmware.zip'):
remove('qmk_firmware.zip')
qmk_storage.delete('cache/qmk_firmware.zip')
checkout_qmk()
# Update redis with the latest data
kb_list = []
cached_json = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
keyboard_info = {
'keyboard_name': keyboard,
'keyboard_folder': keyboard,
'keymaps': [],
'layouts': {},
'maintainer': 'qmk',
}
for layout_name, layout_json in find_all_layouts(keyboard).items():
if not layout_name.startswith('LAYOUT_kc'):
keyboard_info['layouts'][layout_name] = layout_json
for info_json_filename in find_info_json(keyboard):
# Iterate through all the possible info.json files to build the final keyboard JSON.
try:
with open(info_json_filename) as info_file:
keyboard_info = merge_info_json(info_file, keyboard_info)
except Exception as e:
error_msg = 'Error encountered processing %s! %s: %s' % (
keyboard, e.__class__.__name__, e)
error_log.append({
'severity': 'error',
'message': 'Error: ' + error_msg
})
logging.error(error_msg)
logging.exception(e)
# Iterate through all the possible keymaps to build keymap jsons.
for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps(
keyboard):
keyboard_info['keymaps'].append(keymap_name)
keymap_blob = {
'keyboard_name': keyboard,
'keymap_name': keymap_name,
'keymap_folder': keymap_folder,
'layers': keymap,
'layout_macro': layout_macro
}
# Write the keymap to redis
qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name),
keymap_blob)
readme = '%s/%s/readme.md' % (keymap_folder, keymap_name)
if exists(readme):
with open(readme, 'rb') as readme_fd:
readme_text = readme_fd.read()
readme_text = UnicodeDammit(readme_text)
readme_text = readme_text.unicode_markup
else:
readme_text = '%s does not exist.' % readme
qmk_redis.set(
'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name),
readme_text)
# Pull some keyboard information from existing rules.mk and config.h files
config_h = parse_config_h(keyboard)
rules_mk = parse_rules_mk(keyboard)
for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER',
'DESCRIPTION'):
if key in config_h:
if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'):
config_h[key].replace('0x', '')
config_h[key] = config_h[key].upper()
keyboard_info[key.lower()] = config_h[key]
if 'ARMV' in rules_mk:
keyboard_info['processor_type'] = 'arm'
# ARM processors
if 'MCU' in rules_mk:
keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT']
if 'MCU_LDSCRIPT' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
if 'STM32' in keyboard_info['processor']:
keyboard_info['bootloader'] = 'stm32-dfu'
elif keyboard_info.get('manufacturer') == 'Input Club':
keyboard_info['bootloader'] = 'kiibohd-dfu'
else:
keyboard_info['processor_type'] = 'avr'
# AVR processors
if 'ARCH' in rules_mk:
keyboard_info['platform'] = rules_mk['ARCH']
if 'MCU' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
keyboard_info['bootloader'] = 'atmel-dfu'
keyboard_info['identifier'] = ':'.join(
(keyboard_info.get('vendor_id', 'unknown'),
keyboard_info.get('product_id', 'unknown'),
keyboard_info.get('device_ver', 'unknown')))
# Store the keyboard's readme in redis
readme_filename = None
readme_path = ''
for dir in keyboard.split('/'):
readme_path = '/'.join((readme_path, dir))
new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path))
if new_name:
readme_filename = new_name # Last one wins
if readme_filename:
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard),
open(readme_filename).read())
keyboard_info['readme'] = True
else:
error_msg = '%s does not have a readme.md.' % keyboard
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg)
error_log.append({
'severity': 'warning',
'message': 'Warning: ' + error_msg
})
logging.warning(error_msg)
keyboard_info['readme'] = False
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
kb_list.append(keyboard)
cached_json['keyboards'][keyboard] = keyboard_info
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', cached_json)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
return True
def update_kb_redis():
"""Called to update qmk_firmware.
"""
update_kb_redis = Path('update_kb_redis')
qmk_firmware = Path('qmk_firmware')
# Clean up the environment and fetch the latest source
if not debug:
# Create and enter a separate update_kb_redis directory to avoid conflicting with live builds
if update_kb_redis.exists():
rmtree(update_kb_redis)
update_kb_redis.mkdir()
chdir(update_kb_redis)
if not debug or not qmk_firmware.exists():
checkout_qmk(skip_cache=True)
# Enter the qmk_firmware directory
chdir(qmk_firmware)
# Update redis with the latest keyboard data
run(['qmk', 'generate-api'])
api_dir = Path('api_data/v1')
keyboards_dir = api_dir / 'keyboards'
for keyboard_dir in keyboards_dir.glob('**'):
keyboard_name = keyboard_dir.relative_to(keyboards_dir).as_posix()
keyboard_info = keyboard_dir / 'info.json'
keyboard_readme = keyboard_dir / 'readme.md'
if keyboard_info.exists():
info_json = json.load(keyboard_info.open())
redis_json = info_json['keyboards'][keyboard_name]
qmk_redis.set('qmk_api_kb_%s' % (keyboard_name), redis_json)
if keyboard_readme.exists():
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard_name),
keyboard_readme.read_text())
# Update the USB list
usb_json = json.load((api_dir / 'usb.json').open())
redis_usb = usb_json['usb']
qmk_redis.set('qmk_api_usb_list', redis_usb)
# Update the Keyboard list
keyboard_json = json.load((api_dir / 'keyboard_list.json').open())
redis_keyboard = keyboard_json['keyboards']
qmk_redis.set('qmk_api_keyboards', redis_keyboard)
# Leave qmk_firmware
chdir('..')
# Set some metadata
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_needs_update', False)
print('\n*** All keys successfully written to redis!')
if not debug:
# Leave update_kb_redis
chdir('..')
return True
def run(self):
status['current'] = 'good'
last_compile = 0
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list = qmk_redis.get('qmk_api_keyboards')
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
global job_queue_last_compile
global job_queue_last_warning
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last_warning > JOB_QUEUE_TOO_LONG:
job_queue_last_warning = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last_compile.isoformat())
discord_msg(level, message)
# Cycle through each layout for this keyboard and build it
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard))
if not metadata['layouts']:
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
keyboards_tested[
keyboard +
'/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used
failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = {
'severity': 'error',
'message': '%s: No layouts defined.' % keyboard
} # FIXME: Remove when no longer used
configurator_build_status[keyboard +
'/[NO_LAYOUTS]'] = {
'works': False,
'last_tested':
int(time()),
'message':
'%s: No Layouts defined.'
% keyboard
}
continue
for layout_macro in list(metadata['layouts']):
# Skip KEYMAP as it's deprecated
if 'KEYMAP' in layout_macro:
continue
# Build a layout
keyboard_layout_name = '/'.join(
(keyboard, layout_macro))
layers = [
list(
map(
lambda x: 'KC_NO', metadata['layouts']
[layout_macro]['layout']))
]
layers.append(list(map(lambda x: 'KC_TRNS',
layers[0])))
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, layout_macro))
args = (keyboard, 'qmk_api_tasks_test_compile',
layout_macro, layers)
job = qmk_redis.enqueue(compile_firmware,
COMPILE_TIMEOUT, *args)
print(
'Successfully enqueued, polling every 2 seconds...'
)
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s/%s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard, layout_macro,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord_msg(
'warning',
'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, layout_macro,
S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard_layout_name] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard_layout_name] = True # FIXME: Remove this when it's no longer used
if keyboard_layout_name in failed_keyboards:
del failed_keyboards[
keyboard_layout_name] # FIXME: Remove this when it's no longer used
layout_results[keyboard_layout_name] = {
'result': True,
'reason':
layout_macro + ' works in configurator.'
}
else:
if job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, layout_macro,
job.result['returncode']))
print(output)
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
layout_macro
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
layout_results[keyboard_layout_name] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
layout_macro
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard_layout_name] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard_layout_name] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard_layout_name] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested',
keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed',
failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord_msg(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord_msg(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard_layout_name in list(configurator_build_status):
if time() - configurator_build_status[keyboard_layout_name][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard_layout_name,
configurator_build_status[keyboard_layout_name]
['last_tested']))
del configurator_build_status[keyboard_layout_name]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboard/layout combinations.
Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations.
Check out the details here: <%s>"""
discord_msg(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord_msg(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
def periodic_tasks():
"""Jobs that need to run on a regular schedule.
"""
qmk_redis.set('qmk_api_tasks_ping', time())
s3_cleanup()
update_qmk_firmware()
def run(self):
status['current'] = 'good'
last_good_boards = qmk_redis.get('qmk_last_good_boards')
last_bad_boards = qmk_redis.get('qmk_last_bad_boards')
last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard')
keyboards_tested = qmk_redis.get(
'qmk_api_keyboards_tested') # FIXME: Remove when no longer used
if not keyboards_tested:
keyboards_tested = {}
failed_keyboards = qmk_redis.get(
'qmk_api_keyboards_failed') # FIXME: Remove when no longer used
if not failed_keyboards:
failed_keyboards = {}
configurator_build_status = qmk_redis.get(
'qmk_api_configurator_status')
if not configurator_build_status:
configurator_build_status = {}
while True:
good_boards = 0
bad_boards = 0
keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json'
keyboard_list = fetch_json(keyboard_list_url).get('keyboards')
if not keyboard_list:
print(
'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...'
% (keyboard_list_url, COMPILE_TIMEOUT))
periodic_tasks()
sleep(COMPILE_TIMEOUT)
continue
# If we stopped at a known keyboard restart from there
if last_stop in keyboard_list:
good_boards = qmk_redis.get('qmk_good_boards') or 0
bad_boards = qmk_redis.get('qmk_bad_boards') or 0
del (keyboard_list[:keyboard_list.index(last_stop)])
last_stop = None
for keyboard in keyboard_list:
periodic_tasks()
# Cycle through each keyboard and build it
try:
# If we have too many jobs in the queue don't put stress on the infrastructure
while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD:
periodic_tasks()
print(
'Too many jobs in the redis queue (%s)! Sleeping %s seconds...'
% (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT))
sleep(COMPILE_TIMEOUT)
if time(
) - job_queue_last['warning'] > JOB_QUEUE_TOO_LONG:
job_queue_last['warning'] = time()
level = 'warning'
message = 'Compile queue too large (%s) since %s' % (
len(qmk_redis.rq.jobs),
job_queue_last['compile'].isoformat())
discord.message(level, message)
# Find or generate a default keymap for this keyboard.
qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard)
layout_results = {}
metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json'
metadata = fetch_json(metadata_url).get('keyboards',
{}).get(keyboard)
if not metadata:
print(
'*** Sleeping for 60 seconds then continuing to the next keyboard...'
)
sleep(60)
continue
if metadata.get(
'keymaps') and 'default' in metadata['keymaps']:
keymap = fetch_json(
metadata['keymaps']['default']['url'])
keymap['keymap'] = 'default'
else:
keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json'
keymap = fetch_json(keymap_url)
if keymap:
keymap['keymap'] = 'default_configurator'
else:
# Fall back to building an empty keymap
if metadata.get('layouts'):
layout_macro = random.choice(
list(metadata['layouts']))
layout_len = len(metadata['layouts']
[layout_macro]['layout'])
keymap = {
'keyboard':
keyboard,
'keymap':
'generated',
'layout':
layout_macro,
'layers':
[['KC_NO' for i in range(layout_len)],
['KC_TRNS' for i in range(layout_len)]]
}
else:
output = f'No layouts for {keyboard}! Skipping!'
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
print(output)
continue
# Enqueue the job
print('***', strftime(TIME_FORMAT))
print('Beginning test compile for %s, layout %s' %
(keyboard, keymap['layout']))
job = qmk_redis.enqueue(compile_json,
COMPILE_TIMEOUT,
keymap,
send_metrics=False,
public_firmware=True)
print('Successfully enqueued, polling every 2 seconds...')
# Wait for the job to start running
if not wait_for_job_start(job):
print(
'Waited %s seconds for %s to start! Giving up at %s!'
% (S3_CLEANUP_TIMEOUT, keyboard,
strftime(TIME_FORMAT)))
if MSG_ON_BAD_COMPILE:
discord.message(
'warning',
'Keyboard %s waited in queue longer than %s seconds! Queue length %s!'
% (keyboard, S3_CLEANUP_TIMEOUT,
len(qmk_redis.rq.jobs)))
else:
timeout = time() + COMPILE_TIMEOUT + 5
while job.get_status() == 'started':
if time() > timeout:
print(
'Compile timeout reached after %s seconds, giving up on this job.'
% (COMPILE_TIMEOUT))
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
break
sleep(2)
# Check over the job results
if job.result and job.result['returncode'] == 0:
print('Compile job completed successfully!')
good_boards += 1
qmk_redis.set('qmk_good_boards', good_boards)
configurator_build_status[keyboard] = {
'works': True,
'last_tested': int(time()),
'message': job.result['output']
}
keyboards_tested[
keyboard] = True # FIXME: Remove this when it's no longer used
if keyboard in failed_keyboards:
del failed_keyboards[
keyboard] # FIXME: Remove this when it's no longer used
layout_results[keyboard] = {
'result': True,
'reason': keyboard + ' works in configurator.'
}
else:
if job.result and job.result['returncode'] == -3:
output = f'Exception while compiling {keyboard}: {job.result["exception"]}'
print(output)
print(job.result['stacktrace'])
layout_results[keyboard] = {
'result':
False,
'reason':
f'{job.result["exception"]}: {job.result["stacktrace"]}'
}
elif job.result:
output = job.result['output']
print(
'Could not compile %s, layout %s, return code %s'
% (keyboard, keymap['layout'],
job.result['returncode']))
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s** does not work in configurator.' %
keymap['layout']
}
else:
output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT
print(output)
layout_results[keyboard] = {
'result':
False,
'reason':
'**%s**: Compile timeout reached.' %
keymap['layout']
}
bad_boards += 1
qmk_redis.set('qmk_bad_boards', bad_boards)
configurator_build_status[keyboard] = {
'works': False,
'last_tested': int(time()),
'message': output
}
keyboards_tested[
keyboard] = False # FIXME: Remove this when it's no longer used
failed_keyboards[keyboard] = {
'severity': 'error',
'message': output
} # FIXME: Remove this when it's no longer used
# Write our current progress to redis
qmk_redis.set('qmk_api_configurator_status',
configurator_build_status)
qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested)
qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards)
# Report this keyboard to discord
failed_layout = False
for result in layout_results.values():
if not result['result']:
failed_layout = True
break
if (MSG_ON_GOOD_COMPILE
and not failed_layout) or (MSG_ON_BAD_COMPILE
and failed_layout):
level = 'warning' if failed_layout else 'info'
message = 'Configurator summary for **' + keyboard + ':**'
for layout, result in layout_results.items():
icon = ':green_heart:' if result[
'result'] else ':broken_heart:'
message += '\n%s %s' % (icon, result['reason'])
discord.message(level, message, False)
except Exception as e:
print('***', strftime(TIME_FORMAT))
print('Uncaught exception!', e.__class__.__name__)
print(e)
discord.message(
'warning',
'Uncaught exception while testing %s.' % (keyboard, ))
print_exc()
# Remove stale build status entries
print('***', strftime(TIME_FORMAT))
print('Checking configurator_build_status for stale entries.')
for keyboard in list(configurator_build_status):
if time() - configurator_build_status[keyboard][
'last_tested'] > BUILD_STATUS_TIMEOUT:
print(
'Removing stale entry %s because it is %s seconds old'
% (keyboard,
configurator_build_status[keyboard]['last_tested']))
del configurator_build_status[keyboard]
# Notify discord that we've completed a circuit of the keyboards
if MSG_ON_LOOP_COMPLETION:
if last_good_boards is not None:
good_difference = good_boards - last_good_boards
bad_difference = bad_boards - last_bad_boards
if good_difference < 0:
good_difference = '%s fewer than' % (good_difference *
-1, )
elif good_difference > 0:
good_difference = '%s more than' % (good_difference, )
else:
good_difference = 'No change from'
if bad_difference < 0:
bad_difference = '%s fewer than' % (bad_difference *
-1, )
elif bad_difference > 0:
bad_difference = '%s more than' % (bad_difference, )
else:
bad_difference = 'No change from'
last_good_boards = good_boards
last_bad_boards = bad_boards
qmk_redis.set('qmk_last_good_boards', good_boards)
qmk_redis.set('qmk_last_bad_boards', bad_boards)
message = """We've completed a round of testing!
Working: %s the last round, for a total of %s working keyboards.
Non-working: %s the last round, for a total of %s non-working keyboards.
Check out the details here: <%s>"""
discord.message(
'info',
message % (good_difference, good_boards,
bad_difference, bad_boards, ERROR_PAGE_URL))
else:
last_good_boards = good_boards
last_bad_boards = bad_boards
# This comes after our `while True:` above and it should not be possible to break out of that loop.
print(
'How did we get here this should be impossible! HELP! HELP! HELP!')
discord.message(
'error',
'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!'
)
status['current'] = 'bad'
def periodic_tasks():
"""Jobs that need to run on a regular schedule.
"""
job_queue_last['compile'] = datetime.now()
qmk_redis.set('qmk_api_tasks_ping', time())
s3_cleanup()
