def process_keyboard(keyboard, usb_list, kb_list, kb_entries): """Parse all the files associated with a specific keyboard to build an API object for it. """ keyboard_info = build_keyboard_info(keyboard) for layout_name, layout_json in find_all_layouts(keyboard).items(): if not layout_name.startswith('LAYOUT_kc'): keyboard_info['layouts'][layout_name] = layout_json for info_json_filename in find_info_json(keyboard): # Merge info.json files into one. keyboard_info = merge_info_json(info_json_filename, keyboard_info) # Iterate through all the possible keymaps to build keymap jsons. for keymap_name, keymap_folder, layout_macro, layers in find_keymaps(keyboard): keyboard_info['keymaps'].append(keymap_name) write_keymap_redis(keyboard, keymap_name, keymap_folder, layers, layout_macro) # Pull some keyboard information from existing rules.mk and config.h files config_h = parse_config_h(keyboard) rules_mk = parse_rules_mk(keyboard) usb_entry = build_usb_entry(keyboard_info, config_h, usb_list) usb_list[usb_entry['vendor_id']][usb_entry['product_id']][keyboard] = usb_entry # Setup platform specific keys mcu = rules_mk.get('MCU') if mcu in ARM_PROCESSORS: arm_processor_rules(keyboard_info, rules_mk) elif mcu in AVR_PROCESSORS: avr_processor_rules(keyboard_info, rules_mk) else: log_warning("%s: Unknown MCU: %s" % (keyboard, mcu)) unknown_processor_rules(keyboard_info, rules_mk) # Used to identify keyboards in the redis key qmk_api_usb_list. keyboard_info['identifier'] = ':'.join(( keyboard_info.get('vendor_id', 'unknown'), keyboard_info.get('product_id', 'unknown'), keyboard_info.get('device_ver', 'unknown'), )) # Store the keyboard's readme in redis store_keyboard_readme(keyboard_info) # Write the keyboard to redis and add it to the master list. qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info) kb_list.append(keyboard) kb_entries['keyboards'][keyboard] = keyboard_info
def write_keymap_redis(keyboard, keymap_name, keymap_folder, layers, layout_macro): """Write a keymap to redis. """ keymap_blob = { 'keyboard_name': keyboard, 'keymap_name': keymap_name, 'keymap_folder': keymap_folder, 'layers': layers, 'layout_macro': layout_macro, } readme = '%s/%s/readme.md' % (keymap_folder, keymap_name) readme_text = unicode_text(readme) if exists(readme) else '%s does not exist.' % readme qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name), keymap_blob) qmk_redis.set('qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name), readme_text)
def store_keyboard_readme(keyboard_info): """Write a keyboard's readme file to redis. """ keyboard = keyboard_info['keyboard_folder'] readme_filename = None readme_path = '' for dir in keyboard.split('/'): readme_path = '/'.join((readme_path, dir)) new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path)) if new_name: readme_filename = new_name # Last one wins if readme_filename: qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), unicode_text(readme_filename)) keyboard_info['readme'] = True else: log_warning('%s does not have a readme.md.' % keyboard)
def update_kb_redis(): """Called to update qmk_firmware. """ # Clean up the environment and fetch the latest source del error_log[:] if not debug: if exists('update_kb_redis'): rmtree('update_kb_redis') mkdir('update_kb_redis') chdir('update_kb_redis') qmk_redis.set('qmk_needs_update', False) if not debug or not exists('qmk_firmware'): checkout_qmk(skip_cache=True) # Update redis with the latest data kb_list = [] usb_list = { } # Structure: VENDOR_ID: {PRODUCT_ID: {KEYBOARD_FOLDER: {'vendor_id': VENDOR_ID, 'product_id': PRODUCT_ID, 'device_ver': DEVICE_VER, 'manufacturer': MANUFACTURER, 'product': PRODUCT, 'keyboard': KEYBOARD_FOLDER} kb_all = { 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'), 'keyboards': {} } for keyboard in list_keyboards(): try: process_keyboard(keyboard, usb_list, kb_list, kb_all) except Exception as e: # Uncaught exception handler. Ideally this is never hit. log_error( 'Uncaught exception while processing keyboard %s! %s: %s' % (keyboard, e.__class__.__name__, str(e))) logging.exception(e) # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', kb_all) qmk_redis.set('qmk_api_usb_list', usb_list) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) logging.info('*** All keys successfully written to redis! Total size:', len(json.dumps(kb_all))) chdir('..') return True
def update_needed(**update_info): """Called when updates happen to QMK Firmware. """ qmk_redis.set('qmk_needs_update', True)
import qmk_redis qmk_redis.set('qmk_needs_update', True)
def update_kb_redis(): """Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data. """ # Check to see if we need to update if exists('qmk_firmware'): last_update = qmk_redis.get('qmk_api_last_updated') if not debug and isinstance( last_update, dict) and last_update['git_hash'] == git_hash(): # We are already up to date logging.warning( 'update_kb_redis(): Already up to date, skipping...') return False # Clean up the environment and fetch the latest source del (error_log[:]) if exists('qmk_firmware'): rmtree('qmk_firmware') if exists('qmk_firmware.zip'): remove('qmk_firmware.zip') qmk_storage.delete('cache/qmk_firmware.zip') checkout_qmk() # Update redis with the latest data kb_list = [] cached_json = { 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'), 'keyboards': {} } for keyboard in list_keyboards(): keyboard_info = { 'keyboard_name': keyboard, 'keyboard_folder': keyboard, 'keymaps': [], 'layouts': {}, 'maintainer': 'qmk', } for layout_name, layout_json in find_all_layouts(keyboard).items(): if not layout_name.startswith('LAYOUT_kc'): keyboard_info['layouts'][layout_name] = layout_json for info_json_filename in find_info_json(keyboard): # Iterate through all the possible info.json files to build the final keyboard JSON. try: with open(info_json_filename) as info_file: keyboard_info = merge_info_json(info_file, keyboard_info) except Exception as e: error_msg = 'Error encountered processing %s! %s: %s' % ( keyboard, e.__class__.__name__, e) error_log.append({ 'severity': 'error', 'message': 'Error: ' + error_msg }) logging.error(error_msg) logging.exception(e) # Iterate through all the possible keymaps to build keymap jsons. for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps( keyboard): keyboard_info['keymaps'].append(keymap_name) keymap_blob = { 'keyboard_name': keyboard, 'keymap_name': keymap_name, 'keymap_folder': keymap_folder, 'layers': keymap, 'layout_macro': layout_macro } # Write the keymap to redis qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name), keymap_blob) readme = '%s/%s/readme.md' % (keymap_folder, keymap_name) if exists(readme): with open(readme, 'rb') as readme_fd: readme_text = readme_fd.read() readme_text = UnicodeDammit(readme_text) readme_text = readme_text.unicode_markup else: readme_text = '%s does not exist.' % readme qmk_redis.set( 'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name), readme_text) # Pull some keyboard information from existing rules.mk and config.h files config_h = parse_config_h(keyboard) rules_mk = parse_rules_mk(keyboard) for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER', 'DESCRIPTION'): if key in config_h: if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'): config_h[key].replace('0x', '') config_h[key] = config_h[key].upper() keyboard_info[key.lower()] = config_h[key] if 'ARMV' in rules_mk: keyboard_info['processor_type'] = 'arm' # ARM processors if 'MCU' in rules_mk: keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT'] if 'MCU_LDSCRIPT' in rules_mk: keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: if 'STM32' in keyboard_info['processor']: keyboard_info['bootloader'] = 'stm32-dfu' elif keyboard_info.get('manufacturer') == 'Input Club': keyboard_info['bootloader'] = 'kiibohd-dfu' else: keyboard_info['processor_type'] = 'avr' # AVR processors if 'ARCH' in rules_mk: keyboard_info['platform'] = rules_mk['ARCH'] if 'MCU' in rules_mk: keyboard_info['processor'] = rules_mk['MCU'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: keyboard_info['bootloader'] = 'atmel-dfu' keyboard_info['identifier'] = ':'.join( (keyboard_info.get('vendor_id', 'unknown'), keyboard_info.get('product_id', 'unknown'), keyboard_info.get('device_ver', 'unknown'))) # Store the keyboard's readme in redis readme_filename = None readme_path = '' for dir in keyboard.split('/'): readme_path = '/'.join((readme_path, dir)) new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path)) if new_name: readme_filename = new_name # Last one wins if readme_filename: qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), open(readme_filename).read()) keyboard_info['readme'] = True else: error_msg = '%s does not have a readme.md.' % keyboard qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg) error_log.append({ 'severity': 'warning', 'message': 'Warning: ' + error_msg }) logging.warning(error_msg) keyboard_info['readme'] = False # Write the keyboard to redis and add it to the master list. qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info) kb_list.append(keyboard) cached_json['keyboards'][keyboard] = keyboard_info # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', cached_json) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) return True
def update_kb_redis(): """Called to update qmk_firmware. """ update_kb_redis = Path('update_kb_redis') qmk_firmware = Path('qmk_firmware') # Clean up the environment and fetch the latest source if not debug: # Create and enter a separate update_kb_redis directory to avoid conflicting with live builds if update_kb_redis.exists(): rmtree(update_kb_redis) update_kb_redis.mkdir() chdir(update_kb_redis) if not debug or not qmk_firmware.exists(): checkout_qmk(skip_cache=True) # Enter the qmk_firmware directory chdir(qmk_firmware) # Update redis with the latest keyboard data run(['qmk', 'generate-api']) api_dir = Path('api_data/v1') keyboards_dir = api_dir / 'keyboards' for keyboard_dir in keyboards_dir.glob('**'): keyboard_name = keyboard_dir.relative_to(keyboards_dir).as_posix() keyboard_info = keyboard_dir / 'info.json' keyboard_readme = keyboard_dir / 'readme.md' if keyboard_info.exists(): info_json = json.load(keyboard_info.open()) redis_json = info_json['keyboards'][keyboard_name] qmk_redis.set('qmk_api_kb_%s' % (keyboard_name), redis_json) if keyboard_readme.exists(): qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard_name), keyboard_readme.read_text()) # Update the USB list usb_json = json.load((api_dir / 'usb.json').open()) redis_usb = usb_json['usb'] qmk_redis.set('qmk_api_usb_list', redis_usb) # Update the Keyboard list keyboard_json = json.load((api_dir / 'keyboard_list.json').open()) redis_keyboard = keyboard_json['keyboards'] qmk_redis.set('qmk_api_keyboards', redis_keyboard) # Leave qmk_firmware chdir('..') # Set some metadata qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_needs_update', False) print('\n*** All keys successfully written to redis!') if not debug: # Leave update_kb_redis chdir('..') return True
def run(self): status['current'] = 'good' last_compile = 0 last_good_boards = qmk_redis.get('qmk_last_good_boards') last_bad_boards = qmk_redis.get('qmk_last_bad_boards') last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard') keyboards_tested = qmk_redis.get( 'qmk_api_keyboards_tested') # FIXME: Remove when no longer used if not keyboards_tested: keyboards_tested = {} failed_keyboards = qmk_redis.get( 'qmk_api_keyboards_failed') # FIXME: Remove when no longer used if not failed_keyboards: failed_keyboards = {} configurator_build_status = qmk_redis.get( 'qmk_api_configurator_status') if not configurator_build_status: configurator_build_status = {} while True: good_boards = 0 bad_boards = 0 keyboard_list = qmk_redis.get('qmk_api_keyboards') # If we stopped at a known keyboard restart from there if last_stop in keyboard_list: good_boards = qmk_redis.get('qmk_good_boards') or 0 bad_boards = qmk_redis.get('qmk_bad_boards') or 0 del (keyboard_list[:keyboard_list.index(last_stop)]) last_stop = None for keyboard in keyboard_list: global job_queue_last_compile global job_queue_last_warning periodic_tasks() # Cycle through each keyboard and build it try: # If we have too many jobs in the queue don't put stress on the infrastructure while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD: periodic_tasks() print( 'Too many jobs in the redis queue (%s)! Sleeping %s seconds...' % (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT)) sleep(COMPILE_TIMEOUT) if time( ) - job_queue_last_warning > JOB_QUEUE_TOO_LONG: job_queue_last_warning = time() level = 'warning' message = 'Compile queue too large (%s) since %s' % ( len(qmk_redis.rq.jobs), job_queue_last_compile.isoformat()) discord_msg(level, message) # Cycle through each layout for this keyboard and build it qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard) layout_results = {} metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard)) if not metadata['layouts']: bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) keyboards_tested[ keyboard + '/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = { 'severity': 'error', 'message': '%s: No layouts defined.' % keyboard } # FIXME: Remove when no longer used configurator_build_status[keyboard + '/[NO_LAYOUTS]'] = { 'works': False, 'last_tested': int(time()), 'message': '%s: No Layouts defined.' % keyboard } continue for layout_macro in list(metadata['layouts']): # Skip KEYMAP as it's deprecated if 'KEYMAP' in layout_macro: continue # Build a layout keyboard_layout_name = '/'.join( (keyboard, layout_macro)) layers = [ list( map( lambda x: 'KC_NO', metadata['layouts'] [layout_macro]['layout'])) ] layers.append(list(map(lambda x: 'KC_TRNS', layers[0]))) # Enqueue the job print('***', strftime(TIME_FORMAT)) print('Beginning test compile for %s, layout %s' % (keyboard, layout_macro)) args = (keyboard, 'qmk_api_tasks_test_compile', layout_macro, layers) job = qmk_redis.enqueue(compile_firmware, COMPILE_TIMEOUT, *args) print( 'Successfully enqueued, polling every 2 seconds...' ) # Wait for the job to start running if not wait_for_job_start(job): print( 'Waited %s seconds for %s/%s to start! Giving up at %s!' % (S3_CLEANUP_TIMEOUT, keyboard, layout_macro, strftime(TIME_FORMAT))) if MSG_ON_BAD_COMPILE: discord_msg( 'warning', 'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!' % (keyboard, layout_macro, S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs))) else: timeout = time() + COMPILE_TIMEOUT + 5 while job.get_status() == 'started': if time() > timeout: print( 'Compile timeout reached after %s seconds, giving up on this job.' % (COMPILE_TIMEOUT)) layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % layout_macro } break sleep(2) # Check over the job results if job.result and job.result['returncode'] == 0: print('Compile job completed successfully!') good_boards += 1 qmk_redis.set('qmk_good_boards', good_boards) configurator_build_status[keyboard_layout_name] = { 'works': True, 'last_tested': int(time()), 'message': job.result['output'] } keyboards_tested[ keyboard_layout_name] = True # FIXME: Remove this when it's no longer used if keyboard_layout_name in failed_keyboards: del failed_keyboards[ keyboard_layout_name] # FIXME: Remove this when it's no longer used layout_results[keyboard_layout_name] = { 'result': True, 'reason': layout_macro + ' works in configurator.' } else: if job.result: output = job.result['output'] print( 'Could not compile %s, layout %s, return code %s' % (keyboard, layout_macro, job.result['returncode'])) print(output) layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s** does not work in configurator.' % layout_macro } else: output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % layout_macro } bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard_layout_name] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard_layout_name] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard_layout_name] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used # Write our current progress to redis qmk_redis.set('qmk_api_configurator_status', configurator_build_status) qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested) qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards) # Report this keyboard to discord failed_layout = False for result in layout_results.values(): if not result['result']: failed_layout = True break if (MSG_ON_GOOD_COMPILE and not failed_layout) or (MSG_ON_BAD_COMPILE and failed_layout): level = 'warning' if failed_layout else 'info' message = 'Configurator summary for **' + keyboard + ':**' for layout, result in layout_results.items(): icon = ':green_heart:' if result[ 'result'] else ':broken_heart:' message += '\n%s %s' % (icon, result['reason']) discord_msg(level, message, False) except Exception as e: print('***', strftime(TIME_FORMAT)) print('Uncaught exception!', e.__class__.__name__) print(e) discord_msg( 'warning', 'Uncaught exception while testing %s.' % (keyboard, )) print_exc() # Remove stale build status entries print('***', strftime(TIME_FORMAT)) print('Checking configurator_build_status for stale entries.') for keyboard_layout_name in list(configurator_build_status): if time() - configurator_build_status[keyboard_layout_name][ 'last_tested'] > BUILD_STATUS_TIMEOUT: print( 'Removing stale entry %s because it is %s seconds old' % (keyboard_layout_name, configurator_build_status[keyboard_layout_name] ['last_tested'])) del configurator_build_status[keyboard_layout_name] # Notify discord that we've completed a circuit of the keyboards if MSG_ON_LOOP_COMPLETION: if last_good_boards is not None: good_difference = good_boards - last_good_boards bad_difference = bad_boards - last_bad_boards if good_difference < 0: good_difference = '%s fewer than' % (good_difference * -1, ) elif good_difference > 0: good_difference = '%s more than' % (good_difference, ) else: good_difference = 'No change from' if bad_difference < 0: bad_difference = '%s fewer than' % (bad_difference * -1, ) elif bad_difference > 0: bad_difference = '%s more than' % (bad_difference, ) else: bad_difference = 'No change from' last_good_boards = good_boards last_bad_boards = bad_boards qmk_redis.set('qmk_last_good_boards', good_boards) qmk_redis.set('qmk_last_bad_boards', bad_boards) message = """We've completed a round of testing! Working: %s the last round, for a total of %s working keyboard/layout combinations. Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations. Check out the details here: <%s>""" discord_msg( 'info', message % (good_difference, good_boards, bad_difference, bad_boards, ERROR_PAGE_URL)) else: last_good_boards = good_boards last_bad_boards = bad_boards # This comes after our `while True:` above and it should not be possible to break out of that loop. print( 'How did we get here this should be impossible! HELP! HELP! HELP!') discord_msg( 'error', 'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!' ) status['current'] = 'bad'
def periodic_tasks(): """Jobs that need to run on a regular schedule. """ qmk_redis.set('qmk_api_tasks_ping', time()) s3_cleanup() update_qmk_firmware()
def run(self): status['current'] = 'good' last_good_boards = qmk_redis.get('qmk_last_good_boards') last_bad_boards = qmk_redis.get('qmk_last_bad_boards') last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard') keyboards_tested = qmk_redis.get( 'qmk_api_keyboards_tested') # FIXME: Remove when no longer used if not keyboards_tested: keyboards_tested = {} failed_keyboards = qmk_redis.get( 'qmk_api_keyboards_failed') # FIXME: Remove when no longer used if not failed_keyboards: failed_keyboards = {} configurator_build_status = qmk_redis.get( 'qmk_api_configurator_status') if not configurator_build_status: configurator_build_status = {} while True: good_boards = 0 bad_boards = 0 keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json' keyboard_list = fetch_json(keyboard_list_url).get('keyboards') if not keyboard_list: print( 'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...' % (keyboard_list_url, COMPILE_TIMEOUT)) periodic_tasks() sleep(COMPILE_TIMEOUT) continue # If we stopped at a known keyboard restart from there if last_stop in keyboard_list: good_boards = qmk_redis.get('qmk_good_boards') or 0 bad_boards = qmk_redis.get('qmk_bad_boards') or 0 del (keyboard_list[:keyboard_list.index(last_stop)]) last_stop = None for keyboard in keyboard_list: periodic_tasks() # Cycle through each keyboard and build it try: # If we have too many jobs in the queue don't put stress on the infrastructure while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD: periodic_tasks() print( 'Too many jobs in the redis queue (%s)! Sleeping %s seconds...' % (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT)) sleep(COMPILE_TIMEOUT) if time( ) - job_queue_last['warning'] > JOB_QUEUE_TOO_LONG: job_queue_last['warning'] = time() level = 'warning' message = 'Compile queue too large (%s) since %s' % ( len(qmk_redis.rq.jobs), job_queue_last['compile'].isoformat()) discord.message(level, message) # Find or generate a default keymap for this keyboard. qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard) layout_results = {} metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json' metadata = fetch_json(metadata_url).get('keyboards', {}).get(keyboard) if not metadata: print( '*** Sleeping for 60 seconds then continuing to the next keyboard...' ) sleep(60) continue if metadata.get( 'keymaps') and 'default' in metadata['keymaps']: keymap = fetch_json( metadata['keymaps']['default']['url']) keymap['keymap'] = 'default' else: keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json' keymap = fetch_json(keymap_url) if keymap: keymap['keymap'] = 'default_configurator' else: # Fall back to building an empty keymap if metadata.get('layouts'): layout_macro = random.choice( list(metadata['layouts'])) layout_len = len(metadata['layouts'] [layout_macro]['layout']) keymap = { 'keyboard': keyboard, 'keymap': 'generated', 'layout': layout_macro, 'layers': [['KC_NO' for i in range(layout_len)], ['KC_TRNS' for i in range(layout_len)]] } else: output = f'No layouts for {keyboard}! Skipping!' bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used print(output) continue # Enqueue the job print('***', strftime(TIME_FORMAT)) print('Beginning test compile for %s, layout %s' % (keyboard, keymap['layout'])) job = qmk_redis.enqueue(compile_json, COMPILE_TIMEOUT, keymap, send_metrics=False, public_firmware=True) print('Successfully enqueued, polling every 2 seconds...') # Wait for the job to start running if not wait_for_job_start(job): print( 'Waited %s seconds for %s to start! Giving up at %s!' % (S3_CLEANUP_TIMEOUT, keyboard, strftime(TIME_FORMAT))) if MSG_ON_BAD_COMPILE: discord.message( 'warning', 'Keyboard %s waited in queue longer than %s seconds! Queue length %s!' % (keyboard, S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs))) else: timeout = time() + COMPILE_TIMEOUT + 5 while job.get_status() == 'started': if time() > timeout: print( 'Compile timeout reached after %s seconds, giving up on this job.' % (COMPILE_TIMEOUT)) layout_results[keyboard] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % keymap['layout'] } break sleep(2) # Check over the job results if job.result and job.result['returncode'] == 0: print('Compile job completed successfully!') good_boards += 1 qmk_redis.set('qmk_good_boards', good_boards) configurator_build_status[keyboard] = { 'works': True, 'last_tested': int(time()), 'message': job.result['output'] } keyboards_tested[ keyboard] = True # FIXME: Remove this when it's no longer used if keyboard in failed_keyboards: del failed_keyboards[ keyboard] # FIXME: Remove this when it's no longer used layout_results[keyboard] = { 'result': True, 'reason': keyboard + ' works in configurator.' } else: if job.result and job.result['returncode'] == -3: output = f'Exception while compiling {keyboard}: {job.result["exception"]}' print(output) print(job.result['stacktrace']) layout_results[keyboard] = { 'result': False, 'reason': f'{job.result["exception"]}: {job.result["stacktrace"]}' } elif job.result: output = job.result['output'] print( 'Could not compile %s, layout %s, return code %s' % (keyboard, keymap['layout'], job.result['returncode'])) print(output) layout_results[keyboard] = { 'result': False, 'reason': '**%s** does not work in configurator.' % keymap['layout'] } else: output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT print(output) layout_results[keyboard] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % keymap['layout'] } bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used # Write our current progress to redis qmk_redis.set('qmk_api_configurator_status', configurator_build_status) qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested) qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards) # Report this keyboard to discord failed_layout = False for result in layout_results.values(): if not result['result']: failed_layout = True break if (MSG_ON_GOOD_COMPILE and not failed_layout) or (MSG_ON_BAD_COMPILE and failed_layout): level = 'warning' if failed_layout else 'info' message = 'Configurator summary for **' + keyboard + ':**' for layout, result in layout_results.items(): icon = ':green_heart:' if result[ 'result'] else ':broken_heart:' message += '\n%s %s' % (icon, result['reason']) discord.message(level, message, False) except Exception as e: print('***', strftime(TIME_FORMAT)) print('Uncaught exception!', e.__class__.__name__) print(e) discord.message( 'warning', 'Uncaught exception while testing %s.' % (keyboard, )) print_exc() # Remove stale build status entries print('***', strftime(TIME_FORMAT)) print('Checking configurator_build_status for stale entries.') for keyboard in list(configurator_build_status): if time() - configurator_build_status[keyboard][ 'last_tested'] > BUILD_STATUS_TIMEOUT: print( 'Removing stale entry %s because it is %s seconds old' % (keyboard, configurator_build_status[keyboard]['last_tested'])) del configurator_build_status[keyboard] # Notify discord that we've completed a circuit of the keyboards if MSG_ON_LOOP_COMPLETION: if last_good_boards is not None: good_difference = good_boards - last_good_boards bad_difference = bad_boards - last_bad_boards if good_difference < 0: good_difference = '%s fewer than' % (good_difference * -1, ) elif good_difference > 0: good_difference = '%s more than' % (good_difference, ) else: good_difference = 'No change from' if bad_difference < 0: bad_difference = '%s fewer than' % (bad_difference * -1, ) elif bad_difference > 0: bad_difference = '%s more than' % (bad_difference, ) else: bad_difference = 'No change from' last_good_boards = good_boards last_bad_boards = bad_boards qmk_redis.set('qmk_last_good_boards', good_boards) qmk_redis.set('qmk_last_bad_boards', bad_boards) message = """We've completed a round of testing! Working: %s the last round, for a total of %s working keyboards. Non-working: %s the last round, for a total of %s non-working keyboards. Check out the details here: <%s>""" discord.message( 'info', message % (good_difference, good_boards, bad_difference, bad_boards, ERROR_PAGE_URL)) else: last_good_boards = good_boards last_bad_boards = bad_boards # This comes after our `while True:` above and it should not be possible to break out of that loop. print( 'How did we get here this should be impossible! HELP! HELP! HELP!') discord.message( 'error', 'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!' ) status['current'] = 'bad'
def periodic_tasks(): """Jobs that need to run on a regular schedule. """ job_queue_last['compile'] = datetime.now() qmk_redis.set('qmk_api_tasks_ping', time()) s3_cleanup()