def GET_v1_keyboards_keyboard_keymaps_keymap(keyboard, keymap): """Return JSON showing data about a keyboard's keymap """ keyboards = qmk_redis.get('qmk_api_last_updated') keyboards['keyboards'] = {} for kb in keyboard.split(','): kb_data = qmk_redis.get('qmk_api_kb_'+kb) if kb_data: keymaps = {} keymap_list = kb_data['keymaps'] if keymap == 'all' else keymap.split(',') for km in keymap_list: keymaps[km] = qmk_redis.get('qmk_api_kb_%s_keymap_%s' % (kb, km)) if not keymaps: return error('No such keymap: ' + keymap, 404) kb_data['keymaps'] = keymaps keyboards['keyboards'][kb] = kb_data if not keyboards['keyboards']: return error('No such keyboard: ' + keyboard, 404) return jsonify(keyboards)
def update_qmk_firmware(): """Update the API from qmk_firmware after a push. """ if qmk_redis.get('qmk_needs_update'): print('***', strftime(TIME_FORMAT)) print('Beginning qmk_firmware update.') job = qmk_redis.enqueue(update_kb_redis, timeout=QMK_UPDATE_TIMEOUT) print( 'Successfully enqueued job id %s at %s, polling every 2 seconds...' % (job.id, strftime(TIME_FORMAT))) start_time = time() # Wait for the job to start running if not wait_for_job_start(job): print('QMK update queued for %s seconds! Giving up at %s!' % (S3_CLEANUP_TIMEOUT, strftime(TIME_FORMAT))) if MSG_ON_QMK_FAIL: discord_msg( 'warning', 'S3 cleanup queue longer than %s seconds! Queue length %s!' % (S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs))) return False # Monitor the job while it runs while job.get_status() == 'started': if time() - start_time > QMK_UPDATE_TIMEOUT + 5: print( 'QMK update took longer than %s seconds! Cancelling at %s!' % (QMK_UPDATE_TIMEOUT, strftime(TIME_FORMAT))) if MSG_ON_QMK_FAIL: discord_msg( 'error', 'QMK update took longer than %s seconds!' % (QMK_UPDATE_TIMEOUT, )) break sleep(2) # Check over the results if job.result: hash = qmk_redis.get('qmk_api_last_updated')['git_hash'] message = 'QMK update completed successfully! API is current to ' + hash print(message) discord_msg('info', message) if ERROR_LOG_NAG: error_log = qmk_redis.get('qmk_api_update_error_log') if len(error_log) > 5: discord_msg( 'error', 'There are %s errors from the update process. View them here: %s' % (len(error_log), ERROR_LOG_URL)) else: print('Could not update qmk_firmware!') print(job) print(job.result) if MSG_ON_QMK_FAIL: discord_msg('warning', 'QMK update did not complete successfully!')
def GET_v1_keyboards_keyboard(keyboard): """Return JSON showing data about a keyboard """ keyboards = qmk_redis.get('qmk_api_last_updated') keyboards['keyboards'] = {} for kb in keyboard.split(','): kb_data = qmk_redis.get('qmk_api_kb_' + kb) if kb_data: keyboards['keyboards'][kb] = kb_data if not keyboards['keyboards']: return error('No such keyboard: ' + keyboard, 404) return jsonify(keyboards)
def GET_v1_keyboards_all(): """Return JSON showing all available keyboards and their layouts. """ allkb = qmk_redis.get('qmk_api_kb_all') if allkb: return jsonify(allkb) return error('An unknown error occured', 500)
def GET_v1_keyboards_keyboard(keyboard): """Return JSON showing data about a keyboard """ keyboards = { 'last_updated': qmk_redis.get('qmk_api_last_updated'), 'keyboards': {} } for kb in keyboard.split(','): kb_data = qmk_redis.get('qmk_api_kb_'+kb) if kb_data: keyboards['keyboards'][kb] = kb_data if not keyboards['keyboards']: return error('No such keyboard: ' + keyboard, 404) return jsonify(keyboards)
def GET_v1_keyboards_build_log(): """Returns a dictionary of keyboard/layout pairs. Each entry is a dictionary with the following keys: * `works`: Boolean indicating whether the compile was successful * `message`: The compile output for failed builds """ json_data = qmk_redis.get('qmk_api_configurator_status') return jsonify(json_data)
def GET_v1_keyboards_keyboard_readme(keyboard): """Returns the readme for a keyboard. """ readme = qmk_redis.get('qmk_api_kb_%s_readme' % (keyboard)) response = make_response(readme) response.mimetype = 'text/markdown' return response
def GET_v1(): """Return the API's status. """ return jsonify({ 'children': ['compile', 'converters', 'keyboards'], 'last_ping': qmk_redis.get('qmk_api_last_ping'), 'queue_length': len(rq), 'status': 'running', 'version': __VERSION__ })
def GET_v1_update(): """Triggers an update of the API. """ result = { 'result': UPDATE_API, 'last_ping': qmk_redis.get('qmk_api_last_ping'), 'queue_length': len(rq), 'queued_job_ids': rq.job_ids, 'status': 'running', 'version': __VERSION__ } if UPDATE_API: rq.enqueue(update_kb_redis) return jsonify(result)
def GET_v1_healthcheck(): """Checks over the health of the API. Note: This is used for operational purposes. Please don't hit it on the live api.qmk.fm site without talking to us first. Most of this information is available at the /v1 endpoint as well. """ rq.enqueue(ping, at_front=True) return jsonify({ 'last_ping': qmk_redis.get('qmk_api_last_ping'), 'queue_length': len(rq), 'queued_job_ids': rq.job_ids, 'status': 'running', 'version': __VERSION__ })
def check_pings(): """Check the ping values and update api_status with them. """ api_status['queue_length'] = len(rq) for redis_key in ('qmk_api_last_ping', 'qmk_api_tasks_ping'): key = redis_key.replace('qmk_api_', '') value = qmk_redis.get(redis_key) api_status[key] = value if value: if time() - float(value) > CHECK_TIMEOUT: api_status['status'] = 'degraded' api_status['status_%s' % key] = 'degraded' else: api_status['status'] = 'running' api_status['status_%s' % key] = 'good' else: api_status['status'] = 'degraded' api_status['status_%s' % key] = 'degraded'
def GET_v1_usb(): """Returns the list of USB device identifiers used in QMK. """ json_blob = qmk_redis.get('qmk_api_usb_list') return jsonify(json_blob)
def GET_v1_keyboards(): """Return a list of keyboards """ json_blob = qmk_redis.get('qmk_api_keyboards') return jsonify(json_blob)
def GET_v1_keyboards_error_log(): """Return the error log from the last run. """ json_blob = qmk_redis.get('qmk_api_update_error_log') return jsonify(json_blob)
def run(self): status['current'] = 'good' last_good_boards = qmk_redis.get('qmk_last_good_boards') last_bad_boards = qmk_redis.get('qmk_last_bad_boards') last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard') keyboards_tested = qmk_redis.get( 'qmk_api_keyboards_tested') # FIXME: Remove when no longer used if not keyboards_tested: keyboards_tested = {} failed_keyboards = qmk_redis.get( 'qmk_api_keyboards_failed') # FIXME: Remove when no longer used if not failed_keyboards: failed_keyboards = {} configurator_build_status = qmk_redis.get( 'qmk_api_configurator_status') if not configurator_build_status: configurator_build_status = {} while True: good_boards = 0 bad_boards = 0 keyboard_list_url = f'{QMK_JSON_URL}/keyboard_list.json' keyboard_list = fetch_json(keyboard_list_url).get('keyboards') if not keyboard_list: print( 'Could not fetch keyboard list from %s! Running periodic_tasks() then sleeping %s seconds...' % (keyboard_list_url, COMPILE_TIMEOUT)) periodic_tasks() sleep(COMPILE_TIMEOUT) continue # If we stopped at a known keyboard restart from there if last_stop in keyboard_list: good_boards = qmk_redis.get('qmk_good_boards') or 0 bad_boards = qmk_redis.get('qmk_bad_boards') or 0 del (keyboard_list[:keyboard_list.index(last_stop)]) last_stop = None for keyboard in keyboard_list: periodic_tasks() # Cycle through each keyboard and build it try: # If we have too many jobs in the queue don't put stress on the infrastructure while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD: periodic_tasks() print( 'Too many jobs in the redis queue (%s)! Sleeping %s seconds...' % (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT)) sleep(COMPILE_TIMEOUT) if time( ) - job_queue_last['warning'] > JOB_QUEUE_TOO_LONG: job_queue_last['warning'] = time() level = 'warning' message = 'Compile queue too large (%s) since %s' % ( len(qmk_redis.rq.jobs), job_queue_last['compile'].isoformat()) discord.message(level, message) # Find or generate a default keymap for this keyboard. qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard) layout_results = {} metadata_url = f'{QMK_JSON_URL}/keyboards/{keyboard}/info.json' metadata = fetch_json(metadata_url).get('keyboards', {}).get(keyboard) if not metadata: print( '*** Sleeping for 60 seconds then continuing to the next keyboard...' ) sleep(60) continue if metadata.get( 'keymaps') and 'default' in metadata['keymaps']: keymap = fetch_json( metadata['keymaps']['default']['url']) keymap['keymap'] = 'default' else: keymap_url = f'{KEYMAP_JSON_URL}/{keyboard[0]}/{keyboard.replace("/", "_")}_default.json' keymap = fetch_json(keymap_url) if keymap: keymap['keymap'] = 'default_configurator' else: # Fall back to building an empty keymap if metadata.get('layouts'): layout_macro = random.choice( list(metadata['layouts'])) layout_len = len(metadata['layouts'] [layout_macro]['layout']) keymap = { 'keyboard': keyboard, 'keymap': 'generated', 'layout': layout_macro, 'layers': [['KC_NO' for i in range(layout_len)], ['KC_TRNS' for i in range(layout_len)]] } else: output = f'No layouts for {keyboard}! Skipping!' bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used print(output) continue # Enqueue the job print('***', strftime(TIME_FORMAT)) print('Beginning test compile for %s, layout %s' % (keyboard, keymap['layout'])) job = qmk_redis.enqueue(compile_json, COMPILE_TIMEOUT, keymap, send_metrics=False, public_firmware=True) print('Successfully enqueued, polling every 2 seconds...') # Wait for the job to start running if not wait_for_job_start(job): print( 'Waited %s seconds for %s to start! Giving up at %s!' % (S3_CLEANUP_TIMEOUT, keyboard, strftime(TIME_FORMAT))) if MSG_ON_BAD_COMPILE: discord.message( 'warning', 'Keyboard %s waited in queue longer than %s seconds! Queue length %s!' % (keyboard, S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs))) else: timeout = time() + COMPILE_TIMEOUT + 5 while job.get_status() == 'started': if time() > timeout: print( 'Compile timeout reached after %s seconds, giving up on this job.' % (COMPILE_TIMEOUT)) layout_results[keyboard] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % keymap['layout'] } break sleep(2) # Check over the job results if job.result and job.result['returncode'] == 0: print('Compile job completed successfully!') good_boards += 1 qmk_redis.set('qmk_good_boards', good_boards) configurator_build_status[keyboard] = { 'works': True, 'last_tested': int(time()), 'message': job.result['output'] } keyboards_tested[ keyboard] = True # FIXME: Remove this when it's no longer used if keyboard in failed_keyboards: del failed_keyboards[ keyboard] # FIXME: Remove this when it's no longer used layout_results[keyboard] = { 'result': True, 'reason': keyboard + ' works in configurator.' } else: if job.result and job.result['returncode'] == -3: output = f'Exception while compiling {keyboard}: {job.result["exception"]}' print(output) print(job.result['stacktrace']) layout_results[keyboard] = { 'result': False, 'reason': f'{job.result["exception"]}: {job.result["stacktrace"]}' } elif job.result: output = job.result['output'] print( 'Could not compile %s, layout %s, return code %s' % (keyboard, keymap['layout'], job.result['returncode'])) print(output) layout_results[keyboard] = { 'result': False, 'reason': '**%s** does not work in configurator.' % keymap['layout'] } else: output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT print(output) layout_results[keyboard] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % keymap['layout'] } bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used # Write our current progress to redis qmk_redis.set('qmk_api_configurator_status', configurator_build_status) qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested) qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards) # Report this keyboard to discord failed_layout = False for result in layout_results.values(): if not result['result']: failed_layout = True break if (MSG_ON_GOOD_COMPILE and not failed_layout) or (MSG_ON_BAD_COMPILE and failed_layout): level = 'warning' if failed_layout else 'info' message = 'Configurator summary for **' + keyboard + ':**' for layout, result in layout_results.items(): icon = ':green_heart:' if result[ 'result'] else ':broken_heart:' message += '\n%s %s' % (icon, result['reason']) discord.message(level, message, False) except Exception as e: print('***', strftime(TIME_FORMAT)) print('Uncaught exception!', e.__class__.__name__) print(e) discord.message( 'warning', 'Uncaught exception while testing %s.' % (keyboard, )) print_exc() # Remove stale build status entries print('***', strftime(TIME_FORMAT)) print('Checking configurator_build_status for stale entries.') for keyboard in list(configurator_build_status): if time() - configurator_build_status[keyboard][ 'last_tested'] > BUILD_STATUS_TIMEOUT: print( 'Removing stale entry %s because it is %s seconds old' % (keyboard, configurator_build_status[keyboard]['last_tested'])) del configurator_build_status[keyboard] # Notify discord that we've completed a circuit of the keyboards if MSG_ON_LOOP_COMPLETION: if last_good_boards is not None: good_difference = good_boards - last_good_boards bad_difference = bad_boards - last_bad_boards if good_difference < 0: good_difference = '%s fewer than' % (good_difference * -1, ) elif good_difference > 0: good_difference = '%s more than' % (good_difference, ) else: good_difference = 'No change from' if bad_difference < 0: bad_difference = '%s fewer than' % (bad_difference * -1, ) elif bad_difference > 0: bad_difference = '%s more than' % (bad_difference, ) else: bad_difference = 'No change from' last_good_boards = good_boards last_bad_boards = bad_boards qmk_redis.set('qmk_last_good_boards', good_boards) qmk_redis.set('qmk_last_bad_boards', bad_boards) message = """We've completed a round of testing! Working: %s the last round, for a total of %s working keyboards. Non-working: %s the last round, for a total of %s non-working keyboards. Check out the details here: <%s>""" discord.message( 'info', message % (good_difference, good_boards, bad_difference, bad_boards, ERROR_PAGE_URL)) else: last_good_boards = good_boards last_bad_boards = bad_boards # This comes after our `while True:` above and it should not be possible to break out of that loop. print( 'How did we get here this should be impossible! HELP! HELP! HELP!') discord.message( 'error', 'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!' ) status['current'] = 'bad'
def GET_v1_keyboards_build_status(): """Returns a dictionary of keyboard/layout pairs. Each entry is True if the keyboard works in configurator and false if it doesn't. """ json_blob = qmk_redis.get('qmk_api_keyboards_tested') return jsonify(json_blob)
def run(self): status['current'] = 'good' last_compile = 0 last_good_boards = qmk_redis.get('qmk_last_good_boards') last_bad_boards = qmk_redis.get('qmk_last_bad_boards') last_stop = qmk_redis.get('qmk_api_tasks_current_keyboard') keyboards_tested = qmk_redis.get( 'qmk_api_keyboards_tested') # FIXME: Remove when no longer used if not keyboards_tested: keyboards_tested = {} failed_keyboards = qmk_redis.get( 'qmk_api_keyboards_failed') # FIXME: Remove when no longer used if not failed_keyboards: failed_keyboards = {} configurator_build_status = qmk_redis.get( 'qmk_api_configurator_status') if not configurator_build_status: configurator_build_status = {} while True: good_boards = 0 bad_boards = 0 keyboard_list = qmk_redis.get('qmk_api_keyboards') # If we stopped at a known keyboard restart from there if last_stop in keyboard_list: good_boards = qmk_redis.get('qmk_good_boards') or 0 bad_boards = qmk_redis.get('qmk_bad_boards') or 0 del (keyboard_list[:keyboard_list.index(last_stop)]) last_stop = None for keyboard in keyboard_list: global job_queue_last_compile global job_queue_last_warning periodic_tasks() # Cycle through each keyboard and build it try: # If we have too many jobs in the queue don't put stress on the infrastructure while len(qmk_redis.rq.jobs) > JOB_QUEUE_THRESHOLD: periodic_tasks() print( 'Too many jobs in the redis queue (%s)! Sleeping %s seconds...' % (len(qmk_redis.rq.jobs), COMPILE_TIMEOUT)) sleep(COMPILE_TIMEOUT) if time( ) - job_queue_last_warning > JOB_QUEUE_TOO_LONG: job_queue_last_warning = time() level = 'warning' message = 'Compile queue too large (%s) since %s' % ( len(qmk_redis.rq.jobs), job_queue_last_compile.isoformat()) discord_msg(level, message) # Cycle through each layout for this keyboard and build it qmk_redis.set('qmk_api_tasks_current_keyboard', keyboard) layout_results = {} metadata = qmk_redis.get('qmk_api_kb_%s' % (keyboard)) if not metadata['layouts']: bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) keyboards_tested[ keyboard + '/[NO_LAYOUTS]'] = False # FIXME: Remove when no longer used failed_keyboards[keyboard + '/[NO_LAYOUTS]'] = { 'severity': 'error', 'message': '%s: No layouts defined.' % keyboard } # FIXME: Remove when no longer used configurator_build_status[keyboard + '/[NO_LAYOUTS]'] = { 'works': False, 'last_tested': int(time()), 'message': '%s: No Layouts defined.' % keyboard } continue for layout_macro in list(metadata['layouts']): # Skip KEYMAP as it's deprecated if 'KEYMAP' in layout_macro: continue # Build a layout keyboard_layout_name = '/'.join( (keyboard, layout_macro)) layers = [ list( map( lambda x: 'KC_NO', metadata['layouts'] [layout_macro]['layout'])) ] layers.append(list(map(lambda x: 'KC_TRNS', layers[0]))) # Enqueue the job print('***', strftime(TIME_FORMAT)) print('Beginning test compile for %s, layout %s' % (keyboard, layout_macro)) args = (keyboard, 'qmk_api_tasks_test_compile', layout_macro, layers) job = qmk_redis.enqueue(compile_firmware, COMPILE_TIMEOUT, *args) print( 'Successfully enqueued, polling every 2 seconds...' ) # Wait for the job to start running if not wait_for_job_start(job): print( 'Waited %s seconds for %s/%s to start! Giving up at %s!' % (S3_CLEANUP_TIMEOUT, keyboard, layout_macro, strftime(TIME_FORMAT))) if MSG_ON_BAD_COMPILE: discord_msg( 'warning', 'Keyboard %s/%s waited in queue longer than %s seconds! Queue length %s!' % (keyboard, layout_macro, S3_CLEANUP_TIMEOUT, len(qmk_redis.rq.jobs))) else: timeout = time() + COMPILE_TIMEOUT + 5 while job.get_status() == 'started': if time() > timeout: print( 'Compile timeout reached after %s seconds, giving up on this job.' % (COMPILE_TIMEOUT)) layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % layout_macro } break sleep(2) # Check over the job results if job.result and job.result['returncode'] == 0: print('Compile job completed successfully!') good_boards += 1 qmk_redis.set('qmk_good_boards', good_boards) configurator_build_status[keyboard_layout_name] = { 'works': True, 'last_tested': int(time()), 'message': job.result['output'] } keyboards_tested[ keyboard_layout_name] = True # FIXME: Remove this when it's no longer used if keyboard_layout_name in failed_keyboards: del failed_keyboards[ keyboard_layout_name] # FIXME: Remove this when it's no longer used layout_results[keyboard_layout_name] = { 'result': True, 'reason': layout_macro + ' works in configurator.' } else: if job.result: output = job.result['output'] print( 'Could not compile %s, layout %s, return code %s' % (keyboard, layout_macro, job.result['returncode'])) print(output) layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s** does not work in configurator.' % layout_macro } else: output = 'Job took longer than %s seconds, giving up!' % COMPILE_TIMEOUT layout_results[keyboard_layout_name] = { 'result': False, 'reason': '**%s**: Compile timeout reached.' % layout_macro } bad_boards += 1 qmk_redis.set('qmk_bad_boards', bad_boards) configurator_build_status[keyboard_layout_name] = { 'works': False, 'last_tested': int(time()), 'message': output } keyboards_tested[ keyboard_layout_name] = False # FIXME: Remove this when it's no longer used failed_keyboards[keyboard_layout_name] = { 'severity': 'error', 'message': output } # FIXME: Remove this when it's no longer used # Write our current progress to redis qmk_redis.set('qmk_api_configurator_status', configurator_build_status) qmk_redis.set('qmk_api_keyboards_tested', keyboards_tested) qmk_redis.set('qmk_api_keyboards_failed', failed_keyboards) # Report this keyboard to discord failed_layout = False for result in layout_results.values(): if not result['result']: failed_layout = True break if (MSG_ON_GOOD_COMPILE and not failed_layout) or (MSG_ON_BAD_COMPILE and failed_layout): level = 'warning' if failed_layout else 'info' message = 'Configurator summary for **' + keyboard + ':**' for layout, result in layout_results.items(): icon = ':green_heart:' if result[ 'result'] else ':broken_heart:' message += '\n%s %s' % (icon, result['reason']) discord_msg(level, message, False) except Exception as e: print('***', strftime(TIME_FORMAT)) print('Uncaught exception!', e.__class__.__name__) print(e) discord_msg( 'warning', 'Uncaught exception while testing %s.' % (keyboard, )) print_exc() # Remove stale build status entries print('***', strftime(TIME_FORMAT)) print('Checking configurator_build_status for stale entries.') for keyboard_layout_name in list(configurator_build_status): if time() - configurator_build_status[keyboard_layout_name][ 'last_tested'] > BUILD_STATUS_TIMEOUT: print( 'Removing stale entry %s because it is %s seconds old' % (keyboard_layout_name, configurator_build_status[keyboard_layout_name] ['last_tested'])) del configurator_build_status[keyboard_layout_name] # Notify discord that we've completed a circuit of the keyboards if MSG_ON_LOOP_COMPLETION: if last_good_boards is not None: good_difference = good_boards - last_good_boards bad_difference = bad_boards - last_bad_boards if good_difference < 0: good_difference = '%s fewer than' % (good_difference * -1, ) elif good_difference > 0: good_difference = '%s more than' % (good_difference, ) else: good_difference = 'No change from' if bad_difference < 0: bad_difference = '%s fewer than' % (bad_difference * -1, ) elif bad_difference > 0: bad_difference = '%s more than' % (bad_difference, ) else: bad_difference = 'No change from' last_good_boards = good_boards last_bad_boards = bad_boards qmk_redis.set('qmk_last_good_boards', good_boards) qmk_redis.set('qmk_last_bad_boards', bad_boards) message = """We've completed a round of testing! Working: %s the last round, for a total of %s working keyboard/layout combinations. Non-working: %s the last round, for a total of %s non-working keyboard/layout combinations. Check out the details here: <%s>""" discord_msg( 'info', message % (good_difference, good_boards, bad_difference, bad_boards, ERROR_PAGE_URL)) else: last_good_boards = good_boards last_bad_boards = bad_boards # This comes after our `while True:` above and it should not be possible to break out of that loop. print( 'How did we get here this should be impossible! HELP! HELP! HELP!') discord_msg( 'error', 'How did we get here this should impossible! @skullydazed HELP! @skullydazed HELP! @skullydazed HELP!' ) status['current'] = 'bad'
return float(obj) except TypeError: pass return JSONEncoder.default(self, obj) # Useful objects app = Flask(__name__) app.json_encoder = CustomJSONEncoder app.config['JSON_SORT_KEYS'] = False cache_dir = 'kle_cache' gist_url = 'https://api.github.com/gists/%s' cors = CORS(app, resources={'/v*/*': {'origins': '*'}}) rq = Queue(connection=redis) api_status = { 'last_ping': qmk_redis.get('qmk_api_last_ping'), 'queue_length': len(rq), 'status': 'starting', 'version': __VERSION__, } ## Helper functions def check_pings(): """Check the ping values and update api_status with them. """ api_status['queue_length'] = len(rq) for redis_key in ('qmk_api_last_ping', 'qmk_api_tasks_ping'): key = redis_key.replace('qmk_api_', '') value = qmk_redis.get(redis_key) api_status[key] = value
def update_kb_redis(): """Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data. """ # Check to see if we need to update if exists('qmk_firmware'): last_update = qmk_redis.get('qmk_api_last_updated') if not debug and isinstance( last_update, dict) and last_update['git_hash'] == git_hash(): # We are already up to date logging.warning( 'update_kb_redis(): Already up to date, skipping...') return False # Clean up the environment and fetch the latest source del (error_log[:]) if exists('qmk_firmware'): rmtree('qmk_firmware') if exists('qmk_firmware.zip'): remove('qmk_firmware.zip') qmk_storage.delete('cache/qmk_firmware.zip') checkout_qmk() # Update redis with the latest data kb_list = [] cached_json = { 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'), 'keyboards': {} } for keyboard in list_keyboards(): keyboard_info = { 'keyboard_name': keyboard, 'keyboard_folder': keyboard, 'keymaps': [], 'layouts': {}, 'maintainer': 'qmk', } for layout_name, layout_json in find_all_layouts(keyboard).items(): if not layout_name.startswith('LAYOUT_kc'): keyboard_info['layouts'][layout_name] = layout_json for info_json_filename in find_info_json(keyboard): # Iterate through all the possible info.json files to build the final keyboard JSON. try: with open(info_json_filename) as info_file: keyboard_info = merge_info_json(info_file, keyboard_info) except Exception as e: error_msg = 'Error encountered processing %s! %s: %s' % ( keyboard, e.__class__.__name__, e) error_log.append({ 'severity': 'error', 'message': 'Error: ' + error_msg }) logging.error(error_msg) logging.exception(e) # Iterate through all the possible keymaps to build keymap jsons. for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps( keyboard): keyboard_info['keymaps'].append(keymap_name) keymap_blob = { 'keyboard_name': keyboard, 'keymap_name': keymap_name, 'keymap_folder': keymap_folder, 'layers': keymap, 'layout_macro': layout_macro } # Write the keymap to redis qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name), keymap_blob) readme = '%s/%s/readme.md' % (keymap_folder, keymap_name) if exists(readme): with open(readme, 'rb') as readme_fd: readme_text = readme_fd.read() readme_text = UnicodeDammit(readme_text) readme_text = readme_text.unicode_markup else: readme_text = '%s does not exist.' % readme qmk_redis.set( 'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name), readme_text) # Pull some keyboard information from existing rules.mk and config.h files config_h = parse_config_h(keyboard) rules_mk = parse_rules_mk(keyboard) for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER', 'DESCRIPTION'): if key in config_h: if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'): config_h[key].replace('0x', '') config_h[key] = config_h[key].upper() keyboard_info[key.lower()] = config_h[key] if 'ARMV' in rules_mk: keyboard_info['processor_type'] = 'arm' # ARM processors if 'MCU' in rules_mk: keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT'] if 'MCU_LDSCRIPT' in rules_mk: keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: if 'STM32' in keyboard_info['processor']: keyboard_info['bootloader'] = 'stm32-dfu' elif keyboard_info.get('manufacturer') == 'Input Club': keyboard_info['bootloader'] = 'kiibohd-dfu' else: keyboard_info['processor_type'] = 'avr' # AVR processors if 'ARCH' in rules_mk: keyboard_info['platform'] = rules_mk['ARCH'] if 'MCU' in rules_mk: keyboard_info['processor'] = rules_mk['MCU'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: keyboard_info['bootloader'] = 'atmel-dfu' keyboard_info['identifier'] = ':'.join( (keyboard_info.get('vendor_id', 'unknown'), keyboard_info.get('product_id', 'unknown'), keyboard_info.get('device_ver', 'unknown'))) # Store the keyboard's readme in redis readme_filename = None readme_path = '' for dir in keyboard.split('/'): readme_path = '/'.join((readme_path, dir)) new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path)) if new_name: readme_filename = new_name # Last one wins if readme_filename: qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), open(readme_filename).read()) keyboard_info['readme'] = True else: error_msg = '%s does not have a readme.md.' % keyboard qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg) error_log.append({ 'severity': 'warning', 'message': 'Warning: ' + error_msg }) logging.warning(error_msg) keyboard_info['readme'] = False # Write the keyboard to redis and add it to the master list. qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info) kb_list.append(keyboard) cached_json['keyboards'][keyboard] = keyboard_info # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', cached_json) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) return True
def compile_json(keyboard_keymap_data, source_ip=None): """Compile a keymap. Arguments: keyboard_keymap_data A configurator export file that's been deserialized source_ip The IP that submitted the compile job """ result = { 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '', 'source_ip': source_ip, 'output': 'Unknown error', } try: for key in ('keyboard', 'layout', 'keymap'): result[key] = keyboard_keymap_data[key] result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace( '/', '-'), result['keymap'].replace('/', '-')) result['keymap_json'] = json.dumps(keyboard_keymap_data) result['command'] = ['bin/qmk', 'compile', result['keymap_archive']] kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard']) job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + result['keyboard']): print('Unknown keyboard: %s' % (result['keyboard'], )) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } # If this keyboard needs a submodule check it out if kb_data.get('protocol') in ['ChibiOS', 'LUFA']: checkout_lufa() if kb_data.get('protocol') == 'ChibiOS': checkout_chibios() if kb_data.get('protocol') == 'V-USB': checkout_vusb() # Write the keymap file with open(path.join('qmk_firmware', result['keymap_archive']), 'w') as fd: fd.write(result['keymap_json'] + '\n') # Compile the firmware store_firmware_source(result) remove(result['source_archive']) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() return result
def compile_firmware(keyboard, keymap, layout, layers): """Compile a firmware. """ result = { 'keyboard': keyboard, 'layout': layout, 'keymap': keymap, 'command': ['make', 'COLOR=false', ':'.join((keyboard, keymap))], 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '' } try: job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + keyboard): logging.error('Unknown keyboard: %s', keyboard) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } for pathname in ('qmk_firmware/keyboards/%s/keymaps/%s' % (keyboard, keymap), 'qmk_firmware/keyboards/%s/../keymaps/%s' % (keyboard, keymap)): if path.exists(pathname): logging.error( 'Name collision! %s already exists! This should not happen!', pathname) return { 'returncode': -1, 'command': '', 'output': 'Keymap name collision! %s already exists!' % (pathname), 'firmware': None } # If this keyboard needs chibios check it out kb_data = qmk_redis.get('qmk_api_kb_' + keyboard) if kb_data['processor_type'] == 'arm': checkout_chibios() # Build the keyboard firmware create_keymap(result, layers) store_firmware_source(result) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if not result['output']: result['output'] = result['stacktrace'] return result
def compile_firmware(keyboard, keymap, layout, layers, source_ip=None): """Compile a firmware. """ keyboard_safe_chars = keyboard.replace('/', '-') keymap_safe_chars = keymap.replace('/', '-') keymap_json_file = '%s-%s.json' % (keyboard_safe_chars, keymap_safe_chars) keymap_json = json.dumps({ 'keyboard': keyboard, 'keymap': keymap, 'layout': layout, 'layers': layers, 'author': '', 'notes': '', 'version': 1, 'source_ip': source_ip, 'documentation': 'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`' % (keymap_json_file, ) }) result = { 'keyboard': keyboard, 'layout': layout, 'keymap': keymap, 'keymap_archive': keymap_json_file, 'command': ['bin/qmk', 'compile', keymap_json_file], 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '', 'source_ip': source_ip, } try: kb_data = qmk_redis.get('qmk_api_kb_' + keyboard) job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + keyboard): logging.error('Unknown keyboard: %s', keyboard) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } # If this keyboard needs a submodule check it out if kb_data.get('protocol') in ['ChibiOS', 'LUFA']: checkout_lufa() if kb_data.get('protocol') == 'ChibiOS': checkout_chibios() # Write the keymap file with open(path.join('qmk_firmware', keymap_json_file), 'w') as fd: fd.write(keymap_json + '\n') # Compile the firmware store_firmware_source(result) remove(result['source_archive']) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if not result['output']: result['output'] = result['stacktrace'] return result