def compile_firmware(keyboard, keymap, layout, layers): """Compile a firmware. """ result = { 'keyboard': keyboard, 'layout': layout, 'keymap': keymap, 'command': ['make', ':'.join((keyboard, keymap))], 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '' } try: checkout_qmk() job = get_current_job() result['id'] = job.id # Sanity checks if not exists('qmk_firmware/keyboards/%s' % keyboard): logging.error('Unknown keyboard: %s', keyboard) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } if exists('qmk_firmware/keyboards/%s/keymaps/%s' % (keyboard, keymap)) or exists( 'qmk_firmware/keyboards/%s/../keymaps/%s' % (keyboard, keymap)): logging.error('Name collision! This should not happen!') return { 'returncode': -1, 'command': '', 'output': 'Keymap name collision!', 'firmware': None } # Build the keyboard firmware create_keymap(result, layers) store_firmware_source(result) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if not result['output']: result['output'] = result['stacktrace'] return result
def update_kb_redis(): """Called to update qmk_firmware. """ # Clean up the environment and fetch the latest source del error_log[:] if not debug: if exists('update_kb_redis'): rmtree('update_kb_redis') mkdir('update_kb_redis') chdir('update_kb_redis') qmk_redis.set('qmk_needs_update', False) if not debug or not exists('qmk_firmware'): checkout_qmk(skip_cache=True) # Update redis with the latest data kb_list = [] usb_list = { } # Structure: VENDOR_ID: {PRODUCT_ID: {KEYBOARD_FOLDER: {'vendor_id': VENDOR_ID, 'product_id': PRODUCT_ID, 'device_ver': DEVICE_VER, 'manufacturer': MANUFACTURER, 'product': PRODUCT, 'keyboard': KEYBOARD_FOLDER} kb_all = { 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'), 'keyboards': {} } for keyboard in list_keyboards(): try: process_keyboard(keyboard, usb_list, kb_list, kb_all) except Exception as e: # Uncaught exception handler. Ideally this is never hit. log_error( 'Uncaught exception while processing keyboard %s! %s: %s' % (keyboard, e.__class__.__name__, str(e))) logging.exception(e) # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', kb_all) qmk_redis.set('qmk_api_usb_list', usb_list) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) logging.info('*** All keys successfully written to redis! Total size:', len(json.dumps(kb_all))) chdir('..') return True
def compile_firmware(keyboard, keymap, layout, layers, source_ip=None): """Compile a firmware. """ keyboard_safe_chars = keyboard.replace('/', '-') keymap_safe_chars = keymap.replace('/', '-') keymap_json_file = '%s-%s.json' % (keyboard_safe_chars, keymap_safe_chars) keymap_json = json.dumps({ 'keyboard': keyboard, 'keymap': keymap, 'layout': layout, 'layers': layers, 'author': '', 'notes': '', 'version': 1, 'source_ip': source_ip, 'documentation': 'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`' % (keymap_json_file, ) }) result = { 'keyboard': keyboard, 'layout': layout, 'keymap': keymap, 'keymap_archive': keymap_json_file, 'command': ['bin/qmk', 'compile', keymap_json_file], 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '', 'source_ip': source_ip, } try: kb_data = qmk_redis.get('qmk_api_kb_' + keyboard) job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + keyboard): logging.error('Unknown keyboard: %s', keyboard) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } # If this keyboard needs a submodule check it out if kb_data.get('protocol') in ['ChibiOS', 'LUFA']: checkout_lufa() if kb_data.get('protocol') == 'ChibiOS': checkout_chibios() # Write the keymap file with open(path.join('qmk_firmware', keymap_json_file), 'w') as fd: fd.write(keymap_json + '\n') # Compile the firmware store_firmware_source(result) remove(result['source_archive']) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if not result['output']: result['output'] = result['stacktrace'] return result
def compile_firmware(keyboard, keymap, layout, layers): """Compile a firmware. """ result = { 'keyboard': keyboard, 'layout': layout, 'keymap': keymap, 'command': ['make', 'COLOR=false', ':'.join((keyboard, keymap))], 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '' } try: job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + keyboard): logging.error('Unknown keyboard: %s', keyboard) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } for pathname in ('qmk_firmware/keyboards/%s/keymaps/%s' % (keyboard, keymap), 'qmk_firmware/keyboards/%s/../keymaps/%s' % (keyboard, keymap)): if path.exists(pathname): logging.error( 'Name collision! %s already exists! This should not happen!', pathname) return { 'returncode': -1, 'command': '', 'output': 'Keymap name collision! %s already exists!' % (pathname), 'firmware': None } # If this keyboard needs chibios check it out kb_data = qmk_redis.get('qmk_api_kb_' + keyboard) if kb_data['processor_type'] == 'arm': checkout_chibios() # Build the keyboard firmware create_keymap(result, layers) store_firmware_source(result) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if not result['output']: result['output'] = result['stacktrace'] return result
def compile_json(keyboard_keymap_data, source_ip=None): """Compile a keymap. Arguments: keyboard_keymap_data A configurator export file that's been deserialized source_ip The IP that submitted the compile job """ result = { 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '', 'source_ip': source_ip, 'output': 'Unknown error', } try: for key in ('keyboard', 'layout', 'keymap'): result[key] = keyboard_keymap_data[key] result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace( '/', '-'), result['keymap'].replace('/', '-')) result['keymap_json'] = json.dumps(keyboard_keymap_data) result['command'] = ['bin/qmk', 'compile', result['keymap_archive']] kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard']) job = get_current_job() result['id'] = job.id checkout_qmk() # Sanity checks if not path.exists('qmk_firmware/keyboards/' + result['keyboard']): print('Unknown keyboard: %s' % (result['keyboard'], )) return { 'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None } # If this keyboard needs a submodule check it out if kb_data.get('protocol') in ['ChibiOS', 'LUFA']: checkout_lufa() if kb_data.get('protocol') == 'ChibiOS': checkout_chibios() if kb_data.get('protocol') == 'V-USB': checkout_vusb() # Write the keymap file with open(path.join('qmk_firmware', result['keymap_archive']), 'w') as fd: fd.write(result['keymap_json'] + '\n') # Compile the firmware store_firmware_source(result) remove(result['source_archive']) compile_keymap(job, result) store_firmware_binary(result) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() return result
def update_kb_redis(): """Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data. """ # Check to see if we need to update if exists('qmk_firmware'): last_update = qmk_redis.get('qmk_api_last_updated') if not debug and isinstance( last_update, dict) and last_update['git_hash'] == git_hash(): # We are already up to date logging.warning( 'update_kb_redis(): Already up to date, skipping...') return False # Clean up the environment and fetch the latest source del (error_log[:]) if exists('qmk_firmware'): rmtree('qmk_firmware') if exists('qmk_firmware.zip'): remove('qmk_firmware.zip') qmk_storage.delete('cache/qmk_firmware.zip') checkout_qmk() # Update redis with the latest data kb_list = [] cached_json = { 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'), 'keyboards': {} } for keyboard in list_keyboards(): keyboard_info = { 'keyboard_name': keyboard, 'keyboard_folder': keyboard, 'keymaps': [], 'layouts': {}, 'maintainer': 'qmk', } for layout_name, layout_json in find_all_layouts(keyboard).items(): if not layout_name.startswith('LAYOUT_kc'): keyboard_info['layouts'][layout_name] = layout_json for info_json_filename in find_info_json(keyboard): # Iterate through all the possible info.json files to build the final keyboard JSON. try: with open(info_json_filename) as info_file: keyboard_info = merge_info_json(info_file, keyboard_info) except Exception as e: error_msg = 'Error encountered processing %s! %s: %s' % ( keyboard, e.__class__.__name__, e) error_log.append({ 'severity': 'error', 'message': 'Error: ' + error_msg }) logging.error(error_msg) logging.exception(e) # Iterate through all the possible keymaps to build keymap jsons. for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps( keyboard): keyboard_info['keymaps'].append(keymap_name) keymap_blob = { 'keyboard_name': keyboard, 'keymap_name': keymap_name, 'keymap_folder': keymap_folder, 'layers': keymap, 'layout_macro': layout_macro } # Write the keymap to redis qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name), keymap_blob) readme = '%s/%s/readme.md' % (keymap_folder, keymap_name) if exists(readme): with open(readme, 'rb') as readme_fd: readme_text = readme_fd.read() readme_text = UnicodeDammit(readme_text) readme_text = readme_text.unicode_markup else: readme_text = '%s does not exist.' % readme qmk_redis.set( 'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name), readme_text) # Pull some keyboard information from existing rules.mk and config.h files config_h = parse_config_h(keyboard) rules_mk = parse_rules_mk(keyboard) for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER', 'DESCRIPTION'): if key in config_h: if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'): config_h[key].replace('0x', '') config_h[key] = config_h[key].upper() keyboard_info[key.lower()] = config_h[key] if 'ARMV' in rules_mk: keyboard_info['processor_type'] = 'arm' # ARM processors if 'MCU' in rules_mk: keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT'] if 'MCU_LDSCRIPT' in rules_mk: keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: if 'STM32' in keyboard_info['processor']: keyboard_info['bootloader'] = 'stm32-dfu' elif keyboard_info.get('manufacturer') == 'Input Club': keyboard_info['bootloader'] = 'kiibohd-dfu' else: keyboard_info['processor_type'] = 'avr' # AVR processors if 'ARCH' in rules_mk: keyboard_info['platform'] = rules_mk['ARCH'] if 'MCU' in rules_mk: keyboard_info['processor'] = rules_mk['MCU'] if 'BOOTLOADER' in rules_mk: keyboard_info['bootloader'] = rules_mk['BOOTLOADER'] if 'bootloader' not in keyboard_info: keyboard_info['bootloader'] = 'atmel-dfu' keyboard_info['identifier'] = ':'.join( (keyboard_info.get('vendor_id', 'unknown'), keyboard_info.get('product_id', 'unknown'), keyboard_info.get('device_ver', 'unknown'))) # Store the keyboard's readme in redis readme_filename = None readme_path = '' for dir in keyboard.split('/'): readme_path = '/'.join((readme_path, dir)) new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path)) if new_name: readme_filename = new_name # Last one wins if readme_filename: qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), open(readme_filename).read()) keyboard_info['readme'] = True else: error_msg = '%s does not have a readme.md.' % keyboard qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg) error_log.append({ 'severity': 'warning', 'message': 'Warning: ' + error_msg }) logging.warning(error_msg) keyboard_info['readme'] = False # Write the keyboard to redis and add it to the master list. qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info) kb_list.append(keyboard) cached_json['keyboards'][keyboard] = keyboard_info # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', cached_json) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) return True
def update_kb_redis(): """Called to update qmk_firmware. """ update_kb_redis = Path('update_kb_redis') qmk_firmware = Path('qmk_firmware') # Clean up the environment and fetch the latest source if not debug: # Create and enter a separate update_kb_redis directory to avoid conflicting with live builds if update_kb_redis.exists(): rmtree(update_kb_redis) update_kb_redis.mkdir() chdir(update_kb_redis) if not debug or not qmk_firmware.exists(): checkout_qmk(skip_cache=True) # Enter the qmk_firmware directory chdir(qmk_firmware) # Update redis with the latest keyboard data run(['qmk', 'generate-api']) api_dir = Path('api_data/v1') keyboards_dir = api_dir / 'keyboards' for keyboard_dir in keyboards_dir.glob('**'): keyboard_name = keyboard_dir.relative_to(keyboards_dir).as_posix() keyboard_info = keyboard_dir / 'info.json' keyboard_readme = keyboard_dir / 'readme.md' if keyboard_info.exists(): info_json = json.load(keyboard_info.open()) redis_json = info_json['keyboards'][keyboard_name] qmk_redis.set('qmk_api_kb_%s' % (keyboard_name), redis_json) if keyboard_readme.exists(): qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard_name), keyboard_readme.read_text()) # Update the USB list usb_json = json.load((api_dir / 'usb.json').open()) redis_usb = usb_json['usb'] qmk_redis.set('qmk_api_usb_list', redis_usb) # Update the Keyboard list keyboard_json = json.load((api_dir / 'keyboard_list.json').open()) redis_keyboard = keyboard_json['keyboards'] qmk_redis.set('qmk_api_keyboards', redis_keyboard) # Leave qmk_firmware chdir('..') # Set some metadata qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_needs_update', False) print('\n*** All keys successfully written to redis!') if not debug: # Leave update_kb_redis chdir('..') return True
def test_0000_checkout_qmk_skip_cache(): """Make sure that we successfully git clone qmk_firmware and generate the version.txt hash. """ qmk_commands.checkout_qmk(skip_cache=True) assert os.path.exists('qmk_firmware/version.txt')
def compile_json(keyboard_keymap_data, source_ip=None, send_metrics=True, public_firmware=False): """Compile a keymap. Arguments: keyboard_keymap_data A configurator export file that's been deserialized source_ip The IP that submitted the compile job """ start_time = time() base_metric = f'{gethostname()}.qmk_compiler.compile_json' result = { 'keyboard': 'unknown', 'returncode': -2, 'output': '', 'firmware': None, 'firmware_filename': '', 'source_ip': source_ip, 'output': 'Unknown error', 'public_firmware': public_firmware, } if DEBUG: print('Pointing graphite at', GRAPHITE_HOST) graphyte.init(GRAPHITE_HOST, GRAPHITE_PORT) try: for key in ('keyboard', 'layout', 'keymap'): result[key] = keyboard_keymap_data[key] # Gather information result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace('/', '-'), result['keymap'].replace('/', '-')) result['keymap_json'] = json.dumps(keyboard_keymap_data) result['command'] = ['bin/qmk', 'compile', result['keymap_archive']] job = get_current_job() result['id'] = job.id branch = keyboard_keymap_data.get('branch', QMK_GIT_BRANCH) # Fetch the appropriate version of QMK git_start_time = time() checkout_qmk(branch=branch) git_time = time() - git_start_time chdir('qmk_firmware') # Sanity check if not path.exists('keyboards/' + result['keyboard']): print('Unknown keyboard: %s' % (result['keyboard'],)) return {'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None} # Pull in the modules from the QMK we just checked out if './lib/python' not in sys.path: sys.path.append('./lib/python') from qmk.info import info_json # If this keyboard needs a submodule check it out submodule_start_time = time() kb_info = info_json(result['keyboard']) if 'protocol' not in kb_info: kb_info['protocol'] = 'unknown' if kb_info['protocol'] in ['ChibiOS', 'LUFA']: checkout_lufa() if kb_info['protocol'] == 'ChibiOS': checkout_chibios() if kb_info['protocol'] == 'V-USB': checkout_vusb() submodule_time = time() - submodule_start_time # Write the keymap file with open(result['keymap_archive'], 'w') as fd: fd.write(result['keymap_json'] + '\n') # Compile the firmware compile_start_time = time() compile_keymap(job, result) compile_time = time() - compile_start_time # Store the source in S3 storage_start_time = time() store_firmware_binary(result) chdir('..') store_firmware_source(result) remove(result['source_archive']) storage_time = time() - storage_start_time # Send metrics about this build if send_metrics: graphyte.send(f'{base_metric}.{result["keyboard"]}.all_layouts', 1) graphyte.send(f'{base_metric}.{result["keyboard"]}.{result["layout"]}', 1) graphyte.send(f'{base_metric}.{result["keyboard"]}.git_time', git_time) graphyte.send(f'{base_metric}.all_keyboards.git_time', git_time) graphyte.send(f'{base_metric}.{result["keyboard"]}.submodule_time', submodule_time) graphyte.send(f'{base_metric}.all_keyboards.submodule_time', submodule_time) graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time) graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time) if result['returncode'] == 0: graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time) graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time) else: graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1) if source_ip: ip_location = geolite2.lookup(source_ip) if ip_location: if ip_location.subdivisions: location_key = f'{ip_location.country}_{"_".join(ip_location.subdivisions)}' else: location_key = ip_location.country graphyte.send(f'{gethostname()}.qmk_compiler.geoip.{location_key}', 1) total_time = time() - start_time graphyte.send(f'{base_metric}.{result["keyboard"]}.storage_time', storage_time) graphyte.send(f'{base_metric}.all_keyboards.storage_time', storage_time) graphyte.send(f'{base_metric}.{result["keyboard"]}.total_time', total_time) graphyte.send(f'{base_metric}.all_keyboards.total_time', total_time) except Exception as e: result['returncode'] = -3 result['exception'] = e.__class__.__name__ result['stacktrace'] = format_exc() if send_metrics: graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1) store_firmware_metadata(job, result) return result
'message': 'Warning: ' + error_msg }) logging.warning(error_msg) keyboard_info['readme'] = False # Write the keyboard to redis and add it to the master list. qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info) if keyboard_info['processor_type'] != 'arm': kb_list.append(keyboard) cached_json['keyboards'][keyboard] = keyboard_info # Update the global redis information qmk_redis.set('qmk_api_keyboards', kb_list) qmk_redis.set('qmk_api_kb_all', cached_json) qmk_redis.set('qmk_api_last_updated', { 'git_hash': git_hash(), 'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z') }) qmk_redis.set('qmk_api_update_error_log', error_log) return True if __name__ == '__main__': debug = True if not exists('qmk_firmware'): checkout_qmk() update_kb_redis()
def test_0000_checkout_qmk_master(): """Make sure that we successfully git clone qmk_firmware and generate the version.txt hash. """ qmk_commands.checkout_qmk(branch='master') assert os.path.exists('qmk_firmware/version.txt')