def compile_firmware(keyboard, keymap, layout, layers):
"""Compile a firmware.
"""
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'command': ['make', ':'.join((keyboard, keymap))],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': ''
}
try:
checkout_qmk()
job = get_current_job()
result['id'] = job.id
# Sanity checks
if not exists('qmk_firmware/keyboards/%s' % keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
if exists('qmk_firmware/keyboards/%s/keymaps/%s' %
(keyboard, keymap)) or exists(
'qmk_firmware/keyboards/%s/../keymaps/%s' %
(keyboard, keymap)):
logging.error('Name collision! This should not happen!')
return {
'returncode': -1,
'command': '',
'output': 'Keymap name collision!',
'firmware': None
}
# Build the keyboard firmware
create_keymap(result, layers)
store_firmware_source(result)
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def update_kb_redis():
"""Called to update qmk_firmware.
"""
# Clean up the environment and fetch the latest source
del error_log[:]
if not debug:
if exists('update_kb_redis'):
rmtree('update_kb_redis')
mkdir('update_kb_redis')
chdir('update_kb_redis')
qmk_redis.set('qmk_needs_update', False)
if not debug or not exists('qmk_firmware'):
checkout_qmk(skip_cache=True)
# Update redis with the latest data
kb_list = []
usb_list = {
} # Structure: VENDOR_ID: {PRODUCT_ID: {KEYBOARD_FOLDER: {'vendor_id': VENDOR_ID, 'product_id': PRODUCT_ID, 'device_ver': DEVICE_VER, 'manufacturer': MANUFACTURER, 'product': PRODUCT, 'keyboard': KEYBOARD_FOLDER}
kb_all = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
try:
process_keyboard(keyboard, usb_list, kb_list, kb_all)
except Exception as e:
# Uncaught exception handler. Ideally this is never hit.
log_error(
'Uncaught exception while processing keyboard %s! %s: %s' %
(keyboard, e.__class__.__name__, str(e)))
logging.exception(e)
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', kb_all)
qmk_redis.set('qmk_api_usb_list', usb_list)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
logging.info('*** All keys successfully written to redis! Total size:',
len(json.dumps(kb_all)))
chdir('..')
return True
def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
"""Compile a firmware.
"""
keyboard_safe_chars = keyboard.replace('/', '-')
keymap_safe_chars = keymap.replace('/', '-')
keymap_json_file = '%s-%s.json' % (keyboard_safe_chars, keymap_safe_chars)
keymap_json = json.dumps({
'keyboard':
keyboard,
'keymap':
keymap,
'layout':
layout,
'layers':
layers,
'author':
'',
'notes':
'',
'version':
1,
'source_ip':
source_ip,
'documentation':
'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`'
% (keymap_json_file, )
})
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'keymap_archive': keymap_json_file,
'command': ['bin/qmk', 'compile', keymap_json_file],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
}
try:
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
# Write the keymap file
with open(path.join('qmk_firmware', keymap_json_file), 'w') as fd:
fd.write(keymap_json + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def compile_firmware(keyboard, keymap, layout, layers):
"""Compile a firmware.
"""
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'command': ['make', 'COLOR=false', ':'.join((keyboard, keymap))],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': ''
}
try:
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
for pathname in ('qmk_firmware/keyboards/%s/keymaps/%s' %
(keyboard, keymap),
'qmk_firmware/keyboards/%s/../keymaps/%s' %
(keyboard, keymap)):
if path.exists(pathname):
logging.error(
'Name collision! %s already exists! This should not happen!',
pathname)
return {
'returncode':
-1,
'command':
'',
'output':
'Keymap name collision! %s already exists!' % (pathname),
'firmware':
None
}
# If this keyboard needs chibios check it out
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
if kb_data['processor_type'] == 'arm':
checkout_chibios()
# Build the keyboard firmware
create_keymap(result, layers)
store_firmware_source(result)
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def compile_json(keyboard_keymap_data, source_ip=None):
"""Compile a keymap.
Arguments:
keyboard_keymap_data
A configurator export file that's been deserialized
source_ip
The IP that submitted the compile job
"""
result = {
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
'output': 'Unknown error',
}
try:
for key in ('keyboard', 'layout', 'keymap'):
result[key] = keyboard_keymap_data[key]
result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace(
'/', '-'), result['keymap'].replace('/', '-'))
result['keymap_json'] = json.dumps(keyboard_keymap_data)
result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard'])
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + result['keyboard']):
print('Unknown keyboard: %s' % (result['keyboard'], ))
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
if kb_data.get('protocol') == 'V-USB':
checkout_vusb()
# Write the keymap file
with open(path.join('qmk_firmware', result['keymap_archive']),
'w') as fd:
fd.write(result['keymap_json'] + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
return result
def update_kb_redis():
"""Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data.
"""
# Check to see if we need to update
if exists('qmk_firmware'):
last_update = qmk_redis.get('qmk_api_last_updated')
if not debug and isinstance(
last_update, dict) and last_update['git_hash'] == git_hash():
# We are already up to date
logging.warning(
'update_kb_redis(): Already up to date, skipping...')
return False
# Clean up the environment and fetch the latest source
del (error_log[:])
if exists('qmk_firmware'):
rmtree('qmk_firmware')
if exists('qmk_firmware.zip'):
remove('qmk_firmware.zip')
qmk_storage.delete('cache/qmk_firmware.zip')
checkout_qmk()
# Update redis with the latest data
kb_list = []
cached_json = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
keyboard_info = {
'keyboard_name': keyboard,
'keyboard_folder': keyboard,
'keymaps': [],
'layouts': {},
'maintainer': 'qmk',
}
for layout_name, layout_json in find_all_layouts(keyboard).items():
if not layout_name.startswith('LAYOUT_kc'):
keyboard_info['layouts'][layout_name] = layout_json
for info_json_filename in find_info_json(keyboard):
# Iterate through all the possible info.json files to build the final keyboard JSON.
try:
with open(info_json_filename) as info_file:
keyboard_info = merge_info_json(info_file, keyboard_info)
except Exception as e:
error_msg = 'Error encountered processing %s! %s: %s' % (
keyboard, e.__class__.__name__, e)
error_log.append({
'severity': 'error',
'message': 'Error: ' + error_msg
})
logging.error(error_msg)
logging.exception(e)
# Iterate through all the possible keymaps to build keymap jsons.
for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps(
keyboard):
keyboard_info['keymaps'].append(keymap_name)
keymap_blob = {
'keyboard_name': keyboard,
'keymap_name': keymap_name,
'keymap_folder': keymap_folder,
'layers': keymap,
'layout_macro': layout_macro
}
# Write the keymap to redis
qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name),
keymap_blob)
readme = '%s/%s/readme.md' % (keymap_folder, keymap_name)
if exists(readme):
with open(readme, 'rb') as readme_fd:
readme_text = readme_fd.read()
readme_text = UnicodeDammit(readme_text)
readme_text = readme_text.unicode_markup
else:
readme_text = '%s does not exist.' % readme
qmk_redis.set(
'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name),
readme_text)
# Pull some keyboard information from existing rules.mk and config.h files
config_h = parse_config_h(keyboard)
rules_mk = parse_rules_mk(keyboard)
for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER',
'DESCRIPTION'):
if key in config_h:
if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'):
config_h[key].replace('0x', '')
config_h[key] = config_h[key].upper()
keyboard_info[key.lower()] = config_h[key]
if 'ARMV' in rules_mk:
keyboard_info['processor_type'] = 'arm'
# ARM processors
if 'MCU' in rules_mk:
keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT']
if 'MCU_LDSCRIPT' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
if 'STM32' in keyboard_info['processor']:
keyboard_info['bootloader'] = 'stm32-dfu'
elif keyboard_info.get('manufacturer') == 'Input Club':
keyboard_info['bootloader'] = 'kiibohd-dfu'
else:
keyboard_info['processor_type'] = 'avr'
# AVR processors
if 'ARCH' in rules_mk:
keyboard_info['platform'] = rules_mk['ARCH']
if 'MCU' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
keyboard_info['bootloader'] = 'atmel-dfu'
keyboard_info['identifier'] = ':'.join(
(keyboard_info.get('vendor_id', 'unknown'),
keyboard_info.get('product_id', 'unknown'),
keyboard_info.get('device_ver', 'unknown')))
# Store the keyboard's readme in redis
readme_filename = None
readme_path = ''
for dir in keyboard.split('/'):
readme_path = '/'.join((readme_path, dir))
new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path))
if new_name:
readme_filename = new_name # Last one wins
if readme_filename:
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard),
open(readme_filename).read())
keyboard_info['readme'] = True
else:
error_msg = '%s does not have a readme.md.' % keyboard
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg)
error_log.append({
'severity': 'warning',
'message': 'Warning: ' + error_msg
})
logging.warning(error_msg)
keyboard_info['readme'] = False
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
kb_list.append(keyboard)
cached_json['keyboards'][keyboard] = keyboard_info
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', cached_json)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
return True
def update_kb_redis():
"""Called to update qmk_firmware.
"""
update_kb_redis = Path('update_kb_redis')
qmk_firmware = Path('qmk_firmware')
# Clean up the environment and fetch the latest source
if not debug:
# Create and enter a separate update_kb_redis directory to avoid conflicting with live builds
if update_kb_redis.exists():
rmtree(update_kb_redis)
update_kb_redis.mkdir()
chdir(update_kb_redis)
if not debug or not qmk_firmware.exists():
checkout_qmk(skip_cache=True)
# Enter the qmk_firmware directory
chdir(qmk_firmware)
# Update redis with the latest keyboard data
run(['qmk', 'generate-api'])
api_dir = Path('api_data/v1')
keyboards_dir = api_dir / 'keyboards'
for keyboard_dir in keyboards_dir.glob('**'):
keyboard_name = keyboard_dir.relative_to(keyboards_dir).as_posix()
keyboard_info = keyboard_dir / 'info.json'
keyboard_readme = keyboard_dir / 'readme.md'
if keyboard_info.exists():
info_json = json.load(keyboard_info.open())
redis_json = info_json['keyboards'][keyboard_name]
qmk_redis.set('qmk_api_kb_%s' % (keyboard_name), redis_json)
if keyboard_readme.exists():
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard_name),
keyboard_readme.read_text())
# Update the USB list
usb_json = json.load((api_dir / 'usb.json').open())
redis_usb = usb_json['usb']
qmk_redis.set('qmk_api_usb_list', redis_usb)
# Update the Keyboard list
keyboard_json = json.load((api_dir / 'keyboard_list.json').open())
redis_keyboard = keyboard_json['keyboards']
qmk_redis.set('qmk_api_keyboards', redis_keyboard)
# Leave qmk_firmware
chdir('..')
# Set some metadata
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_needs_update', False)
print('\n*** All keys successfully written to redis!')
if not debug:
# Leave update_kb_redis
chdir('..')
return True
def test_0000_checkout_qmk_skip_cache():
"""Make sure that we successfully git clone qmk_firmware and generate the version.txt hash.
"""
qmk_commands.checkout_qmk(skip_cache=True)
assert os.path.exists('qmk_firmware/version.txt')
def compile_json(keyboard_keymap_data, source_ip=None, send_metrics=True, public_firmware=False):
"""Compile a keymap.
Arguments:
keyboard_keymap_data
A configurator export file that's been deserialized
source_ip
The IP that submitted the compile job
"""
start_time = time()
base_metric = f'{gethostname()}.qmk_compiler.compile_json'
result = {
'keyboard': 'unknown',
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
'output': 'Unknown error',
'public_firmware': public_firmware,
}
if DEBUG:
print('Pointing graphite at', GRAPHITE_HOST)
graphyte.init(GRAPHITE_HOST, GRAPHITE_PORT)
try:
for key in ('keyboard', 'layout', 'keymap'):
result[key] = keyboard_keymap_data[key]
# Gather information
result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace('/', '-'), result['keymap'].replace('/', '-'))
result['keymap_json'] = json.dumps(keyboard_keymap_data)
result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
job = get_current_job()
result['id'] = job.id
branch = keyboard_keymap_data.get('branch', QMK_GIT_BRANCH)
# Fetch the appropriate version of QMK
git_start_time = time()
checkout_qmk(branch=branch)
git_time = time() - git_start_time
chdir('qmk_firmware')
# Sanity check
if not path.exists('keyboards/' + result['keyboard']):
print('Unknown keyboard: %s' % (result['keyboard'],))
return {'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None}
# Pull in the modules from the QMK we just checked out
if './lib/python' not in sys.path:
sys.path.append('./lib/python')
from qmk.info import info_json
# If this keyboard needs a submodule check it out
submodule_start_time = time()
kb_info = info_json(result['keyboard'])
if 'protocol' not in kb_info:
kb_info['protocol'] = 'unknown'
if kb_info['protocol'] in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_info['protocol'] == 'ChibiOS':
checkout_chibios()
if kb_info['protocol'] == 'V-USB':
checkout_vusb()
submodule_time = time() - submodule_start_time
# Write the keymap file
with open(result['keymap_archive'], 'w') as fd:
fd.write(result['keymap_json'] + '\n')
# Compile the firmware
compile_start_time = time()
compile_keymap(job, result)
compile_time = time() - compile_start_time
# Store the source in S3
storage_start_time = time()
store_firmware_binary(result)
chdir('..')
store_firmware_source(result)
remove(result['source_archive'])
storage_time = time() - storage_start_time
# Send metrics about this build
if send_metrics:
graphyte.send(f'{base_metric}.{result["keyboard"]}.all_layouts', 1)
graphyte.send(f'{base_metric}.{result["keyboard"]}.{result["layout"]}', 1)
graphyte.send(f'{base_metric}.{result["keyboard"]}.git_time', git_time)
graphyte.send(f'{base_metric}.all_keyboards.git_time', git_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.submodule_time', submodule_time)
graphyte.send(f'{base_metric}.all_keyboards.submodule_time', submodule_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time)
graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time)
if result['returncode'] == 0:
graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time)
graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time)
else:
graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1)
if source_ip:
ip_location = geolite2.lookup(source_ip)
if ip_location:
if ip_location.subdivisions:
location_key = f'{ip_location.country}_{"_".join(ip_location.subdivisions)}'
else:
location_key = ip_location.country
graphyte.send(f'{gethostname()}.qmk_compiler.geoip.{location_key}', 1)
total_time = time() - start_time
graphyte.send(f'{base_metric}.{result["keyboard"]}.storage_time', storage_time)
graphyte.send(f'{base_metric}.all_keyboards.storage_time', storage_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.total_time', total_time)
graphyte.send(f'{base_metric}.all_keyboards.total_time', total_time)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if send_metrics:
graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1)
store_firmware_metadata(job, result)
return result
'message': 'Warning: ' + error_msg
})
logging.warning(error_msg)
keyboard_info['readme'] = False
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
if keyboard_info['processor_type'] != 'arm':
kb_list.append(keyboard)
cached_json['keyboards'][keyboard] = keyboard_info
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', cached_json)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
return True
if __name__ == '__main__':
debug = True
if not exists('qmk_firmware'):
checkout_qmk()
update_kb_redis()
def test_0000_checkout_qmk_master():
"""Make sure that we successfully git clone qmk_firmware and generate the version.txt hash.
"""
qmk_commands.checkout_qmk(branch='master')
assert os.path.exists('qmk_firmware/version.txt')