def update_kb_redis():
"""Called to update qmk_firmware.
"""
# Clean up the environment and fetch the latest source
del error_log[:]
if not debug:
if exists('update_kb_redis'):
rmtree('update_kb_redis')
mkdir('update_kb_redis')
chdir('update_kb_redis')
qmk_redis.set('qmk_needs_update', False)
if not debug or not exists('qmk_firmware'):
checkout_qmk(skip_cache=True)
# Update redis with the latest data
kb_list = []
usb_list = {
} # Structure: VENDOR_ID: {PRODUCT_ID: {KEYBOARD_FOLDER: {'vendor_id': VENDOR_ID, 'product_id': PRODUCT_ID, 'device_ver': DEVICE_VER, 'manufacturer': MANUFACTURER, 'product': PRODUCT, 'keyboard': KEYBOARD_FOLDER}
kb_all = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
try:
process_keyboard(keyboard, usb_list, kb_list, kb_all)
except Exception as e:
# Uncaught exception handler. Ideally this is never hit.
log_error(
'Uncaught exception while processing keyboard %s! %s: %s' %
(keyboard, e.__class__.__name__, str(e)))
logging.exception(e)
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', kb_all)
qmk_redis.set('qmk_api_usb_list', usb_list)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
logging.info('*** All keys successfully written to redis! Total size:',
len(json.dumps(kb_all)))
chdir('..')
return True
def update_kb_redis():
"""Called to update qmk_firmware.
"""
update_kb_redis = Path('update_kb_redis')
qmk_firmware = Path('qmk_firmware')
# Clean up the environment and fetch the latest source
if not debug:
# Create and enter a separate update_kb_redis directory to avoid conflicting with live builds
if update_kb_redis.exists():
rmtree(update_kb_redis)
update_kb_redis.mkdir()
chdir(update_kb_redis)
if not debug or not qmk_firmware.exists():
checkout_qmk(skip_cache=True)
# Enter the qmk_firmware directory
chdir(qmk_firmware)
# Update redis with the latest keyboard data
run(['qmk', 'generate-api'])
api_dir = Path('api_data/v1')
keyboards_dir = api_dir / 'keyboards'
for keyboard_dir in keyboards_dir.glob('**'):
keyboard_name = keyboard_dir.relative_to(keyboards_dir).as_posix()
keyboard_info = keyboard_dir / 'info.json'
keyboard_readme = keyboard_dir / 'readme.md'
if keyboard_info.exists():
info_json = json.load(keyboard_info.open())
redis_json = info_json['keyboards'][keyboard_name]
qmk_redis.set('qmk_api_kb_%s' % (keyboard_name), redis_json)
if keyboard_readme.exists():
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard_name),
keyboard_readme.read_text())
# Update the USB list
usb_json = json.load((api_dir / 'usb.json').open())
redis_usb = usb_json['usb']
qmk_redis.set('qmk_api_usb_list', redis_usb)
# Update the Keyboard list
keyboard_json = json.load((api_dir / 'keyboard_list.json').open())
redis_keyboard = keyboard_json['keyboards']
qmk_redis.set('qmk_api_keyboards', redis_keyboard)
# Leave qmk_firmware
chdir('..')
# Set some metadata
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_needs_update', False)
print('\n*** All keys successfully written to redis!')
if not debug:
# Leave update_kb_redis
chdir('..')
return True
def update_kb_redis():
"""Called when updates happen to QMK Firmware. Responsible for updating the cached source code and API data.
"""
# Check to see if we need to update
if exists('qmk_firmware'):
last_update = qmk_redis.get('qmk_api_last_updated')
if not debug and isinstance(
last_update, dict) and last_update['git_hash'] == git_hash():
# We are already up to date
logging.warning(
'update_kb_redis(): Already up to date, skipping...')
return False
# Clean up the environment and fetch the latest source
del (error_log[:])
if exists('qmk_firmware'):
rmtree('qmk_firmware')
if exists('qmk_firmware.zip'):
remove('qmk_firmware.zip')
qmk_storage.delete('cache/qmk_firmware.zip')
checkout_qmk()
# Update redis with the latest data
kb_list = []
cached_json = {
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z'),
'keyboards': {}
}
for keyboard in list_keyboards():
keyboard_info = {
'keyboard_name': keyboard,
'keyboard_folder': keyboard,
'keymaps': [],
'layouts': {},
'maintainer': 'qmk',
}
for layout_name, layout_json in find_all_layouts(keyboard).items():
if not layout_name.startswith('LAYOUT_kc'):
keyboard_info['layouts'][layout_name] = layout_json
for info_json_filename in find_info_json(keyboard):
# Iterate through all the possible info.json files to build the final keyboard JSON.
try:
with open(info_json_filename) as info_file:
keyboard_info = merge_info_json(info_file, keyboard_info)
except Exception as e:
error_msg = 'Error encountered processing %s! %s: %s' % (
keyboard, e.__class__.__name__, e)
error_log.append({
'severity': 'error',
'message': 'Error: ' + error_msg
})
logging.error(error_msg)
logging.exception(e)
# Iterate through all the possible keymaps to build keymap jsons.
for keymap_name, keymap_folder, layout_macro, keymap in find_keymaps(
keyboard):
keyboard_info['keymaps'].append(keymap_name)
keymap_blob = {
'keyboard_name': keyboard,
'keymap_name': keymap_name,
'keymap_folder': keymap_folder,
'layers': keymap,
'layout_macro': layout_macro
}
# Write the keymap to redis
qmk_redis.set('qmk_api_kb_%s_keymap_%s' % (keyboard, keymap_name),
keymap_blob)
readme = '%s/%s/readme.md' % (keymap_folder, keymap_name)
if exists(readme):
with open(readme, 'rb') as readme_fd:
readme_text = readme_fd.read()
readme_text = UnicodeDammit(readme_text)
readme_text = readme_text.unicode_markup
else:
readme_text = '%s does not exist.' % readme
qmk_redis.set(
'qmk_api_kb_%s_keymap_%s_readme' % (keyboard, keymap_name),
readme_text)
# Pull some keyboard information from existing rules.mk and config.h files
config_h = parse_config_h(keyboard)
rules_mk = parse_rules_mk(keyboard)
for key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER', 'MANUFACTURER',
'DESCRIPTION'):
if key in config_h:
if key in ('VENDOR_ID', 'PRODUCT_ID', 'DEVICE_VER'):
config_h[key].replace('0x', '')
config_h[key] = config_h[key].upper()
keyboard_info[key.lower()] = config_h[key]
if 'ARMV' in rules_mk:
keyboard_info['processor_type'] = 'arm'
# ARM processors
if 'MCU' in rules_mk:
keyboard_info['platform'] = rules_mk['MCU_LDSCRIPT']
if 'MCU_LDSCRIPT' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU_LDSCRIPT']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
if 'STM32' in keyboard_info['processor']:
keyboard_info['bootloader'] = 'stm32-dfu'
elif keyboard_info.get('manufacturer') == 'Input Club':
keyboard_info['bootloader'] = 'kiibohd-dfu'
else:
keyboard_info['processor_type'] = 'avr'
# AVR processors
if 'ARCH' in rules_mk:
keyboard_info['platform'] = rules_mk['ARCH']
if 'MCU' in rules_mk:
keyboard_info['processor'] = rules_mk['MCU']
if 'BOOTLOADER' in rules_mk:
keyboard_info['bootloader'] = rules_mk['BOOTLOADER']
if 'bootloader' not in keyboard_info:
keyboard_info['bootloader'] = 'atmel-dfu'
keyboard_info['identifier'] = ':'.join(
(keyboard_info.get('vendor_id', 'unknown'),
keyboard_info.get('product_id', 'unknown'),
keyboard_info.get('device_ver', 'unknown')))
# Store the keyboard's readme in redis
readme_filename = None
readme_path = ''
for dir in keyboard.split('/'):
readme_path = '/'.join((readme_path, dir))
new_name = find_readme('qmk_firmware/keyboards%s' % (readme_path))
if new_name:
readme_filename = new_name # Last one wins
if readme_filename:
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard),
open(readme_filename).read())
keyboard_info['readme'] = True
else:
error_msg = '%s does not have a readme.md.' % keyboard
qmk_redis.set('qmk_api_kb_%s_readme' % (keyboard), error_msg)
error_log.append({
'severity': 'warning',
'message': 'Warning: ' + error_msg
})
logging.warning(error_msg)
keyboard_info['readme'] = False
# Write the keyboard to redis and add it to the master list.
qmk_redis.set('qmk_api_kb_%s' % (keyboard), keyboard_info)
kb_list.append(keyboard)
cached_json['keyboards'][keyboard] = keyboard_info
# Update the global redis information
qmk_redis.set('qmk_api_keyboards', kb_list)
qmk_redis.set('qmk_api_kb_all', cached_json)
qmk_redis.set('qmk_api_last_updated', {
'git_hash': git_hash(),
'last_updated': strftime('%Y-%m-%d %H:%M:%S %Z')
})
qmk_redis.set('qmk_api_update_error_log', error_log)
return True
def test_0013_git_hash():
"""Make sure that we get a valid hex string in the git hash.
"""
hash = qmk_commands.git_hash()
assert re.match(r'^[0-9a-f]+$', hash)
assert len(hash) == 40