def compile_firmware(keyboard, keymap, layout, layers, source_ip=None):
"""Compile a firmware.
"""
keyboard_safe_chars = keyboard.replace('/', '-')
keymap_safe_chars = keymap.replace('/', '-')
keymap_json_file = '%s-%s.json' % (keyboard_safe_chars, keymap_safe_chars)
keymap_json = json.dumps({
'keyboard':
keyboard,
'keymap':
keymap,
'layout':
layout,
'layers':
layers,
'author':
'',
'notes':
'',
'version':
1,
'source_ip':
source_ip,
'documentation':
'This file is a configurator export. You can compile it directly inside QMK using the command `bin/qmk compile %s`'
% (keymap_json_file, )
})
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'keymap_archive': keymap_json_file,
'command': ['bin/qmk', 'compile', keymap_json_file],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
}
try:
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
# Write the keymap file
with open(path.join('qmk_firmware', keymap_json_file), 'w') as fd:
fd.write(keymap_json + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def compile_json(keyboard_keymap_data, source_ip=None):
"""Compile a keymap.
Arguments:
keyboard_keymap_data
A configurator export file that's been deserialized
source_ip
The IP that submitted the compile job
"""
result = {
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
'output': 'Unknown error',
}
try:
for key in ('keyboard', 'layout', 'keymap'):
result[key] = keyboard_keymap_data[key]
result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace(
'/', '-'), result['keymap'].replace('/', '-'))
result['keymap_json'] = json.dumps(keyboard_keymap_data)
result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
kb_data = qmk_redis.get('qmk_api_kb_' + result['keyboard'])
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + result['keyboard']):
print('Unknown keyboard: %s' % (result['keyboard'], ))
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
# If this keyboard needs a submodule check it out
if kb_data.get('protocol') in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_data.get('protocol') == 'ChibiOS':
checkout_chibios()
if kb_data.get('protocol') == 'V-USB':
checkout_vusb()
# Write the keymap file
with open(path.join('qmk_firmware', result['keymap_archive']),
'w') as fd:
fd.write(result['keymap_json'] + '\n')
# Compile the firmware
store_firmware_source(result)
remove(result['source_archive'])
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
return result
def compile_firmware(keyboard, keymap, layout, layers):
"""Compile a firmware.
"""
result = {
'keyboard': keyboard,
'layout': layout,
'keymap': keymap,
'command': ['make', 'COLOR=false', ':'.join((keyboard, keymap))],
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': ''
}
try:
job = get_current_job()
result['id'] = job.id
checkout_qmk()
# Sanity checks
if not path.exists('qmk_firmware/keyboards/' + keyboard):
logging.error('Unknown keyboard: %s', keyboard)
return {
'returncode': -1,
'command': '',
'output': 'Unknown keyboard!',
'firmware': None
}
for pathname in ('qmk_firmware/keyboards/%s/keymaps/%s' %
(keyboard, keymap),
'qmk_firmware/keyboards/%s/../keymaps/%s' %
(keyboard, keymap)):
if path.exists(pathname):
logging.error(
'Name collision! %s already exists! This should not happen!',
pathname)
return {
'returncode':
-1,
'command':
'',
'output':
'Keymap name collision! %s already exists!' % (pathname),
'firmware':
None
}
# If this keyboard needs chibios check it out
kb_data = qmk_redis.get('qmk_api_kb_' + keyboard)
if kb_data['processor_type'] == 'arm':
checkout_chibios()
# Build the keyboard firmware
create_keymap(result, layers)
store_firmware_source(result)
compile_keymap(job, result)
store_firmware_binary(result)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if not result['output']:
result['output'] = result['stacktrace']
return result
def compile_json(keyboard_keymap_data, source_ip=None, send_metrics=True, public_firmware=False):
"""Compile a keymap.
Arguments:
keyboard_keymap_data
A configurator export file that's been deserialized
source_ip
The IP that submitted the compile job
"""
start_time = time()
base_metric = f'{gethostname()}.qmk_compiler.compile_json'
result = {
'keyboard': 'unknown',
'returncode': -2,
'output': '',
'firmware': None,
'firmware_filename': '',
'source_ip': source_ip,
'output': 'Unknown error',
'public_firmware': public_firmware,
}
if DEBUG:
print('Pointing graphite at', GRAPHITE_HOST)
graphyte.init(GRAPHITE_HOST, GRAPHITE_PORT)
try:
for key in ('keyboard', 'layout', 'keymap'):
result[key] = keyboard_keymap_data[key]
# Gather information
result['keymap_archive'] = '%s-%s.json' % (result['keyboard'].replace('/', '-'), result['keymap'].replace('/', '-'))
result['keymap_json'] = json.dumps(keyboard_keymap_data)
result['command'] = ['bin/qmk', 'compile', result['keymap_archive']]
job = get_current_job()
result['id'] = job.id
branch = keyboard_keymap_data.get('branch', QMK_GIT_BRANCH)
# Fetch the appropriate version of QMK
git_start_time = time()
checkout_qmk(branch=branch)
git_time = time() - git_start_time
chdir('qmk_firmware')
# Sanity check
if not path.exists('keyboards/' + result['keyboard']):
print('Unknown keyboard: %s' % (result['keyboard'],))
return {'returncode': -1, 'command': '', 'output': 'Unknown keyboard!', 'firmware': None}
# Pull in the modules from the QMK we just checked out
if './lib/python' not in sys.path:
sys.path.append('./lib/python')
from qmk.info import info_json
# If this keyboard needs a submodule check it out
submodule_start_time = time()
kb_info = info_json(result['keyboard'])
if 'protocol' not in kb_info:
kb_info['protocol'] = 'unknown'
if kb_info['protocol'] in ['ChibiOS', 'LUFA']:
checkout_lufa()
if kb_info['protocol'] == 'ChibiOS':
checkout_chibios()
if kb_info['protocol'] == 'V-USB':
checkout_vusb()
submodule_time = time() - submodule_start_time
# Write the keymap file
with open(result['keymap_archive'], 'w') as fd:
fd.write(result['keymap_json'] + '\n')
# Compile the firmware
compile_start_time = time()
compile_keymap(job, result)
compile_time = time() - compile_start_time
# Store the source in S3
storage_start_time = time()
store_firmware_binary(result)
chdir('..')
store_firmware_source(result)
remove(result['source_archive'])
storage_time = time() - storage_start_time
# Send metrics about this build
if send_metrics:
graphyte.send(f'{base_metric}.{result["keyboard"]}.all_layouts', 1)
graphyte.send(f'{base_metric}.{result["keyboard"]}.{result["layout"]}', 1)
graphyte.send(f'{base_metric}.{result["keyboard"]}.git_time', git_time)
graphyte.send(f'{base_metric}.all_keyboards.git_time', git_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.submodule_time', submodule_time)
graphyte.send(f'{base_metric}.all_keyboards.submodule_time', submodule_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time)
graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time)
if result['returncode'] == 0:
graphyte.send(f'{base_metric}.{result["keyboard"]}.compile_time', compile_time)
graphyte.send(f'{base_metric}.all_keyboards.compile_time', compile_time)
else:
graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1)
if source_ip:
ip_location = geolite2.lookup(source_ip)
if ip_location:
if ip_location.subdivisions:
location_key = f'{ip_location.country}_{"_".join(ip_location.subdivisions)}'
else:
location_key = ip_location.country
graphyte.send(f'{gethostname()}.qmk_compiler.geoip.{location_key}', 1)
total_time = time() - start_time
graphyte.send(f'{base_metric}.{result["keyboard"]}.storage_time', storage_time)
graphyte.send(f'{base_metric}.all_keyboards.storage_time', storage_time)
graphyte.send(f'{base_metric}.{result["keyboard"]}.total_time', total_time)
graphyte.send(f'{base_metric}.all_keyboards.total_time', total_time)
except Exception as e:
result['returncode'] = -3
result['exception'] = e.__class__.__name__
result['stacktrace'] = format_exc()
if send_metrics:
graphyte.send(f'{base_metric}.{result["keyboard"]}.errors', 1)
store_firmware_metadata(job, result)
return result