def lock(connection,
connection_lock,
guard_time_seconds,
printer,
prompt,
keep_going_flag=None,
hw_reset=True):
'''Lock the given connection'''
timeout_seconds = guard_time_seconds
success = False
if connection:
instance_text = u_utils.get_instance_text(get_instance(connection))
if connection_lock:
# Wait on the lock
printer.string("{}instance {} waiting up to {} second(s)" \
" to lock connection...". \
format(prompt, instance_text, guard_time_seconds))
count = 0
while not connection_lock.acquire(False) and \
((guard_time_seconds == 0) or (timeout_seconds > 0)) and \
u_utils.keep_going(keep_going_flag, printer, prompt):
sleep(1)
timeout_seconds -= 1
count += 1
if count == 30:
printer.string("{}instance {} still waiting {} second(s)" \
" for a connection lock (locker is" \
" currently {}).". \
format(prompt, instance_text, timeout_seconds,
connection_lock))
count = 0
if (guard_time_seconds == 0) or (timeout_seconds > 0):
printer.string("{}instance {} has locked a connection ({}).". \
format(prompt, instance_text, connection_lock))
if hw_reset:
kmtronic = get_kmtronic(connection)
usb_cutter_id_str = get_usb_cutter_id_str(connection)
if kmtronic:
printer.string("{}using KMTronic to reset {}...". \
format(prompt, instance_text))
u_utils.kmtronic_reset(kmtronic["ip_address"],
kmtronic["hex_bitmap"], printer,
prompt)
if usb_cutter_id_str:
printer.string("{}using USB cutter to reset {}...". \
format(prompt, instance_text))
u_utils.usb_cutter_reset([usb_cutter_id_str], printer,
prompt)
success = True
else:
success = True
printer.string("{}note: instance {} lock is empty.". \
format(prompt, instance_text))
return success
def watch_items(in_handle, connection_type, results, guard_time_seconds,
inactivity_time_seconds, terminator, printer, reporter, prompt,
keep_going_flag):
'''Watch output'''
return_value = -1
start_time = time()
last_activity_time = time()
printer.string("{}watching output until run completes...".format(prompt))
# Start a thread to read lines from in_handle
# This is done in a separate thread as it can block or
# hang; this way we get to detect that and time out.
read_queue = queue.Queue()
readline_thread = threading.Thread(target=readline_and_queue,
args=(results, read_queue, in_handle,
connection_type, terminator))
readline_thread.start()
try:
while u_utils.keep_going(keep_going_flag, printer, prompt) and \
not results["finished"] and \
(not guard_time_seconds or \
(time() - start_time < guard_time_seconds)) and \
(not inactivity_time_seconds or \
(time() - last_activity_time < inactivity_time_seconds)):
try:
line = read_queue.get(timeout=0.5)
last_activity_time = time()
printer.string("{}{}".format(prompt, line), file_only=True)
for entry in INTERESTING:
match = re.match(entry[0], line)
if match:
entry[1](match, results, printer, prompt, reporter)
except queue.Empty:
pass
# Let others in
sleep(0.01)
# Set this to stop the read thread
results["finished"] = True
readline_thread.join()
if guard_time_seconds and (time() - start_time >= guard_time_seconds):
printer.string("{}guard timer ({} second(s))" \
" expired.".format(prompt, guard_time_seconds))
elif inactivity_time_seconds and (time() - last_activity_time >=
inactivity_time_seconds):
printer.string("{}inactivity timer ({} second(s))" \
" expired.".format(prompt, inactivity_time_seconds))
else:
return_value = results["items_failed"] + results["reboots"]
except (serial.SerialException, EOFError) as ex:
printer.string("{}{} while accessing port {}: {}.".format(
prompt,
type(ex).__name__, in_handle.name, str(ex)))
return return_value
def run(instance,
mcu,
board,
toolchain,
connection,
connection_lock,
platform_lock,
misc_locks,
clean,
defines,
ubxlib_dir,
working_dir,
printer,
reporter,
test_report_handle,
keep_going_flag=None,
unity_dir=None):
'''Build/run on Arduino'''
return_value = -1
monitor_dtr_rts_on = None
installed = False
sketch_paths = []
build_paths = []
return_values = []
instance_text = u_utils.get_instance_text(instance)
# None of the misc locks are required
del misc_locks
# Since we only currently support ESP-IDF we don't need
# unity as ESP-IDF already has a copy built in
del unity_dir
prompt = PROMPT + instance_text + ": "
toolchain = toolchain.lower()
# Print out what we've been told to do and at the
# same time check for the DTR/RTS off marker
text = "running Arduino for " + mcu + " with SDK " + toolchain + \
" on board with FQBN \"" + board + "\""
if connection and connection["serial_port"]:
text += ", serial port " + connection["serial_port"]
if clean:
text += ", clean build"
if defines:
text += ", with #define(s)"
for idx, define in enumerate(defines):
if define == MONITOR_DTR_RTS_OFF_MARKER:
monitor_dtr_rts_on = False
if idx == 0:
text += " \"" + define + "\""
else:
text += ", \"" + define + "\""
if ubxlib_dir:
text += ", ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
printer.string("{}{}.".format(prompt, text))
reporter.event(u_report.EVENT_TYPE_BUILD, u_report.EVENT_START, "Arduino")
printer.string("{}CD to {}...".format(prompt, working_dir))
with u_utils.ChangeDir(working_dir):
# Lock the Arduino platform while we install the tools
with u_utils.Lock(platform_lock, PLATFORM_LOCK_GUARD_TIME_SECONDS,
"platform", printer, prompt,
keep_going_flag) as locked_platform:
if locked_platform:
installed = install(board, ARDUINO_BOARDS_URLS, printer,
prompt, keep_going_flag)
if installed:
arduino_dir = os.path.join(ubxlib_dir, ARDUINO_SUB_DIR)
library_path = os.path.join(working_dir, LIBRARIES_SUB_DIR)
# Clear out any pre-built ubxlib library or rebuilds
# won't pick up changes
clear_prebuilt_library(os.path.join(library_path, LIBRARY_NAME),
mcu, printer, prompt)
# Create the ubxlib Arduino library
if create_library(ubxlib_dir, arduino_dir, toolchain,
os.path.join(library_path, LIBRARY_NAME),
LIBRARY_NAME_LIB_POSTFIX, clean, printer, prompt,
keep_going_flag):
# Create the ubxlib Arduino test library
if create_library(ubxlib_dir, arduino_dir, toolchain,
os.path.join(library_path, LIBRARY_NAME),
LIBRARY_NAME_TEST_POSTFIX, clean, printer,
prompt, keep_going_flag):
# We now build both libraries with the test sketch and we also
# build the examples with the just the ubxlib Arduino library.
# Make a list of the sketches to build
sketch_paths.append(
os.path.join(arduino_dir, TEST_SKETCH_SUB_PATH))
for root, _directories, files in os.walk(library_path):
for file in files:
if os.sep + "examples" + os.sep in root and file.endswith(
".ino"):
sketch_paths.append(os.path.join(root, file))
printer.string("{}{} thing(s) to build.".format(
prompt, len(sketch_paths)))
# Build the sketches: note that the first build of the ubxlib
# Arduino library to a .a file will be copied back into the library
# directory for use in the following builds
build_dir = os.path.join(working_dir, BUILD_SUBDIR)
for sketch_path in sketch_paths:
build_start_time = time()
build_path = build(build_dir, sketch_path,
library_path, mcu, board, defines,
clean, printer, prompt, reporter,
keep_going_flag)
if not u_utils.keep_going(keep_going_flag, printer,
prompt) or not build_path:
break
build_paths.append(build_path)
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_PASSED,
"build {} of {} took {:.0f} second(s)". \
format(len(build_paths), len(sketch_paths),
time() - build_start_time))
if len(build_paths) == len(sketch_paths):
# Download and run the builds
with u_connection.Lock(
connection, connection_lock,
CONNECTION_LOCK_GUARD_TIME_SECONDS, printer,
prompt, keep_going_flag) as locked:
if locked:
for build_path in build_paths:
# I have seen download failures occur if two
# are initiated at the same time so lock the
# platform for this
downloaded = False
with u_utils.Lock(
platform_lock,
PLATFORM_LOCK_GUARD_TIME_SECONDS,
"platform", printer, prompt,
keep_going_flag
) as locked_platform:
if locked_platform:
# Have seen this fail, so give it a few goes
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_START)
retries = 0
while u_utils.keep_going(keep_going_flag, printer,
prompt) and \
not downloaded and (retries < 3):
downloaded = download(
build_path, board,
connection["serial_port"],
printer, prompt,
keep_going_flag)
if not downloaded:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_WARNING,
"unable to download, will" \
" retry...")
retries += 1
sleep(5)
if downloaded:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_COMPLETE)
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_START)
# Open the COM port to get debug output
serial_handle = u_utils.open_serial(
connection["serial_port"],
115200,
printer,
prompt,
dtr_set_on=monitor_dtr_rts_on,
rts_set_on=monitor_dtr_rts_on)
if serial_handle is not None:
# Monitor progress
return_values.append(
u_monitor.main(
serial_handle,
u_monitor.
CONNECTION_SERIAL,
RUN_GUARD_TIME_SECONDS,
RUN_INACTIVITY_TIME_SECONDS,
"\r",
instance,
printer,
reporter,
test_report_handle,
keep_going_flag=
keep_going_flag))
# Delays and flushes here to make sure
# that the serial port actually closes
# since we might need to re-open it for
# another download going around the loop
serial_handle.cancel_read()
sleep(1)
serial_handle.reset_input_buffer()
serial_handle.reset_output_buffer()
sleep(1)
serial_handle.close()
reporter.event(
u_report.
EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_COMPLETE,
"serial port closed")
sleep(5)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to open serial port " + \
connection["serial_port"])
if return_values and return_values[
-1] == 0:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_COMPLETE)
else:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_FAILED)
else:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_FAILED,
"unable to download to the target")
if return_values:
return_value = 0
for item in return_values:
# If a return value goes negative then
# only count the negative values, i.e. the
# number of infrastructure failures
if (item < 0) and (return_value >= 0):
return_value = item
else:
if (((item > 0) and (return_value >= 0)) or \
((item < 0) and (return_value < 0))):
return_value += item
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to lock a connection")
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED, "failed a build")
else:
reporter.event(
u_report.EVENT_TYPE_BUILD, u_report.EVENT_FAILED,
"unable to build library, check debug log for details")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE, u_report.EVENT_FAILED,
"unable to create library, check debug log for details")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE, u_report.EVENT_FAILED,
"unable to install tools, check debug log for details")
return return_value
def install(esp_idf_url, esp_idf_dir, esp_idf_branch, system_lock,
keep_going_flag, printer, prompt, reporter):
'''Install the Espressif tools and ESP-IDF'''
returned_env = {}
count = 0
# Acquire the install lock as this is a global operation
if system_lock is None or u_utils.install_lock_acquire(
system_lock, printer, prompt, keep_going_flag):
# Fetch the repo
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
u_utils.fetch_repo(esp_idf_url, esp_idf_dir,
esp_idf_branch, printer, prompt):
# Set up the environment variable IDF_TOOLS_PATH
my_env = os.environ
my_env["IDF_TOOLS_PATH"] = IDF_TOOLS_PATH
if u_utils.is_linux():
install_cmd = ["./install.sh"]
export_cmd = [".", "./export.sh"]
bash_cmd = True
else:
install_cmd = ["install.bat"]
export_cmd = ["export.bat"]
bash_cmd = None
printer.string("{}installing the Espressif tools to \"{}\" and" \
" ESP-IDF to \"{}\".". \
format(prompt, IDF_TOOLS_PATH, esp_idf_dir))
# Switch to where the stuff should have already
# been fetched to
with u_utils.ChangeDir(esp_idf_dir):
if not u_utils.has_admin():
printer.string("{}NOTE: if {} fails (the return" \
" code may still be 0), then try re-running" \
" as administrator.".format(prompt, " ".join(install_cmd)))
# First call install.bat
# set shell to True to keep Jenkins happy
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
u_utils.exe_run(install_cmd, INSTALL_GUARD_TIME_SECONDS,
printer, prompt, shell_cmd=True,
keep_going_flag=keep_going_flag):
# ...then export.bat to set up paths etc.
# which we return attached to returned_env.
# It is possible for the process of extracting
# the environment variables to fail due to machine
# loading (see comments against EXE_RUN_QUEUE_WAIT_SECONDS
# in exe_run) so give this up to three chances to succeed
while u_utils.keep_going(keep_going_flag, printer, prompt) and \
not returned_env and (count < 3):
# set shell to True to keep Jenkins happy
u_utils.exe_run(export_cmd,
INSTALL_GUARD_TIME_SECONDS,
printer,
prompt,
shell_cmd=True,
returned_env=returned_env,
bash_cmd=bash_cmd,
keep_going_flag=keep_going_flag)
if not returned_env:
printer.string("{}warning: retrying {} to" \
" capture the environment variables...".
format(prompt, " ".join(export_cmd)))
count += 1
if not returned_env:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"{} failed".format(" ".join(export_cmd)))
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"{} failed".format(" ".join(install_cmd)))
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to fetch " + esp_idf_url)
if system_lock is not None:
u_utils.install_lock_release(system_lock, printer, prompt)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED, "could not acquire install lock")
return returned_env
def run(instance,
mcu,
toolchain,
connection,
connection_lock,
platform_lock,
misc_locks,
clean,
defines,
ubxlib_dir,
working_dir,
printer,
reporter,
test_report_handle,
keep_going_flag=None):
'''Build/run on ESP-IDF'''
return_value = -1
monitor_dtr_rts_on = None
instance_text = u_utils.get_instance_text(instance)
# No issues with running in parallel on ESP-IDF
del platform_lock
# Only one toolchain for ESP-IDF
del toolchain
prompt = PROMPT + instance_text + ": "
# Print out what we've been told to do and at the
# same time check for the DTR/RTS off marker
text = "running ESP-IDF for " + mcu
if connection and connection["serial_port"]:
text += ", on serial port " + connection["serial_port"]
if clean:
text += ", clean build"
if defines:
text += ", with #define(s)"
for idx, define in enumerate(defines):
if define == MONITOR_DTR_RTS_OFF_MARKER:
monitor_dtr_rts_on = False
if idx == 0:
text += " \"" + define + "\""
else:
text += ", \"" + define + "\""
if ubxlib_dir:
text += ", ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
printer.string("{}{}.".format(prompt, text))
reporter.event(u_report.EVENT_TYPE_BUILD, u_report.EVENT_START, "ESP-IDF")
printer.string("{}CD to {}...".format(prompt, working_dir))
with u_utils.ChangeDir(working_dir):
# Fetch ESP-IDF into the right sub directory
# and install the tools
esp_idf_dir = ESP_IDF_ROOT + os.sep + ESP_IDF_LOCATION["subdir"]
system_lock = None
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
misc_locks and ("system_lock" in misc_locks):
system_lock = misc_locks["system_lock"]
returned_env = install(ESP_IDF_LOCATION["url"], esp_idf_dir,
ESP_IDF_LOCATION["branch"], system_lock,
keep_going_flag, printer, prompt, reporter)
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
returned_env:
# From here on the ESP-IDF tools need to set up
# and use the set of environment variables
# returned above.
print_env(returned_env, printer, prompt)
# Now do the build
build_dir = working_dir + os.sep + BUILD_SUBDIR
build_start_time = time()
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
build(esp_idf_dir, ubxlib_dir, build_dir,
defines, returned_env, clean,
printer, prompt, reporter, keep_going_flag):
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_PASSED,
"build took {:.0f} second(s)". \
format(time() - build_start_time))
with u_connection.Lock(connection, connection_lock,
CONNECTION_LOCK_GUARD_TIME_SECONDS,
printer, prompt,
keep_going_flag) as locked:
if locked:
# Have seen this fail, only with Python 3 for
# some reason, so give it a few goes
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_START)
retries = 0
downloaded = False
while u_utils.keep_going(keep_going_flag, printer, prompt) and \
not downloaded and (retries < 3):
downloaded = download(esp_idf_dir, ubxlib_dir,
build_dir,
connection["serial_port"],
returned_env, printer,
prompt)
if not downloaded:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_WARNING,
"unable to download, will retry...")
retries += 1
sleep(5)
if downloaded:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_COMPLETE)
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_START)
# Open the COM port to get debug output
serial_handle = u_utils.open_serial(
connection["serial_port"],
115200,
printer,
prompt,
dtr_set_on=monitor_dtr_rts_on,
rts_set_on=monitor_dtr_rts_on)
if serial_handle is not None:
# Monitor progress
return_value = u_monitor.main(
serial_handle,
u_monitor.CONNECTION_SERIAL,
RUN_GUARD_TIME_SECONDS,
RUN_INACTIVITY_TIME_SECONDS,
"\r",
instance,
printer,
reporter,
test_report_handle,
keep_going_flag=keep_going_flag)
serial_handle.close()
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to open serial port " + \
connection["serial_port"])
if return_value == 0:
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_COMPLETE)
else:
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_FAILED)
else:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_FAILED,
"unable to download to the target")
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to lock a connection")
else:
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_FAILED,
"unable to build, check debug log for details")
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED, "tools installation failed")
return return_value
def run(instance,
mcu,
board,
toolchain,
connection,
connection_lock,
platform_lock,
misc_locks,
clean,
defines,
ubxlib_dir,
working_dir,
printer,
reporter,
test_report_handle,
keep_going_flag=None):
'''Build/run on Zephyr'''
return_value = -1
build_dir = None
instance_text = u_utils.get_instance_text(instance)
# Don't need the platform or misc locks
del platform_lock
del misc_locks
# Only one toolchain for Zephyr
del toolchain
prompt = PROMPT + instance_text + ": "
# Print out what we've been told to do
text = "running Zephyr for " + mcu + " (on a \"" + board + "\" board)"
if connection and "debugger" in connection and connection["debugger"]:
text += ", on JLink debugger serial number " + connection["debugger"]
if clean:
text += ", clean build"
if defines:
text += ", with #define(s)"
for idx, define in enumerate(defines):
if idx == 0:
text += " \"" + define + "\""
else:
text += ", \"" + define + "\""
if ubxlib_dir:
text += ", ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
printer.string("{}{}.".format(prompt, text))
reporter.event(u_report.EVENT_TYPE_BUILD, u_report.EVENT_START, "Zephyr")
# Switch to the working directory
with u_utils.ChangeDir(working_dir):
# Check that everything we need is installed
# and configured
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
check_installation(TOOLS_LIST, printer, prompt):
# Set up the environment variables for Zephyr
returned_env = set_env(printer, prompt)
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
returned_env:
# The west tools need to use the environment
# configured above.
print_env(returned_env, printer, prompt)
# Note that Zephyr brings in its own
# copy of Unity so there is no need to
# fetch it here.
if board:
# Do the build
build_start_time = time()
build_dir = build(board, clean, ubxlib_dir, defines,
returned_env, printer, prompt, reporter,
keep_going_flag)
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
build_dir:
# Build succeeded, need to lock some things to do the download
reporter.event(
u_report.EVENT_TYPE_BUILD, u_report.EVENT_PASSED,
"build took {:.0f} second(s)".format(
time() - build_start_time))
# Do the download
with u_connection.Lock(
connection, connection_lock,
CONNECTION_LOCK_GUARD_TIME_SECONDS, printer,
prompt, keep_going_flag) as locked_connection:
if locked_connection:
# Get the device name for JLink
jlink_device_name = jlink_device(mcu)
if not jlink_device_name:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_WARNING,
"MCU not found in JLink devices")
# Do the download
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_START)
retries = 0
downloaded = False
while u_utils.keep_going(keep_going_flag, printer, prompt) and \
not downloaded and (retries < 3):
downloaded = download(
connection, jlink_device_name,
DOWNLOAD_GUARD_TIME_SECONDS, build_dir,
returned_env, printer, prompt)
if not downloaded:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_WARNING,
"unable to download, will retry..."
)
retries += 1
sleep(5)
if downloaded:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_COMPLETE)
# Now the target can be reset
u_utils.reset_nrf_target(
connection, printer, prompt)
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_START)
with URttReader(
jlink_device(mcu),
jlink_serial=connection[
"debugger"],
printer=printer,
prompt=prompt) as rtt_reader:
return_value = u_monitor.main(
rtt_reader,
u_monitor.CONNECTION_RTT,
RUN_GUARD_TIME_SECONDS,
RUN_INACTIVITY_TIME_SECONDS,
"\n",
instance,
printer,
reporter,
test_report_handle,
keep_going_flag=keep_going_flag)
if return_value == 0:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_COMPLETE)
else:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_FAILED)
else:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_FAILED,
"check debug log for details")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to lock a connection")
else:
return_value = 1
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_FAILED,
"check debug log for details")
else:
return_value = 1
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to find overlay file")
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"environment setup failed")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE, u_report.EVENT_FAILED,
"there is a problem with the tools installation for Zephyr")
return return_value
def run(instance,
mcu,
toolchain,
connection,
connection_lock,
platform_lock,
misc_locks,
clean,
defines,
ubxlib_dir,
working_dir,
printer,
reporter,
test_report_handle,
keep_going_flag=None,
unity_dir=None):
'''Build/run on STM32Cube'''
return_value = -1
mcu_dir = ubxlib_dir + os.sep + SDK_DIR + os.sep + "mcu" + os.sep + mcu.lower(
)
instance_text = u_utils.get_instance_text(instance)
workspace_dir = working_dir + os.sep + STM32CUBE_IDE_WORKSPACE_SUBDIR
output_project_dir = working_dir + os.sep + "project"
elf_path = None
downloaded = False
download_list = None
# Only one toolchain for STM32Cube
del toolchain
prompt = PROMPT + instance_text + ": "
# Print out what we've been told to do
text = "running STM32Cube for " + mcu
if connection and "debugger" in connection and connection["debugger"]:
text += ", on STLink debugger serial number " + connection["debugger"]
if clean:
text += ", clean build"
if defines:
text += ", with #define(s)"
for idx, define in enumerate(defines):
if idx == 0:
text += " \"" + define + "\""
else:
text += ", \"" + define + "\""
if ubxlib_dir:
text += ", ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
if unity_dir:
text += ", using Unity from \"" + unity_dir + "\""
printer.string("{}{}.".format(prompt, text))
# On STM32F4 we can get USB errors if we try to do a download
# on one platform while another is performing SWO logging.
# Since each board only runs a single instance of stuff we
# can work around this be ensuring that all downloads are
# completed before SWO logging begins.
# Add us to the list of pending downloads
if misc_locks and ("stm32f4_downloads_list" in misc_locks):
download_list = misc_locks["stm32f4_downloads_list"]
download_list.append(instance_text)
reporter.event(u_report.EVENT_TYPE_BUILD, u_report.EVENT_START,
"STM32Cube")
# Switch to the working directory
with u_utils.ChangeDir(working_dir):
# Check that everything we need is installed
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
check_installation(PATHS_LIST, printer, prompt):
# Fetch Unity, if necessary
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
not unity_dir:
if u_utils.fetch_repo(u_utils.UNITY_URL,
u_utils.UNITY_SUBDIR,
None,
printer,
prompt,
submodule_init=False):
unity_dir = os.getcwd() + os.sep + u_utils.UNITY_SUBDIR
if unity_dir:
# I've no idea why but on every other
# build STM32Cube loses track of where
# most of the files are: you'll see it
# say that it can't find u_cfg_sw.h and
# fail. Until we find out why just
# give it two goes, deleting the project
# we created before trying again.
retries = 2
while u_utils.keep_going(keep_going_flag, printer, prompt) and \
(elf_path is None) and (retries > 0):
# The STM32Cube IDE, based on Eclipse
# has no mechanism for overriding the locations
# of things so here we read the .project
# file and replace the locations of the
# STM32Cube SDK and Unity files as
# appropriate
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
create_project(PROJECT_NAME,
mcu_dir,
output_project_dir,
STM32CUBE_FW_PATH, unity_dir,
printer, prompt):
# Do the build
build_start_time = time()
elf_path = build_binary(output_project_dir,
workspace_dir, PROJECT_NAME,
clean, defines, printer,
prompt, keep_going_flag)
if elf_path is None:
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_INFORMATION,
"unable to build, will retry")
printer.string("{}if the compilation." \
" failure was because" \
" the build couldn't" \
" even find u_cfg_sw.h" \
" then ignore it, Eclipse" \
" lost its head, happens" \
" a lot, we will try again.". \
format(prompt))
else:
reporter.event(
u_report.EVENT_TYPE_BUILD, u_report.EVENT_WARNING,
"unable to create STM32Cube project, will retry")
retries -= 1
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
elf_path:
reporter.event(
u_report.EVENT_TYPE_BUILD, u_report.EVENT_PASSED,
"build took {:.0f} second(s)".format(time() -
build_start_time))
# Lock the connection.
with u_connection.Lock(
connection, connection_lock,
CONNECTION_LOCK_GUARD_TIME_SECONDS, printer,
prompt, keep_going_flag) as locked_connection:
if locked_connection:
# I have seen download failures occur if two
# ST-Link connections are initiated at the same time.
with u_utils.Lock(
platform_lock,
PLATFORM_LOCK_GUARD_TIME_SECONDS,
"platform", printer, prompt,
keep_going_flag) as locked_platform:
if locked_platform:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_START)
# Do the download. I have seen the STM32F4 debugger
# barf on occasions so give this two bites of
# the cherry
retries = 2
while u_utils.keep_going(keep_going_flag,
printer, prompt) and \
not downloaded and (retries > 0):
downloaded = download(
connection,
DOWNLOAD_GUARD_TIME_SECONDS,
elf_path, printer, prompt)
retries -= 1
if not downloaded:
if connection and "serial_port" in connection \
and connection["serial_port"]:
# Before retrying, reset the USB port
u_utils.usb_reset("STMicroelectronics STLink" \
"Virtual COM Port (" +
connection["serial_port"] +
")", printer, prompt)
sleep(5)
if platform_lock:
# Once the download has been done (or not) the platform lock
# can be released, after a little safety sleep
sleep(1)
platform_lock.release()
if downloaded:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_COMPLETE)
# Remove us from the list of pending downloads
if download_list:
download_list.remove(instance_text)
# Wait for all the other downloads to complete before
# starting SWO logging
u_utils.wait_for_completion(
download_list,
"STM32F4 downloads",
DOWNLOADS_COMPLETE_GUARD_TIME_SECONDS,
printer, prompt,
keep_going_flag)
# So that all STM32Cube instances don't start up at
# once, which can also cause problems, wait the
# instance-number number of seconds.
hold_off = instance[0]
if hold_off > 30:
hold_off = 30
sleep(hold_off)
# Create and empty the SWO data file and decoded text file
file_handle = open(SWO_DATA_FILE,
"w").close()
file_handle = open(
SWO_DECODED_TEXT_FILE,
"w").close()
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_START)
try:
# Start a process which reads the
# SWO output from a file, decodes it and
# writes it back to a file
process = Process(
target=swo_decode_process,
args=(SWO_DATA_FILE,
SWO_DECODED_TEXT_FILE))
process.start()
# Now start Open OCD to reset the target
# and capture SWO output
sleep(1)
with u_utils.ExeRun(
open_ocd(
OPENOCD_COMMANDS,
connection), printer,
prompt):
# Open the SWO decoded text file for
# reading, binary to prevent the line
# endings being munged.
file_handle = open(
SWO_DECODED_TEXT_FILE,
"rb")
# Monitor progress based on the decoded
# SWO text
return_value = u_monitor. \
main(file_handle,
u_monitor.CONNECTION_PIPE,
RUN_GUARD_TIME_SECONDS,
RUN_INACTIVITY_TIME_SECONDS,
"\r", instance, printer,
reporter,
test_report_handle,
keep_going_flag=keep_going_flag)
file_handle.close()
process.terminate()
except KeyboardInterrupt:
# Tidy up process on SIGINT
printer.string(
"{}caught CTRL-C, terminating..."
.format(prompt))
process.terminate()
return_value = -1
if return_value == 0:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_COMPLETE)
else:
reporter.event(
u_report.EVENT_TYPE_TEST,
u_report.EVENT_FAILED)
else:
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_FAILED)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to lock a connection")
else:
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_FAILED,
"check debug log for details")
# To avoid a build up of stuff, delete the temporary build and
# workspace on exit
if os.path.exists(output_project_dir):
printer.string("{}deleting temporary build directory {}...". \
format(prompt, output_project_dir))
u_utils.deltree(output_project_dir, printer, prompt)
if os.path.exists(workspace_dir):
printer.string("{}deleting temporary workspace directory {}...". \
format(prompt, workspace_dir))
u_utils.deltree(workspace_dir, printer, prompt)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED, "unable to fetch Unity")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE, u_report.EVENT_FAILED,
"there is a problem with the tools installation for STM32F4")
# Remove us from the list of pending downloads for safety
try:
misc_locks["stm32f4_downloads_list"].remove(instance_text)
except (AttributeError, ValueError, TypeError):
pass
return return_value
def run(instance,
mcu,
toolchain,
connection,
connection_lock,
platform_lock,
misc_locks,
clean,
defines,
ubxlib_dir,
working_dir,
printer,
reporter,
test_report_handle,
keep_going_flag=None,
unity_dir=None):
'''Build/run on nRF5'''
return_value = -1
hex_file_path = None
instance_text = u_utils.get_instance_text(instance)
downloaded = False
_ = (misc_locks) # Suppress unused variable
# Don't need the platform lock
del platform_lock
prompt = PROMPT + instance_text + ": "
# Print out what we've been told to do
text = "running nRF5 for " + mcu + " under " + toolchain
if connection and "debugger" in connection and connection["debugger"]:
text += ", on JLink debugger serial number " + connection["debugger"]
if clean:
text += ", clean build"
if defines:
text += ", with #define(s)"
for idx, define in enumerate(defines):
if idx == 0:
text += " \"" + define + "\""
else:
text += ", \"" + define + "\""
if ubxlib_dir:
text += ", ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
if unity_dir:
text += ", using Unity from \"" + unity_dir + "\""
printer.string("{}{}.".format(prompt, text))
reporter.event(u_report.EVENT_TYPE_BUILD, u_report.EVENT_START,
"nRF5SDK/" + toolchain)
# Switch to the working directory
with u_utils.ChangeDir(working_dir):
# Check that everything we need is installed
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
check_installation(toolchain, TOOLS_LIST, printer, prompt):
# Fetch Unity, if necessary
if u_utils.keep_going(keep_going_flag, printer, prompt) and \
not unity_dir:
if u_utils.fetch_repo(u_utils.UNITY_URL,
u_utils.UNITY_SUBDIR,
None,
printer,
prompt,
submodule_init=False):
unity_dir = os.getcwd() + os.sep + u_utils.UNITY_SUBDIR
if unity_dir:
# Do the build
build_start_time = time()
if u_utils.keep_going(keep_going_flag, printer, prompt):
if toolchain.lower() == "gcc":
build_subdir_gcc = BUILD_SUBDIR_PREFIX_GCC + instance_text.replace(
".", "_")
hex_file_path = build_gcc(clean, build_subdir_gcc,
ubxlib_dir, unity_dir,
defines, printer, prompt,
reporter, keep_going_flag)
elif toolchain.lower() == "ses":
hex_file_path = build_ses(clean, ubxlib_dir, unity_dir,
defines, printer, prompt,
reporter, keep_going_flag)
if hex_file_path:
# Build succeeded, need to lock a connection to do the download
reporter.event(
u_report.EVENT_TYPE_BUILD, u_report.EVENT_PASSED,
"build took {:.0f} second(s)".format(time() -
build_start_time))
# Do the download
with u_connection.Lock(
connection, connection_lock,
CONNECTION_LOCK_GUARD_TIME_SECONDS, printer,
prompt, keep_going_flag) as locked_connection:
if locked_connection:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_START)
# I have seen the download fail on occasion
# so give this two bites of the cherry
retries = 2
while not downloaded and (retries > 0):
downloaded = download(
connection, DOWNLOAD_GUARD_TIME_SECONDS,
hex_file_path, printer, prompt)
retries -= 1
if not downloaded and (retries > 0):
reporter.event(
u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_WARNING,
"unable to download, will retry...")
sleep(5)
if downloaded:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_COMPLETE)
# Now the target can be reset
u_utils.reset_nrf_target(
connection, printer, prompt)
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_START)
with URttReader(
"NRF52840_XXAA",
jlink_serial=connection["debugger"],
printer=printer,
prompt=prompt) as rtt_reader:
return_value = u_monitor.main(
rtt_reader, u_monitor.CONNECTION_RTT,
RUN_GUARD_TIME_SECONDS,
RUN_INACTIVITY_TIME_SECONDS, "\n",
instance, printer, reporter,
test_report_handle)
if return_value == 0:
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_COMPLETE)
else:
reporter.event(u_report.EVENT_TYPE_TEST,
u_report.EVENT_FAILED)
else:
reporter.event(u_report.EVENT_TYPE_DOWNLOAD,
u_report.EVENT_FAILED,
"check debug log for details")
# Wait for a short while before giving
# the connection lock away to make sure
# that everything really has shut down
# in the debugger
sleep(5)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"unable to lock a connection")
else:
return_value = 1
reporter.event(u_report.EVENT_TYPE_BUILD,
u_report.EVENT_FAILED,
"check debug log for details")
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED, "unable to fetch Unity")
else:
reporter.event(
u_report.EVENT_TYPE_INFRASTRUCTURE, u_report.EVENT_FAILED,
"there is a problem with the tools installation for nRF5 SDK")
return return_value
def run(instance,
ubxlib_dir,
working_dir,
printer,
reporter,
keep_going_flag=None):
'''Run Pylint'''
return_value = 1
got_pylint = False
min_rating = 10
instance_text = u_utils.get_instance_text(instance)
prompt = PROMPT + instance_text + ": "
# Print out what we've been told to do
text = "running Pylint from ubxlib directory \"" + ubxlib_dir + "\""
if working_dir:
text += ", working directory \"" + working_dir + "\""
text += ", checking for minimum rating " + str(MIN_RATING)
printer.string("{}{}.".format(prompt, text))
reporter.event(u_report.EVENT_TYPE_CHECK, u_report.EVENT_START, "Pylint")
got_pylint = u_utils.exe_where("pylint", \
"ERROR: can't find pylint, please make" \
" sure that it is installed and on the path.", \
printer, prompt)
if got_pylint:
# Run Pylint on all the .py files in PYTHON_PATHS
return_value = 0
for py_path in PYTHON_PATHS:
abs_py_path = ubxlib_dir + os.sep + py_path
if os.path.exists(abs_py_path):
printer.string("{}CD to {}...".format(prompt, abs_py_path))
with u_utils.ChangeDir(abs_py_path):
popen_keywords = {
'stderr': subprocess.STDOUT,
'shell': True # Stop Jenkins hanging
}
for py_file in os.listdir(abs_py_path):
if py_file.endswith(".py"):
if not u_utils.keep_going(keep_going_flag, printer,
prompt):
return_value = -1
break
printer.string("{}running Pylint on {}...".format(
prompt, py_file))
got_rating = False
try:
# ignore u_settings module as it sets members programatically and
# will thus generate a bunch of lint warnings
text = subprocess.check_output(
u_utils.subprocess_osify([
"pylint", "--exit-zero",
"--ignored-modules=u_settings", py_file
]), **popen_keywords)
rating = 0
for line in text.splitlines():
line = line.decode()
printer.string("{}{}".format(prompt, line))
# See if there is a rating in this line
outcome = line.rpartition(
"Your code has been rated at ")
if len(outcome) == 3:
outcome = outcome[2].split("/")
# Get the bit before the "/" in the "x/y" rating
try:
rating = float(outcome[0])
got_rating = True
if rating < min_rating:
min_rating = rating
if rating < MIN_RATING:
return_value += 1
except ValueError:
# Can't have been a rating line
pass
# Let other things in
sleep(0.01)
if got_rating:
if rating < MIN_RATING:
reporter.event(u_report.EVENT_TYPE_CHECK,
u_report.EVENT_ERROR,
"rating {} < minimum ({})". \
format(rating, MIN_RATING))
for line in text.splitlines():
line = line.strip().decode()
if line:
reporter.event_extra_information(
line)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"Pylint returned no rating for file \"{}\"". \
format(py_file))
printer.string("{}Pylint returned no rating.". \
format(prompt))
# No rating returned, flag an error
return_value += 1
except subprocess.CalledProcessError as error:
reporter.event(u_report.EVENT_TYPE_CHECK,
u_report.EVENT_FAILED)
printer.string(
"{}Pylint returned error {}:".format(
prompt, error.returncode))
for line in error.output.splitlines():
line = line.strip().decode()
if line:
reporter.event_extra_information(line)
printer.string("{}{}".format(
prompt, line))
else:
# Flag an error if a path doesn't exist
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"path {} does not exit".format(abs_py_path))
printer.string("{}path \"{}\" does not exist.". \
format(prompt, abs_py_path))
return_value += 1
printer.string("{}Pylint check complete, minimum rating {}.".format(
prompt, min_rating))
if min_rating < MIN_RATING:
reporter.event(u_report.EVENT_TYPE_CHECK, u_report.EVENT_FAILED)
else:
reporter.event(u_report.EVENT_TYPE_CHECK, u_report.EVENT_PASSED)
else:
reporter.event(u_report.EVENT_TYPE_INFRASTRUCTURE,
u_report.EVENT_FAILED,
"there is a problem with the Pylint installation")
return return_value