def schedule_software_update(self, sleep_time_before_reboot):
"""Wrapper function to safely schedule the execute update function with the required
arguments.
Args:
sleep_time_before_reboot (int): Time in seconds to sleep before initiating
a device reboot after installation.
Returns:
bool: True if the execute_update was successfully schedule else False.
"""
with utility.AutoLock(self.state_guard):
if self.update_state == software_update_config.SoftwareUpdateState.UPDATE_AVAILABLE:
self.get_logger().info("Update is available, scheduling...")
sleep_time_before_reboot = \
software_update_config.MIN_TIME_BEFORE_REBOOT_IN_SECONDS \
if sleep_time_before_reboot < software_update_config.MIN_TIME_BEFORE_REBOOT_IN_SECONDS \
else sleep_time_before_reboot
# Update kickoff: schedule update.
self.scheduler.schedule_action(
self.execute_update, reboot_after=sleep_time_before_reboot)
# Change state to requested.
self.update_state = software_update_config.SoftwareUpdateState.UPDATE_PENDING
return True
else:
self.get_logger().info(
"Ignoring update request, state: "
f"{self.get_state_description(self.update_state)}")
return False
def software_update_state(self, req, res):
"""Callback for the console_model_action service. Handles the upload model and delete model
actions from the console.
Args:
req (SoftwareUpdateStateSrv.Request): Request object with the request flag set.
res (SoftwareUpdateStateSrv.Response): Response object with update_state(int)
with the software udpate state information.
Returns:
SoftwareUpdateStateSrv.Response: Response object with update_state(int)
with the software udpate state information.
"""
self.get_logger().info(f"software_update_state({req.request})")
try:
with utility.AutoLock(self.state_guard):
update_state = self.update_state
res.update_state = update_state
self.get_logger().info(
f"software_update_state response({update_state})")
return res
except Exception:
self.get_logger().error("Invalid update status")
res.update_state = software_update_config.SoftwareUpdateState.UPDATE_UNKNOWN
return res
def get_update_state(self, force_update_check=False):
"""Return the update status after perfoming software update check or waiting for current
ongoing check to complete.
Args:
force_update_check (bool, optional): Set to True to force schedule update check immediately.
Defaults to False.
Returns:
int: Software update state information.
"""
self.get_logger().info("Get software update state...")
with utility.AutoLock(self.state_guard):
# Force update check if it hasn"t been completed yet.
if self.update_state == software_update_config.SoftwareUpdateState.UPDATE_UNKNOWN:
force_update_check = True
# Kick off update check if needed.
if not self.check_in_progress:
if force_update_check:
self.check_complete.clear()
self.scheduler.schedule_action(self.do_update_check)
else:
self.get_logger().info(
"Force software update check flag is not set")
else:
self.get_logger().info(
"Previous scheduled software update check in progress...")
# Wait for the check to complete.
while not self.check_complete.wait(2):
self.get_logger().info(
"Waiting for software update check complete...")
self.get_logger().info("Software update check complete.")
with utility.AutoLock(self.state_guard):
# State here can still be unknown if it had been unknown before the last update
# and the last update failed. Report and do the forced check next time.
update_state = self.update_state
self.get_logger().info(
"Software update status: "
f"{self.get_state_description(update_state)}")
return update_state
def check_otg_connection(self):
"""Wrapper function to schedule the otg_connection_change function whenver
there is a chagne in the host connecton status.
"""
with utility.AutoLock(self.otg_guard):
host_connected = "U0" in file_system_utils.read_line(
os.path.join(otg_config.OTG_STATE_DIRECTORY,
otg_config.OTG_LINK_STATE))
if host_connected != self.otg_connected:
self.otg_connected = host_connected
self.scheduler.schedule_action(self.otg_connection_change, connected=host_connected)
def console_model_action(self, model_path, action):
"""Function to delete the model from model_path if action == 0 or else transfer the model
from model_path to /opt/aws/deepracer/artifacts.
Args:
model_path (str): Path where the model is located.
action (int): 0 to delete the model and 1 to transfer the model.
Returns:
str: Status(str) containing the details about the action execution.
"""
self.get_logger().info(
f"console_model_action...model_path : {model_path} action : {action}"
)
if action == 0:
# Delete the installed model
status = file_system_utils.remove_dir_tree(model_path)
return "done-delete" if status else "failure-on-delete"
# Else upload the model to the DeepRacer device
# Wait until all threads finished.
for model in self.models_in_progress:
state = self.models_in_progress[model]
state.wait_complete()
self.get_logger().info(
"consoleloadModels... copy from model directory to local temporary directories"
)
# Transfer models from model directoy to local temporary directories.
with utility.AutoLock(self.progress_guard):
self.transfer_models({
"path": model_path,
"name": "",
"mount_point": None
})
# Start the threads.
for model in self.models_in_progress:
state = self.models_in_progress[model]
state.start_install()
# Wait till the model upload is completed to show the proper size
for model in self.models_in_progress:
state = self.models_in_progress[model]
state.wait_complete()
return "done-upload"
def load_models(self, keyworded_args):
"""Wrapper function to trigger the transfer models function to copy the model artifacts
from USB to /opt/aws/deepracer/artifacts folder, after verifying that its safe
to copy the files.
Args:
keyworded_args (dict): Keyworded arguments passed to the function while scheduling.
"""
# Wait until all threads finished.
for model in self.models_in_progress:
state = self.models_in_progress[model]
state.wait_complete()
# Transfer models from media to local temporary directories.
with utility.AutoLock(self.progress_guard):
self.transfer_models(keyworded_args)
# Start the threads.
for model in self.models_in_progress:
state = self.models_in_progress[model]
state.start_install()
def do_update_check(self):
"""Main function to perform the cache update, update the candidate list and
update_state flag.
"""
self.check_in_progress = True
self.get_logger().info("Checking software update...")
# Make sure we have network connection.
if not self.is_network_connected:
self.get_logger().info("Scheduling software update check to wait "
"for network connection...")
self.reschedule_software_update_check = True
return
# Update and Re-open the cache to read the updated package list
if not self.update_deepracer_cache():
self.check_complete.set()
self.check_in_progress = False
return
# Reset the package update list.
self.update_pkg_candidate_list()
with utility.AutoLock(self.state_guard):
# No packages found to update?
if len(self.update_list) == 0:
self.update_state = software_update_config.SoftwareUpdateState.UP_TO_DATE
else:
self.update_state = software_update_config.SoftwareUpdateState.UPDATE_AVAILABLE
self.pct_dict_db.put({
software_update_config.PROG_STATE_KEY:
software_update_config.PROGRESS_STATES[1],
software_update_config.PROG_PCT_KEY:
0.0
})
self.get_logger().info(
self.get_state_description(self.update_state))
self.check_in_progress = False
self.check_complete.set()
def verify_model_ready(self, model_name):
"""Helper function to wait for model loading to complete if its not already,
else verify if the model folder has a checksum file created.
Args:
model_name (str): Model folder name in /opt/aws/deepracer/artifacts folder.
Returns:
bool: True if the model is properly loaded/has checksum.txt in the model folder else False.
"""
self.get_logger().info(
f"Verifying readiness of model \"{model_name}\"...")
try:
with utility.AutoLock(self.progress_guard):
state = self.models_in_progress[model_name]
try:
# Wait for the model to become ready.
self.get_logger().info(
f"Making sure \"{model_name}\" thread is complete...")
state.wait_complete()
# Return install status.
ready = state.installed.isSet()
if ready:
self.get_logger().info(
f"Model \"{model_name}\" is successfully installed.")
else:
self.get_logger().info(
f"Model \"{model_name}\" installation failed.")
except Exception as ex:
self.get_logger().error(
f"Failed to wait for installation of \"{model_name}\": {ex}"
)
ready = False
except Exception as ex:
self.get_logger().info(
f"Model \"{model_name}\" not being currently installed,"
" checking existing installation...")
model_install_directory = os.path.join(
model_loader_config.MODEL_INSTALL_ROOT_DIRECTORY, model_name)
if not os.path.isdir(model_install_directory):
self.get_logger().info(
f"Model \"{model_name}\" name is not recognized.")
ready = False
else:
checksum_path = os.path.join(
model_install_directory,
model_loader_config.MODEL_CHECKSUM_FILE)
checksum = file_system_utils.read_line(checksum_path).strip()
ready = (checksum != "")
if ready:
self.get_logger().info(
f"Model \"{model_name}\" is installed.")
else:
self.get_logger().info(
f"Model \"{model_name}\" installation appears incomplete."
)
return ready
def execute_update(self, keyworded_args):
"""Main function to execute the software update process triggered from console. It commits
the udpated packages found, verify the package installation, publish update percentage
post installation, update power LED status during the process and reboot the device
after installation process is completed.
Args:
keyworded_args (dict): Keyworded arguments passed to the function while scheduling.
"""
self.get_logger().info("Starting software update...")
self.publish_pct_timer = self.create_timer(2.0,
self.publish_update_pct)
with utility.AutoLock(self.state_guard):
# Ignore if already up to date.
if self.update_state == software_update_config.SoftwareUpdateState.UP_TO_DATE:
self.get_logger().info(
f"Software is up to date, ignoring: {self.update_state}")
return
# Ignore if in progress.
if self.update_state == software_update_config.SoftwareUpdateState.UPDATE_IN_PROGRESS:
self.get_logger().info(
"Software update is already in progress, ignoring.")
return
# Mark as in progress.
self.update_state = software_update_config.SoftwareUpdateState.UPDATE_IN_PROGRESS
self.led_blink_request.led_index = constants.LEDIndex.POWER_LED
self.led_blink_request.color1 = constants.LEDColor.BLUE
self.led_blink_request.color2 = constants.LEDColor.NO_COLOR
self.led_blink_request.delay = 0.2
self.led_blink_service.call(self.led_blink_request)
try:
for package in self.update_list:
self.get_logger().info(
f"** Installing Software update for {package.name}")
package.mark_install()
commit_response = self.cache.commit(
fetch_progress.FetchProgress(self.pct_dict_db,
self.get_logger()),
install_progress.InstallProgress(self.pct_dict_db,
self.get_logger()))
self.get_logger().info(
f"** Apt cache committed with latest packages: {commit_response}"
)
retry_verification = 0
installation_successful = False
while retry_verification < software_update_config.MAX_UPDATE_VERIFICATION_RETRY_COUNT:
if self.verify_software_update():
self.get_logger().info("Software update verified")
installation_successful = True
break
else:
self.get_logger().error(
"Software update could not be verified. "
f"Retrying {retry_verification+1}/{5}..")
retry_verification += 1
time.sleep(5)
if not installation_successful:
raise Exception("Failed to install packages")
# Update is only 100% after the commit call exits
# Set the state to complete
self.publish_pct_timer.cancel()
self.pct_dict_db.put({
software_update_config.PROG_STATE_KEY:
software_update_config.PROGRESS_STATES[5],
software_update_config.PROG_PCT_KEY:
100.0
})
self.publish_update_pct()
with utility.AutoLock(self.state_guard):
self.update_state = software_update_config.SoftwareUpdateState.UP_TO_DATE
self.get_logger().info("Software update complete.")
except Exception as ex:
self.led_solid_request.led_index = constants.LEDIndex.POWER_LED
self.led_solid_request.color = constants.LEDColor.RED
self.led_solid_request.hold = 2.0
self.led_solid_service.call(self.led_solid_request)
with utility.AutoLock(self.state_guard):
self.update_state = software_update_config.SoftwareUpdateState.UPDATE_AVAILABLE
self.get_logger().info(f"Software update failed: {ex}")
self.pct_dict_db.put({
software_update_config.PROG_STATE_KEY:
software_update_config.PROGRESS_STATES[0],
software_update_config.PROG_PCT_KEY:
0.0
})
self.publish_update_pct()
#
# Not rebooting the software update is dangerous and leave it in unstable state
# The parameter passed to the software update is to reboot after some sleep time.
# This is required so that the webserver can return 100% compeletion to the client
# Before it reboots itself.
#
reboot_after = keyworded_args.get("reboot_after", 0)
self.get_logger().info(f"Reboot after: {reboot_after} seconds")
time.sleep(reboot_after)
self.led_solid_request.led_index = constants.LEDIndex.POWER_LED
self.led_solid_request.color = constants.LEDColor.NO_COLOR
self.led_solid_request.hold = 0.0
self.led_solid_service.call(self.led_solid_request)
self.get_logger().info("Rebooting...")
os.system("reboot")