def main() -> None: params = Params() if params.get_bool("DisableUpdates"): cloudlog.warning("updates are disabled by the DisableUpdates param") exit(0) ov_lock_fd = open(LOCK_FILE, 'w') try: fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as e: raise RuntimeError( "couldn't get overlay lock; is another instance running?") from e # Set low io priority proc = psutil.Process() if psutil.LINUX: proc.ionice(psutil.IOPRIO_CLASS_BE, value=7) # Check if we just performed an update if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") if not params.get("InstallDate"): t = datetime.datetime.utcnow().isoformat() params.put("InstallDate", t.encode('utf8')) overlay_init = Path(os.path.join(BASEDIR, ".overlay_init")) overlay_init.unlink(missing_ok=True) update_failed_count = 0 # TODO: Load from param? wait_helper = WaitTimeHelper(proc) # Run the update loop while not wait_helper.shutdown: wait_helper.ready_event.clear() # Attempt an update exception = None new_version = False update_failed_count += 1 try: init_overlay() # TODO: still needed? skip this and just fetch? # Lightweight internt check internet_ok, update_available = check_for_update() if internet_ok and not update_available: update_failed_count = 0 # Fetch update if internet_ok: new_version = fetch_update(wait_helper) update_failed_count = 0 except subprocess.CalledProcessError as e: cloudlog.event("update process failed", cmd=e.cmd, output=e.output, returncode=e.returncode) exception = f"command failed: {e.cmd}\n{e.output}" overlay_init.unlink(missing_ok=True) except Exception as e: cloudlog.exception("uncaught updated exception, shouldn't happen") exception = str(e) overlay_init.unlink(missing_ok=True) if not wait_helper.shutdown: try: set_params(new_version, update_failed_count, exception) except Exception: cloudlog.exception( "uncaught updated exception while setting params, shouldn't happen" ) # infrequent attempts if we successfully updated recently wait_helper.sleep(5 * 60 if update_failed_count > 0 else 90 * 60) dismount_overlay()
def main(): update_failed_count = 0 overlay_init_done = False params = Params() if params.get("DisableUpdates") == b"1": raise RuntimeError("updates are disabled by param") if not os.geteuid() == 0: raise RuntimeError("updated must be launched as root!") # Set low io priority p = psutil.Process() if psutil.LINUX: p.ionice(psutil.IOPRIO_CLASS_BE, value=7) ov_lock_fd = open('/tmp/safe_staging_overlay.lock', 'w') try: fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: raise RuntimeError( "couldn't get overlay lock; is another updated running?") # Wait a short time before our first update attempt # Avoids race with IsOffroad not being set, reduces manager startup load time.sleep(30) wait_helper = WaitTimeHelper() time_offroad = 0 need_reboot = False while True: update_failed_count += 1 time_wrong = datetime.datetime.utcnow().year < 2019 ping_failed = subprocess.call( ["ping", "-W", "4", "-c", "1", "8.8.8.8"]) # Wait until we have a valid datetime to initialize the overlay if not (ping_failed or time_wrong): try: # If the git directory has modifcations after we created the overlay # we need to recreate the overlay if overlay_init_done: overlay_init_fn = os.path.join(BASEDIR, ".overlay_init") git_dir_path = os.path.join(BASEDIR, ".git") new_files = run( ["find", git_dir_path, "-newer", overlay_init_fn]) if len(new_files.splitlines()): cloudlog.info( ".git directory changed, recreating overlay") overlay_init_done = False if not overlay_init_done: init_ovfs() overlay_init_done = True if params.get("IsOffroad") == b"1": need_reboot = attempt_update(time_offroad, need_reboot) update_failed_count = 0 else: time_offroad = sec_since_boot() cloudlog.info("not running updater, openpilot running") except subprocess.CalledProcessError as e: cloudlog.event("update process failed", cmd=e.cmd, output=e.output, returncode=e.returncode) overlay_init_done = False except Exception: cloudlog.exception( "uncaught updated exception, shouldn't happen") params.put("UpdateFailedCount", str(update_failed_count)) wait_between_updates(wait_helper.ready_event) if wait_helper.shutdown: break # We've been signaled to shut down dismount_ovfs()
def register(show_spinner=False): params = Params() params.put("Version", version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_git_commit(default="")) params.put("GitBranch", get_git_branch(default="")) params.put("GitRemote", get_git_remote(default="")) params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') needs_registration = (None in [IMEI, HardwareSerial]) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") # make key readable by app users (ai.comma.plus.offroad) os.chmod(PERSIST + '/comma/', 0o755) os.chmod(PERSIST + '/comma/id_rsa', 0o744) dongle_id = params.get("DongleId", encoding='utf8') needs_registration = needs_registration or dongle_id is None if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly private_key = open(PERSIST + "/comma/id_rsa").read() public_key = open(PERSIST + "/comma/id_rsa.pub").read() register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') # Block until we get the imei imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) serial = HARDWARE.get_serial() params.put("IMEI", imei1) params.put("HardwareSerial", serial) while True: try: cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] params.put("DongleId", dongle_id) break except Exception: cloudlog.exception("failed to authenticate") time.sleep(1) if show_spinner: spinner.close() return dongle_id
def calculate(self, pandaState): try: now = sec_since_boot() # If pandaState is None, we're probably not in a car, so we don't care if pandaState is None or pandaState.pandaState.pandaType == log.PandaState.PandaType.unknown: with self.integration_lock: self.last_measurement_time = None self.next_pulsed_measurement_time = None self.power_used_uWh = 0 return # Low-pass battery voltage self.car_voltage_instant_mV = pandaState.pandaState.voltage self.car_voltage_mV = ( (pandaState.pandaState.voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K))) # Cap the car battery power and save it in a param every 10-ish seconds self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0) self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh) if now - self.last_save_time >= 10: put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh))) self.last_save_time = now # First measurement, set integration time with self.integration_lock: if self.last_measurement_time is None: self.last_measurement_time = now return if (pandaState.pandaState.ignitionLine or pandaState.pandaState.ignitionCan): # If there is ignition, we integrate the charging rate of the car with self.integration_lock: self.power_used_uWh = 0 integration_time_h = (now - self.last_measurement_time) / 3600 if integration_time_h < 0: raise ValueError( f"Negative integration time: {integration_time_h}h" ) self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h) self.last_measurement_time = now else: # No ignition, we integrate the offroad power used by the device is_uno = pandaState.pandaState.pandaType == log.PandaState.PandaType.uno # Get current power draw somehow current_power = HARDWARE.get_current_power_draw() # pylint: disable=assignment-from-none if current_power is not None: pass elif HARDWARE.get_battery_status() == 'Discharging': # If the battery is discharging, we can use this measurement # On C2: this is low by about 10-15%, probably mostly due to UNO draw not being factored in current_power = ( (HARDWARE.get_battery_voltage() / 1000000) * (HARDWARE.get_battery_current() / 1000000)) elif (self.next_pulsed_measurement_time is not None) and ( self.next_pulsed_measurement_time <= now): # TODO: Figure out why this is off by a factor of 3/4??? FUDGE_FACTOR = 1.33 # Turn off charging for about 10 sec in a thread that does not get killed on SIGINT, and perform measurement here to avoid blocking thermal def perform_pulse_measurement(now): try: HARDWARE.set_battery_charging(False) time.sleep(5) # Measure for a few sec to get a good average voltages = [] currents = [] for _ in range(6): voltages.append(HARDWARE.get_battery_voltage()) currents.append(HARDWARE.get_battery_current()) time.sleep(1) current_power = ((mean(voltages) / 1000000) * (mean(currents) / 1000000)) self._perform_integration( now, current_power * FUDGE_FACTOR) # Enable charging again HARDWARE.set_battery_charging(True) except Exception: cloudlog.exception( "Pulsed power measurement failed") # Start pulsed measurement and return threading.Thread(target=perform_pulse_measurement, args=(now, )).start() self.next_pulsed_measurement_time = None return elif self.next_pulsed_measurement_time is None and not is_uno: # On a charging EON with black panda, or drawing more than 400mA out of a white/grey one # Only way to get the power draw is to turn off charging for a few sec and check what the discharging rate is # We shouldn't do this very often, so make sure it has been some long-ish random time interval self.next_pulsed_measurement_time = now + random.randint( 120, 180) return else: # Do nothing return # Do the integration self._perform_integration(now, current_power) except Exception: cloudlog.exception("Power monitoring calculation failed")
def thermald_thread(): pm = messaging.PubMaster(['deviceState']) pandaState_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected pandaState frequency pandaState_sock = messaging.sub_sock('pandaState', timeout=pandaState_timeout) location_sock = messaging.sub_sock('gpsLocationExternal') managerState_sock = messaging.sub_sock('managerState', conflate=True) fan_speed = 0 count = 0 startup_conditions = { "ignition": False, } startup_conditions_prev = startup_conditions.copy() off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green usb_power = True current_branch = get_git_branch() network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) pandaState_prev = None should_start_prev = False handle_fan = None is_uno = False ui_running_prev = False params = Params() power_monitor = PowerMonitoring() no_panda_cnt = 0 thermal_config = HARDWARE.get_thermal_config() # CPR3 logging if EON: base_path = "/sys/kernel/debug/cpr3-regulator/" cpr_files = [p for p in Path(base_path).glob("**/*") if p.is_file()] cpr_data = {} for cf in cpr_files: with open(cf, "r") as f: try: cpr_data[str(cf)] = f.read().strip() except Exception: pass cloudlog.event("CPR", data=cpr_data) while 1: pandaState = messaging.recv_sock(pandaState_sock, wait=True) msg = read_thermal(thermal_config) if pandaState is not None: usb_power = pandaState.pandaState.usbPowerMode != log.PandaState.UsbPowerMode.client # If we lose connection to the panda, wait 5 seconds before going offroad if pandaState.pandaState.pandaType == log.PandaState.PandaType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if startup_conditions["ignition"]: cloudlog.error("Lost panda connection while onroad") startup_conditions["ignition"] = False else: no_panda_cnt = 0 startup_conditions[ "ignition"] = pandaState.pandaState.ignitionLine or pandaState.pandaState.ignitionCan startup_conditions[ "hardware_supported"] = pandaState.pandaState.pandaType not in [ log.PandaState.PandaType.whitePanda, log.PandaState.PandaType.greyPanda ] set_offroad_alert_if_changed( "Offroad_HardwareUnsupported", not startup_conditions["hardware_supported"]) # Setup fan handler on first connect to panda if handle_fan is None and pandaState.pandaState.pandaType != log.PandaState.PandaType.unknown: is_uno = pandaState.pandaState.pandaType == log.PandaState.PandaType.uno if (not EON) or is_uno: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if pandaState_prev is not None: if pandaState.pandaState.pandaType == log.PandaState.PandaType.unknown and \ pandaState_prev.pandaState.pandaType != log.PandaState.PandaType.unknown: params.panda_disconnect() pandaState_prev = pandaState # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() network_strength = HARDWARE.get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.deviceState.freeSpacePercent = get_available_percent(default=100.0) msg.deviceState.memoryUsagePercent = int( round(psutil.virtual_memory().percent)) msg.deviceState.cpuUsagePercent = int(round(psutil.cpu_percent())) msg.deviceState.networkType = network_type msg.deviceState.networkStrength = network_strength msg.deviceState.batteryPercent = HARDWARE.get_battery_capacity() msg.deviceState.batteryStatus = HARDWARE.get_battery_status() msg.deviceState.batteryCurrent = HARDWARE.get_battery_current() msg.deviceState.batteryVoltage = HARDWARE.get_battery_voltage() msg.deviceState.usbOnline = HARDWARE.get_usb_present() # Fake battery levels on uno for frame if (not EON) or is_uno: msg.deviceState.batteryPercent = 100 msg.deviceState.batteryStatus = "Charging" msg.deviceState.batteryTempC = 0 current_filter.update(msg.deviceState.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update(max(msg.deviceState.cpuTempC)) max_comp_temp = max(max_cpu_temp, msg.deviceState.memoryTempC, max(msg.deviceState.gpuTempC)) bat_temp = msg.deviceState.batteryTempC if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, startup_conditions["ignition"]) msg.deviceState.fanSpeedPercentDesired = fan_speed # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 # We only do this if there is a relay that prevents the car from faulting is_offroad_for_5_min = (started_ts is None) and ( (not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if max_cpu_temp > 107. or bat_temp >= 63. or (is_offroad_for_5_min and max_cpu_temp > 70.0): # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 96.0 or bat_temp > 60.: # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 94.0: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: thermal_status = ThermalStatus.green # default to good condition # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert startup_conditions["time_valid"] = (now.year > 2020) or ( now.year == 2020 and now.month >= 10) set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) # Show update prompt try: last_update = datetime.datetime.fromisoformat( params.get("LastUpdateTime", encoding='utf8')) except (TypeError, ValueError): last_update = now dt = now - last_update update_failed_count = params.get("UpdateFailedCount") update_failed_count = 0 if update_failed_count is None else int( update_failed_count) last_update_exception = params.get("LastUpdateException", encoding='utf8') if update_failed_count > 15 and last_update_exception is not None: if current_branch in ["release2", "dashcam"]: extra_text = "Ensure the software is correctly installed" else: extra_text = last_update_exception set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text) elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True) elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.") else: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) startup_conditions["up_to_date"] = params.get( "Offroad_ConnectivityNeeded") is None or params.get_bool( "DisableUpdates") startup_conditions["not_uninstalling"] = not params.get_bool( "DoUninstall") startup_conditions["accepted_terms"] = params.get( "HasAcceptedTerms") == terms_version panda_signature = params.get("PandaFirmware") startup_conditions["fw_version_match"] = (panda_signature is None) or ( panda_signature == FW_SIGNATURE ) # don't show alert is no panda is connected (None) set_offroad_alert_if_changed( "Offroad_PandaFirmwareMismatch", (not startup_conditions["fw_version_match"])) # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ (current_branch in ['dashcam', 'dashcam-staging']) startup_conditions["not_driver_view"] = not params.get_bool( "IsDriverViewEnabled") startup_conditions["not_taking_snapshot"] = not params.get_bool( "IsTakingSnapshot") # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 startup_conditions[ "device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed( "Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"])) # Handle offroad/onroad transition should_start = all(startup_conditions.values()) if should_start: if not should_start_prev: params.delete("IsOffroad") if TICI and DISABLE_LTE_ONROAD: os.system("sudo systemctl stop --no-block lte") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions) if should_start_prev or (count == 0): params.put_bool("IsOffroad", True) if TICI and DISABLE_LTE_ONROAD: os.system("sudo systemctl start --no-block lte") started_ts = None if off_ts is None: off_ts = sec_since_boot() # Offroad power monitoring power_monitor.calculate(pandaState) msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used() msg.deviceState.carBatteryCapacityUwh = max( 0, power_monitor.get_car_battery_capacity()) # Check if we need to disable charging (handled by boardd) msg.deviceState.chargingDisabled = power_monitor.should_disable_charging( pandaState, off_ts) # Check if we need to shut down if power_monitor.should_shutdown(pandaState, off_ts, started_seen): cloudlog.info(f"shutting device down, offroad since {off_ts}") # TODO: add function for blocking cloudlog instead of sleep time.sleep(10) HARDWARE.shutdown() # If UI has crashed, set the brightness to reasonable non-zero value manager_state = messaging.recv_one_or_none(managerState_sock) if manager_state is not None: ui_running = "ui" in (p.name for p in manager_state.managerState.processes if p.running) if ui_running_prev and not ui_running: HARDWARE.set_screen_brightness(20) ui_running_prev = ui_running msg.deviceState.chargingError = current_filter.x > 0. and msg.deviceState.batteryPercent < 90 # if current is positive, then battery is being discharged msg.deviceState.started = started_ts is not None msg.deviceState.startedMonoTime = int(1e9 * (started_ts or 0)) msg.deviceState.thermalStatus = thermal_status pm.send("deviceState", msg) if EON and not is_uno: set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power)) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # report to server once every 10 minutes if (count % int(600. / DT_TRML)) == 0: location = messaging.recv_sock(location_sock) cloudlog.event("STATUS_PACKET", count=count, pandaState=(strip_deprecated_keys( pandaState.to_dict()) if pandaState else None), location=(strip_deprecated_keys( location.gpsLocationExternal.to_dict()) if location else None), deviceState=strip_deprecated_keys(msg.to_dict())) count += 1
def hw_state_thread(end_event, hw_queue): """Handles non critical hardware state, and sends over queue""" count = 0 registered_count = 0 prev_hw_state = None modem_version = None modem_nv = None modem_configured = False while not end_event.is_set(): # these are expensive calls. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() modem_temps = HARDWARE.get_modem_temperatures() if len(modem_temps) == 0 and prev_hw_state is not None: modem_temps = prev_hw_state.modem_temps # Log modem version once if TICI and ((modem_version is None) or (modem_nv is None)): modem_version = HARDWARE.get_modem_version() # pylint: disable=assignment-from-none modem_nv = HARDWARE.get_modem_nv() # pylint: disable=assignment-from-none if (modem_version is not None) and (modem_nv is not None): cloudlog.event("modem version", version=modem_version, nv=modem_nv) hw_state = HardwareState( network_type=network_type, network_metered=HARDWARE.get_network_metered(network_type), network_strength=HARDWARE.get_network_strength( network_type), network_info=HARDWARE.get_network_info(), nvme_temps=HARDWARE.get_nvme_temperatures(), modem_temps=modem_temps, ) try: hw_queue.put_nowait(hw_state) except queue.Full: pass if TICI and (hw_state.network_info is not None) and (hw_state.network_info.get( 'state', None) == "REGISTERED"): registered_count += 1 else: registered_count = 0 if registered_count > 10: cloudlog.warning( f"Modem stuck in registered state {hw_state.network_info}. nmcli conn up lte" ) os.system("nmcli conn up lte") registered_count = 0 # TODO: remove this once the config is in AGNOS if not modem_configured and len(HARDWARE.get_sim_info().get( 'sim_id', '')) > 0: cloudlog.warning("configuring modem") HARDWARE.configure_modem() modem_configured = True prev_hw_state = hw_state except Exception: cloudlog.exception("Error getting hardware state") count += 1 time.sleep(DT_TRML)
# This is needed otherwise touched files might show up as modified try: subprocess.check_call(["git", "update-index", "--refresh"]) except subprocess.CalledProcessError: pass dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0) # Log dirty files if dirty and comma_remote: try: dirty_files = run_cmd(["git", "diff-index", branch, "--"]) cloudlog.event("dirty comma branch", version=version, dirty=dirty, origin=origin, branch=branch, dirty_files=dirty_files, commit=commit, origin_commit=get_git_commit(branch)) except subprocess.CalledProcessError: pass dirty = dirty or (not comma_remote) dirty = dirty or ('master' in branch) except subprocess.CalledProcessError: dirty = True cloudlog.exception("git subprocess failed while checking dirty") if __name__ == "__main__": print("Dirty: %s" % dirty) print("Version: %s" % version) print("Remote: %s" % origin) print("Branch: %s" % branch) print("Prebuilt: %s" % prebuilt)
def register(show_spinner=False) -> Optional[str]: params = Params() params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id: Optional[str] = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) pubkey = Path(PERSIST + "/comma/id_rsa.pub") if not pubkey.is_file(): dongle_id = UNREGISTERED_DONGLE_ID cloudlog.warning(f"missing public key: {pubkey}") elif needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2: public_key = f1.read() private_key = f2.read() # Block until we get the imei serial = HARDWARE.get_serial() start_time = time.monotonic() imei1: Optional[str] = None imei2: Optional[str] = None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) params.put("IMEI", imei1) params.put("HardwareSerial", serial) backoff = 0 start_time = time.monotonic() while True: try: register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code in (402, 403): cloudlog.info( f"Unable to register device, got {resp.status_code}") dongle_id = UNREGISTERED_DONGLE_ID else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] break except Exception: cloudlog.exception("failed to authenticate") backoff = min(backoff + 1, 15) time.sleep(backoff) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) if show_spinner: spinner.close() if dongle_id: params.put("DongleId", dongle_id) set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC) return dongle_id
def register(show_spinner=False) -> str: params = Params() if "commadotai" in API_HOST and (Params().get_bool("dp_jetson") or Params().get_bool("dp_atl")): return UNREGISTERED_DONGLE_ID if not params.get_bool('dp_reg'): return UNREGISTERED_DONGLE_ID params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST+"/comma") assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa") os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub") if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly with open(PERSIST+"/comma/id_rsa.pub") as f1, open(PERSIST+"/comma/id_rsa") as f2: public_key = f1.read() private_key = f2.read() # Block until we get the imei serial = HARDWARE.get_serial() start_time = time.monotonic() imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) if time.monotonic() - start_time > 60 and show_spinner: spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})") params.put("IMEI", imei1) params.put("HardwareSerial", serial) backoff = 0 start_time = time.monotonic() while True: try: register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code in (402, 403): cloudlog.info(f"Unable to register device, got {resp.status_code}") dongle_id = UNREGISTERED_DONGLE_ID else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] break except Exception: cloudlog.exception("failed to authenticate") return UNREGISTERED_DONGLE_ID # backoff = min(backoff + 1, 15) # time.sleep(backoff) # if time.monotonic() - start_time > 60 and show_spinner: # spinner.update(f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})") if show_spinner: spinner.close() if dongle_id: params.put("DongleId", dongle_id) set_offroad_alert("Offroad_UnofficialHardware", dongle_id == UNREGISTERED_DONGLE_ID) return dongle_id
def thermald_thread(): # prevent LEECO from undervoltage BATT_PERC_OFF = 100 health_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected health frequency # now loop thermal_sock = messaging.pub_sock('thermal') health_sock = messaging.sub_sock('health', timeout=health_timeout) location_sock = messaging.sub_sock('gpsLocation') ignition = False fan_speed = 0 count = 0 off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green thermal_status_prev = ThermalStatus.green usb_power = True usb_power_prev = True current_branch = get_git_branch() network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) health_prev = None fw_version_match_prev = True charging_disabled = False current_update_alert = None time_valid_prev = True should_start_prev = False handle_fan = None is_uno = False has_relay = False params = Params() pm = PowerMonitoring() no_panda_cnt = 0 IsOpenpilotViewEnabled = 0 ts_last_ip = 0 ip_addr = '255.255.255.255' # sound trigger sound_trigger = 1 opkrAutoShutdown = 0 env = dict(os.environ) env['LD_LIBRARY_PATH'] = mediaplayer getoff_alert = Params().get('OpkrEnableGetoffAlert') == b'1' params = Params() if int(params.get('OpkrAutoShutdown')) == 0: opkrAutoShutdown = 0 elif int(params.get('OpkrAutoShutdown')) == 1: opkrAutoShutdown = 5 elif int(params.get('OpkrAutoShutdown')) == 2: opkrAutoShutdown = 30 elif int(params.get('OpkrAutoShutdown')) == 3: opkrAutoShutdown = 60 elif int(params.get('OpkrAutoShutdown')) == 4: opkrAutoShutdown = 180 elif int(params.get('OpkrAutoShutdown')) == 5: opkrAutoShutdown = 300 elif int(params.get('OpkrAutoShutdown')) == 6: opkrAutoShutdown = 600 elif int(params.get('OpkrAutoShutdown')) == 7: opkrAutoShutdown = 1800 elif int(params.get('OpkrAutoShutdown')) == 8: opkrAutoShutdown = 3600 elif int(params.get('OpkrAutoShutdown')) == 9: opkrAutoShutdown = 10800 else: opkrAutoShutdown = 18000 while 1: ts = sec_since_boot() health = messaging.recv_sock(health_sock, wait=True) location = messaging.recv_sock(location_sock) location = location.gpsLocation if location else None msg = read_thermal() if health is not None: usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client ignition = health.health.ignitionLine or health.health.ignitionCan # Setup fan handler on first connect to panda if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown: is_uno = health.health.hwType == log.HealthData.HwType.uno if is_uno or not ANDROID: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if health_prev is not None: if health.health.hwType == log.HealthData.HwType.unknown and \ health_prev.health.hwType != log.HealthData.HwType.unknown: params.panda_disconnect() health_prev = health # If we lose connection to the panda, wait 5 seconds before going offroad if health.health.hwType == log.HealthData.HwType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if ignition: cloudlog.error("Lost panda connection while onroad") ignition = False else: no_panda_cnt = 0 ignition = health.health.ignitionLine or health.health.ignitionCan # Setup fan handler on first connect to panda if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown: is_uno = health.health.hwType == log.HealthData.HwType.uno has_relay = health.health.hwType in [log.HealthData.HwType.blackPanda, log.HealthData.HwType.uno, log.HealthData.HwType.dos] if is_uno or not ANDROID: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if health_prev is not None: if health.health.hwType == log.HealthData.HwType.unknown and \ health_prev.health.hwType != log.HealthData.HwType.unknown: params.panda_disconnect() health_prev = health elif ignition == False or IsOpenpilotViewEnabled: IsOpenpilotViewEnabled = int( params.get("IsOpenpilotViewEnabled") ) ignition = IsOpenpilotViewEnabled # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = get_network_type() network_strength = get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0 msg.thermal.memUsedPercent = int(round(psutil.virtual_memory().percent)) msg.thermal.cpuPerc = int(round(psutil.cpu_percent())) msg.thermal.networkType = network_type msg.thermal.networkStrength = network_strength msg.thermal.batteryPercent = get_battery_capacity() msg.thermal.batteryStatus = get_battery_status() msg.thermal.batteryCurrent = get_battery_current() msg.thermal.batteryVoltage = get_battery_voltage() msg.thermal.usbOnline = get_usb_present() # Fake battery levels on uno for frame if is_uno: msg.thermal.batteryPercent = 100 msg.thermal.batteryStatus = "Charging" msg.thermal.bat = 0 # update ip every 10 seconds ts = sec_since_boot() if ts - ts_last_ip >= 10.: try: result = subprocess.check_output(["ifconfig", "wlan0"], encoding='utf8') # pylint: disable=unexpected-keyword-arg ip_addr = re.findall(r"inet addr:((\d+\.){3}\d+)", result)[0][0] except: ip_addr = 'N/A' ts_last_ip = ts msg.thermal.ipAddr = ip_addr current_filter.update(msg.thermal.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update( max(msg.thermal.cpu0, msg.thermal.cpu1, msg.thermal.cpu2, msg.thermal.cpu3) / 10.0) max_comp_temp = max(max_cpu_temp, msg.thermal.mem / 10., msg.thermal.gpu / 10.) bat_temp = msg.thermal.bat / 1000. if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, ignition) msg.thermal.fanSpeed = fan_speed # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 # We only do this if there is a relay that prevents the car from faulting if max_cpu_temp > 107. or bat_temp >= 63. or (has_relay and (started_ts is None) and max_cpu_temp > 70.0): # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 96.0 or bat_temp > 60.: # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 94.0: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: # all good thermal_status = ThermalStatus.green # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert time_valid = now.year >= 2019 if time_valid and not time_valid_prev: set_offroad_alert("Offroad_InvalidTime", False) if not time_valid and time_valid_prev: set_offroad_alert("Offroad_InvalidTime", True) time_valid_prev = time_valid # Show update prompt # try: # last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) # except (TypeError, ValueError): # last_update = now # dt = now - last_update # update_failed_count = params.get("UpdateFailedCount") # update_failed_count = 0 if update_failed_count is None else int(update_failed_count) # last_update_exception = params.get("LastUpdateException", encoding='utf8') # if update_failed_count > 15 and last_update_exception is not None: # if current_branch in ["release2", "dashcam"]: # extra_text = "Ensure the software is correctly installed" # else: # extra_text = last_update_exception # if current_update_alert != "update" + extra_text: # current_update_alert = "update" + extra_text # set_offroad_alert("Offroad_ConnectivityNeeded", False) # set_offroad_alert("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert("Offroad_UpdateFailed", True, extra_text=extra_text) # elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: # if current_update_alert != "expired": # current_update_alert = "expired" # set_offroad_alert("Offroad_UpdateFailed", False) # set_offroad_alert("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert("Offroad_ConnectivityNeeded", True) # elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: # remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) # if current_update_alert != "prompt" + remaining_time: # current_update_alert = "prompt" + remaining_time # set_offroad_alert("Offroad_UpdateFailed", False) # set_offroad_alert("Offroad_ConnectivityNeeded", False) # set_offroad_alert("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.") # elif current_update_alert is not None: # current_update_alert = None # set_offroad_alert("Offroad_UpdateFailed", False) # set_offroad_alert("Offroad_ConnectivityNeeded", False) # set_offroad_alert("Offroad_ConnectivityNeededPrompt", False) do_uninstall = params.get("DoUninstall") == b"1" accepted_terms = params.get("HasAcceptedTerms") == terms_version completed_training = params.get("CompletedTrainingVersion") == training_version panda_signature = params.get("PandaFirmware") fw_version_match = (panda_signature is None) or (panda_signature == FW_SIGNATURE) # don't show alert is no panda is connected (None) should_start = ignition # with 2% left, we killall, otherwise the phone will take a long time to boot should_start = should_start and msg.thermal.freeSpace > 0.02 # confirm we have completed training and aren't uninstalling should_start = should_start and accepted_terms and completed_training and (not do_uninstall) # check for firmware mismatch #should_start = should_start and fw_version_match # check if system time is valid should_start = should_start and time_valid # don't start while taking snapshot if not should_start_prev: is_viewing_driver = params.get("IsDriverViewEnabled") == b"1" is_taking_snapshot = params.get("IsTakingSnapshot") == b"1" should_start = should_start and (not is_taking_snapshot) and (not is_viewing_driver) if fw_version_match and not fw_version_match_prev: set_offroad_alert("Offroad_PandaFirmwareMismatch", False) if not fw_version_match and fw_version_match_prev: set_offroad_alert("Offroad_PandaFirmwareMismatch", True) # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 if thermal_status >= ThermalStatus.danger: should_start = False if thermal_status_prev < ThermalStatus.danger: set_offroad_alert("Offroad_TemperatureTooHigh", True) else: if thermal_status_prev >= ThermalStatus.danger: set_offroad_alert("Offroad_TemperatureTooHigh", False) if should_start: if not should_start_prev: params.delete("IsOffroad") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True os.system('echo performance > /sys/class/devfreq/soc:qcom,cpubw/governor') else: if should_start_prev or (count == 0): put_nonblocking("IsOffroad", "1") started_ts = None if off_ts is None: off_ts = sec_since_boot() os.system('echo powersave > /sys/class/devfreq/soc:qcom,cpubw/governor') if sound_trigger == 1 and msg.thermal.batteryStatus == "Discharging" and started_seen and (sec_since_boot() - off_ts) > 1 and getoff_alert: subprocess.Popen([mediaplayer + 'mediaplayer', '/data/openpilot/selfdrive/assets/sounds/eondetach.wav'], shell = False, stdin=None, stdout=None, stderr=None, env = env, close_fds=True) sound_trigger = 0 # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for # more than a minute but we were running if msg.thermal.batteryPercent <= BATT_PERC_OFF and msg.thermal.batteryStatus == "Discharging" and \ started_seen and opkrAutoShutdown and (sec_since_boot() - off_ts) > opkrAutoShutdown: os.system('LD_LIBRARY_PATH="" svc power shutdown') charging_disabled = check_car_battery_voltage(should_start, health, charging_disabled, msg) if msg.thermal.batteryCurrent > 0: msg.thermal.batteryStatus = "Discharging" else: msg.thermal.batteryStatus = "Charging" msg.thermal.chargingDisabled = charging_disabled prebuiltlet = Params().get('PutPrebuiltOn') == b'1' if not os.path.isfile(prebuiltfile) and prebuiltlet: os.system("cd /data/openpilot; touch prebuilt") elif os.path.isfile(prebuiltfile) and not prebuiltlet: os.system("cd /data/openpilot; rm -f prebuilt") # Offroad power monitoring pm.calculate(health) msg.thermal.offroadPowerUsage = pm.get_power_used() msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90 # if current is positive, then battery is being discharged msg.thermal.started = started_ts is not None msg.thermal.startedTs = int(1e9*(started_ts or 0)) msg.thermal.thermalStatus = thermal_status thermal_sock.send(msg.to_bytes()) if usb_power_prev and not usb_power: set_offroad_alert("Offroad_ChargeDisabled", True) elif usb_power and not usb_power_prev: set_offroad_alert("Offroad_ChargeDisabled", False) thermal_status_prev = thermal_status usb_power_prev = usb_power fw_version_match_prev = fw_version_match should_start_prev = should_start print(msg) # report to server once per minute if (count % int(60. / DT_TRML)) == 0: cloudlog.event("STATUS_PACKET", count=count, health=(health.to_dict() if health else None), location=(location.to_dict() if location else None), thermal=msg.to_dict()) count += 1
from selfdrive.swaglog import cloudlog with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "common", "version.h")) as _versionf: version = _versionf.read().split('"')[1] try: origin = subprocess.check_output(["git", "config", "--get", "remote.origin.url"]).rstrip() if origin.startswith('[email protected]:commaai') or origin.startswith('https://github.com/commaai'): if origin.endswith('/one.git'): dirty = True else: branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).rstrip() branch = 'origin/' + branch dirty = subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0 if dirty: dirty_files = subprocess.check_output(["git", "diff-index", branch, "--"]) commit = subprocess.check_output(["git", "rev-parse", "--verify", "HEAD"]).rstrip() origin_commit = subprocess.check_output(["git", "rev-parse", "--verify", branch]).rstrip() cloudlog.event("dirty comma branch", vesion=version, dirty=dirty, origin=origin, branch=branch, dirty_files=dirty_files, commit=commit, origin_commit=origin_commit) else: dirty = True except subprocess.CalledProcessError: try: cloudlog.exception("git subprocess failed while finding version") except: pass dirty = True # put this here training_version = "0.1.0"
def register(): params = Params() params.put("Version", version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_git_commit(default="")) params.put("GitBranch", get_git_branch(default="")) params.put("GitRemote", get_git_remote(default="")) params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") # make key readable by app users (ai.comma.plus.offroad) os.chmod(PERSIST + '/comma/', 0o755) os.chmod(PERSIST + '/comma/id_rsa', 0o744) dongle_id = params.get("DongleId", encoding='utf8') public_key = open(PERSIST + "/comma/id_rsa.pub").read() # create registration token # in the future, this key will make JWTs directly private_key = open(PERSIST + "/comma/id_rsa").read() # late import import jwt register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') try: cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=HARDWARE.get_imei(0), imei2=HARDWARE.get_imei(1), serial=HARDWARE.get_serial(), public_key=public_key, register_token=register_token) dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] params.put("DongleId", dongle_id) return dongle_id except Exception: cloudlog.exception("failed to authenticate") if dongle_id is not None: return dongle_id else: return None
def upload_handler(end_event: threading.Event) -> None: sm = messaging.SubMaster(['deviceState']) tid = threading.get_ident() while not end_event.is_set(): cur_upload_items[tid] = None try: cur_upload_items[tid] = upload_queue.get(timeout=1)._replace( current=True) if cur_upload_items[tid].id in cancelled_uploads: cancelled_uploads.remove(cur_upload_items[tid].id) continue # Remove item if too old age = datetime.now() - datetime.fromtimestamp( cur_upload_items[tid].created_at / 1000) if age.total_seconds() > MAX_AGE: cloudlog.event("athena.upload_handler.expired", item=cur_upload_items[tid], error=True) continue # Check if uploading over metered connection is allowed sm.update(0) metered = sm['deviceState'].networkMetered network_type = sm['deviceState'].networkType.raw if metered and (not cur_upload_items[tid].allow_cellular): retry_upload(tid, end_event, False) continue try: def cb(sz, cur): # Abort transfer if connection changed to metered after starting upload sm.update(0) metered = sm['deviceState'].networkMetered if metered and (not cur_upload_items[tid].allow_cellular): raise AbortTransferException cur_upload_items[tid] = cur_upload_items[tid]._replace( progress=cur / sz if sz else 1) fn = cur_upload_items[tid].path try: sz = os.path.getsize(fn) except OSError: sz = -1 cloudlog.event("athena.upload_handler.upload_start", fn=fn, sz=sz, network_type=network_type, metered=metered, retry_count=cur_upload_items[tid].retry_count) response = _do_upload(cur_upload_items[tid], cb) if response.status_code not in (200, 201, 401, 403, 412): cloudlog.event("athena.upload_handler.retry", status_code=response.status_code, fn=fn, sz=sz, network_type=network_type, metered=metered) retry_upload(tid, end_event) else: cloudlog.event("athena.upload_handler.success", fn=fn, sz=sz, network_type=network_type, metered=metered) UploadQueueCache.cache(upload_queue) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.SSLError): cloudlog.event("athena.upload_handler.timeout", fn=fn, sz=sz, network_type=network_type, metered=metered) retry_upload(tid, end_event) except AbortTransferException: cloudlog.event("athena.upload_handler.abort", fn=fn, sz=sz, network_type=network_type, metered=metered) retry_upload(tid, end_event, False) except queue.Empty: pass except Exception: cloudlog.exception("athena.upload_handler.exception")
def thermald_thread(): pm = messaging.PubMaster(['deviceState']) pandaState_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected pandaState frequency pandaState_sock = messaging.sub_sock('pandaState', timeout=pandaState_timeout) location_sock = messaging.sub_sock('gpsLocationExternal') managerState_sock = messaging.sub_sock('managerState', conflate=True) fan_speed = 0 count = 0 startup_conditions = { "ignition": False, } startup_conditions_prev = startup_conditions.copy() off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green usb_power = True current_branch = get_git_branch() network_type = NetworkType.none network_strength = NetworkStrength.unknown wifiIpAddress = 'N/A' current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) pandaState_prev = None charging_disabled = False should_start_prev = False handle_fan = None is_uno = False ui_running_prev = False params = Params() power_monitor = PowerMonitoring() no_panda_cnt = 0 thermal_config = HARDWARE.get_thermal_config() # CPR3 logging if EON: base_path = "/sys/kernel/debug/cpr3-regulator/" cpr_files = [p for p in Path(base_path).glob("**/*") if p.is_file()] cpr_files = ["/sys/kernel/debug/regulator/pm8994_s11/voltage" ] + cpr_files cpr_data = {} for cf in cpr_files: with open(cf, "r") as f: try: cpr_data[str(cf)] = f.read().strip() except Exception: pass cloudlog.event("CPR", data=cpr_data) # sound trigger sound_trigger = 1 opkrAutoShutdown = 0 shutdown_trigger = 1 is_openpilot_view_enabled = 0 env = dict(os.environ) env['LD_LIBRARY_PATH'] = mediaplayer getoff_alert = params.get_bool("OpkrEnableGetoffAlert") hotspot_on_boot = params.get_bool("OpkrHotspotOnBoot") hotspot_run = False if int(params.get("OpkrAutoShutdown", encoding="utf8")) == 0: opkrAutoShutdown = 0 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 1: opkrAutoShutdown = 5 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 2: opkrAutoShutdown = 30 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 3: opkrAutoShutdown = 60 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 4: opkrAutoShutdown = 180 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 5: opkrAutoShutdown = 300 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 6: opkrAutoShutdown = 600 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 7: opkrAutoShutdown = 1800 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 8: opkrAutoShutdown = 3600 elif int(params.get("OpkrAutoShutdown", encoding="utf8")) == 9: opkrAutoShutdown = 10800 else: opkrAutoShutdown = 18000 while 1: ts = sec_since_boot() pandaState = messaging.recv_sock(pandaState_sock, wait=True) msg = read_thermal(thermal_config) if pandaState is not None: usb_power = pandaState.pandaState.usbPowerMode != log.PandaState.UsbPowerMode.client # If we lose connection to the panda, wait 5 seconds before going offroad if pandaState.pandaState.pandaType == log.PandaState.PandaType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if startup_conditions["ignition"]: cloudlog.error("Lost panda connection while onroad") startup_conditions["ignition"] = False shutdown_trigger = 1 else: no_panda_cnt = 0 startup_conditions[ "ignition"] = pandaState.pandaState.ignitionLine or pandaState.pandaState.ignitionCan sound_trigger == 1 #startup_conditions["hardware_supported"] = pandaState.pandaState.pandaType not in [log.PandaState.PandaType.whitePanda, # log.PandaState.PandaType.greyPanda] #set_offroad_alert_if_changed("Offroad_HardwareUnsupported", not startup_conditions["hardware_supported"]) # Setup fan handler on first connect to panda if handle_fan is None and pandaState.pandaState.pandaType != log.PandaState.PandaType.unknown: is_uno = pandaState.pandaState.pandaType == log.PandaState.PandaType.uno if (not EON) or is_uno: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if pandaState_prev is not None: if pandaState.pandaState.pandaType == log.PandaState.PandaType.unknown and \ pandaState_prev.pandaState.pandaType != log.PandaState.PandaType.unknown: params.clear_all(ParamKeyType.CLEAR_ON_PANDA_DISCONNECT) pandaState_prev = pandaState elif params.get_bool("IsOpenpilotViewEnabled") and not params.get_bool( "IsDriverViewEnabled") and is_openpilot_view_enabled == 0: is_openpilot_view_enabled = 1 startup_conditions["ignition"] = True elif not params.get_bool( "IsOpenpilotViewEnabled") and not params.get_bool( "IsDriverViewEnabled") and is_openpilot_view_enabled == 1: shutdown_trigger = 0 sound_trigger == 0 is_openpilot_view_enabled = 0 startup_conditions["ignition"] = False # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() network_strength = HARDWARE.get_network_strength(network_type) wifiIpAddress = HARDWARE.get_ip_address() except Exception: cloudlog.exception("Error getting network status") msg.deviceState.freeSpacePercent = get_available_percent(default=100.0) msg.deviceState.memoryUsagePercent = int( round(psutil.virtual_memory().percent)) msg.deviceState.cpuUsagePercent = int(round(psutil.cpu_percent())) msg.deviceState.networkType = network_type msg.deviceState.networkStrength = network_strength msg.deviceState.wifiIpAddress = wifiIpAddress msg.deviceState.batteryPercent = HARDWARE.get_battery_capacity() msg.deviceState.batteryStatus = HARDWARE.get_battery_status() msg.deviceState.batteryCurrent = HARDWARE.get_battery_current() msg.deviceState.batteryVoltage = HARDWARE.get_battery_voltage() msg.deviceState.usbOnline = HARDWARE.get_usb_present() # Fake battery levels on uno for frame if (not EON) or is_uno: msg.deviceState.batteryPercent = 100 msg.deviceState.batteryStatus = "Charging" msg.deviceState.batteryTempC = 0 current_filter.update(msg.deviceState.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update(max(msg.deviceState.cpuTempC)) max_comp_temp = max(max_cpu_temp, msg.deviceState.memoryTempC, max(msg.deviceState.gpuTempC)) bat_temp = msg.deviceState.batteryTempC if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, startup_conditions["ignition"]) msg.deviceState.fanSpeedPercentDesired = fan_speed # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 # We only do this if there is a relay that prevents the car from faulting is_offroad_for_5_min = (started_ts is None) and ( (not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if max_cpu_temp > 107. or bat_temp >= 63. or (is_offroad_for_5_min and max_cpu_temp > 70.0): # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 96.0 or bat_temp > 60.: # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 94.0: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: thermal_status = ThermalStatus.green # default to good condition # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert startup_conditions["time_valid"] = True if ( (now.year > 2020) or (now.year == 2020 and now.month >= 10) ) else True # set True for battery less EON otherwise, set False. set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) # Show update prompt # try: # last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) # except (TypeError, ValueError): # last_update = now # dt = now - last_update # update_failed_count = params.get("UpdateFailedCount") # update_failed_count = 0 if update_failed_count is None else int(update_failed_count) # last_update_exception = params.get("LastUpdateException", encoding='utf8') # if update_failed_count > 15 and last_update_exception is not None: # if current_branch in ["release2", "dashcam"]: # extra_text = "Ensure the software is correctly installed" # else: # extra_text = last_update_exception # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text) # elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True) # elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: # remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.") # else: # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) #startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") startup_conditions["not_uninstalling"] = not params.get_bool( "DoUninstall") startup_conditions["accepted_terms"] = params.get( "HasAcceptedTerms") == terms_version panda_signature = params.get("PandaFirmware") startup_conditions["fw_version_match"] = (panda_signature is None) or ( panda_signature == FW_SIGNATURE ) # don't show alert is no panda is connected (None) set_offroad_alert_if_changed( "Offroad_PandaFirmwareMismatch", (not startup_conditions["fw_version_match"])) # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ (current_branch in ['dashcam', 'dashcam-staging']) startup_conditions["not_driver_view"] = not params.get_bool( "IsDriverViewEnabled") startup_conditions["not_taking_snapshot"] = not params.get_bool( "IsTakingSnapshot") # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 startup_conditions[ "device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed( "Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"])) # Handle offroad/onroad transition should_start = all(startup_conditions.values()) if should_start != should_start_prev or (count == 0): params.put_bool("IsOffroad", not should_start) HARDWARE.set_power_save(not should_start) if TICI and not params.get_bool("EnableLteOnroad"): fxn = "off" if should_start else "on" os.system(f"nmcli radio wwan {fxn}") if should_start: off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions) started_ts = None if off_ts is None: off_ts = sec_since_boot() if shutdown_trigger == 1 and sound_trigger == 1 and msg.deviceState.batteryStatus == "Discharging" and started_seen and ( sec_since_boot() - off_ts) > 1 and getoff_alert: subprocess.Popen([ mediaplayer + 'mediaplayer', '/data/openpilot/selfdrive/assets/sounds/eondetach.wav' ], shell=False, stdin=None, stdout=None, stderr=None, env=env, close_fds=True) sound_trigger = 0 # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for # more than a minute but we were running if shutdown_trigger == 1 and msg.deviceState.batteryStatus == "Discharging" and \ started_seen and opkrAutoShutdown and (sec_since_boot() - off_ts) > opkrAutoShutdown and not os.path.isfile(pandaflash_ongoing): os.system('LD_LIBRARY_PATH="" svc power shutdown') charging_disabled = check_car_battery_voltage(should_start, pandaState, charging_disabled, msg) if msg.deviceState.batteryCurrent > 0: msg.deviceState.batteryStatus = "Discharging" else: msg.deviceState.batteryStatus = "Charging" msg.deviceState.chargingDisabled = charging_disabled prebuiltlet = params.get_bool("PutPrebuiltOn") if not os.path.isfile(prebuiltfile) and prebuiltlet: os.system("cd /data/openpilot; touch prebuilt") elif os.path.isfile(prebuiltfile) and not prebuiltlet: os.system("cd /data/openpilot; rm -f prebuilt") sshkeylet = params.get_bool("OpkrSSHLegacy") if not os.path.isfile(sshkeyfile) and sshkeylet: os.system( "cp -f /data/openpilot/selfdrive/assets/addon/key/GithubSshKeys_legacy /data/params/d/GithubSshKeys; chmod 600 /data/params/d/GithubSshKeys; touch /data/public_key" ) elif os.path.isfile(sshkeyfile) and not sshkeylet: os.system( "cp -f /data/openpilot/selfdrive/assets/addon/key/GithubSshKeys_new /data/params/d/GithubSshKeys; chmod 600 /data/params/d/GithubSshKeys; rm -f /data/public_key" ) # opkr hotspot if hotspot_on_boot and not hotspot_run and sec_since_boot() > 120: os.system("service call wifi 37 i32 0 i32 1 &") hotspot_run = True # Offroad power monitoring power_monitor.calculate(pandaState) msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used() msg.deviceState.carBatteryCapacityUwh = max( 0, power_monitor.get_car_battery_capacity()) # # Check if we need to disable charging (handled by boardd) # msg.deviceState.chargingDisabled = power_monitor.should_disable_charging(pandaState, off_ts) # # # Check if we need to shut down # if power_monitor.should_shutdown(pandaState, off_ts, started_seen): # cloudlog.info(f"shutting device down, offroad since {off_ts}") # # TODO: add function for blocking cloudlog instead of sleep # time.sleep(10) # HARDWARE.shutdown() # If UI has crashed, set the brightness to reasonable non-zero value manager_state = messaging.recv_one_or_none(managerState_sock) if manager_state is not None: ui_running = "ui" in (p.name for p in manager_state.managerState.processes if p.running) if ui_running_prev and not ui_running: HARDWARE.set_screen_brightness(20) ui_running_prev = ui_running msg.deviceState.chargingError = current_filter.x > 0. and msg.deviceState.batteryPercent < 90 # if current is positive, then battery is being discharged msg.deviceState.started = started_ts is not None msg.deviceState.startedMonoTime = int(1e9 * (started_ts or 0)) msg.deviceState.thermalStatus = thermal_status pm.send("deviceState", msg) if EON and not is_uno: set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power)) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # report to server once every 10 minutes if (count % int(600. / DT_TRML)) == 0: if EON and started_ts is None and msg.deviceState.memoryUsagePercent > 40: cloudlog.event("High offroad memory usage", mem=msg.deviceState.memoryUsagePercent) location = messaging.recv_sock(location_sock) cloudlog.event("STATUS_PACKET", count=count, pandaState=(strip_deprecated_keys( pandaState.to_dict()) if pandaState else None), location=(strip_deprecated_keys( location.gpsLocationExternal.to_dict()) if location else None), deviceState=strip_deprecated_keys(msg.to_dict())) count += 1
def upload_handler(end_event: threading.Event) -> None: sm = messaging.SubMaster(['deviceState']) tid = threading.get_ident() while not end_event.is_set(): cur_upload_items[tid] = None try: cur_upload_items[tid] = upload_queue.get(timeout=1)._replace( current=True) if cur_upload_items[tid].id in cancelled_uploads: cancelled_uploads.remove(cur_upload_items[tid].id) continue # TODO: remove item if too old # Check if uploading over cell is allowed sm.update(0) cell = sm['deviceState'].networkType not in [ NetworkType.wifi, NetworkType.ethernet ] if cell and (not cur_upload_items[tid].allow_cellular): retry_upload(tid, end_event, False) continue try: def cb(sz, cur): # Abort transfer if connection changed to cell after starting upload sm.update(0) cell = sm['deviceState'].networkType not in [ NetworkType.wifi, NetworkType.ethernet ] if cell and (not cur_upload_items[tid].allow_cellular): raise AbortTransferException cur_upload_items[tid] = cur_upload_items[tid]._replace( progress=cur / sz if sz else 1) response = _do_upload(cur_upload_items[tid], cb) if response.status_code not in (200, 201, 403, 412): cloudlog.warning( f"athena.upload_handler.retry {response.status_code} {cur_upload_items[tid]}" ) retry_upload(tid, end_event) UploadQueueCache.cache(upload_queue) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.SSLError) as e: cloudlog.warning( f"athena.upload_handler.retry {e} {cur_upload_items[tid]}") retry_upload(tid, end_event) except AbortTransferException: cloudlog.warning( f"athena.upload_handler.abort {cur_upload_items[tid]}") retry_upload(tid, end_event, False) except queue.Empty: pass except Exception: cloudlog.exception("athena.upload_handler.exception")
def main(): params = Params() if params.get("DisableUpdates") == b"1": raise RuntimeError("updates are disabled by the DisableUpdates param") if os.geteuid() != 0: raise RuntimeError("updated must be launched as root!") # Set low io priority proc = psutil.Process() if psutil.LINUX: proc.ionice(psutil.IOPRIO_CLASS_BE, value=7) ov_lock_fd = open(LOCK_FILE, 'w') try: fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: raise RuntimeError( "couldn't get overlay lock; is another updated running?") # Wait for IsOffroad to be set before our first update attempt wait_helper = WaitTimeHelper(proc) wait_helper.sleep(30) update_failed_count = 0 update_available = False overlay_initialized = False while not wait_helper.shutdown: wait_helper.ready_event.clear() # Check for internet every 30s time_wrong = datetime.datetime.utcnow().year < 2019 ping_failed = os.system(f"ping -W 4 -c 1 {TEST_IP}") != 0 if ping_failed or time_wrong: wait_helper.sleep(30) continue # Attempt an update exception = None update_failed_count += 1 try: # Re-create the overlay if BASEDIR/.git has changed since we created the overlay if overlay_initialized: overlay_init_fn = os.path.join(BASEDIR, ".overlay_init") git_dir_path = os.path.join(BASEDIR, ".git") new_files = run( ["find", git_dir_path, "-newer", overlay_init_fn]) if len(new_files.splitlines()): cloudlog.info(".git directory changed, recreating overlay") overlay_initialized = False if not overlay_initialized: init_ovfs() overlay_initialized = True if params.get("IsOffroad") == b"1": update_available = attempt_update( wait_helper) or update_available update_failed_count = 0 if not update_available and os.path.isdir(NEOSUPDATE_DIR): shutil.rmtree(NEOSUPDATE_DIR) else: cloudlog.info("not running updater, openpilot running") except subprocess.CalledProcessError as e: cloudlog.event("update process failed", cmd=e.cmd, output=e.output, returncode=e.returncode) exception = e overlay_initialized = False except Exception: cloudlog.exception("uncaught updated exception, shouldn't happen") params.put("UpdateFailedCount", str(update_failed_count)) if exception is None: params.delete("LastUpdateException") else: params.put("LastUpdateException", f"command failed: {exception.cmd}\n{exception.output}") # Wait 10 minutes between update attempts wait_helper.sleep(60 * 10) dismount_ovfs()
def log_handler(end_event): if PC: return log_files = [] last_scan = 0 while not end_event.is_set(): try: curr_scan = sec_since_boot() if curr_scan - last_scan > 10: log_files = get_logs_to_send_sorted() last_scan = curr_scan # send one log curr_log = None if len(log_files) > 0: log_entry = log_files.pop() # newest log file cloudlog.debug( f"athena.log_handler.forward_request {log_entry}") try: curr_time = int(time.time()) log_path = os.path.join(SWAGLOG_DIR, log_entry) setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder)) with open(log_path) as f: jsonrpc = { "method": "forwardLogs", "params": { "logs": f.read() }, "jsonrpc": "2.0", "id": log_entry } low_priority_send_queue.put_nowait(json.dumps(jsonrpc)) curr_log = log_entry except OSError: pass # file could be deleted by log rotation # wait for response up to ~100 seconds # always read queue at least once to process any old responses that arrive for _ in range(100): if end_event.is_set(): break try: log_resp = json.loads(log_recv_queue.get(timeout=1)) log_entry = log_resp.get("id") log_success = "result" in log_resp and log_resp[ "result"].get("success") cloudlog.debug( f"athena.log_handler.forward_response {log_entry} {log_success}" ) if log_entry and log_success: log_path = os.path.join(SWAGLOG_DIR, log_entry) try: setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME) except OSError: pass # file could be deleted by log rotation if curr_log == log_entry: break except queue.Empty: if curr_log is None: break except Exception: cloudlog.exception("athena.log_handler.exception")
def calculate(self, health): try: now = time.time() # Check that time is valid if datetime.datetime.fromtimestamp(now).year < 2019: return # Only integrate when there is no ignition # If health is None, we're probably not in a car, so we don't care if health is None or (health.health.ignitionLine or health.health.ignitionCan) or \ health.health.hwType == log.HealthData.HwType.unknown: with self.integration_lock: self.last_measurement_time = None self.next_pulsed_measurement_time = None self.power_used_uWh = 0 return # First measurement, set integration time with self.integration_lock: if self.last_measurement_time is None: self.last_measurement_time = now return # Get current power draw somehow current_power = 0 if get_battery_status() == 'Discharging': # If the battery is discharging, we can use this measurement # On C2: this is low by about 10-15%, probably mostly due to UNO draw not being factored in current_power = ((get_battery_voltage() / 1000000) * (get_battery_current() / 1000000)) elif (health.health.hwType in [ log.HealthData.HwType.whitePanda, log.HealthData.HwType.greyPanda ]) and (health.health.current > 1): # If white/grey panda, use the integrated current measurements if the measurement is not 0 # If the measurement is 0, the current is 400mA or greater, and out of the measurement range of the panda # This seems to be accurate to about 5% current_power = ( PANDA_OUTPUT_VOLTAGE * panda_current_to_actual_current(health.health.current)) elif (self.next_pulsed_measurement_time is not None) and (self.next_pulsed_measurement_time <= now): # TODO: Figure out why this is off by a factor of 3/4??? FUDGE_FACTOR = 1.33 # Turn off charging for about 10 sec in a thread that does not get killed on SIGINT, and perform measurement here to avoid blocking thermal def perform_pulse_measurement(now): try: set_battery_charging(False) time.sleep(5) # Measure for a few sec to get a good average voltages = [] currents = [] for i in range(6): voltages.append(get_battery_voltage()) currents.append(get_battery_current()) time.sleep(1) current_power = ((mean(voltages) / 1000000) * (mean(currents) / 1000000)) self._perform_integration(now, current_power * FUDGE_FACTOR) # Enable charging again set_battery_charging(True) except Exception: cloudlog.exception("Pulsed power measurement failed") # Start pulsed measurement and return threading.Thread(target=perform_pulse_measurement, args=(now, )).start() self.next_pulsed_measurement_time = None return elif self.next_pulsed_measurement_time is None and not self.is_uno: # On a charging EON with black panda, or drawing more than 400mA out of a white/grey one # Only way to get the power draw is to turn off charging for a few sec and check what the discharging rate is # We shouldn't do this very often, so make sure it has been some long-ish random time interval self.next_pulsed_measurement_time = now + random.randint( 120, 180) return else: # Do nothing return # Do the integration self._perform_integration(now, current_power) except Exception: cloudlog.exception("Power monitoring calculation failed")
def thermald_thread(): pm = messaging.PubMaster(['deviceState']) pandaState_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected pandaState frequency pandaState_sock = messaging.sub_sock('pandaStates', timeout=pandaState_timeout) sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "managerState"]) fan_speed = 0 count = 0 startup_conditions = { "ignition": False, } startup_conditions_prev = startup_conditions.copy() off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green usb_power = True network_type = NetworkType.none network_strength = NetworkStrength.unknown network_info = None modem_version = None registered_count = 0 nvme_temps = None modem_temps = None current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_TRML) pandaState_prev = None should_start_prev = False in_car = False handle_fan = None is_uno = False ui_running_prev = False params = Params() power_monitor = PowerMonitoring() no_panda_cnt = 0 HARDWARE.initialize_hardware() thermal_config = HARDWARE.get_thermal_config() # TODO: use PI controller for UNO controller = PIController(k_p=0, k_i=2e-3, neg_limit=-80, pos_limit=0, rate=(1 / DT_TRML)) # Leave flag for loggerd to indicate device was left onroad if params.get_bool("IsOnroad"): params.put_bool("BootedOnroad", True) while True: pandaStates = messaging.recv_sock(pandaState_sock, wait=True) sm.update(0) peripheralState = sm['peripheralState'] msg = read_thermal(thermal_config) if pandaStates is not None and len(pandaStates.pandaStates) > 0: pandaState = pandaStates.pandaStates[0] # If we lose connection to the panda, wait 5 seconds before going offroad if pandaState.pandaType == log.PandaState.PandaType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if startup_conditions["ignition"]: cloudlog.error("Lost panda connection while onroad") startup_conditions["ignition"] = False else: no_panda_cnt = 0 startup_conditions["ignition"] = pandaState.ignitionLine or pandaState.ignitionCan in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected usb_power = peripheralState.usbPowerMode != log.PeripheralState.UsbPowerMode.client # Setup fan handler on first connect to panda if handle_fan is None and peripheralState.pandaType != log.PandaState.PandaType.unknown: is_uno = peripheralState.pandaType == log.PandaState.PandaType.uno if TICI: cloudlog.info("Setting up TICI fan handler") handle_fan = handle_fan_tici elif is_uno or PC: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if pandaState_prev is not None: if pandaState.pandaType == log.PandaState.PandaType.unknown and \ pandaState_prev.pandaType != log.PandaState.PandaType.unknown: params.clear_all(ParamKeyType.CLEAR_ON_PANDA_DISCONNECT) pandaState_prev = pandaState # these are expensive calls. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() network_strength = HARDWARE.get_network_strength(network_type) network_info = HARDWARE.get_network_info() # pylint: disable=assignment-from-none nvme_temps = HARDWARE.get_nvme_temperatures() modem_temps = HARDWARE.get_modem_temperatures() # Log modem version once if modem_version is None: modem_version = HARDWARE.get_modem_version() # pylint: disable=assignment-from-none if modem_version is not None: cloudlog.warning(f"Modem version: {modem_version}") if TICI and (network_info.get('state', None) == "REGISTERED"): registered_count += 1 else: registered_count = 0 if registered_count > 10: cloudlog.warning(f"Modem stuck in registered state {network_info}. nmcli conn up lte") os.system("nmcli conn up lte") registered_count = 0 except Exception: cloudlog.exception("Error getting network status") msg.deviceState.freeSpacePercent = get_available_percent(default=100.0) msg.deviceState.memoryUsagePercent = int(round(psutil.virtual_memory().percent)) msg.deviceState.cpuUsagePercent = [int(round(n)) for n in psutil.cpu_percent(percpu=True)] msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent())) msg.deviceState.networkType = network_type msg.deviceState.networkStrength = network_strength if network_info is not None: msg.deviceState.networkInfo = network_info if nvme_temps is not None: msg.deviceState.nvmeTempC = nvme_temps if modem_temps is not None: msg.deviceState.modemTempC = modem_temps msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness() msg.deviceState.batteryPercent = HARDWARE.get_battery_capacity() msg.deviceState.batteryCurrent = HARDWARE.get_battery_current() msg.deviceState.usbOnline = HARDWARE.get_usb_present() current_filter.update(msg.deviceState.batteryCurrent / 1e6) max_comp_temp = temp_filter.update( max(max(msg.deviceState.cpuTempC), msg.deviceState.memoryTempC, max(msg.deviceState.gpuTempC)) ) if handle_fan is not None: fan_speed = handle_fan(controller, max_comp_temp, fan_speed, startup_conditions["ignition"]) msg.deviceState.fanSpeedPercentDesired = fan_speed is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if is_offroad_for_5_min and max_comp_temp > OFFROAD_DANGER_TEMP: # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 thermal_status = ThermalStatus.danger else: current_band = THERMAL_BANDS[thermal_status] band_idx = list(THERMAL_BANDS.keys()).index(thermal_status) if current_band.min_temp is not None and max_comp_temp < current_band.min_temp: thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1] elif current_band.max_temp is not None and max_comp_temp > current_band.max_temp: thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1] # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert startup_conditions["time_valid"] = (now.year > 2020) or (now.year == 2020 and now.month >= 10) set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) # Show update prompt try: last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) except (TypeError, ValueError): last_update = now dt = now - last_update update_failed_count = params.get("UpdateFailedCount") update_failed_count = 0 if update_failed_count is None else int(update_failed_count) last_update_exception = params.get("LastUpdateException", encoding='utf8') if update_failed_count > 15 and last_update_exception is not None: if tested_branch: extra_text = "Ensure the software is correctly installed" else: extra_text = last_update_exception set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text) elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True) elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: remaining = max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 1) set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining} day{'' if remaining == 1 else 's'}.") else: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate") startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall") startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ params.get_bool("Passive") startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled") startup_conditions["not_taking_snapshot"] = not params.get_bool("IsTakingSnapshot") # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 startup_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"])) if TICI: set_offroad_alert_if_changed("Offroad_StorageMissing", (not Path("/data/media").is_mount())) # Handle offroad/onroad transition should_start = all(startup_conditions.values()) if should_start != should_start_prev or (count == 0): params.put_bool("IsOnroad", should_start) params.put_bool("IsOffroad", not should_start) HARDWARE.set_power_save(not should_start) if should_start: off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions) started_ts = None if off_ts is None: off_ts = sec_since_boot() # Offroad power monitoring power_monitor.calculate(peripheralState, startup_conditions["ignition"]) msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used() msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity()) # Check if we need to disable charging (handled by boardd) msg.deviceState.chargingDisabled = power_monitor.should_disable_charging(startup_conditions["ignition"], in_car, off_ts) # Check if we need to shut down if power_monitor.should_shutdown(peripheralState, startup_conditions["ignition"], in_car, off_ts, started_seen): cloudlog.info(f"shutting device down, offroad since {off_ts}") # TODO: add function for blocking cloudlog instead of sleep time.sleep(10) HARDWARE.shutdown() # If UI has crashed, set the brightness to reasonable non-zero value ui_running = "ui" in (p.name for p in sm["managerState"].processes if p.running) if ui_running_prev and not ui_running: HARDWARE.set_screen_brightness(20) ui_running_prev = ui_running msg.deviceState.chargingError = current_filter.x > 0. and msg.deviceState.batteryPercent < 90 # if current is positive, then battery is being discharged msg.deviceState.started = started_ts is not None msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0)) last_ping = params.get("LastAthenaPingTime") if last_ping is not None: msg.deviceState.lastAthenaPingTime = int(last_ping) msg.deviceState.thermalStatus = thermal_status pm.send("deviceState", msg) if EON and not is_uno: set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power)) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # report to server once every 10 minutes if (count % int(600. / DT_TRML)) == 0: if EON and started_ts is None and msg.deviceState.memoryUsagePercent > 40: cloudlog.event("High offroad memory usage", mem=msg.deviceState.memoryUsagePercent) cloudlog.event("STATUS_PACKET", count=count, pandaStates=(strip_deprecated_keys(pandaStates.to_dict()) if pandaStates else None), peripheralState=strip_deprecated_keys(peripheralState.to_dict()), location=(strip_deprecated_keys(sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None), deviceState=strip_deprecated_keys(msg.to_dict())) count += 1
def get_expected_signature(): try: return Panda.get_signature_from_firmware(PANDA_FW_FN) except Exception: cloudlog.exception("Error computing expected signature") return b""
def main(): params = Params() if params.get_bool("DisableUpdates"): cloudlog.warning("updates are disabled by the DisableUpdates param") exit(0) ov_lock_fd = open(LOCK_FILE, 'w') try: fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: raise RuntimeError( "couldn't get overlay lock; is another instance running?") from e # Set low io priority proc = psutil.Process() if psutil.LINUX: proc.ionice(psutil.IOPRIO_CLASS_BE, value=7) # Check if we just performed an update if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") if not params.get("InstallDate"): t = datetime.datetime.utcnow().isoformat() params.put("InstallDate", t.encode('utf8')) overlay_init = Path(os.path.join(BASEDIR, ".overlay_init")) overlay_init.unlink(missing_ok=True) first_run = True last_fetch_time = 0 update_failed_count = 0 # Set initial params for offroad alerts set_params(False, 0, None) # Wait for IsOffroad to be set before our first update attempt wait_helper = WaitTimeHelper(proc) wait_helper.sleep(30) # Run the update loop # * every 1m, do a lightweight internet/update check # * every 10m, do a full git fetch while not wait_helper.shutdown: update_now = wait_helper.ready_event.is_set() wait_helper.ready_event.clear() # Don't run updater while onroad or if the time's wrong time_wrong = datetime.datetime.utcnow().year < 2019 is_onroad = not params.get_bool("IsOffroad") if is_onroad or time_wrong: wait_helper.sleep(30) cloudlog.info("not running updater, not offroad") continue # Attempt an update exception = None new_version = False update_failed_count += 1 try: init_overlay() internet_ok, update_available = check_for_update() if internet_ok and not update_available: update_failed_count = 0 # Fetch updates at most every 10 minutes if internet_ok and (update_now or time.monotonic() - last_fetch_time > 60 * 10): new_version = fetch_update(wait_helper) update_failed_count = 0 last_fetch_time = time.monotonic() if first_run and not new_version and os.path.isdir( NEOSUPDATE_DIR): shutil.rmtree(NEOSUPDATE_DIR) first_run = False except subprocess.CalledProcessError as e: cloudlog.event("update process failed", cmd=e.cmd, output=e.output, returncode=e.returncode) exception = f"command failed: {e.cmd}\n{e.output}" overlay_init.unlink(missing_ok=True) except Exception as e: cloudlog.exception("uncaught updated exception, shouldn't happen") exception = str(e) overlay_init.unlink(missing_ok=True) try: set_params(new_version, update_failed_count, exception) except Exception: cloudlog.exception( "uncaught updated exception while setting params, shouldn't happen" ) wait_helper.sleep(60) dismount_overlay()
def thermald_thread(): # prevent LEECO from undervoltage BATT_PERC_OFF = 10 if LEON else 3 health_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected health frequency # now loop thermal_sock = messaging.pub_sock('thermal') health_sock = messaging.sub_sock('health', timeout=health_timeout) location_sock = messaging.sub_sock('gpsLocation') ignition = False fan_speed = 0 count = 0 off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green thermal_status_prev = ThermalStatus.green usb_power = True usb_power_prev = True network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) health_prev = None fw_version_match_prev = True current_connectivity_alert = None time_valid_prev = True should_start_prev = False handle_fan = None is_uno = False params = Params() pm = PowerMonitoring() no_panda_cnt = 0 while 1: health = messaging.recv_sock(health_sock, wait=True) location = messaging.recv_sock(location_sock) location = location.gpsLocation if location else None msg = read_thermal() if health is not None: usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client # If we lose connection to the panda, wait 5 seconds before going offroad if health.health.hwType == log.HealthData.HwType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if ignition: cloudlog.error("Lost panda connection while onroad") ignition = False else: no_panda_cnt = 0 ignition = health.health.ignitionLine or health.health.ignitionCan # Setup fan handler on first connect to panda if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown: is_uno = health.health.hwType == log.HealthData.HwType.uno if is_uno or not ANDROID: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if health_prev is not None: if health.health.hwType == log.HealthData.HwType.unknown and \ health_prev.health.hwType != log.HealthData.HwType.unknown: params.panda_disconnect() health_prev = health # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = get_network_type() network_strength = get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0 msg.thermal.memUsedPercent = int(round( psutil.virtual_memory().percent)) msg.thermal.cpuPerc = int(round(psutil.cpu_percent())) msg.thermal.networkType = network_type msg.thermal.networkStrength = network_strength msg.thermal.batteryPercent = get_battery_capacity() msg.thermal.batteryStatus = get_battery_status() msg.thermal.batteryCurrent = get_battery_current() msg.thermal.batteryVoltage = get_battery_voltage() msg.thermal.usbOnline = get_usb_present() # Fake battery levels on uno for frame if is_uno: msg.thermal.batteryPercent = 100 msg.thermal.batteryStatus = "Charging" current_filter.update(msg.thermal.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update( max(msg.thermal.cpu0, msg.thermal.cpu1, msg.thermal.cpu2, msg.thermal.cpu3) / 10.0) max_comp_temp = max(max_cpu_temp, msg.thermal.mem / 10., msg.thermal.gpu / 10.) bat_temp = msg.thermal.bat / 1000. if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, ignition) msg.thermal.fanSpeed = fan_speed # thermal logic with hysterisis if max_cpu_temp > 107. or bat_temp >= 63.: # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 92.5 or bat_temp > 60.: # CPU throttling starts around ~90C # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 87.5: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: # all good thermal_status = ThermalStatus.green # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert time_valid = now.year >= 2019 if time_valid and not time_valid_prev: params.delete("Offroad_InvalidTime") if not time_valid and time_valid_prev: put_nonblocking("Offroad_InvalidTime", json.dumps(OFFROAD_ALERTS["Offroad_InvalidTime"])) time_valid_prev = time_valid # Show update prompt try: last_update = datetime.datetime.fromisoformat( params.get("LastUpdateTime", encoding='utf8')) except (TypeError, ValueError): last_update = now dt = now - last_update update_failed_count = params.get("UpdateFailedCount") update_failed_count = 0 if update_failed_count is None else int( update_failed_count) if dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: if current_connectivity_alert != "expired": current_connectivity_alert = "expired" params.delete("Offroad_ConnectivityNeededPrompt") put_nonblocking( "Offroad_ConnectivityNeeded", json.dumps(OFFROAD_ALERTS["Offroad_ConnectivityNeeded"])) elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) if current_connectivity_alert != "prompt" + remaining_time: current_connectivity_alert = "prompt" + remaining_time alert_connectivity_prompt = copy.copy( OFFROAD_ALERTS["Offroad_ConnectivityNeededPrompt"]) alert_connectivity_prompt["text"] += remaining_time + " days." params.delete("Offroad_ConnectivityNeeded") put_nonblocking("Offroad_ConnectivityNeededPrompt", json.dumps(alert_connectivity_prompt)) elif current_connectivity_alert is not None: current_connectivity_alert = None params.delete("Offroad_ConnectivityNeeded") params.delete("Offroad_ConnectivityNeededPrompt") do_uninstall = params.get("DoUninstall") == b"1" accepted_terms = params.get("HasAcceptedTerms") == terms_version completed_training = params.get( "CompletedTrainingVersion") == training_version panda_signature = params.get("PandaFirmware") fw_version_match = (panda_signature is None) or ( panda_signature == FW_SIGNATURE ) # don't show alert is no panda is connected (None) should_start = ignition # with 2% left, we killall, otherwise the phone will take a long time to boot should_start = should_start and msg.thermal.freeSpace > 0.02 # confirm we have completed training and aren't uninstalling should_start = should_start and accepted_terms and completed_training and ( not do_uninstall) # check for firmware mismatch should_start = should_start and fw_version_match # check if system time is valid should_start = should_start and time_valid # don't start while taking snapshot if not should_start_prev: is_viewing_driver = params.get("IsDriverViewEnabled") == b"1" is_taking_snapshot = params.get("IsTakingSnapshot") == b"1" should_start = should_start and (not is_taking_snapshot) and ( not is_viewing_driver) if fw_version_match and not fw_version_match_prev: params.delete("Offroad_PandaFirmwareMismatch") if not fw_version_match and fw_version_match_prev: put_nonblocking( "Offroad_PandaFirmwareMismatch", json.dumps(OFFROAD_ALERTS["Offroad_PandaFirmwareMismatch"])) # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 if thermal_status >= ThermalStatus.danger: should_start = False if thermal_status_prev < ThermalStatus.danger: put_nonblocking( "Offroad_TemperatureTooHigh", json.dumps(OFFROAD_ALERTS["Offroad_TemperatureTooHigh"])) else: if thermal_status_prev >= ThermalStatus.danger: params.delete("Offroad_TemperatureTooHigh") if should_start: if not should_start_prev: params.delete("IsOffroad") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True os.system( 'echo performance > /sys/class/devfreq/soc:qcom,cpubw/governor' ) else: if should_start_prev or (count == 0): put_nonblocking("IsOffroad", "1") started_ts = None if off_ts is None: off_ts = sec_since_boot() os.system( 'echo powersave > /sys/class/devfreq/soc:qcom,cpubw/governor' ) # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for # more than a minute but we were running if msg.thermal.batteryPercent < BATT_PERC_OFF and msg.thermal.batteryStatus == "Discharging" and \ started_seen and (sec_since_boot() - off_ts) > 60: os.system('LD_LIBRARY_PATH="" svc power shutdown') # Offroad power monitoring pm.calculate(health) msg.thermal.offroadPowerUsage = pm.get_power_used() msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90 # if current is positive, then battery is being discharged msg.thermal.started = started_ts is not None msg.thermal.startedTs = int(1e9 * (started_ts or 0)) msg.thermal.thermalStatus = thermal_status thermal_sock.send(msg.to_bytes()) if usb_power_prev and not usb_power: put_nonblocking( "Offroad_ChargeDisabled", json.dumps(OFFROAD_ALERTS["Offroad_ChargeDisabled"])) elif usb_power and not usb_power_prev: params.delete("Offroad_ChargeDisabled") thermal_status_prev = thermal_status usb_power_prev = usb_power fw_version_match_prev = fw_version_match should_start_prev = should_start # report to server once per minute if (count % int(60. / DT_TRML)) == 0: cloudlog.event("STATUS_PACKET", count=count, health=(health.to_dict() if health else None), location=(location.to_dict() if location else None), thermal=msg.to_dict()) count += 1
def register(show_spinner=False): params = Params() params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly private_key = open(PERSIST + "/comma/id_rsa").read() public_key = open(PERSIST + "/comma/id_rsa.pub").read() # Block until we get the imei imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) serial = HARDWARE.get_serial() params.put("IMEI", imei1) params.put("HardwareSerial", serial) backoff = 0 while True: try: register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code == 402: cloudlog.info( "Uknown serial number while trying to register device") dongle_id = None else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] params.put("DongleId", dongle_id) break except Exception: cloudlog.exception("failed to authenticate") backoff = min(backoff + 1, 15) time.sleep(backoff) if show_spinner: spinner.close() return dongle_id
def cache(upload_queue): try: items = [i._asdict() for i in upload_queue.queue if i.id not in cancelled_uploads] UploadQueueCache.params.put("AthenadUploadQueue", json.dumps(items)) except Exception: cloudlog.exception("athena.UploadQueueCache.cache.exception")
def thermald_thread(): health_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected health frequency # now loop thermal_sock = messaging.pub_sock('thermal') health_sock = messaging.sub_sock('health', timeout=health_timeout) location_sock = messaging.sub_sock('gpsLocation') fan_speed = 0 count = 0 startup_conditions = { "ignition": False, } startup_conditions_prev = startup_conditions.copy() off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green usb_power = True current_branch = get_git_branch() network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) health_prev = None charging_disabled = False should_start_prev = False handle_fan = None is_uno = False params = Params() pm = PowerMonitoring() no_panda_cnt = 0 thermal_config = get_thermal_config() ts_last_ip = 0 ip_addr = '255.255.255.255' # sound trigger sound_trigger = 1 opkrAutoShutdown = 0 shutdown_trigger = 1 is_openpilot_view_enabled = 0 env = dict(os.environ) env['LD_LIBRARY_PATH'] = mediaplayer getoff_alert = Params().get('OpkrEnableGetoffAlert') == b'1' if int(params.get('OpkrAutoShutdown')) == 0: opkrAutoShutdown = 0 elif int(params.get('OpkrAutoShutdown')) == 1: opkrAutoShutdown = 5 elif int(params.get('OpkrAutoShutdown')) == 2: opkrAutoShutdown = 30 elif int(params.get('OpkrAutoShutdown')) == 3: opkrAutoShutdown = 60 elif int(params.get('OpkrAutoShutdown')) == 4: opkrAutoShutdown = 180 elif int(params.get('OpkrAutoShutdown')) == 5: opkrAutoShutdown = 300 elif int(params.get('OpkrAutoShutdown')) == 6: opkrAutoShutdown = 600 elif int(params.get('OpkrAutoShutdown')) == 7: opkrAutoShutdown = 1800 elif int(params.get('OpkrAutoShutdown')) == 8: opkrAutoShutdown = 3600 elif int(params.get('OpkrAutoShutdown')) == 9: opkrAutoShutdown = 10800 else: opkrAutoShutdown = 18000 lateral_control_method = int(params.get("LateralControlMethod")) lateral_control_method_prev = int(params.get("LateralControlMethod")) lateral_control_method_cnt = 0 lateral_control_method_trigger = 0 while 1: ts = sec_since_boot() health = messaging.recv_sock(health_sock, wait=True) location = messaging.recv_sock(location_sock) location = location.gpsLocation if location else None msg = read_thermal(thermal_config) if health is not None: usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client # If we lose connection to the panda, wait 5 seconds before going offroad lateral_control_method = int(params.get("LateralControlMethod")) if lateral_control_method != lateral_control_method_prev and lateral_control_method_trigger == 0: startup_conditions["ignition"] = False lateral_control_method_trigger = 1 elif lateral_control_method != lateral_control_method_prev: lateral_control_method_cnt += 1 if lateral_control_method_cnt > 1 / DT_TRML: lateral_control_method_prev = lateral_control_method elif health.health.hwType == log.HealthData.HwType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if startup_conditions["ignition"]: cloudlog.error("Lost panda connection while onroad") startup_conditions["ignition"] = False shutdown_trigger = 1 else: no_panda_cnt = 0 startup_conditions[ "ignition"] = health.health.ignitionLine or health.health.ignitionCan sound_trigger == 1 lateral_control_method_cnt = 0 lateral_control_method_trigger = 0 # Setup fan handler on first connect to panda if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown: is_uno = health.health.hwType == log.HealthData.HwType.uno if (not EON) or is_uno: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if health_prev is not None: if health.health.hwType == log.HealthData.HwType.unknown and \ health_prev.health.hwType != log.HealthData.HwType.unknown: params.panda_disconnect() health_prev = health elif int(params.get("IsOpenpilotViewEnabled")) == 1 and int( params.get("IsDriverViewEnabled") ) == 0 and is_openpilot_view_enabled == 0: is_openpilot_view_enabled = 1 startup_conditions["ignition"] = True elif int(params.get("IsOpenpilotViewEnabled")) == 0 and int( params.get("IsDriverViewEnabled") ) == 0 and is_openpilot_view_enabled == 1: shutdown_trigger = 0 sound_trigger == 0 is_openpilot_view_enabled = 0 startup_conditions["ignition"] = False # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() network_strength = HARDWARE.get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0 msg.thermal.memUsedPercent = int(round( psutil.virtual_memory().percent)) msg.thermal.cpuPerc = int(round(psutil.cpu_percent())) msg.thermal.networkType = network_type msg.thermal.networkStrength = network_strength msg.thermal.batteryPercent = HARDWARE.get_battery_capacity() msg.thermal.batteryStatus = HARDWARE.get_battery_status() msg.thermal.batteryCurrent = HARDWARE.get_battery_current() msg.thermal.batteryVoltage = HARDWARE.get_battery_voltage() msg.thermal.usbOnline = HARDWARE.get_usb_present() # Fake battery levels on uno for frame if (not EON) or is_uno: msg.thermal.batteryPercent = 100 msg.thermal.batteryStatus = "Charging" msg.thermal.bat = 0 # update ip every 10 seconds ts = sec_since_boot() if ts - ts_last_ip >= 10.: try: result = subprocess.check_output(["ifconfig", "wlan0"], encoding='utf8') # pylint: disable=unexpected-keyword-arg ip_addr = re.findall(r"inet addr:((\d+\.){3}\d+)", result)[0][0] except: ip_addr = 'N/A' ts_last_ip = ts msg.thermal.ipAddr = ip_addr current_filter.update(msg.thermal.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update(max(msg.thermal.cpu)) max_comp_temp = max(max_cpu_temp, msg.thermal.mem, max(msg.thermal.gpu)) bat_temp = msg.thermal.bat if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, startup_conditions["ignition"]) msg.thermal.fanSpeed = fan_speed # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 # We only do this if there is a relay that prevents the car from faulting is_offroad_for_5_min = (started_ts is None) and ( (not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if max_cpu_temp > 107. or bat_temp >= 63. or (is_offroad_for_5_min and max_cpu_temp > 70.0): # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 96.0 or bat_temp > 60.: # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 94.0: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: # all good thermal_status = ThermalStatus.green # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert startup_conditions["time_valid"] = (now.year > 2020) or ( now.year == 2020 and now.month >= 10) set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) # Show update prompt # try: # last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) # except (TypeError, ValueError): # last_update = now # dt = now - last_update # # update_failed_count = params.get("UpdateFailedCount") # update_failed_count = 0 if update_failed_count is None else int(update_failed_count) # last_update_exception = params.get("LastUpdateException", encoding='utf8') # # if update_failed_count > 15 and last_update_exception is not None: # if current_branch in ["release2", "dashcam"]: # extra_text = "Ensure the software is correctly installed" # else: # extra_text = last_update_exception # # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text) # elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True) # elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: # remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.") # else: # set_offroad_alert_if_changed("Offroad_UpdateFailed", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) # set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) startup_conditions["not_uninstalling"] = not params.get( "DoUninstall") == b"1" startup_conditions["accepted_terms"] = params.get( "HasAcceptedTerms") == terms_version panda_signature = params.get("PandaFirmware") startup_conditions["fw_version_match"] = (panda_signature is None) or ( panda_signature == FW_SIGNATURE ) # don't show alert is no panda is connected (None) set_offroad_alert_if_changed( "Offroad_PandaFirmwareMismatch", (not startup_conditions["fw_version_match"])) # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.thermal.freeSpace > 0.02 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ (current_branch in ['dashcam', 'dashcam-staging']) startup_conditions["not_driver_view"] = not params.get( "IsDriverViewEnabled") == b"1" startup_conditions["not_taking_snapshot"] = not params.get( "IsTakingSnapshot") == b"1" # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 startup_conditions[ "device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed( "Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"])) should_start = all(startup_conditions.values()) if should_start: if not should_start_prev: params.delete("IsOffroad") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions) if should_start_prev or (count == 0): params.put("IsOffroad", "1") started_ts = None if off_ts is None: off_ts = sec_since_boot() if shutdown_trigger == 1 and sound_trigger == 1 and msg.thermal.batteryStatus == "Discharging" and started_seen and ( sec_since_boot() - off_ts) > 1 and getoff_alert: subprocess.Popen([ mediaplayer + 'mediaplayer', '/data/openpilot/selfdrive/assets/sounds/eondetach.wav' ], shell=False, stdin=None, stdout=None, stderr=None, env=env, close_fds=True) sound_trigger = 0 # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for # more than a minute but we were running if shutdown_trigger == 1 and msg.thermal.batteryStatus == "Discharging" and \ started_seen and opkrAutoShutdown and (sec_since_boot() - off_ts) > opkrAutoShutdown and not os.path.isfile(pandaflash_ongoing): os.system('LD_LIBRARY_PATH="" svc power shutdown') charging_disabled = check_car_battery_voltage(should_start, health, charging_disabled, msg) if msg.thermal.batteryCurrent > 0: msg.thermal.batteryStatus = "Discharging" else: msg.thermal.batteryStatus = "Charging" msg.thermal.chargingDisabled = charging_disabled prebuiltlet = Params().get('PutPrebuiltOn') == b'1' if not os.path.isfile(prebuiltfile) and prebuiltlet: os.system("cd /data/openpilot; touch prebuilt") elif os.path.isfile(prebuiltfile) and not prebuiltlet: os.system("cd /data/openpilot; rm -f prebuilt") # Offroad power monitoring pm.calculate(health) msg.thermal.offroadPowerUsage = pm.get_power_used() msg.thermal.carBatteryCapacity = max(0, pm.get_car_battery_capacity()) # # Check if we need to disable charging (handled by boardd) # msg.thermal.chargingDisabled = pm.should_disable_charging(health, off_ts) # # # Check if we need to shut down # if pm.should_shutdown(health, off_ts, started_seen, LEON): # cloudlog.info(f"shutting device down, offroad since {off_ts}") # # TODO: add function for blocking cloudlog instead of sleep # time.sleep(10) # os.system('LD_LIBRARY_PATH="" svc power shutdown') msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90 # if current is positive, then battery is being discharged msg.thermal.started = started_ts is not None msg.thermal.startedTs = int(1e9 * (started_ts or 0)) msg.thermal.thermalStatus = thermal_status thermal_sock.send(msg.to_bytes()) set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power)) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # report to server once per minute if (count % int(60. / DT_TRML)) == 0: cloudlog.event("STATUS_PACKET", count=count, health=(health.to_dict() if health else None), location=(location.to_dict() if location else None), thermal=msg.to_dict()) count += 1
def get_data(self, timeout, total_timeout=None): if total_timeout is None: total_timeout = 10 * timeout self._drain_rx() # Create message objects msgs = {} request_counter = {} request_done = {} for tx_addr, rx_addr in self.msg_addrs.items(): # rx_addr not set when using functional tx addr id_addr = rx_addr or tx_addr[0] sub_addr = tx_addr[1] can_client = CanClient(self._can_tx, partial(self._can_rx, id_addr, sub_addr=sub_addr), tx_addr[0], rx_addr, self.bus, sub_addr=sub_addr, debug=self.debug) max_len = 8 if sub_addr is None else 7 msg = IsoTpMessage(can_client, timeout=0, max_len=max_len, debug=self.debug) msg.send(self.request[0]) msgs[tx_addr] = msg request_counter[tx_addr] = 0 request_done[tx_addr] = False results = {} start_time = time.monotonic() last_response_time = start_time while True: self.rx() if all(request_done.values()): break for tx_addr, msg in msgs.items(): try: dat: Optional[bytes] = msg.recv() except Exception: cloudlog.exception("Error processing UDS response") request_done[tx_addr] = True continue if not dat: continue counter = request_counter[tx_addr] expected_response = self.response[counter] response_valid = dat[:len(expected_response )] == expected_response if response_valid: last_response_time = time.monotonic() if counter + 1 < len(self.request): msg.send(self.request[counter + 1]) request_counter[tx_addr] += 1 else: results[tx_addr] = dat[len(expected_response):] request_done[tx_addr] = True else: request_done[tx_addr] = True cloudlog.warning( f"iso-tp query bad response: 0x{dat.hex()}") cur_time = time.monotonic() if cur_time - last_response_time > timeout: for tx_addr in msgs: if (request_counter[tx_addr] > 0) and (not request_done[tx_addr]): cloudlog.warning( f"iso-tp query timeout after receiving response: {tx_addr}" ) break if cur_time - start_time > total_timeout: cloudlog.warning("iso-tp query timeout while receiving data") break return results
def thermald_thread(): health_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected health frequency # now loop thermal_sock = messaging.pub_sock('thermal') health_sock = messaging.sub_sock('health', timeout=health_timeout) location_sock = messaging.sub_sock('gpsLocation') fan_speed = 0 count = 0 startup_conditions = { "ignition": False, } startup_conditions_prev = startup_conditions.copy() off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green usb_power = True current_branch = get_git_branch() network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML) health_prev = None should_start_prev = False handle_fan = None is_uno = False params = Params() pm = PowerMonitoring() no_panda_cnt = 0 thermal_config = get_thermal_config() while 1: health = messaging.recv_sock(health_sock, wait=True) location = messaging.recv_sock(location_sock) location = location.gpsLocation if location else None msg = read_thermal(thermal_config) if health is not None: usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client # If we lose connection to the panda, wait 5 seconds before going offroad if health.health.hwType == log.HealthData.HwType.unknown: no_panda_cnt += 1 if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML: if startup_conditions["ignition"]: cloudlog.error("Lost panda connection while onroad") startup_conditions["ignition"] = False else: no_panda_cnt = 0 startup_conditions["ignition"] = health.health.ignitionLine or health.health.ignitionCan # Setup fan handler on first connect to panda if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown: is_uno = health.health.hwType == log.HealthData.HwType.uno if (not EON) or is_uno: cloudlog.info("Setting up UNO fan handler") handle_fan = handle_fan_uno else: cloudlog.info("Setting up EON fan handler") setup_eon_fan() handle_fan = handle_fan_eon # Handle disconnect if health_prev is not None: if health.health.hwType == log.HealthData.HwType.unknown and \ health_prev.health.hwType != log.HealthData.HwType.unknown: params.panda_disconnect() health_prev = health # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() network_strength = HARDWARE.get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0 msg.thermal.memUsedPercent = int(round(psutil.virtual_memory().percent)) msg.thermal.cpuPerc = int(round(psutil.cpu_percent())) msg.thermal.networkType = network_type msg.thermal.networkStrength = network_strength msg.thermal.batteryPercent = get_battery_capacity() msg.thermal.batteryStatus = get_battery_status() msg.thermal.batteryCurrent = get_battery_current() msg.thermal.batteryVoltage = get_battery_voltage() msg.thermal.usbOnline = get_usb_present() # Fake battery levels on uno for frame if (not EON) or is_uno: msg.thermal.batteryPercent = 100 msg.thermal.batteryStatus = "Charging" msg.thermal.bat = 0 current_filter.update(msg.thermal.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = cpu_temp_filter.update(max(msg.thermal.cpu)) max_comp_temp = max(max_cpu_temp, msg.thermal.mem, max(msg.thermal.gpu)) bat_temp = msg.thermal.bat if handle_fan is not None: fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, startup_conditions["ignition"]) msg.thermal.fanSpeed = fan_speed # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 # We only do this if there is a relay that prevents the car from faulting is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if max_cpu_temp > 107. or bat_temp >= 63. or (is_offroad_for_5_min and max_cpu_temp > 70.0): # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 96.0 or bat_temp > 60.: # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 94.0: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 80.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 75.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: # all good thermal_status = ThermalStatus.green # **** starting logic **** # Check for last update time and display alerts if needed now = datetime.datetime.utcnow() # show invalid date/time alert startup_conditions["time_valid"] = (now.year > 2020) or (now.year == 2020 and now.month >= 10) set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) # Show update prompt try: last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) except (TypeError, ValueError): last_update = now dt = now - last_update update_failed_count = params.get("UpdateFailedCount") update_failed_count = 0 if update_failed_count is None else int(update_failed_count) last_update_exception = params.get("LastUpdateException", encoding='utf8') if update_failed_count > 15 and last_update_exception is not None: if current_branch in ["release2", "dashcam"]: extra_text = "Ensure the software is correctly installed" else: extra_text = last_update_exception set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text) elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True) elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.") else: set_offroad_alert_if_changed("Offroad_UpdateFailed", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False) set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False) startup_conditions["not_uninstalling"] = not params.get("DoUninstall") == b"1" startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version panda_signature = params.get("PandaFirmware") startup_conditions["fw_version_match"] = (panda_signature is None) or (panda_signature == FW_SIGNATURE) # don't show alert is no panda is connected (None) set_offroad_alert_if_changed("Offroad_PandaFirmwareMismatch", (not startup_conditions["fw_version_match"])) # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.thermal.freeSpace > 0.02 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ (current_branch in ['dashcam', 'dashcam-staging']) startup_conditions["not_driver_view"] = not params.get("IsDriverViewEnabled") == b"1" startup_conditions["not_taking_snapshot"] = not params.get("IsTakingSnapshot") == b"1" # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 startup_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"])) should_start = all(startup_conditions.values()) startup_conditions["hardware_supported"] = health is not None and health.health.hwType not in [log.HealthData.HwType.whitePanda, log.HealthData.HwType.greyPanda] set_offroad_alert_if_changed("Offroad_HardwareUnsupported", health is not None and not startup_conditions["hardware_supported"]) if should_start: if not should_start_prev: params.delete("IsOffroad") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions) if should_start_prev or (count == 0): params.put("IsOffroad", "1") started_ts = None if off_ts is None: off_ts = sec_since_boot() # Offroad power monitoring pm.calculate(health) msg.thermal.offroadPowerUsage = pm.get_power_used() msg.thermal.carBatteryCapacity = max(0, pm.get_car_battery_capacity()) # Check if we need to disable charging (handled by boardd) msg.thermal.chargingDisabled = pm.should_disable_charging(health, off_ts) # Check if we need to shut down if pm.should_shutdown(health, off_ts, started_seen, LEON): cloudlog.info(f"shutting device down, offroad since {off_ts}") # TODO: add function for blocking cloudlog instead of sleep time.sleep(10) os.system('LD_LIBRARY_PATH="" svc power shutdown') msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90 # if current is positive, then battery is being discharged msg.thermal.started = started_ts is not None msg.thermal.startedTs = int(1e9*(started_ts or 0)) msg.thermal.thermalStatus = thermal_status thermal_sock.send(msg.to_bytes()) set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power)) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # report to server once per minute if (count % int(60. / DT_TRML)) == 0: cloudlog.event("STATUS_PACKET", count=count, health=(health.to_dict() if health else None), location=(location.to_dict() if location else None), thermal=msg.to_dict()) count += 1
def main(): params = Params() if params.get("DisableUpdates") == b"1": raise RuntimeError("updates are disabled by the DisableUpdates param") # TODO: remove this after next release if EON and "letv" not in open("/proc/cmdline").read(): raise RuntimeError("updates are disabled due to device deprecation") if EON and os.geteuid() != 0: raise RuntimeError("updated must be launched as root!") # Set low io priority proc = psutil.Process() if psutil.LINUX: proc.ionice(psutil.IOPRIO_CLASS_BE, value=7) ov_lock_fd = open(LOCK_FILE, 'w') try: fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: raise RuntimeError( "couldn't get overlay lock; is another updated running?") from e # Wait for IsOffroad to be set before our first update attempt wait_helper = WaitTimeHelper(proc) wait_helper.sleep(30) overlay_init = Path(os.path.join(BASEDIR, ".overlay_init")) if overlay_init.exists(): overlay_init.unlink() first_run = True last_fetch_time = 0 update_failed_count = 0 # Run the update loop # * every 1m, do a lightweight internet/update check # * every 10m, do a full git fetch while not wait_helper.shutdown: update_now = wait_helper.ready_event.is_set() wait_helper.ready_event.clear() # Don't run updater while onroad or if the time's wrong time_wrong = datetime.datetime.utcnow().year < 2019 is_onroad = params.get("IsOffroad") != b"1" if is_onroad or time_wrong: wait_helper.sleep(30) cloudlog.info("not running updater, not offroad") continue # Attempt an update exception = None new_version = False update_failed_count += 1 try: init_overlay() internet_ok, update_available = check_for_update() if internet_ok and not update_available: update_failed_count = 0 # Fetch updates at most every 10 minutes if internet_ok and (update_now or time.monotonic() - last_fetch_time > 60 * 10): new_version = fetch_update(wait_helper) update_failed_count = 0 last_fetch_time = time.monotonic() if first_run and not new_version and os.path.isdir( NEOSUPDATE_DIR): shutil.rmtree(NEOSUPDATE_DIR) first_run = False except subprocess.CalledProcessError as e: cloudlog.event("update process failed", cmd=e.cmd, output=e.output, returncode=e.returncode) exception = f"command failed: {e.cmd}\n{e.output}" except Exception as e: cloudlog.exception("uncaught updated exception, shouldn't happen") exception = str(e) set_params(new_version, update_failed_count, exception) wait_helper.sleep(60) dismount_overlay()
try: manager_thread() except Exception: traceback.print_exc() crash.capture_exception() finally: cleanup_all_processes(None, None) if params.get("DoUninstall", encoding='utf8') == "1": uninstall() if __name__ == "__main__": try: main() except Exception: add_logentries_handler(cloudlog) cloudlog.exception("Manager failed to start") # Show last 3 lines of traceback error = traceback.format_exc(-3) error = "Manager failed to start\n \n" + error with TextWindow(error) as t: t.wait_for_exit() raise # manual exit because we are forked sys.exit(0)
def thermald_thread(): # prevent LEECO from undervoltage BATT_PERC_OFF = 10 if LEON else 3 health_timeout = int(1000 * 2.5 * DT_TRML) # 2.5x the expected health frequency # now loop thermal_sock = messaging.pub_sock('thermal') health_sock = messaging.sub_sock('health', timeout=health_timeout) location_sock = messaging.sub_sock('gpsLocation') ignition = False fan_speed = 0 count = 0 off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green thermal_status_prev = ThermalStatus.green usb_power = True usb_power_prev = True network_type = NetworkType.none network_strength = NetworkStrength.unknown current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) health_prev = None fw_version_match_prev = True current_connectivity_alert = None time_valid_prev = True should_start_prev = False is_uno = (read_tz(29, clip=False) < -1000) if is_uno or not ANDROID: handle_fan = handle_fan_uno else: setup_eon_fan() handle_fan = handle_fan_eon pm = PowerMonitoring(is_uno) # dragonpilot ts_last_ip = None ts_last_update_vars = 0 ts_last_charging_ctrl = None dp_last_modified = None ip_addr = '255.255.255.255' dragon_charging_ctrl = False dragon_charging_ctrl_prev = False dragon_to_discharge = 70 dragon_to_charge = 60 dp_temp_monitor = True while 1: # dragonpilot ts = sec_since_boot() # update variable status every 10 secs if ts - ts_last_update_vars >= 10.: modified = dp_get_last_modified() if dp_last_modified != modified: dp_temp_monitor = True if params.get('DragonEnableTempMonitor', encoding='utf8') == "1" else False if not is_uno: dragon_charging_ctrl = True if params.get('DragonChargingCtrl', encoding='utf8') == "1" else False if dragon_charging_ctrl: try: dragon_to_discharge = int(params.get('DragonCharging', encoding='utf8')) except (TypeError, ValueError): dragon_to_discharge = 70 try: dragon_to_charge = int(params.get('DragonDisCharging', encoding='utf8')) except (TypeError, ValueError): dragon_to_charge = 60 else: dragon_charging_ctrl = False dp_last_modified = modified ts_last_update_vars = ts health = messaging.recv_sock(health_sock, wait=True) location = messaging.recv_sock(location_sock) location = location.gpsLocation if location else None msg = read_thermal() # clear car params when panda gets disconnected if health is None and health_prev is not None: params.panda_disconnect() health_prev = health if health is not None: usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client # get_network_type is an expensive call. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = get_network_type() network_strength = get_network_strength(network_type) except Exception: cloudlog.exception("Error getting network status") msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0 msg.thermal.memUsedPercent = int(round(psutil.virtual_memory().percent)) msg.thermal.cpuPerc = int(round(psutil.cpu_percent())) msg.thermal.networkType = network_type msg.thermal.networkStrength = network_strength msg.thermal.batteryPercent = get_battery_capacity() msg.thermal.batteryStatus = get_battery_status() msg.thermal.batteryCurrent = get_battery_current() msg.thermal.batteryVoltage = get_battery_voltage() msg.thermal.usbOnline = get_usb_present() # Fake battery levels on uno for frame if is_uno: msg.thermal.batteryPercent = 100 msg.thermal.batteryStatus = "Charging" # dragonpilot ip Mod # update ip every 10 seconds ts = sec_since_boot() if ts_last_ip is None or ts - ts_last_ip >= 10.: try: result = subprocess.check_output(["ifconfig", "wlan0"], encoding='utf8') # pylint: disable=unexpected-keyword-arg ip_addr = re.findall(r"inet addr:((\d+\.){3}\d+)", result)[0][0] except: ip_addr = 'N/A' ts_last_ip = ts msg.thermal.ipAddr = ip_addr current_filter.update(msg.thermal.batteryCurrent / 1e6) # TODO: add car battery voltage check max_cpu_temp = max(msg.thermal.cpu0, msg.thermal.cpu1, msg.thermal.cpu2, msg.thermal.cpu3) / 10.0 max_comp_temp = max(max_cpu_temp, msg.thermal.mem / 10., msg.thermal.gpu / 10.) bat_temp = msg.thermal.bat / 1000. fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, ignition) msg.thermal.fanSpeed = fan_speed # thermal logic with hysterisis if max_cpu_temp > 107.: # onroad not allowed thermal_status = ThermalStatus.danger elif max_comp_temp > 92.5: # CPU throttling starts around ~90C # hysteresis between onroad not allowed and engage not allowed thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger) elif max_cpu_temp > 87.5: # hysteresis between engage not allowed and uploader not allowed thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red) elif max_cpu_temp > 86.0: # uploader not allowed thermal_status = ThermalStatus.yellow elif max_cpu_temp > 78.0: # hysteresis between uploader not allowed and all good thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow) else: # all good thermal_status = ThermalStatus.green # **** starting logic **** # Check for last update time and display alerts if needed # now = datetime.datetime.now() # # # show invalid date/time alert # time_valid = now.year >= 2019 # if time_valid and not time_valid_prev: # params.delete("Offroad_InvalidTime") # if not time_valid and time_valid_prev: # params.put("Offroad_InvalidTime", json.dumps(OFFROAD_ALERTS["Offroad_InvalidTime"])) # time_valid_prev = time_valid time_valid = True # Show update prompt # try: # last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8')) # except (TypeError, ValueError): # last_update = now # dt = now - last_update # # update_failed_count = params.get("UpdateFailedCount") # update_failed_count = 0 if update_failed_count is None else int(update_failed_count) # # if dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1: # if current_connectivity_alert != "expired": # current_connectivity_alert = "expired" # params.delete("Offroad_ConnectivityNeededPrompt") # params.put("Offroad_ConnectivityNeeded", json.dumps(OFFROAD_ALERTS["Offroad_ConnectivityNeeded"])) # elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT: # remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0)) # if current_connectivity_alert != "prompt" + remaining_time: # current_connectivity_alert = "prompt" + remaining_time # alert_connectivity_prompt = copy.copy(OFFROAD_ALERTS["Offroad_ConnectivityNeededPrompt"]) # alert_connectivity_prompt["text"] += remaining_time + " days." # params.delete("Offroad_ConnectivityNeeded") # params.put("Offroad_ConnectivityNeededPrompt", json.dumps(alert_connectivity_prompt)) # elif current_connectivity_alert is not None: # current_connectivity_alert = None # params.delete("Offroad_ConnectivityNeeded") # params.delete("Offroad_ConnectivityNeededPrompt") # start constellation of processes when the car starts ignition = health is not None and (health.health.ignitionLine or health.health.ignitionCan) do_uninstall = params.get("DoUninstall") == b"1" accepted_terms = params.get("HasAcceptedTerms") == terms_version completed_training = params.get("CompletedTrainingVersion") == training_version panda_signature = params.get("PandaFirmware") fw_version_match = (panda_signature is None) or (panda_signature == FW_SIGNATURE) # don't show alert is no panda is connected (None) should_start = ignition # with 2% left, we killall, otherwise the phone will take a long time to boot should_start = should_start and msg.thermal.freeSpace > 0.02 # confirm we have completed training and aren't uninstalling should_start = should_start and accepted_terms and completed_training and (not do_uninstall) # check for firmware mismatch should_start = should_start and fw_version_match # check if system time is valid should_start = should_start and time_valid # don't start while taking snapshot if not should_start_prev: is_taking_snapshot = params.get("IsTakingSnapshot") == b"1" should_start = should_start and (not is_taking_snapshot) if fw_version_match and not fw_version_match_prev: params.delete("Offroad_PandaFirmwareMismatch") if not fw_version_match and fw_version_match_prev: params.put("Offroad_PandaFirmwareMismatch", json.dumps(OFFROAD_ALERTS["Offroad_PandaFirmwareMismatch"])) # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 if dp_temp_monitor: if thermal_status >= ThermalStatus.danger: should_start = False if thermal_status_prev < ThermalStatus.danger: params.put("Offroad_TemperatureTooHigh", json.dumps(OFFROAD_ALERTS["Offroad_TemperatureTooHigh"])) else: if thermal_status_prev >= ThermalStatus.danger: params.delete("Offroad_TemperatureTooHigh") if should_start: if not should_start_prev: params.delete("IsOffroad") off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True os.system('echo performance > /sys/class/devfreq/soc:qcom,cpubw/governor') else: if should_start_prev or (count == 0): params.put("IsOffroad", "1") started_ts = None if off_ts is None: off_ts = sec_since_boot() os.system('echo powersave > /sys/class/devfreq/soc:qcom,cpubw/governor') # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for # more than a minute but we were running if msg.thermal.batteryPercent < BATT_PERC_OFF and msg.thermal.batteryStatus == "Discharging" and \ started_seen and (sec_since_boot() - off_ts) > 60: os.system('LD_LIBRARY_PATH="" svc power shutdown') # Offroad power monitoring pm.calculate(health) msg.thermal.offroadPowerUsage = pm.get_power_used() msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90 # if current is positive, then battery is being discharged msg.thermal.started = started_ts is not None msg.thermal.startedTs = int(1e9*(started_ts or 0)) msg.thermal.thermalStatus = thermal_status thermal_sock.send(msg.to_bytes()) if usb_power_prev and not usb_power: params.put("Offroad_ChargeDisabled", json.dumps(OFFROAD_ALERTS["Offroad_ChargeDisabled"])) elif usb_power and not usb_power_prev: params.delete("Offroad_ChargeDisabled") thermal_status_prev = thermal_status usb_power_prev = usb_power fw_version_match_prev = fw_version_match should_start_prev = should_start if dragon_charging_ctrl != dragon_charging_ctrl_prev: set_battery_charging(True) if dragon_charging_ctrl: if ts_last_charging_ctrl is None or ts - ts_last_charging_ctrl >= 60.: if msg.thermal.batteryPercent >= dragon_to_discharge and get_battery_charging(): set_battery_charging(False) elif msg.thermal.batteryPercent <= dragon_to_charge and not get_battery_charging(): set_battery_charging(True) ts_last_charging_ctrl = ts dragon_charging_ctrl_prev = dragon_charging_ctrl # report to server once per minute if (count % int(60. / DT_TRML)) == 0: cloudlog.event("STATUS_PACKET", count=count, health=(health.to_dict() if health else None), location=(location.to_dict() if location else None), thermal=msg.to_dict()) count += 1