def test_clean_exit(self): """ Ensure all processes exit cleanly when stopped. """ HARDWARE.set_power_save(False) manager.manager_prepare() for p in ALL_PROCESSES: managed_processes[p].start() time.sleep(10) for p in reversed(ALL_PROCESSES): with self.subTest(proc=p): state = managed_processes[p].get_process_state_msg() self.assertTrue(state.running, f"{p} not running") exit_code = managed_processes[p].stop(retry=False) self.assertTrue(exit_code is not None, f"{p} failed to exit") # TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code exit_codes = [0, 1] if managed_processes[p].sigkill: exit_codes = [-signal.SIGKILL] self.assertIn(exit_code, exit_codes, f"{p} died with {exit_code}")
def setBandwithLimit(upload_speed_kbps, download_speed_kbps): if not AGNOS: return {"success": 0, "error": "only supported on AGNOS"} try: HARDWARE.set_bandwidth_limit(upload_speed_kbps, download_speed_kbps) return {"success": 1} except subprocess.CalledProcessError as e: return { "success": 0, "error": "failed to set limit", "stdout": e.stdout, "stderr": e.stderr }
def handle_agnos_update(wait_helper: WaitTimeHelper) -> None: from system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number cur_version = HARDWARE.get_os_version() updated_version = run([ "bash", "-c", r"unset AGNOS_VERSION && source launch_env.sh && \ echo -n $AGNOS_VERSION" ], OVERLAY_MERGED).strip() cloudlog.info(f"AGNOS version check: {cur_version} vs {updated_version}") if cur_version == updated_version: return # prevent an openpilot getting swapped in with a mismatched or partially downloaded agnos set_consistent_flag(False) cloudlog.info( f"Beginning background installation for AGNOS {updated_version}") set_offroad_alert("Offroad_NeosUpdate", True) manifest_path = os.path.join(OVERLAY_MERGED, "system/hardware/tici/agnos.json") target_slot_number = get_target_slot_number() flash_agnos_update(manifest_path, target_slot_number, cloudlog) set_offroad_alert("Offroad_NeosUpdate", False)
def init(project: SentryProject) -> None: # forks like to mess with this, so double check comma_remote = is_comma_remote() and "commaai" in get_origin(default="") if not comma_remote or not is_registered_device() or PC: return env = "release" if is_tested_branch() else "master" dongle_id = Params().get("DongleId", encoding='utf-8') integrations = [] if project == SentryProject.SELFDRIVE: integrations.append(ThreadingIntegration(propagate_hub=True)) else: sentry_sdk.utils.MAX_STRING_LENGTH = 8192 sentry_sdk.init(project.value, default_integrations=False, release=get_version(), integrations=integrations, traces_sample_rate=1.0, environment=env) sentry_sdk.set_user({"id": dongle_id}) sentry_sdk.set_tag("dirty", is_dirty()) sentry_sdk.set_tag("origin", get_origin()) sentry_sdk.set_tag("branch", get_branch()) sentry_sdk.set_tag("commit", get_commit()) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session()
def stop(self, retry: bool=True, block: bool=True) -> Optional[int]: if self.proc is None: return None if self.proc.exitcode is None: if not self.shutting_down: cloudlog.info(f"killing {self.name}") sig = signal.SIGKILL if self.sigkill else signal.SIGINT self.signal(sig) self.shutting_down = True if not block: return None join_process(self.proc, 5) # If process failed to die send SIGKILL or reboot if self.proc.exitcode is None and retry: if self.unkillable: cloudlog.critical(f"unkillable process {self.name} failed to exit! rebooting in 15 if it doesn't die") join_process(self.proc, 15) if self.proc.exitcode is None: cloudlog.critical(f"unkillable process {self.name} failed to die!") os.system("date >> /data/unkillable_reboot") os.sync() HARDWARE.reboot() raise RuntimeError else: cloudlog.info(f"killing {self.name} with SIGKILL") self.signal(signal.SIGKILL) self.proc.join() ret = self.proc.exitcode cloudlog.info(f"{self.name} is dead with {ret}") if self.proc.exitcode is not None: self.shutting_down = False self.proc = None return ret
def main() -> None: prepare_only = os.getenv("PREPAREONLY") is not None manager_init() # Start UI early so prepare can happen in the background if not prepare_only: managed_processes['ui'].start() manager_prepare() if prepare_only: return # SystemExit on sigterm signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1)) try: manager_thread() except Exception: traceback.print_exc() sentry.capture_exception() finally: manager_cleanup() params = Params() if params.get_bool("DoUninstall"): cloudlog.warning("uninstalling") HARDWARE.uninstall() elif params.get_bool("DoReboot"): cloudlog.warning("reboot") HARDWARE.reboot() elif params.get_bool("DoShutdown"): cloudlog.warning("shutdown") HARDWARE.shutdown()
def flash_panda(panda_serial: str) -> Panda: panda = Panda(panda_serial) fw_signature = get_expected_signature(panda) internal_panda = panda.is_internal() and not panda.bootstub panda_version = "bootstub" if panda.bootstub else panda.get_version() panda_signature = b"" if panda.bootstub else panda.get_signature() cloudlog.warning( f"Panda {panda_serial} connected, version: {panda_version}, signature {panda_signature.hex()[:16]}, expected {fw_signature.hex()[:16]}" ) if panda.bootstub or panda_signature != fw_signature: cloudlog.info("Panda firmware out of date, update required") panda.flash() cloudlog.info("Done flashing") if panda.bootstub: bootstub_version = panda.get_version() cloudlog.info( f"Flashed firmware not booting, flashing development bootloader. Bootstub version: {bootstub_version}" ) if internal_panda: HARDWARE.recover_internal_panda() panda.recover(reset=(not internal_panda)) cloudlog.info("Done flashing bootloader") if panda.bootstub: cloudlog.info("Panda still not booting, exiting") raise AssertionError panda_signature = panda.get_signature() if panda_signature != fw_signature: cloudlog.info("Version mismatch after flashing, exiting") raise AssertionError return panda
def test_clean_exit(self): HARDWARE.set_power_save(False) manager.manager_prepare() for p in ALL_PROCESSES: managed_processes[p].start() time.sleep(10) for p in reversed(ALL_PROCESSES): state = managed_processes[p].get_process_state_msg() self.assertTrue(state.running, f"{p} not running") exit_code = managed_processes[p].stop(retry=False) if (AGNOS and p in [ 'ui', ]): # TODO: make Qt UI exit gracefully continue # TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code exit_codes = [0, 1] if managed_processes[p].sigkill: exit_codes = [-signal.SIGKILL] assert exit_code in exit_codes, f"{p} died with {exit_code}"
def calculate(self, peripheralState, ignition): try: now = sec_since_boot() # If peripheralState is None, we're probably not in a car, so we don't care if peripheralState is None or peripheralState.pandaType == log.PandaState.PandaType.unknown: with self.integration_lock: self.last_measurement_time = None self.next_pulsed_measurement_time = None self.power_used_uWh = 0 return # Low-pass battery voltage self.car_voltage_instant_mV = peripheralState.voltage self.car_voltage_mV = ((peripheralState.voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K))) statlog.gauge("car_voltage", self.car_voltage_mV / 1e3) # Cap the car battery power and save it in a param every 10-ish seconds self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0) self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh) if now - self.last_save_time >= 10: put_nonblocking("CarBatteryCapacity", str(int(self.car_battery_capacity_uWh))) self.last_save_time = now # First measurement, set integration time with self.integration_lock: if self.last_measurement_time is None: self.last_measurement_time = now return if ignition: # If there is ignition, we integrate the charging rate of the car with self.integration_lock: self.power_used_uWh = 0 integration_time_h = (now - self.last_measurement_time) / 3600 if integration_time_h < 0: raise ValueError(f"Negative integration time: {integration_time_h}h") self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h) self.last_measurement_time = now else: # Get current power draw somehow current_power = HARDWARE.get_current_power_draw() # Do the integration self._perform_integration(now, current_power) except Exception: cloudlog.exception("Power monitoring calculation failed")
def setUp(self): HARDWARE.initialize_hardware() HARDWARE.set_power_save(False) # wait a bit for power save to disable time.sleep(5)
def setUp(self): HARDWARE.initialize_hardware() HARDWARE.set_power_save(False)
def setUp(self): os.environ['PASSIVE'] = '0' HARDWARE.set_power_save(False)
def test_sound_card_init(self): assert HARDWARE.get_sound_card_online()
def getNetworks(): return HARDWARE.get_networks()
def getNetworkType(): return HARDWARE.get_network_type()
def thermald_thread(end_event, hw_queue): pm = messaging.PubMaster(['deviceState']) sm = messaging.SubMaster([ "peripheralState", "gpsLocationExternal", "controlsState", "pandaStates" ], poll=["pandaStates"]) count = 0 onroad_conditions: Dict[str, bool] = { "ignition": False, } startup_conditions: Dict[str, bool] = {} startup_conditions_prev: Dict[str, bool] = {} off_ts = None started_ts = None started_seen = False thermal_status = ThermalStatus.green last_hw_state = HardwareState( network_type=NetworkType.none, network_metered=False, network_strength=NetworkStrength.unknown, network_info=None, nvme_temps=[], modem_temps=[], ) current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML) temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_TRML) should_start_prev = False in_car = False engaged_prev = False params = Params() power_monitor = PowerMonitoring() HARDWARE.initialize_hardware() thermal_config = HARDWARE.get_thermal_config() fan_controller = None while not end_event.is_set(): sm.update(PANDA_STATES_TIMEOUT) pandaStates = sm['pandaStates'] peripheralState = sm['peripheralState'] msg = read_thermal(thermal_config) if sm.updated['pandaStates'] and len(pandaStates) > 0: # Set ignition based on any panda connected onroad_conditions["ignition"] = any( ps.ignitionLine or ps.ignitionCan for ps in pandaStates if ps.pandaType != log.PandaState.PandaType.unknown) pandaState = pandaStates[0] in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected # Setup fan handler on first connect to panda if fan_controller is None and peripheralState.pandaType != log.PandaState.PandaType.unknown: if TICI: fan_controller = TiciFanController() elif (sec_since_boot() - sm.rcv_time['pandaStates']) > DISCONNECT_TIMEOUT: if onroad_conditions["ignition"]: onroad_conditions["ignition"] = False cloudlog.error("panda timed out onroad") try: last_hw_state = hw_queue.get_nowait() except queue.Empty: pass msg.deviceState.freeSpacePercent = get_available_percent(default=100.0) msg.deviceState.memoryUsagePercent = int( round(psutil.virtual_memory().percent)) msg.deviceState.cpuUsagePercent = [ int(round(n)) for n in psutil.cpu_percent(percpu=True) ] msg.deviceState.gpuUsagePercent = int( round(HARDWARE.get_gpu_usage_percent())) msg.deviceState.networkType = last_hw_state.network_type msg.deviceState.networkMetered = last_hw_state.network_metered msg.deviceState.networkStrength = last_hw_state.network_strength if last_hw_state.network_info is not None: msg.deviceState.networkInfo = last_hw_state.network_info msg.deviceState.nvmeTempC = last_hw_state.nvme_temps msg.deviceState.modemTempC = last_hw_state.modem_temps msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness( ) msg.deviceState.usbOnline = HARDWARE.get_usb_present() current_filter.update(msg.deviceState.batteryCurrent / 1e6) max_comp_temp = temp_filter.update( max(max(msg.deviceState.cpuTempC), msg.deviceState.memoryTempC, max(msg.deviceState.gpuTempC))) if fan_controller is not None: msg.deviceState.fanSpeedPercentDesired = fan_controller.update( max_comp_temp, onroad_conditions["ignition"]) is_offroad_for_5_min = (started_ts is None) and ( (not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5)) if is_offroad_for_5_min and max_comp_temp > OFFROAD_DANGER_TEMP: # If device is offroad we want to cool down before going onroad # since going onroad increases load and can make temps go over 107 thermal_status = ThermalStatus.danger else: current_band = THERMAL_BANDS[thermal_status] band_idx = list(THERMAL_BANDS.keys()).index(thermal_status) if current_band.min_temp is not None and max_comp_temp < current_band.min_temp: thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1] elif current_band.max_temp is not None and max_comp_temp > current_band.max_temp: thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1] # **** starting logic **** # Ensure date/time are valid now = datetime.datetime.utcnow() startup_conditions["time_valid"] = (now.year > 2020) or ( now.year == 2020 and now.month >= 10) set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"])) startup_conditions["up_to_date"] = params.get( "Offroad_ConnectivityNeeded") is None or params.get_bool( "DisableUpdates") or params.get_bool("SnoozeUpdate") startup_conditions["not_uninstalling"] = not params.get_bool( "DoUninstall") startup_conditions["accepted_terms"] = params.get( "HasAcceptedTerms") == terms_version # with 2% left, we killall, otherwise the phone will take a long time to boot startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2 startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \ params.get_bool("Passive") startup_conditions["not_driver_view"] = not params.get_bool( "IsDriverViewEnabled") startup_conditions["not_taking_snapshot"] = not params.get_bool( "IsTakingSnapshot") # if any CPU gets above 107 or the battery gets above 63, kill all processes # controls will warn with CPU above 95 or battery above 60 onroad_conditions[ "device_temp_good"] = thermal_status < ThermalStatus.danger set_offroad_alert_if_changed( "Offroad_TemperatureTooHigh", (not onroad_conditions["device_temp_good"])) # TODO: this should move to TICI.initialize_hardware, but we currently can't import params there if TICI: if not os.path.isfile("/persist/comma/living-in-the-moment"): if not Path("/data/media").is_mount(): set_offroad_alert_if_changed("Offroad_StorageMissing", True) else: # check for bad NVMe try: with open("/sys/block/nvme0n1/device/model") as f: model = f.read().strip() if not model.startswith( "Samsung SSD 980") and params.get( "Offroad_BadNvme") is None: set_offroad_alert_if_changed( "Offroad_BadNvme", True) cloudlog.event("Unsupported NVMe", model=model, error=True) except Exception: pass # Handle offroad/onroad transition should_start = all(onroad_conditions.values()) if started_ts is None: should_start = should_start and all(startup_conditions.values()) if should_start != should_start_prev or (count == 0): params.put_bool("IsOnroad", should_start) params.put_bool("IsOffroad", not should_start) params.put_bool("IsEngaged", False) engaged_prev = False HARDWARE.set_power_save(not should_start) if sm.updated['controlsState']: engaged = sm['controlsState'].enabled if engaged != engaged_prev: params.put_bool("IsEngaged", engaged) engaged_prev = engaged try: with open('/dev/kmsg', 'w') as kmsg: kmsg.write(f"<3>[thermald] engaged: {engaged}\n") except Exception: pass if should_start: off_ts = None if started_ts is None: started_ts = sec_since_boot() started_seen = True else: if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev): cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions) started_ts = None if off_ts is None: off_ts = sec_since_boot() # Offroad power monitoring power_monitor.calculate(peripheralState, onroad_conditions["ignition"]) msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used() msg.deviceState.carBatteryCapacityUwh = max( 0, power_monitor.get_car_battery_capacity()) current_power_draw = HARDWARE.get_current_power_draw() statlog.sample("power_draw", current_power_draw) msg.deviceState.powerDrawW = current_power_draw som_power_draw = HARDWARE.get_som_power_draw() statlog.sample("som_power_draw", som_power_draw) msg.deviceState.somPowerDrawW = som_power_draw # Check if we need to disable charging (handled by boardd) msg.deviceState.chargingDisabled = power_monitor.should_disable_charging( onroad_conditions["ignition"], in_car, off_ts) # Check if we need to shut down if power_monitor.should_shutdown(peripheralState, onroad_conditions["ignition"], in_car, off_ts, started_seen): cloudlog.warning(f"shutting device down, offroad since {off_ts}") params.put_bool("DoShutdown", True) msg.deviceState.chargingError = current_filter.x > 0. and msg.deviceState.batteryPercent < 90 # if current is positive, then battery is being discharged msg.deviceState.started = started_ts is not None msg.deviceState.startedMonoTime = int(1e9 * (started_ts or 0)) last_ping = params.get("LastAthenaPingTime") if last_ping is not None: msg.deviceState.lastAthenaPingTime = int(last_ping) msg.deviceState.thermalStatus = thermal_status pm.send("deviceState", msg) should_start_prev = should_start startup_conditions_prev = startup_conditions.copy() # Log to statsd statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent) statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent) statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent) for i, usage in enumerate(msg.deviceState.cpuUsagePercent): statlog.gauge(f"cpu{i}_usage_percent", usage) for i, temp in enumerate(msg.deviceState.cpuTempC): statlog.gauge(f"cpu{i}_temperature", temp) for i, temp in enumerate(msg.deviceState.gpuTempC): statlog.gauge(f"gpu{i}_temperature", temp) statlog.gauge("memory_temperature", msg.deviceState.memoryTempC) statlog.gauge("ambient_temperature", msg.deviceState.ambientTempC) for i, temp in enumerate(msg.deviceState.pmicTempC): statlog.gauge(f"pmic{i}_temperature", temp) for i, temp in enumerate(last_hw_state.nvme_temps): statlog.gauge(f"nvme_temperature{i}", temp) for i, temp in enumerate(last_hw_state.modem_temps): statlog.gauge(f"modem_temperature{i}", temp) statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired) statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent) # report to server once every 10 minutes if (count % int(600. / DT_TRML)) == 0: cloudlog.event( "STATUS_PACKET", count=count, pandaStates=[ strip_deprecated_keys(p.to_dict()) for p in pandaStates ], peripheralState=strip_deprecated_keys( peripheralState.to_dict()), location=(strip_deprecated_keys( sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None), deviceState=strip_deprecated_keys(msg.to_dict())) count += 1
def manager_init() -> None: # update system time from panda set_time(cloudlog) # save boot log subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) params = Params() params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) default_params: List[Tuple[str, Union[str, bytes]]] = [ ("CompletedTrainingVersion", "0"), ("DisengageOnAccelerator", "1"), ("HasAcceptedTerms", "0"), ("OpenpilotEnabledToggle", "1"), ] if not PC: default_params.append( ("LastUpdateTime", datetime.datetime.utcnow().isoformat().encode('utf8'))) if params.get_bool("RecordFrontLock"): params.put_bool("RecordFront", True) if not params.get_bool("DisableRadar_Allow"): params.delete("DisableRadar") # set unset params for k, v in default_params: if params.get(k) is None: params.put(k, v) # is this dashcam? if os.getenv("PASSIVE") is not None: params.put_bool("Passive", bool(int(os.getenv("PASSIVE", "0")))) if params.get("Passive") is None: raise Exception("Passive must be set to continue") # Create folders needed for msgq try: os.mkdir("/dev/shm") except FileExistsError: pass except PermissionError: print("WARNING: failed to make /dev/shm") # set version params params.put("Version", get_version()) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_commit(default="")) params.put("GitBranch", get_short_branch(default="")) params.put("GitRemote", get_origin(default="")) params.put_bool("IsTestedBranch", is_tested_branch()) # set dongle id reg_res = register(show_spinner=True) if reg_res: dongle_id = reg_res else: serial = params.get("HardwareSerial") raise Exception(f"Registration failed for device {serial}") os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog if not is_dirty(): os.environ['CLEAN'] = '1' # init logging sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, version=get_version(), dirty=is_dirty(), device=HARDWARE.get_device_type())
def do_reboot(): time.sleep(2) HARDWARE.reboot()
#!/usr/bin/env python3 from system.hardware import HARDWARE if __name__ == "__main__": HARDWARE.set_power_save(False)
def main() -> NoReturn: first_run = True params = Params() while True: try: params.delete("PandaSignatures") # Flash all Pandas in DFU mode for p in PandaDFU.list(): cloudlog.info( f"Panda in DFU mode found, flashing recovery {p}") PandaDFU(p).recover() time.sleep(1) panda_serials = Panda.list() if len(panda_serials) == 0: if first_run: cloudlog.info("Resetting internal panda") HARDWARE.reset_internal_panda() time.sleep(2) # wait to come back up continue cloudlog.info( f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}" ) # Flash pandas pandas: List[Panda] = [] for serial in panda_serials: pandas.append(flash_panda(serial)) # check health for lost heartbeat for panda in pandas: health = panda.health() if health["heartbeat_lost"]: params.put_bool("PandaHeartbeatLost", True) cloudlog.event("heartbeat lost", deviceState=health, serial=panda.get_usb_serial()) if first_run: cloudlog.info(f"Resetting panda {panda.get_usb_serial()}") panda.reset() # sort pandas to have deterministic order pandas.sort(key=cmp_to_key(panda_sort_cmp)) panda_serials = list(map(lambda p: p.get_usb_serial(), pandas)) # type: ignore # log panda fw versions params.put("PandaSignatures", b','.join(p.get_signature() for p in pandas)) # close all pandas for p in pandas: p.close() except (usb1.USBErrorNoDevice, usb1.USBErrorPipe): # a panda was disconnected while setting everything up. let's try again cloudlog.exception("Panda USB exception while setting up") continue first_run = False # run boardd with all connected serials as arguments os.environ['MANAGER_DAEMON'] = 'boardd' os.chdir(os.path.join(BASEDIR, "selfdrive/boardd")) subprocess.run(["./boardd", *panda_serials], check=True)
def __init__(self, sm=None, pm=None, can_sock=None, CI=None): config_realtime_process(4, Priority.CTRL_HIGH) # Setup sockets self.pm = pm if self.pm is None: self.pm = messaging.PubMaster([ 'sendcan', 'controlsState', 'carState', 'carControl', 'carEvents', 'carParams' ]) self.camera_packets = [ "roadCameraState", "driverCameraState", "wideRoadCameraState" ] self.can_sock = can_sock if can_sock is None: can_timeout = None if os.environ.get('NO_CAN_TIMEOUT', False) else 20 self.can_sock = messaging.sub_sock('can', timeout=can_timeout) self.log_sock = messaging.sub_sock('androidLog') if CI is None: # wait for one pandaState and one CAN packet print("Waiting for CAN messages...") get_one_can(self.can_sock) self.CI, self.CP = get_car(self.can_sock, self.pm.sock['sendcan']) else: self.CI, self.CP = CI, CI.CP params = Params() self.joystick_mode = params.get_bool("JoystickDebugMode") or ( self.CP.notCar and sm is None) joystick_packet = ['testJoystick'] if self.joystick_mode else [] self.sm = sm if self.sm is None: ignore = ['driverCameraState', 'managerState' ] if SIMULATION else None self.sm = messaging.SubMaster( [ 'deviceState', 'pandaStates', 'peripheralState', 'modelV2', 'liveCalibration', 'driverMonitoringState', 'longitudinalPlan', 'lateralPlan', 'liveLocationKalman', 'managerState', 'liveParameters', 'radarState' ] + self.camera_packets + joystick_packet, ignore_alive=ignore, ignore_avg_freq=['radarState', 'longitudinalPlan']) # set alternative experiences from parameters self.disengage_on_accelerator = params.get_bool( "DisengageOnAccelerator") self.CP.alternativeExperience = 0 if not self.disengage_on_accelerator: self.CP.alternativeExperience |= ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS if self.CP.dashcamOnly and params.get_bool("DashcamOverride"): self.CP.dashcamOnly = False # read params self.is_metric = params.get_bool("IsMetric") self.is_ldw_enabled = params.get_bool("IsLdwEnabled") openpilot_enabled_toggle = params.get_bool("OpenpilotEnabledToggle") passive = params.get_bool("Passive") or not openpilot_enabled_toggle # detect sound card presence and ensure successful init sounds_available = HARDWARE.get_sound_card_online() car_recognized = self.CP.carName != 'mock' controller_available = self.CI.CC is not None and not passive and not self.CP.dashcamOnly self.read_only = not car_recognized or not controller_available or self.CP.dashcamOnly if self.read_only: safety_config = car.CarParams.SafetyConfig.new_message() safety_config.safetyModel = car.CarParams.SafetyModel.noOutput self.CP.safetyConfigs = [safety_config] # Write CarParams for radard cp_bytes = self.CP.to_bytes() params.put("CarParams", cp_bytes) put_nonblocking("CarParamsCache", cp_bytes) self.CC = car.CarControl.new_message() self.CS_prev = car.CarState.new_message() self.AM = AlertManager() self.events = Events() self.LoC = LongControl(self.CP) self.VM = VehicleModel(self.CP) self.LaC: LatControl if self.CP.steerControlType == car.CarParams.SteerControlType.angle: self.LaC = LatControlAngle(self.CP, self.CI) elif self.CP.lateralTuning.which() == 'pid': self.LaC = LatControlPID(self.CP, self.CI) elif self.CP.lateralTuning.which() == 'indi': self.LaC = LatControlINDI(self.CP, self.CI) elif self.CP.lateralTuning.which() == 'torque': self.LaC = LatControlTorque(self.CP, self.CI) self.initialized = False self.state = State.disabled self.enabled = False self.active = False self.can_rcv_error = False self.soft_disable_timer = 0 self.v_cruise_kph = 255 self.v_cruise_kph_last = 0 self.mismatch_counter = 0 self.cruise_mismatch_counter = 0 self.can_rcv_error_counter = 0 self.last_blinker_frame = 0 self.distance_traveled = 0 self.last_functional_fan_frame = 0 self.events_prev = [] self.current_alert_types = [ET.PERMANENT] self.logged_comm_issue = None self.button_timers = { ButtonEvent.Type.decelCruise: 0, ButtonEvent.Type.accelCruise: 0 } self.last_actuators = car.CarControl.Actuators.new_message() self.desired_curvature = 0.0 self.desired_curvature_rate = 0.0 # TODO: no longer necessary, aside from process replay self.sm['liveParameters'].valid = True self.startup_event = get_startup_event(car_recognized, controller_available, len(self.CP.carFw) > 0) if not sounds_available: self.events.add(EventName.soundsUnavailable, static=True) if not car_recognized: self.events.add(EventName.carUnrecognized, static=True) if len(self.CP.carFw) > 0: set_offroad_alert("Offroad_CarUnrecognized", True) else: set_offroad_alert("Offroad_NoFirmware", True) elif self.read_only: self.events.add(EventName.dashcamMode, static=True) elif self.joystick_mode: self.events.add(EventName.joystickDebug, static=True) self.startup_event = None # controlsd is driven by can recv, expected at 100Hz self.rk = Ratekeeper(100, print_delay_threshold=None) self.prof = Profiler(False) # off by default
def getNetworkMetered(): network_type = HARDWARE.get_network_type() return HARDWARE.get_network_metered(network_type)
def hw_state_thread(end_event, hw_queue): """Handles non critical hardware state, and sends over queue""" count = 0 registered_count = 0 prev_hw_state = None modem_version = None modem_nv = None modem_configured = False while not end_event.is_set(): # these are expensive calls. update every 10s if (count % int(10. / DT_TRML)) == 0: try: network_type = HARDWARE.get_network_type() modem_temps = HARDWARE.get_modem_temperatures() if len(modem_temps) == 0 and prev_hw_state is not None: modem_temps = prev_hw_state.modem_temps # Log modem version once if AGNOS and ((modem_version is None) or (modem_nv is None)): modem_version = HARDWARE.get_modem_version() # pylint: disable=assignment-from-none modem_nv = HARDWARE.get_modem_nv() # pylint: disable=assignment-from-none if (modem_version is not None) and (modem_nv is not None): cloudlog.event("modem version", version=modem_version, nv=modem_nv) hw_state = HardwareState( network_type=network_type, network_metered=HARDWARE.get_network_metered(network_type), network_strength=HARDWARE.get_network_strength( network_type), network_info=HARDWARE.get_network_info(), nvme_temps=HARDWARE.get_nvme_temperatures(), modem_temps=modem_temps, ) try: hw_queue.put_nowait(hw_state) except queue.Full: pass if AGNOS and (hw_state.network_info is not None) and (hw_state.network_info.get( 'state', None) == "REGISTERED"): registered_count += 1 else: registered_count = 0 if registered_count > 10: cloudlog.warning( f"Modem stuck in registered state {hw_state.network_info}. nmcli conn up lte" ) os.system("nmcli conn up lte") registered_count = 0 # TODO: remove this once the config is in AGNOS if not modem_configured and len(HARDWARE.get_sim_info().get( 'sim_id', '')) > 0: cloudlog.warning("configuring modem") HARDWARE.configure_modem() modem_configured = True prev_hw_state = hw_state except Exception: cloudlog.exception("Error getting hardware state") count += 1 time.sleep(DT_TRML)
def register(show_spinner=False) -> Optional[str]: params = Params() params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id: Optional[str] = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) pubkey = Path(PERSIST + "/comma/id_rsa.pub") if not pubkey.is_file(): dongle_id = UNREGISTERED_DONGLE_ID cloudlog.warning(f"missing public key: {pubkey}") elif needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2: public_key = f1.read() private_key = f2.read() # Block until we get the imei serial = HARDWARE.get_serial() start_time = time.monotonic() imei1: Optional[str] = None imei2: Optional[str] = None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) params.put("IMEI", imei1) params.put("HardwareSerial", serial) backoff = 0 start_time = time.monotonic() while True: try: register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code in (402, 403): cloudlog.info( f"Unable to register device, got {resp.status_code}") dongle_id = UNREGISTERED_DONGLE_ID else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] break except Exception: cloudlog.exception("failed to authenticate") backoff = min(backoff + 1, 15) time.sleep(backoff) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) if show_spinner: spinner.close() if dongle_id: params.put("DongleId", dongle_id) set_offroad_alert("Offroad_UnofficialHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC) return dongle_id
def main() -> NoReturn: dongle_id = Params().get("DongleId", encoding='utf-8') def get_influxdb_line(measurement: str, value: Union[float, Dict[str, float]], timestamp: datetime, tags: dict) -> str: res = f"{measurement}" for k, v in tags.items(): res += f",{k}={str(v)}" res += " " if isinstance(value, float): value = {'value': value} for k, v in value.items(): res += f"{k}={v}," res += f"dongle_id=\"{dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n" return res # open statistics socket ctx = zmq.Context().instance() sock = ctx.socket(zmq.PULL) sock.bind(STATS_SOCKET) # initialize stats directory Path(STATS_DIR).mkdir(parents=True, exist_ok=True) # initialize tags tags = { 'started': False, 'version': get_short_version(), 'branch': get_short_branch(), 'dirty': is_dirty(), 'origin': get_normalized_origin(), 'deviceType': HARDWARE.get_device_type(), } # subscribe to deviceState for started state sm = SubMaster(['deviceState']) idx = 0 last_flush_time = time.monotonic() gauges = {} samples: Dict[str, List[float]] = defaultdict(list) while True: started_prev = sm['deviceState'].started sm.update() # Update metrics while True: try: metric = sock.recv_string(zmq.NOBLOCK) try: metric_type = metric.split('|')[1] metric_name = metric.split(':')[0] metric_value = float(metric.split('|')[0].split(':')[1]) if metric_type == METRIC_TYPE.GAUGE: gauges[metric_name] = metric_value elif metric_type == METRIC_TYPE.SAMPLE: samples[metric_name].append(metric_value) else: cloudlog.event("unknown metric type", metric_type=metric_type) except Exception: cloudlog.event("malformed metric", metric=metric) except zmq.error.Again: break # flush when started state changes or after FLUSH_TIME_S if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev): result = "" current_time = datetime.utcnow().replace(tzinfo=timezone.utc) tags['started'] = sm['deviceState'].started for key, value in gauges.items(): result += get_influxdb_line(f"gauge.{key}", value, current_time, tags) for key, values in samples.items(): values.sort() sample_count = len(values) sample_sum = sum(values) stats = { 'count': sample_count, 'min': values[0], 'max': values[-1], 'mean': sample_sum / sample_count, } for percentile in [0.05, 0.5, 0.95]: value = values[int(round(percentile * (sample_count - 1)))] stats[f"p{int(percentile * 100)}"] = value result += get_influxdb_line(f"sample.{key}", stats, current_time, tags) # clear intermediate data gauges.clear() samples.clear() last_flush_time = time.monotonic() # check that we aren't filling up the drive if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT: if len(result) > 0: stats_path = os.path.join(STATS_DIR, f"{current_time.timestamp():.0f}_{idx}") with atomic_write_in_dir(stats_path) as f: f.write(result) idx += 1 else: cloudlog.error("stats dir full")
def getSimInfo(): return HARDWARE.get_sim_info()