Example #1
0
                    def perform_pulse_measurement(now):
                        try:
                            HARDWARE.set_battery_charging(False)
                            time.sleep(5)

                            # Measure for a few sec to get a good average
                            voltages = []
                            currents = []
                            for _ in range(6):
                                voltages.append(HARDWARE.get_battery_voltage())
                                currents.append(HARDWARE.get_battery_current())
                                time.sleep(1)
                            current_power = ((mean(voltages) / 1000000) *
                                             (mean(currents) / 1000000))

                            self._perform_integration(
                                now, current_power * FUDGE_FACTOR)

                            # Enable charging again
                            HARDWARE.set_battery_charging(True)
                        except Exception:
                            cloudlog.exception(
                                "Pulsed power measurement failed")
Example #2
0
def thermald_thread():
  health_timeout = int(1000 * 2.5 * DT_TRML)  # 2.5x the expected health frequency

  # now loop
  thermal_sock = messaging.pub_sock('thermal')
  health_sock = messaging.sub_sock('health', timeout=health_timeout)
  location_sock = messaging.sub_sock('gpsLocation')

  fan_speed = 0
  count = 0

  startup_conditions = {
    "ignition": False,
  }
  startup_conditions_prev = startup_conditions.copy()

  off_ts = None
  started_ts = None
  started_seen = False
  thermal_status = ThermalStatus.green
  usb_power = True
  current_branch = get_git_branch()

  network_type = NetworkType.none
  network_strength = NetworkStrength.unknown

  current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML)
  cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML)
  health_prev = None
  should_start_prev = False
  handle_fan = None
  is_uno = False

  params = Params()
  pm = PowerMonitoring()
  no_panda_cnt = 0

  thermal_config = get_thermal_config()

  while 1:
    health = messaging.recv_sock(health_sock, wait=True)
    location = messaging.recv_sock(location_sock)
    location = location.gpsLocation if location else None
    msg = read_thermal(thermal_config)

    if health is not None:
      usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client

      # If we lose connection to the panda, wait 5 seconds before going offroad
      if health.health.hwType == log.HealthData.HwType.unknown:
        no_panda_cnt += 1
        if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML:
          if startup_conditions["ignition"]:
            cloudlog.error("Lost panda connection while onroad")
          startup_conditions["ignition"] = False
      else:
        no_panda_cnt = 0
        startup_conditions["ignition"] = health.health.ignitionLine or health.health.ignitionCan

      # Setup fan handler on first connect to panda
      if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown:
        is_uno = health.health.hwType == log.HealthData.HwType.uno

        if (not EON) or is_uno:
          cloudlog.info("Setting up UNO fan handler")
          handle_fan = handle_fan_uno
        else:
          cloudlog.info("Setting up EON fan handler")
          setup_eon_fan()
          handle_fan = handle_fan_eon

      # Handle disconnect
      if health_prev is not None:
        if health.health.hwType == log.HealthData.HwType.unknown and \
          health_prev.health.hwType != log.HealthData.HwType.unknown:
          params.panda_disconnect()
      health_prev = health

    # get_network_type is an expensive call. update every 10s
    if (count % int(10. / DT_TRML)) == 0:
      try:
        network_type = HARDWARE.get_network_type()
        network_strength = HARDWARE.get_network_strength(network_type)
      except Exception:
        cloudlog.exception("Error getting network status")

    msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0
    msg.thermal.memUsedPercent = int(round(psutil.virtual_memory().percent))
    msg.thermal.cpuPerc = int(round(psutil.cpu_percent()))
    msg.thermal.networkType = network_type
    msg.thermal.networkStrength = network_strength
    msg.thermal.batteryPercent = HARDWARE.get_battery_capacity()
    msg.thermal.batteryStatus = HARDWARE.get_battery_status()
    msg.thermal.batteryCurrent = HARDWARE.get_battery_current()
    msg.thermal.batteryVoltage = HARDWARE.get_battery_voltage()
    msg.thermal.usbOnline = HARDWARE.get_usb_present()

    # Fake battery levels on uno for frame
    if (not EON) or is_uno:
      msg.thermal.batteryPercent = 100
      msg.thermal.batteryStatus = "Charging"
      msg.thermal.bat = 0

    current_filter.update(msg.thermal.batteryCurrent / 1e6)

    # TODO: add car battery voltage check
    max_cpu_temp = cpu_temp_filter.update(max(msg.thermal.cpu))
    max_comp_temp = max(max_cpu_temp, msg.thermal.mem, max(msg.thermal.gpu))
    bat_temp = msg.thermal.bat

    if handle_fan is not None:
      fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, startup_conditions["ignition"])
      msg.thermal.fanSpeed = fan_speed

    # If device is offroad we want to cool down before going onroad
    # since going onroad increases load and can make temps go over 107
    # We only do this if there is a relay that prevents the car from faulting
    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (sec_since_boot() - off_ts > 60 * 5))
    if max_cpu_temp > 107. or bat_temp >= 63. or (is_offroad_for_5_min and max_cpu_temp > 70.0):
      # onroad not allowed
      thermal_status = ThermalStatus.danger
    elif max_comp_temp > 96.0 or bat_temp > 60.:
      # hysteresis between onroad not allowed and engage not allowed
      thermal_status = clip(thermal_status, ThermalStatus.red, ThermalStatus.danger)
    elif max_cpu_temp > 94.0:
      # hysteresis between engage not allowed and uploader not allowed
      thermal_status = clip(thermal_status, ThermalStatus.yellow, ThermalStatus.red)
    elif max_cpu_temp > 80.0:
      # uploader not allowed
      thermal_status = ThermalStatus.yellow
    elif max_cpu_temp > 75.0:
      # hysteresis between uploader not allowed and all good
      thermal_status = clip(thermal_status, ThermalStatus.green, ThermalStatus.yellow)
    else:
      # all good
      thermal_status = ThermalStatus.green

    # **** starting logic ****

    # Check for last update time and display alerts if needed
    now = datetime.datetime.utcnow()

    # show invalid date/time alert
    startup_conditions["time_valid"] = (now.year > 2020) or (now.year == 2020 and now.month >= 10)
    set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"]))

    # Show update prompt
    try:
      last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8'))
    except (TypeError, ValueError):
      last_update = now
    dt = now - last_update

    update_failed_count = params.get("UpdateFailedCount")
    update_failed_count = 0 if update_failed_count is None else int(update_failed_count)
    last_update_exception = params.get("LastUpdateException", encoding='utf8')

    if update_failed_count > 15 and last_update_exception is not None:
      if current_branch in ["release2", "dashcam"]:
        extra_text = "Ensure the software is correctly installed"
      else:
        extra_text = last_update_exception

      set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False)
      set_offroad_alert_if_changed("Offroad_UpdateFailed", True, extra_text=extra_text)
    elif dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1:
      set_offroad_alert_if_changed("Offroad_UpdateFailed", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", True)
    elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT:
      remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0))
      set_offroad_alert_if_changed("Offroad_UpdateFailed", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining_time} days.")
    else:
      set_offroad_alert_if_changed("Offroad_UpdateFailed", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeeded", False)
      set_offroad_alert_if_changed("Offroad_ConnectivityNeededPrompt", False)

    startup_conditions["not_uninstalling"] = not params.get("DoUninstall") == b"1"
    startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version

    panda_signature = params.get("PandaFirmware")
    startup_conditions["fw_version_match"] = (panda_signature is None) or (panda_signature == FW_SIGNATURE)   # don't show alert is no panda is connected (None)
    set_offroad_alert_if_changed("Offroad_PandaFirmwareMismatch", (not startup_conditions["fw_version_match"]))

    # with 2% left, we killall, otherwise the phone will take a long time to boot
    startup_conditions["free_space"] = msg.thermal.freeSpace > 0.02
    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \
                                               (current_branch in ['dashcam', 'dashcam-staging'])
    startup_conditions["not_driver_view"] = not params.get("IsDriverViewEnabled") == b"1"
    startup_conditions["not_taking_snapshot"] = not params.get("IsTakingSnapshot") == b"1"
    # if any CPU gets above 107 or the battery gets above 63, kill all processes
    # controls will warn with CPU above 95 or battery above 60
    startup_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger
    set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", (not startup_conditions["device_temp_good"]))
    should_start = all(startup_conditions.values())

    startup_conditions["hardware_supported"] = health is not None and health.health.hwType not in [log.HealthData.HwType.whitePanda,
                                                                                                   log.HealthData.HwType.greyPanda]
    set_offroad_alert_if_changed("Offroad_HardwareUnsupported", health is not None and not startup_conditions["hardware_supported"])

    if should_start:
      if not should_start_prev:
        params.delete("IsOffroad")

      off_ts = None
      if started_ts is None:
        started_ts = sec_since_boot()
        started_seen = True
    else:
      if startup_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
        cloudlog.event("Startup blocked", startup_conditions=startup_conditions)
      if should_start_prev or (count == 0):
        params.put("IsOffroad", "1")

      started_ts = None
      if off_ts is None:
        off_ts = sec_since_boot()

    # Offroad power monitoring
    pm.calculate(health)
    msg.thermal.offroadPowerUsage = pm.get_power_used()
    msg.thermal.carBatteryCapacity = max(0, pm.get_car_battery_capacity())

    # Check if we need to disable charging (handled by boardd)
    msg.thermal.chargingDisabled = pm.should_disable_charging(health, off_ts)

    # Check if we need to shut down
    if pm.should_shutdown(health, off_ts, started_seen, LEON):
      cloudlog.info(f"shutting device down, offroad since {off_ts}")
      # TODO: add function for blocking cloudlog instead of sleep
      time.sleep(10)
      os.system('LD_LIBRARY_PATH="" svc power shutdown')

    msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90  # if current is positive, then battery is being discharged
    msg.thermal.started = started_ts is not None
    msg.thermal.startedTs = int(1e9*(started_ts or 0))

    msg.thermal.thermalStatus = thermal_status
    thermal_sock.send(msg.to_bytes())

    set_offroad_alert_if_changed("Offroad_ChargeDisabled", (not usb_power))

    should_start_prev = should_start
    startup_conditions_prev = startup_conditions.copy()

    # report to server once per minute
    if (count % int(60. / DT_TRML)) == 0:
      cloudlog.event("STATUS_PACKET",
                     count=count,
                     health=(health.to_dict() if health else None),
                     location=(location.to_dict() if location else None),
                     thermal=msg.to_dict())

    count += 1
Example #3
0
    def calculate(self, health):
        try:
            now = sec_since_boot()

            # If health is None, we're probably not in a car, so we don't care
            if health is None or health.health.hwType == log.HealthData.HwType.unknown:
                with self.integration_lock:
                    self.last_measurement_time = None
                    self.next_pulsed_measurement_time = None
                    self.power_used_uWh = 0
                return

            # Low-pass battery voltage
            self.car_voltage_mV = (
                (health.health.voltage * CAR_VOLTAGE_LOW_PASS_K) +
                (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))

            # Cap the car battery power and save it in a param every 10-ish seconds
            self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh,
                                                0)
            self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh,
                                                CAR_BATTERY_CAPACITY_uWh)
            if now - self.last_save_time >= 10:
                put_nonblocking("CarBatteryCapacity",
                                str(int(self.car_battery_capacity_uWh)))
                self.last_save_time = now

            # First measurement, set integration time
            with self.integration_lock:
                if self.last_measurement_time is None:
                    self.last_measurement_time = now
                    return

            if (health.health.ignitionLine or health.health.ignitionCan):
                # If there is ignition, we integrate the charging rate of the car
                with self.integration_lock:
                    self.power_used_uWh = 0
                    integration_time_h = (now -
                                          self.last_measurement_time) / 3600
                    if integration_time_h < 0:
                        raise ValueError(
                            f"Negative integration time: {integration_time_h}h"
                        )
                    self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W *
                                                      1e6 * integration_time_h)
                    self.last_measurement_time = now
            else:
                # No ignition, we integrate the offroad power used by the device
                is_uno = health.health.hwType == log.HealthData.HwType.uno
                # Get current power draw somehow
                current_power = HARDWARE.get_current_power_draw()
                if current_power is not None:
                    pass
                elif HARDWARE.get_battery_status() == 'Discharging':
                    # If the battery is discharging, we can use this measurement
                    # On C2: this is low by about 10-15%, probably mostly due to UNO draw not being factored in
                    current_power = (
                        (HARDWARE.get_battery_voltage() / 1000000) *
                        (HARDWARE.get_battery_current() / 1000000))
                elif (self.next_pulsed_measurement_time is not None) and (
                        self.next_pulsed_measurement_time <= now):
                    # TODO: Figure out why this is off by a factor of 3/4???
                    FUDGE_FACTOR = 1.33

                    # Turn off charging for about 10 sec in a thread that does not get killed on SIGINT, and perform measurement here to avoid blocking thermal
                    def perform_pulse_measurement(now):
                        try:
                            HARDWARE.set_battery_charging(False)
                            time.sleep(5)

                            # Measure for a few sec to get a good average
                            voltages = []
                            currents = []
                            for _ in range(6):
                                voltages.append(HARDWARE.get_battery_voltage())
                                currents.append(HARDWARE.get_battery_current())
                                time.sleep(1)
                            current_power = ((mean(voltages) / 1000000) *
                                             (mean(currents) / 1000000))

                            self._perform_integration(
                                now, current_power * FUDGE_FACTOR)

                            # Enable charging again
                            HARDWARE.set_battery_charging(True)
                        except Exception:
                            cloudlog.exception(
                                "Pulsed power measurement failed")

                    # Start pulsed measurement and return
                    threading.Thread(target=perform_pulse_measurement,
                                     args=(now, )).start()
                    self.next_pulsed_measurement_time = None
                    return

                elif self.next_pulsed_measurement_time is None and not is_uno:
                    # On a charging EON with black panda, or drawing more than 400mA out of a white/grey one
                    # Only way to get the power draw is to turn off charging for a few sec and check what the discharging rate is
                    # We shouldn't do this very often, so make sure it has been some long-ish random time interval
                    self.next_pulsed_measurement_time = now + random.randint(
                        120, 180)
                    return
                else:
                    # Do nothing
                    return

                # Do the integration
                self._perform_integration(now, current_power)
        except Exception:
            cloudlog.exception("Power monitoring calculation failed")