def get_data(self, timeout, total_timeout=60.):
self._drain_rx()
# Create message objects
msgs = {}
request_counter = {}
request_done = {}
for tx_addr, rx_addr in self.msg_addrs.items():
# rx_addr not set when using functional tx addr
id_addr = rx_addr or tx_addr[0]
sub_addr = tx_addr[1]
can_client = CanClient(self._can_tx,
partial(self._can_rx,
id_addr,
sub_addr=sub_addr),
tx_addr[0],
rx_addr,
self.bus,
sub_addr=sub_addr,
debug=self.debug)
max_len = 8 if sub_addr is None else 7
msg = IsoTpMessage(can_client,
timeout=0,
max_len=max_len,
debug=self.debug)
msg.send(self.request[0])
msgs[tx_addr] = msg
request_counter[tx_addr] = 0
request_done[tx_addr] = False
results = {}
start_time = time.monotonic()
response_timeouts = {
tx_addr: start_time + timeout
for tx_addr in self.msg_addrs
}
while True:
self.rx()
if all(request_done.values()):
break
for tx_addr, msg in msgs.items():
try:
dat: Optional[bytes] = msg.recv()
except Exception:
cloudlog.exception("Error processing UDS response")
request_done[tx_addr] = True
continue
if not dat:
continue
counter = request_counter[tx_addr]
expected_response = self.response[counter]
response_valid = dat[:len(expected_response
)] == expected_response
if response_valid:
response_timeouts[tx_addr] = time.monotonic() + timeout
if counter + 1 < len(self.request):
msg.send(self.request[counter + 1])
request_counter[tx_addr] += 1
else:
results[(tx_addr, msg._can_client.rx_addr
)] = dat[len(expected_response):]
request_done[tx_addr] = True
else:
error_code = dat[2] if len(dat) > 2 else -1
if error_code == 0x78:
response_timeouts[tx_addr] = time.monotonic(
) + self.response_pending_timeout
if self.debug:
cloudlog.warning(
f"iso-tp query response pending: {tx_addr}")
else:
response_timeouts[tx_addr] = 0
request_done[tx_addr] = True
cloudlog.warning(
f"iso-tp query bad response: {tx_addr} - 0x{dat.hex()}"
)
cur_time = time.monotonic()
if cur_time - max(response_timeouts.values()) > 0:
for tx_addr in msgs:
if request_counter[tx_addr] > 0 and not request_done[
tx_addr]:
cloudlog.warning(
f"iso-tp query timeout after receiving response: {tx_addr}"
)
break
if cur_time - start_time > total_timeout:
cloudlog.warning("iso-tp query timeout while receiving data")
break
return results
def calculate_route(self, destination):
cloudlog.warning(
f"Calculating route {self.last_position} -> {destination}")
self.nav_destination = destination
params = {
'access_token': self.mapbox_token,
'annotations': 'maxspeed',
'geometries': 'geojson',
'overview': 'full',
'steps': 'true',
'banner_instructions': 'true',
'alternatives': 'false',
}
if self.last_bearing is not None:
params['bearings'] = f"{(self.last_bearing + 360) % 360:.0f},90;"
url = self.mapbox_host + f'/directions/v5/mapbox/driving-traffic/{self.last_position.longitude},{self.last_position.latitude};{destination.longitude},{destination.latitude}'
try:
resp = requests.get(url, params=params)
resp.raise_for_status()
r = resp.json()
if len(r['routes']):
self.route = r['routes'][0]['legs'][0]['steps']
self.route_geometry = []
maxspeed_idx = 0
maxspeeds = r['routes'][0]['legs'][0]['annotation']['maxspeed']
# Convert coordinates
for step in self.route:
coords = []
for c in step['geometry']['coordinates']:
coord = Coordinate.from_mapbox_tuple(c)
# Last step does not have maxspeed
if (maxspeed_idx < len(maxspeeds)):
maxspeed = maxspeeds[maxspeed_idx]
if ('unknown'
not in maxspeed) and ('none'
not in maxspeed):
coord.annotations['maxspeed'] = maxspeed_to_ms(
maxspeed)
coords.append(coord)
maxspeed_idx += 1
self.route_geometry.append(coords)
maxspeed_idx -= 1 # Every segment ends with the same coordinate as the start of the next
self.step_idx = 0
else:
cloudlog.warning("Got empty route response")
self.clear_route()
except requests.exceptions.RequestException:
cloudlog.exception("failed to get route")
self.clear_route()
self.send_route()
def main() -> None:
params = Params()
if params.get_bool("DisableUpdates"):
cloudlog.warning("updates are disabled by the DisableUpdates param")
exit(0)
ov_lock_fd = open(LOCK_FILE, 'w')
try:
fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
raise RuntimeError(
"couldn't get overlay lock; is another instance running?") from e
# Set low io priority
proc = psutil.Process()
if psutil.LINUX:
proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
# Check if we just performed an update
if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
cloudlog.event("update installed")
if not params.get("InstallDate"):
t = datetime.datetime.utcnow().isoformat()
params.put("InstallDate", t.encode('utf8'))
overlay_init = Path(os.path.join(BASEDIR, ".overlay_init"))
overlay_init.unlink(missing_ok=True)
update_failed_count = 0 # TODO: Load from param?
wait_helper = WaitTimeHelper(proc)
# Run the update loop
while not wait_helper.shutdown:
wait_helper.ready_event.clear()
# Attempt an update
exception = None
new_version = False
update_failed_count += 1
try:
init_overlay()
# TODO: still needed? skip this and just fetch?
# Lightweight internt check
internet_ok, update_available = check_for_update()
if internet_ok and not update_available:
update_failed_count = 0
# Fetch update
if internet_ok:
new_version = fetch_update(wait_helper)
update_failed_count = 0
except subprocess.CalledProcessError as e:
cloudlog.event("update process failed",
cmd=e.cmd,
output=e.output,
returncode=e.returncode)
exception = f"command failed: {e.cmd}\n{e.output}"
overlay_init.unlink(missing_ok=True)
except Exception as e:
cloudlog.exception("uncaught updated exception, shouldn't happen")
exception = str(e)
overlay_init.unlink(missing_ok=True)
if not wait_helper.shutdown:
try:
set_params(new_version, update_failed_count, exception)
except Exception:
cloudlog.exception(
"uncaught updated exception while setting params, shouldn't happen"
)
# infrequent attempts if we successfully updated recently
wait_helper.sleep(5 * 60 if update_failed_count > 0 else 90 * 60)
dismount_overlay()
elif params.get_bool("DoReboot"):
cloudlog.warning("reboot")
HARDWARE.reboot()
elif params.get_bool("DoShutdown"):
cloudlog.warning("shutdown")
HARDWARE.shutdown()
if __name__ == "__main__":
unblock_stdout()
try:
main()
except Exception:
add_file_handler(cloudlog)
cloudlog.exception("Manager failed to start")
try:
managed_processes['ui'].stop()
except Exception:
pass
# Show last 3 lines of traceback
error = traceback.format_exc(-3)
error = "Manager failed to start\n\n" + error
with TextWindow(error) as t:
t.wait_for_exit()
raise
# manual exit because we are forked
def log_handler(end_event):
if PC:
return
log_files = []
last_scan = 0
while not end_event.is_set():
try:
curr_scan = sec_since_boot()
if curr_scan - last_scan > 10:
log_files = get_logs_to_send_sorted()
last_scan = curr_scan
# send one log
curr_log = None
if len(log_files) > 0:
log_entry = log_files.pop() # newest log file
cloudlog.debug(
f"athena.log_handler.forward_request {log_entry}")
try:
curr_time = int(time.time())
log_path = os.path.join(SWAGLOG_DIR, log_entry)
setxattr(log_path, LOG_ATTR_NAME,
int.to_bytes(curr_time, 4, sys.byteorder))
with open(log_path) as f:
jsonrpc = {
"method": "forwardLogs",
"params": {
"logs": f.read()
},
"jsonrpc": "2.0",
"id": log_entry
}
low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
curr_log = log_entry
except OSError:
pass # file could be deleted by log rotation
# wait for response up to ~100 seconds
# always read queue at least once to process any old responses that arrive
for _ in range(100):
if end_event.is_set():
break
try:
log_resp = json.loads(log_recv_queue.get(timeout=1))
log_entry = log_resp.get("id")
log_success = "result" in log_resp and log_resp[
"result"].get("success")
cloudlog.debug(
f"athena.log_handler.forward_response {log_entry} {log_success}"
)
if log_entry and log_success:
log_path = os.path.join(SWAGLOG_DIR, log_entry)
try:
setxattr(log_path, LOG_ATTR_NAME,
LOG_ATTR_VALUE_MAX_UNIX_TIME)
except OSError:
pass # file could be deleted by log rotation
if curr_log == log_entry:
break
except queue.Empty:
if curr_log is None:
break
except Exception:
cloudlog.exception("athena.log_handler.exception")
def upload_handler(end_event: threading.Event) -> None:
sm = messaging.SubMaster(['deviceState'])
tid = threading.get_ident()
while not end_event.is_set():
cur_upload_items[tid] = None
try:
cur_upload_items[tid] = upload_queue.get(timeout=1)._replace(
current=True)
if cur_upload_items[tid].id in cancelled_uploads:
cancelled_uploads.remove(cur_upload_items[tid].id)
continue
# Remove item if too old
age = datetime.now() - datetime.fromtimestamp(
cur_upload_items[tid].created_at / 1000)
if age.total_seconds() > MAX_AGE:
cloudlog.event("athena.upload_handler.expired",
item=cur_upload_items[tid],
error=True)
continue
# Check if uploading over metered connection is allowed
sm.update(0)
metered = sm['deviceState'].networkMetered
network_type = sm['deviceState'].networkType.raw
if metered and (not cur_upload_items[tid].allow_cellular):
retry_upload(tid, end_event, False)
continue
try:
def cb(sz, cur):
# Abort transfer if connection changed to metered after starting upload
sm.update(0)
metered = sm['deviceState'].networkMetered
if metered and (not cur_upload_items[tid].allow_cellular):
raise AbortTransferException
cur_upload_items[tid] = cur_upload_items[tid]._replace(
progress=cur / sz if sz else 1)
fn = cur_upload_items[tid].path
try:
sz = os.path.getsize(fn)
except OSError:
sz = -1
cloudlog.event("athena.upload_handler.upload_start",
fn=fn,
sz=sz,
network_type=network_type,
metered=metered,
retry_count=cur_upload_items[tid].retry_count)
response = _do_upload(cur_upload_items[tid], cb)
if response.status_code not in (200, 201, 401, 403, 412):
cloudlog.event("athena.upload_handler.retry",
status_code=response.status_code,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
retry_upload(tid, end_event)
else:
cloudlog.event("athena.upload_handler.success",
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
UploadQueueCache.cache(upload_queue)
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
requests.exceptions.SSLError):
cloudlog.event("athena.upload_handler.timeout",
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
retry_upload(tid, end_event)
except AbortTransferException:
cloudlog.event("athena.upload_handler.abort",
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
retry_upload(tid, end_event, False)
except queue.Empty:
pass
except Exception:
cloudlog.exception("athena.upload_handler.exception")
def main() -> NoReturn:
first_run = True
params = Params()
while True:
try:
params.delete("PandaSignatures")
# Flash all Pandas in DFU mode
for p in PandaDFU.list():
cloudlog.info(
f"Panda in DFU mode found, flashing recovery {p}")
PandaDFU(p).recover()
time.sleep(1)
panda_serials = Panda.list()
if len(panda_serials) == 0:
if first_run:
cloudlog.info("Resetting internal panda")
HARDWARE.reset_internal_panda()
time.sleep(2) # wait to come back up
continue
cloudlog.info(
f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}"
)
# Flash pandas
pandas: List[Panda] = []
for serial in panda_serials:
pandas.append(flash_panda(serial))
# check health for lost heartbeat
for panda in pandas:
health = panda.health()
if health["heartbeat_lost"]:
params.put_bool("PandaHeartbeatLost", True)
cloudlog.event("heartbeat lost",
deviceState=health,
serial=panda.get_usb_serial())
if first_run:
cloudlog.info(f"Resetting panda {panda.get_usb_serial()}")
panda.reset()
# sort pandas to have deterministic order
pandas.sort(key=cmp_to_key(panda_sort_cmp))
panda_serials = list(map(lambda p: p.get_usb_serial(),
pandas)) # type: ignore
# log panda fw versions
params.put("PandaSignatures",
b','.join(p.get_signature() for p in pandas))
# close all pandas
for p in pandas:
p.close()
except (usb1.USBErrorNoDevice, usb1.USBErrorPipe):
# a panda was disconnected while setting everything up. let's try again
cloudlog.exception("Panda USB exception while setting up")
continue
first_run = False
# run boardd with all connected serials as arguments
os.environ['MANAGER_DAEMON'] = 'boardd'
os.chdir(os.path.join(BASEDIR, "selfdrive/boardd"))
subprocess.run(["./boardd", *panda_serials], check=True)
def register(show_spinner=False) -> Optional[str]:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id: Optional[str] = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
pubkey = Path(PERSIST + "/comma/id_rsa.pub")
if not pubkey.is_file():
dongle_id = UNREGISTERED_DONGLE_ID
cloudlog.warning(f"missing public key: {pubkey}")
elif needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST +
"/comma/id_rsa.pub") as f1, open(PERSIST +
"/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
serial = HARDWARE.get_serial()
start_time = time.monotonic()
imei1: Optional[str] = None
imei2: Optional[str] = None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
backoff = 0
start_time = time.monotonic()
while True:
try:
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
if resp.status_code in (402, 403):
cloudlog.info(
f"Unable to register device, got {resp.status_code}")
dongle_id = UNREGISTERED_DONGLE_ID
else:
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
break
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
if show_spinner:
spinner.close()
if dongle_id:
params.put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware",
(dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
return dongle_id
def hw_state_thread(end_event, hw_queue):
"""Handles non critical hardware state, and sends over queue"""
count = 0
registered_count = 0
prev_hw_state = None
modem_version = None
modem_nv = None
modem_configured = False
while not end_event.is_set():
# these are expensive calls. update every 10s
if (count % int(10. / DT_TRML)) == 0:
try:
network_type = HARDWARE.get_network_type()
modem_temps = HARDWARE.get_modem_temperatures()
if len(modem_temps) == 0 and prev_hw_state is not None:
modem_temps = prev_hw_state.modem_temps
# Log modem version once
if AGNOS and ((modem_version is None) or (modem_nv is None)):
modem_version = HARDWARE.get_modem_version() # pylint: disable=assignment-from-none
modem_nv = HARDWARE.get_modem_nv() # pylint: disable=assignment-from-none
if (modem_version is not None) and (modem_nv is not None):
cloudlog.event("modem version",
version=modem_version,
nv=modem_nv)
hw_state = HardwareState(
network_type=network_type,
network_metered=HARDWARE.get_network_metered(network_type),
network_strength=HARDWARE.get_network_strength(
network_type),
network_info=HARDWARE.get_network_info(),
nvme_temps=HARDWARE.get_nvme_temperatures(),
modem_temps=modem_temps,
)
try:
hw_queue.put_nowait(hw_state)
except queue.Full:
pass
if AGNOS and (hw_state.network_info
is not None) and (hw_state.network_info.get(
'state', None) == "REGISTERED"):
registered_count += 1
else:
registered_count = 0
if registered_count > 10:
cloudlog.warning(
f"Modem stuck in registered state {hw_state.network_info}. nmcli conn up lte"
)
os.system("nmcli conn up lte")
registered_count = 0
# TODO: remove this once the config is in AGNOS
if not modem_configured and len(HARDWARE.get_sim_info().get(
'sim_id', '')) > 0:
cloudlog.warning("configuring modem")
HARDWARE.configure_modem()
modem_configured = True
prev_hw_state = hw_state
except Exception:
cloudlog.exception("Error getting hardware state")
count += 1
time.sleep(DT_TRML)
