def camera_replay(lr, fr, desire=None, calib=None):
spinner = Spinner()
spinner.update("starting model replay")
pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan'])
sm = messaging.SubMaster(['modelV2'])
# TODO: add dmonitoringmodeld
print("preparing procs")
managed_processes['camerad'].prepare()
managed_processes['modeld'].prepare()
try:
print("starting procs")
managed_processes['camerad'].start()
managed_processes['modeld'].start()
time.sleep(5)
sm.update(1000)
print("procs started")
desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()}
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), replace_calib(msg, calib))
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibration":
pm.send(msg.which(), replace_calib(msg, calib))
elif msg.which() == "roadCameraState":
if desire is not None:
for i in desire[frame_idx].nonzero()[0]:
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = desires_by_index[i]
pm.send('lateralPlan', dat)
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1]
f.roadCameraState.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
managed_processes['modeld'].stop()
time.sleep(2)
managed_processes['camerad'].stop()
return log_msgs
def parse_cifar(dataset, mode):
features = []
labels = []
coarse_labels = []
batch_names = []
TARFILE, label_data, label_labels, label_coarse = get_data_params(dataset)
datanames = get_datanames(dataset, mode)
try:
spinner = Spinner(prefix="Loading {} data...".format(mode))
spinner.start()
tf = tarfile.open(TARFILE)
for dataname in datanames:
ti = tf.getmember(dataname)
data = unpickle(tf.extractfile(ti))
features.append(data[label_data])
labels.append(data[label_labels])
batch_names.extend([dataname.split('/')[1]] *
len(data[label_data]))
if dataset == 'cifar100superclass':
coarse_labels.append(data[label_coarse])
features = np.concatenate(features)
features = features.reshape(features.shape[0], 3, 32, 32)
features = features.transpose(0, 2, 3, 1).astype('uint8')
labels = np.concatenate(labels)
if dataset == 'cifar100superclass':
coarse_labels = np.concatenate(coarse_labels)
spinner.stop()
except KeyboardInterrupt:
spinner.stop()
sys.exit(1)
return features, labels, coarse_labels, batch_names
def update_hash_dict(self):
if self.num_proc is None:
self.num_proc = cpu_count() - 1
# check current hash_dict
current_files = set(self.image_filenames)
cache_files = self.hash_dict.keys()
lost_set = cache_files - current_files
target_files = list(current_files - cache_files)
if len(lost_set) + len(target_files) > 0:
try:
if len(self.hash_dict) == 0:
spinner = Spinner(
prefix=
"Calculating image hashes (hash-bits={} num-proc={})..."
.format(self.hash_bits, self.num_proc))
else:
spinner = Spinner(
prefix=
"Updating image hashes (hash-bits={} num-proc={})...".
format(self.hash_bits, self.num_proc))
spinner.start()
# del lost_set from hash_dict
for f in lost_set:
del self.hash_dict[f]
if six.PY2:
from pathos.multiprocessing import ProcessPool as Pool
elif six.PY3:
from multiprocessing import Pool
pool = Pool(self.num_proc)
hashes = pool.map(self.gen_hash, target_files)
for filename, hash_value in zip(target_files, hashes):
self.hash_dict[filename] = hash_value
spinner.stop()
except KeyboardInterrupt:
pool.terminate()
pool.join()
spinner.stop()
sys.exit(1)
return True
else:
return False
def test_boardd_loopback():
# wait for boardd to init
spinner = Spinner()
time.sleep(2)
with Timeout(60, "boardd didn't start"):
sm = messaging.SubMaster(['pandaState'])
while sm.rcv_frame['pandaState'] < 1:
sm.update(1000)
# boardd blocks on CarVin and CarParams
cp = car.CarParams.new_message()
cp.safetyModel = car.CarParams.SafetyModel.allOutput
Params().put("CarVin", b"0" * 17)
Params().put_bool("ControlsReady", True)
Params().put("CarParams", cp.to_bytes())
sendcan = messaging.pub_sock('sendcan')
can = messaging.sub_sock('can', conflate=False, timeout=100)
time.sleep(1)
n = 1000
for i in range(n):
spinner.update(f"boardd loopback {i}/{n}")
sent_msgs = defaultdict(set)
for _ in range(random.randrange(10)):
to_send = []
for __ in range(random.randrange(100)):
bus = random.randrange(3)
addr = random.randrange(1, 1 << 29)
dat = bytes([
random.getrandbits(8)
for _ in range(random.randrange(1, 9))
])
sent_msgs[bus].add((addr, dat))
to_send.append(make_can_msg(addr, dat, bus))
sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan'))
max_recv = 10
while max_recv > 0 and any(len(sent_msgs[bus]) for bus in range(3)):
recvd = messaging.drain_sock(can, wait_for_one=True)
for msg in recvd:
for m in msg.can:
if m.src >= 128:
k = (m.address, m.dat)
assert k in sent_msgs[m.src - 128]
sent_msgs[m.src - 128].discard(k)
max_recv -= 1
# if a set isn't empty, messages got dropped
for bus in range(3):
assert not len(
sent_msgs[bus]
), f"loop {i}: bus {bus} missing {len(sent_msgs[bus])} messages"
spinner.close()
def model_replay(lr_list, frs):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_YUV_BACK, 40, False,
*(tici_f_frame_size if TICI else eon_f_frame_size))
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_YUV_FRONT, 40, False,
*(tici_d_frame_size if TICI else eon_d_frame_size))
vipc_server.start_listener()
pm = messaging.PubMaster([
'roadCameraState', 'driverCameraState', 'liveCalibration',
'lateralPlan'
])
sm = messaging.SubMaster(['modelV2', 'driverState'])
try:
managed_processes['modeld'].start()
managed_processes['dmonitoringmodeld'].start()
time.sleep(5)
sm.update(1000)
last_desire = None
log_msgs = []
frame_idxs = dict.fromkeys(['roadCameraState', 'driverCameraState'], 0)
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), replace_calib(msg, None))
for msg in tqdm(lr_list):
if msg.which() == "liveCalibration":
last_calib = list(msg.liveCalibration.rpyCalib)
pm.send(msg.which(), replace_calib(msg, last_calib))
elif msg.which() == "lateralPlan":
last_desire = msg.lateralPlan.desire
elif msg.which() in ["roadCameraState", "driverCameraState"]:
ret = process_frame(msg, pm, sm, log_msgs, vipc_server,
spinner, frs, frame_idxs, last_desire)
if ret is None:
break
except KeyboardInterrupt:
pass
finally:
spinner.close()
managed_processes['modeld'].stop()
managed_processes['dmonitoringmodeld'].stop()
return log_msgs
def load_hash_dict(self, load_path, use_cache, target_dir):
if load_path and Path(load_path).exists() and use_cache:
logger.debug("Load hash cache: {}".format(load_path))
spinner = Spinner(prefix="Loading hash cache...")
spinner.start()
self.hash_dict = joblib.load(load_path)
spinner.stop()
is_update = self.update_hash_dict()
return not is_update
else:
self.hash_dict = {}
self.update_hash_dict()
return False
def camera_replay(lr, fr):
spinner = Spinner()
pm = messaging.PubMaster(['frame', 'liveCalibration'])
sm = messaging.SubMaster(['model'])
# TODO: add dmonitoringmodeld
print("preparing procs")
manager.prepare_managed_process("camerad")
manager.prepare_managed_process("modeld")
try:
print("starting procs")
manager.start_managed_process("camerad")
manager.start_managed_process("modeld")
time.sleep(5)
print("procs started")
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), msg.as_builder())
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibrationd":
pm.send(msg.which(), msg.as_builder())
elif msg.which() == "frame":
f = msg.as_builder()
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1]
f.frame.image = img.flatten().tobytes()
frame_idx += 1
pm.send(msg.which(), f)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['model']))
spinner.update("modeld replay %d/%d" %
(frame_idx, fr.frame_count))
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
print("replay done")
spinner.close()
manager.kill_managed_process('modeld')
time.sleep(2)
manager.kill_managed_process('camerad')
return log_msgs
def make_hash_list(self):
if self.num_proc is None:
self.num_proc = cpu_count() - 1
try:
spinner = Spinner(
prefix="Calculating image hashes (hash-bits={} num-proc={})..."
.format(self.hash_bits, self.num_proc))
spinner.start()
if six.PY2:
from pathos.multiprocessing import ProcessPool as Pool
elif six.PY3:
from multiprocessing import Pool
pool = Pool(self.num_proc)
self.cache = pool.map(self.gen_hash, self.image_filenames)
spinner.stop()
except KeyboardInterrupt:
pool.terminate()
pool.join()
spinner.stop()
sys.exit(1)
def update_apks(show_spinner=False):
# install apks
installed = get_installed_apks()
install_apks = glob.glob(
os.path.join(BASEDIR, "selfdrive/assets/addon/apk/*.apk"))
if show_spinner:
spinner = Spinner()
show_spinner = False
for apk in install_apks:
app = os.path.basename(apk)[:-4]
if app not in installed:
installed[app] = None
cloudlog.info("installed apks %s" % (str(installed), ))
for app in installed.keys():
apk_path = os.path.join(BASEDIR,
"selfdrive/assets/addon/apk/" + app + ".apk")
if not os.path.exists(apk_path):
continue
h1 = hashlib.sha1(open(apk_path, 'rb').read()).hexdigest()
h2 = None
if installed[app] is not None:
h2 = hashlib.sha1(open(installed[app], 'rb').read()).hexdigest()
cloudlog.info("comparing version of %s %s vs %s" % (app, h1, h2))
if h2 is None or h1 != h2:
show_spinner = True
spinner.update("installing %s" % app)
cloudlog.info("installing %s" % app)
success = install_apk(apk_path)
if not success:
cloudlog.info("needing to uninstall %s" % app)
system("pm uninstall %s" % app)
success = install_apk(apk_path)
assert success
if show_spinner:
spinner.close()
def load(self, load_path, use_cache, target_dir):
if load_path and Path(load_path).exists() and use_cache:
cache_mtime = self.check_mtime(load_path)
target_mtime = self.check_latest_dir_mtime(target_dir)
if cache_mtime > target_mtime:
logger.debug("Load hash cache: {}".format(load_path))
spinner = Spinner(prefix="Loading hash cache...")
spinner.start()
self.cache = joblib.load(load_path)
spinner.stop()
return True
else:
self.cache = []
self.make_hash_list()
return False
else:
self.cache = []
self.make_hash_list()
return False
exit_status = os.wait()[1] >> 8
os._exit(exit_status)
if __name__ == "__main__":
unblock_stdout()
from common.spinner import Spinner
from common.text_window import TextWindow
import importlib
import traceback
from multiprocessing import Process
# Run scons
spinner = Spinner(noop=(__name__ != "__main__" or not ANDROID))
spinner.update("0")
if not prebuilt:
for retry in [True, False]:
# run scons
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
env['SCONS_CACHE'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
scons = subprocess.Popen(["scons", j_flag],
cwd=BASEDIR,
env=env,
stderr=subprocess.PIPE)
try:
manager_thread(spinner)
except Exception:
traceback.print_exc()
crash.capture_exception()
finally:
manager_cleanup()
if Params().get("DoUninstall", encoding='utf8') == "1":
cloudlog.warning("uninstalling")
HARDWARE.uninstall()
if __name__ == "__main__":
unblock_stdout()
spinner = Spinner()
spinner.update_progress(MAX_BUILD_PROGRESS, 100)
try:
main(spinner)
except Exception:
add_logentries_handler(cloudlog)
cloudlog.exception("Manager failed to start")
# Show last 3 lines of traceback
error = traceback.format_exc(-3)
error = "Manager failed to start\n\n" + error
spinner.close()
with TextWindow(error) as t:
t.wait_for_exit()
def build(spinner: Spinner, dirty: bool = False) -> None:
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else f"-j{nproc - 1}"
for retry in [False]:
scons: subprocess.Popen = subprocess.Popen(
["scons", j_flag, "--cache-populate"],
cwd=BASEDIR,
env=env,
stderr=subprocess.PIPE)
assert scons.stderr is not None
compile_output = []
# Read progress from stderr and update spinner
while scons.poll() is None:
try:
line = scons.stderr.readline()
if line is None:
continue
line = line.rstrip()
prefix = b'progress: '
if line.startswith(prefix):
i = int(line[len(prefix):])
spinner.update_progress(
MAX_BUILD_PROGRESS * min(1., i / TOTAL_SCONS_NODES),
100.)
elif len(line):
compile_output.append(line)
print(line.decode('utf8', 'replace'))
except Exception:
pass
if scons.returncode != 0:
# Read remaining output
r = scons.stderr.read().split(b'\n')
compile_output += r
if retry and (not dirty):
if not os.getenv("CI"):
print("scons build failed, cleaning in")
for i in range(3, -1, -1):
print("....%d" % i)
time.sleep(1)
subprocess.check_call(["scons", "-c"],
cwd=BASEDIR,
env=env)
else:
print("scons build failed after retry")
sys.exit(1)
else:
# Build failed log errors
errors = [
line.decode('utf8', 'replace') for line in compile_output
if any(err in line for err in
[b'error: ', b'not found, needed by target'])
]
error_s = "\n".join(errors)
add_file_handler(cloudlog)
cloudlog.error("scons build failed\n" + error_s)
# Show TextWindow
spinner.close()
if not os.getenv("CI"):
error_s = "\n \n".join("\n".join(textwrap.wrap(e, 65))
for e in errors)
with TextWindow("openpilot failed to build\n \n" +
error_s) as t:
t.wait_for_exit()
exit(1)
else:
break
# enforce max cache size
cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
cache_files.sort(key=lambda f: f.stat().st_mtime)
cache_size = sum(f.stat().st_size for f in cache_files)
for f in cache_files:
if cache_size < MAX_CACHE_SIZE:
break
cache_size -= f.stat().st_size
f.unlink()
exit_status = os.wait()[1] >> 8
os._exit(exit_status)
if __name__ == "__main__":
unblock_stdout()
from common.spinner import Spinner
else:
from common.spinner import FakeSpinner as Spinner
import importlib
import traceback
from multiprocessing import Process
# Run scons
spinner = Spinner()
spinner.update("0", 'starting openpilot')
if not prebuilt:
for retry in [True, False]:
# run scons
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
env['SCONS_CACHE'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
scons_finished_progress = 70.0
# Read progress from stderr and update spinner
def setUpClass(cls):
os.environ['STARTED'] = '1'
os.environ['BOARDD_LOOPBACK'] = '1'
cls.spinner = Spinner()
def flash_panda(panda_serial: str) -> Panda:
panda = Panda(panda_serial)
fw_signature = get_expected_signature(panda)
panda_version = "bootstub" if panda.bootstub else panda.get_version()
panda_signature = b"" if panda.bootstub else panda.get_signature()
cloudlog.warning(
f"Panda {panda_serial} connected, version: {panda_version}, signature {panda_signature.hex()[:16]}, expected {fw_signature.hex()[:16]}"
)
if panda.bootstub or panda_signature != fw_signature:
cloudlog.info("Panda firmware out of date, update required")
panda.flash()
cloudlog.info("Done flashing")
if panda.bootstub:
spinner = Spinner()
spinner.update("Restoring panda")
panda.recover()
spinner.close()
if panda.bootstub:
spinner = Spinner()
spinner.update("Restoring panda")
try:
if panda.get_mcu_type() == MCU_TYPE_H7:
subprocess.run(
"cd /data/openpilot/panda/board; ./recover_h7.sh",
capture_output=True,
shell=True)
else:
subprocess.run("cd /data/openpilot/panda/board; ./recover.sh",
capture_output=True,
shell=True)
panda.reset()
panda.reconnect()
finally:
spinner.close()
if panda.bootstub:
cloudlog.info("Panda still not booting, exiting")
raise AssertionError
panda_signature = panda.get_signature()
if panda_signature != fw_signature:
cloudlog.info("Version mismatch after flashing, exiting")
raise AssertionError
return panda
exit_status = os.wait()[1] >> 8
os._exit(exit_status)
if __name__ == "__main__":
unblock_stdout()
from common.spinner import Spinner
from common.text_window import TextWindow
if not (os.system("python3 -m pip list | grep 'scipy' ") == 0):
os.system(
"cd /data/openpilot/installer/scipy_installer/ && ./scipy_installer")
# Run scons
spinner = Spinner(noop=(__name__ != "__main__"))
spinner.update("0")
def build():
for retry in [True, False]:
# run scons
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
env['SCONS_CACHE'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else f"-j{nproc - 1}"
scons = subprocess.Popen(["scons", j_flag],
cwd=BASEDIR,
env=env,
def register(show_spinner=False) -> str:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0
assert os.system(
"openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub")
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
serial = HARDWARE.get_serial()
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
import requests, json
backoff = 0
while True:
try:
hostURL = "http://47.117.4.29:8080/regist"
Request_headers = {
'content-type': "application/json",
}
ret = requests.post(hostURL, headers=Request_headers, data=json.dumps(HardwareSerial))
dongle_id = json.loads(ret.text)
if dongle_id:
break
else:
if dongle_id == None:
raise Exception("请联系马威！")
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if show_spinner:
spinner.close()
if True:
params.put("DongleId", dongle_id)
return dongle_id
def model_replay(lr, frs):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_ROAD, 40, False,
*(tici_f_frame_size if TICI else eon_f_frame_size))
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_DRIVER, 40, False,
*(tici_d_frame_size if TICI else eon_d_frame_size))
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 40,
False, *(tici_f_frame_size))
vipc_server.start_listener()
sm = messaging.SubMaster(['modelV2', 'driverState'])
pm = messaging.PubMaster([
'roadCameraState', 'wideRoadCameraState', 'driverCameraState',
'liveCalibration', 'lateralPlan'
])
try:
managed_processes['modeld'].start()
managed_processes['dmonitoringmodeld'].start()
time.sleep(5)
sm.update(1000)
log_msgs = []
last_desire = None
recv_cnt = defaultdict(int)
frame_idxs = defaultdict(int)
# init modeld with valid calibration
cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"]
for _ in range(5):
pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder())
time.sleep(0.1)
msgs = defaultdict(list)
for msg in lr:
msgs[msg.which()].append(msg)
for cam_msgs in zip_longest(msgs['roadCameraState'],
msgs['wideRoadCameraState'],
msgs['driverCameraState']):
# need a pair of road/wide msgs
if TICI and None in (cam_msgs[0], cam_msgs[1]):
break
for msg in cam_msgs:
if msg is None:
continue
if SEND_EXTRA_INPUTS:
if msg.which() == "liveCalibration":
last_calib = list(msg.liveCalibration.rpyCalib)
pm.send(msg.which(), replace_calib(msg, last_calib))
elif msg.which() == "lateralPlan":
last_desire = msg.lateralPlan.desire
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = last_desire
pm.send('lateralPlan', dat)
if msg.which() in VIPC_STREAM:
msg = msg.as_builder()
camera_state = getattr(msg, msg.which())
img = frs[msg.which()].get(frame_idxs[msg.which()],
pix_fmt="nv12")[0]
frame_idxs[msg.which()] += 1
# send camera state and frame
camera_state.frameId = frame_idxs[msg.which()]
pm.send(msg.which(), msg)
vipc_server.send(VIPC_STREAM[msg.which()],
img.flatten().tobytes(),
camera_state.frameId,
camera_state.timestampSof,
camera_state.timestampEof)
recv = None
if msg.which() in ('roadCameraState',
'wideRoadCameraState'):
if not TICI or min(frame_idxs['roadCameraState'],
frame_idxs['wideRoadCameraState']
) > recv_cnt['modelV2']:
recv = "modelV2"
elif msg.which() == 'driverCameraState':
recv = "driverState"
# wait for a response
with Timeout(15, f"timed out waiting for {recv}"):
if recv:
recv_cnt[recv] += 1
log_msgs.append(messaging.recv_one(sm.sock[recv]))
spinner.update(
"replaying models: road %d/%d, driver %d/%d" %
(frame_idxs['roadCameraState'],
frs['roadCameraState'].frame_count,
frame_idxs['driverCameraState'],
frs['driverCameraState'].frame_count))
if any(frame_idxs[c] >= frs[c].frame_count
for c in frame_idxs.keys()):
break
finally:
spinner.close()
managed_processes['modeld'].stop()
managed_processes['dmonitoringmodeld'].stop()
return log_msgs
def register(show_spinner=False) -> Optional[str]:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id: Optional[str] = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
pubkey = Path(PERSIST + "/comma/id_rsa.pub")
if not pubkey.is_file():
dongle_id = UNREGISTERED_DONGLE_ID
cloudlog.warning(f"missing public key: {pubkey}")
elif needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST +
"/comma/id_rsa.pub") as f1, open(PERSIST +
"/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
serial = HARDWARE.get_serial()
start_time = time.monotonic()
imei1: Optional[str] = None
imei2: Optional[str] = None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
backoff = 0
start_time = time.monotonic()
while True:
try:
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
if resp.status_code in (402, 403):
cloudlog.info(
f"Unable to register device, got {resp.status_code}")
dongle_id = UNREGISTERED_DONGLE_ID
else:
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
break
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
if show_spinner:
spinner.close()
if dongle_id:
params.put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware",
(dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
return dongle_id
def main():
# the flippening!
os.system('LD_LIBRARY_PATH="" content insert --uri content://settings/system --bind name:s:user_rotation --bind value:i:1')
# disable bluetooth
os.system('service call bluetooth_manager 8')
if os.getenv("NOLOG") is not None:
del managed_processes['loggerd']
del managed_processes['tombstoned']
if os.getenv("NOUPLOAD") is not None:
del managed_processes['uploader']
if os.getenv("NOVISION") is not None:
del managed_processes['visiond']
if os.getenv("LEAN") is not None:
del managed_processes['uploader']
del managed_processes['loggerd']
del managed_processes['logmessaged']
del managed_processes['logcatd']
del managed_processes['tombstoned']
del managed_processes['proclogd']
if os.getenv("NOCONTROL") is not None:
del managed_processes['controlsd']
del managed_processes['plannerd']
del managed_processes['radard']
# support additional internal only extensions
try:
import selfdrive.manager_extensions
selfdrive.manager_extensions.register(register_managed_process) # pylint: disable=no-member
except ImportError:
pass
params = Params()
params.manager_start()
# set unset params
if params.get("CompletedTrainingVersion") is None:
params.put("CompletedTrainingVersion", "0")
if params.get("IsMetric") is None:
params.put("IsMetric", "0")
if params.get("RecordFront") is None:
params.put("RecordFront", "0")
if params.get("HasAcceptedTerms") is None:
params.put("HasAcceptedTerms", "0")
if params.get("HasCompletedSetup") is None:
params.put("HasCompletedSetup", "0")
if params.get("IsUploadRawEnabled") is None:
params.put("IsUploadRawEnabled", "1")
if params.get("IsUploadVideoOverCellularEnabled") is None:
params.put("IsUploadVideoOverCellularEnabled", "1")
if params.get("IsGeofenceEnabled") is None:
params.put("IsGeofenceEnabled", "-1")
if params.get("SpeedLimitOffset") is None:
params.put("SpeedLimitOffset", "0")
if params.get("LongitudinalControl") is None:
params.put("LongitudinalControl", "0")
if params.get("LimitSetSpeed") is None:
params.put("LimitSetSpeed", "0")
if params.get("LimitSetSpeedNeural") is None:
params.put("LimitSetSpeedNeural", "0")
if params.get("LastUpdateTime") is None:
t = datetime.datetime.now().isoformat()
params.put("LastUpdateTime", t.encode('utf8'))
if params.get("OpenpilotEnabledToggle") is None:
params.put("OpenpilotEnabledToggle", "1")
# is this chffrplus?
if os.getenv("PASSIVE") is not None:
params.put("Passive", str(int(os.getenv("PASSIVE"))))
if params.get("Passive") is None:
raise Exception("Passive must be set to continue")
with Spinner() as spinner:
spinner.update("0") # Show progress bar
manager_update()
manager_init()
manager_prepare(spinner)
if os.getenv("PREPAREONLY") is not None:
return
# SystemExit on sigterm
signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
try:
manager_thread()
except Exception:
traceback.print_exc()
crash.capture_exception()
print ("EXIT ON EXCEPTION")
finally:
cleanup_all_processes(None, None)
if params.get("DoUninstall", encoding='utf8') == "1":
uninstall()
if __name__ == "__main__":
unblock_stdout()
if __name__ == "__main__" and ANDROID:
from common.spinner import Spinner
from common.text_window import TextWindow
else:
from common.spinner import FakeSpinner as Spinner
from common.text_window import FakeTextWindow as TextWindow
import importlib
import traceback
from multiprocessing import Process
# Run scons
spinner = Spinner()
spinner.update("0")
if not prebuilt:
for retry in [True, False]:
# run scons
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
env['SCONS_CACHE'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else "-j%d" % (nproc - 1)
scons = subprocess.Popen(["scons", j_flag],
cwd=BASEDIR,
env=env,
stderr=subprocess.PIPE)
try:
manager_thread()
except Exception:
traceback.print_exc()
crash.capture_exception()
finally:
manager_cleanup()
if Params().get("DoUninstall", encoding='utf8') == "1":
cloudlog.warning("uninstalling")
HARDWARE.uninstall()
if __name__ == "__main__":
unblock_stdout()
spinner = Spinner()
try:
main(spinner)
except Exception:
add_logentries_handler(cloudlog)
cloudlog.exception("Manager failed to start")
# Show last 3 lines of traceback
error = traceback.format_exc(-3)
error = "Manager failed to start\n\n" + error
spinner.close()
with TextWindow(error) as t:
t.wait_for_exit()
raise
def model_replay(lr, frs):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 40, False, *(tici_f_frame_size if TICI else eon_f_frame_size))
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_DRIVER, 40, False, *(tici_d_frame_size if TICI else eon_d_frame_size))
vipc_server.start_listener()
sm = messaging.SubMaster(['modelV2', 'driverState'])
pm = messaging.PubMaster(['roadCameraState', 'driverCameraState', 'liveCalibration', 'lateralPlan'])
try:
managed_processes['modeld'].start()
managed_processes['dmonitoringmodeld'].start()
time.sleep(2)
sm.update(1000)
log_msgs = []
last_desire = None
frame_idxs = defaultdict(lambda: 0)
# init modeld with valid calibration
cal_msgs = [msg for msg in lr if msg.which() == "liveCalibration"]
for _ in range(5):
pm.send(cal_msgs[0].which(), cal_msgs[0].as_builder())
time.sleep(0.1)
for msg in tqdm(lr):
if SEND_EXTRA_INPUTS:
if msg.which() == "liveCalibration":
last_calib = list(msg.liveCalibration.rpyCalib)
pm.send(msg.which(), replace_calib(msg, last_calib))
elif msg.which() == "lateralPlan":
last_desire = msg.lateralPlan.desire
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = last_desire
pm.send('lateralPlan', dat)
if msg.which() in ["roadCameraState", "driverCameraState"]:
camera_state = getattr(msg, msg.which())
stream = VisionStreamType.VISION_STREAM_ROAD if msg.which() == "roadCameraState" else VisionStreamType.VISION_STREAM_DRIVER
img = frs[msg.which()].get(frame_idxs[msg.which()], pix_fmt="yuv420p")[0]
# send camera state and frame
pm.send(msg.which(), msg.as_builder())
vipc_server.send(stream, img.flatten().tobytes(), camera_state.frameId,
camera_state.timestampSof, camera_state.timestampEof)
# wait for a response
with Timeout(seconds=15):
packet_from_camera = {"roadCameraState": "modelV2", "driverCameraState": "driverState"}
log_msgs.append(messaging.recv_one(sm.sock[packet_from_camera[msg.which()]]))
frame_idxs[msg.which()] += 1
if frame_idxs[msg.which()] >= frs[msg.which()].frame_count:
break
spinner.update("replaying models: road %d/%d, driver %d/%d" % (frame_idxs['roadCameraState'],
frs['roadCameraState'].frame_count, frame_idxs['driverCameraState'], frs['driverCameraState'].frame_count))
finally:
spinner.close()
managed_processes['modeld'].stop()
managed_processes['dmonitoringmodeld'].stop()
return log_msgs
def register(show_spinner=False):
params = Params()
params.put("Version", version)
params.put("TermsVersion", terms_version)
params.put("TrainingVersion", training_version)
params.put("GitCommit", get_git_commit(default=""))
params.put("GitBranch", get_git_branch(default=""))
params.put("GitRemote", get_git_remote(default=""))
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
needs_registration = (None in [IMEI, HardwareSerial])
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST +
"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in " + PERSIST +
"/comma/id_rsa.tmp -pubout -out " + PERSIST +
"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub",
PERSIST + "/comma/id_rsa.pub")
# make key readable by app users (ai.comma.plus.offroad)
os.chmod(PERSIST + '/comma/', 0o755)
os.chmod(PERSIST + '/comma/id_rsa', 0o744)
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = needs_registration or dongle_id is None
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
private_key = open(PERSIST + "/comma/id_rsa").read()
public_key = open(PERSIST + "/comma/id_rsa.pub").read()
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
# Block until we get the imei
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
serial = HARDWARE.get_serial()
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
while True:
try:
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
params.put("DongleId", dongle_id)
break
except Exception:
cloudlog.exception("failed to authenticate")
time.sleep(1)
if show_spinner:
spinner.close()
return dongle_id
if downloader.sha256.hexdigest().lower(
) != partition['hash'].lower():
raise Exception("Uncompressed hash mismatch")
if out.tell() != partition['size']:
raise Exception("Uncompressed size mismatch")
# Write hash after successfull flash
os.sync()
out.write(partition['hash_raw'].lower().encode())
cloudlog.info(f"AGNOS ready on slot {target_slot}")
if __name__ == "__main__":
import logging
import time
import sys
if len(sys.argv) != 2:
print("Usage: ./agnos.py <manifest.json>")
exit(1)
spinner = Spinner()
spinner.update("Updating AGNOS")
time.sleep(5)
logging.basicConfig(level=logging.INFO)
flash_agnos_update(sys.argv[1], logging, spinner)
def register(show_spinner=False) -> str:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST +
"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in " + PERSIST +
"/comma/id_rsa.tmp -pubout -out " + PERSIST +
"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub",
PERSIST + "/comma/id_rsa.pub")
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST +
"/comma/id_rsa.pub") as f1, open(PERSIST +
"/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
serial = HARDWARE.get_serial()
start_time = time.monotonic()
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
backoff = 0
start_time = time.monotonic()
while True:
try:
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
if resp.status_code in (402, 403):
cloudlog.info(
f"Unable to register device, got {resp.status_code}")
dongle_id = UNREGISTERED_DONGLE_ID
else:
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
break
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
if show_spinner:
spinner.close()
if dongle_id:
params.put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware",
dongle_id == UNREGISTERED_DONGLE_ID)
return dongle_id
sys.stdout.write(dat.decode('utf8'))
except (OSError, IOError, UnicodeDecodeError):
pass
# os.wait() returns a tuple with the pid and a 16 bit value
# whose low byte is the signal number and whose high byte is the exit satus
exit_status = os.wait()[1] >> 8
os._exit(exit_status)
if __name__ == "__main__":
unblock_stdout()
# Run scons
spinner = Spinner()
spinner.update("0")
if __name__ != "__main__":
spinner.close()
def build():
for retry in [True, False]:
# run scons
env = os.environ.copy()
env['SCONS_PROGRESS'] = "1"
env['SCONS_CACHE'] = "1"
nproc = os.cpu_count()
j_flag = "" if nproc is None else f"-j{nproc - 1}"
scons = subprocess.Popen(["scons", j_flag], cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
# install pip from git
package = 'git+https://github.com/move-fast/opspline.git@master'
pip = subprocess.Popen([sys.executable, "-m", "pip", "install", "-v", package], stdout=subprocess.PIPE)
# Read progress from pip and update spinner
steps = 0
while True:
output = pip.stdout.readline()
if pip.poll() is not None:
break
if output:
steps += 1
spinner.update_progress(MAX_BUILD_PROGRESS * min(1., steps / TOTAL_PIP_STEPS), 100.)
print(output.decode('utf8', 'replace'))
if TICI:
shutil.rmtree(TMP_DIR)
os.unsetenv('TMPDIR')
# remove numpy installed to PYEXTRA_DIR since numpy is already present in the AGNOS image
if OPSPLINE_SPEC is None:
for directory in glob(f'{PYEXTRA_DIR}/numpy*'):
shutil.rmtree(directory)
shutil.rmtree(f'{PYEXTRA_DIR}/bin')
if __name__ == "__main__" and (OPSPLINE_SPEC is None or OVERPY_SPEC is None):
spinner = Spinner()
spinner.update_progress(0, 100)
install_dep(spinner)
def model_replay(lr, fr, desire=None, calib=None):
spinner = Spinner()
spinner.update("starting model replay")
vipc_server = None
pm = messaging.PubMaster(
['roadCameraState', 'liveCalibration', 'lateralPlan'])
sm = messaging.SubMaster(['modelV2'])
# TODO: add dmonitoringmodeld
try:
managed_processes['modeld'].start()
time.sleep(5)
sm.update(1000)
desires_by_index = {
v: k
for k, v in log.LateralPlan.Desire.schema.enumerants.items()
}
cal = [msg for msg in lr if msg.which() == "liveCalibration"]
for msg in cal[:5]:
pm.send(msg.which(), replace_calib(msg, calib))
log_msgs = []
frame_idx = 0
for msg in tqdm(lr):
if msg.which() == "liveCalibration":
pm.send(msg.which(), replace_calib(msg, calib))
elif msg.which() == "roadCameraState":
if desire is not None:
for i in desire[frame_idx].nonzero()[0]:
dat = messaging.new_message('lateralPlan')
dat.lateralPlan.desire = desires_by_index[i]
pm.send('lateralPlan', dat)
f = msg.as_builder()
pm.send(msg.which(), f)
img = fr.get(frame_idx, pix_fmt="yuv420p")[0]
if vipc_server is None:
w, h = {
int(3 * w * h / 2): (w, h)
for (w, h) in [tici_f_frame_size, eon_f_frame_size]
}[len(img)]
vipc_server = VisionIpcServer("camerad")
vipc_server.create_buffers(
VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w,
h)
vipc_server.start_listener()
time.sleep(1) # wait for modeld to connect
vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK,
img.flatten().tobytes(),
f.roadCameraState.frameId,
f.roadCameraState.timestampSof,
f.roadCameraState.timestampEof)
with Timeout(seconds=15):
log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
spinner.update("modeld replay %d/%d" %
(frame_idx, fr.frame_count))
frame_idx += 1
if frame_idx >= fr.frame_count:
break
except KeyboardInterrupt:
pass
finally:
spinner.close()
managed_processes['modeld'].stop()
return log_msgs
