def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None):
if ignore_fields is None:
ignore_fields = []
if ignore_msgs is None:
ignore_msgs = []
cmp_log_path = cmp_log_fn if os.path.exists(cmp_log_fn) else BASE_URL + os.path.basename(cmp_log_fn)
cmp_log_msgs = list(LogReader(cmp_log_path))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
# TODO: update routes so enable check can run
# failed enable check: honda bosch, hyundai, chrysler, and subaru
if cfg.proc_name == "controlsd" and FULL_TEST and False:
for msg in log_msgs:
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception("Route never enabled: %s" % segment)
try:
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs, cfg.tolerance)
except Exception as e:
return str(e)
def test_process(cfg,
lr,
ref_log_path,
new_log_path,
ignore_fields=None,
ignore_msgs=None):
if ignore_fields is None:
ignore_fields = []
if ignore_msgs is None:
ignore_msgs = []
ref_log_msgs = list(LogReader(ref_log_path))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
if cfg.proc_name == "controlsd":
if not check_enabled(log_msgs):
return f"Route did not enable at all or for long enough: {new_log_path}", log_msgs
try:
return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore,
ignore_msgs, cfg.tolerance,
cfg.field_tolerances), log_msgs
except Exception as e:
return str(e), log_msgs
def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None):
if ignore_fields is None:
ignore_fields = []
if ignore_msgs is None:
ignore_msgs = []
cmp_log_path = cmp_log_fn if os.path.exists(
cmp_log_fn) else BASE_URL + os.path.basename(cmp_log_fn)
cmp_log_msgs = list(LogReader(cmp_log_path))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
if cfg.proc_name == "controlsd":
for msg in log_msgs:
if msg.which() == "carParams":
if msg.carParams.notCar:
break
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception(f"Route never enabled: {segment}")
try:
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields + cfg.ignore,
ignore_msgs, cfg.tolerance)
except Exception as e:
return str(e)
def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]):
if not os.path.isfile(cmp_log_fn):
req = requests.get(BASE_URL + os.path.basename(cmp_log_fn))
assert req.status_code == 200, ("Failed to download %s" % cmp_log_fn)
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(cmp_log_fn))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
# TODO: update routes so enable check can run
# failed enable check: honda bosch, hyundai, chrysler, and subaru
if cfg.proc_name == "controlsd" and FULL_TEST and False:
for msg in log_msgs:
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception("Route never enabled: %s" % segment)
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields + cfg.ignore,
ignore_msgs)
def test_process(cfg, lr, ref_log_path, ignore_fields=None, ignore_msgs=None):
if ignore_fields is None:
ignore_fields = []
if ignore_msgs is None:
ignore_msgs = []
ref_log_msgs = list(LogReader(ref_log_path))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
if cfg.proc_name == "controlsd":
if not check_enabled(log_msgs):
raise Exception(f"Route never enabled: {ref_log_path}")
try:
return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs
except Exception as e:
return str(e), log_msgs
def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]):
if not os.path.isfile(cmp_log_fn):
assert False, ("Failed to open %s" % cmp_log_fn)
else:
print("Opening file [%s]" % cmp_log_fn)
cmp_log_msgs = list(LogReader(cmp_log_fn))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
# TODO: update routes so enable check can run
# failed enable check: honda bosch, hyundai, chrysler, and subaru
if cfg.proc_name == "controlsd" and FULL_TEST and False:
for msg in log_msgs:
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception("Route never enabled: %s" % segment)
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
def test_process(cfg, lr, ref_log_fn, ignore_fields=None, ignore_msgs=None):
if ignore_fields is None:
ignore_fields = []
if ignore_msgs is None:
ignore_msgs = []
ref_log_path = ref_log_fn if os.path.exists(
ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn)
ref_log_msgs = list(LogReader(ref_log_path))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
if cfg.proc_name == "controlsd":
if not check_enabled(log_msgs):
segment = ref_log_fn.split("/")[-1].split("_")[0]
raise Exception(f"Route never enabled: {segment}")
try:
return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore,
ignore_msgs, cfg.tolerance), log_msgs
except Exception as e:
return str(e), log_msgs
log_msgs = camera_replay(list(lr), fr)
failed = False
if not update:
ref_commit = open(ref_commit_fn).read().strip()
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
cmp_log = LogReader(BASE_URL + log_fn)
ignore = [
'logMonoTime', 'valid', 'modelV2.frameDropPerc',
'modelV2.modelExecutionTime'
]
results: Any = {TEST_ROUTE: {}}
results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log,
log_msgs,
ignore_fields=ignore)
diff1, diff2, failed = format_diff(results, ref_commit)
print(diff1)
with open("model_diff.txt", "w") as f:
f.write(diff2)
if update or failed:
from selfdrive.test.openpilotci import upload_file
print("Uploading new refs")
new_commit = get_git_commit()
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", new_commit)
save_log(log_fn, log_msgs)
if not update:
with open(ref_commit_fn) as f:
ref_commit = f.read().strip()
log_fn = get_log_fn(ref_commit, TEST_ROUTE, tici=TICI)
try:
cmp_log = LogReader(BASE_URL + log_fn)
ignore = [
'logMonoTime', 'modelV2.frameDropPerc',
'modelV2.modelExecutionTime', 'driverState.modelExecutionTime',
'driverState.dspExecutionTime'
]
tolerance = None if not PC else 1e-3
results: Any = {TEST_ROUTE: {}}
results[TEST_ROUTE]["models"] = compare_logs(cmp_log,
log_msgs,
tolerance=tolerance,
ignore_fields=ignore)
diff1, diff2, failed = format_diff(results, ref_commit)
print(diff2)
print('-------------\n' * 5)
print(diff1)
with open("model_diff.txt", "w") as f:
f.write(diff2)
except Exception as e:
print(str(e))
failed = True
# upload new refs
if update or failed:
from selfdrive.test.openpilotci import upload_file
url = "https://commadataci.blob.core.windows.net/openpilotci/"
req = requests.get(url + os.path.basename(log_fn))
if req.status_code != 200:
results[segment][
cfg.proc_name] = "failed to download comparison log"
continue
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(log_fn))
diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore)
results[segment][cfg.proc_name] = diff
os.remove(rlog_fn)
failed = False
with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
f.write("***** tested against commit %s *****\n" % ref_commit)
for segment, result in list(results.items()):
f.write("***** differences for segment %s *****\n" % segment)
print("***** results for segment %s *****" % segment)
for proc, diff in list(result.items()):
f.write("*** process: %s ***\n" % proc)
print("\t%s" % proc)
update = "--update" in sys.argv
lr = LogReader(get_url(TEST_ROUTE, 0))
fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera"))
log_msgs = camera_replay(list(lr), fr)
failed = False
if not update:
ref_commit = open("model_replay_ref_commit").read().strip()
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
cmp_log = LogReader(BASE_URL + log_fn)
results: Any = {TEST_ROUTE: {}}
results[TEST_ROUTE]["modeld"] = compare_logs(
cmp_log,
log_msgs,
ignore_fields=['logMonoTime', 'valid', 'model.frameDropPerc'])
diff1, diff2, failed = format_diff(results, ref_commit)
print(diff1)
with open("model_diff.txt", "w") as f:
f.write(diff2)
if update or failed:
from selfdrive.test.openpilotci import upload_file
print("Uploading new refs")
new_commit = get_git_commit()
log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", new_commit)
save_log(log_fn, log_msgs)