def map_column(self, path, dtype, shape):
npy_path = os.path.join(self._path, path)
mkdirs_exists_ok(os.path.dirname(npy_path))
return np.lib.format.open_memmap(npy_path,
mode='w+',
dtype=dtype,
shape=shape)
def setup_output():
output_dir = os.path.join(os.getcwd(), 'out/longitudinal')
if not os.path.exists(os.path.join(output_dir, "index.html")):
# write test output header
css_style = """
.maneuver_title {
font-size: 24px;
text-align: center;
}
.maneuver_graph {
width: 100%;
}
"""
view_html = "<html><head><style>%s</style></head><body><table>" % (
css_style, )
for i, man in enumerate(maneuvers):
view_html += "<tr><td class='maneuver_title' colspan=5><div>%s</div></td></tr><tr>" % (
man.title, )
for c in [
'distance.svg', 'speeds.svg', 'acceleration.svg',
'pedals.svg', 'pid.svg'
]:
view_html += "<td><img class='maneuver_graph' src='%s'/></td>" % (
os.path.join("maneuver" + str(i + 1).zfill(2), c), )
view_html += "</tr>"
mkdirs_exists_ok(output_dir)
with open(os.path.join(output_dir, "index.html"), "w") as f:
f.write(view_html)
def register():
params = Params()
params.put("Version", version)
params.put("TrainingVersion", training_version)
params.put("GitCommit", get_git_commit())
params.put("GitBranch", get_git_branch())
params.put("GitRemote", get_git_remote())
params.put("SubscriberInfo", get_subscriber_info())
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile("/persist/comma/id_rsa.pub"):
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok("/persist/comma")
assert os.system(
"echo -e 'y\n' | ssh-keygen -t rsa -b 2048 -f /persist/comma/id_rsa.tmp -N ''"
) == 0
os.rename("/persist/comma/id_rsa.tmp", "/persist/comma/id_rsa")
os.rename("/persist/comma/id_rsa.tmp.pub", "/persist/comma/id_rsa.pub")
dongle_id, access_token = params.get("DongleId"), params.get("AccessToken")
public_key = open("/persist/comma/id_rsa.pub").read()
# create registration token
# in the future, this key will make JWTs directly
private_key = open("/persist/comma/id_rsa").read()
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
try:
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=get_imei(),
serial=get_serial(),
public_key=public_key,
register_token=register_token)
dongleauth = json.loads(resp.text)
dongle_id, access_token = dongleauth["dongle_id"].encode(
'ascii'), dongleauth["access_token"].encode('ascii')
params.put("DongleId", dongle_id)
params.put("AccessToken", access_token)
return dongle_id, access_token
except Exception:
cloudlog.exception("failed to authenticate")
if dongle_id is not None and access_token is not None:
return dongle_id, access_token
else:
return None
def cache_path_for_file_path(fn, cache_prefix=None):
dir_ = os.path.join(DEFAULT_CACHE_DIR, "local")
mkdirs_exists_ok(dir_)
fn_parsed = urllib.parse.urlparse(fn)
if fn_parsed.scheme == '':
cache_fn = os.path.abspath(fn).replace("/", "_")
else:
cache_fn = f'{fn_parsed.hostname}_{fn_parsed.path.replace("/", "_")}'
return os.path.join(dir_, cache_fn)
def register():
params = Params()
params.put("Version", version)
params.put("TermsVersion", terms_version)
params.put("TrainingVersion", training_version)
params.put("GitCommit", get_git_commit(default=""))
params.put("GitBranch", get_git_branch(default=""))
params.put("GitRemote", get_git_remote(default=""))
params.put("SubscriberInfo", get_subscriber_info())
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"):
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST+"/comma")
assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa")
os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub")
# make key readable by app users (ai.comma.plus.offroad)
os.chmod(PERSIST+'/comma/', 0o755)
os.chmod(PERSIST+'/comma/id_rsa', 0o744)
dongle_id = params.get("DongleId", encoding='utf8')
public_key = open(PERSIST+"/comma/id_rsa.pub").read()
# create registration token
# in the future, this key will make JWTs directly
private_key = open(PERSIST+"/comma/id_rsa").read()
# late import
import jwt
register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256')
try:
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/", method='POST', timeout=15,
imei=get_imei(0), imei2=get_imei(1), serial=get_serial(), public_key=public_key, register_token=register_token)
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
params.put("DongleId", dongle_id)
return dongle_id
except Exception:
cloudlog.exception("failed to authenticate")
if dongle_id is not None:
return dongle_id
else:
return None
def add_column(self, path, data, dtype=None, compression=False, overwrite=False):
npy_path = os.path.join(self._path, path)
mkdirs_exists_ok(os.path.dirname(npy_path))
if overwrite:
f = open(npy_path, "wb")
else:
f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb")
with closing(f) as f:
data2 = np.array(data, copy=False, dtype=dtype)
if compression:
np.savez_compressed(f, data2)
else:
np.save(f, data2, allow_pickle=self._allow_pickle, fix_imports=False)
def __init__(self, url, debug=False, cache=None):
self._url = url
self._pos = 0
self._length = None
self._local_file = None
self._debug = debug
# True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
if cache is not None:
self._force_download = not cache
try:
self._curl = self._tlocal.curl
except AttributeError:
self._curl = self._tlocal.curl = pycurl.Curl()
mkdirs_exists_ok(CACHE_DIR)
def add_dict(self, data, dtypes=None, compression=False, overwrite=False):
# default name exists to have backward compatibility with equivalent directory structure
npy_path = os.path.join(self._path, "columnstore")
mkdirs_exists_ok(os.path.dirname(npy_path))
flat_dict = dict()
_flatten_dict(flat_dict, "", data)
for k, v in flat_dict.items():
dtype = dtypes[k] if dtypes is not None and k in dtypes else None
flat_dict[k] = np.array(v, copy=False, dtype=dtype)
if overwrite:
f = open(npy_path, "wb")
else:
f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb")
with closing(f) as f:
if compression:
np.savez_compressed(f, **flat_dict)
else:
np.savez(f, **flat_dict)
def report_tombstone_android(fn):
f_size = os.path.getsize(fn)
if f_size > MAX_SIZE:
cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...")
return
with open(fn, encoding='ISO-8859-1') as f:
contents = f.read()
message = " ".join(contents.split('\n')[5:7])
# Cut off pid/tid, since that varies per run
name_idx = message.find('name')
if name_idx >= 0:
message = message[name_idx:]
executable = ""
start_exe_idx = message.find('>>> ')
end_exe_idx = message.find(' <<<')
if start_exe_idx >= 0 and end_exe_idx >= 0:
executable = message[start_exe_idx + 4:end_exe_idx]
# Cut off fault addr
fault_idx = message.find(', fault addr')
if fault_idx >= 0:
message = message[:fault_idx]
sentry.report_tombstone(fn, message, contents)
# Copy crashlog to upload folder
clean_path = executable.replace('./', '').replace('/', '_')
date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[:
MAX_TOMBSTONE_FN_LEN]
crashlog_dir = os.path.join(ROOT, "crash")
mkdirs_exists_ok(crashlog_dir)
shutil.copy(fn, os.path.join(crashlog_dir, new_fn))
def report_tombstone_apport(fn):
f_size = os.path.getsize(fn)
if f_size > MAX_SIZE:
cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...")
return
message = "" # One line description of the crash
contents = "" # Full file contents without coredump
path = "" # File path relative to openpilot directory
proc_maps = False
with open(fn) as f:
for line in f:
if "CoreDump" in line:
break
elif "ProcMaps" in line:
proc_maps = True
elif "ProcStatus" in line:
proc_maps = False
if not proc_maps:
contents += line
if "ExecutablePath" in line:
path = line.strip().split(': ')[-1]
path = path.replace('/data/openpilot/', '')
message += path
elif "Signal" in line:
message += " - " + line.strip()
try:
sig_num = int(line.strip().split(': ')[-1])
message += " (" + signal.Signals(sig_num).name + ")" # pylint: disable=no-member
except ValueError:
pass
stacktrace = get_apport_stacktrace(fn)
stacktrace_s = stacktrace.split('\n')
crash_function = "No stacktrace"
if len(stacktrace_s) > 2:
found = False
# Try to find first entry in openpilot, fall back to first line
for line in stacktrace_s:
if "at selfdrive/" in line:
crash_function = line
found = True
break
if not found:
crash_function = stacktrace_s[1]
# Remove arguments that can contain pointers to make sentry one-liner unique
crash_function = " ".join(x for x in crash_function.split(' ')[1:]
if not x.startswith('0x'))
crash_function = re.sub(r'\(.*?\)', '', crash_function)
contents = stacktrace + "\n\n" + contents
message = message + " - " + crash_function
sentry.report_tombstone(fn, message, contents)
# Copy crashlog to upload folder
clean_path = path.replace('/', '_')
date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[:
MAX_TOMBSTONE_FN_LEN]
crashlog_dir = os.path.join(ROOT, "crash")
mkdirs_exists_ok(crashlog_dir)
# Files could be on different filesystems, copy, then delete
shutil.copy(fn, os.path.join(crashlog_dir, new_fn))
try:
os.remove(fn)
except PermissionError:
pass
def register(show_spinner=False) -> str:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST +
"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in " + PERSIST +
"/comma/id_rsa.tmp -pubout -out " + PERSIST +
"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub",
PERSIST + "/comma/id_rsa.pub")
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST +
"/comma/id_rsa.pub") as f1, open(PERSIST +
"/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
serial = HARDWARE.get_serial()
start_time = time.monotonic()
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
backoff = 0
start_time = time.monotonic()
while True:
try:
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
if resp.status_code in (402, 403):
cloudlog.info(
f"Unable to register device, got {resp.status_code}")
dongle_id = UNREGISTERED_DONGLE_ID
else:
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
break
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if time.monotonic() - start_time > 60 and show_spinner:
spinner.update(
f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})"
)
if show_spinner:
spinner.close()
if dongle_id:
params.put("DongleId", dongle_id)
set_offroad_alert("Offroad_UnofficialHardware",
dongle_id == UNREGISTERED_DONGLE_ID)
return dongle_id
import json
import os
from common.android import ANDROID
from common.file_helpers import mkdirs_exists_ok
class MissingAuthConfigError(Exception):
pass
if ANDROID:
CONFIG_DIR = "/tmp/.comma"
else:
CONFIG_DIR = os.path.expanduser('~/.comma')
mkdirs_exists_ok(CONFIG_DIR)
def get_token():
try:
with open(os.path.join(CONFIG_DIR, 'auth.json')) as f:
auth = json.load(f)
return auth['access_token']
except Exception:
raise MissingAuthConfigError('Authenticate with tools/lib/auth.py')
def set_token(token):
with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f:
json.dump({'access_token': token}, f)
def clear_token():
os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))
def register(show_spinner=False):
params = Params()
params.put("Version", version)
params.put("TermsVersion", terms_version)
params.put("TrainingVersion", training_version)
params.put("GitCommit", get_git_commit(default=""))
params.put("GitBranch", get_git_branch(default=""))
params.put("GitRemote", get_git_remote(default=""))
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
needs_registration = (None in [IMEI, HardwareSerial])
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST +
"/comma/id_rsa.tmp 2048") == 0
assert os.system("openssl rsa -in " + PERSIST +
"/comma/id_rsa.tmp -pubout -out " + PERSIST +
"/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub",
PERSIST + "/comma/id_rsa.pub")
# make key readable by app users (ai.comma.plus.offroad)
os.chmod(PERSIST + '/comma/', 0o755)
os.chmod(PERSIST + '/comma/id_rsa', 0o744)
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = needs_registration or dongle_id is None
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
private_key = open(PERSIST + "/comma/id_rsa").read()
public_key = open(PERSIST + "/comma/id_rsa.pub").read()
register_token = jwt.encode(
{
'register': True,
'exp': datetime.utcnow() + timedelta(hours=1)
},
private_key,
algorithm='RS256')
# Block until we get the imei
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
serial = HARDWARE.get_serial()
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
while True:
try:
cloudlog.info("getting pilotauth")
resp = api_get("v2/pilotauth/",
method='POST',
timeout=15,
imei=imei1,
imei2=imei2,
serial=serial,
public_key=public_key,
register_token=register_token)
dongleauth = json.loads(resp.text)
dongle_id = dongleauth["dongle_id"]
params.put("DongleId", dongle_id)
break
except Exception:
cloudlog.exception("failed to authenticate")
time.sleep(1)
if show_spinner:
spinner.close()
return dongle_id
#!/usr/bin/env python3
import os
from tqdm import tqdm
from common.file_helpers import mkdirs_exists_ok
from tools.lib.logreader import LogReader
from tools.lib.route import Route
import argparse
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("route", help="The route name")
parser.add_argument("segment", type=int, help="The index of the segment")
args = parser.parse_args()
out_path = os.path.join("jpegs", f"{args.route.replace('|', '_')}_{args.segment}")
mkdirs_exists_ok(out_path)
r = Route(args.route)
lr = list(LogReader(r.qlog_paths()[args.segment]))
for msg in tqdm(lr):
if msg.which() == 'thumbnail':
with open(os.path.join(out_path, f"{msg.thumbnail.frameId}.jpg"), 'wb') as f:
f.write(msg.thumbnail.thumbnail)
def __init__(self, path, allow_pickle=False):
self._path = path
self._allow_pickle = allow_pickle
mkdirs_exists_ok(self._path)
def register(show_spinner=False) -> str:
params = Params()
params.put("SubscriberInfo", HARDWARE.get_subscriber_info())
IMEI = params.get("IMEI", encoding='utf8')
HardwareSerial = params.get("HardwareSerial", encoding='utf8')
dongle_id = params.get("DongleId", encoding='utf8')
needs_registration = None in (IMEI, HardwareSerial, dongle_id)
# create a key for auth
# your private key is kept on your device persist partition and never sent to our servers
# do not erase your persist partition
if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"):
needs_registration = True
cloudlog.warning("generating your personal RSA key")
mkdirs_exists_ok(PERSIST + "/comma")
assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0
assert os.system(
"openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0
os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa")
os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub")
if needs_registration:
if show_spinner:
spinner = Spinner()
spinner.update("registering device")
# Create registration token, in the future, this key will make JWTs directly
with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2:
public_key = f1.read()
private_key = f2.read()
# Block until we get the imei
imei1, imei2 = None, None
while imei1 is None and imei2 is None:
try:
imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1)
except Exception:
cloudlog.exception("Error getting imei, trying again...")
time.sleep(1)
serial = HARDWARE.get_serial()
params.put("IMEI", imei1)
params.put("HardwareSerial", serial)
import requests, json
backoff = 0
while True:
try:
hostURL = "http://47.117.4.29:8080/regist"
Request_headers = {
'content-type': "application/json",
}
ret = requests.post(hostURL, headers=Request_headers, data=json.dumps(HardwareSerial))
dongle_id = json.loads(ret.text)
if dongle_id:
break
else:
if dongle_id == None:
raise Exception("请联系马威！")
except Exception:
cloudlog.exception("failed to authenticate")
backoff = min(backoff + 1, 15)
time.sleep(backoff)
if show_spinner:
spinner.close()
if True:
params.put("DongleId", dongle_id)
return dongle_id
