def map_column(self, path, dtype, shape): npy_path = os.path.join(self._path, path) mkdirs_exists_ok(os.path.dirname(npy_path)) return np.lib.format.open_memmap(npy_path, mode='w+', dtype=dtype, shape=shape)
def setup_output(): output_dir = os.path.join(os.getcwd(), 'out/longitudinal') if not os.path.exists(os.path.join(output_dir, "index.html")): # write test output header css_style = """ .maneuver_title { font-size: 24px; text-align: center; } .maneuver_graph { width: 100%; } """ view_html = "<html><head><style>%s</style></head><body><table>" % ( css_style, ) for i, man in enumerate(maneuvers): view_html += "<tr><td class='maneuver_title' colspan=5><div>%s</div></td></tr><tr>" % ( man.title, ) for c in [ 'distance.svg', 'speeds.svg', 'acceleration.svg', 'pedals.svg', 'pid.svg' ]: view_html += "<td><img class='maneuver_graph' src='%s'/></td>" % ( os.path.join("maneuver" + str(i + 1).zfill(2), c), ) view_html += "</tr>" mkdirs_exists_ok(output_dir) with open(os.path.join(output_dir, "index.html"), "w") as f: f.write(view_html)
def register(): params = Params() params.put("Version", version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_git_commit()) params.put("GitBranch", get_git_branch()) params.put("GitRemote", get_git_remote()) params.put("SubscriberInfo", get_subscriber_info()) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile("/persist/comma/id_rsa.pub"): cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok("/persist/comma") assert os.system( "echo -e 'y\n' | ssh-keygen -t rsa -b 2048 -f /persist/comma/id_rsa.tmp -N ''" ) == 0 os.rename("/persist/comma/id_rsa.tmp", "/persist/comma/id_rsa") os.rename("/persist/comma/id_rsa.tmp.pub", "/persist/comma/id_rsa.pub") dongle_id, access_token = params.get("DongleId"), params.get("AccessToken") public_key = open("/persist/comma/id_rsa.pub").read() # create registration token # in the future, this key will make JWTs directly private_key = open("/persist/comma/id_rsa").read() register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') try: cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=get_imei(), serial=get_serial(), public_key=public_key, register_token=register_token) dongleauth = json.loads(resp.text) dongle_id, access_token = dongleauth["dongle_id"].encode( 'ascii'), dongleauth["access_token"].encode('ascii') params.put("DongleId", dongle_id) params.put("AccessToken", access_token) return dongle_id, access_token except Exception: cloudlog.exception("failed to authenticate") if dongle_id is not None and access_token is not None: return dongle_id, access_token else: return None
def cache_path_for_file_path(fn, cache_prefix=None): dir_ = os.path.join(DEFAULT_CACHE_DIR, "local") mkdirs_exists_ok(dir_) fn_parsed = urllib.parse.urlparse(fn) if fn_parsed.scheme == '': cache_fn = os.path.abspath(fn).replace("/", "_") else: cache_fn = f'{fn_parsed.hostname}_{fn_parsed.path.replace("/", "_")}' return os.path.join(dir_, cache_fn)
def register(): params = Params() params.put("Version", version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_git_commit(default="")) params.put("GitBranch", get_git_branch(default="")) params.put("GitRemote", get_git_remote(default="")) params.put("SubscriberInfo", get_subscriber_info()) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST+"/comma/id_rsa.pub"): cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST+"/comma") assert os.system("openssl genrsa -out "+PERSIST+"/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in "+PERSIST+"/comma/id_rsa.tmp -pubout -out "+PERSIST+"/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST+"/comma/id_rsa.tmp", PERSIST+"/comma/id_rsa") os.rename(PERSIST+"/comma/id_rsa.tmp.pub", PERSIST+"/comma/id_rsa.pub") # make key readable by app users (ai.comma.plus.offroad) os.chmod(PERSIST+'/comma/', 0o755) os.chmod(PERSIST+'/comma/id_rsa', 0o744) dongle_id = params.get("DongleId", encoding='utf8') public_key = open(PERSIST+"/comma/id_rsa.pub").read() # create registration token # in the future, this key will make JWTs directly private_key = open(PERSIST+"/comma/id_rsa").read() # late import import jwt register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256') try: cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=get_imei(0), imei2=get_imei(1), serial=get_serial(), public_key=public_key, register_token=register_token) dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] params.put("DongleId", dongle_id) return dongle_id except Exception: cloudlog.exception("failed to authenticate") if dongle_id is not None: return dongle_id else: return None
def add_column(self, path, data, dtype=None, compression=False, overwrite=False): npy_path = os.path.join(self._path, path) mkdirs_exists_ok(os.path.dirname(npy_path)) if overwrite: f = open(npy_path, "wb") else: f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb") with closing(f) as f: data2 = np.array(data, copy=False, dtype=dtype) if compression: np.savez_compressed(f, data2) else: np.save(f, data2, allow_pickle=self._allow_pickle, fix_imports=False)
def __init__(self, url, debug=False, cache=None): self._url = url self._pos = 0 self._length = None self._local_file = None self._debug = debug # True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0")) if cache is not None: self._force_download = not cache try: self._curl = self._tlocal.curl except AttributeError: self._curl = self._tlocal.curl = pycurl.Curl() mkdirs_exists_ok(CACHE_DIR)
def add_dict(self, data, dtypes=None, compression=False, overwrite=False): # default name exists to have backward compatibility with equivalent directory structure npy_path = os.path.join(self._path, "columnstore") mkdirs_exists_ok(os.path.dirname(npy_path)) flat_dict = dict() _flatten_dict(flat_dict, "", data) for k, v in flat_dict.items(): dtype = dtypes[k] if dtypes is not None and k in dtypes else None flat_dict[k] = np.array(v, copy=False, dtype=dtype) if overwrite: f = open(npy_path, "wb") else: f = os.fdopen(os.open(npy_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL), "wb") with closing(f) as f: if compression: np.savez_compressed(f, **flat_dict) else: np.savez(f, **flat_dict)
def report_tombstone_android(fn): f_size = os.path.getsize(fn) if f_size > MAX_SIZE: cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...") return with open(fn, encoding='ISO-8859-1') as f: contents = f.read() message = " ".join(contents.split('\n')[5:7]) # Cut off pid/tid, since that varies per run name_idx = message.find('name') if name_idx >= 0: message = message[name_idx:] executable = "" start_exe_idx = message.find('>>> ') end_exe_idx = message.find(' <<<') if start_exe_idx >= 0 and end_exe_idx >= 0: executable = message[start_exe_idx + 4:end_exe_idx] # Cut off fault addr fault_idx = message.find(', fault addr') if fault_idx >= 0: message = message[:fault_idx] sentry.report_tombstone(fn, message, contents) # Copy crashlog to upload folder clean_path = executable.replace('./', '').replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[: MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(ROOT, "crash") mkdirs_exists_ok(crashlog_dir) shutil.copy(fn, os.path.join(crashlog_dir, new_fn))
def report_tombstone_apport(fn): f_size = os.path.getsize(fn) if f_size > MAX_SIZE: cloudlog.error(f"Tombstone {fn} too big, {f_size}. Skipping...") return message = "" # One line description of the crash contents = "" # Full file contents without coredump path = "" # File path relative to openpilot directory proc_maps = False with open(fn) as f: for line in f: if "CoreDump" in line: break elif "ProcMaps" in line: proc_maps = True elif "ProcStatus" in line: proc_maps = False if not proc_maps: contents += line if "ExecutablePath" in line: path = line.strip().split(': ')[-1] path = path.replace('/data/openpilot/', '') message += path elif "Signal" in line: message += " - " + line.strip() try: sig_num = int(line.strip().split(': ')[-1]) message += " (" + signal.Signals(sig_num).name + ")" # pylint: disable=no-member except ValueError: pass stacktrace = get_apport_stacktrace(fn) stacktrace_s = stacktrace.split('\n') crash_function = "No stacktrace" if len(stacktrace_s) > 2: found = False # Try to find first entry in openpilot, fall back to first line for line in stacktrace_s: if "at selfdrive/" in line: crash_function = line found = True break if not found: crash_function = stacktrace_s[1] # Remove arguments that can contain pointers to make sentry one-liner unique crash_function = " ".join(x for x in crash_function.split(' ')[1:] if not x.startswith('0x')) crash_function = re.sub(r'\(.*?\)', '', crash_function) contents = stacktrace + "\n\n" + contents message = message + " - " + crash_function sentry.report_tombstone(fn, message, contents) # Copy crashlog to upload folder clean_path = path.replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[: MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(ROOT, "crash") mkdirs_exists_ok(crashlog_dir) # Files could be on different filesystems, copy, then delete shutil.copy(fn, os.path.join(crashlog_dir, new_fn)) try: os.remove(fn) except PermissionError: pass
def register(show_spinner=False) -> str: params = Params() params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2: public_key = f1.read() private_key = f2.read() # Block until we get the imei serial = HARDWARE.get_serial() start_time = time.monotonic() imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) params.put("IMEI", imei1) params.put("HardwareSerial", serial) backoff = 0 start_time = time.monotonic() while True: try: register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) if resp.status_code in (402, 403): cloudlog.info( f"Unable to register device, got {resp.status_code}") dongle_id = UNREGISTERED_DONGLE_ID else: dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] break except Exception: cloudlog.exception("failed to authenticate") backoff = min(backoff + 1, 15) time.sleep(backoff) if time.monotonic() - start_time > 60 and show_spinner: spinner.update( f"registering device - serial: {serial}, IMEI: ({imei1}, {imei2})" ) if show_spinner: spinner.close() if dongle_id: params.put("DongleId", dongle_id) set_offroad_alert("Offroad_UnofficialHardware", dongle_id == UNREGISTERED_DONGLE_ID) return dongle_id
import json import os from common.android import ANDROID from common.file_helpers import mkdirs_exists_ok class MissingAuthConfigError(Exception): pass if ANDROID: CONFIG_DIR = "/tmp/.comma" else: CONFIG_DIR = os.path.expanduser('~/.comma') mkdirs_exists_ok(CONFIG_DIR) def get_token(): try: with open(os.path.join(CONFIG_DIR, 'auth.json')) as f: auth = json.load(f) return auth['access_token'] except Exception: raise MissingAuthConfigError('Authenticate with tools/lib/auth.py') def set_token(token): with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f: json.dump({'access_token': token}, f) def clear_token(): os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))
def register(show_spinner=False): params = Params() params.put("Version", version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_git_commit(default="")) params.put("GitBranch", get_git_branch(default="")) params.put("GitRemote", get_git_remote(default="")) params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') needs_registration = (None in [IMEI, HardwareSerial]) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system("openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") # make key readable by app users (ai.comma.plus.offroad) os.chmod(PERSIST + '/comma/', 0o755) os.chmod(PERSIST + '/comma/id_rsa', 0o744) dongle_id = params.get("DongleId", encoding='utf8') needs_registration = needs_registration or dongle_id is None if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly private_key = open(PERSIST + "/comma/id_rsa").read() public_key = open(PERSIST + "/comma/id_rsa.pub").read() register_token = jwt.encode( { 'register': True, 'exp': datetime.utcnow() + timedelta(hours=1) }, private_key, algorithm='RS256') # Block until we get the imei imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) serial = HARDWARE.get_serial() params.put("IMEI", imei1) params.put("HardwareSerial", serial) while True: try: cloudlog.info("getting pilotauth") resp = api_get("v2/pilotauth/", method='POST', timeout=15, imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token) dongleauth = json.loads(resp.text) dongle_id = dongleauth["dongle_id"] params.put("DongleId", dongle_id) break except Exception: cloudlog.exception("failed to authenticate") time.sleep(1) if show_spinner: spinner.close() return dongle_id
#!/usr/bin/env python3 import os from tqdm import tqdm from common.file_helpers import mkdirs_exists_ok from tools.lib.logreader import LogReader from tools.lib.route import Route import argparse if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("route", help="The route name") parser.add_argument("segment", type=int, help="The index of the segment") args = parser.parse_args() out_path = os.path.join("jpegs", f"{args.route.replace('|', '_')}_{args.segment}") mkdirs_exists_ok(out_path) r = Route(args.route) lr = list(LogReader(r.qlog_paths()[args.segment])) for msg in tqdm(lr): if msg.which() == 'thumbnail': with open(os.path.join(out_path, f"{msg.thumbnail.frameId}.jpg"), 'wb') as f: f.write(msg.thumbnail.thumbnail)
def __init__(self, path, allow_pickle=False): self._path = path self._allow_pickle = allow_pickle mkdirs_exists_ok(self._path)
def register(show_spinner=False) -> str: params = Params() params.put("SubscriberInfo", HARDWARE.get_subscriber_info()) IMEI = params.get("IMEI", encoding='utf8') HardwareSerial = params.get("HardwareSerial", encoding='utf8') dongle_id = params.get("DongleId", encoding='utf8') needs_registration = None in (IMEI, HardwareSerial, dongle_id) # create a key for auth # your private key is kept on your device persist partition and never sent to our servers # do not erase your persist partition if not os.path.isfile(PERSIST + "/comma/id_rsa.pub"): needs_registration = True cloudlog.warning("generating your personal RSA key") mkdirs_exists_ok(PERSIST + "/comma") assert os.system("openssl genrsa -out " + PERSIST + "/comma/id_rsa.tmp 2048") == 0 assert os.system( "openssl rsa -in " + PERSIST + "/comma/id_rsa.tmp -pubout -out " + PERSIST + "/comma/id_rsa.tmp.pub") == 0 os.rename(PERSIST + "/comma/id_rsa.tmp", PERSIST + "/comma/id_rsa") os.rename(PERSIST + "/comma/id_rsa.tmp.pub", PERSIST + "/comma/id_rsa.pub") if needs_registration: if show_spinner: spinner = Spinner() spinner.update("registering device") # Create registration token, in the future, this key will make JWTs directly with open(PERSIST + "/comma/id_rsa.pub") as f1, open(PERSIST + "/comma/id_rsa") as f2: public_key = f1.read() private_key = f2.read() # Block until we get the imei imei1, imei2 = None, None while imei1 is None and imei2 is None: try: imei1, imei2 = HARDWARE.get_imei(0), HARDWARE.get_imei(1) except Exception: cloudlog.exception("Error getting imei, trying again...") time.sleep(1) serial = HARDWARE.get_serial() params.put("IMEI", imei1) params.put("HardwareSerial", serial) import requests, json backoff = 0 while True: try: hostURL = "http://47.117.4.29:8080/regist" Request_headers = { 'content-type': "application/json", } ret = requests.post(hostURL, headers=Request_headers, data=json.dumps(HardwareSerial)) dongle_id = json.loads(ret.text) if dongle_id: break else: if dongle_id == None: raise Exception("请联系马威!") except Exception: cloudlog.exception("failed to authenticate") backoff = min(backoff + 1, 15) time.sleep(backoff) if show_spinner: spinner.close() if True: params.put("DongleId", dongle_id) return dongle_id