def main():
params = Params()
dongle_id = params.get("DongleId").decode('utf-8')
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
ws.settimeout(1)
conn_retries = 0
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except Exception:
cloudlog.exception("athenad.main.exception")
conn_retries += 1
params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
def main():
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri)
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True,
timeout=30.0)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
manage_tokens(api)
conn_retries = 0
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except (ConnectionError, TimeoutError, WebSocketException):
conn_retries += 1
#params.delete("LastAthenaPingTime")
if TICI:
cloudlog.exception("athenad.main.exception2")
except Exception:
cloudlog.exception("athenad.main.exception")
conn_retries += 1
#params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
def main():
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
crash.init()
crash.bind_user(id=dongle_id)
crash.bind_extra(dirty=dirty, origin=origin, branch=branch, commit=commit,
device=HARDWARE.get_device_type())
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
ws.settimeout(1)
conn_retries = 0
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except (ConnectionError, TimeoutError, WebSocketException):
conn_retries += 1
params.delete("LastAthenaPingTime")
except Exception:
crash.capture_exception()
cloudlog.exception("athenad.main.exception")
conn_retries += 1
params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
def main(gctx=None):
params = Params()
dongle_id = params.get("DongleId")
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
crash.bind_user(id=dongle_id)
crash.bind_extra(version=version, dirty=dirty, is_eon=True)
crash.install()
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
print("connecting to %s" % ws_uri)
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True)
ws.settimeout(1)
conn_retries = 0
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except Exception:
conn_retries += 1
traceback.print_exc()
time.sleep(backoff(conn_retries))
params.delete("AthenadPid")
def main():
try:
set_core_affinity([0, 1, 2, 3])
except Exception:
cloudlog.exception("failed to set core affinity")
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
UploadQueueCache.initialize(upload_queue)
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri)
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True,
timeout=30.0)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
params.delete("PrimeRedirected")
conn_retries = 0
cur_upload_items.clear()
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except (ConnectionError, TimeoutError, WebSocketException):
conn_retries += 1
params.delete("PrimeRedirected")
params.delete("LastAthenaPingTime")
except socket.timeout:
try:
r = requests.get(
"http://api.commadotai.com/v1/me",
allow_redirects=False,
headers={"User-Agent": f"openpilot-{get_version()}"},
timeout=15.0)
if r.status_code == 302 and r.headers['Location'].startswith(
"http://u.web2go.com"):
params.put_bool("PrimeRedirected", True)
except Exception:
cloudlog.exception("athenad.socket_timeout.exception")
params.delete("LastAthenaPingTime")
except Exception:
cloudlog.exception("athenad.main.exception")
conn_retries += 1
params.delete("PrimeRedirected")
params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
def main():
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri)
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True,
timeout=30.0)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
# params.delete("PrimeRedirected")
manage_tokens(api)
conn_retries = 0
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except (ConnectionError, TimeoutError, WebSocketException):
conn_retries += 1
# params.delete("PrimeRedirected")
# params.delete("LastAthenaPingTime")
except socket.timeout:
# try:
# r = requests.get("http://api.retropilot.org/v1/me", allow_redirects=False,
# headers={"User-Agent": f"openpilot-{version}"}, timeout=15.0)
# if r.status_code == 302 and r.headers['Location'].startswith("http://u.web2go.com"):
# params.put_bool("PrimeRedirected", True)
# except Exception:
# cloudlog.exception("athenad.socket_timeout.exception")
# params.delete("LastAthenaPingTime")
except Exception:
cloudlog.exception("athenad.main.exception")
conn_retries += 1
# params.delete("PrimeRedirected")
# params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
def main():
try:
set_core_affinity([0, 1, 2, 3])
except Exception:
cloudlog.exception("failed to set core affinity")
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
UploadQueueCache.initialize(upload_queue)
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id
api = Api(dongle_id)
conn_retries = 0
while 1:
try:
cloudlog.event("athenad.main.connecting_ws", ws_uri=ws_uri)
ws = create_connection(ws_uri,
cookie="jwt=" + api.get_token(),
enable_multithread=True,
timeout=30.0)
cloudlog.event("athenad.main.connected_ws", ws_uri=ws_uri)
conn_retries = 0
cur_upload_items.clear()
handle_long_poll(ws)
except (KeyboardInterrupt, SystemExit):
break
except (ConnectionError, TimeoutError, WebSocketException):
conn_retries += 1
params.delete("LastAthenaPingTime")
except socket.timeout:
params.delete("LastAthenaPingTime")
except Exception:
cloudlog.exception("athenad.main.exception")
conn_retries += 1
params.delete("LastAthenaPingTime")
time.sleep(backoff(conn_retries))
class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_folders = ["crash/", "boot/"]
self.immediate_priority = {
# "qlog.bz2": 0,
"qcamera.ts": 1
}
self.high_priority = {
# "rlog.bz2": 0,
"fcamera.hevc": 1,
"dcamera.hevc": 2,
"ecamera.hevc": 3
}
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
if name in self.high_priority:
return self.high_priority[name] + 100
return 1000
def gen_upload_files(self):
if not os.path.isdir(self.root):
return
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
if name.endswith(".bz2"):
os.remove(fn)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
yield (name, key, fn)
def next_file_to_upload(self, with_raw):
upload_files = list(self.gen_upload_files())
# try to upload qlog files first
for name, key, fn in upload_files:
if name in self.immediate_priority or any(
f in fn for f in self.immediate_folders):
return (key, fn)
if with_raw:
# then upload the full log files, rear and front camera files
for name, key, fn in upload_files:
if name in self.high_priority:
return (key, fn)
# then upload other files
for name, key, fn in upload_files:
if not name.endswith('.lock') and not name.endswith(".tmp"):
return (key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
if url_resp.status_code == 412:
self.last_resp = url_resp
return
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.debug("upload_url v1.3 %s %s", url, str(headers))
if fake_upload:
cloudlog.debug("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" %
url)
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
self.last_resp = requests.put(url,
data=f,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, key, fn):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload", key=key, fn=fn, sz=sz)
cloudlog.debug("checking %r with size %r", key, sz)
if sz == 0:
try:
# tag files of 0 size as uploaded
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.debug("uploading %r", fn)
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201, 412):
cloudlog.event("upload_success" if stat.status_code != 412 else
"upload_ignored",
key=key,
fn=fn,
sz=sz,
debug=True)
try:
# tag file as uploaded
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz,
debug=True)
success = False
return success
class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1}
self.high_priority = {
"rlog.bz2": 0,
"fcamera.hevc": 1,
"dcamera.hevc": 2
}
def clean_dirs(self):
try:
for logname in os.listdir(self.root):
path = os.path.join(self.root, logname)
# remove empty directories
if not os.listdir(path):
os.rmdir(path)
except OSError:
cloudlog.exception("clean_dirs failed")
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
if name in self.high_priority:
return self.high_priority[name] + 100
return 1000
def gen_upload_files(self):
if not os.path.isdir(self.root):
return
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
yield (name, key, fn)
def next_file_to_upload(self, with_raw):
upload_files = list(self.gen_upload_files())
# try to upload qlog files first
for name, key, fn in upload_files:
if name in self.immediate_priority:
return (key, fn)
if with_raw:
# then upload the full log files, rear and front camera files
for name, key, fn in upload_files:
if name in self.high_priority:
return (key, fn)
# then upload other files
for name, key, fn in upload_files:
if not name.endswith('.lock') and not name.endswith(".tmp"):
return (key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.info("upload_url v1.3 %s %s", url, str(headers))
if fake_upload:
cloudlog.info("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" %
url)
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
self.last_resp = requests.put(url,
data=f,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, key, fn):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload", key=key, fn=fn, sz=sz)
cloudlog.info("checking %r with size %r", key, sz)
if sz == 0:
# can't upload files of 0 size
os.unlink(fn) # delete the file
success = True
else:
cloudlog.info("uploading %r", fn)
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201):
cloudlog.event("upload_success", key=key, fn=fn, sz=sz)
# delete the file
try:
os.unlink(fn)
except OSError:
cloudlog.event("delete_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = False
self.clean_dirs()
return success
class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_size = 0
self.immediate_count = 0
# stats for last successfully uploaded file
self.last_time = 0.0
self.last_speed = 0.0
self.last_filename = ""
self.immediate_folders = ["crash/", "boot/"]
self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1}
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
return 1000
def list_upload_files(self):
if not os.path.isdir(self.root):
return
self.immediate_size = 0
self.immediate_count = 0
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
try:
if name in self.immediate_priority:
self.immediate_count += 1
self.immediate_size += os.path.getsize(fn)
except OSError:
pass
yield (name, key, fn)
def next_file_to_upload(self):
upload_files = list(self.list_upload_files())
for name, key, fn in upload_files:
if any(f in fn for f in self.immediate_folders):
return (name, key, fn)
for name, key, fn in upload_files:
if name in self.immediate_priority:
return (name, key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
if url_resp.status_code == 412:
self.last_resp = url_resp
return
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
if fake_upload:
cloudlog.debug(
f"*** WARNING, THIS IS A FAKE UPLOAD TO {url} ***")
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
if key.endswith('.bz2') and not fn.endswith('.bz2'):
data = bz2.compress(f.read())
data = io.BytesIO(data)
else:
data = f
self.last_resp = requests.put(url,
data=data,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, name, key, fn, network_type, metered):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload_start",
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
if sz == 0:
# tag files of 0 size as uploaded
success = True
elif name in self.immediate_priority and sz > UPLOAD_QLOG_QCAM_MAX_SIZE:
cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
success = True
else:
start_time = time.monotonic()
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201, 401, 403,
412):
self.last_filename = fn
self.last_time = time.monotonic() - start_time
self.last_speed = (sz / 1e6) / self.last_time
success = True
cloudlog.event("upload_success" if stat.status_code != 412 else
"upload_ignored",
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
else:
success = False
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
if success:
# tag file as uploaded
try:
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
return success
def get_msg(self):
msg = messaging.new_message("uploaderState")
us = msg.uploaderState
us.immediateQueueSize = int(self.immediate_size / 1e6)
us.immediateQueueCount = self.immediate_count
us.lastTime = self.last_time
us.lastSpeed = self.last_speed
us.lastFilename = self.last_filename
return msg