class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1}
self.high_priority = {
"rlog.bz2": 0,
"fcamera.hevc": 1,
"dcamera.hevc": 2
}
def clean_dirs(self):
try:
for logname in os.listdir(self.root):
path = os.path.join(self.root, logname)
# remove empty directories
if not os.listdir(path):
os.rmdir(path)
except OSError:
cloudlog.exception("clean_dirs failed")
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
if name in self.high_priority:
return self.high_priority[name] + 100
return 1000
def gen_upload_files(self):
if not os.path.isdir(self.root):
return
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
yield (name, key, fn)
def next_file_to_upload(self, with_raw):
upload_files = list(self.gen_upload_files())
# try to upload qlog files first
for name, key, fn in upload_files:
if name in self.immediate_priority:
return (key, fn)
if with_raw:
# then upload the full log files, rear and front camera files
for name, key, fn in upload_files:
if name in self.high_priority:
return (key, fn)
# then upload other files
for name, key, fn in upload_files:
if not name.endswith('.lock') and not name.endswith(".tmp"):
return (key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.info("upload_url v1.3 %s %s", url, str(headers))
if fake_upload:
cloudlog.info("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" %
url)
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
self.last_resp = requests.put(url,
data=f,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, key, fn):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload", key=key, fn=fn, sz=sz)
cloudlog.info("checking %r with size %r", key, sz)
if sz == 0:
# can't upload files of 0 size
os.unlink(fn) # delete the file
success = True
else:
cloudlog.info("uploading %r", fn)
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201):
cloudlog.event("upload_success", key=key, fn=fn, sz=sz)
# delete the file
try:
os.unlink(fn)
except OSError:
cloudlog.event("delete_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = False
self.clean_dirs()
return success
class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_folders = ["crash/", "boot/"]
self.immediate_priority = {
# "qlog.bz2": 0,
"qcamera.ts": 1
}
self.high_priority = {
# "rlog.bz2": 0,
"fcamera.hevc": 1,
"dcamera.hevc": 2,
"ecamera.hevc": 3
}
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
if name in self.high_priority:
return self.high_priority[name] + 100
return 1000
def gen_upload_files(self):
if not os.path.isdir(self.root):
return
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
if name.endswith(".bz2"):
os.remove(fn)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
yield (name, key, fn)
def next_file_to_upload(self, with_raw):
upload_files = list(self.gen_upload_files())
# try to upload qlog files first
for name, key, fn in upload_files:
if name in self.immediate_priority or any(
f in fn for f in self.immediate_folders):
return (key, fn)
if with_raw:
# then upload the full log files, rear and front camera files
for name, key, fn in upload_files:
if name in self.high_priority:
return (key, fn)
# then upload other files
for name, key, fn in upload_files:
if not name.endswith('.lock') and not name.endswith(".tmp"):
return (key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
if url_resp.status_code == 412:
self.last_resp = url_resp
return
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.debug("upload_url v1.3 %s %s", url, str(headers))
if fake_upload:
cloudlog.debug("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" %
url)
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
self.last_resp = requests.put(url,
data=f,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, key, fn):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload", key=key, fn=fn, sz=sz)
cloudlog.debug("checking %r with size %r", key, sz)
if sz == 0:
try:
# tag files of 0 size as uploaded
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.debug("uploading %r", fn)
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201, 412):
cloudlog.event("upload_success" if stat.status_code != 412 else
"upload_ignored",
key=key,
fn=fn,
sz=sz,
debug=True)
try:
# tag file as uploaded
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
success = True
else:
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz,
debug=True)
success = False
return success
class Uploader():
def __init__(self, dongle_id, root):
self.dongle_id = dongle_id
self.api = Api(dongle_id)
self.root = root
self.upload_thread = None
self.last_resp = None
self.last_exc = None
self.immediate_size = 0
self.immediate_count = 0
# stats for last successfully uploaded file
self.last_time = 0.0
self.last_speed = 0.0
self.last_filename = ""
self.immediate_folders = ["crash/", "boot/"]
self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1}
def get_upload_sort(self, name):
if name in self.immediate_priority:
return self.immediate_priority[name]
return 1000
def list_upload_files(self):
if not os.path.isdir(self.root):
return
self.immediate_size = 0
self.immediate_count = 0
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
try:
if name in self.immediate_priority:
self.immediate_count += 1
self.immediate_size += os.path.getsize(fn)
except OSError:
pass
yield (name, key, fn)
def next_file_to_upload(self):
upload_files = list(self.list_upload_files())
for name, key, fn in upload_files:
if any(f in fn for f in self.immediate_folders):
return (name, key, fn)
for name, key, fn in upload_files:
if name in self.immediate_priority:
return (name, key, fn)
return None
def do_upload(self, key, fn):
try:
url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/",
timeout=10,
path=key,
access_token=self.api.get_token())
if url_resp.status_code == 412:
self.last_resp = url_resp
return
url_resp_json = json.loads(url_resp.text)
url = url_resp_json['url']
headers = url_resp_json['headers']
cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
if fake_upload:
cloudlog.debug(
f"*** WARNING, THIS IS A FAKE UPLOAD TO {url} ***")
class FakeResponse():
def __init__(self):
self.status_code = 200
self.last_resp = FakeResponse()
else:
with open(fn, "rb") as f:
if key.endswith('.bz2') and not fn.endswith('.bz2'):
data = bz2.compress(f.read())
data = io.BytesIO(data)
else:
data = f
self.last_resp = requests.put(url,
data=data,
headers=headers,
timeout=10)
except Exception as e:
self.last_exc = (e, traceback.format_exc())
raise
def normal_upload(self, key, fn):
self.last_resp = None
self.last_exc = None
try:
self.do_upload(key, fn)
except Exception:
pass
return self.last_resp
def upload(self, name, key, fn, network_type, metered):
try:
sz = os.path.getsize(fn)
except OSError:
cloudlog.exception("upload: getsize failed")
return False
cloudlog.event("upload_start",
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
if sz == 0:
# tag files of 0 size as uploaded
success = True
elif name in self.immediate_priority and sz > UPLOAD_QLOG_QCAM_MAX_SIZE:
cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
success = True
else:
start_time = time.monotonic()
stat = self.normal_upload(key, fn)
if stat is not None and stat.status_code in (200, 201, 401, 403,
412):
self.last_filename = fn
self.last_time = time.monotonic() - start_time
self.last_speed = (sz / 1e6) / self.last_time
success = True
cloudlog.event("upload_success" if stat.status_code != 412 else
"upload_ignored",
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
else:
success = False
cloudlog.event("upload_failed",
stat=stat,
exc=self.last_exc,
key=key,
fn=fn,
sz=sz,
network_type=network_type,
metered=metered)
if success:
# tag file as uploaded
try:
setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
except OSError:
cloudlog.event("uploader_setxattr_failed",
exc=self.last_exc,
key=key,
fn=fn,
sz=sz)
return success
def get_msg(self):
msg = messaging.new_message("uploaderState")
us = msg.uploaderState
us.immediateQueueSize = int(self.immediate_size / 1e6)
us.immediateQueueCount = self.immediate_count
us.lastTime = self.last_time
us.lastSpeed = self.last_speed
us.lastFilename = self.last_filename
return msg