class Uploader(): def __init__(self, dongle_id, root): self.dongle_id = dongle_id self.api = Api(dongle_id) self.root = root self.upload_thread = None self.last_resp = None self.last_exc = None self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1} self.high_priority = { "rlog.bz2": 0, "fcamera.hevc": 1, "dcamera.hevc": 2 } def clean_dirs(self): try: for logname in os.listdir(self.root): path = os.path.join(self.root, logname) # remove empty directories if not os.listdir(path): os.rmdir(path) except OSError: cloudlog.exception("clean_dirs failed") def get_upload_sort(self, name): if name in self.immediate_priority: return self.immediate_priority[name] if name in self.high_priority: return self.high_priority[name] + 100 return 1000 def gen_upload_files(self): if not os.path.isdir(self.root): return for logname in listdir_by_creation(self.root): path = os.path.join(self.root, logname) try: names = os.listdir(path) except OSError: continue if any(name.endswith(".lock") for name in names): continue for name in sorted(names, key=self.get_upload_sort): key = os.path.join(logname, name) fn = os.path.join(path, name) yield (name, key, fn) def next_file_to_upload(self, with_raw): upload_files = list(self.gen_upload_files()) # try to upload qlog files first for name, key, fn in upload_files: if name in self.immediate_priority: return (key, fn) if with_raw: # then upload the full log files, rear and front camera files for name, key, fn in upload_files: if name in self.high_priority: return (key, fn) # then upload other files for name, key, fn in upload_files: if not name.endswith('.lock') and not name.endswith(".tmp"): return (key, fn) return None def do_upload(self, key, fn): try: url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token()) url_resp_json = json.loads(url_resp.text) url = url_resp_json['url'] headers = url_resp_json['headers'] cloudlog.info("upload_url v1.3 %s %s", url, str(headers)) if fake_upload: cloudlog.info("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" % url) class FakeResponse(): def __init__(self): self.status_code = 200 self.last_resp = FakeResponse() else: with open(fn, "rb") as f: self.last_resp = requests.put(url, data=f, headers=headers, timeout=10) except Exception as e: self.last_exc = (e, traceback.format_exc()) raise def normal_upload(self, key, fn): self.last_resp = None self.last_exc = None try: self.do_upload(key, fn) except Exception: pass return self.last_resp def upload(self, key, fn): try: sz = os.path.getsize(fn) except OSError: cloudlog.exception("upload: getsize failed") return False cloudlog.event("upload", key=key, fn=fn, sz=sz) cloudlog.info("checking %r with size %r", key, sz) if sz == 0: # can't upload files of 0 size os.unlink(fn) # delete the file success = True else: cloudlog.info("uploading %r", fn) stat = self.normal_upload(key, fn) if stat is not None and stat.status_code in (200, 201): cloudlog.event("upload_success", key=key, fn=fn, sz=sz) # delete the file try: os.unlink(fn) except OSError: cloudlog.event("delete_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz) success = True else: cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz) success = False self.clean_dirs() return success
class Uploader(): def __init__(self, dongle_id, root): self.dongle_id = dongle_id self.api = Api(dongle_id) self.root = root self.upload_thread = None self.last_resp = None self.last_exc = None self.immediate_folders = ["crash/", "boot/"] self.immediate_priority = { # "qlog.bz2": 0, "qcamera.ts": 1 } self.high_priority = { # "rlog.bz2": 0, "fcamera.hevc": 1, "dcamera.hevc": 2, "ecamera.hevc": 3 } def get_upload_sort(self, name): if name in self.immediate_priority: return self.immediate_priority[name] if name in self.high_priority: return self.high_priority[name] + 100 return 1000 def gen_upload_files(self): if not os.path.isdir(self.root): return for logname in listdir_by_creation(self.root): path = os.path.join(self.root, logname) try: names = os.listdir(path) except OSError: continue if any(name.endswith(".lock") for name in names): continue for name in sorted(names, key=self.get_upload_sort): key = os.path.join(logname, name) fn = os.path.join(path, name) if name.endswith(".bz2"): os.remove(fn) # skip files already uploaded try: is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) except OSError: cloudlog.event("uploader_getxattr_failed", exc=self.last_exc, key=key, fn=fn) is_uploaded = True # deleter could have deleted if is_uploaded: continue yield (name, key, fn) def next_file_to_upload(self, with_raw): upload_files = list(self.gen_upload_files()) # try to upload qlog files first for name, key, fn in upload_files: if name in self.immediate_priority or any( f in fn for f in self.immediate_folders): return (key, fn) if with_raw: # then upload the full log files, rear and front camera files for name, key, fn in upload_files: if name in self.high_priority: return (key, fn) # then upload other files for name, key, fn in upload_files: if not name.endswith('.lock') and not name.endswith(".tmp"): return (key, fn) return None def do_upload(self, key, fn): try: url_resp = self.api.get("v1.3/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token()) if url_resp.status_code == 412: self.last_resp = url_resp return url_resp_json = json.loads(url_resp.text) url = url_resp_json['url'] headers = url_resp_json['headers'] cloudlog.debug("upload_url v1.3 %s %s", url, str(headers)) if fake_upload: cloudlog.debug("*** WARNING, THIS IS A FAKE UPLOAD TO %s ***" % url) class FakeResponse(): def __init__(self): self.status_code = 200 self.last_resp = FakeResponse() else: with open(fn, "rb") as f: self.last_resp = requests.put(url, data=f, headers=headers, timeout=10) except Exception as e: self.last_exc = (e, traceback.format_exc()) raise def normal_upload(self, key, fn): self.last_resp = None self.last_exc = None try: self.do_upload(key, fn) except Exception: pass return self.last_resp def upload(self, key, fn): try: sz = os.path.getsize(fn) except OSError: cloudlog.exception("upload: getsize failed") return False cloudlog.event("upload", key=key, fn=fn, sz=sz) cloudlog.debug("checking %r with size %r", key, sz) if sz == 0: try: # tag files of 0 size as uploaded setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) except OSError: cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) success = True else: cloudlog.debug("uploading %r", fn) stat = self.normal_upload(key, fn) if stat is not None and stat.status_code in (200, 201, 412): cloudlog.event("upload_success" if stat.status_code != 412 else "upload_ignored", key=key, fn=fn, sz=sz, debug=True) try: # tag file as uploaded setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) except OSError: cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) success = True else: cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz, debug=True) success = False return success
class Uploader(): def __init__(self, dongle_id, root): self.dongle_id = dongle_id self.api = Api(dongle_id) self.root = root self.upload_thread = None self.last_resp = None self.last_exc = None self.immediate_size = 0 self.immediate_count = 0 # stats for last successfully uploaded file self.last_time = 0.0 self.last_speed = 0.0 self.last_filename = "" self.immediate_folders = ["crash/", "boot/"] self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1} def get_upload_sort(self, name): if name in self.immediate_priority: return self.immediate_priority[name] return 1000 def list_upload_files(self): if not os.path.isdir(self.root): return self.immediate_size = 0 self.immediate_count = 0 for logname in listdir_by_creation(self.root): path = os.path.join(self.root, logname) try: names = os.listdir(path) except OSError: continue if any(name.endswith(".lock") for name in names): continue for name in sorted(names, key=self.get_upload_sort): key = os.path.join(logname, name) fn = os.path.join(path, name) # skip files already uploaded try: is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) except OSError: cloudlog.event("uploader_getxattr_failed", exc=self.last_exc, key=key, fn=fn) is_uploaded = True # deleter could have deleted if is_uploaded: continue try: if name in self.immediate_priority: self.immediate_count += 1 self.immediate_size += os.path.getsize(fn) except OSError: pass yield (name, key, fn) def next_file_to_upload(self): upload_files = list(self.list_upload_files()) for name, key, fn in upload_files: if any(f in fn for f in self.immediate_folders): return (name, key, fn) for name, key, fn in upload_files: if name in self.immediate_priority: return (name, key, fn) return None def do_upload(self, key, fn): try: url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token()) if url_resp.status_code == 412: self.last_resp = url_resp return url_resp_json = json.loads(url_resp.text) url = url_resp_json['url'] headers = url_resp_json['headers'] cloudlog.debug("upload_url v1.4 %s %s", url, str(headers)) if fake_upload: cloudlog.debug( f"*** WARNING, THIS IS A FAKE UPLOAD TO {url} ***") class FakeResponse(): def __init__(self): self.status_code = 200 self.last_resp = FakeResponse() else: with open(fn, "rb") as f: if key.endswith('.bz2') and not fn.endswith('.bz2'): data = bz2.compress(f.read()) data = io.BytesIO(data) else: data = f self.last_resp = requests.put(url, data=data, headers=headers, timeout=10) except Exception as e: self.last_exc = (e, traceback.format_exc()) raise def normal_upload(self, key, fn): self.last_resp = None self.last_exc = None try: self.do_upload(key, fn) except Exception: pass return self.last_resp def upload(self, name, key, fn, network_type, metered): try: sz = os.path.getsize(fn) except OSError: cloudlog.exception("upload: getsize failed") return False cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered) if sz == 0: # tag files of 0 size as uploaded success = True elif name in self.immediate_priority and sz > UPLOAD_QLOG_QCAM_MAX_SIZE: cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz) success = True else: start_time = time.monotonic() stat = self.normal_upload(key, fn) if stat is not None and stat.status_code in (200, 201, 401, 403, 412): self.last_filename = fn self.last_time = time.monotonic() - start_time self.last_speed = (sz / 1e6) / self.last_time success = True cloudlog.event("upload_success" if stat.status_code != 412 else "upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered) else: success = False cloudlog.event("upload_failed", stat=stat, exc=self.last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered) if success: # tag file as uploaded try: setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) except OSError: cloudlog.event("uploader_setxattr_failed", exc=self.last_exc, key=key, fn=fn, sz=sz) return success def get_msg(self): msg = messaging.new_message("uploaderState") us = msg.uploaderState us.immediateQueueSize = int(self.immediate_size / 1e6) us.immediateQueueCount = self.immediate_count us.lastTime = self.last_time us.lastSpeed = self.last_speed us.lastFilename = self.last_filename return msg