def final_status(self, upload, temp, final=False): end_time = time.time() start_time = self.st total_uploaded_b = 0 total_downloaded_b = 0 total_copied_b = 0 total_uploaded_t = 0 total_downloaded_t = 0 total_copied_t = 0 for item in self.up_complete: total_uploaded_t += item["timespan"] total_uploaded_b += item["size_uploaded"] for item in self.down_complete: total_downloaded_t += item["timespan"] total_downloaded_b += item["size_downloaded"] for item in self.copy_complete: total_copied_t += item["timespan"] total_copied_b += item["size_copied"] downspeed = utils.get_speed(total_downloaded_b, total_downloaded_t) copyspeed = utils.get_speed(total_copied_b, total_copied_t) upspeed = utils.get_speed(total_uploaded_b, total_uploaded_t) runspeed = utils.get_speed(total_downloaded_b, (end_time - start_time)) uprunspeed = utils.get_speed(total_uploaded_b, (end_time - start_time)) copyrunspeed = utils.get_speed(total_copied_b, (end_time - start_time)) if final or total_downloaded_t > 0: log.info("Total download time: %s", utils.convert_time(total_downloaded_t)) if upload and (final or total_uploaded_t): log.info("Total upload time: %s", utils.convert_time(total_uploaded_t)) elif temp and (final or total_copied_t): log.info("Total copy time: %s", utils.convert_time(total_copied_t)) log.info("Total run time: %s", utils.convert_time(end_time - start_time)) if final or total_downloaded_b: log.info("Download speed: %s at %s", utils.convert_size(total_downloaded_b), downspeed) if upload and (final or total_uploaded_b): log.info("Upload speed: %s at %s", utils.convert_size(total_uploaded_b), upspeed) elif temp and (final or total_copied_b): log.info("Copy speed: %s at %s", utils.convert_size(total_copied_b), copyspeed) if final or total_downloaded_b: log.info("Overall download speed: %s at %s", utils.convert_size(total_downloaded_b), runspeed) if upload and (final or total_uploaded_b): log.info("Overall upload speed: %s at %s", utils.convert_size(total_uploaded_b), uprunspeed) elif temp and (final or total_copied_b): log.info("Overall copy speed: %s at %s", utils.convert_size(total_copied_b), copyrunspeed)
def download(status, should_exit, session, results, command_args): log.debug("Starting up") while not should_exit.is_set(): item = None try: item = status.down() except EmptyException: if status.queuers_active > 0 or status.queues > 0: continue else: log.debug("Nothing more to download. Shutting down") return if item is None: continue random.seed(item["filepath"]) filename = item["filename"] size_bytes = item["filesize"] size_str = utils.convert_size(size_bytes) temp_dir = item["filedir"] temp_file = item["fullpath"] log.info("%s size %s", filename, size_str) timespan = 0 if command_args.temp: temp_dir = command_args.temp temp_file = os.path.join(temp_dir, filename) apidownloaduri = "%smyfile.ext?access_token=%s&path=%s" % (BASE_URL, command_args.token, item["filepath"]) if (not command_args.upload and not os.path.exists(item["filedir"])) or (command_args.temp and not os.path.exists(temp_dir)): cleanUpAfterError("Missing temp or destination parent directory", item, results) continue if command_args.temp: filehash = sha1("blob " + str(size_bytes) + "\0" + item["filepath"]) tmpname = filehash.hexdigest() temp_file = os.path.join(command_args.temp, tmpname) item["temppath"] = temp_file log.debug("Downloading file to %s", temp_file) retriesleft = 10 apiratecount = 1 down_failed = True while retriesleft > 0 and not should_exit.is_set(): sleeptime = random.randint(10, 70) if apiratecount > 5: apiratecount = 5 try: mode = "wb" filecomplete = False seek = 0 sizecopied = 0 try: seek = os.path.getsize(temp_file) if seek > size_bytes: seek = 0 elif seek == size_bytes: log.info("Found temp. Nothing to download") filecomplete = True elif seek > 0: sizecopied += seek log.info("continuing download") mode = "ab" except: pass if not filecomplete: req = session.get(apidownloaduri, stream=True, timeout=120, headers={'Range':"bytes=%s-" % seek}) req.raise_for_status() with open(temp_file, mode) as tmpfile: st = time.time() cr = st progress_time = st + 60 timespan = 0 chunk_size = 1024 * 1024 for chunk in req.iter_content(chunk_size=chunk_size): cr = time.time() if should_exit.is_set() or not chunk: break tmpfile.write(chunk) sizecopied += len(chunk) if command_args.progress and progress_time < cr: progress_time = cr + 60 speed = utils.get_speed(sizecopied-seek, (cr-st)) sizecopied_str = utils.convert_size(sizecopied) time_left = utils.get_remaining_time(sizecopied-seek, size_bytes-sizecopied, (cr-st)) log.info(filename) log.info("Downloaded %s of %s at %s. %s left", sizecopied_str, size_str, speed, time_left) if should_exit.is_set(): log.info("Stopping download") elif sizecopied != size_bytes: raise SizeMismatchError("Download size mismatch downloaded %s expected %s" % (sizecopied, size_bytes)) else: timespan = (cr-st) except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError, requests.exceptions.RequestException): retriesleft -= 1 if req.status_code == 429: apiratecount += 1 retriesleft += 1 log.warn("API rate limit reached. Will retry") else: log.exception("Network error %s. Will retry %s more times", req.status_code, retriesleft) if retriesleft > 0: time.sleep(10 * apiratecount) else: log.error("Error downloading %s", filename) cleanUpAfterError("Maximum retries reached", item, results) except SizeMismatchError: retriesleft -= 1 log.exception("%s File size mismatch. Will retry %s more times", filename, retriesleft) if retriesleft > 0: time.sleep(sleeptime) else: cleanUpAfterError("Error downloading %s Maximum retries reached" % filename, item, results) except IOError as e: if e.errno == errno.ENOSPC: if status.up_active > 0 or status.copy_active > 0: log.warn("Out of space. Waiting %s secs", sleeptime) time.sleep(sleeptime) else: log.critical("No space left on target or temp disk. Exiting") should_exit.set() else: retriesleft -= 1 if retriesleft > 0: log.exception("Error downloading %s. Will retry %s more times", filename, retriesleft) time.sleep(sleeptime) else: cleanUpAfterError("An unknown error occurred", item, results) except: retriesleft -= 1 if retriesleft > 0: log.exception("Error downloading %s. Will retry %s more times", filename, retriesleft) time.sleep(sleeptime) else: cleanUpAfterError("An unknown error occurred", item, results) else: if not should_exit.is_set(): down_failed = False retriesleft = 0 if timespan <= 0: timespan = 1 status.down({ "timespan": timespan, "size_downloaded": sizecopied, "temppath": temp_file }) if command_args.upload: log.info("Finished download %s in %s", filename, utils.convert_time(timespan)) log.debug("Adding to upload queue") status.queue_up(item) elif command_args.temp: log.info("Finished download %s in %s", filename, utils.convert_time(timespan)) log.debug("Adding to move queue") status.queue_copy(item) else: log.info("Finished download %s in %s", item["fullpath"], utils.convert_time(timespan)) results.writeSuccess(item["fullpath"], item["filepath"]) if command_args.progress: speed = utils.get_speed(sizecopied-seek, timespan) log.info("%s downloaded at %s", size_str, speed) else: cleanUpAfterError("Download stopped", item, results) if down_failed: status.down_fail(item) log.debug("End of thread") log.debug("Shutting down")
def upload(status, should_exit, results, args): log.debug("Starting up") g = GoogleDrive() while not should_exit.is_set(): apiratecount = 1 try: item = status.up() except EmptyException: if status.queuers_active or status.down_active: continue else: log.debug("Nothing left to upload. Shutting down") if item is None: continue filename = item["filename"] size_bytes = item["filesize"] size_str = utils.convert_size(size_bytes) temp_file = item["temppath"] parent_id = item["filedir"] log.info("Uploading %s %s", filename, size_str) retriesleft = 10 up_failed=True while retriesleft > 0 and not should_exit.is_set(): if apiratecount > 5: apiratecount = 5 try: g.get_service() needtoupload = g.need_to_upload(filename, parent_id, size_bytes) if needtoupload: st = time.time() timespan = 0 log.debug("Uploading file %s to parent %s", filename, parent_id) result = g.upload_file(temp_file, filename, parent=parent_id) if not result: raise UploadError("Upload failed") timespan = (time.time()-st) except HttpError as e: retriesleft -= 1 if e.resp.status == 403: apiratecount += 1 retriesleft += 1 log.warn("Google API rate limit reached. Will retry") else: log.exception("Error uploading file will retry %s more times", retriesleft) if retriesleft > 0: time.sleep(10 * apiratecount) else: results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename) except: retriesleft -= 1 if retriesleft > 0: log.exception("Error uploading file will retry %s more times", retriesleft) time.sleep(10 * apiratecount) else: results.writeError(item["filename"], item["fullpath"], item["filepath"], "Error %s could not be uploaded" % filename) else: retriesleft = 0 up_failed = False if not args.local: try: os.remove(temp_file) except: log.exception("Failed cleaning up temp file %s", temp_file) status.up({ "timespan": timespan, "size_uploaded": size_bytes, "temppath": temp_file }) if args.progress: speed = utils.get_speed(size_bytes, timespan) log.info("%s uploaded at %s", size_str, speed) log.info("Finished uploading %s", filename) results.writeSuccess(item["fullpath"], result["id"]) if up_failed: status.up_fail(item) log.debug("End of thread") log.debug("Shutting down")
def copy(status, should_exit, results, args): log.debug("Starting up") while not should_exit.is_set(): try: item = status.copy() except EmptyException: if status.queuers_active or status.down_active: continue else: log.debug("Nothing left to move. Shutting down") if item is None: continue filename = item["filename"] size_bytes = item["filesize"] size_str = utils.convert_size(size_bytes) tmppath = item["temppath"] destpath = item["fullpath"] timespan = 0 log.info("Moving %s %s", filename, size_str) retriesleft = 3 sizecopied = 0 seek = 0 copy_failed = True while retriesleft > 0 and not should_exit.is_set(): try: mode = "wb" filecomplete = False try: seek = os.path.getsize(destpath) tmpsize = os.path.getsize(tmppath) if seek > tmpsize: seek = 0 elif seek == tmpsize: filecomplete = True elif seek > 0: mode = "ab" except: pass if not filecomplete: timespan = 0 st = time.time() progress = st + 60 cr = st with open(tmppath, 'rb') as f, open(destpath, mode) as fo: if seek: f.seek(seek) while not should_exit.is_set(): piece = f.read(1024) cr = time.time() if args.progress and progress < cr: progress = cr + 60 speed = utils.get_speed(sizecopied-seek, (cr-st)) log.info("%s Moved %s of %s at %s", filename, utils.convert_size(sizecopied), size_str, speed) if piece: fo.write(piece) sizecopied += len(piece) else: break timespan = (cr-st) except IOError as e: if e.errno == errno.ENOSPC: log.critical("No space left on target disk. Exiting") should_exit.set() else: retriesleft -= 1 if retriesleft > 0: log.exception("Error downloading %s. Will retry %s more times", filename, retriesleft) time.sleep(10) else: log.exception("Error file could not be moved to %s", destpath) results.writeError(item["filename"], item["fullpath"], item["filepath"], "Move failed") except: retriesleft -= 1 if retriesleft > 0: log.exception("Error moving file will retry %s more times", retriesleft) time.sleep(10) else: log.exception("Error file could not be moved to %s", destpath) results.writeError(item["filename"], item["fullpath"], item["filepath"], "Move failed") else: if not should_exit.is_set(): copy_failed = False retriesleft = 0 if args.progress: speed = utils.get_speed(sizecopied-seek, timespan) log.info("%s %s moved at %s", filename, size_str, speed) try: os.remove(tmppath) except: log.exception("Failed cleaning up temp file %s", tmppath) results.writeSuccess(destpath, item["filepath"]) status.copy({ "timespan": timespan, "size_copied": sizecopied, "temppath": tmppath, "destpath": destpath }) log.info("Finished moving %s", filename) else: results.writeError(item["filename"], item["fullpath"], item["filepath"], "Move stopped") if copy_failed: status.copy_fail(item) log.debug("End of thread") log.info("Shutting down")