def download_server(memory, options, url, output_name_item_percentage=None): """ download_server @type memory: Memory, None @type options: optparse.Values, instance @type url: str, unicode @type output_name_item_percentage: str """ if memory: url = os.path.normpath(url) url = options.server + options.cryptobox + "/" + url session = memory.get("session") result = session.get(url, timeout=3600, stream=True, verify=False) else: result = requests.get(url, timeout=3600, stream=True, verify=False) if result.status_code == 403: raise ServerForbidden(result.reason) if result.status_code == 500: raise ServerError(result.reason) tf_download = get_named_temporary_file(auto_delete=False) if "Content-Length" in result.headers: size = int(result.headers['Content-Length'].strip()) downloaded_bytes = 0 prev_percenage = -1 last_update = time.time() for buf in result.iter_content(1024): if buf: tf_download.write(buf) downloaded_bytes += len(buf) divider = float(size) / 100 if divider > 0: percentage = int(float(downloaded_bytes) / divider) if percentage != prev_percenage: if time.time() - last_update > 0.5: if output_name_item_percentage: update_item_progress( percentage, output_name_item_percentage) else: update_item_progress(percentage) last_update = time.time() prev_percenage = percentage return tf_download else: tf_download.write(result.content) return tf_download.name
def download_server(memory, options, url, output_name_item_percentage=None): """ download_server @type memory: Memory, None @type options: optparse.Values, instance @type url: str, unicode @type output_name_item_percentage: str """ if memory: url = os.path.normpath(url) url = options.server + options.cryptobox + "/" + url session = memory.get("session") result = session.get(url, timeout=3600, stream=True, verify=False) else: result = requests.get(url, timeout=3600, stream=True, verify=False) if result.status_code == 403: raise ServerForbidden(result.reason) if result.status_code == 500: raise ServerError(result.reason) tf_download = get_named_temporary_file(auto_delete=False) if "Content-Length" in result.headers: size = int(result.headers["Content-Length"].strip()) downloaded_bytes = 0 prev_percenage = -1 last_update = time.time() for buf in result.iter_content(1024): if buf: tf_download.write(buf) downloaded_bytes += len(buf) divider = float(size) / 100 if divider > 0: percentage = int(float(downloaded_bytes) / divider) if percentage != prev_percenage: if time.time() - last_update > 0.5: if output_name_item_percentage: update_item_progress(percentage, output_name_item_percentage) else: update_item_progress(percentage) last_update = time.time() prev_percenage = percentage return tf_download else: tf_download.write(result.content) return tf_download.name
def prog_callback(param, current, total): """ @type param: @type current: @type total: prog_callback """ try: if param: if time.time() - param.last_cb_time > 0.5: param.last_cb_time = time.time() percentage = 100 - ((total - current) * 100) / total if percentage != last_progress[0]: last_progress[0] = percentage update_item_progress(percentage) except Exception, exc: print "cba_sync.py:1078", "updating upload progress failed", str(exc)
def get_unique_content(memory, options, all_unique_nodes, local_paths): """ @type memory: Memory @type options: istance @type all_unique_nodes: dict @type local_paths: tuple """ if len(local_paths) == 0: return memory unique_nodes_hashes = [fhash for fhash in all_unique_nodes if not have_blob(options, fhash)] unique_nodes = [all_unique_nodes[fhash] for fhash in all_unique_nodes if fhash in unique_nodes_hashes] downloaded_files_cnt = 0 unique_nodes = [node for node in unique_nodes if not os.path.exists(os.path.join(options.dir, node["doc"]["m_path_p64s"].lstrip(os.path.sep)))] unique_nodes = sorted(unique_nodes, key=lambda k: k["doc"]["m_size_p64s"]) for node in unique_nodes: update_progress(downloaded_files_cnt, len(unique_nodes), "downloading " + str(node["doc"]["m_name"])) content_path, content_hash = download_blob(memory, options, node) update_item_progress(100) output_json({"item_progress": 0}) memory, file_nodes_left = write_blobs_to_filepaths(memory, options, local_paths, None, content_hash, content_path) downloaded_files_cnt += 1 update_progress(downloaded_files_cnt, len(unique_nodes), "downloading done") for lp in local_paths: memory = add_local_path_history(memory, os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.sep))) source_path = None file_path = os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.path.sep)) if not os.path.exists(file_path): for lph in all_unique_nodes: if lph == lp["content_hash_latest_timestamp"][0]: source_path = os.path.join(options.dir, all_unique_nodes[lph]["doc"]["m_path_p64s"].lstrip(os.path.sep)) break datapath = data = None if source_path: if not os.path.exists(source_path): fhash = lp["content_hash_latest_timestamp"][0] source_path = os.path.join(get_blob_dir(options), fhash[:2]) source_path = os.path.join(source_path, fhash[2:]) memory = add_path_history(file_path, memory) secret = password_derivation(options.password, base64.decodestring(memory.get("salt_b64"))) dec_file = decrypt_file(source_path, secret) datapath = dec_file.name if not datapath: st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath = read_file(source_path, True) else: st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath_dummy = read_file(source_path) st_mtime = int(lp["content_hash_latest_timestamp"][1]) write_file(file_path, data, datapath, st_mtime, st_mtime, st_mode, st_uid, st_gid) local_paths_not_written = [fp for fp in local_paths if not os.path.exists(os.path.join(options.dir, fp["doc"]["m_path_p64s"].lstrip(os.path.sep)))] if len(local_paths_not_written) > 0: local_index = get_localindex(memory) local_path_hashes = {} for ldir in local_index["dirnames"]: for f in local_index["dirnames"][ldir]["filenames"]: if "hash" in f: local_path_hashes[f["hash"]] = os.path.join(local_index["dirnames"][ldir]["dirname"], f["name"]) for lfnw in local_paths_not_written: w = False for lfh in local_path_hashes: if not w: if strcmp(lfnw["content_hash_latest_timestamp"][0], lfh): w = True open(os.path.join(options.dir, lfnw["doc"]["m_path_p64s"].lstrip(os.path.sep)), "w").write(open(local_path_hashes[lfh]).read()) return memory