def download_server(memory, options, url, output_name_item_percentage=None):
"""
download_server
@type memory: Memory, None
@type options: optparse.Values, instance
@type url: str, unicode
@type output_name_item_percentage: str
"""
if memory:
url = os.path.normpath(url)
url = options.server + options.cryptobox + "/" + url
session = memory.get("session")
result = session.get(url, timeout=3600, stream=True, verify=False)
else:
result = requests.get(url, timeout=3600, stream=True, verify=False)
if result.status_code == 403:
raise ServerForbidden(result.reason)
if result.status_code == 500:
raise ServerError(result.reason)
tf_download = get_named_temporary_file(auto_delete=False)
if "Content-Length" in result.headers:
size = int(result.headers['Content-Length'].strip())
downloaded_bytes = 0
prev_percenage = -1
last_update = time.time()
for buf in result.iter_content(1024):
if buf:
tf_download.write(buf)
downloaded_bytes += len(buf)
divider = float(size) / 100
if divider > 0:
percentage = int(float(downloaded_bytes) / divider)
if percentage != prev_percenage:
if time.time() - last_update > 0.5:
if output_name_item_percentage:
update_item_progress(
percentage, output_name_item_percentage)
else:
update_item_progress(percentage)
last_update = time.time()
prev_percenage = percentage
return tf_download
else:
tf_download.write(result.content)
return tf_download.name
def download_server(memory, options, url, output_name_item_percentage=None):
"""
download_server
@type memory: Memory, None
@type options: optparse.Values, instance
@type url: str, unicode
@type output_name_item_percentage: str
"""
if memory:
url = os.path.normpath(url)
url = options.server + options.cryptobox + "/" + url
session = memory.get("session")
result = session.get(url, timeout=3600, stream=True, verify=False)
else:
result = requests.get(url, timeout=3600, stream=True, verify=False)
if result.status_code == 403:
raise ServerForbidden(result.reason)
if result.status_code == 500:
raise ServerError(result.reason)
tf_download = get_named_temporary_file(auto_delete=False)
if "Content-Length" in result.headers:
size = int(result.headers["Content-Length"].strip())
downloaded_bytes = 0
prev_percenage = -1
last_update = time.time()
for buf in result.iter_content(1024):
if buf:
tf_download.write(buf)
downloaded_bytes += len(buf)
divider = float(size) / 100
if divider > 0:
percentage = int(float(downloaded_bytes) / divider)
if percentage != prev_percenage:
if time.time() - last_update > 0.5:
if output_name_item_percentage:
update_item_progress(percentage, output_name_item_percentage)
else:
update_item_progress(percentage)
last_update = time.time()
prev_percenage = percentage
return tf_download
else:
tf_download.write(result.content)
return tf_download.name
def prog_callback(param, current, total):
"""
@type param:
@type current:
@type total:
prog_callback
"""
try:
if param:
if time.time() - param.last_cb_time > 0.5:
param.last_cb_time = time.time()
percentage = 100 - ((total - current) * 100) / total
if percentage != last_progress[0]:
last_progress[0] = percentage
update_item_progress(percentage)
except Exception, exc:
print "cba_sync.py:1078", "updating upload progress failed", str(exc)
def get_unique_content(memory, options, all_unique_nodes, local_paths):
"""
@type memory: Memory
@type options: istance
@type all_unique_nodes: dict
@type local_paths: tuple
"""
if len(local_paths) == 0:
return memory
unique_nodes_hashes = [fhash for fhash in all_unique_nodes if not have_blob(options, fhash)]
unique_nodes = [all_unique_nodes[fhash] for fhash in all_unique_nodes if fhash in unique_nodes_hashes]
downloaded_files_cnt = 0
unique_nodes = [node for node in unique_nodes if not os.path.exists(os.path.join(options.dir, node["doc"]["m_path_p64s"].lstrip(os.path.sep)))]
unique_nodes = sorted(unique_nodes, key=lambda k: k["doc"]["m_size_p64s"])
for node in unique_nodes:
update_progress(downloaded_files_cnt, len(unique_nodes), "downloading " + str(node["doc"]["m_name"]))
content_path, content_hash = download_blob(memory, options, node)
update_item_progress(100)
output_json({"item_progress": 0})
memory, file_nodes_left = write_blobs_to_filepaths(memory, options, local_paths, None, content_hash, content_path)
downloaded_files_cnt += 1
update_progress(downloaded_files_cnt, len(unique_nodes), "downloading done")
for lp in local_paths:
memory = add_local_path_history(memory, os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.sep)))
source_path = None
file_path = os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.path.sep))
if not os.path.exists(file_path):
for lph in all_unique_nodes:
if lph == lp["content_hash_latest_timestamp"][0]:
source_path = os.path.join(options.dir, all_unique_nodes[lph]["doc"]["m_path_p64s"].lstrip(os.path.sep))
break
datapath = data = None
if source_path:
if not os.path.exists(source_path):
fhash = lp["content_hash_latest_timestamp"][0]
source_path = os.path.join(get_blob_dir(options), fhash[:2])
source_path = os.path.join(source_path, fhash[2:])
memory = add_path_history(file_path, memory)
secret = password_derivation(options.password, base64.decodestring(memory.get("salt_b64")))
dec_file = decrypt_file(source_path, secret)
datapath = dec_file.name
if not datapath:
st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath = read_file(source_path, True)
else:
st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath_dummy = read_file(source_path)
st_mtime = int(lp["content_hash_latest_timestamp"][1])
write_file(file_path, data, datapath, st_mtime, st_mtime, st_mode, st_uid, st_gid)
local_paths_not_written = [fp for fp in local_paths if not os.path.exists(os.path.join(options.dir, fp["doc"]["m_path_p64s"].lstrip(os.path.sep)))]
if len(local_paths_not_written) > 0:
local_index = get_localindex(memory)
local_path_hashes = {}
for ldir in local_index["dirnames"]:
for f in local_index["dirnames"][ldir]["filenames"]:
if "hash" in f:
local_path_hashes[f["hash"]] = os.path.join(local_index["dirnames"][ldir]["dirname"], f["name"])
for lfnw in local_paths_not_written:
w = False
for lfh in local_path_hashes:
if not w:
if strcmp(lfnw["content_hash_latest_timestamp"][0], lfh):
w = True
open(os.path.join(options.dir, lfnw["doc"]["m_path_p64s"].lstrip(os.path.sep)), "w").write(open(local_path_hashes[lfh]).read())
return memory
