def decrypt_and_build_filetree(memory, options, secret): """ decrypt_and_build_filetree @type memory: Memory @type options: optparse.Values, instance @type secret: str """ if not secret: raise Exception("decrypt_and_build_filetree: no secret given") datadir = get_data_dir(options) if not os.path.exists(datadir): print "cba_index.py:365", "nothing to decrypt", datadir, "does not exists" return memory output_json({"msg": "preparing decrypt"}) blobdir = os.path.join(datadir, "blobs") localindex = get_localindex(memory) hashes = set() restored_hashes = [] if localindex: for dirhash in localindex["dirnames"]: if "dirname" in localindex["dirnames"][dirhash]: if not os.path.exists( localindex["dirnames"][dirhash]["dirname"]): ensure_directory( localindex["dirnames"][dirhash]["dirname"]) for cfile in localindex["dirnames"][dirhash]["filenames"]: fpath = os.path.join( localindex["dirnames"][dirhash]["dirname"], cfile["name"]) if not os.path.exists(fpath): hashes.add((cfile["hash"], cfile["name"])) else: restored_hashes.append(cfile["hash"]) processed_files = 0 numfiles = len(hashes) for cfile in hashes: processed_files += 1 update_progress(processed_files, numfiles, cfile[1]) #noinspection PyUnusedLocal paths = decrypt_blob_to_filepaths(blobdir, localindex, cfile[0], secret) memory = store_localindex(memory, localindex) cleanup_tempfiles() return memory
def encrypt_new_blobs(secret, new_blobs): """ @type secret: str or unicode @type new_blobs: dict """ processed_files = 0 numfiles = len(new_blobs) for fhash in new_blobs: ensure_directory(new_blobs[fhash]["blobdir"]) update_progress(processed_files, numfiles, "encrypting: " + os.path.basename(new_blobs[fhash]["fpath"])) read_and_encrypt_file(new_blobs[fhash]["fpath"], new_blobs[fhash]["blobpath"], secret) processed_files += 1
def decrypt_and_build_filetree(memory, options, secret): """ decrypt_and_build_filetree @type memory: Memory @type options: optparse.Values, instance @type secret: str """ if not secret: raise Exception("decrypt_and_build_filetree: no secret given") datadir = get_data_dir(options) if not os.path.exists(datadir): print "cba_index.py:365", "nothing to decrypt", datadir, "does not exists" return memory output_json({"msg": "preparing decrypt"}) blobdir = os.path.join(datadir, "blobs") localindex = get_localindex(memory) hashes = set() restored_hashes = [] if localindex: for dirhash in localindex["dirnames"]: if "dirname" in localindex["dirnames"][dirhash]: if not os.path.exists(localindex["dirnames"][dirhash]["dirname"]): ensure_directory(localindex["dirnames"][dirhash]["dirname"]) for cfile in localindex["dirnames"][dirhash]["filenames"]: fpath = os.path.join(localindex["dirnames"][dirhash]["dirname"], cfile["name"]) if not os.path.exists(fpath): hashes.add((cfile["hash"], cfile["name"])) else: restored_hashes.append(cfile["hash"]) processed_files = 0 numfiles = len(hashes) for cfile in hashes: processed_files += 1 update_progress(processed_files, numfiles, cfile[1]) #noinspection PyUnusedLocal paths = decrypt_blob_to_filepaths(blobdir, localindex, cfile[0], secret) memory = store_localindex(memory, localindex) cleanup_tempfiles() return memory
def wait_for_tasks(memory, options, result_message_param=None): """ wait_for_tasks @type memory: Memory @type options: optparse.Values, instance """ initial_num_tasks = -1 while True: session = None if memory.has("session"): session = memory.get("session") result, memory = on_server(memory, options, "crypto_tasks", payload={}, session=session) if result: if len(result) > 1: if result[1]: num_tasks = len([x for x in result[1] if x["m_command_object"] != "StorePassword"]) if initial_num_tasks == -1: initial_num_tasks = num_tasks if not result_message_param: result_message = "waiting for tasks to finish on server" else: result_message = result_message_param update_progress(initial_num_tasks - num_tasks, initial_num_tasks, result_message) if num_tasks == 0: return memory if not result_message: if num_tasks > 3: time.sleep(1) if num_tasks > 6: log_json("waiting for tasks " + str(num_tasks)) else: return memory if not result_message_param: time.sleep(1) else: time.sleep(0.5)
def upload_files(memory, options, serverindex, file_uploads): """ upload_files @type memory: Memory @type options: optparse.Values, instance @type serverindex: dict @type file_uploads: tuple """ path_guid_cache = {} cnt = 0 for uf in file_uploads: try: parent_path = uf["local_path"].replace(options.dir, "") parent_path = os.path.dirname(parent_path) uf["parent_path"] = parent_path if parent_path in path_guid_cache: uf["parent_short_id"] = path_guid_cache[parent_path] else: uf["parent_short_id"], memory = path_to_server_guid(memory, options, serverindex, parent_path) path_guid_cache[parent_path] = uf["parent_short_id"] update_progress(cnt, len(file_uploads), "checking path " + parent_path) cnt += 1 except NoParentFound: uf["parent_short_id"] = uf["parent_path"] = "" files_uploaded = [] file_uploads = sorted(file_uploads, key=lambda k: k["size"]) for uf in file_uploads: update_progress(len(files_uploaded) + 1, len(file_uploads), "uploading " + os.path.basename(uf["local_path"])) if os.path.exists(uf["local_path"]): file_path = upload_file(memory.get("session"), options.server, options.cryptobox, uf["local_path"], path_to_relative_path_unix_style(memory, uf["local_path"]), uf["parent_short_id"]) files_uploaded.append(file_path) output_json({"item_progress": 0}) return memory, tuple(files_uploaded)
def get_unique_content(memory, options, all_unique_nodes, local_paths): """ @type memory: Memory @type options: istance @type all_unique_nodes: dict @type local_paths: tuple """ if len(local_paths) == 0: return memory unique_nodes_hashes = [fhash for fhash in all_unique_nodes if not have_blob(options, fhash)] unique_nodes = [all_unique_nodes[fhash] for fhash in all_unique_nodes if fhash in unique_nodes_hashes] downloaded_files_cnt = 0 unique_nodes = [node for node in unique_nodes if not os.path.exists(os.path.join(options.dir, node["doc"]["m_path_p64s"].lstrip(os.path.sep)))] unique_nodes = sorted(unique_nodes, key=lambda k: k["doc"]["m_size_p64s"]) for node in unique_nodes: update_progress(downloaded_files_cnt, len(unique_nodes), "downloading " + str(node["doc"]["m_name"])) content_path, content_hash = download_blob(memory, options, node) update_item_progress(100) output_json({"item_progress": 0}) memory, file_nodes_left = write_blobs_to_filepaths(memory, options, local_paths, None, content_hash, content_path) downloaded_files_cnt += 1 update_progress(downloaded_files_cnt, len(unique_nodes), "downloading done") for lp in local_paths: memory = add_local_path_history(memory, os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.sep))) source_path = None file_path = os.path.join(options.dir, lp["doc"]["m_path_p64s"].lstrip(os.path.sep)) if not os.path.exists(file_path): for lph in all_unique_nodes: if lph == lp["content_hash_latest_timestamp"][0]: source_path = os.path.join(options.dir, all_unique_nodes[lph]["doc"]["m_path_p64s"].lstrip(os.path.sep)) break datapath = data = None if source_path: if not os.path.exists(source_path): fhash = lp["content_hash_latest_timestamp"][0] source_path = os.path.join(get_blob_dir(options), fhash[:2]) source_path = os.path.join(source_path, fhash[2:]) memory = add_path_history(file_path, memory) secret = password_derivation(options.password, base64.decodestring(memory.get("salt_b64"))) dec_file = decrypt_file(source_path, secret) datapath = dec_file.name if not datapath: st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath = read_file(source_path, True) else: st_ctime, st_atime, st_mtime, st_mode, st_uid, st_gid, st_size, datapath_dummy = read_file(source_path) st_mtime = int(lp["content_hash_latest_timestamp"][1]) write_file(file_path, data, datapath, st_mtime, st_mtime, st_mode, st_uid, st_gid) local_paths_not_written = [fp for fp in local_paths if not os.path.exists(os.path.join(options.dir, fp["doc"]["m_path_p64s"].lstrip(os.path.sep)))] if len(local_paths_not_written) > 0: local_index = get_localindex(memory) local_path_hashes = {} for ldir in local_index["dirnames"]: for f in local_index["dirnames"][ldir]["filenames"]: if "hash" in f: local_path_hashes[f["hash"]] = os.path.join(local_index["dirnames"][ldir]["dirname"], f["name"]) for lfnw in local_paths_not_written: w = False for lfh in local_path_hashes: if not w: if strcmp(lfnw["content_hash_latest_timestamp"][0], lfh): w = True open(os.path.join(options.dir, lfnw["doc"]["m_path_p64s"].lstrip(os.path.sep)), "w").write(open(local_path_hashes[lfh]).read()) return memory
def index_and_encrypt(memory, options): """ index_and_encrypt @type memory: Memory @type options: optparse.Values, instance @rtype salt, secret, memory, localindex: str, str, Memory, dict """ localindex = make_local_index(options) datadir = get_data_dir(options) if quick_lock_check(options): output_json({ "message": "cryptobox is locked, nothing can be added now first decrypt (-d)" }) return None, None, memory, localindex salt = None if memory.has("salt_b64"): salt = base64.decodestring(memory.get("salt_b64")) if not salt: salt = Random.new().read(32) memory.set("salt_b64", base64.encodestring(salt)) output_json({"msg": "preparing encrypt"}) secret = password_derivation(options.password, salt) ensure_directory(datadir) new_blobs = {} file_cnt = 0 new_objects = 0 hash_set_on_disk = set() processed_files = 0 numfiles = 0 for dirhash in localindex["dirnames"]: numfiles += len(localindex["dirnames"][dirhash]["filenames"]) for dirhash in localindex["dirnames"]: for fname in localindex["dirnames"][dirhash]["filenames"]: file_cnt += 1 file_dir = localindex["dirnames"][dirhash]["dirname"] file_path = os.path.join(file_dir, fname["name"]) if os.path.exists(file_path): update_progress(processed_files, numfiles, "indexing " + os.path.basename(file_path)) filedata, localindex = make_cryptogit_hash( file_path, datadir, localindex) fname["hash"] = filedata["filehash"] hash_set_on_disk.add(filedata["filehash"]) if not filedata["blob_exists"]: new_blobs[filedata["filehash"]] = filedata new_objects += 1 if len(new_blobs) > 1500: encrypt_new_blobs(secret, new_blobs) new_blobs = {} if len(new_blobs) > 0: if len(new_blobs) > 0: encrypt_new_blobs(secret, new_blobs) cleanup_tempfiles() memory = store_localindex(memory, localindex) if options.remove: ld = os.listdir(options.dir) ld.remove(".cryptobox") processed_files = 0 numfiles = len(ld) for fname in ld: fpath = os.path.join(options.dir, fname) processed_files += 1 update_progress(processed_files, numfiles, "delete " + os.path.basename(fpath)) if os.path.isdir(fpath): if os.path.exists(fpath): shutil.rmtree(fpath, True) else: if os.path.exists(fpath): os.remove(fpath) obsolute_blob_store_entries = set() blob_dirs = os.path.join(datadir, "blobs") ensure_directory(blob_dirs) for blob_dir in os.listdir(blob_dirs): blob_store = os.path.join(blob_dirs, blob_dir.lstrip(os.path.sep)) if os.path.isdir(blob_store): for blob_file in os.listdir(blob_store): found = False for fhash in hash_set_on_disk: if fhash in (blob_dir + blob_file): found = True if not found: obsolute_blob_store_entries.add(blob_dir + blob_file) for f_hash in obsolute_blob_store_entries: blob_dir = os.path.join(blob_dirs, f_hash[:2]) blob_path = os.path.join(blob_dir, f_hash[2:]) os.remove(blob_path) if os.path.isdir(blob_dir): blob_entries = [ f for f in os.listdir(blob_dir) if not f.startswith('.') ] if len(blob_entries) == 0: shutil.rmtree(blob_dir, True) cleanup_tempfiles() return salt, secret, memory, localindex
def index_and_encrypt(memory, options): """ index_and_encrypt @type memory: Memory @type options: optparse.Values, instance @rtype salt, secret, memory, localindex: str, str, Memory, dict """ localindex = make_local_index(options) datadir = get_data_dir(options) if quick_lock_check(options): output_json({"message": "cryptobox is locked, nothing can be added now first decrypt (-d)"}) return None, None, memory, localindex salt = None if memory.has("salt_b64"): salt = base64.decodestring(memory.get("salt_b64")) if not salt: salt = Random.new().read(32) memory.set("salt_b64", base64.encodestring(salt)) output_json({"msg": "preparing encrypt"}) secret = password_derivation(options.password, salt) ensure_directory(datadir) new_blobs = {} file_cnt = 0 new_objects = 0 hash_set_on_disk = set() processed_files = 0 numfiles = 0 for dirhash in localindex["dirnames"]: numfiles += len(localindex["dirnames"][dirhash]["filenames"]) for dirhash in localindex["dirnames"]: for fname in localindex["dirnames"][dirhash]["filenames"]: file_cnt += 1 file_dir = localindex["dirnames"][dirhash]["dirname"] file_path = os.path.join(file_dir, fname["name"]) if os.path.exists(file_path): update_progress(processed_files, numfiles, "indexing " + os.path.basename(file_path)) filedata, localindex = make_cryptogit_hash(file_path, datadir, localindex) fname["hash"] = filedata["filehash"] hash_set_on_disk.add(filedata["filehash"]) if not filedata["blob_exists"]: new_blobs[filedata["filehash"]] = filedata new_objects += 1 if len(new_blobs) > 1500: encrypt_new_blobs(secret, new_blobs) new_blobs = {} if len(new_blobs) > 0: if len(new_blobs) > 0: encrypt_new_blobs(secret, new_blobs) cleanup_tempfiles() memory = store_localindex(memory, localindex) if options.remove: ld = os.listdir(options.dir) ld.remove(".cryptobox") processed_files = 0 numfiles = len(ld) for fname in ld: fpath = os.path.join(options.dir, fname) processed_files += 1 update_progress(processed_files, numfiles, "delete " + os.path.basename(fpath)) if os.path.isdir(fpath): if os.path.exists(fpath): shutil.rmtree(fpath, True) else: if os.path.exists(fpath): os.remove(fpath) obsolute_blob_store_entries = set() blob_dirs = os.path.join(datadir, "blobs") ensure_directory(blob_dirs) for blob_dir in os.listdir(blob_dirs): blob_store = os.path.join(blob_dirs, blob_dir.lstrip(os.path.sep)) if os.path.isdir(blob_store): for blob_file in os.listdir(blob_store): found = False for fhash in hash_set_on_disk: if fhash in (blob_dir + blob_file): found = True if not found: obsolute_blob_store_entries.add(blob_dir + blob_file) for f_hash in obsolute_blob_store_entries: blob_dir = os.path.join(blob_dirs, f_hash[:2]) blob_path = os.path.join(blob_dir, f_hash[2:]) os.remove(blob_path) if os.path.isdir(blob_dir): blob_entries = [f for f in os.listdir(blob_dir) if not f.startswith('.')] if len(blob_entries) == 0: shutil.rmtree(blob_dir, True) cleanup_tempfiles() return salt, secret, memory, localindex