def test_mutation_history(self): """ test_sync_delete_local_folder """ self.reset_cb_dir() self.reset_cb_db_clean() ensure_directory(self.cboptions.dir) localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.mkdir("testdata/test/foo") localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.system("rm -Rf testdata/test/foo") localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) self.assertEqual(self.directories_synced(), True) self.all_changes_asserted_zero() os.mkdir("testdata/test/foo") self.assertEqual(os.path.exists("testdata/test/foo"), True) dir_del_local, dir_del_server, dir_make_local, dir_make_server, file_del_local, file_del_server, file_downloads, file_uploads, rename_server, folder_rename_server, rename_local_folders = self.get_sync_changes() self.assertEqual(len(dir_make_server), 1) localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.mkdir("testdata/test/foo2") os.system("ls > testdata/test/foo2/test.txt") localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.system("rm -Rf testdata/test/foo2/test.txt") dir_del_local, dir_del_server, dir_make_local, dir_make_server, file_del_local, file_del_server, file_downloads, file_uploads, rename_server, folder_rename_server, rename_local_folders = self.get_sync_changes() self.assertEqual(len(file_del_server), 1) localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.system("ls > testdata/test/foo2/test.txt") dir_del_local, dir_del_server, dir_make_local, dir_make_server, file_del_local, file_del_server, file_downloads, file_uploads, rename_server, folder_rename_server, rename_local_folders = self.get_sync_changes() self.assertEqual(len(file_uploads), 1)
def reset_cb_dir(self): """ complete_reset """ os.system("rm -Rf testdata/test") ensure_directory(self.cboptions.dir) ensure_directory(get_data_dir(self.cboptions))
def unzip_testfiles_clean(self): """ unzip_testfiles_clean """ ensure_directory(self.cboptions.dir) ensure_directory(get_data_dir(self.cboptions)) os.system("cd testdata; cp testmap_clean.zip test.zip") os.system("cd testdata; unzip -o test.zip > /dev/null") os.system("rm testdata/test.zip")
def unzip_testfiles_configonly(self): """ unzip_testfiles_configonly """ ensure_directory(self.cboptions.dir) ensure_directory(get_data_dir(self.cboptions)) os.system("cd testdata; cp testmap_config.zip test.zip") os.system("cd testdata; unzip -o test.zip > /dev/null") os.system("rm testdata/test.zip") self.cbmemory.load(get_data_dir(self.cboptions))
def test_sync_changed_file(self): """ test_sync_changed_file """ self.reset_cb_db_clean() ensure_directory(self.cboptions.dir) os.system("echo 'hello' > testdata/test/hello.txt") localindex, self.cbmemory = sync_server(self.cbmemory, self.cboptions) os.system("echo 'hello world' > testdata/test/hello.txt") dir_del_local, dir_del_server, dir_make_local, dir_make_server, file_del_local, file_del_server, file_downloads, file_uploads, rename_server, folder_rename_server, rename_local_folders = self.get_sync_changes() self.assertEqual(len(file_uploads), 1)
def decrypt_and_build_filetree(memory, options, secret): """ decrypt_and_build_filetree @type memory: Memory @type options: optparse.Values, instance @type secret: str """ if not secret: raise Exception("decrypt_and_build_filetree: no secret given") datadir = get_data_dir(options) if not os.path.exists(datadir): print "cba_index.py:365", "nothing to decrypt", datadir, "does not exists" return memory output_json({"msg": "preparing decrypt"}) blobdir = os.path.join(datadir, "blobs") localindex = get_localindex(memory) hashes = set() restored_hashes = [] if localindex: for dirhash in localindex["dirnames"]: if "dirname" in localindex["dirnames"][dirhash]: if not os.path.exists( localindex["dirnames"][dirhash]["dirname"]): ensure_directory( localindex["dirnames"][dirhash]["dirname"]) for cfile in localindex["dirnames"][dirhash]["filenames"]: fpath = os.path.join( localindex["dirnames"][dirhash]["dirname"], cfile["name"]) if not os.path.exists(fpath): hashes.add((cfile["hash"], cfile["name"])) else: restored_hashes.append(cfile["hash"]) processed_files = 0 numfiles = len(hashes) for cfile in hashes: processed_files += 1 update_progress(processed_files, numfiles, cfile[1]) #noinspection PyUnusedLocal paths = decrypt_blob_to_filepaths(blobdir, localindex, cfile[0], secret) memory = store_localindex(memory, localindex) cleanup_tempfiles() return memory
def encrypt_new_blobs(secret, new_blobs): """ @type secret: str or unicode @type new_blobs: dict """ processed_files = 0 numfiles = len(new_blobs) for fhash in new_blobs: ensure_directory(new_blobs[fhash]["blobdir"]) update_progress(processed_files, numfiles, "encrypting: " + os.path.basename(new_blobs[fhash]["fpath"])) read_and_encrypt_file(new_blobs[fhash]["fpath"], new_blobs[fhash]["blobpath"], secret) processed_files += 1
def decrypt_and_build_filetree(memory, options, secret): """ decrypt_and_build_filetree @type memory: Memory @type options: optparse.Values, instance @type secret: str """ if not secret: raise Exception("decrypt_and_build_filetree: no secret given") datadir = get_data_dir(options) if not os.path.exists(datadir): print "cba_index.py:365", "nothing to decrypt", datadir, "does not exists" return memory output_json({"msg": "preparing decrypt"}) blobdir = os.path.join(datadir, "blobs") localindex = get_localindex(memory) hashes = set() restored_hashes = [] if localindex: for dirhash in localindex["dirnames"]: if "dirname" in localindex["dirnames"][dirhash]: if not os.path.exists(localindex["dirnames"][dirhash]["dirname"]): ensure_directory(localindex["dirnames"][dirhash]["dirname"]) for cfile in localindex["dirnames"][dirhash]["filenames"]: fpath = os.path.join(localindex["dirnames"][dirhash]["dirname"], cfile["name"]) if not os.path.exists(fpath): hashes.add((cfile["hash"], cfile["name"])) else: restored_hashes.append(cfile["hash"]) processed_files = 0 numfiles = len(hashes) for cfile in hashes: processed_files += 1 update_progress(processed_files, numfiles, cfile[1]) #noinspection PyUnusedLocal paths = decrypt_blob_to_filepaths(blobdir, localindex, cfile[0], secret) memory = store_localindex(memory, localindex) cleanup_tempfiles() return memory
def make_directories_local(memory, options, localindex, folders): """ @type memory: Memory @type options: optparse.Values, instance @type localindex: dict @type folders: tuple """ for f in folders: ensure_directory(f["name"]) memory = add_local_path_history(memory, f["name"]) memory = add_server_path_history(memory, f["relname"]) arg = {"DIR": options.dir, "folders": {"dirnames": {}}, "numfiles": 0} index_files_visit(arg, f["name"], []) for k in arg["folders"]["dirnames"]: localindex["dirnames"][k] = arg["folders"]["dirnames"][k] return memory
def index_and_encrypt(memory, options): """ index_and_encrypt @type memory: Memory @type options: optparse.Values, instance @rtype salt, secret, memory, localindex: str, str, Memory, dict """ localindex = make_local_index(options) datadir = get_data_dir(options) if quick_lock_check(options): output_json({ "message": "cryptobox is locked, nothing can be added now first decrypt (-d)" }) return None, None, memory, localindex salt = None if memory.has("salt_b64"): salt = base64.decodestring(memory.get("salt_b64")) if not salt: salt = Random.new().read(32) memory.set("salt_b64", base64.encodestring(salt)) output_json({"msg": "preparing encrypt"}) secret = password_derivation(options.password, salt) ensure_directory(datadir) new_blobs = {} file_cnt = 0 new_objects = 0 hash_set_on_disk = set() processed_files = 0 numfiles = 0 for dirhash in localindex["dirnames"]: numfiles += len(localindex["dirnames"][dirhash]["filenames"]) for dirhash in localindex["dirnames"]: for fname in localindex["dirnames"][dirhash]["filenames"]: file_cnt += 1 file_dir = localindex["dirnames"][dirhash]["dirname"] file_path = os.path.join(file_dir, fname["name"]) if os.path.exists(file_path): update_progress(processed_files, numfiles, "indexing " + os.path.basename(file_path)) filedata, localindex = make_cryptogit_hash( file_path, datadir, localindex) fname["hash"] = filedata["filehash"] hash_set_on_disk.add(filedata["filehash"]) if not filedata["blob_exists"]: new_blobs[filedata["filehash"]] = filedata new_objects += 1 if len(new_blobs) > 1500: encrypt_new_blobs(secret, new_blobs) new_blobs = {} if len(new_blobs) > 0: if len(new_blobs) > 0: encrypt_new_blobs(secret, new_blobs) cleanup_tempfiles() memory = store_localindex(memory, localindex) if options.remove: ld = os.listdir(options.dir) ld.remove(".cryptobox") processed_files = 0 numfiles = len(ld) for fname in ld: fpath = os.path.join(options.dir, fname) processed_files += 1 update_progress(processed_files, numfiles, "delete " + os.path.basename(fpath)) if os.path.isdir(fpath): if os.path.exists(fpath): shutil.rmtree(fpath, True) else: if os.path.exists(fpath): os.remove(fpath) obsolute_blob_store_entries = set() blob_dirs = os.path.join(datadir, "blobs") ensure_directory(blob_dirs) for blob_dir in os.listdir(blob_dirs): blob_store = os.path.join(blob_dirs, blob_dir.lstrip(os.path.sep)) if os.path.isdir(blob_store): for blob_file in os.listdir(blob_store): found = False for fhash in hash_set_on_disk: if fhash in (blob_dir + blob_file): found = True if not found: obsolute_blob_store_entries.add(blob_dir + blob_file) for f_hash in obsolute_blob_store_entries: blob_dir = os.path.join(blob_dirs, f_hash[:2]) blob_path = os.path.join(blob_dir, f_hash[2:]) os.remove(blob_path) if os.path.isdir(blob_dir): blob_entries = [ f for f in os.listdir(blob_dir) if not f.startswith('.') ] if len(blob_entries) == 0: shutil.rmtree(blob_dir, True) cleanup_tempfiles() return salt, secret, memory, localindex
def index_and_encrypt(memory, options): """ index_and_encrypt @type memory: Memory @type options: optparse.Values, instance @rtype salt, secret, memory, localindex: str, str, Memory, dict """ localindex = make_local_index(options) datadir = get_data_dir(options) if quick_lock_check(options): output_json({"message": "cryptobox is locked, nothing can be added now first decrypt (-d)"}) return None, None, memory, localindex salt = None if memory.has("salt_b64"): salt = base64.decodestring(memory.get("salt_b64")) if not salt: salt = Random.new().read(32) memory.set("salt_b64", base64.encodestring(salt)) output_json({"msg": "preparing encrypt"}) secret = password_derivation(options.password, salt) ensure_directory(datadir) new_blobs = {} file_cnt = 0 new_objects = 0 hash_set_on_disk = set() processed_files = 0 numfiles = 0 for dirhash in localindex["dirnames"]: numfiles += len(localindex["dirnames"][dirhash]["filenames"]) for dirhash in localindex["dirnames"]: for fname in localindex["dirnames"][dirhash]["filenames"]: file_cnt += 1 file_dir = localindex["dirnames"][dirhash]["dirname"] file_path = os.path.join(file_dir, fname["name"]) if os.path.exists(file_path): update_progress(processed_files, numfiles, "indexing " + os.path.basename(file_path)) filedata, localindex = make_cryptogit_hash(file_path, datadir, localindex) fname["hash"] = filedata["filehash"] hash_set_on_disk.add(filedata["filehash"]) if not filedata["blob_exists"]: new_blobs[filedata["filehash"]] = filedata new_objects += 1 if len(new_blobs) > 1500: encrypt_new_blobs(secret, new_blobs) new_blobs = {} if len(new_blobs) > 0: if len(new_blobs) > 0: encrypt_new_blobs(secret, new_blobs) cleanup_tempfiles() memory = store_localindex(memory, localindex) if options.remove: ld = os.listdir(options.dir) ld.remove(".cryptobox") processed_files = 0 numfiles = len(ld) for fname in ld: fpath = os.path.join(options.dir, fname) processed_files += 1 update_progress(processed_files, numfiles, "delete " + os.path.basename(fpath)) if os.path.isdir(fpath): if os.path.exists(fpath): shutil.rmtree(fpath, True) else: if os.path.exists(fpath): os.remove(fpath) obsolute_blob_store_entries = set() blob_dirs = os.path.join(datadir, "blobs") ensure_directory(blob_dirs) for blob_dir in os.listdir(blob_dirs): blob_store = os.path.join(blob_dirs, blob_dir.lstrip(os.path.sep)) if os.path.isdir(blob_store): for blob_file in os.listdir(blob_store): found = False for fhash in hash_set_on_disk: if fhash in (blob_dir + blob_file): found = True if not found: obsolute_blob_store_entries.add(blob_dir + blob_file) for f_hash in obsolute_blob_store_entries: blob_dir = os.path.join(blob_dirs, f_hash[:2]) blob_path = os.path.join(blob_dir, f_hash[2:]) os.remove(blob_path) if os.path.isdir(blob_dir): blob_entries = [f for f in os.listdir(blob_dir) if not f.startswith('.')] if len(blob_entries) == 0: shutil.rmtree(blob_dir, True) cleanup_tempfiles() return salt, secret, memory, localindex