def content_input_file(): filename = 'tests/temp/multipart_content_input.txt' # generate large file for multipart testing util.create_large_file(filename, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) yield filename if os.path.exists(filename): os.remove(filename)
def setup_module(): if os.getenv('RECORD_ONLY', '') == '1': return # generate large file for multipart testing util.create_large_file(GENERATED_CONTENT_INPUT_FILE, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) # generate files, each containing a base64 string of an AES encryption key for fname in [GENERATED_ENC_KEY1_FILE, GENERATED_ENC_KEY2_FILE]: enc_key_str = generate_aes256_key_str() with open(fname, 'w') as f: f.write(enc_key_str)
def setup_module(): if os.getenv('RECORD_ONLY', '') == '1': return # generate large file for multipart testing util.create_large_file(GENERATED_CONTENT_INPUT_FILE, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) # generate files, each containing a base64 string of an AES encryption key for fname in [GENERATED_ENC_KEY1_FILE, GENERATED_ENC_KEY2_FILE]: enc_key_str = generate_aes256_key_str() with open(fname, 'w') as f: f.write(enc_key_str) # generate file with leading, trailing whitespaces and newline added to it with open(GENERATED_ENC_KEY1_FILE) as fh: with open(GENERATED_ENC_KEY1_FILE_WITH_NEWLINE, 'w') as fw: fw.write(' ' + fh.read() + os.linesep + ' ')
def generate_test_data(object_storage_client): global bulk_get_object_to_content, bulk_get_bucket_name, root_bulk_put_folder, bulk_put_large_files, bulk_put_mid_sized_files, bulk_put_bucket_name # Create a test bucket create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkGetTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) bulk_get_bucket_name = create_bucket_request.name # Create items at various heirarchy levels (to be surfaced as different directories on disk) for i in range(OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET): if i % 5 == 4: object_name = 'a/b/c/d/Object_{}'.format(i) bulk_get_prefix_to_object['a/b/c/d'].append(object_name) elif i % 5 == 3: object_name = 'a/b/c/Object_{}'.format(i) bulk_get_prefix_to_object['a/b/c'].append(object_name) elif i % 5 == 2: object_name = 'a/b/Object_{}'.format(i) bulk_get_prefix_to_object['a/b'].append(object_name) elif i % 5 == 1: # This is equivalent to a/ on the file system because we drop the leading slash (we drop path separators from the front to avoid unexpected results) object_name = '/a/Object_{}'.format(i) bulk_get_prefix_to_object['/a'].append(object_name) else: # At the root of the bucket object_name = 'Object_{}'.format(i) bulk_get_prefix_to_object[''].append(object_name) object_content = generate_random_string(CONTENT_STRING_LENGTH) object_storage_client.put_object(util.NAMESPACE, create_bucket_request.name, object_name, object_content) bulk_get_object_to_content[object_name] = object_content # makedirs creates all subfolders recursively root_bulk_put_folder = 'tests/temp/bulk_put_{}'.format(util.random_number_string()) bulk_put_folder_leaf = '{}/subfolder1/subfolder2/subfolder3'.format(root_bulk_put_folder) if not os.path.exists(bulk_put_folder_leaf): os.makedirs(bulk_put_folder_leaf) create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkPutTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) bulk_put_bucket_name = create_bucket_request.name subfolders = ['', 'subfolder1', 'subfolder1/subfolder2', 'subfolder1/subfolder2/subfolder3'] for subfolder in subfolders: if subfolder == '': full_folder = root_bulk_put_folder else: full_folder = os.path.join(root_bulk_put_folder, subfolder) for i in range(OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT + 1): file_path = '{}/object_{}'.format(full_folder, i) if i != 0 and i % OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT == 0: # Put in one big file per subfolder util.create_large_file(file_path, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) bulk_put_large_files.add(file_path) elif i != 0 and i % 10 == 0: # Put in the occasional file with a reasonable size so that we can force multipart util.create_large_file(file_path, MID_SIZED_FILE_IN_MEBIBTYES) bulk_put_mid_sized_files.add(file_path) else: with open(file_path, 'w') as f: f.write(generate_random_string(CONTENT_STRING_LENGTH)) yield # Tear down stuff by deleting all the things and then deleting the buckets delete_bucket_and_all_items(object_storage_client, bulk_get_bucket_name) delete_bucket_and_all_items(object_storage_client, bulk_put_bucket_name) # Remove all directories recursively shutil.rmtree(root_bulk_put_folder)
def test_get_multipart(object_storage_client): create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkGetMultipartsTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) large_file_root_dir = os.path.join('tests', 'temp', 'multipart_get_large_files') if not os.path.exists(large_file_root_dir): os.makedirs(large_file_root_dir) util.create_large_file(os.path.join(large_file_root_dir, '1.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '2.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '3.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '4.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '5.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '6.bin'), 1) # Creates a 1 MiB file for variety invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', large_file_root_dir ]) large_file_verify_dir = os.path.join('tests', 'temp', 'multipart_get_large_files_verify') invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--download-dir', large_file_verify_dir, '--multipart-download-threshold', '128']) assert get_count_of_files_in_folder_and_subfolders(large_file_verify_dir) == 6 assert filecmp.cmp(os.path.join(large_file_root_dir, '1.bin'), os.path.join(large_file_verify_dir, '1.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '2.bin'), os.path.join(large_file_verify_dir, '2.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '3.bin'), os.path.join(large_file_verify_dir, '3.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '4.bin'), os.path.join(large_file_verify_dir, '4.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '5.bin'), os.path.join(large_file_verify_dir, '5.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '6.bin'), os.path.join(large_file_verify_dir, '6.bin')) shutil.rmtree(large_file_root_dir) shutil.rmtree(large_file_verify_dir) delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)