def test_delete(object_storage_client): create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkDelete_{}'.format(random.randint(0, 1000000)) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder]) num_objects_to_delete = get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder) # Sanity check that the bucket has things in it assert get_number_of_objects_in_bucket(object_storage_client, create_bucket_request.name) > 0 result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name]) if num_objects_to_delete >= 1000: confirm_prompt = 'WARNING: This command will delete at least {} objects. Are you sure you wish to continue?'.format(num_objects_to_delete) else: confirm_prompt = 'WARNING: This command will delete {} objects. Are you sure you wish to continue?'.format(num_objects_to_delete) assert confirm_prompt in result.output result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--force']) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['delete-failures'] == {} assert len(parsed_result['deleted-objects']) == num_objects_to_delete # Check that the bucket is now empty assert get_number_of_objects_in_bucket(object_storage_client, create_bucket_request.name) == 0 delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)
def test_bulk_operation_table_output_query(object_storage_client): create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageTableOutput_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) result = invoke(['os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder, '--output', 'table', '--query', "[?action=='Uploaded'].{file: file, \"opc-content-md5\": \"opc-content-md5\"}"]) assert 'file' in result.output assert 'opc-content-md5' in result.output assert 'etag' not in result.output result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--dry-run', '--output', 'table']) assert 'action' in result.output assert 'object' in result.output assert '/a/Object_1' in result.output result = invoke(['os', 'object', 'bulk-delete', '--namespace', util.NAMESPACE, '--bucket-name', bulk_get_bucket_name, '--dry-run', '--output', 'table', '--query', "[?object=='Object_0'][object]"]) assert 'action' not in result.output assert '/a/Object_1' not in result.output assert 'Object_0' in result.output target_download_folder = os.path.join('tests', 'temp', create_bucket_request.name) result = invoke([ 'os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--download-dir', target_download_folder, '--output', 'table', ]) delete_bucket_and_all_items(object_storage_client, create_bucket_request.name) shutil.rmtree(target_download_folder)
def test_bulk_put_with_multipart_params(object_storage_client): create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkPutMultipartsTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder, '--part-size', '10' ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['skipped-objects'] == [] assert parsed_result['upload-failures'] == {} assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder) result = invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', root_bulk_put_folder, '--no-multipart', '--overwrite' ]) parsed_result = parse_json_response_from_mixed_output(result.output) assert parsed_result['skipped-objects'] == [] assert parsed_result['upload-failures'] == {} assert len(parsed_result['uploaded-objects']) == get_count_of_files_in_folder_and_subfolders(root_bulk_put_folder) delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)
def test_data(object_storage_client): # Setup test data # Create the bucket with 2 objects util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, BUCKET_NAME) util.create_bucket(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, BUCKET_NAME, objects=['object1', 'object2'])
def test_get_multipart(object_storage_client): create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkGetMultipartsTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) large_file_root_dir = os.path.join('tests', 'temp', 'multipart_get_large_files') if not os.path.exists(large_file_root_dir): os.makedirs(large_file_root_dir) util.create_large_file(os.path.join(large_file_root_dir, '1.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '2.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '3.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '4.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '5.bin'), LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) util.create_large_file(os.path.join(large_file_root_dir, '6.bin'), 1) # Creates a 1 MiB file for variety invoke([ 'os', 'object', 'bulk-upload', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--src-dir', large_file_root_dir ]) large_file_verify_dir = os.path.join('tests', 'temp', 'multipart_get_large_files_verify') invoke(['os', 'object', 'bulk-download', '--namespace', util.NAMESPACE, '--bucket-name', create_bucket_request.name, '--download-dir', large_file_verify_dir, '--multipart-download-threshold', '128']) assert get_count_of_files_in_folder_and_subfolders(large_file_verify_dir) == 6 assert filecmp.cmp(os.path.join(large_file_root_dir, '1.bin'), os.path.join(large_file_verify_dir, '1.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '2.bin'), os.path.join(large_file_verify_dir, '2.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '3.bin'), os.path.join(large_file_verify_dir, '3.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '4.bin'), os.path.join(large_file_verify_dir, '4.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '5.bin'), os.path.join(large_file_verify_dir, '5.bin')) assert filecmp.cmp(os.path.join(large_file_root_dir, '6.bin'), os.path.join(large_file_verify_dir, '6.bin')) shutil.rmtree(large_file_root_dir) shutil.rmtree(large_file_verify_dir) delete_bucket_and_all_items(object_storage_client, create_bucket_request.name)
def generate_test_data(object_storage_client): global bulk_get_object_to_content, bulk_get_bucket_name, root_bulk_put_folder, bulk_put_large_files, bulk_put_mid_sized_files, bulk_put_bucket_name # Create a test bucket create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkGetTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) bulk_get_bucket_name = create_bucket_request.name # Create items at various heirarchy levels (to be surfaced as different directories on disk) for i in range(OBJECTS_TO_CREATE_IN_BUCKET_FOR_BULK_GET): if i % 5 == 4: object_name = 'a/b/c/d/Object_{}'.format(i) bulk_get_prefix_to_object['a/b/c/d'].append(object_name) elif i % 5 == 3: object_name = 'a/b/c/Object_{}'.format(i) bulk_get_prefix_to_object['a/b/c'].append(object_name) elif i % 5 == 2: object_name = 'a/b/Object_{}'.format(i) bulk_get_prefix_to_object['a/b'].append(object_name) elif i % 5 == 1: # This is equivalent to a/ on the file system because we drop the leading slash (we drop path separators from the front to avoid unexpected results) object_name = '/a/Object_{}'.format(i) bulk_get_prefix_to_object['/a'].append(object_name) else: # At the root of the bucket object_name = 'Object_{}'.format(i) bulk_get_prefix_to_object[''].append(object_name) object_content = generate_random_string(CONTENT_STRING_LENGTH) object_storage_client.put_object(util.NAMESPACE, create_bucket_request.name, object_name, object_content) bulk_get_object_to_content[object_name] = object_content # makedirs creates all subfolders recursively root_bulk_put_folder = 'tests/temp/bulk_put_{}'.format(util.random_number_string()) bulk_put_folder_leaf = '{}/subfolder1/subfolder2/subfolder3'.format(root_bulk_put_folder) if not os.path.exists(bulk_put_folder_leaf): os.makedirs(bulk_put_folder_leaf) create_bucket_request = oci.object_storage.models.CreateBucketDetails() create_bucket_request.name = 'ObjectStorageBulkPutTest_{}'.format(util.random_number_string()) create_bucket_request.compartment_id = util.COMPARTMENT_ID util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, create_bucket_request.name) object_storage_client.create_bucket(util.NAMESPACE, create_bucket_request) bulk_put_bucket_name = create_bucket_request.name subfolders = ['', 'subfolder1', 'subfolder1/subfolder2', 'subfolder1/subfolder2/subfolder3'] for subfolder in subfolders: if subfolder == '': full_folder = root_bulk_put_folder else: full_folder = os.path.join(root_bulk_put_folder, subfolder) for i in range(OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT + 1): file_path = '{}/object_{}'.format(full_folder, i) if i != 0 and i % OBJECTS_TO_CREATE_IN_FOLDER_FOR_BULK_PUT == 0: # Put in one big file per subfolder util.create_large_file(file_path, LARGE_CONTENT_FILE_SIZE_IN_MEBIBYTES) bulk_put_large_files.add(file_path) elif i != 0 and i % 10 == 0: # Put in the occasional file with a reasonable size so that we can force multipart util.create_large_file(file_path, MID_SIZED_FILE_IN_MEBIBTYES) bulk_put_mid_sized_files.add(file_path) else: with open(file_path, 'w') as f: f.write(generate_random_string(CONTENT_STRING_LENGTH)) yield # Tear down stuff by deleting all the things and then deleting the buckets delete_bucket_and_all_items(object_storage_client, bulk_get_bucket_name) delete_bucket_and_all_items(object_storage_client, bulk_put_bucket_name) # Remove all directories recursively shutil.rmtree(root_bulk_put_folder)
def temp_bucket(runner, config_file, config_profile, object_storage_client): bucket_name = 'cli_temp_multipart_bucket_' + str(random.randint( 0, 1000000)) print("Bucket Name: ", bucket_name) # ns get result = invoke(runner, config_file, config_profile, ['ns', 'get']) validate_response(result) assert util.NAMESPACE in result.output # bucket create util.clear_test_data(object_storage_client, util.NAMESPACE, util.COMPARTMENT_ID, bucket_name) result = invoke(runner, config_file, config_profile, [ 'bucket', 'create', '-ns', util.NAMESPACE, '--compartment-id', util.COMPARTMENT_ID, '--name', bucket_name ]) validate_response(result) # create the SSE-C encryption key enc_key_str = generate_aes256_key_str() with open(GENERATED_ENC_KEY_FILE, 'w') as f: f.write(enc_key_str) yield bucket_name # clean up, delete bucket error_count = 0 try: result = invoke( runner, config_file, config_profile, ['object', 'list', '-ns', util.NAMESPACE, '-bn', bucket_name]) validate_response(result) response = json.loads(result.output) if 'data' in response: objects = response['data'] for obj in objects: invoke(runner, config_file, config_profile, [ 'object', 'delete', '-ns', util.NAMESPACE, '-bn', bucket_name, '--name', obj['name'], '--force' ]) validate_response(result) except Exception as error: util.print_latest_exception(error) error_count = error_count + 1 try: print("Deleting bucket") result = invoke(runner, config_file, config_profile, [ 'bucket', 'delete', '-ns', util.NAMESPACE, '--name', bucket_name, '--force' ]) validate_response(result) except Exception as error: util.print_latest_exception(error) error_count = error_count + 1 assert 0 == error_count # remove the SSE-C key file if os.path.exists(GENERATED_ENC_KEY_FILE): os.remove(GENERATED_ENC_KEY_FILE)