def test_config(test_job): global logger host = '127.0.0.1' port = '8081' # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) # Connection config config = os.path.join(os.path.dirname(__file__), '..', '..', 'src', 'config', 'config.yaml') with open(config, 'r') as file_config: config = yaml.safe_load(file_config) host = config['replicator'].get('host') port = str(config['replicator'].get('port')) # URL for non-secure http endpoint url = 'http://' + host + ':' + port return {'url': url, 'test_job': test_job}
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object name object_name = str(config.object_name_prefix) bucket_name = config.source_bucket_name request_id = "dummy-request-id" obj = S3AsyncCreateMultipartUpload(session, request_id, bucket_name, object_name) await obj.create() logger.info("S3AsyncCreateMultipartUpload test passed!") await session.close()
def logger(): """Setup logger for tests.""" # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') logger = setup_logger('client_tests', log_config_file) if logger is None: sys.exit(-1) return logger
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate bucket names bucket_name = config.source_bucket_name # Generate object names object_name = str(config.object_name_prefix) object_size = config.object_size range_read_offset = config.range_read_offset range_read_length = config.range_read_length request_id = "dummy-request-id" object_reader = S3AsyncGetObject(session, request_id, bucket_name, object_name, object_size, range_read_offset, range_read_length) reader_generator = object_reader.fetch(object_size) async for _ in reader_generator: pass content_length = object_reader.get_content_length() if range_read_length >= 0: # Validate if content length matches to total object range if object_reader.get_total_object_range() == content_length: logger.info("Content-Length matched!") logger.info("S3AsyncGetObjectRangeRead test passed!") else: logger.error("Error : size mismatched") logger.info("S3AsyncGetObjectRangeRead test failed!") else: # Validate if content length matches to object size in config if object_size == content_length: logger.info("Content-Length matched!") logger.info("S3AsyncGetObject test passed!") else: logger.error("Error : size mismatched") logger.info("S3AsyncGetObject test failed!") await session.close()
def init_logger(): log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) return logger
def __init__(self, config_file, log_config_file): """Initialise logger and configuration.""" self._config = Config(config_file) if self._config.load() is None: print("Failed to load configuration.\n") sys.exit(-1) # Setup logging. self._logger = setup_logger('s3replicationmanager', log_config_file) if self._logger is None: print("Failed to configure logging.\n") sys.exit(-1) self._config.print_with(self._logger) self._jobs = Jobs(self._logger, "all-jobs") self._subscribers = Subscribers()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object name object_name = str(config.object_name_prefix) bucket_name = config.source_bucket_name request_id = "dummy-request-id" # For Cortx, default value of version_id=None # For AWS, user of this class should provide version-id as well. version_id = None # For Cortx, default value of part_number=None # For AWS, user of this class should provide part-number as well. part_number = None head_obj = S3AsyncHeadObject(session, request_id, bucket_name, object_name, version_id) # Pass the part_number to get specific part's information await head_obj.get(part_number) # Validate if content length matches to object size if config.object_size == head_obj.get_content_length(): logger.info("Content-Length matched!") logger.info("S3AsyncHeadObject test passed!") else: logger.error("Error : Content-Length mismatched") logger.info("S3AsyncHeadObject test failed!") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) # Read configs for future comparision replication_prefix = str(config.object_name_prefix) test_replication_object = replication_prefix replication_dest_bucket = config.target_bucket_name s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object names request_id = "dummy-request-id" obj = S3AsyncGetBucketReplication(session, request_id, config.source_bucket_name) # Start transfer await obj.get() replication_rule = obj.get_replication_rule(test_replication_object) logger.debug(replication_rule) assert replication_rule._prefix == replication_prefix, \ "replication_prefix mismatched" assert replication_rule._dest_bucket == replication_dest_bucket, \ "replication_dest_bucket mismatched" await session.close() logger.info("AsyncS3GetBucketReplication test passed!")
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object name object_name = str(config.object_name_prefix) bucket_name = config.source_bucket_name total_parts = config.total_parts request_id = "dummy-request-id" # Provide actual upload ID in place of __UPLOAD_ID__ upload_id = "__UPLOAD_ID__" obj_data_generator = MultipartObjectDataGenerator(logger, config.object_size, total_parts) obj = S3AsyncUploadPart(session, request_id, bucket_name, object_name, upload_id) total_chunks = int(config.object_size / total_parts) print("Total chunks {}".format(total_chunks)) for part_no in range(1, int(total_parts + 1)): await obj.upload(obj_data_generator, part_no, total_chunks) print("\n") logger.info("S3AsyncUploadPart test passed!") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate bucket names bucket_name = config.source_bucket_name # Generate object names object_name = str(config.object_name_prefix) request_id = "dummy-request-id" # Get tag-name and tag-value from config tagset = {} tag_name = config.object_tag_name tag_value = config.object_tag_value tagset[tag_name] = tag_value tag_object = S3AsyncPutObjectTagging(session, request_id, bucket_name, object_name, tagset) await tag_object.send() logger.info("S3AsyncPutObjectTagging test passed!") await session.close()
def __init__(self, config_file, log_config_file): """Initialise logger and configuration.""" self._config = Config(config_file) if self._config.load() is None: print("Failed to load configuration.\n") sys.exit(-1) # Setup logging. self._logger = setup_logger('s3replicator', log_config_file) if self._logger is None: print("Failed to configure logging.\n") sys.exit(-1) self._inprogress_jobs = Jobs(self._logger, "all-jobs") if self._config.job_cache_enabled: self._completed_jobs = Jobs(self._logger, "completed-jobs", self._config.job_cache_timeout_secs) else: self._completed_jobs = Jobs(self._logger, "completed-jobs") self._replication_managers = ReplicationManagers() self._config.print_with(self._logger)
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate bucket names bucket_name = config.source_bucket_name # Generate object names object_name = str(config.object_name_prefix) request_id = "dummy-request-id" tag_object = S3AsyncGetObjectTagging(session, request_id, bucket_name, object_name) await tag_object.fetch() # Validate if tags value matches to object tag value in config if config.object_tag_value == tag_object.get_tags_value( config.object_tag_name): logger.info("Tag value matched!") logger.info("S3AsyncGetObjectTagging test passed!") else: logger.error("Error : Tag value mismatched") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object name object_name = str(config.object_name_prefix) bucket_name = config.source_bucket_name request_id = "dummy-request-id" # Provide actual upload ID in place of __UPLOAD_ID__ upload_id = "__UPLOAD_ID__" # Etag dictionary # Example : etag_dict = {1: '"7ade650b8547b04d5fa6ba96e10c8b5f"'} etag_dict = {1: "__ETAG_PART1__", 2: "__ETAG_PART2__"} obj = S3AsyncCompleteMultipartUpload(session, request_id, bucket_name, object_name, upload_id, etag_dict) await obj.complete_upload() logger.info("Final ETag : {}".format(obj.get_final_etag())) logger.info("S3AsyncCompleteMultipartUpload test passed!") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate object names source_object_name = str(config.object_name_prefix) target_object_name = str(config.object_name_prefix) request_id = "dummy-request-id" object_reader = S3AsyncGetObject(session, request_id, config.source_bucket_name, source_object_name, config.object_size, config.range_read_offset, config.range_read_length) object_writer = S3AsyncPutObject(session, request_id, config.target_bucket_name, target_object_name, config.object_size) # Start transfer await object_writer.send(object_reader, config.transfer_chunk_size) logger.info("S3AsyncTransferObject test passed!") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) # Generate bucket names bucket_name = config.source_bucket_name # Generate object names object_name = str(config.object_name_prefix) object_size = config.object_size request_id = "dummy-request-id" object_reader = FixedObjectDataGenerator(logger, object_name, object_size) object_writer = S3AsyncPutObject(session, request_id, bucket_name, object_name, object_size) # Write to the object await object_writer.send(object_reader, object_size) logger.info("S3AsyncPutObject test passed!") await session.close()
async def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key) source_bucket_name = config.source_bucket_name # Generate object names source_object_name = str(config.object_name_prefix) request_id = "dummy-request-id" total_parts = config.total_parts # Start Multipart Upload # Create multipart print("\nCreate multipart upload") print("_______________________\n") obj_create = S3AsyncCreateMultipartUpload(session, request_id, source_bucket_name, source_object_name) await obj_create.create() upload_id = obj_create.get_response_header("UploadId") # Object data generator obj_data_generator = MultipartObjectDataGenerator(logger, config.object_size, total_parts) # Upload part print("\nUpload part") print("___________\n") obj_upload = S3AsyncUploadPart(session, request_id, source_bucket_name, source_object_name, upload_id) total_chunks = int(config.object_size / total_parts) for part_no in range(1, int(total_parts + 1)): await obj_upload.upload(obj_data_generator, part_no, total_chunks) print("\n") etag_dict = obj_upload.get_etag_dict() # Complete multipart upload print("Complete multipart upload") print("__________________________\n") obj_complete = S3AsyncCompleteMultipartUpload(session, request_id, source_bucket_name, source_object_name, upload_id, etag_dict) await obj_complete.complete_upload() final_etag = obj_complete.get_final_etag() logger.info("Final ETag : {}".format(final_etag)) logger.info("S3AsyncMultipartUpload test passed!") await session.close()
async def main(): """Main function for calling various REST requests.""" host = '127.0.0.1' port = '8081' # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) # Create parser object parser = argparse.ArgumentParser(description='''Replicator server help''') # Adding an arguments parser.add_argument( '--configfile', type=str, metavar='path', help='Path to replication manager configuration file(format: yaml)') parser.add_argument('--jobfile', type=str, metavar='path', help='Test job record') # Parsing arguments args = parser.parse_args() # Read input config file and get host, port if args.configfile is None: args.configfile = os.path.join(os.path.dirname(__file__), "..", "..", "src", 'config', 'config.yaml') with open(args.configfile, 'r') as file_config: config = yaml.safe_load(file_config) host = config['replicator'].get('host') port = str(config['replicator'].get('port')) # URL for non-secure http endpoint url = 'http://' + host + ':' + port # Load the test job record. test_job = {} if args.jobfile is None: args.jobfile = os.path.join(os.path.dirname(__file__), 'data', 'test_job.json') with open(args.jobfile, 'r') as file_config: test_job = json.load(file_config) # Start client session async with aiohttp.ClientSession() as session: # Post the replication job async with session.post(url + '/jobs', json=test_job) as response: logger.info('POST jobs Status: {}'.format(response.status)) html = await response.json() logger.info('Body: {}'.format(html))
def main(): config = Config() # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) # CONFIG OPTIONS bucket_name = config.source_bucket_name dest_bucket_name = config.target_bucket_name object_name_prefix = str(config.object_name_prefix) iam_role = config.iam_role # Create temp replication policy file. os.system( 'cp ./tests/system/config/replication_policy_sample.json temp_policy.json' ) matches = ['_ACCOUNT_ID_', '_REPLICATION_ENABLED_BUCKET_'] # Read policy and make replacements based on config options. with fileinput.FileInput('temp_policy.json', inplace=True) as file: # Read each line and match the pattern and do replacement. for line in file: if all(x in line for x in matches): line = re.sub('(_ACCOUNT_ID_)', str(iam_role), line) line = re.sub('(_REPLICATION_ENABLED_BUCKET_)', bucket_name, line) print(line) elif '_DESTINATION_BUCKET_' in line: line = re.sub('(_DESTINATION_BUCKET_)', dest_bucket_name, line) print(line) elif '_PREFIX_' in line: line = re.sub('(_PREFIX_)', object_name_prefix, line) print(line) else: print(line, end='') # updated replication policy. # os.system('cat temp_policy.json') command = 'aws s3api put-bucket-replication --bucket ' + \ bucket_name + ' --replication-configuration file://temp_policy.json' exit_status = os.system(command) if exit_status == 0: logger.info("put-bucket-replication passed! ") else: os.system('rm -rf temp_policy.json') logger.error("put-bucket-replication failed! ") os._exit(exit_status) # Delete temp file. os.system('rm -rf temp_policy.json')
async def main(): config = Config() bucket_name = config.source_bucket_name total_count = config.total_objects # Number of objects to upload. object_size = config.object_size # Bytes. # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) s3_site = S3Site(config.endpoint, config.s3_service_name, config.s3_region) session = S3Session(logger, s3_site, config.access_key, config.secret_key, config.max_s3_connections) reader_list = [] writer_list = [] put_task_list = [] start_time = time.perf_counter() # Prepare for upload. for i in range(total_count): # Generate object name object_name = "test_object_" + str(i) + "_sz" + str(object_size) request_id = "request-id" + str(i) object_reader = FixedObjectDataGenerator(logger, object_name, object_size) object_writer = S3AsyncPutObject(session, request_id, bucket_name, object_name, object_size) reader_list.append(object_reader) writer_list.append(object_writer) task = asyncio.ensure_future(writer_list[i].send( reader_list[i], reader_list[i].object_size)) put_task_list.append(task) # Trigger upload and Wait for uploads to complete. await asyncio.gather(*put_task_list) end_time = time.perf_counter() total_time_ms = int(round((end_time - start_time) * 1000)) for index in range(total_count): # Validate object uploaded successfully. assert writer_list[index].get_state() == S3RequestState.COMPLETED source_etag = reader_list[index].get_etag() target_etag = writer_list[index].get_etag() assert target_etag == source_etag, \ "PUT ETag = {} and Data ETag = {}".format(target_etag, source_etag) logger.info("Total time to upload {} objects = {} ms.".format( total_count, total_time_ms)) logger.info("Avg time per upload = {} ms.".format(total_time_ms / total_count)) await session.close()
def main(): config = Config() bucket_name = config.source_bucket_name total_count = config.total_objects # Number of objects to upload. # Must be multiple of total_count max_pool_connections = config.max_s3_connections # Must be less than and multiple of total_count max_threads = config.max_threads_for_boto3 object_size = config.object_size # Bytes. assert total_count % max_pool_connections == 0, \ "max_pool_connections must be multiple of total_count" assert max_threads <= total_count and total_count % max_threads == 0, \ "max_threads must be less than or equal to and multiple of total_count" # Setup logging and get logger log_config_file = os.path.join(os.path.dirname(__file__), 'config', 'logger_config.yaml') print("Using log config {}".format(log_config_file)) logger = setup_logger('client_tests', log_config_file) if logger is None: print("Failed to configure logging.\n") sys.exit(-1) # Init Global. GlobalTestDataBlock.create(object_size) session = boto3.session.Session() client = session.client("s3", use_ssl=False, endpoint_url=config.endpoint, aws_access_key_id=config.access_key, aws_secret_access_key=config.secret_key, config=botocore.client.Config( max_pool_connections=max_pool_connections)) # Create resources for each thread. work_items = [] start_time = time.perf_counter() with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) \ as executor: futures = [] for i in range(total_count): # Generate object name object_name = "test_object_" + str(i) + "_sz" + str(object_size) work_item = WorkItem(bucket_name, object_name, object_size, client) work_items.append(work_item) futures.append( executor.submit(upload_object, logger=logger, work_item=work_items[i])) # Wait for all threads to complete. for future in concurrent.futures.as_completed(futures): future.result() end_time = time.perf_counter() total_time_ms_threads_requests = int(round((end_time - start_time) * 1000)) logger.info( "Total time to upload {} objects including thread creation = {} ms.". format(total_count, total_time_ms_threads_requests)) logger.info("Avg time per upload = {} ms.".format( total_time_ms_threads_requests / total_count))