def run():
file_path = '/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list.txt'
output_path = 'firstname_list_filtered.txt'
query_name = "sequential_search_firstname"
query_key = 'firstname'
endpoints = [
"http://10.25.2.147:9876",
"http://10.25.2.148:9876",
"http://10.25.2.149:9876",
"http://10.25.2.151:9876"
]
index = 0
with open(file_path, 'r') as f_source:
for key in f_source.readlines():
key = key.strip()
#for key in ['MARY']:
workload_item = workload.WorkloadItem(key, query_name, endpoints[index % len(endpoints)], query_key, key)
worker_queue.put(workload_item)
index += 1
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
print('Write output file')
with open(output_path,'w') as f:
for key in sorted(results):
f.write("{}\n".format(key))
def run():
target_list = {
'firstname':
('/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list.txt',
'/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_2000.txt',
'sequential_search_firstname', 'firstname'),
'lastname':
('/home/chsu6/elastic-hpcc/benchmark/dataset/lastname_list.txt',
'/home/chsu6/elastic-hpcc/benchmark/dataset/lastname_list_2000.txt',
'sequential_search_lastname', 'lastname'),
'city':
('/home/chsu6/elastic-hpcc/benchmark/dataset/city_list.txt',
'/home/chsu6/elastic-hpcc/benchmark/dataset/city_list_2000.txt',
'sequential_search_city', 'city'),
'zip': ('/home/chsu6/elastic-hpcc/benchmark/dataset/zip_list.txt',
'/home/chsu6/elastic-hpcc/benchmark/dataset/zip_list_2000.txt',
'sequential_search_zip', 'zip'),
}
endpoints = [
"http://10.25.2.147:9876", "http://10.25.2.148:9876",
"http://10.25.2.149:9876", "http://10.25.2.151:9876"
]
index = 0
for target in target_list.keys():
keys_input_path, keys_output_path, query_name, query_key = target_list[
target]
results[query_key] = []
with open(keys_input_path, 'r') as f_source:
for key in f_source.readlines():
key = key.strip()
workload_item = workload.WorkloadItem(
key, query_name, endpoints[index % len(endpoints)],
query_key, key)
worker_queue.put(workload_item)
index += 1
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
print('Writing output file')
for target in target_list.keys():
keys_input_path, keys_output_path, query_name, query_key = target_list[
target]
with open(keys_output_path, 'w') as f:
for key in sorted(results[query_key]):
f.write("{}\n".format(key))
def run():
endpoints = [
"http://10.25.2.131:9876",
"http://10.25.2.132:9876",
"http://10.25.2.133:9876",
"http://10.25.2.134:9876",
"http://10.25.2.136:9876",
"http://10.25.2.138:9876",
"http://10.25.2.139:9876",
"http://10.25.2.140:9876",
]
endpoints = endpoints[:2]
num_queries = num_workers * 100
key_list = open(
'/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_1085.txt',
'r').readlines()
for app_id in range(21, 22):
#for app_id in range(1022, 1023):
query_name = "sequential_search_firstname_{}".format(app_id)
query_key = 'firstname'
for key_index in range(len(key_list)):
key_firstname = key_list[key_index].strip()
for endpoint_index in range(len(endpoints)):
endpoint = endpoints[endpoint_index]
workload_item = workload.WorkloadItem(
"{}-{}-{}".format(endpoint_index, app_id, key_index),
query_name, endpoint, query_key, key_firstname)
worker_queue.put(workload_item)
print(len(worker_queue))
with helper.Timer() as t:
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
count_success = sum(results.values())
print('Success:', count_success)
print('Failure:', len(results) - count_success)
print('Average Time:',
sum(elapsed_time_records.values()) / len(elapsed_time_records))
print('Elapsed Time:', t.elapsed)
print('Complete stress testing')
def runner(self):
endpoints = [
"http://10.25.2.131:9876",
"http://10.25.2.132:9876",
"http://10.25.2.133:9876",
"http://10.25.2.134:9876",
"http://10.25.2.136:9876",
"http://10.25.2.138:9876",
"http://10.25.2.139:9876",
"http://10.25.2.140:9876",
]
key_list = open(
'/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_1085.txt',
'r').readlines()
count = 0
for i in range(self.num_requests):
app_id = random.randint(1, 1024)
query_name = "sequential_search_firstname_{}".format(app_id)
query_key = 'firstname'
key_firstname = key_list[random.randint(0,
len(key_list) -
1)].strip()
endpoint = None
if app_id <= 256:
endpoint = random.choice(endpoints[0:2])
elif app_id <= 512:
endpoint = random.choice(endpoints[2:4])
elif app_id <= 768:
endpoint = random.choice(endpoints[4:6])
elif app_id <= 1024:
endpoint = random.choice(endpoints[6:8])
workload_item = workload.WorkloadItem(str(count), query_name,
endpoint, query_key,
key_firstname)
req = Request(i, workload_item)
#request_dispatcher = request_dispatchers[random.randint(0, len(request_dispatchers) - 1)]
request_dispatcher = request_dispatchers[app_id - 1]
request_dispatcher.put(req)
for request_dispatcher in self.request_dispatchers:
request_dispatcher.complete()
print('Completed request generator')
def run(output_path):
endpoints = [
"http://10.25.2.131:9876",
"http://10.25.2.132:9876",
"http://10.25.2.133:9876",
"http://10.25.2.134:9876",
"http://10.25.2.136:9876",
"http://10.25.2.138:9876",
"http://10.25.2.139:9876",
"http://10.25.2.140:9876",
]
endpoints = endpoints[:1]
num_queries = num_workers * 100
key_list = open('/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_2000.txt', 'r').readlines()
for app_id in range(1, 2):
query_name = "sequential_search_firstname_{}".format(app_id)
query_key = 'firstname'
for key_index in range(len(key_list)):
key_firstname = key_list[key_index].strip()
for endpoint_index in range(len(endpoints)):
endpoint = endpoints[endpoint_index]
workload_item = workload.WorkloadItem("{}-{}-{}".format(endpoint_index, app_id, key_index), query_name, endpoint, query_key, key_firstname)
worker_queue.put(workload_item)
#print(len(worker_queue))
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
count_success = sum(results.values())
print('Success:', count_success)
print('Failure:', len(results) - count_success)
with open(output_path, 'w') as f_out:
for key in sorted(keys_exists):
f_out.write(key + "\n")
print('Complete stress testing')
def run():
endpoints = [
"http://10.25.2.131:9876",
"http://10.25.2.132:9876",
"http://10.25.2.133:9876",
"http://10.25.2.134:9876",
"http://10.25.2.136:9876",
"http://10.25.2.138:9876",
"http://10.25.2.139:9876",
"http://10.25.2.140:9876",
]
endpoints = endpoints[:]
#endpoints = endpoints[-1:]
num_queries = num_workers * 100
index = 0
query_name = "sequential_search_firstname_16"
query_key = 'firstname'
# key_list = open('/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_1085.txt', 'r').readlines()
key_list = open(
'/home/chsu6/elastic-hpcc/benchmark/dataset/firstname_list_2000.txt',
'r').readlines()
#key = 'MARY'
print('total queries={}'.format(num_queries))
for i in range(num_queries):
key = key_list[random.randint(0, len(key_list) - 1)].strip()
workload_item = workload.WorkloadItem(
query_key, query_name, endpoints[index % len(endpoints)],
query_key, key)
worker_queue.put(workload_item)
index += 1
print(len(worker_queue))
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
print('Complete stress testing')
def retrieve_requests(self, node_bucket_id, num_requests):
# self.logger.info("Retrieve from node bucket {} for {} requests".format(node_bucket_id, num_requests))
if len(self.node_buckets) < 1:
# self.logger.info("Node bucket {} is not initialized".format(node_bucket_id))
return [Request.new_empty()]
elif node_bucket_id not in self.node_buckets:
# self.logger.info("Node bucket {} is not empty".format(node_bucket_id))
return [Request.new_empty()]
else:
node_bucket = self.node_buckets[node_bucket_id]
# self.logger.info("Current node bucket {} has {} requests".format(node_bucket.xid, node_bucket.qsize()))
request_list = []
while len(request_list) < num_requests:
request = node_bucket.get_request()
#self.logger.info("Request type={}".format(request.xtype))
if request.xtype == RequestType.run:
# self.logger.info("Correcting request information")
workload_item = request.workload
workload_correct = workload.WorkloadItem(
workload_item.wid, workload_item.query_name,
node_bucket.endpoint, workload_item.query_key,
workload_item.key)
request_correct = Request(request.xtype, request.xid,
workload_correct)
request_list.append(request_correct)
else:
request_list.append(request)
break
#for request in request_list:
# self.logger.info("xid={}, xtype={}".format(request.xid, request.xtype))
#self.logger.info('Get {} requests from node bucket {}'.format(len(request_list)-1, node_bucket_id))
self.logger.info("Current queue size={}".format(
self.get_current_queue_size()))
num_requests = len(request_list) if request_list[
-1].xtype != RequestType.run else len(request_list) - 1
self.current_workload_record.record_dispatch_result(
node_bucket_id, num_requests)
return request_list
def run():
endpoints = [
"http://10.25.2.131:9876",
"http://10.25.2.132:9876",
"http://10.25.2.133:9876",
"http://10.25.2.134:9876",
"http://10.25.2.136:9876",
"http://10.25.2.138:9876",
"http://10.25.2.139:9876",
"http://10.25.2.140:9876",
]
endpoints = endpoints[:]
num_queries = num_workers * 100
for app_id in range(1, 129):
query_name = "sequential_search_firstname_{}".format(app_id)
query_key = 'firstname'
key_firstname = 'MARY'
for endpoint_index in range(len(endpoints)):
endpoint = endpoints[endpoint_index]
workload_item = workload.WorkloadItem(
"{}-{}".format(app_id, endpoint_index), query_name, endpoint,
query_key, key_firstname)
worker_queue.put(workload_item)
print(len(worker_queue))
for i in range(num_workers):
worker_pool.spawn(worker, i)
worker_pool.join()
count_success = sum(results.values())
print('Success:', count_success)
print('Failure:', len(results) - count_success)
print('Complete stress testing')