def run(run_output_dir, workload_file):
#hpcc_cluster = HPCCCluster.parse_config("/etc/HPCCSystems/source/hpcc_t5_r5_120worker_cyclic.xml")
#hpcc_cluster = HPCCCluster.parse_config("/etc/HPCCSystems/source/hpcc_16r_cyclic_2replica.xml")
#hpcc_cluster = HPCCCluster.parse_config("/etc/HPCCSystems/source/elastic_cyclic_4node_2replica.xml")
#hpcc_cluster = HPCCCluster.parse_config("/etc/HPCCSystems/source/cyclic/elastic_4node_2replica.xml")
hpcc_cluster = HPCCCluster.parse_config(
"/etc/HPCCSystems/source/cyclic/elastic_4node_4replica.xml")
benchmark_config = BenchmarkConfig.parse_file(
"/home/chsu6/elastic-hpcc/conf/6driver.yaml")
#benchmark_config = BenchmarkConfig.parse_file("/home/chsu6/elastic-hpcc/conf/1driver.yaml")
script_dir = os.path.dirname(os.path.realpath(__file__))
workload_config_template = WorkloadConfig.parse_file(
os.path.join(script_dir, workload_file))
application_db = workload_config_template.lookup_config(
'workload.applications')
for arrival_type in arrival_types:
for num_queries in num_queries_list:
for period in period_list:
workload_config = copy.deepcopy(workload_config_template)
workload_config.set_config('workload.type', arrival_type)
workload_config.set_config('workload.num_queries', num_queries)
workload_config.set_config('workload.period', period)
workload = Workload.from_config(workload_config)
workload_timeline = WorkloadExecutionTimeline.from_workload(
workload)
for app_name in applications:
for distribution_type in distribution_types:
per_workload_config = copy.deepcopy(workload_config)
per_workload_config.set_config('workload.distribution',
distribution_type)
per_workload_config.set_config(
'workload.applications',
{app_name: application_db[app_name]})
per_workload = Workload.from_config(
per_workload_config)
per_workload_timeline = WorkloadExecutionTimeline.from_timeline(
workload_timeline, per_workload)
output_dir = os.path.join(
"results", run_output_dir,
"5roxie_{}_{}_{}_{}queries_{}sec".format(
arrival_type, distribution_type['type'],
app_name, num_queries, period))
if os.path.exists(output_dir):
continue
bm = RoxieBenchmark(hpcc_cluster,
benchmark_config,
per_workload_timeline,
output_dir=output_dir)
time.sleep(60)
bm.run()
def submit_workload(self, workload):
if isinstance(workload, Workload):
self.logger.debug("submit a workload object")
workload_timeline = WorkloadExecutionTimeline.from_workload(
workload)
else:
self.logger.debug("submit a workload timeline")
workload_timeline = workload
self.logger.debug(workload_timeline)
return self.commander.workload_submit(workload_timeline)
def cache(self, workload_config, workload, update=False, name=None):
cache_path = self._generate_cache_path(workload_config) if name is None else os.path.join(self.store_dir, name)
if (not update) and (os.path.exists(cache_path)):
print("loading workload timeline from {}".format(cache_path))
return WorkloadTimelineManager.load_timeline(cache_path)
else:
print("generate a new workload timeline")
workload_timeline = WorkloadExecutionTimeline.from_workload(workload)
print("caching the workload timeline to {}".format(cache_path))
WorkloadTimelineManager.save_timeline(workload_timeline, cache_path)
return workload_timeline
def run(ctx, config, output_dir):
hpcc_cluster = HPCCCluster.parse_config(
"/etc/HPCCSystems/source/hpcc_t5_r5_cyclic.xml")
benchmark_config = BenchmarkConfig.parse_file(ctx.obj['config'])
w = Workload.parse_config(config)
workload_timeline = WorkloadExecutionTimeline.from_workload(w)
bm = RoxieBenchmark(hpcc_cluster,
benchmark_config,
workload_timeline,
output_dir=output_dir)
bm.run()
def run(ctx, config, hpcc_config, output_dir):
# workload
w = Workload.parse_config(config)
workload_timeline = WorkloadExecutionTimeline.from_workload(w)
# hpcc setting
hpcc_cluster = HPCCCluster.parse_config(hpcc_config)
# benchmark setting
benchmark_config = BenchmarkConfig.parse_file(ctx.obj['config'])
bm = RoxieBenchmark(hpcc_cluster,
benchmark_config,
workload_timeline,
output_dir=output_dir)
bm.run()
def submit(ctx, config, output_dir):
w = Workload.parse_config(config)
workload_timeline = WorkloadExecutionTimeline.from_workload(w)
benchmark_service = BenchmarkService.new(ctx.obj['config'])
benchmark_service.submit_workload(workload_timeline)
def load_timeline(cache_path):
return WorkloadExecutionTimeline.from_pickle(cache_path)