def measure(tools, scenarios, sizes, queries, optional_arguments: dict):
"""Benchmark function.
"""
for tool in tools:
if tool in optional_arguments:
args = optional_arguments[tool]
else:
args = [""]
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(
TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in scenarios:
for size in sizes:
for query in queries:
print("Run benchmark: <tool: " + tool +
", scenario: " + scenario + ", query: " + query +
", size: " + str(size) +
(", arg: " + arg if arg != "" else "") + ">")
cmd = [
"java", "-Xmx" + java_xmx, "-jar", target,
"-scenario", scenario, "-query", query, "-size",
str(size), arg
]
subprocess.call(cmd)
util.set_working_directory("..")
def benchmark(conf):
"""Runs measurements.
"""
header = "../output/header.tsv"
result_file = "../output/output.tsv"
if os.path.exists(result_file):
os.remove(result_file)
shutil.copy(header, result_file)
for change_set in conf.change_sets:
for tool in conf.tools:
for query in conf.queries:
for args in conf.optional_arguments:
for size in conf.sizes:
target = util.get_tool_jar(tool)
print("Running benchmark: tool = " + tool + ", change set = " + change_set +
", query = " + query + ", size = " + str(size) + ", extra arguments = " + str(args))
try:
command = ["java", conf.vmargs,
"-jar", target,
"-runs", str(conf.runs),
"-size", str(size),
"-query", query,
"-changeSet", change_set,
"-iterationCount", str(conf.iterations)]
command += args
command = flatten(command)
output = subprocess.check_output(command, timeout=conf.timeout)
with open(result_file, "ab") as file:
file.write(output)
except TimeoutExpired:
print("Timed out after", conf.timeout, "s, continuing with the next query.")
break
except CalledProcessError as e:
print("Program exited with error")
break
def measure(tools, scenarios, sizes, queries, optional_arguments: dict):
"""Benchmark function.
"""
for tool in tools:
if tool in optional_arguments:
args = optional_arguments[tool]
else:
args = [""]
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in scenarios:
for size in sizes:
for query in queries:
print("Run benchmark: <tool: " + tool +
", scenario: " + scenario +
", query: " + query +
", size: " + str(size) +
(", arg: " + arg if arg != "" else "") +
">")
cmd = ["java", "-Xmx" + java_xmx, "-jar", target,
"-scenario", scenario,
"-query", query,
"-size", str(size),
arg
]
subprocess.call(cmd)
util.set_working_directory("..")
def benchmark(conf):
"""Runs measurements.
"""
header = "../output/header.tsv"
result_file = "../output/output.tsv"
if os.path.exists(result_file):
os.remove(result_file)
shutil.copy(header, result_file)
for change_set in conf.change_sets:
for tool in conf.tools:
for query in conf.queries:
for size in conf.sizes:
target = util.get_tool_jar(tool)
print("Running benchmark: tool = " + tool + ", change set = " + change_set +
", query = " + query + ", size = " + str(size))
try:
output = subprocess.check_output(flatten(
["java", conf.vmargs,
"-jar", target,
"-runs", str(conf.runs),
"-size", str(size),
"-query", query,
"-changeSet", change_set,
"-iterationCount", str(conf.iterations)]), timeout=conf.timeout)
with open(result_file, "ab") as file:
file.write(output)
except TimeoutExpired:
print("Timed out after", conf.timeout, "s, continuing with the next query.")
break
except CalledProcessError as e:
print("Program exited with error")
break
def measure(config):
for tool in config["tools"]:
args = [""]
if tool in config["benchmark_optional_arguments"]:
for optional_argument in config["benchmark_optional_arguments"][tool]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in config["scenarios"]:
for query in config["queries"]:
for size in config["sizes"]:
print("Running benchmark... " +
"runs: " + str(config["runs"]) +
", tool: " + tool +
", scenario: " + scenario +
", query: " + query +
", size: " + str(size) +
(", argument: " + arg if arg != "" else ""))
cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target,
"-runs", str(config["runs"]),
"-scenario", scenario,
"-query", query,
"-size", str(size),
arg]
try:
subprocess.check_output(cmd, timeout=config["timeout"])
except subprocess.TimeoutExpired:
print("Timeout, skipping larger sizes for this tool/scenario/query.")
break
except subprocess.CalledProcessError:
print("An error occured, skipping larger sizes for this tool/scenario/query.")
break
util.set_working_directory("..")
def measure(config):
for scenario in config["scenarios"]:
transformation_arguments = []
# dict only has one item
for (scenario_name, scenario_arguments) in scenario.items():
if scenario_arguments is not None:
for arg, value in scenario_arguments.items():
transformation_arguments.append("-" + arg)
transformation_arguments.append(str(value))
for tool in config["tools"]:
args = [""]
if tool in config["benchmark_optional_arguments"]:
for optional_argument in config["benchmark_optional_arguments"][tool]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for queries in config["queries"]:
for size in config["sizes"]:
# remove all files in the temporary results directory
prev_files = glob.glob('../results/json/*')
for f in prev_files:
os.remove(f)
print("Running benchmark... " +
"runs: " + str(config["runs"]) +
", tool: " + tool +
", scenario: " + scenario_name +
", queries: " + queries +
", size: " + str(size) +
(", argument: " + arg if arg != "" else ""))
cmd = flatten(["java",
config["java_opts"],
"-jar", target,
"-runs", str(config["runs"]),
"-scenario", scenario_name,
"-queries", queries.split(" "),
"-size", str(size),
transformation_arguments,
arg])
try:
subprocess.check_call(cmd, timeout=config["timeout"])
except subprocess.TimeoutExpired:
print("Timeout, skipping larger sizes for this tool/scenario/queries.")
break
except subprocess.CalledProcessError:
print("An error occured, skipping larger sizes for this tool/scenario/queries.")
break
# if the runs were successful, move all files to the results
result_files = glob.glob('../results/json/*')
for f in result_files:
name = os.path.basename(f)
os.rename(f, '../results/completed/' + name)
util.set_working_directory("..")
