def measure(tools, scenarios, sizes, queries, optional_arguments: dict): """Benchmark function. """ for tool in tools: if tool in optional_arguments: args = optional_arguments[tool] else: args = [""] for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format( TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in scenarios: for size in sizes: for query in queries: print("Run benchmark: <tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", arg: " + arg if arg != "" else "") + ">") cmd = [ "java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-query", query, "-size", str(size), arg ] subprocess.call(cmd) util.set_working_directory("..")
def benchmark(conf): """Runs measurements. """ header = "../output/header.tsv" result_file = "../output/output.tsv" if os.path.exists(result_file): os.remove(result_file) shutil.copy(header, result_file) for change_set in conf.change_sets: for tool in conf.tools: for query in conf.queries: for args in conf.optional_arguments: for size in conf.sizes: target = util.get_tool_jar(tool) print("Running benchmark: tool = " + tool + ", change set = " + change_set + ", query = " + query + ", size = " + str(size) + ", extra arguments = " + str(args)) try: command = ["java", conf.vmargs, "-jar", target, "-runs", str(conf.runs), "-size", str(size), "-query", query, "-changeSet", change_set, "-iterationCount", str(conf.iterations)] command += args command = flatten(command) output = subprocess.check_output(command, timeout=conf.timeout) with open(result_file, "ab") as file: file.write(output) except TimeoutExpired: print("Timed out after", conf.timeout, "s, continuing with the next query.") break except CalledProcessError as e: print("Program exited with error") break
def measure(tools, scenarios, sizes, queries, optional_arguments: dict): """Benchmark function. """ for tool in tools: if tool in optional_arguments: args = optional_arguments[tool] else: args = [""] for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in scenarios: for size in sizes: for query in queries: print("Run benchmark: <tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", arg: " + arg if arg != "" else "") + ">") cmd = ["java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-query", query, "-size", str(size), arg ] subprocess.call(cmd) util.set_working_directory("..")
def benchmark(conf): """Runs measurements. """ header = "../output/header.tsv" result_file = "../output/output.tsv" if os.path.exists(result_file): os.remove(result_file) shutil.copy(header, result_file) for change_set in conf.change_sets: for tool in conf.tools: for query in conf.queries: for size in conf.sizes: target = util.get_tool_jar(tool) print("Running benchmark: tool = " + tool + ", change set = " + change_set + ", query = " + query + ", size = " + str(size)) try: output = subprocess.check_output(flatten( ["java", conf.vmargs, "-jar", target, "-runs", str(conf.runs), "-size", str(size), "-query", query, "-changeSet", change_set, "-iterationCount", str(conf.iterations)]), timeout=conf.timeout) with open(result_file, "ab") as file: file.write(output) except TimeoutExpired: print("Timed out after", conf.timeout, "s, continuing with the next query.") break except CalledProcessError as e: print("Program exited with error") break
def measure(config): for tool in config["tools"]: args = [""] if tool in config["benchmark_optional_arguments"]: for optional_argument in config["benchmark_optional_arguments"][tool]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in config["scenarios"]: for query in config["queries"]: for size in config["sizes"]: print("Running benchmark... " + "runs: " + str(config["runs"]) + ", tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", argument: " + arg if arg != "" else "")) cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target, "-runs", str(config["runs"]), "-scenario", scenario, "-query", query, "-size", str(size), arg] try: subprocess.check_output(cmd, timeout=config["timeout"]) except subprocess.TimeoutExpired: print("Timeout, skipping larger sizes for this tool/scenario/query.") break except subprocess.CalledProcessError: print("An error occured, skipping larger sizes for this tool/scenario/query.") break util.set_working_directory("..")
def measure(config): for scenario in config["scenarios"]: transformation_arguments = [] # dict only has one item for (scenario_name, scenario_arguments) in scenario.items(): if scenario_arguments is not None: for arg, value in scenario_arguments.items(): transformation_arguments.append("-" + arg) transformation_arguments.append(str(value)) for tool in config["tools"]: args = [""] if tool in config["benchmark_optional_arguments"]: for optional_argument in config["benchmark_optional_arguments"][tool]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for queries in config["queries"]: for size in config["sizes"]: # remove all files in the temporary results directory prev_files = glob.glob('../results/json/*') for f in prev_files: os.remove(f) print("Running benchmark... " + "runs: " + str(config["runs"]) + ", tool: " + tool + ", scenario: " + scenario_name + ", queries: " + queries + ", size: " + str(size) + (", argument: " + arg if arg != "" else "")) cmd = flatten(["java", config["java_opts"], "-jar", target, "-runs", str(config["runs"]), "-scenario", scenario_name, "-queries", queries.split(" "), "-size", str(size), transformation_arguments, arg]) try: subprocess.check_call(cmd, timeout=config["timeout"]) except subprocess.TimeoutExpired: print("Timeout, skipping larger sizes for this tool/scenario/queries.") break except subprocess.CalledProcessError: print("An error occured, skipping larger sizes for this tool/scenario/queries.") break # if the runs were successful, move all files to the results result_files = glob.glob('../results/json/*') for f in result_files: name = os.path.basename(f) os.rename(f, '../results/completed/' + name) util.set_working_directory("..")