def measure(tools, scenarios, sizes, queries, optional_arguments: dict): """Benchmark function. """ for tool in tools: if tool in optional_arguments: args = optional_arguments[tool] else: args = [""] for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in scenarios: for size in sizes: for query in queries: print("Run benchmark: <tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", arg: " + arg if arg != "" else "") + ">") cmd = ["java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-query", query, "-size", str(size), arg ] subprocess.call(cmd) util.set_working_directory("..")
def load(self): """ Loads a config.json file and run a validation process. If the configurations seem valid, returns Configuration object. """ util.set_working_directory() # paths relatively to this script's location schema_json = util.json_decode(self.schema_path) if schema_json is None: msg = "Problem has occurred during the decoding procedure" + \ " with the following file: " + self.schema_path + "." logging.error(msg) raise IOError(msg) tools_json = util.json_decode(self.tools_path) if tools_json is None: msg = "Problem has occurred during the decoding procedure" + \ " with the following file: " + self.tools_path + "." logging.error(msg) raise IOError(msg) try: with open(self.config_path, mode="r") as file: config_string = file.read() decoder = json.JSONDecoder(object_pairs_hook=checking_hook) config_json = decoder.decode(config_string) except IOError: msg = "The file does not exist or cannot be read:" + \ (os.path.split(self.config_path))[1] logging.error(msg) raise IOError(msg) except ValueError as value_error: msg = (os.path.split(self.config_path))[1] + " file is not valid" logging.error(msg) print(value_error) raise ValidationError(msg) except KeyError as k_error: msg = "Duplicate key specified." logging.error(msg) print(k_error) print("Modify: " + (os.path.split(self.config_path))[1]) raise ValidationError(msg) valid = validation.is_valid_json(config_json, schema_json) if not valid: msg = "Validation failed for " + \ (os.path.split(self.config_path))[1] + "." logging.error(msg) raise ValidationError(msg) config = Configuration() config.iterations = config_json["IterationCount"] config.runs = config_json["Runs"] config.queries = config_json["Queries"] config.scenarios = config_json["Scenarios"] config.sizes = util.get_power_of_two(config_json["MinSize"], config_json["MaxSize"]) config.tools = config_json["Tools"] config.optional_arguments = config_json["OptionalArguments"] return config
def generate(config, formats): for format in formats: for scenario in config["scenarios"]: # dict only has one item for (scenario_name, _) in scenario.items(): pass args = [""] if format in config["generator_optional_arguments"]: for optional_argument in config["generator_optional_arguments"][format]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format) util.set_working_directory(path) target = util.get_generator_jar(format) for size in config["sizes"]: cmd = flatten(["java", config["java_opts"], "-jar", target, "-scenario", scenario_name, "-size", str(size), arg]) try: subprocess.check_call(cmd) except subprocess.CalledProcessError: print("An error occured during model generation, skipping larger sizes for this scenario/format.") break util.set_working_directory("..")
def init_log(): """Initialize logger handlers. """ util.set_working_directory() os.makedirs("log/dist", exist_ok=True) os.makedirs("log/all", exist_ok=True) if not os.path.exists("log/all/logs.txt"): open("log/all/logs.txt", mode="a").close() # unique logging file log_file = "log/dist/" + time.strftime("%Y-%m-%d %H:%M:%S") + ".txt" log_file = log_file.replace(" ", "_") log_file = log_file.replace(":", "_") open(log_file, mode="w").close() logging.basicConfig(filename=log_file, format="[%(levelname)s] " + "Module:%(module)s, Message:%(message)s", level=logging.INFO) console_handler = logging.StreamHandler() # common logging file file_handler = logging.FileHandler("log/all/logs.txt") file_handler.setLevel(logging.INFO) console_handler.setLevel(logging.ERROR) c_formatter = logging.Formatter("[%(levelname)s] Module:%(module)s" + ", Message:%(message)s") console_handler.setFormatter(c_formatter) f_formatter = logging.Formatter("%(asctime)s [%(levelname)s] Module:" + "%(module)s, Message:%(message)s") file_handler.setFormatter(f_formatter) logging.getLogger().addHandler(console_handler) logging.getLogger().addHandler(file_handler)
def measure(tools, scenarios, sizes, queries, optional_arguments: dict): """Benchmark function. """ for tool in tools: if tool in optional_arguments: args = optional_arguments[tool] else: args = [""] for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format( TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in scenarios: for size in sizes: for query in queries: print("Run benchmark: <tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", arg: " + arg if arg != "" else "") + ">") cmd = [ "java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-query", query, "-size", str(size), arg ] subprocess.call(cmd) util.set_working_directory("..")
def visualize(): """Visualizes the benchmark results """ clean_dir("../diagrams") util.set_working_directory("../reporting") subprocess.call(["Rscript", "visualize.R", "../config/reporting-1.json"]) subprocess.call(["Rscript", "visualize.R", "../config/reporting-2.json"])
def build(skip_tests): """Builds the project. """ util.set_working_directory("../") args = ["mvn", "clean", "install"] if skip_tests: args.append("-DskipTests") subprocess.check_call(args) util.set_working_directory()
def build(skip_tests): """Builds the project. """ util.set_working_directory("../") if skip_tests: subprocess.check_call("mvn clean install -DskipTests", shell=True) else: subprocess.check_call("mvn clean install", shell=True) util.set_working_directory()
def build(conf, skip_tests): """Builds the project. """ util.set_working_directory("../") args = flatten(["mvn", conf.vmargs, "clean", "install", "--fail-at-end"]) if skip_tests: args.append("-DskipTests") subprocess.check_call(args) util.set_working_directory()
def build(conf, skip_tests): """Builds the project. """ util.set_working_directory("../") args = flatten(["mvn", conf.vmargs, "clean", "install", "--fail-at-end"]) if skip_tests: args.append("-DskipTests") print(args) """ angepasst mit shell=True subprocess.check_call(args, shell=True) """ subprocess.check_output(args, stderr=subprocess.STDOUT, shell=True) util.set_working_directory()
def run_benchmark(configurations): """Run the benchmark after the configurations parameter. Parameters: @param configurations: a list of Configuration objects """ logging.info("benchmark.run_benchmark called.") util.set_working_directory(configurations[0].common.path) if not os.path.exists("./results"): os.mkdir("results") for config in configurations: execute(config)
def get_format(self, tool): """ Returns the tool's format as string. """ # path relatively to this script's location current_directory = os.getcwd() util.set_working_directory() formats_json = util.json_decode(self.dependencies_path) if formats_json is None: raise IOError("Problem has occurred during the decoding procedure" + " with the following file: " + self.dependencies_path) util.set_working_directory(current_directory) if tool not in formats_json: raise IOError("Format for " + tool + " not specified.") else: return formats_json[tool]["format"]
def load(self): """ Loads a config.json file and run a validation process. If the configurations seem valid, returns a list with Configuration objects. """ util.set_working_directory() try: with open(self.config_path, mode="r") as file: config_string = file.read() decoder = json.JSONDecoder(object_pairs_hook=checking_hook) config_json = decoder.decode(config_string) except IOError: msg = "The file does not exist or cannot read:" + \ (os.path.split(self.config_path))[1] logging.error(msg) raise IOError(msg) except ValueError as value_error: msg = (os.path.split(self.config_path))[1] + " file is not valid" logging.error(msg) logging.error(value_error) raise IOError(msg) except KeyError as k_error: msg = "Duplicate key specified." logging.error(msg) logging.error(k_error) logging.error("Modify: " + (os.path.split(self.config_path))[1]) raise IOError(msg) util.check_validation(config_json) config = Configuration() sizes = util.get_power_of_two(config_json["MinSize"], config_json["MaxSize"]) config.tools = config_json["Tools"] config.queries = config_json["Queries"] config.change_sets = config_json["ChangeSets"] config.sizes = sizes config.iterations = config_json["IterationCount"] config.runs = config_json["Runs"] config.vmargs = config_json["JVM"]["vmargs"] config.xmx = config_json["JVM"]["Xmx"] config.timeout = config_json["Timeout"] config.optional_arguments = config_json["OptionalArguments"] return config
def load(self): """ Loads a config.json file and run a validation process. If the configurations seem valid, returns a list with Configuration objects. """ util.set_working_directory() try: with open(self.config_path, mode="r") as file: config_string = file.read() decoder = json.JSONDecoder(object_pairs_hook=checking_hook) config_json = decoder.decode(config_string) except IOError: msg = "The file does not exist or cannot read:" + \ (os.path.split(self.config_path))[1] logging.error(msg) raise IOError(msg) except ValueError as value_error: msg = (os.path.split(self.config_path))[1] + " file is not valid" logging.error(msg) logging.error(value_error) raise IOError(msg) except KeyError as k_error: msg = "Duplicate key specified." logging.error(msg) logging.error(k_error) logging.error("Modify: " + (os.path.split(self.config_path))[1]) raise IOError(msg) util.check_validation(config_json) config = Configuration() sizes = util.get_power_of_two(config_json["MinSize"], config_json["MaxSize"]) config.tools = config_json["Tools"] config.queries = config_json["Queries"] config.change_sets = config_json["ChangeSets"] config.sizes = sizes config.iterations = config_json["IterationCount"] config.runs = config_json["Runs"] config.vmargs = config_json["JVM"]["vmargs"] config.timeout = config_json["Timeout"] config.optional_arguments = config_json["OptionalArguments"] return config
def generate(formats, scenarios, sizes): """ Generates the models after the configurations parameter. """ for scenario in scenarios: for format in formats: path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format) util.set_working_directory(path) target = util.get_generator_jar(format) for size in sizes: print("Generate model: <format: " + format + ", scenario: " + scenario + ", size: " + str(size) + ">") subprocess.call(["java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-size", str(size)]) util.set_working_directory("..")
def generate(formats, scenarios, sizes): """ Generates the models after the configurations parameter. """ for scenario in scenarios: for format in formats: path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format( FORMAT=format) util.set_working_directory(path) target = util.get_generator_jar(format) for size in sizes: print("Generate model: <format: " + format + ", scenario: " + scenario + ", size: " + str(size) + ">") subprocess.call([ "java", "-Xmx" + java_xmx, "-jar", target, "-scenario", scenario, "-size", str(size) ]) util.set_working_directory("..")
def measure(config): for tool in config["tools"]: args = [""] if tool in config["benchmark_optional_arguments"]: for optional_argument in config["benchmark_optional_arguments"][tool]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for scenario in config["scenarios"]: for query in config["queries"]: for size in config["sizes"]: print("Running benchmark... " + "runs: " + str(config["runs"]) + ", tool: " + tool + ", scenario: " + scenario + ", query: " + query + ", size: " + str(size) + (", argument: " + arg if arg != "" else "")) cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target, "-runs", str(config["runs"]), "-scenario", scenario, "-query", query, "-size", str(size), arg] try: subprocess.check_output(cmd, timeout=config["timeout"]) except subprocess.TimeoutExpired: print("Timeout, skipping larger sizes for this tool/scenario/query.") break except subprocess.CalledProcessError: print("An error occured, skipping larger sizes for this tool/scenario/query.") break util.set_working_directory("..")
def build_allegro(package): """ Installs maven dependencies to the local repository. """ current_directory = os.getcwd() util.set_working_directory() util.set_working_directory("shell-scripts/") subprocess.call(["./build_allegro.sh"]) util.set_working_directory(current_directory)
def install_neo4j_deps(path): """ Clones and builds neo4j-shell-tools and rdf-graph-drivers. """ # change back working directory later, so store it now current_directory = os.getcwd() # change working directory to this module's location util.set_working_directory() # jump to the project parent folder since path can be relative if os.path.exists("../../neo4j-shell-tools"): logging.info("Neo4j-shell-tools has been deployed.") else: util.set_working_directory("shell-scripts/") subprocess.call(["./install_neo4j.sh"]) # Set the working directory to this script's folder. util.set_working_directory(current_directory)
def extract_results(): """Extracts the benchmark results """ clean_dir("../results") util.set_working_directory("../reporting") subprocess.call(["Rscript", "extract_results.R"])
import util if __name__ == "__main__": with open("config/config.yml", 'r') as stream: config = yaml.load(stream) formats = ["emf", "graph", "rdf", "sql"] for format in formats: for query in config["queries"]: args = [""] if format in config["generator_optional_arguments"]: for optional_argument in config["generator_optional_arguments"][format]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format) util.set_working_directory(path) target = util.get_generator_jar(format) cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target, "-scenario", "Minimal", "-query", query, arg] try: subprocess.check_call(cmd) except subprocess.CalledProcessError: print("An error occured during model generation.") util.set_working_directory("..")
parser.add_argument("-g", "--generate", help="generate models", action="store_true") parser.add_argument("-m", "--measure", help="run the benchmark", action="store_true") parser.add_argument("-s", "--skip-tests", help="skip JUnit tests", action="store_true") args = parser.parse_args() # set working directory to this file's path util.set_working_directory() loader = Loader() config = loader.load() formats = set() for tool in config.tools: formats.add(loader.get_format(tool)) java_xmx = "2g" # if there are no args, execute a full sequence # with the test and the visualization/reporting no_args = all(val is False for val in vars(args).values()) if no_args: args.build = True args.generate = True
parser.add_argument("-b", "--build", help="build the projects", action="store_true") parser.add_argument("-g", "--generate", help="generate models", action="store_true") parser.add_argument("-m", "--measure", help="run the benchmark", action="store_true") parser.add_argument("-s", "--skip-tests", help="skip JUnit tests", action="store_true") args = parser.parse_args() # set working directory to this file's path util.set_working_directory() with open("config/config.yml", 'r') as stream: config = yaml.load(stream) config["sizes"] = util.get_power_of_two(config["min_size"], config["max_size"]) with open("config/formats.yml", 'r') as stream: tool_formats = yaml.load(stream) formats = set() for tool in config["tools"]: formats.add(tool_formats[tool]) # if there are no args, execute a full sequence # with the test and the visualization/reporting no_args = all(val is False for val in vars(args).values())
def measure(config): for scenario in config["scenarios"]: transformation_arguments = [] # dict only has one item for (scenario_name, scenario_arguments) in scenario.items(): if scenario_arguments is not None: for arg, value in scenario_arguments.items(): transformation_arguments.append("-" + arg) transformation_arguments.append(str(value)) for tool in config["tools"]: args = [""] if tool in config["benchmark_optional_arguments"]: for optional_argument in config["benchmark_optional_arguments"][tool]: args.append("-" + optional_argument) for arg in args: path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool) util.set_working_directory(path) target = util.get_tool_jar(tool) for queries in config["queries"]: for size in config["sizes"]: # remove all files in the temporary results directory prev_files = glob.glob('../results/json/*') for f in prev_files: os.remove(f) print("Running benchmark... " + "runs: " + str(config["runs"]) + ", tool: " + tool + ", scenario: " + scenario_name + ", queries: " + queries + ", size: " + str(size) + (", argument: " + arg if arg != "" else "")) cmd = flatten(["java", config["java_opts"], "-jar", target, "-runs", str(config["runs"]), "-scenario", scenario_name, "-queries", queries.split(" "), "-size", str(size), transformation_arguments, arg]) try: subprocess.check_call(cmd, timeout=config["timeout"]) except subprocess.TimeoutExpired: print("Timeout, skipping larger sizes for this tool/scenario/queries.") break except subprocess.CalledProcessError: print("An error occured, skipping larger sizes for this tool/scenario/queries.") break # if the runs were successful, move all files to the results result_files = glob.glob('../results/json/*') for f in result_files: name = os.path.basename(f) os.rename(f, '../results/completed/' + name) util.set_working_directory("..")