def measure(tools, scenarios, sizes, queries, optional_arguments: dict):
"""Benchmark function.
"""
for tool in tools:
if tool in optional_arguments:
args = optional_arguments[tool]
else:
args = [""]
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in scenarios:
for size in sizes:
for query in queries:
print("Run benchmark: <tool: " + tool +
", scenario: " + scenario +
", query: " + query +
", size: " + str(size) +
(", arg: " + arg if arg != "" else "") +
">")
cmd = ["java", "-Xmx" + java_xmx, "-jar", target,
"-scenario", scenario,
"-query", query,
"-size", str(size),
arg
]
subprocess.call(cmd)
util.set_working_directory("..")
def load(self):
"""
Loads a config.json file and run a validation process.
If the configurations seem valid, returns Configuration object.
"""
util.set_working_directory()
# paths relatively to this script's location
schema_json = util.json_decode(self.schema_path)
if schema_json is None:
msg = "Problem has occurred during the decoding procedure" + \
" with the following file: " + self.schema_path + "."
logging.error(msg)
raise IOError(msg)
tools_json = util.json_decode(self.tools_path)
if tools_json is None:
msg = "Problem has occurred during the decoding procedure" + \
" with the following file: " + self.tools_path + "."
logging.error(msg)
raise IOError(msg)
try:
with open(self.config_path, mode="r") as file:
config_string = file.read()
decoder = json.JSONDecoder(object_pairs_hook=checking_hook)
config_json = decoder.decode(config_string)
except IOError:
msg = "The file does not exist or cannot be read:" + \
(os.path.split(self.config_path))[1]
logging.error(msg)
raise IOError(msg)
except ValueError as value_error:
msg = (os.path.split(self.config_path))[1] + " file is not valid"
logging.error(msg)
print(value_error)
raise ValidationError(msg)
except KeyError as k_error:
msg = "Duplicate key specified."
logging.error(msg)
print(k_error)
print("Modify: " + (os.path.split(self.config_path))[1])
raise ValidationError(msg)
valid = validation.is_valid_json(config_json, schema_json)
if not valid:
msg = "Validation failed for " + \
(os.path.split(self.config_path))[1] + "."
logging.error(msg)
raise ValidationError(msg)
config = Configuration()
config.iterations = config_json["IterationCount"]
config.runs = config_json["Runs"]
config.queries = config_json["Queries"]
config.scenarios = config_json["Scenarios"]
config.sizes = util.get_power_of_two(config_json["MinSize"], config_json["MaxSize"])
config.tools = config_json["Tools"]
config.optional_arguments = config_json["OptionalArguments"]
return config
def generate(config, formats):
for format in formats:
for scenario in config["scenarios"]:
# dict only has one item
for (scenario_name, _) in scenario.items():
pass
args = [""]
if format in config["generator_optional_arguments"]:
for optional_argument in config["generator_optional_arguments"][format]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format)
util.set_working_directory(path)
target = util.get_generator_jar(format)
for size in config["sizes"]:
cmd = flatten(["java",
config["java_opts"],
"-jar", target,
"-scenario", scenario_name,
"-size", str(size),
arg])
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
print("An error occured during model generation, skipping larger sizes for this scenario/format.")
break
util.set_working_directory("..")
def init_log():
"""Initialize logger handlers.
"""
util.set_working_directory()
os.makedirs("log/dist", exist_ok=True)
os.makedirs("log/all", exist_ok=True)
if not os.path.exists("log/all/logs.txt"):
open("log/all/logs.txt", mode="a").close()
# unique logging file
log_file = "log/dist/" + time.strftime("%Y-%m-%d %H:%M:%S") + ".txt"
log_file = log_file.replace(" ", "_")
log_file = log_file.replace(":", "_")
open(log_file, mode="w").close()
logging.basicConfig(filename=log_file,
format="[%(levelname)s] " +
"Module:%(module)s, Message:%(message)s",
level=logging.INFO)
console_handler = logging.StreamHandler()
# common logging file
file_handler = logging.FileHandler("log/all/logs.txt")
file_handler.setLevel(logging.INFO)
console_handler.setLevel(logging.ERROR)
c_formatter = logging.Formatter("[%(levelname)s] Module:%(module)s" +
", Message:%(message)s")
console_handler.setFormatter(c_formatter)
f_formatter = logging.Formatter("%(asctime)s [%(levelname)s] Module:" +
"%(module)s, Message:%(message)s")
file_handler.setFormatter(f_formatter)
logging.getLogger().addHandler(console_handler)
logging.getLogger().addHandler(file_handler)
def measure(tools, scenarios, sizes, queries, optional_arguments: dict):
"""Benchmark function.
"""
for tool in tools:
if tool in optional_arguments:
args = optional_arguments[tool]
else:
args = [""]
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(
TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in scenarios:
for size in sizes:
for query in queries:
print("Run benchmark: <tool: " + tool +
", scenario: " + scenario + ", query: " + query +
", size: " + str(size) +
(", arg: " + arg if arg != "" else "") + ">")
cmd = [
"java", "-Xmx" + java_xmx, "-jar", target,
"-scenario", scenario, "-query", query, "-size",
str(size), arg
]
subprocess.call(cmd)
util.set_working_directory("..")
def visualize():
"""Visualizes the benchmark results
"""
clean_dir("../diagrams")
util.set_working_directory("../reporting")
subprocess.call(["Rscript", "visualize.R", "../config/reporting-1.json"])
subprocess.call(["Rscript", "visualize.R", "../config/reporting-2.json"])
def visualize():
"""Visualizes the benchmark results
"""
clean_dir("../diagrams")
util.set_working_directory("../reporting")
subprocess.call(["Rscript", "visualize.R", "../config/reporting-1.json"])
subprocess.call(["Rscript", "visualize.R", "../config/reporting-2.json"])
def init_log():
"""Initialize logger handlers.
"""
util.set_working_directory()
os.makedirs("log/dist", exist_ok=True)
os.makedirs("log/all", exist_ok=True)
if not os.path.exists("log/all/logs.txt"):
open("log/all/logs.txt", mode="a").close()
# unique logging file
log_file = "log/dist/" + time.strftime("%Y-%m-%d %H:%M:%S") + ".txt"
log_file = log_file.replace(" ", "_")
log_file = log_file.replace(":", "_")
open(log_file, mode="w").close()
logging.basicConfig(filename=log_file, format="[%(levelname)s] " +
"Module:%(module)s, Message:%(message)s",
level=logging.INFO)
console_handler = logging.StreamHandler()
# common logging file
file_handler = logging.FileHandler("log/all/logs.txt")
file_handler.setLevel(logging.INFO)
console_handler.setLevel(logging.ERROR)
c_formatter = logging.Formatter("[%(levelname)s] Module:%(module)s" +
", Message:%(message)s")
console_handler.setFormatter(c_formatter)
f_formatter = logging.Formatter("%(asctime)s [%(levelname)s] Module:" +
"%(module)s, Message:%(message)s")
file_handler.setFormatter(f_formatter)
logging.getLogger().addHandler(console_handler)
logging.getLogger().addHandler(file_handler)
def build(skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
args = ["mvn", "clean", "install"]
if skip_tests:
args.append("-DskipTests")
subprocess.check_call(args)
util.set_working_directory()
def build(skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
if skip_tests:
subprocess.check_call("mvn clean install -DskipTests", shell=True)
else:
subprocess.check_call("mvn clean install", shell=True)
util.set_working_directory()
def build(skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
args = ["mvn", "clean", "install"]
if skip_tests:
args.append("-DskipTests")
subprocess.check_call(args)
util.set_working_directory()
def build(skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
if skip_tests:
subprocess.check_call("mvn clean install -DskipTests", shell=True)
else:
subprocess.check_call("mvn clean install", shell=True)
util.set_working_directory()
def build(conf, skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
args = flatten(["mvn", conf.vmargs, "clean", "install", "--fail-at-end"])
if skip_tests:
args.append("-DskipTests")
subprocess.check_call(args)
util.set_working_directory()
def build(conf, skip_tests):
"""Builds the project.
"""
util.set_working_directory("../")
args = flatten(["mvn", conf.vmargs, "clean", "install", "--fail-at-end"])
if skip_tests:
args.append("-DskipTests")
print(args)
""" angepasst mit shell=True
subprocess.check_call(args, shell=True)
"""
subprocess.check_output(args, stderr=subprocess.STDOUT, shell=True)
util.set_working_directory()
def run_benchmark(configurations):
"""Run the benchmark after the configurations parameter.
Parameters:
@param configurations: a list of Configuration objects
"""
logging.info("benchmark.run_benchmark called.")
util.set_working_directory(configurations[0].common.path)
if not os.path.exists("./results"):
os.mkdir("results")
for config in configurations:
execute(config)
def get_format(self, tool):
"""
Returns the tool's format as string.
"""
# path relatively to this script's location
current_directory = os.getcwd()
util.set_working_directory()
formats_json = util.json_decode(self.dependencies_path)
if formats_json is None:
raise IOError("Problem has occurred during the decoding procedure" +
" with the following file: " + self.dependencies_path)
util.set_working_directory(current_directory)
if tool not in formats_json:
raise IOError("Format for " + tool + " not specified.")
else:
return formats_json[tool]["format"]
def load(self):
"""
Loads a config.json file and run a validation process.
If the configurations seem valid, returns a list with
Configuration objects.
"""
util.set_working_directory()
try:
with open(self.config_path, mode="r") as file:
config_string = file.read()
decoder = json.JSONDecoder(object_pairs_hook=checking_hook)
config_json = decoder.decode(config_string)
except IOError:
msg = "The file does not exist or cannot read:" + \
(os.path.split(self.config_path))[1]
logging.error(msg)
raise IOError(msg)
except ValueError as value_error:
msg = (os.path.split(self.config_path))[1] + " file is not valid"
logging.error(msg)
logging.error(value_error)
raise IOError(msg)
except KeyError as k_error:
msg = "Duplicate key specified."
logging.error(msg)
logging.error(k_error)
logging.error("Modify: " + (os.path.split(self.config_path))[1])
raise IOError(msg)
util.check_validation(config_json)
config = Configuration()
sizes = util.get_power_of_two(config_json["MinSize"],
config_json["MaxSize"])
config.tools = config_json["Tools"]
config.queries = config_json["Queries"]
config.change_sets = config_json["ChangeSets"]
config.sizes = sizes
config.iterations = config_json["IterationCount"]
config.runs = config_json["Runs"]
config.vmargs = config_json["JVM"]["vmargs"]
config.xmx = config_json["JVM"]["Xmx"]
config.timeout = config_json["Timeout"]
config.optional_arguments = config_json["OptionalArguments"]
return config
def load(self):
"""
Loads a config.json file and run a validation process.
If the configurations seem valid, returns a list with
Configuration objects.
"""
util.set_working_directory()
try:
with open(self.config_path, mode="r") as file:
config_string = file.read()
decoder = json.JSONDecoder(object_pairs_hook=checking_hook)
config_json = decoder.decode(config_string)
except IOError:
msg = "The file does not exist or cannot read:" + \
(os.path.split(self.config_path))[1]
logging.error(msg)
raise IOError(msg)
except ValueError as value_error:
msg = (os.path.split(self.config_path))[1] + " file is not valid"
logging.error(msg)
logging.error(value_error)
raise IOError(msg)
except KeyError as k_error:
msg = "Duplicate key specified."
logging.error(msg)
logging.error(k_error)
logging.error("Modify: " + (os.path.split(self.config_path))[1])
raise IOError(msg)
util.check_validation(config_json)
config = Configuration()
sizes = util.get_power_of_two(config_json["MinSize"],
config_json["MaxSize"])
config.tools = config_json["Tools"]
config.queries = config_json["Queries"]
config.change_sets = config_json["ChangeSets"]
config.sizes = sizes
config.iterations = config_json["IterationCount"]
config.runs = config_json["Runs"]
config.vmargs = config_json["JVM"]["vmargs"]
config.timeout = config_json["Timeout"]
config.optional_arguments = config_json["OptionalArguments"]
return config
def generate(formats, scenarios, sizes):
"""
Generates the models after the configurations parameter.
"""
for scenario in scenarios:
for format in formats:
path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format)
util.set_working_directory(path)
target = util.get_generator_jar(format)
for size in sizes:
print("Generate model: <format: " + format +
", scenario: " + scenario +
", size: " + str(size) + ">")
subprocess.call(["java", "-Xmx" + java_xmx,
"-jar", target,
"-scenario", scenario,
"-size", str(size)])
util.set_working_directory("..")
def generate(formats, scenarios, sizes):
"""
Generates the models after the configurations parameter.
"""
for scenario in scenarios:
for format in formats:
path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(
FORMAT=format)
util.set_working_directory(path)
target = util.get_generator_jar(format)
for size in sizes:
print("Generate model: <format: " + format + ", scenario: " +
scenario + ", size: " + str(size) + ">")
subprocess.call([
"java", "-Xmx" + java_xmx, "-jar", target, "-scenario",
scenario, "-size",
str(size)
])
util.set_working_directory("..")
def measure(config):
for tool in config["tools"]:
args = [""]
if tool in config["benchmark_optional_arguments"]:
for optional_argument in config["benchmark_optional_arguments"][tool]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for scenario in config["scenarios"]:
for query in config["queries"]:
for size in config["sizes"]:
print("Running benchmark... " +
"runs: " + str(config["runs"]) +
", tool: " + tool +
", scenario: " + scenario +
", query: " + query +
", size: " + str(size) +
(", argument: " + arg if arg != "" else ""))
cmd = ["java", "-Xmx" + config["java_opts"]["xmx"], "-jar", target,
"-runs", str(config["runs"]),
"-scenario", scenario,
"-query", query,
"-size", str(size),
arg]
try:
subprocess.check_output(cmd, timeout=config["timeout"])
except subprocess.TimeoutExpired:
print("Timeout, skipping larger sizes for this tool/scenario/query.")
break
except subprocess.CalledProcessError:
print("An error occured, skipping larger sizes for this tool/scenario/query.")
break
util.set_working_directory("..")
def build_allegro(package):
""" Installs maven dependencies to the local repository.
"""
current_directory = os.getcwd()
util.set_working_directory()
util.set_working_directory("shell-scripts/")
subprocess.call(["./build_allegro.sh"])
util.set_working_directory(current_directory)
def build_allegro(package):
""" Installs maven dependencies to the local repository.
"""
current_directory = os.getcwd()
util.set_working_directory()
util.set_working_directory("shell-scripts/")
subprocess.call(["./build_allegro.sh"])
util.set_working_directory(current_directory)
def install_neo4j_deps(path):
"""
Clones and builds neo4j-shell-tools and rdf-graph-drivers.
"""
# change back working directory later, so store it now
current_directory = os.getcwd()
# change working directory to this module's location
util.set_working_directory()
# jump to the project parent folder since path can be relative
if os.path.exists("../../neo4j-shell-tools"):
logging.info("Neo4j-shell-tools has been deployed.")
else:
util.set_working_directory("shell-scripts/")
subprocess.call(["./install_neo4j.sh"])
# Set the working directory to this script's folder.
util.set_working_directory(current_directory)
def install_neo4j_deps(path):
"""
Clones and builds neo4j-shell-tools and rdf-graph-drivers.
"""
# change back working directory later, so store it now
current_directory = os.getcwd()
# change working directory to this module's location
util.set_working_directory()
# jump to the project parent folder since path can be relative
if os.path.exists("../../neo4j-shell-tools"):
logging.info("Neo4j-shell-tools has been deployed.")
else:
util.set_working_directory("shell-scripts/")
subprocess.call(["./install_neo4j.sh"])
# Set the working directory to this script's folder.
util.set_working_directory(current_directory)
def extract_results():
"""Extracts the benchmark results
"""
clean_dir("../results")
util.set_working_directory("../reporting")
subprocess.call(["Rscript", "extract_results.R"])
import util
if __name__ == "__main__":
with open("config/config.yml", 'r') as stream:
config = yaml.load(stream)
formats = ["emf", "graph", "rdf", "sql"]
for format in formats:
for query in config["queries"]:
args = [""]
if format in config["generator_optional_arguments"]:
for optional_argument in config["generator_optional_arguments"][format]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.generator.{FORMAT}/".format(FORMAT=format)
util.set_working_directory(path)
target = util.get_generator_jar(format)
cmd = ["java", "-Xmx" + config["java_opts"]["xmx"],
"-jar", target,
"-scenario", "Minimal",
"-query", query,
arg]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
print("An error occured during model generation.")
util.set_working_directory("..")
parser.add_argument("-g",
"--generate",
help="generate models",
action="store_true")
parser.add_argument("-m",
"--measure",
help="run the benchmark",
action="store_true")
parser.add_argument("-s",
"--skip-tests",
help="skip JUnit tests",
action="store_true")
args = parser.parse_args()
# set working directory to this file's path
util.set_working_directory()
loader = Loader()
config = loader.load()
formats = set()
for tool in config.tools:
formats.add(loader.get_format(tool))
java_xmx = "2g"
# if there are no args, execute a full sequence
# with the test and the visualization/reporting
no_args = all(val is False for val in vars(args).values())
if no_args:
args.build = True
args.generate = True
def extract_results():
"""Extracts the benchmark results
"""
clean_dir("../results")
util.set_working_directory("../reporting")
subprocess.call(["Rscript", "extract_results.R"])
parser.add_argument("-b", "--build",
help="build the projects",
action="store_true")
parser.add_argument("-g", "--generate",
help="generate models",
action="store_true")
parser.add_argument("-m", "--measure",
help="run the benchmark",
action="store_true")
parser.add_argument("-s", "--skip-tests",
help="skip JUnit tests",
action="store_true")
args = parser.parse_args()
# set working directory to this file's path
util.set_working_directory()
with open("config/config.yml", 'r') as stream:
config = yaml.load(stream)
config["sizes"] = util.get_power_of_two(config["min_size"], config["max_size"])
with open("config/formats.yml", 'r') as stream:
tool_formats = yaml.load(stream)
formats = set()
for tool in config["tools"]:
formats.add(tool_formats[tool])
# if there are no args, execute a full sequence
# with the test and the visualization/reporting
no_args = all(val is False for val in vars(args).values())
def measure(config):
for scenario in config["scenarios"]:
transformation_arguments = []
# dict only has one item
for (scenario_name, scenario_arguments) in scenario.items():
if scenario_arguments is not None:
for arg, value in scenario_arguments.items():
transformation_arguments.append("-" + arg)
transformation_arguments.append(str(value))
for tool in config["tools"]:
args = [""]
if tool in config["benchmark_optional_arguments"]:
for optional_argument in config["benchmark_optional_arguments"][tool]:
args.append("-" + optional_argument)
for arg in args:
path = "./hu.bme.mit.trainbenchmark.benchmark.{TOOL}/".format(TOOL=tool)
util.set_working_directory(path)
target = util.get_tool_jar(tool)
for queries in config["queries"]:
for size in config["sizes"]:
# remove all files in the temporary results directory
prev_files = glob.glob('../results/json/*')
for f in prev_files:
os.remove(f)
print("Running benchmark... " +
"runs: " + str(config["runs"]) +
", tool: " + tool +
", scenario: " + scenario_name +
", queries: " + queries +
", size: " + str(size) +
(", argument: " + arg if arg != "" else ""))
cmd = flatten(["java",
config["java_opts"],
"-jar", target,
"-runs", str(config["runs"]),
"-scenario", scenario_name,
"-queries", queries.split(" "),
"-size", str(size),
transformation_arguments,
arg])
try:
subprocess.check_call(cmd, timeout=config["timeout"])
except subprocess.TimeoutExpired:
print("Timeout, skipping larger sizes for this tool/scenario/queries.")
break
except subprocess.CalledProcessError:
print("An error occured, skipping larger sizes for this tool/scenario/queries.")
break
# if the runs were successful, move all files to the results
result_files = glob.glob('../results/json/*')
for f in result_files:
name = os.path.basename(f)
os.rename(f, '../results/completed/' + name)
util.set_working_directory("..")
