def run_main_plugin(self, _compiler_: Compiler, _plugin_: Plugin, _program_: Program):
"""
Runs the given plugin against the given program
:param _compiler_: the compiler used to compile the program
:param _plugin_: the plugin to test
:param _program_: the program to test
"""
_program_.is_installed.wait()
with _program_.lock, TestRunner.EnvManager(_compiler_, "{}-{}".format(_plugin_.name, _program_.name)):
plugin = [
subclass for subclass in get_subclasses(BasePlugin) if subclass.__name__.lower() == _plugin_.name
][0]
if hasattr(_plugin_, "main_plugin"):
analysis_plugins = [plugin]
plugin = _plugin_.main_plugin
else:
analysis_plugins = []
_plugin_.pre_run()
# noinspection PyBroadException
try:
self.assertFalse(run.trigger_bug(_program_.name, plugin(), analysis_plugins=analysis_plugins))
except ProgramNotInstalledException:
raise unittest.SkipTest("{} is not installed".format(_program_.name))
except Exception: # with concurrency, tests might fail. Let's retry once
time.sleep(2) # let's sleep a bit before, timing might be bad
self.assertFalse(run.trigger_bug(_program_.name, plugin(), analysis_plugins=analysis_plugins))
time.sleep(2)
def register_for_trigger(**kwargs) -> None:
"""
Allows each plugin to register to be called during the trigger phases
:param kwargs: keyword arguments to pass to the plugins
"""
for plugin in get_subclasses(BasePlugin):
plugin.register_for_trigger(**kwargs)
def configure(force: bool) -> None:
"""
Calls all configure options for enabled plugins
:param force: True in order to reinstall if it was already installed. If not installed, has no difference
"""
for plugin in get_subclasses(BasePlugin):
plugin().configure(force)
def configure(self, compiler: Compiler) -> None:
"""
Configures the environment to run with the given compiler
:param compiler: the compiler to use
"""
try:
with TestRunner.EnvManager(compiler, "configure"):
self.assertFalse(
dependencies.install(False),
"Could not install dependencies for {} with{} bitcode".format(
compiler.name, "out" if compiler.bitcode else ""
),
)
for plugin_info in compiler.plugins:
log_file = "{}-{}-configuration".format(plugin_info.package, plugin_info.name)
with TestRunner.EnvManager(compiler, log_file):
plugin = [
subclass
for subclass in get_subclasses(BasePlugin)
if subclass.__name__.lower() == plugin_info.name
][0]
self.assertFalse(plugin().configure(force=False))
finally:
compiler.is_configured.set()
def create_executables(*args, **kwargs) -> None:
"""
Allows each enabled plugin to create a special executable if needed
:param args: arguments to pass to plugins
:param kwargs: keyword arguments to pass to plugins
"""
for plugin in get_subclasses(MainPlugin):
plugin().create_executable(*args, **kwargs)
def setup_db(self) -> None:
""" Configures the database with the models subclassing lib.models.Model """
conn = mysql.connector.connect(
user=self.app.config["DATABASE_USER"], password=self.app.config["DATABASE_PASSWORD"],
host=self.app.config["DATABASE_HOST"], port=self.app.config["DATABASE_PORT"], raise_on_warnings=True
)
try:
cursor = conn.cursor()
cursor.execute(
"CREATE DATABASE IF NOT EXISTS {} CHARACTER SET utf8".format(self.app.config["DATABASE_NAME"])
)
conn.commit()
except:
raise
else:
with self.DBManager(self.app) as connection:
for model in sorted(lib.get_subclasses(lib.models.Model), key=lambda x: x.index):
model.setup_table(connection=connection)
finally:
conn.close()
def register_for_trigger(cls, subparser: _SubParsersAction, *args, **kwargs):
"""
Registers for the trigger, add option to specify a number when selecting the plugin
:param subparser: the parser to use
:param args: additional arguments to pass to parents
:param kwargs: additional keyword arguments to pass to parents
"""
for plugin in get_subclasses(MainPlugin):
if plugin == Success:
continue
cls.available_plugins[str(plugin).rsplit(".", 1)[1].rstrip("'>").lower()] = plugin()
parser = super().register_for_trigger(subparser, *args, **kwargs)
parser.add_argument(
"-p", "--plugin", action="append", help="the bug to benchmark. Can be used multiple times",
dest="overhead_plugins", required=True, choices=[plugin for plugin in cls.available_plugins]
)
parser.add_argument(
"-g", "--graph", dest="graph_destination",
help="Generate a overhead report as a graph (you will need matplotlib for this)"
)
def run_main_plugin(self, _compiler_: Compiler, _plugin_: Plugin,
_program_: Program):
"""
Runs the given plugin against the given program
:param _compiler_: the compiler used to compile the program
:param _plugin_: the plugin to test
:param _program_: the program to test
"""
_program_.is_installed.wait()
with _program_.lock, TestRunner.EnvManager(
_compiler_, "{}-{}".format(_plugin_.name, _program_.name)):
plugin = [
subclass for subclass in get_subclasses(BasePlugin)
if subclass.__name__.lower() == _plugin_.name
][0]
if hasattr(_plugin_, "main_plugin"):
analysis_plugins = [plugin]
plugin = _plugin_.main_plugin
else:
analysis_plugins = []
_plugin_.pre_run()
# noinspection PyBroadException
try:
self.assertFalse(
run.trigger_bug(_program_.name,
plugin(),
analysis_plugins=analysis_plugins))
except ProgramNotInstalledException:
raise unittest.SkipTest("{} is not installed".format(
_program_.name))
except Exception: # with concurrency, tests might fail. Let's retry once
time.sleep(2) # let's sleep a bit before, timing might be bad
self.assertFalse(
run.trigger_bug(_program_.name,
plugin(),
analysis_plugins=analysis_plugins))
time.sleep(2)
def configure(self, compiler: Compiler) -> None:
"""
Configures the environment to run with the given compiler
:param compiler: the compiler to use
"""
try:
with TestRunner.EnvManager(compiler, "configure"):
self.assertFalse(
dependencies.install(False),
"Could not install dependencies for {} with{} bitcode".
format(compiler.name, "out" if compiler.bitcode else ""))
for plugin_info in compiler.plugins:
log_file = "{}-{}-configuration".format(
plugin_info.package, plugin_info.name)
with TestRunner.EnvManager(compiler, log_file):
plugin = [
subclass for subclass in get_subclasses(BasePlugin)
if subclass.__name__.lower() == plugin_info.name
][0]
self.assertFalse(plugin().configure(force=False))
finally:
compiler.is_configured.set()
:param _compilers_: the list of compilers to use
"""
for compiler_class in _compilers_:
compiler = compiler_class(RESOURCES_MANAGER.Event())
function_name = "test_1{}_{}_{}_{}wllvm".format(
bound_value(compiler.priority), compiler.package, compiler.name, "no-" if not compiler.bitcode else ""
)
setattr(TestRunner, function_name, lambda x, comp=compiler: TestRunner.configure(x, comp))
setattr(getattr(TestRunner, function_name), "__name__", function_name)
add_programs_compile(compiler)
def clean_working_directory() -> None:
""" Removes old logs before running """
with suppress(FileNotFoundError):
shutil.rmtree(TestRunner.log_directory)
with suppress(FileNotFoundError):
shutil.rmtree(SAVE_DIRECTORY)
os.makedirs(TestRunner.log_directory)
os.makedirs(SAVE_DIRECTORY)
# Add all functions to TestRunner on initialization
clean_working_directory()
load_compilers()
COMPILERS = get_subclasses(Compiler)
add_compilers(COMPILERS)
"test_1{}_{}_{}_{}wllvm".format(
bound_value(compiler.priority),
compiler.package,
compiler.name,
"no-" if not compiler.bitcode else ""
)
setattr(TestRunner,
function_name,
lambda x, comp=compiler: TestRunner.configure(x, comp))
setattr(getattr(TestRunner, function_name), "__name__", function_name)
add_programs_compile(compiler)
def clean_working_directory() -> None:
""" Removes old logs before running """
with suppress(FileNotFoundError):
shutil.rmtree(TestRunner.log_directory)
with suppress(FileNotFoundError):
shutil.rmtree(SAVE_DIRECTORY)
os.makedirs(TestRunner.log_directory)
os.makedirs(SAVE_DIRECTORY)
# Add all functions to TestRunner on initialization
clean_working_directory()
load_compilers()
COMPILERS = get_subclasses(Compiler)
add_compilers(COMPILERS)
