def retry(self, *args, run_number: int = 0, **kwargs) -> int:
"""
Updates the number of time the program has run and relaunches it
:param args: additional arguments
:param run_number: the number of time the benchmark has run
:param kwargs: additional keyword arguments
:return: 0|1|None on success|failure|unexpected result
"""
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
with suppress(FileNotFoundError), \
open(os.path.join(self.trigger.conf.getdir("install", "install_directory"))) as httpd_pid:
pid = int(httpd_pid.read())
launch_and_log(["kill", str(pid)])
run_number += 1
if run_number > self.maximum_tries:
return 1
logging.warning("An error occurred while launching apache, retrying")
self.trigger.clean_logs()
return self.run(*args, run_number=run_number, **kwargs)
def run(self):
"""
Launches the command and waits for the output
"""
with suppress(subprocess.CalledProcessError):
launch_and_log(self.command.split(" "),
preexec_fn=TriggerWithHelper.__preexec_fn__)
def configure(self) -> None:
"""
Configures the sources
"""
if self.conf["configure"] == "configure":
cmd = [
os.path.join(self.sources_dir, "configure"),
"--prefix={}".format(self.install_dir)
]
elif self.conf["configure"] == "cmake":
cmd = ["cmake", self.sources_dir]
else:
logging.verbose("{} does not need configuration".format(
self.conf["display_name"]))
return
cmd += self.conf.getlist("configure_args", [])
logging.info("Configuring %(name)s",
dict(name=self.conf["display_name"]))
self.env["WLLVM_CONFIGURE_ONLY"] = "1"
helper.launch_and_log(cmd,
cwd=self.working_dir,
env=self.env,
error_msg="Configuration failed")
del self.env["WLLVM_CONFIGURE_ONLY"]
def patch(self,
patches: list,
directory: str,
reverse: bool = False,
patches_path=None) -> None:
"""
Applies different patches to the sources or the installed files
:param patches: list of patches to apply
:param directory: the top directory where to apply these patches
:param reverse: if the patch is to be reversed
:param patches_path: the path where to find the patches. If not set, will use data/program_name/patches
"""
if not patches:
return
for _patch in patches:
logging.verbose("Applying {}patch {}".format(
"Reverse " if reverse else "", _patch))
cmd = [
"patch", "-p1", "-i",
os.path.join(patches_path or self.patches_path, _patch)
]
if reverse:
cmd.insert(2, "-R")
helper.launch_and_log(cmd,
cwd=directory,
error_msg="A patch failed to apply")
def run(self, *args, **kwargs) -> int:
"""
Benchmarks the execution time of 20 runs and stores the last 10 results (to avoid side effects) in
self.trigger.result.
Runs at most 100 times before deciding the run is a failure.
:param args: additional arguments
:param kwargs: additional keyword arguments
:return: 0|1 on success|failure
"""
results = []
tries = 0
while len(results
) < self.expected_results and tries < self.maximum_tries:
tries += 1
try:
proc_start = self.trigger.Server(self.trigger.cmd)
proc_start.start()
time.sleep(self.trigger.delay)
results_queue = multiprocessing.Queue() # pylint: disable=no-member
self.triggers = []
for command in self.trigger.helper_commands:
self.triggers.append(
self.trigger.helper(command,
results=results_queue,
**self.trigger.named_helper_args))
result = timeit.repeat(self.client_run, number=1, repeat=1)
finally:
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
for thread in self.triggers:
thread.terminate()
values = []
for _ in self.triggers:
values.append(results_queue.get_nowait())
if self.trigger.check_success(values) != 0:
logging.warning("Trigger did not work, retrying")
continue
results += result
show_progress(len(results),
self.expected_results,
section="trigger")
time.sleep(2)
if tries >= 100:
return 1
logging.verbose("Run times : {} secs".format(results))
self.trigger.returned_information = results[self.expected_results -
self.kept_runs:]
return 0
def retry(self, *args, run_number: int=0, **kwargs) -> int:
"""
Updates the number of time the program has run and relaunches it
:param args: additional arguments
:param run_number: the number of time the benchmark has run
:param kwargs: additional keyword arguments
:return: 0|1|None on success|failure|unexpected result
"""
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
with suppress(FileNotFoundError), \
open(os.path.join(self.trigger.conf.getdir("install", "install_directory"))) as httpd_pid:
pid = int(httpd_pid.read())
launch_and_log(["kill", str(pid)])
run_number += 1
if run_number > self.maximum_tries:
return 1
logging.warning("An error occurred while launching apache, retrying")
self.trigger.clean_logs()
return self.run(*args, run_number=run_number, **kwargs)
def make(self) -> None:
"""
runs 'make'
"""
logging.info("Compiling %(name)s", dict(name=self.conf["display_name"]))
cmd = ["make"] + get_global_conf().getlist("install", "make_args")
if self.conf.getlist("make_args", None):
cmd += self.conf.getlist("make_args")
helper.launch_and_log(cmd, cwd=self.working_dir, env=self.env, error_msg="Compilation failed")
def run(self, *args, **kwargs) -> int:
"""
Benchmarks the execution time of 20 runs and stores the last 10 results (to avoid side effects) in
self.trigger.result.
Runs at most 100 times before deciding the run is a failure.
:param args: additional arguments
:param kwargs: additional keyword arguments
:return: 0|1 on success|failure
"""
results = []
tries = 0
while len(results) < self.expected_results and tries < self.maximum_tries:
tries += 1
try:
proc_start = self.trigger.Server(self.trigger.cmd)
proc_start.start()
time.sleep(self.trigger.delay)
results_queue = multiprocessing.Queue() # pylint: disable=no-member
self.triggers = []
for command in self.trigger.helper_commands:
self.triggers.append(
self.trigger.helper(command, results=results_queue, **self.trigger.named_helper_args)
)
result = timeit.repeat(self.client_run, number=1, repeat=1)
finally:
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
for thread in self.triggers:
thread.terminate()
values = []
for _ in self.triggers:
values.append(results_queue.get_nowait())
if self.trigger.check_success(values) != 0:
logging.warning("Trigger did not work, retrying")
continue
results += result
show_progress(len(results), self.expected_results, section="trigger")
time.sleep(2)
if tries >= 100:
return 1
logging.verbose("Run times : {} secs".format(results))
self.trigger.returned_information = results[self.expected_results - self.kept_runs:]
return 0
def extract_bitcode(self) -> None:
"""
Extracts and copies the bitcode file to the bin directory
"""
logging.info("Copying bitcode file")
source = os.path.join(self.working_dir, self.conf["bitcode_file"].lstrip("/"))
cmd = "{}/wllvm/extract-bc {}".format(get_global_conf().getdir("utilities", "install_directory"), source)
helper.launch_and_log(cmd.split(" "), env=self.env, error_msg="Bitcode extraction failed")
shutil.copy(source + ".bc", self.install_dir + "/bin/")
def install(self) -> None:
"""
Runs 'make install'
"""
logging.info("Installing %(name)s", dict(name=self.conf["display_name"]))
if self.conf.get("install", None) == "copy":
shutil.copytree(self.sources_dir, self.install_dir)
else:
helper.launch_and_log(
["make", "install"], cwd=self.working_dir, env=self.env, error_msg="Installation failed"
)
def make(self) -> None:
"""
runs 'make'
"""
logging.info("Compiling %(name)s",
dict(name=self.conf["display_name"]))
cmd = ["make"] + get_global_conf().getlist("install", "make_args")
if self.conf.getlist("make_args", None):
cmd += self.conf.getlist("make_args")
helper.launch_and_log(cmd,
cwd=self.working_dir,
env=self.env,
error_msg="Compilation failed")
def install_python_modules() -> None:
"""
Install necessary python modules for the scripts and aesthetics
:raise subprocess.CalledProcessError
"""
logging.verbose("Installing python dependencies")
requirements = os.path.join(constants.CONF_PATH, "requirements.pip")
cmd = ["pip3", "install", "-r", requirements]
# pylint: disable=no-member
if (not (hasattr(sys, 'real_prefix') and sys.prefix != sys.real_prefix)) and (sys.prefix == sys.base_prefix):
# we are not in a virtualenv. Let's install the packages as user
cmd.insert(2, "--user")
launch_and_log(cmd, error_msg="Could not install python module")
def install(self) -> None:
"""
Runs 'make install'
"""
logging.info("Installing %(name)s",
dict(name=self.conf["display_name"]))
if self.conf.get("install", None) == "copy":
shutil.copytree(self.sources_dir, self.install_dir)
else:
helper.launch_and_log(["make", "install"],
cwd=self.working_dir,
env=self.env,
error_msg="Installation failed")
def run(self, *args, run_number: int = 0, **kwargs) -> int:
"""
Benchmarks the number of requests per second an apache server can handle
Runs at most 100 times before deciding the run is a failure
:param args: additional arguments
:param run_number: the number of time the benchmark has run
:param kwargs: additional keyword arguments
:return: 0|1|None on success|failure|unexpected result
"""
proc_start = self.trigger.Server(self.trigger.cmd)
proc_start.start()
time.sleep(self.trigger.delay)
cmd = "ab -n 30000 -c 1 {}".format(
self.trigger.benchmark_url).split(" ")
logging.verbose(cmd)
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
**kwargs)
except subprocess.CalledProcessError as exc:
for line in exc.output.decode().split("\n"):
logging.debug(line)
return self.retry(*args, run_number=run_number, **kwargs)
else:
success = self.trigger.check_success()
if success:
return self.retry(*args, run_number=run_number, **kwargs)
self.trigger.result = []
for line in output.decode().split("\n"):
if line.startswith("Requests per second:"):
self.trigger.returned_information = [
float(line.split(":")[1].strip().split(" ")[0])
]
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
if len(self.trigger.returned_information) == 0:
return self.retry(*args, run_number=run_number, **kwargs)
logging.verbose("Requests per second : {}".format(
self.trigger.returned_information[0]))
return success
def run(self) -> int:
"""
Runs the cmd program in a subprocess, with rlimit set and checks the output to be sure that it is the correct
bug
:return: 0|1|None on success|failure|unexpected result
"""
logging.verbose(self.cmd)
error_code = 0
try:
# noinspection PyTypeChecker
launch_and_log(self.cmd, shell=True, preexec_fn=self.__preexec_fn__)
except subprocess.CalledProcessError as exc:
error_code = exc.returncode
return self.check_success(error_code=error_code)
def extract_bitcode(self) -> None:
"""
Extracts and copies the bitcode file to the bin directory
"""
logging.info("Copying bitcode file")
source = os.path.join(self.working_dir,
self.conf["bitcode_file"].lstrip("/"))
cmd = "{}/wllvm/extract-bc {}".format(
get_global_conf().getdir("utilities", "install_directory"), source)
helper.launch_and_log(cmd.split(" "),
env=self.env,
error_msg="Bitcode extraction failed")
shutil.copy(source + ".bc", self.install_dir + "/bin/")
def run(self) -> int:
"""
Runs the cmd program in a subprocess, with rlimit set and checks the output to be sure that it is the correct
bug
:return: 0|1|None on success|failure|unexpected result
"""
logging.verbose(self.cmd)
error_code = 0
try:
# noinspection PyTypeChecker
launch_and_log(self.cmd,
shell=True,
preexec_fn=self.__preexec_fn__)
except subprocess.CalledProcessError as exc:
error_code = exc.returncode
return self.check_success(error_code=error_code)
def patch(self, patches: list, directory: str, reverse: bool=False, patches_path=None) -> None:
"""
Applies different patches to the sources or the installed files
:param patches: list of patches to apply
:param directory: the top directory where to apply these patches
:param reverse: if the patch is to be reversed
:param patches_path: the path where to find the patches. If not set, will use data/program_name/patches
"""
if not patches:
return
for _patch in patches:
logging.verbose("Applying {}patch {}".format("Reverse " if reverse else "", _patch))
cmd = ["patch", "-p1", "-i", os.path.join(patches_path or self.patches_path, _patch)]
if reverse:
cmd.insert(2, "-R")
helper.launch_and_log(cmd, cwd=directory, error_msg="A patch failed to apply")
def run(self) -> int:
"""
Main function. Calls every other one in order to make the bug trigger
:return: 0|1|None on success|failure|unexpected event
"""
try:
logging.verbose(self.cmd)
proc_start = self.Server(
self.cmd
) # this is not a typo. Using cmd is REQUIRED for the sake of plugins
proc_start.start()
time.sleep(self.delay)
triggers = []
results_queue = multiprocessing.Queue() # pylint: disable=no-member
for command in self.helper_commands:
# noinspection PyCallingNonCallable
triggers.append(
self.helper(command,
results=results_queue,
**self.named_helper_args))
for thread in triggers:
thread.start()
for thread in triggers:
thread.join(self.timeout)
for thread in triggers:
thread.terminate()
finally:
with suppress(subprocess.CalledProcessError):
launch_and_log(self.stop_cmd.split(" "))
results = []
for _ in triggers:
with suppress(queue.Empty):
results.append(results_queue.get_nowait())
time.sleep(self.delay)
return self.check_success(results=results)
def run(self, *args, run_number: int=0, **kwargs) -> int:
"""
Benchmarks the number of requests per second an apache server can handle
Runs at most 100 times before deciding the run is a failure
:param args: additional arguments
:param run_number: the number of time the benchmark has run
:param kwargs: additional keyword arguments
:return: 0|1|None on success|failure|unexpected result
"""
proc_start = self.trigger.Server(self.trigger.cmd)
proc_start.start()
time.sleep(self.trigger.delay)
cmd = "ab -n 30000 -c 1 {}".format(self.trigger.benchmark_url).split(" ")
logging.verbose(cmd)
try:
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, **kwargs)
except subprocess.CalledProcessError as exc:
for line in exc.output.decode().split("\n"):
logging.debug(line)
return self.retry(*args, run_number=run_number, **kwargs)
else:
success = self.trigger.check_success()
if success:
return self.retry(*args, run_number=run_number, **kwargs)
self.trigger.result = []
for line in output.decode().split("\n"):
if line.startswith("Requests per second:"):
self.trigger.returned_information = [float(line.split(":")[1].strip().split(" ")[0])]
with suppress(subprocess.CalledProcessError):
launch_and_log(self.trigger.stop_cmd.split(" "))
if len(self.trigger.returned_information) == 0:
return self.retry(*args, run_number=run_number, **kwargs)
logging.verbose("Requests per second : {}".format(self.trigger.returned_information[0]))
return success
def run(self) -> int:
"""
Main function. Calls every other one in order to make the bug trigger
:return: 0|1|None on success|failure|unexpected event
"""
try:
logging.verbose(self.cmd)
proc_start = self.Server(self.cmd) # this is not a typo. Using cmd is REQUIRED for the sake of plugins
proc_start.start()
time.sleep(self.delay)
triggers = []
results_queue = multiprocessing.Queue() # pylint: disable=no-member
for command in self.helper_commands:
# noinspection PyCallingNonCallable
triggers.append(self.helper(command, results=results_queue, **self.named_helper_args))
for thread in triggers:
thread.start()
for thread in triggers:
thread.join(self.timeout)
for thread in triggers:
thread.terminate()
finally:
with suppress(subprocess.CalledProcessError):
launch_and_log(self.stop_cmd.split(" "))
results = []
for _ in triggers:
with suppress(queue.Empty):
results.append(results_queue.get_nowait())
time.sleep(self.delay)
return self.check_success(results=results)
def configure(self) -> None:
"""
Configures the sources
"""
if self.conf["configure"] == "configure":
cmd = [
os.path.join(self.sources_dir, "configure"),
"--prefix={}".format(self.install_dir)
]
elif self.conf["configure"] == "cmake":
cmd = [
"cmake",
self.sources_dir
]
else:
logging.verbose("{} does not need configuration".format(self.conf["display_name"]))
return
cmd += self.conf.getlist("configure_args", [])
logging.info("Configuring %(name)s", dict(name=self.conf["display_name"]))
self.env["WLLVM_CONFIGURE_ONLY"] = "1"
helper.launch_and_log(cmd, cwd=self.working_dir, env=self.env, error_msg="Configuration failed")
del self.env["WLLVM_CONFIGURE_ONLY"]
def run(self):
"""
Launches the command and waits for the output
"""
with suppress(subprocess.CalledProcessError):
launch_and_log(self.command.split(" "), preexec_fn=TriggerWithHelper.__preexec_fn__)