def execute(self, compile_base_path: str, timeout: Optional[int],
logger: Logger):
detector_invocation = ["java"] + self.detector.java_options + [
"-jar", _quote(self.detector.jar_path)
]
command = detector_invocation + self._get_detector_arguments(
self.version.get_compile(compile_base_path))
command = " ".join(command)
start = time.time()
try:
Shell.exec(command, logger=logger, timeout=timeout)
self.result = Result.success
except CommandFailedError as e:
logger.error("Detector failed: %s", e)
self.result = Result.error
message = str(e)
message_lines = str.splitlines(message)
if len(message_lines) > 5000:
self.message = "\n".join(
message_lines[0:500]) + "\n" + "\n".join(
message_lines[-4500:])
else:
self.message = message
except TimeoutError:
logger.error("Detector took longer than the maximum of %s seconds",
timeout)
self.result = Result.timeout
finally:
end = time.time()
runtime = end - start
self.runtime = runtime
logger.info("Run took {0:.2f} seconds.".format(runtime))
self.save()
def create(self) -> None:
self._logger.debug("Create checkout directory %s", self.checkout_dir)
makedirs(self.checkout_dir, exist_ok=True)
self._logger.debug("Checkout from %s", self.url)
Shell.exec("svn checkout \"{}@{}\" .".format(self.url, self.revision),
cwd=self.checkout_dir)
Shell.exec("svn upgrade", cwd=self.checkout_dir)
def test_error_contains_stderr(self):
with assert_raises(CommandFailedError):
Shell.exec("echo 'test' 1>&2 && exit 1")
try:
Shell.exec("echo 'test' 1>&2 && exit 1")
except CommandFailedError as e:
assert_equals("test" + os.linesep, e.error)
def _compile_correct_usages(source: str, destination: str, classpath: str):
makedirs(destination, exist_ok=True)
for root, dirs, files in os.walk(source):
for java_file in [f for f in files if f.endswith(".java")]:
destination = dirname(join(destination, java_file))
Shell.exec('javac "{}" -d "{}" -cp "{}"'.format(join(root, java_file), destination, classpath),
logger=logging.getLogger("task.compile_correct_usages.compile"))
def __create_image(dot_graph, working_directory, image_name):
image_path = join(working_directory, image_name)
if not exists(image_path):
makedirs(working_directory, exist_ok=True)
dot_path = image_path + ".dot"
safe_write(dot_graph, dot_path, append=False)
Shell.exec("dot -v -Tsvg -o'{}' '{}'".format(image_path, dot_path))
remove(dot_path)
def __create_image(dot_graph, working_directory, image_name):
image_path = join(working_directory, image_name)
if not exists(image_path):
makedirs(working_directory, exist_ok=True)
dot_path = image_path + ".dot"
#can be used to restrict iterations during graph layouting
#dot_graph = dot_graph.replace("\n", "\nnslimit=100;\n",1)
safe_write(dot_graph, dot_path, append=False)
Shell.exec("dot -v -Tsvg -o'{}' '{}'".format(image_path, dot_path))
remove(dot_path)
def _compile(commands: List[str], project_dir: str, dep_dir: str) -> None:
logger = logging.getLogger("compile.tasks.exec")
for command in commands:
if command.startswith("mvn "):
command = "mvn dependency:build-classpath " + command[4:]
output = Shell.exec(command, cwd=project_dir, logger=logger)
dependencies = Compile._parse_maven_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
elif command.startswith("ant "):
command += " -debug -verbose"
output = Shell.exec(command, cwd=project_dir, logger=logger)
dependencies = Compile._parse_ant_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
elif command.startswith("gradle "):
Shell.exec(command, cwd=project_dir, logger=logger)
shutil.copy(
os.path.dirname(__file__) + '/classpath.gradle',
project_dir)
output = Shell.exec(
"gradle :printClasspath -b classpath.gradle",
cwd=project_dir,
logger=logger)
dependencies = Compile._parse_gradle_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
else:
Shell.exec(command, cwd=project_dir, logger=logger)
def _compile(commands: List[str], project_dir: str, dep_dir: str) -> None:
logger = logging.getLogger("compile.tasks.exec")
for command in commands:
if command.startswith("mvn "):
command = "mvn dependency:build-classpath -DincludeScope=compile " + command[
4:]
output = Shell.exec(command, cwd=project_dir, logger=logger)
dependencies = Compile.__parse_maven_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
elif command.startswith("ant"):
command += " -debug -verbose"
output = Shell.exec(command, cwd=project_dir, logger=logger)
dependencies = Compile.__parse_ant_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
elif command.startswith("gradle "):
Shell.exec(command, cwd=project_dir, logger=logger)
buildfile_dir = Compile.__parse_buildfile_dir(command)
shutil.copy(
os.path.join(os.path.dirname(__file__),
'classpath.gradle'),
os.path.join(project_dir, buildfile_dir))
command = "gradle :printClasspath -b '{}'".format(
os.path.join(buildfile_dir, "classpath.gradle"))
output = Shell.exec(command, cwd=project_dir, logger=logger)
dependencies = Compile.__parse_gradle_classpath(output)
Compile._copy_classpath(dependencies, dep_dir)
else:
Shell.exec(command, cwd=project_dir, logger=logger)
def setup(self):
self.temp_dir = mkdtemp(prefix='mubench-checkout-git_')
self.git_url = join(self.temp_dir, "remote")
os.makedirs(self.git_url)
Shell.exec("git init .", cwd=self.git_url)
Shell.exec("touch foo", cwd=self.git_url)
Shell.exec("git add -A", cwd=self.git_url)
Shell.exec("git commit -a -m \"Initial commit.\"", cwd=self.git_url)
self.checkouts_dir = join(self.temp_dir, "checkouts")
def _get_dependencies(self, shell_output: str, project_dir: str,
logger: Logger) -> Set[str]:
buildfile_dir = self._parse_buildfile_dir(self.args)
shutil.copy(
os.path.join(os.path.dirname(__file__), 'classpath.gradle'),
os.path.join(project_dir, buildfile_dir))
command = "gradle :printClasspath -b '{}'".format(
os.path.join(buildfile_dir, "classpath.gradle"))
output = Shell.exec(command, cwd=project_dir, logger=logger)
return self._parse_classpath(output)
def execute(self, project_dir: str, logger: Logger) -> Set[str]:
command = self._get_command(self.args)
try:
output = Shell.exec(command, cwd=project_dir, logger=logger)
return self._get_dependencies(output, project_dir, logger)
except CommandFailedError as e:
error_output = '\n' + self._get_errors(e.output, e.error)
e.output = error_output
e.error = ""
raise
def exec_util(main: str, args: str = ""):
base_path = dirname(__file__)
utils_jar_path = join(base_path, UTILS_JAR_NAME)
if exists(utils_jar_path) and not is_valid_file(utils_jar_path, UTILS_MD5):
remove(utils_jar_path)
if not exists(utils_jar_path):
try:
download_file(UTILS_JAR_URL, utils_jar_path, UTILS_MD5)
except (URLError, ValueError, FileNotFoundError) as e:
raise ValueError("utils unavailable: {}".format(e))
return Shell.exec(
"java -cp \"{}\" de.tu_darmstadt.stg.mubench.utils.{} {}".format(
utils_jar_path, main, args))
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../mubench.pipeline")
from utils.shell import Shell, CommandFailedError
class CustomHelpFormatter(HelpFormatter):
def _format_action(self, action):
if type(action) == _SubParsersAction:
msg = ''
for subaction in action._get_subactions():
msg += self._format_action(subaction)
return msg
else:
return super(CustomHelpFormatter, self)._format_action(action)
try:
__version__ = Shell.exec('version').strip()
except CommandFailedError as e:
__version__ = "v???"
parser = ArgumentParser(
usage=SUPPRESS,
description="This is MUBench " + __version__ + ".",
epilog="For details, check out `<command> -h` or https://github.com/stg-tud/MUBench. ",
add_help=False,
formatter_class=CustomHelpFormatter)
parser.add_argument('-v', '--version', action='version', version=__version__, help=SUPPRESS)
parser._positionals.title = "MUBench provides the following commands"
subparsers = parser.add_subparsers(dest='command')
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
example_projects_by_API = {}
with open(INDEX_PATH) as index:
for row in csv.reader(index, delimiter="\t"):
# skip blank lines, e.g., on trailing newline
if not row:
continue
target_type = row[6]
try:
if target_type not in example_projects_by_API:
logger.info("Preparing examples for type: %s...", target_type)
target_example_file = os.path.join(CHECKOUTS_PATH,
target_type + ".yml")
example_projects = {}
with open_yamls_if_exists(target_example_file) as projects:
for project in projects:
hash = Shell.exec(
"cd \"{}\"; git rev-parse HEAD".format(
join(MUBENCH_ROOT_PATH, project["path"])))
example_projects[project["url"]] = hash.strip()
example_projects_by_API[target_type] = example_projects
except Exception as error:
logger.exception("failed", exc_info=error)
write_yaml(example_projects_by_API,
join(CHECKOUTS_PATH, "example_projects_by_API.yml"))
def execute(self, version: ProjectVersion, detector_arguments: Dict[str,
str],
timeout: Optional[int], logger: Logger):
detector_arguments = self._filter_args(detector_arguments, logger)
command = self._get_command(detector_arguments)
Shell.exec(command, logger=logger, timeout=timeout)
def check(self):
Shell.exec("git --version")
def check(self):
Shell.exec("java -version")
def test_timeout(self):
with assert_raises(TimeoutError):
Shell.exec("sleep 10", timeout=1)
def test_output(self):
out = Shell.exec("echo test")
assert_equals("test" + os.linesep, out)
def _clone(self, url: str, revision: str, path: str):
Shell.exec("git clone {} . --quiet".format(url), cwd=path, logger=self._logger)
def check(self):
Shell.exec("dot -V")
def check(self):
Shell.exec("gradle -version")
def check(self):
Shell.exec("mvn -v")
def clone(self):
io.makedirs(self.path)
clone_command = "git clone --depth 1 {} . --quiet -c core.askpass=true".format(self.url)
Shell.exec(clone_command, cwd=self.path, logger=self._logger)
def _update(self, url: str, revision: str, path: str):
Shell.exec("git checkout {} --quiet".format(revision), cwd=path, logger=self._logger)
def test_output_contains_non_empty_stderr(self):
out = Shell.exec("echo 'test' 1>&2")
expected = "=== ERROR ===" + os.linesep + "test" + os.linesep
assert_equals(expected, out)
def test_command_failure(self):
with assert_raises(CommandFailedError):
Shell.exec("unknown command")
def test_output_contains_stdout_and_stderr(self):
out = Shell.exec("echo '-stdout-' && echo '-stderr-' 1>&2")
expected = "=== OUT ===" + os.linesep + \
"-stdout-" + os.linesep + \
"=== ERROR ===" + os.linesep + \
"-stderr-" + os.linesep
def test_runs(self):
Shell.exec("echo 'test'")
def check(self):
Shell.exec("svn --version")