def __copy_misuse_sources(sources_path, misuses, destination):
remove_tree(destination)
for misuse in misuses:
file = misuse.location.file
dst = join(destination, file)
makedirs(dirname(dst), exist_ok=True)
shutil.copy(join(sources_path, file), dst)
def run(self, version: ProjectVersion):
logger = logging.getLogger("tasks.checkout")
try:
checkout = version.get_checkout(self.checkouts_path)
except ValueError as e:
raise UserWarning("Checkout data corrupted: %s", e)
if self.force_checkout:
checkout.delete()
checkout_exists = checkout.exists()
logger.debug("Checkout exists = %r", checkout_exists)
if checkout_exists:
logger.debug("Already checked out %s.", version)
else:
logger.info("Fetching %s from %s...", version, checkout)
if self.use_temp_dir:
temp_dir = mkdtemp(prefix="mubench-checkout_")
temp_checkout = version.get_checkout(temp_dir)
temp_checkout.create(self.run_timestamp)
logger.debug("Copying checkout to persistent directory...")
copy_tree(temp_dir, self.checkouts_path)
remove_tree(temp_dir)
else:
checkout.create(self.run_timestamp)
return checkout
def __copy_pattern_sources(misuses: List[Misuse],
project_compile: ProjectCompile):
remove_tree(project_compile.pattern_sources_base_path)
for misuse in misuses:
pattern_source_path = project_compile.get_pattern_source_path(
misuse)
for pattern in misuse.patterns:
pattern.copy(pattern_source_path)
def test_removes_folder_completely(self):
create_file(join(self.temp_dir, "dir1", "dir2", "file1"))
create_file(join(self.temp_dir, "dir1", "file2"))
create_file(join(self.temp_dir, "file3"))
remove_tree(self.temp_dir)
assert not exists(self.temp_dir)
def __copy_misuse_classes(classes_path, misuses, destination):
remove_tree(destination)
for misuse in misuses:
basepath = join(classes_path, splitext(misuse.location.file)[0])
classes = glob(basepath + ".class") + glob(basepath + "$*.class")
for clazz in classes:
dst = join(destination, relpath(clazz, classes_path))
makedirs(dirname(dst), exist_ok=True)
shutil.copy(clazz, dst)
def __copy_pattern_classes(misuses: List[Misuse], classes_path: str,
project_compile: ProjectCompile):
remove_tree(project_compile.pattern_classes_base_path)
for misuse in misuses:
pattern_classes_path = project_compile.get_pattern_classes_path(
misuse)
for pattern in misuse.patterns:
pattern_class_file_name = pattern.relative_path_without_extension + ".class"
new_name = join(pattern_classes_path, pattern_class_file_name)
makedirs(dirname(new_name), exist_ok=True)
shutil.copy(join(classes_path, pattern_class_file_name),
new_name)
def test_existent_misuse(self, _):
self.uut._report_unknown_misuse = MagicMock()
project = create_project("-project-", meta={})
version = create_version("-id-", meta={"misuses": ["-misuse-"]}, project=project)
project.path = mkdtemp(prefix="mubench_test-dataset-check_")
try:
misuse_yml_path = join(project.path, Project.MISUSES_DIR, "-misuse-", Misuse.MISUSE_FILE)
create_file(misuse_yml_path)
self.uut.run(project, version)
self.uut._report_unknown_misuse.assert_not_called()
finally:
remove_tree(project.path)
def run(self, version: ProjectVersion, checkout: ProjectCheckout):
logger = logging.getLogger("task.compile")
version_compile = version.get_compile(self.compiles_base_path)
build_path = mkdtemp(prefix='mubench-compile_') if self.use_temp_dir else version_compile.build_dir
if self.force_compile or checkout.timestamp > version_compile.timestamp:
logger.debug("Force compile - removing previous compiles...")
version_compile.delete()
try:
if not version.is_compilable:
raise UserWarning("Skipping compilation: not configured.")
if version_compile.needs_compile():
logger.info("Compiling %s...", version)
logger.debug("Copying checkout to build directory...")
checkout_path = checkout.checkout_dir
copy_tree(checkout_path, build_path)
logger.debug("Copying additional resources...")
self.__copy_additional_compile_sources(version, build_path)
logger.debug("Compiling project...")
self._compile(version.compile_commands,
build_path,
version_compile.dependencies_path,
self.compiles_base_path,
logger)
logger.debug("Create project jar...")
zip_dir_contents(version_compile.original_classes_paths, version_compile.original_classpath)
version_compile.save(self.run_timestamp)
if self.use_temp_dir:
logger.debug("Moving complete build to persistent directory...")
copy_tree(build_path, version_compile.build_dir)
remove_tree(build_path)
except Exception:
version_compile.delete()
raise
return version_compile
def process_project_version(self, project: Project,
version: ProjectVersion) -> List[str]:
logger = logging.getLogger("checkout")
try:
checkout = version.get_checkout(self.checkouts_path)
except ValueError as e:
logger.error("Checkout data corrupted: %s", e)
return self.skip(version)
try:
if self.force_checkout:
checkout.delete()
checkout_exists = checkout.exists()
logger.debug("Checkout exists = %r", checkout_exists)
if checkout_exists:
logger.debug("Already checked out %s.", version)
else:
logger.info("Fetching %s from %s...", version, checkout)
if self.use_temp_dir:
temp_dir = mkdtemp(prefix="mubench-checkout_")
temp_checkout = version.get_checkout(temp_dir)
temp_checkout.create()
logger.debug("Copying checkout to persistent directory...")
copy_tree(temp_dir, self.checkouts_path)
remove_tree(temp_dir)
else:
checkout.create()
return self.ok()
except CommandFailedError as e:
logger.error("Checkout failed: %s", e)
return self.skip(version)
except IOError:
logger.error("Checkout failed.", exc_info=True)
return self.skip(version)
def delete(self):
remove_tree(self.correct_usage_sources_path)
remove_tree(self.correct_usage_classes_path)
remove_tree(self.misuse_source_path)
remove_tree(self.misuse_classes_path)
remove_tree(self._misuse_compile_file)
def reset(self):
remove_tree(self._get_findings_path())
makedirs(self._get_findings_path(), exist_ok=True)
DetectorExecution.__init__(self, self.run_mode, self.detector,
self.version, self._findings_base_path,
self.findings_filter)
def teardown(self):
remove_tree(self.temp_dir)
self.compile_patch.stop()
def reset(self):
remove_tree(self.findings_path)
makedirs(self.findings_path, exist_ok=True)
DetectorRun.__init__(self, self.detector, self.version,
self.findings_path)
def delete(self) -> None:
remove_tree(self.checkout_dir)
def delete(self) -> None:
self._logger.debug("Delete %s", self.checkout_dir)
remove_tree(self.checkout_dir)
def delete(self):
self._logger.debug("Delete %s", self.__base_checkout_dir)
remove_tree(self.__base_checkout_dir)
self.__child.delete()
def process_project_version(self, project: Project,
version: ProjectVersion):
logger = logging.getLogger("compile")
logger.info("Compiling %s...", version)
logger = logging.getLogger("compile.tasks")
project_compile = version.get_compile(self.compiles_base_path)
build_path = mkdtemp(
prefix='mubench-compile_'
) if self.use_temp_dir else project_compile.build_dir
sources_path = join(build_path, version.source_dir)
classes_path = join(build_path, version.classes_dir)
if self.force_compile:
logger.debug("Force compile - removing previous compiles...")
project_compile.delete()
try:
needs_copy_sources = project_compile.needs_copy_sources()
needs_compile = project_compile.needs_compile()
if needs_copy_sources or needs_compile:
logger.debug("Copying checkout to build directory...")
checkout_path = version.get_checkout(
self.checkouts_base_path).checkout_dir
copy_tree(checkout_path, build_path)
logger.debug("Copying additional resources...")
self.__copy_additional_compile_sources(version, build_path)
if not needs_copy_sources:
logger.debug("Already copied source.")
else:
logger.info("Copying sources...")
logger.debug("Copying project sources...")
copy_tree(sources_path, project_compile.original_sources_path)
logger.debug("Copying misuse sources...")
self.__copy_misuse_sources(sources_path, version.misuses,
project_compile.misuse_source_path)
logger.info("Copying pattern sources...")
self.__copy_pattern_sources(version.misuses, project_compile)
if not version.compile_commands:
logger.warning("Skipping compilation: not configured.")
return self.skip(version)
if not needs_compile:
logger.debug("Already compiled project.")
else:
logger.info("Compiling project...")
logger.debug("Copying patterns to source directory...")
self.__copy(version.patterns, sources_path)
self._compile(version.compile_commands, build_path,
project_compile.dependencies_path,
self.compiles_base_path, logger)
logger.debug("Move pattern classes...")
self.__copy_pattern_classes(version.misuses, classes_path,
project_compile)
self.__remove_pattern_classes(version.misuses, classes_path)
logger.debug("Copy project classes...")
copy_tree(classes_path, project_compile.original_classes_path)
logger.debug("Create project jar...")
self.__create_jar(project_compile.original_classes_path,
project_compile.original_classpath)
logger.debug("Copy misuse classes...")
self.__copy_misuse_classes(classes_path, version.misuses,
project_compile.misuse_classes_path)
if self.use_temp_dir:
logger.debug(
"Moving complete build to persistent directory...")
copy_tree(build_path, project_compile.build_dir)
remove_tree(build_path)
except Exception as e:
logger.error("Compilation failed: %s", e)
project_compile.delete()
return self.skip(version)
return self.ok()
def delete(self):
remove_tree(self.original_sources_path)
remove_tree(self.misuse_source_path)
remove_tree(self.pattern_sources_base_path)
remove_tree(self.original_classes_path)
remove_tree(self.misuse_classes_path)
remove_tree(self.pattern_classes_base_path)
remove_tree(self.build_dir)
def delete(self):
io.remove_tree(self.path)
def delete(self):
remove_tree(self.build_dir)
def process_project_version(self, project: Project,
version: ProjectVersion):
logger = logging.getLogger("compile")
logger.info("Compiling %s...", version)
logger.debug("- Force compile = %r", self.force_compile)
logger = logging.getLogger("compile.tasks")
project_compile = version.get_compile(self.compiles_base_path)
build_path = join(project_compile.base_path, Compile.__BUILD_DIR)
sources_path = join(build_path, version.source_dir)
classes_path = join(build_path, version.classes_dir)
needs_copy_sources = project_compile.needs_copy_sources(
) or self.force_compile
needs_compile = project_compile.needs_compile() or self.force_compile
if needs_copy_sources or needs_compile:
logger.debug("Copying to build directory...")
checkout_path = version.get_checkout(
self.checkouts_base_path).checkout_dir
self.__clean_copy(checkout_path, build_path)
logger.debug("Copying additional resources...")
self.__copy_additional_compile_sources(version, build_path)
if not needs_copy_sources:
logger.debug("Already copied project source.")
else:
try:
logger.info("Copying project sources...")
self.__clean_copy(sources_path,
project_compile.original_sources_path)
self.__copy_misuse_sources(sources_path, version.misuses,
project_compile.misuse_source_path)
except IOError as e:
logger.error("Failed to copy project sources: %s", e)
return self.skip(version)
if not version.compile_commands:
logger.warn("Skipping compilation: not configured.")
return self.skip(version)
if not needs_compile:
logger.info("Already compiled project.")
else:
try:
logger.info("Compiling project...")
self._compile(version.compile_commands, build_path,
project_compile.dependency_base_path)
logger.debug("Copying project classes...")
self.__clean_copy(classes_path,
project_compile.original_classes_path)
self.__copy_misuse_classes(classes_path, version.misuses,
project_compile.misuse_classes_path)
except CommandFailedError as e:
logger.error("Compilation failed: %s", e)
return self.skip(version)
except FileNotFoundError as e:
logger.error("Failed to copy classes: %s", e)
return self.skip(version)
if not version.patterns:
logger.info("Skipping pattern compilation: no patterns.")
return self.ok()
needs_copy_pattern_sources = project_compile.needs_copy_pattern_sources(
) or self.force_compile
needs_compile_patterns = project_compile.needs_compile_patterns(
) or self.force_compile
if needs_copy_pattern_sources or needs_compile_patterns:
logger.debug("Copying to build directory...")
checkout_path = version.get_checkout(
self.checkouts_base_path).checkout_dir
self.__clean_copy(checkout_path, build_path)
logger.debug("Copying additional resources...")
self.__copy_additional_compile_sources(version, build_path)
if not needs_copy_pattern_sources:
logger.debug("Already copied pattern sources.")
else:
try:
logger.info("Copying pattern sources...")
self.__copy_pattern_sources(version.misuses, project_compile)
except IOError as e:
logger.error("Failed to copy pattern sources: %s", e)
return self.skip(version)
if not needs_compile_patterns:
logger.info("Already compiled patterns.")
else:
try:
logger.debug("Copying patterns to source directory...")
self.__copy(version.patterns, sources_path)
logger.info("Compiling patterns...")
self._compile(version.compile_commands, build_path,
project_compile.dependency_base_path)
logger.debug("Copying pattern classes...")
self.__copy_pattern_classes(version.misuses, classes_path,
project_compile)
except FileNotFoundError as e:
remove_tree(project_compile.pattern_classes_base_path)
logger.error("Compilation failed: %s", e)
return self.skip(version)
except CommandFailedError as e:
logger.error("Compilation failed: %s", e)
return self.skip(version)
return self.ok()
def __clean_copy(sources_path: str, destination: str):
remove_tree(destination)
copy_tree(sources_path, destination)
def _copy_classpath(dependencies: List[str], dep_dir: str):
remove_tree(dep_dir)
makedirs(dirname(dep_dir + "/"), exist_ok=True)
for dependency in dependencies:
shutil.copy(dependency, dep_dir)
def teardown(self):
remove_tree(self.temp_dir)
def _copy_classpath(dependencies: List[str], dep_dir: str):
remove_tree(dep_dir)
makedirs(dep_dir, exist_ok=True)
for dependency in dependencies:
if not os.path.isdir(dependency):
shutil.copy(dependency, dep_dir)