def java_codebase_to_units(location, build_command, build_root): """Convert a Java codebase found at 'location' into a list of units. Works by compiling it with 'build_command' and then inspecting the class files under 'location/build_root'.""" # Compile the classes logger.info("Starting build process") if "CLASSPATH" in os.environ: logger.debug("CLASSPATH variable set") if config.java_classpath() != "": logger.debug("java_native/classpath set, using that as classpath") os.environ["CLASSPATH"] = config.java_classpath() if "CLASSPATH" not in os.environ: logger.warning("No CLASSPATH set") else: logger.info("CLASSPATH is:\n\t{}".format(os.environ["CLASSPATH"])) try: subprocess.run(build_command, cwd=location, shell=True, check=True, stdout=sys.stderr, stderr=sys.stderr) except subprocess.CalledProcessError: logger.error("Failed to call {}".format(build_command)) exit(1) logger.info("Build completed") # Get absolute path to build root build_root = os.path.join(location, build_root) logger.debug("Absolute build root is {}".format(build_root)) # Get a list of fully-qualified class names fqns = [] for root, dirs, files in os.walk(build_root): dirs[:] = [d for d in dirs if not config.dir_ignored(d)] classfiles = [ f for f in files if f.endswith(".class") and not config.file_ignored(f) ] prefix = root[len(build_root):].replace(os.sep, ".") if len(prefix) > 0 and prefix[0] == ".": prefix = prefix[1:] fqns = fqns + [( (prefix + ".") if prefix != "" else "") + os.path.splitext(n)[0] for n in classfiles] logger.debug("{} classes identified".format(len(fqns))) # Convert the XML representation of these classes to Unit xml = _run_inspector(build_root, fqns) root = ElementTree.fromstring(xml) units = dict() for child in root: # TODO: Validation of the XML shouldn't be done using assertions. # Is validation even necessary in this case? assert child.tag == "class", "Found element in XML that's not <class>!" unit = _xml_element_to_unit(child) units[unit.name] = unit return units
def clojure_codebase_to_units(location): """Returns a list of Units representing a Clojure codebase in 'location'.""" # Resolve classpath if "CLASSPATH" in os.environ: logger.debug( "CLASSPATH variable set (it's not guaranteed Clojure will it)") if config.clojure_classpath() != "": logger.debug( "clojure/classpath set, setting that as CLASSPATH (it's not guaranteed Clojure will use it" ) os.environ["CLASSPATH"] = config.clojure_classpath() if "CLASSPATH" not in os.environ: logger.warning("No CLASSPATH set") else: logger.info("CLASSPATH is:\n\t{}".format(os.environ["CLASSPATH"])) # Inspect .clj files cljfiles = [] for root, dirs, files in os.walk(location): dirs[:] = [d for d in dirs if not config.dir_ignored(d)] cljfiles += [ os.path.join(root, f) for f in files if f.endswith(_source_file_ext) and not config.file_ignored(f) ] units = dict() for cljfile in cljfiles: try: inspector_out = _run_inspector(cljfile, location) units.update(_sexp_read(inspector_out)) except _ClojureUtilityException: logger.warn("File {} failed to parse".format(cljfile)) if not config.clojure_omit_on_error(): raise return units
def java_codebase_to_units(location): """Return a list of Units representing a Java codebase in 'location'.""" # This needs to perform two passes on all Java files. # # In the first pass: # 1) Java files are translated to compilation units (aka AST). # 2) The type system is enriched with all types declared in this codebase. # # In the second pass: # 1) Referential types are resolved when referring to this codebase. # 2) All compilation units are translated into Units. units = dict() # First pass type_system = _JavaTypeSystem() compilations = [] for root, dirs, files in os.walk(location): dirs[:] = [d for d in dirs if not config.dir_ignored(d)] javafiles = [ f for f in files if f.endswith(_source_file_ext) and not config.file_ignored(f) ] for javafile in javafiles: with open(os.path.join(root, javafile), "r") as f: compilation = _source_to_compilation(javafile, f.read()) for type_name, type_node in _compilation_get_types( compilation): type_system.add_qualified_type(type_name, (type_node, compilation)) compilations.append(compilation) type_system.finalize() # Second pass for compilation in compilations: unit = _compilation_to_unit(compilation, type_system) units[unit.name] = unit return units
def python_codebase_to_units(location): """Returns a list of Units representing a Python codebase in 'location'.""" if config.type_hinting(): # When the handler is invoked, the 'ast' module needs to start # pointing to 'ast35' from 'typed_ast' if type hinting is to be used. # Note that 'ast' must be changed globally, as the other functions in this # module rely on it as well. global ast from typed_ast import ast35 ast = ast35 units = dict() for root, dirs, files in os.walk(location): dirs[:] = [d for d in dirs if not config.dir_ignored(d)] pyfiles = [ f for f in files if f.endswith(_source_file_ext) and not config.file_ignored(f) ] for pyfile in pyfiles: pymodule = pyfile[:-(len(_source_file_ext))] # Strip extension with codecs.open(os.path.join(root, pyfile), "r", encoding="utf-8", errors="replace") as f: try: units[pymodule] = _module_to_unit(pymodule, ast.parse(f.read())) except Exception: print(traceback.format_exc(), file=sys.stderr) msg = "Failed to parse file {}".format( os.path.join(root, pyfile)) if config.python_omit_on_error(): logger.warning(msg) else: logger.error(msg) exit(1) return units
def test_only_dir(self): self.assertFalse(config.dir_ignored("special")) self.assertTrue(config.dir_ignored("somethingelse"))
def test_ignore_multiple_dirs(self): self.assertTrue(config.dir_ignored("dirname1")) self.assertTrue(config.dir_ignored("dirname2"))
def test_ignore_dir(self): self.assertTrue(config.dir_ignored("dirname"))