def java_codebase_to_units(location, build_command, build_root):
"""Convert a Java codebase found at 'location' into a list of units.
Works by compiling it with 'build_command' and then inspecting the
class files under 'location/build_root'."""
# Compile the classes
logger.info("Starting build process")
if "CLASSPATH" in os.environ:
logger.debug("CLASSPATH variable set")
if config.java_classpath() != "":
logger.debug("java_native/classpath set, using that as classpath")
os.environ["CLASSPATH"] = config.java_classpath()
if "CLASSPATH" not in os.environ:
logger.warning("No CLASSPATH set")
else:
logger.info("CLASSPATH is:\n\t{}".format(os.environ["CLASSPATH"]))
try:
subprocess.run(build_command,
cwd=location,
shell=True,
check=True,
stdout=sys.stderr,
stderr=sys.stderr)
except subprocess.CalledProcessError:
logger.error("Failed to call {}".format(build_command))
exit(1)
logger.info("Build completed")
# Get absolute path to build root
build_root = os.path.join(location, build_root)
logger.debug("Absolute build root is {}".format(build_root))
# Get a list of fully-qualified class names
fqns = []
for root, dirs, files in os.walk(build_root):
dirs[:] = [d for d in dirs if not config.dir_ignored(d)]
classfiles = [
f for f in files
if f.endswith(".class") and not config.file_ignored(f)
]
prefix = root[len(build_root):].replace(os.sep, ".")
if len(prefix) > 0 and prefix[0] == ".":
prefix = prefix[1:]
fqns = fqns + [(
(prefix + ".") if prefix != "" else "") + os.path.splitext(n)[0]
for n in classfiles]
logger.debug("{} classes identified".format(len(fqns)))
# Convert the XML representation of these classes to Unit
xml = _run_inspector(build_root, fqns)
root = ElementTree.fromstring(xml)
units = dict()
for child in root:
# TODO: Validation of the XML shouldn't be done using assertions.
# Is validation even necessary in this case?
assert child.tag == "class", "Found element in XML that's not <class>!"
unit = _xml_element_to_unit(child)
units[unit.name] = unit
return units
def clojure_codebase_to_units(location):
"""Returns a list of Units representing a Clojure codebase in 'location'."""
# Resolve classpath
if "CLASSPATH" in os.environ:
logger.debug(
"CLASSPATH variable set (it's not guaranteed Clojure will it)")
if config.clojure_classpath() != "":
logger.debug(
"clojure/classpath set, setting that as CLASSPATH (it's not guaranteed Clojure will use it"
)
os.environ["CLASSPATH"] = config.clojure_classpath()
if "CLASSPATH" not in os.environ:
logger.warning("No CLASSPATH set")
else:
logger.info("CLASSPATH is:\n\t{}".format(os.environ["CLASSPATH"]))
# Inspect .clj files
cljfiles = []
for root, dirs, files in os.walk(location):
dirs[:] = [d for d in dirs if not config.dir_ignored(d)]
cljfiles += [
os.path.join(root, f) for f in files
if f.endswith(_source_file_ext) and not config.file_ignored(f)
]
units = dict()
for cljfile in cljfiles:
try:
inspector_out = _run_inspector(cljfile, location)
units.update(_sexp_read(inspector_out))
except _ClojureUtilityException:
logger.warn("File {} failed to parse".format(cljfile))
if not config.clojure_omit_on_error():
raise
return units
def java_codebase_to_units(location):
"""Return a list of Units representing a Java codebase in 'location'."""
# This needs to perform two passes on all Java files.
#
# In the first pass:
# 1) Java files are translated to compilation units (aka AST).
# 2) The type system is enriched with all types declared in this codebase.
#
# In the second pass:
# 1) Referential types are resolved when referring to this codebase.
# 2) All compilation units are translated into Units.
units = dict()
# First pass
type_system = _JavaTypeSystem()
compilations = []
for root, dirs, files in os.walk(location):
dirs[:] = [d for d in dirs if not config.dir_ignored(d)]
javafiles = [
f for f in files
if f.endswith(_source_file_ext) and not config.file_ignored(f)
]
for javafile in javafiles:
with open(os.path.join(root, javafile), "r") as f:
compilation = _source_to_compilation(javafile, f.read())
for type_name, type_node in _compilation_get_types(
compilation):
type_system.add_qualified_type(type_name,
(type_node, compilation))
compilations.append(compilation)
type_system.finalize()
# Second pass
for compilation in compilations:
unit = _compilation_to_unit(compilation, type_system)
units[unit.name] = unit
return units
def python_codebase_to_units(location):
"""Returns a list of Units representing a Python codebase in 'location'."""
if config.type_hinting():
# When the handler is invoked, the 'ast' module needs to start
# pointing to 'ast35' from 'typed_ast' if type hinting is to be used.
# Note that 'ast' must be changed globally, as the other functions in this
# module rely on it as well.
global ast
from typed_ast import ast35
ast = ast35
units = dict()
for root, dirs, files in os.walk(location):
dirs[:] = [d for d in dirs if not config.dir_ignored(d)]
pyfiles = [
f for f in files
if f.endswith(_source_file_ext) and not config.file_ignored(f)
]
for pyfile in pyfiles:
pymodule = pyfile[:-(len(_source_file_ext))] # Strip extension
with codecs.open(os.path.join(root, pyfile),
"r",
encoding="utf-8",
errors="replace") as f:
try:
units[pymodule] = _module_to_unit(pymodule,
ast.parse(f.read()))
except Exception:
print(traceback.format_exc(), file=sys.stderr)
msg = "Failed to parse file {}".format(
os.path.join(root, pyfile))
if config.python_omit_on_error():
logger.warning(msg)
else:
logger.error(msg)
exit(1)
return units
def test_only_dir(self):
self.assertFalse(config.dir_ignored("special"))
self.assertTrue(config.dir_ignored("somethingelse"))
def test_ignore_multiple_dirs(self):
self.assertTrue(config.dir_ignored("dirname1"))
self.assertTrue(config.dir_ignored("dirname2"))
def test_ignore_dir(self):
self.assertTrue(config.dir_ignored("dirname"))