def test_lstm(self):
file_path = self._get_test_path("lstm.py")
result = gather_library_usage(file_path)
assert "report" in result
assert file_path in result["report"]
assert len(result["report"].keys()) == 1
assert result["report"][file_path] == {
"tensorflow": [
"tensorflow.Session",
"tensorflow.Variable",
"tensorflow.argmax",
"tensorflow.cast",
"tensorflow.contrib.rnn.BasicLSTMCell",
"tensorflow.contrib.rnn.static_rnn",
"tensorflow.equal",
"tensorflow.examples.tutorials.mnist.input_data.read_data_sets",
"tensorflow.float32",
"tensorflow.global_variables_initializer",
"tensorflow.matmul",
"tensorflow.nn.softmax",
"tensorflow.nn.softmax_cross_entropy_with_logits",
"tensorflow.placeholder",
"tensorflow.random_normal",
"tensorflow.reduce_mean",
"tensorflow.train.GradientDescentOptimizer",
"tensorflow.unstack",
]
}
def cli(
ctx=None,
verbose: bool = False,
path: str = None,
ignore_errors: bool = False,
without_standard_imports: bool = False,
without_builtin_imports: bool = False,
):
"""Statically analyze sources and extract information about called library functions in Python applications."""
if ctx:
ctx.auto_envvar_prefix = "INVECTIO"
if verbose:
_LOGGER.setLevel(logging.DEBUG)
_LOGGER.debug("Debug mode is on")
_LOGGER.debug("Version: %s", __version__)
result = gather_library_usage(
path,
ignore_errors=ignore_errors,
without_standard_imports=without_standard_imports,
without_builtin_imports=without_builtin_imports,
)
click.echo(json.dumps(result, indent=2, sort_keys=True))
def _get_static_analysis() -> typing.Optional[dict]:
"""Get static analysis of files used in project."""
# We are running in the root directory of project, use the root part for gathering static analysis.
_LOGGER.info(
"Performing static analysis of sources to gather library usage")
try:
library_usage = gather_library_usage(".",
ignore_errors=True,
without_standard_imports=True)
except FileNotFoundError:
_LOGGER.warning(
"No library usage was aggregated - no Python sources found")
return None
report = {} # type: Dict[Any, Any]
for file_record in library_usage["report"].values():
for library, usage in file_record.items():
# We could filter out some of the libraries which were used.
if library not in report:
report[library] = set()
report[library].update(usage)
return {
"report": {k: list(v)
for k, v in report.items()},
"version": library_usage["version"],
}
def test_app_6(self):
file_path = self._get_test_path("app_6.py")
result = gather_library_usage(file_path)
assert result == {
file_path: {
"tensorflow": ["tensorflow.layers.conv2d"]
}
}
def test_standard_imports_detection(self):
file_path = self._get_test_path("app_7.py")
result = gather_library_usage(file_path, without_standard_imports=True)
assert "report" in result
assert "version" in result
assert file_path in result["report"]
assert result["report"][file_path] == {}
result = gather_library_usage(file_path,
without_standard_imports=False)
assert "report" in result
assert "version" in result
assert file_path in result["report"]
assert result["report"][file_path] == {
"collections": ["collections.deque"],
"datetime": ["datetime.datetime.utcnow"]
}
def test_app_5(self) -> None:
file_path = self._get_test_path("app_5_test.py")
result = gather_library_usage(file_path)
assert "report" in result
assert result["report"] == {
file_path: {
"tensorflow": ["tensorflow.layers.conv2d"]
},
}
def test_without_builtin_and_standard_imports(self):
file_path = self._get_test_path("app_8.py")
result = gather_library_usage(file_path,
without_standard_imports=True,
without_builtin_imports=True)
assert "version" in result
assert "report" in result
assert file_path in result["report"]
assert result["report"][file_path] == {
'tensorflow': ['tensorflow.python.keras.layers.LSTM']
}
def whatuses(
path: str,
ignore_errors: bool = False,
without_standard_imports: bool = False,
without_builtin_imports: bool = False,
) -> None:
"""Gather information about symbol usage by a module or a source file."""
result = gather_library_usage(
path,
ignore_errors=ignore_errors,
without_standard_imports=without_standard_imports,
without_builtin_imports=without_builtin_imports,
)
click.echo(json.dumps(result, indent=2, sort_keys=True))
def test_without_builtin_imports(self):
file_path = self._get_test_path("app_8.py")
result = gather_library_usage(file_path,
without_standard_imports=False,
without_builtin_imports=True)
assert "version" in result
assert "report" in result
assert file_path in result["report"]
assert {k: set(v)
for k, v in result["report"][file_path].items()} == {
"collections": {"collections.deque"},
"signal": {"signal.SIGKILL", "signal.signal"},
"tensorflow": {"tensorflow.python.keras.layers.LSTM"},
}
def test_project_dir(self):
project_path = self._get_test_path("project_dir")
result = gather_library_usage(project_path)
assert result == {
"tests/data/project_dir/main.py": {
"flask": ["flask.Flask"],
"proj": ["proj.get_model"],
},
"tests/data/project_dir/proj/__init__.py": {},
"tests/data/project_dir/proj/model.py": {
"numpy": ["numpy.random.random"],
"tensorflow": [
"tensorflow.keras.Sequential",
"tensorflow.keras.layers.Dense",
],
},
"tests/data/project_dir/proj/utils.py": {
"datetime": ["datetime.datetime.utcnow"]
},
}
def _get_static_analysis() -> dict:
"""Get static analysis of files used in project."""
# We are running in the root directory of project, use the root part for gathering static analysis.
_LOGGER.info("Performing static analysis")
try:
library_usage = gather_library_usage(".", ignore_errors=True)
except FileNotFoundError:
_LOGGER.warning("No library usage was aggregated - no Python sources found")
return {}
result = {}
for file_record in library_usage.values():
for library, usage in file_record.items():
if library not in _LIBRARIES_USAGE:
_LOGGER.debug("Omitting usage of library %r", library)
continue
if library not in result:
result[library] = []
result[library].extend(usage)
return result
def test_no_files(self):
file_path = self._get_test_path("somenonexistingfileorfilepath")
with pytest.raises(FileNotFoundError):
gather_library_usage(file_path)
def test_empty(self):
file_path = self._get_test_path("empty.py")
result = gather_library_usage(file_path)
assert "report" in result
assert result["report"] == {file_path: {}}
def test_version(self):
file_path = self._get_test_path("empty.py")
result = gather_library_usage(file_path)
assert "version" in result
assert result["version"] == invectio_version
