def parse(source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Optional[Options] = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
if options is None:
options = Options()
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(options=options,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno, e.offset, e.msg, blocker=True)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def parse(
source: Union[str, bytes],
fnam: Optional[str] = None,
errors: Optional[Errors] = None,
options: Options = Options()) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
errors.set_file('<input>' if fnam is None else fnam, None)
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(
options=options,
is_stub=is_stub_file,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno, e.offset, e.msg)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None,
pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
custom_typing_module: str = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
The pyversion (major, minor) argument determines the Python syntax variant.
"""
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
assert pyversion[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(pyversion=pyversion,
is_stub=is_stub_file,
custom_typing_module=custom_typing_module,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
return tree
except (SyntaxError, TypeCommentParseError) as e:
if errors:
errors.set_file('<input>' if fnam is None else fnam)
errors.report(e.lineno, e.offset, e.msg)
else:
raise
return MypyFile([], [], False, set())
def parse(source: Union[str, bytes],
fnam: str = None,
errors: Errors = None,
pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
custom_typing_module: str = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
The pyversion (major, minor) argument determines the Python syntax variant.
"""
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
assert pyversion[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(
pyversion=pyversion,
is_stub=is_stub_file,
custom_typing_module=custom_typing_module,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
return tree
except (SyntaxError, TypeCommentParseError) as e:
if errors:
errors.set_file('<input>' if fnam is None else fnam)
errors.report(e.lineno, e.offset, e.msg)
else:
raise
return MypyFile([], [], False, set())
def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]:
filename = "<unknown>"
if sys.version_info >= (3, 8):
# TODO: support Python 4+ ;)
for minor_version in range(sys.version_info[1], 4, -1):
try:
return ast.parse(src,
filename,
feature_version=(3, minor_version))
except SyntaxError:
continue
else:
for feature_version in (7, 6):
try:
return ast3.parse(src,
filename,
feature_version=feature_version)
except SyntaxError:
continue
if ast27.__name__ == "ast":
raise SyntaxError(
"The requested source code has invalid Python 3 syntax.\n"
"If you are trying to format Python 2 files please reinstall Black"
" with the 'python2' extra: `python3 -m pip install black[python2]`."
)
return ast27.parse(src)
def parse_single_version(
src: str, version: Tuple[int,
int]) -> Union[ast.AST, ast3.AST, ast27.AST]:
filename = "<unknown>"
# typed_ast is needed because of feature version limitations in the builtin ast
if sys.version_info >= (3, 8) and version >= (3, ):
return ast.parse(src, filename, feature_version=version)
elif version >= (3, ):
return ast3.parse(src, filename, feature_version=version[1])
elif version == (2, 7):
return ast27.parse(src)
raise AssertionError(
"INTERNAL ERROR: Tried parsing unsupported Python version!")
def process_file(options, source_text=None, kythe_args=None):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
kythe_args: Extra args for generating the kythe index
Returns:
An Indexer object with the indexed code
Raises:
PytypeError if pytype fails.
"""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
a = ast27.parse(src, options.input)
ast = ast27
else:
a = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(source, vm.loader, module_name, kythe_args)
ix.index(a)
ix.finalize()
return ix
def _load_27_code(self, code):
"""
Loads code from a python 27 project and returns the AST once converted to a python 3 AST
:param code: The string containing the python code
:return: Python 3 AST
"""
try:
# Try parsing as python 2.7 code
tree = ast27.parse(code)
# convert ast to v3
return conversions.py2to3(tree)
except SyntaxError as e:
logger.warning(
"There was a problem parsing the syntax in this code: %s", e)
def scan(self, code, filename=''):
code = utils.sanitize_code(code)
self.code = code.splitlines()
self.filename = filename
try:
node = ast.parse(code, filename=self.filename)
except SyntaxError as err:
pass
except (TypeError, ValueError) as err:
# Python < 3.5 raises TypeError and Python >= 3.5 raises
# ValueError if source contains null bytes.
print('{0}: invalid source code "{1}"'.format(
utils.format_path(filename), err),
file=sys.stderr)
self.found_dead_code_or_error = True
else:
self.visit(node)
def parse_file(source_code: str) -> Optional[Tuple]:
try:
# NB: The Python 3 ast is generally backwards-compatible with earlier versions. The only
# breaking change is `async` `await` becoming reserved keywords in Python 3.7 (deprecated
# in 3.6). If the std-lib fails to parse, we could use typed-ast to try parsing with a
# target version of Python 3.5, but we don't because Python 3.5 is almost EOL and has very
# low usage.
# We will also fail to parse Python 3.8 syntax if Pants is run with Python 3.6 or 3.7.
# There is no known workaround for this, beyond users changing their `./pants` script to
# always use >= 3.8.
tree = ast3.parse(source_code)
visitor_cls = _Py3AstVisitor if sys.version_info[:2] < (3, 8) else _Py38AstVisitor
return tree, visitor_cls
except SyntaxError:
try:
return ast27.parse(source_code), _Py27AstVisitor
except SyntaxError:
return None
def process_file(options):
"""Process a single file and return cross references."""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
try:
analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
except utils.UsageError as e:
logging.error("Usage error: %s\n", utils.message(e))
return 1
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
a = ast27.parse(src, options.input)
ast = ast27
else:
a = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(source, module_name)
ix.index(a)
ix.lookup_refs()
ix.process_deflocs()
ix.process_links()
return ix
def parse(source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Optional[Options] = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
if options is None:
options = Options()
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
# Disable deprecation warnings about <>.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(
options=options,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno if e.lineno is not None else -1,
e.offset,
e.msg,
blocker=True,
code=codes.SYNTAX)
tree = MypyFile([], [], False, {})
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def parse_second_level_ast(code):
second_levels = list()
try:
root_node = ast.parse(code)
except SyntaxError:
try:
root_node = ast27.parse(code)
except (SyntaxError, RuntimeError):
return list()
# for node in root_node.body:
# if isinstance(node, ast27.Import) or isinstance(node, ast.Import):
# second_levels.extend(parse_import_node_second_levels(node))
# if isinstance(node, ast27.ImportFrom) or isinstance(node, ast.ImportFrom):
# second_levels.extend(parse_importfrom_node_second_levels(node))
# second_levels = list(set(second_levels))
second_levels = parse_body(root_node.body, second_levels,
parse_import_node_second_levels,
parse_importfrom_node_second_levels)
return second_levels
def parse_top_level_ast(code):
top_levels = list()
try:
root_node = ast.parse(code)
except SyntaxError:
try:
root_node = ast27.parse(code)
# root_node = py2to3(root_node)
except SyntaxError:
return list()
# for node in root_node.body:
# if isinstance(node, ast27.Import) or isinstance(node, ast.Import):
# top_levels.extend(parse_import_node_top_levels(node))
# if isinstance(node, ast27.ImportFrom) or isinstance(node, ast.ImportFrom):
# top_levels.extend(parse_importfrom_node_top_level(node))
# top_levels = list(set(top_levels))
top_levels = parse_body(root_node.body, top_levels,
parse_import_node_top_levels,
parse_importfrom_node_top_levels)
return top_levels
def process_file(options,
source_text=None,
kythe_args=None,
keep_pytype_data=False,
ast_factory=None,
annotate_ast=False):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
kythe_args: Extra args for generating the kythe index
keep_pytype_data: Whether to preserve the Reference.data field. If true, the
field will hold the type of the reference as a str or Tuple[str, str] (for
attributes). Otherwise, it will be inaccessible.
ast_factory: Callable to return an ast-module-compatible object to parse the
source text into an ast-compatible object. It is passed the pytype Options
object. If not specified, typed_ast will be used.
annotate_ast: Whether to annotate the ast with type information. Nodes with
type information will have these attributes added:
* `.resolved_type`: the pytd information about the type
* `.resolved_annotation`: A string representation of the type, as would
be written in an annotation.
Returns:
The Indexer object used for indexing, and the created AST object. The
AST object may have been modified if `annotate_ast=True`.
Raises:
PytypeError if pytype fails.
"""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(
errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
pytype_ast, _ = analyze.infer_types(
src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
if ast_factory:
ast = ast_factory(options)
ast_root_node = ast.parse(src, options.input)
else:
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
ast_root_node = ast27.parse(src, options.input)
ast = ast27
else:
ast_root_node = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(
source, vm.loader, module_name, kythe_args, annotate_ast=annotate_ast)
ix.index(ast_root_node)
ix.finalize(keep_pytype_data, pytype_ast)
return ix, ast_root_node
def _compute_ast(self) -> _ast27.AST:
return _ast27.parse(self.source)
def process_file(filename):
metrics = []
with lock:
file_info = get_file_info(filename)
file_total_length = get_file_total_length(filename)
file_lines = file_info.lines
file_content = file_info.content
if not file_lines:
raise Exception
# layout features
tabs_ratio = get_tabs_ratio(file_lines, file_total_length)
metrics.append(tabs_ratio)
space_ratio = get_spaces_ratio(file_lines, file_total_length)
metrics.append(space_ratio)
empty_lines = get_empty_lines_ratio(file_lines)
metrics.append(empty_lines)
whitespaces_ratio = get_whitespaces_ratio(file_lines, file_total_length)
metrics.append(whitespaces_ratio)
before_bracket = get_spaces_before_bracket(file_lines, file_total_length)
metrics.append(before_bracket)
after_bracket = get_spaces_after_bracket(file_lines, file_total_length)
metrics.append(after_bracket)
after_equal = get_spaces_after_equal(file_lines, file_total_length)
metrics.append(after_equal)
before_equal = get_spaces_before_equal(file_lines, file_total_length)
metrics.append(before_equal)
after_comma = get_spaces_after_comma(file_lines, file_total_length)
metrics.append(after_comma)
# lexical features
line_count = len(file_lines)
if line_count == 0:
raise Exception
# get tree and is_ast3
is_ast3 = False
try:
tree = ast27.parse(file_content)
except Exception as ex:
try:
tree = ast3.parse(file_content)
is_ast3 = True
except Exception as ex:
raise ex
# end of tree generating
comments_info = count_comments(file_lines)
function_info = get_functions_info(tree, is_ast3)
line_lengths = get_line_lengths(file_lines)
name_lengths = get_names_lengths(tree, is_ast3)
metrics.append(comments_info.string / line_count)
metrics.append(comments_info.multiline / line_count)
literals_count = get_literals_count(tree, is_ast3)
metrics.append(literals_count / line_count)
unique_keywords = get_unique_keywords(tree, is_ast3)
metrics.append(len(unique_keywords) / line_count)
metrics.append(function_info.func_count / line_count)
branching_factor = get_branching_factor(tree, is_ast3)
metrics.append(branching_factor)
metrics.append(get_func_args_avg(function_info.args_count))
metrics.append(get_func_args_std(function_info.args_count))
metrics.append(get_avg_line_len(line_lengths))
metrics.append(get_std_line_len(line_lengths))
metrics.append(get_avg_function_name_length(function_info.name_lengths))
metrics.append(get_std_function_name_length(function_info.name_lengths))
metrics.append(get_avg_name_lengths(name_lengths))
metrics.append(get_std_name_lengths(name_lengths))
# syntactic features
try:
slices_count = get_slices_count(tree, is_ast3)
metrics.append(slices_count / line_count)
except Exception:
print('Slices error')
try:
compreh_count = get_compreh_count(tree, is_ast3)
metrics.append(compreh_count / line_count)
except Exception:
print('Compreh error')
try:
max_tree_depth = get_max_tree_depth(tree)
metrics.append(max_tree_depth)
except Exception:
print('Depth error')
try:
depth_info = get_depth_info(tree, is_ast3)
avg_depth = get_avg_depth(depth_info, is_ast3)
metrics.extend(avg_depth) # list
except Exception:
print('Avg depth error')
try:
max_depths = get_max_depths(depth_info)
metrics.extend(max_depths)
except Exception:
print('Max depths error')
try:
keywords_avg = [
k / line_count for k in get_keywords_count(tree, is_ast3)
]
metrics.extend(keywords_avg) # list
except Exception:
print('Kw count error')
try:
tf_freq, idf_freq = get_term_frequency(tree, is_ast3)
metrics.extend(tf_freq)
except Exception:
print('TF error')
try:
bigram = get_bigrams_freq(tree, is_ast3)
metrics.extend(bigram)
except Exception as ex:
print('Bigram error')
return metrics, tf_freq, idf_freq
def regen(s):
return self.gen.generate(ast27.parse(s), 0)
def process_file(options,
source_text=None,
generate_callgraphs=False,
preserve_pytype_vm=False):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
generate_callgraphs: Collect call graph information
preserve_pytype_vm: Preserve the pytype vm in the indexer
Returns:
The Indexer object used for indexing.
Raises:
PytypeError if pytype fails.
"""
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=True,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
pytd_module, _ = analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
ast_root_node = ast27.parse(src, options.input)
ast = ast27
else:
ast_root_node = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
src_code = source.Code(src,
vm.opcode_traces,
VmTrace,
filename=options.input)
ix = Indexer(ast=ast,
src=src_code,
loader=vm.loader,
module_name=module_name,
pytd_module=pytd_module)
ix.index(ast_root_node)
ix.finalize()
# Make the vm available via indexer.vm for post-finalize() functions.
ix.vm = vm
# Use the indexer as a single object to hold data for calling processes.
if generate_callgraphs:
ix.function_map = callgraph.collect_function_map(ix)
# Release the vm before returning
if not preserve_pytype_vm:
ix.vm = None
return ix
