def parse(source: Union[str, bytes], fnam: str, module: Optional[str], errors: Optional[Errors] = None, options: Optional[Options] = None) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ raise_on_error = False if errors is None: errors = Errors() raise_on_error = True if options is None: options = Options() errors.set_file(fnam, module) is_stub_file = fnam.endswith('.pyi') try: assert options.python_version[0] < 3 and not is_stub_file ast = ast27.parse(source, fnam, 'exec') tree = ASTConverter(options=options, errors=errors, ).visit(ast) assert isinstance(tree, MypyFile) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: errors.report(e.lineno, e.offset, e.msg, blocker=True) tree = MypyFile([], [], False, set()) if raise_on_error and errors.is_errors(): errors.raise_error() return tree
def parse( source: Union[str, bytes], fnam: Optional[str] = None, errors: Optional[Errors] = None, options: Options = Options()) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ raise_on_error = False if errors is None: errors = Errors() raise_on_error = True errors.set_file('<input>' if fnam is None else fnam, None) is_stub_file = bool(fnam) and fnam.endswith('.pyi') try: assert options.python_version[0] < 3 and not is_stub_file ast = ast27.parse(source, fnam, 'exec') tree = ASTConverter( options=options, is_stub=is_stub_file, errors=errors, ).visit(ast) assert isinstance(tree, MypyFile) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: errors.report(e.lineno, e.offset, e.msg) tree = MypyFile([], [], False, set()) if raise_on_error and errors.is_errors(): errors.raise_error() return tree
def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION, custom_typing_module: str = None) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. The pyversion (major, minor) argument determines the Python syntax variant. """ is_stub_file = bool(fnam) and fnam.endswith('.pyi') try: assert pyversion[0] < 3 and not is_stub_file ast = ast27.parse(source, fnam, 'exec') tree = ASTConverter(pyversion=pyversion, is_stub=is_stub_file, custom_typing_module=custom_typing_module, ).visit(ast) assert isinstance(tree, MypyFile) tree.path = fnam tree.is_stub = is_stub_file return tree except (SyntaxError, TypeCommentParseError) as e: if errors: errors.set_file('<input>' if fnam is None else fnam) errors.report(e.lineno, e.offset, e.msg) else: raise return MypyFile([], [], False, set())
def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION, custom_typing_module: str = None) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. The pyversion (major, minor) argument determines the Python syntax variant. """ is_stub_file = bool(fnam) and fnam.endswith('.pyi') try: assert pyversion[0] < 3 and not is_stub_file ast = ast27.parse(source, fnam, 'exec') tree = ASTConverter( pyversion=pyversion, is_stub=is_stub_file, custom_typing_module=custom_typing_module, ).visit(ast) assert isinstance(tree, MypyFile) tree.path = fnam tree.is_stub = is_stub_file return tree except (SyntaxError, TypeCommentParseError) as e: if errors: errors.set_file('<input>' if fnam is None else fnam) errors.report(e.lineno, e.offset, e.msg) else: raise return MypyFile([], [], False, set())
def parse_ast(src: str) -> Union[ast.AST, ast3.AST, ast27.AST]: filename = "<unknown>" if sys.version_info >= (3, 8): # TODO: support Python 4+ ;) for minor_version in range(sys.version_info[1], 4, -1): try: return ast.parse(src, filename, feature_version=(3, minor_version)) except SyntaxError: continue else: for feature_version in (7, 6): try: return ast3.parse(src, filename, feature_version=feature_version) except SyntaxError: continue if ast27.__name__ == "ast": raise SyntaxError( "The requested source code has invalid Python 3 syntax.\n" "If you are trying to format Python 2 files please reinstall Black" " with the 'python2' extra: `python3 -m pip install black[python2]`." ) return ast27.parse(src)
def parse_single_version( src: str, version: Tuple[int, int]) -> Union[ast.AST, ast3.AST, ast27.AST]: filename = "<unknown>" # typed_ast is needed because of feature version limitations in the builtin ast if sys.version_info >= (3, 8) and version >= (3, ): return ast.parse(src, filename, feature_version=version) elif version >= (3, ): return ast3.parse(src, filename, feature_version=version[1]) elif version == (2, 7): return ast27.parse(src) raise AssertionError( "INTERNAL ERROR: Tried parsing unsupported Python version!")
def process_file(options, source_text=None, kythe_args=None): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. kythe_args: Extra args for generating the kythe index Returns: An Indexer object with the indexed code Raises: PytypeError if pytype fails. """ # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) with io.wrap_pytype_exceptions(PytypeError, filename=options.input): analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. a = ast27.parse(src, options.input) ast = ast27 else: a = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer(source, vm.loader, module_name, kythe_args) ix.index(a) ix.finalize() return ix
def _load_27_code(self, code): """ Loads code from a python 27 project and returns the AST once converted to a python 3 AST :param code: The string containing the python code :return: Python 3 AST """ try: # Try parsing as python 2.7 code tree = ast27.parse(code) # convert ast to v3 return conversions.py2to3(tree) except SyntaxError as e: logger.warning( "There was a problem parsing the syntax in this code: %s", e)
def scan(self, code, filename=''): code = utils.sanitize_code(code) self.code = code.splitlines() self.filename = filename try: node = ast.parse(code, filename=self.filename) except SyntaxError as err: pass except (TypeError, ValueError) as err: # Python < 3.5 raises TypeError and Python >= 3.5 raises # ValueError if source contains null bytes. print('{0}: invalid source code "{1}"'.format( utils.format_path(filename), err), file=sys.stderr) self.found_dead_code_or_error = True else: self.visit(node)
def parse_file(source_code: str) -> Optional[Tuple]: try: # NB: The Python 3 ast is generally backwards-compatible with earlier versions. The only # breaking change is `async` `await` becoming reserved keywords in Python 3.7 (deprecated # in 3.6). If the std-lib fails to parse, we could use typed-ast to try parsing with a # target version of Python 3.5, but we don't because Python 3.5 is almost EOL and has very # low usage. # We will also fail to parse Python 3.8 syntax if Pants is run with Python 3.6 or 3.7. # There is no known workaround for this, beyond users changing their `./pants` script to # always use >= 3.8. tree = ast3.parse(source_code) visitor_cls = _Py3AstVisitor if sys.version_info[:2] < (3, 8) else _Py38AstVisitor return tree, visitor_cls except SyntaxError: try: return ast27.parse(source_code), _Py27AstVisitor except SyntaxError: return None
def process_file(options): """Process a single file and return cross references.""" # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) try: analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. a = ast27.parse(src, options.input) ast = ast27 else: a = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer(source, module_name) ix.index(a) ix.lookup_refs() ix.process_deflocs() ix.process_links() return ix
def parse(source: Union[str, bytes], fnam: str, module: Optional[str], errors: Optional[Errors] = None, options: Optional[Options] = None) -> MypyFile: """Parse a source file, without doing any semantic analysis. Return the parse tree. If errors is not provided, raise ParseError on failure. Otherwise, use the errors object to report parse errors. """ raise_on_error = False if errors is None: errors = Errors() raise_on_error = True if options is None: options = Options() errors.set_file(fnam, module) is_stub_file = fnam.endswith('.pyi') try: assert options.python_version[0] < 3 and not is_stub_file # Disable deprecation warnings about <>. with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) ast = ast27.parse(source, fnam, 'exec') tree = ASTConverter( options=options, errors=errors, ).visit(ast) assert isinstance(tree, MypyFile) tree.path = fnam tree.is_stub = is_stub_file except SyntaxError as e: errors.report(e.lineno if e.lineno is not None else -1, e.offset, e.msg, blocker=True, code=codes.SYNTAX) tree = MypyFile([], [], False, {}) if raise_on_error and errors.is_errors(): errors.raise_error() return tree
def parse_second_level_ast(code): second_levels = list() try: root_node = ast.parse(code) except SyntaxError: try: root_node = ast27.parse(code) except (SyntaxError, RuntimeError): return list() # for node in root_node.body: # if isinstance(node, ast27.Import) or isinstance(node, ast.Import): # second_levels.extend(parse_import_node_second_levels(node)) # if isinstance(node, ast27.ImportFrom) or isinstance(node, ast.ImportFrom): # second_levels.extend(parse_importfrom_node_second_levels(node)) # second_levels = list(set(second_levels)) second_levels = parse_body(root_node.body, second_levels, parse_import_node_second_levels, parse_importfrom_node_second_levels) return second_levels
def parse_top_level_ast(code): top_levels = list() try: root_node = ast.parse(code) except SyntaxError: try: root_node = ast27.parse(code) # root_node = py2to3(root_node) except SyntaxError: return list() # for node in root_node.body: # if isinstance(node, ast27.Import) or isinstance(node, ast.Import): # top_levels.extend(parse_import_node_top_levels(node)) # if isinstance(node, ast27.ImportFrom) or isinstance(node, ast.ImportFrom): # top_levels.extend(parse_importfrom_node_top_level(node)) # top_levels = list(set(top_levels)) top_levels = parse_body(root_node.body, top_levels, parse_import_node_top_levels, parse_importfrom_node_top_levels) return top_levels
def process_file(options, source_text=None, kythe_args=None, keep_pytype_data=False, ast_factory=None, annotate_ast=False): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. kythe_args: Extra args for generating the kythe index keep_pytype_data: Whether to preserve the Reference.data field. If true, the field will hold the type of the reference as a str or Tuple[str, str] (for attributes). Otherwise, it will be inaccessible. ast_factory: Callable to return an ast-module-compatible object to parse the source text into an ast-compatible object. It is passed the pytype Options object. If not specified, typed_ast will be used. annotate_ast: Whether to annotate the ast with type information. Nodes with type information will have these attributes added: * `.resolved_type`: the pytd information about the type * `.resolved_annotation`: A string representation of the type, as would be written in an annotation. Returns: The Indexer object used for indexing, and the created AST object. The AST object may have been modified if `annotate_ast=True`. Raises: PytypeError if pytype fails. """ # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer( errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) with io.wrap_pytype_exceptions(PytypeError, filename=options.input): pytype_ast, _ = analyze.infer_types( src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) if ast_factory: ast = ast_factory(options) ast_root_node = ast.parse(src, options.input) else: major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. ast_root_node = ast27.parse(src, options.input) ast = ast27 else: ast_root_node = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer( source, vm.loader, module_name, kythe_args, annotate_ast=annotate_ast) ix.index(ast_root_node) ix.finalize(keep_pytype_data, pytype_ast) return ix, ast_root_node
def _compute_ast(self) -> _ast27.AST: return _ast27.parse(self.source)
def process_file(filename): metrics = [] with lock: file_info = get_file_info(filename) file_total_length = get_file_total_length(filename) file_lines = file_info.lines file_content = file_info.content if not file_lines: raise Exception # layout features tabs_ratio = get_tabs_ratio(file_lines, file_total_length) metrics.append(tabs_ratio) space_ratio = get_spaces_ratio(file_lines, file_total_length) metrics.append(space_ratio) empty_lines = get_empty_lines_ratio(file_lines) metrics.append(empty_lines) whitespaces_ratio = get_whitespaces_ratio(file_lines, file_total_length) metrics.append(whitespaces_ratio) before_bracket = get_spaces_before_bracket(file_lines, file_total_length) metrics.append(before_bracket) after_bracket = get_spaces_after_bracket(file_lines, file_total_length) metrics.append(after_bracket) after_equal = get_spaces_after_equal(file_lines, file_total_length) metrics.append(after_equal) before_equal = get_spaces_before_equal(file_lines, file_total_length) metrics.append(before_equal) after_comma = get_spaces_after_comma(file_lines, file_total_length) metrics.append(after_comma) # lexical features line_count = len(file_lines) if line_count == 0: raise Exception # get tree and is_ast3 is_ast3 = False try: tree = ast27.parse(file_content) except Exception as ex: try: tree = ast3.parse(file_content) is_ast3 = True except Exception as ex: raise ex # end of tree generating comments_info = count_comments(file_lines) function_info = get_functions_info(tree, is_ast3) line_lengths = get_line_lengths(file_lines) name_lengths = get_names_lengths(tree, is_ast3) metrics.append(comments_info.string / line_count) metrics.append(comments_info.multiline / line_count) literals_count = get_literals_count(tree, is_ast3) metrics.append(literals_count / line_count) unique_keywords = get_unique_keywords(tree, is_ast3) metrics.append(len(unique_keywords) / line_count) metrics.append(function_info.func_count / line_count) branching_factor = get_branching_factor(tree, is_ast3) metrics.append(branching_factor) metrics.append(get_func_args_avg(function_info.args_count)) metrics.append(get_func_args_std(function_info.args_count)) metrics.append(get_avg_line_len(line_lengths)) metrics.append(get_std_line_len(line_lengths)) metrics.append(get_avg_function_name_length(function_info.name_lengths)) metrics.append(get_std_function_name_length(function_info.name_lengths)) metrics.append(get_avg_name_lengths(name_lengths)) metrics.append(get_std_name_lengths(name_lengths)) # syntactic features try: slices_count = get_slices_count(tree, is_ast3) metrics.append(slices_count / line_count) except Exception: print('Slices error') try: compreh_count = get_compreh_count(tree, is_ast3) metrics.append(compreh_count / line_count) except Exception: print('Compreh error') try: max_tree_depth = get_max_tree_depth(tree) metrics.append(max_tree_depth) except Exception: print('Depth error') try: depth_info = get_depth_info(tree, is_ast3) avg_depth = get_avg_depth(depth_info, is_ast3) metrics.extend(avg_depth) # list except Exception: print('Avg depth error') try: max_depths = get_max_depths(depth_info) metrics.extend(max_depths) except Exception: print('Max depths error') try: keywords_avg = [ k / line_count for k in get_keywords_count(tree, is_ast3) ] metrics.extend(keywords_avg) # list except Exception: print('Kw count error') try: tf_freq, idf_freq = get_term_frequency(tree, is_ast3) metrics.extend(tf_freq) except Exception: print('TF error') try: bigram = get_bigrams_freq(tree, is_ast3) metrics.extend(bigram) except Exception as ex: print('Bigram error') return metrics, tf_freq, idf_freq
def regen(s): return self.gen.generate(ast27.parse(s), 0)
def process_file(options, source_text=None, generate_callgraphs=False, preserve_pytype_vm=False): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. generate_callgraphs: Collect call graph information preserve_pytype_vm: Preserve the pytype vm in the indexer Returns: The Indexer object used for indexing. Raises: PytypeError if pytype fails. """ errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=True, loader=loader) with io.wrap_pytype_exceptions(PytypeError, filename=options.input): pytd_module, _ = analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. ast_root_node = ast27.parse(src, options.input) ast = ast27 else: ast_root_node = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" src_code = source.Code(src, vm.opcode_traces, VmTrace, filename=options.input) ix = Indexer(ast=ast, src=src_code, loader=vm.loader, module_name=module_name, pytd_module=pytd_module) ix.index(ast_root_node) ix.finalize() # Make the vm available via indexer.vm for post-finalize() functions. ix.vm = vm # Use the indexer as a single object to hold data for calling processes. if generate_callgraphs: ix.function_map = callgraph.collect_function_map(ix) # Release the vm before returning if not preserve_pytype_vm: ix.vm = None return ix