def post_db_load_blob(self, blob): filepath = blob.get("src", "") if filepath.endswith( ".phtml") and filepath.find("/views/scripts/") > 0: # Wrap the blob data in an implicit Zend view: # class implicit_zend_view extends Zend_View { # function implicit_render() { # ... data ... # } # } child_nodes = blob.getchildren() implicit_view_class = ET.SubElement( blob, "scope", ilk="class", name="(Zend_View)", classrefs="Zend_View", attributes="__fabricated__ __hidden__", line="0") view_method = ET.SubElement(implicit_view_class, "scope", ilk="function", name="(render)", signature="", attributes="__fabricated__ __hidden__", line="0") for child in child_nodes: blob.remove(child) view_method.append(child)
def scan_buf(buf, mtime=None, lang="Go"): """Scan the given GoBuffer return an ElementTree (conforming to the CIX schema) giving a summary of its code elements. @param buf {GoBuffer} is the Go buffer to scan @param mtime {int} is a modified time for the file (in seconds since the "epoch"). If it is not specified the _current_ time is used. Note that the default is not to stat() the file and use that because the given content might not reflect the saved file state. """ # Dev Notes: # - This stub implementation of the Go CILE return an "empty" # summary for the given content, i.e. CIX content that says "there # are no code elements in this Go content". # - Use the following command (in the extension source dir) to # debug/test your scanner: # codeintel scan -p -l Go <example-Go-file> # "codeintel" is a script available in the Komodo SDK. log.info("scan '%s'", buf.path) if mtime is None: mtime = int(time.time()) # The 'path' attribute must use normalized dir separators. if sys.platform.startswith("win"): path = buf.path.replace('\\', '/') else: path = buf.path tree = ET.Element("codeintel", version="2.0", xmlns="urn:activestate:cix:2.0") file = ET.SubElement(tree, "file", lang=lang, mtime=str(mtime)) blob = ET.SubElement(file, "scope", ilk="blob", lang=lang, name=os.path.basename(path)) #TODO: # - A 'package' -> 'blob'. Problem is a single go package can be from # multiple files... so really would want `lib.get_blobs(name)` instead # of `lib.get_blob(name)` in the codeintel API. How does Ruby deal with # this? Perl? # - How do the multi-platform stdlib syscall_linux.go et al fit together? # Dev Note: # This is where you process the Go content and add CIX elements # to 'blob' as per the CIX schema (cix-2.0.rng). Use the # "buf.accessor" API (see class Accessor in codeintel2.accessor) to # analyze. For example: # - A token stream of the content is available via: # buf.accessor.gen_tokens() # Use the "codeintel html -b <example-Go-file>" command as # a debugging tool. # - "buf.accessor.text" is the whole content of the file. If you have # a separate tokenizer/scanner tool for Go content, you may # want to use it. return tree
def post_db_load_blob(self, blob): # Add this to the module top-level: # class Foo(object): # def foo(self): # pass foo_class = ET.SubElement(blob, "scope", ilk="class", name="Foo", classrefs="object", line="0") foo_method = ET.SubElement(foo_class, "scope", ilk="function", name="foo", signature="foo()", line="0") self_arg = ET.SubElement(foo_method, "variable", citdl="Foo", ilk="argument", name="self")
def add_var(elt, name, citdl=None, local=True, arg=False): if not citdl: citdl = BASE_TYPE return ET.SubElement(elt, "variable", name=name, citdl=citdl, attributes=("__local__" if local else ""), ilk=("argument" if arg else ""))
def convert_mixinrefs(elem): global adjusted_node for child in elem: convert_mixinrefs(child) mixinrefs = elem.get("mixinrefs") if mixinrefs and elem.tag == "scope": for m in mixinrefs.split(" "): if m == "Kernel" and elem.get("classrefs"): continue ET.SubElement(elem, 'import', symbol=m) adjusted_node += 1 #XXX Trent - non-deprecated way to delete an attribute? del elem.attrib['mixinrefs']
def tree(self): """The CIX tree for this buffer. Will lazily scan if necessary.""" self.acquire_lock() try: # SIDE-EFFECT: scan if necessary blob_from_lang = self.blob_from_lang tree = ET.Element("codeintel", version="2.0") path = self.path if os.sep != '/': path = path.replace(os.sep, '/') file = ET.SubElement(tree, "file", path=path, lang=self.lang, mtime=str(self._scan_time_cache)) if self._scan_error_cache: file.set("error", self._scan_error_cache) if blob_from_lang: for lang, blob in sorted(blob_from_lang.items()): file.append(blob) return tree finally: self.release_lock()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [ child for child in function if child.tag != 'variable' ] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write( '<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def scan_buf(buf, mtime=None, lang="TorqueScript"): """Scan the given TorqueScriptBuffer return an ElementTree (conforming to the CIX schema) giving a summary of its code elements. @param buf {TorqueScriptBuffer} is the TorqueScript buffer to scan @param mtime {int} is a modified time for the file (in seconds since the "epoch"). If it is not specified the _current_ time is used. Note that the default is not to stat() the file and use that because the given content might not reflect the saved file state. """ # Dev Notes: # - This stub implementation of the TorqueScript CILE return an "empty" # summary for the given content, i.e. CIX content that says "there # are no code elements in this TorqueScript content". # - Use the following command (in the extension source dir) to # debug/test your scanner: # codeintel scan -p -l TorqueScript <example-TorqueScript-file> # "codeintel" is a script available in the Komodo SDK. #log.info("scan '%s'", buf.path) if mtime is None: mtime = int(time.time()) # The 'path' attribute must use normalized dir separators. if sys.platform.startswith("win"): path = buf.path.replace('\\', '/') else: path = buf.path tree = ET.Element("codeintel", version="2.0", xmlns="urn:activestate:cix:2.0") file_elt = ET.SubElement(tree, "file", lang=lang, mtime=str(mtime)) blob = ET.SubElement(file_elt, "scope", ilk="blob", lang=lang, name=os.path.basename(path)) # Dev Note: # This is where you process the TorqueScript content and add CIX elements # to 'blob' as per the CIX schema (cix-2.0.rng). Use the # "buf.accessor" API (see class Accessor in codeintel2.accessor) to # analyze. For example: # - A token stream of the content is available via: # buf.accessor.gen_tokens() # Use the "codeintel html -b <example-TorqueScript-file>" command as # a debugging tool. # - "buf.accessor.text" is the whole content of the file. If you have # a separate tokenizer/scanner tool for TorqueScript content, you may # want to use it. #log.info("Setting scan buffer") old_stdout = sys.stdout sys.stdout = sys.stderr ytab.yy_clear_stacks() torque_lex.set_scan_buffer(buf.accessor.text, is_filename=False) try: #log.info("Attempting parse") successful_parse = not ytab.yyparse() except Exception: successful_parse = False import traceback traceback.print_exc(file=sys.stderr) traceback.print_tb(sys.exc_info()[2], file=sys.stderr) if successful_parse: #let's extract something here ts_ast = ytab.yyvs[1] #log.info("Extracting blob") extract_blob_data(ts_ast, blob, file_elt) else: file_elt.set("error", "Error parsing file") sys.stdout = old_stdout return tree
def extract_blob_data(ast, default_blob, script_root, local_namespace=None): # we know this is a statement list # that's all that yyparse will ever give us exports_symbols = False # Let's see what we find here # Assignment: 'AssignExprNode', 'AssignOpExprNode', SlotAssignNode','SlotAssignOpNode', # List: 'StmtList' # Recurse Tree: 'ReturnStmtNode', 'ExprNode' # 'FloatBinaryExprNode', 'FloatNode', 'FloatUnaryExprNode', 'FuncCallExprNode', # Declaration: 'FunctionDeclStmtNode',ObjectDeclNode # Tricky: IfStmtNode, LoopStmtNode # Useless: 'BreakStmtNode','ContinueStmtNode', # Ignore: 'VarNode' # Dunno: InternalSlotAccessNode if not local_namespace: local_namespace = default_blob for statement in ast.stmts: if isinstance(statement, StmtList): extract_blob_data(statement, default_blob, script_root, local_namespace=local_namespace) # We need a better heuristic for who we might be doc'ing elif isinstance(statement, StrConstNode) and statement.doc: pass elif isinstance(statement, IfStmtNode): if statement.test_expr: handle_expr(statement.test_expr, default_blob, script_root, local_namespace) elif isinstance(statement, LoopStmtNode): pass elif isinstance(statement, FunctionDeclStmtNode): # declare function if statement.package_name: # add a new module or get an existing one lang = default_blob.get('lang') for parent_module in script_root.findall("./scope"): if test_elt_attrs(parent_module, ilk="blob", lang=lang, name=statement.package_name): break else: parent_module = ET.SubElement(script_root, "scope", ilk="blob", lang=lang, name=statement.package_name) else: parent_module = default_blob if statement.namespace: # find the appropriate namespace, or add a new one for parent_elt in parent_module.findall("./scope"): if test_elt_attrs(parent_elt, ilk="class", name=statement.namespace): break else: parent_elt = ET.SubElement(parent_module, "scope", ilk="class", name=statement.namespace) else: parent_elt = parent_module #log.debug("-->Declaring function: %s%s%s", # ((statement.package_name + "::") if statement.package_name else ""), # ((statement.namespace + "::") if statement.namespace else ""), # statement.fn_name) new_function = ET.SubElement(parent_elt, "scope", ilk="function", name=statement.fn_name) add_function_args(new_function, statement) extract_blob_data(statement.stmts, default_blob, script_root, local_namespace=new_function) elif isinstance(statement, ObjectDeclNode): # declare namespace pass # default behavior, this should be last, since many of the above are subclasses of ExprNode that we know how to handle better elif isinstance(statement, (ExprNode, ReturnStmtNode)): # recurse the tree for this expression, and extract any assignment expressions handle_expr((statement if isinstance(statement, ExprNode) else statement.expr), default_blob, script_root, local_namespace)
def handle_expr(statement, default_blob, script_root, local_namespace): if is_func(statement, "activatepackage"): is_import = IMPORT_PACKAGE elif is_func(statement, "exec"): is_import = IMPORT_EXEC else: is_import = False #log.info("-->Evaluating expression for variable declarations and type inferencing") if is_import: #log.info("-->It's an %s-type import", ("exec" if is_import == IMPORT_EXEC else "package")) num_args = len(statement.args.stmts) if not num_args: #log.warn("--->With no arguments, skipping") pack_name_node = None elif num_args == 1: package_name_node, = statement.args.stmts elif is_import == IMPORT_EXEC and num_args > 1: package_name_node = statement.args.stmts[0] else: #log.warn("--->Something was wrong with it") package_name_node = None if not package_name_node: name_data = "" elif isinstance(package_name_node, (ConstantNode, StrConstNode)): name_data = package_name_node.value elif isinstance(package_name_node, ExprNode): #log.warn("--->This is a programmatically generated import, let's skip it") name_name, name_data, name_type = handle_expr( package_name_node, default_blob, script_root, local_namespace) # this really needs to be processed more else: pass #log.warn("--->We have an import expression of unknown provenance") if name_data and is_import == IMPORT_EXEC: name_data = find_exec_file(name_data) if name_data: do_import = ET.SubElement(default_blob, "import", name=str(name_data), symbol="*") expr_name = "" expr_data = "" expr_types = [ type_name for type_name in (BASE_TYPE, "IntType") if type_name == BASE_TYPE or is_import == IMPORT_EXEC ] else: expr_name = "" expr_data = "" expr_types = [BASE_TYPE] if isinstance(statement, (AssignExprNode, AssignOpExprNode)): # it's a variable # handle assignment pass elif isinstance(statement, (SlotAssignNode, SlotAssignOpNode)): # it's an array # handle assignment pass #nv_name, expr_data return (expr_name, expr_data, " ".join(expr_types))