def post_db_load_blob(self, blob):
filepath = blob.get("src", "")
if filepath.endswith(
".phtml") and filepath.find("/views/scripts/") > 0:
# Wrap the blob data in an implicit Zend view:
# class implicit_zend_view extends Zend_View {
# function implicit_render() {
# ... data ...
# }
# }
child_nodes = blob.getchildren()
implicit_view_class = ET.SubElement(
blob,
"scope",
ilk="class",
name="(Zend_View)",
classrefs="Zend_View",
attributes="__fabricated__ __hidden__",
line="0")
view_method = ET.SubElement(implicit_view_class,
"scope",
ilk="function",
name="(render)",
signature="",
attributes="__fabricated__ __hidden__",
line="0")
for child in child_nodes:
blob.remove(child)
view_method.append(child)
def scan_buf(buf, mtime=None, lang="Go"):
"""Scan the given GoBuffer return an ElementTree (conforming
to the CIX schema) giving a summary of its code elements.
@param buf {GoBuffer} is the Go buffer to scan
@param mtime {int} is a modified time for the file (in seconds since
the "epoch"). If it is not specified the _current_ time is used.
Note that the default is not to stat() the file and use that
because the given content might not reflect the saved file state.
"""
# Dev Notes:
# - This stub implementation of the Go CILE return an "empty"
# summary for the given content, i.e. CIX content that says "there
# are no code elements in this Go content".
# - Use the following command (in the extension source dir) to
# debug/test your scanner:
# codeintel scan -p -l Go <example-Go-file>
# "codeintel" is a script available in the Komodo SDK.
log.info("scan '%s'", buf.path)
if mtime is None:
mtime = int(time.time())
# The 'path' attribute must use normalized dir separators.
if sys.platform.startswith("win"):
path = buf.path.replace('\\', '/')
else:
path = buf.path
tree = ET.Element("codeintel",
version="2.0",
xmlns="urn:activestate:cix:2.0")
file = ET.SubElement(tree, "file", lang=lang, mtime=str(mtime))
blob = ET.SubElement(file,
"scope",
ilk="blob",
lang=lang,
name=os.path.basename(path))
#TODO:
# - A 'package' -> 'blob'. Problem is a single go package can be from
# multiple files... so really would want `lib.get_blobs(name)` instead
# of `lib.get_blob(name)` in the codeintel API. How does Ruby deal with
# this? Perl?
# - How do the multi-platform stdlib syscall_linux.go et al fit together?
# Dev Note:
# This is where you process the Go content and add CIX elements
# to 'blob' as per the CIX schema (cix-2.0.rng). Use the
# "buf.accessor" API (see class Accessor in codeintel2.accessor) to
# analyze. For example:
# - A token stream of the content is available via:
# buf.accessor.gen_tokens()
# Use the "codeintel html -b <example-Go-file>" command as
# a debugging tool.
# - "buf.accessor.text" is the whole content of the file. If you have
# a separate tokenizer/scanner tool for Go content, you may
# want to use it.
return tree
def post_db_load_blob(self, blob):
# Add this to the module top-level:
# class Foo(object):
# def foo(self):
# pass
foo_class = ET.SubElement(blob, "scope", ilk="class", name="Foo",
classrefs="object", line="0")
foo_method = ET.SubElement(foo_class, "scope", ilk="function",
name="foo", signature="foo()", line="0")
self_arg = ET.SubElement(foo_method, "variable", citdl="Foo",
ilk="argument", name="self")
def add_var(elt, name, citdl=None, local=True, arg=False):
if not citdl:
citdl = BASE_TYPE
return ET.SubElement(elt,
"variable",
name=name,
citdl=citdl,
attributes=("__local__" if local else ""),
ilk=("argument" if arg else ""))
def convert_mixinrefs(elem):
global adjusted_node
for child in elem:
convert_mixinrefs(child)
mixinrefs = elem.get("mixinrefs")
if mixinrefs and elem.tag == "scope":
for m in mixinrefs.split(" "):
if m == "Kernel" and elem.get("classrefs"): continue
ET.SubElement(elem, 'import', symbol=m)
adjusted_node += 1
#XXX Trent - non-deprecated way to delete an attribute?
del elem.attrib['mixinrefs']
def tree(self):
"""The CIX tree for this buffer. Will lazily scan if necessary."""
self.acquire_lock()
try:
# SIDE-EFFECT: scan if necessary
blob_from_lang = self.blob_from_lang
tree = ET.Element("codeintel", version="2.0")
path = self.path
if os.sep != '/':
path = path.replace(os.sep, '/')
file = ET.SubElement(tree, "file", path=path,
lang=self.lang,
mtime=str(self._scan_time_cache))
if self._scan_error_cache:
file.set("error", self._scan_error_cache)
if blob_from_lang:
for lang, blob in sorted(blob_from_lang.items()):
file.append(blob)
return tree
finally:
self.release_lock()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def scan_buf(buf, mtime=None, lang="TorqueScript"):
"""Scan the given TorqueScriptBuffer return an ElementTree (conforming
to the CIX schema) giving a summary of its code elements.
@param buf {TorqueScriptBuffer} is the TorqueScript buffer to scan
@param mtime {int} is a modified time for the file (in seconds since
the "epoch"). If it is not specified the _current_ time is used.
Note that the default is not to stat() the file and use that
because the given content might not reflect the saved file state.
"""
# Dev Notes:
# - This stub implementation of the TorqueScript CILE return an "empty"
# summary for the given content, i.e. CIX content that says "there
# are no code elements in this TorqueScript content".
# - Use the following command (in the extension source dir) to
# debug/test your scanner:
# codeintel scan -p -l TorqueScript <example-TorqueScript-file>
# "codeintel" is a script available in the Komodo SDK.
#log.info("scan '%s'", buf.path)
if mtime is None:
mtime = int(time.time())
# The 'path' attribute must use normalized dir separators.
if sys.platform.startswith("win"):
path = buf.path.replace('\\', '/')
else:
path = buf.path
tree = ET.Element("codeintel",
version="2.0",
xmlns="urn:activestate:cix:2.0")
file_elt = ET.SubElement(tree, "file", lang=lang, mtime=str(mtime))
blob = ET.SubElement(file_elt,
"scope",
ilk="blob",
lang=lang,
name=os.path.basename(path))
# Dev Note:
# This is where you process the TorqueScript content and add CIX elements
# to 'blob' as per the CIX schema (cix-2.0.rng). Use the
# "buf.accessor" API (see class Accessor in codeintel2.accessor) to
# analyze. For example:
# - A token stream of the content is available via:
# buf.accessor.gen_tokens()
# Use the "codeintel html -b <example-TorqueScript-file>" command as
# a debugging tool.
# - "buf.accessor.text" is the whole content of the file. If you have
# a separate tokenizer/scanner tool for TorqueScript content, you may
# want to use it.
#log.info("Setting scan buffer")
old_stdout = sys.stdout
sys.stdout = sys.stderr
ytab.yy_clear_stacks()
torque_lex.set_scan_buffer(buf.accessor.text, is_filename=False)
try:
#log.info("Attempting parse")
successful_parse = not ytab.yyparse()
except Exception:
successful_parse = False
import traceback
traceback.print_exc(file=sys.stderr)
traceback.print_tb(sys.exc_info()[2], file=sys.stderr)
if successful_parse:
#let's extract something here
ts_ast = ytab.yyvs[1]
#log.info("Extracting blob")
extract_blob_data(ts_ast, blob, file_elt)
else:
file_elt.set("error", "Error parsing file")
sys.stdout = old_stdout
return tree
def extract_blob_data(ast, default_blob, script_root, local_namespace=None):
# we know this is a statement list
# that's all that yyparse will ever give us
exports_symbols = False # Let's see what we find here
# Assignment: 'AssignExprNode', 'AssignOpExprNode', SlotAssignNode','SlotAssignOpNode',
# List: 'StmtList'
# Recurse Tree: 'ReturnStmtNode', 'ExprNode'
# 'FloatBinaryExprNode', 'FloatNode', 'FloatUnaryExprNode', 'FuncCallExprNode',
# Declaration: 'FunctionDeclStmtNode',ObjectDeclNode
# Tricky: IfStmtNode, LoopStmtNode
# Useless: 'BreakStmtNode','ContinueStmtNode',
# Ignore: 'VarNode'
# Dunno: InternalSlotAccessNode
if not local_namespace:
local_namespace = default_blob
for statement in ast.stmts:
if isinstance(statement, StmtList):
extract_blob_data(statement,
default_blob,
script_root,
local_namespace=local_namespace)
# We need a better heuristic for who we might be doc'ing
elif isinstance(statement, StrConstNode) and statement.doc:
pass
elif isinstance(statement, IfStmtNode):
if statement.test_expr:
handle_expr(statement.test_expr, default_blob, script_root,
local_namespace)
elif isinstance(statement, LoopStmtNode):
pass
elif isinstance(statement, FunctionDeclStmtNode):
# declare function
if statement.package_name:
# add a new module or get an existing one
lang = default_blob.get('lang')
for parent_module in script_root.findall("./scope"):
if test_elt_attrs(parent_module,
ilk="blob",
lang=lang,
name=statement.package_name):
break
else:
parent_module = ET.SubElement(script_root,
"scope",
ilk="blob",
lang=lang,
name=statement.package_name)
else:
parent_module = default_blob
if statement.namespace:
# find the appropriate namespace, or add a new one
for parent_elt in parent_module.findall("./scope"):
if test_elt_attrs(parent_elt,
ilk="class",
name=statement.namespace):
break
else:
parent_elt = ET.SubElement(parent_module,
"scope",
ilk="class",
name=statement.namespace)
else:
parent_elt = parent_module
#log.debug("-->Declaring function: %s%s%s",
# ((statement.package_name + "::") if statement.package_name else ""),
# ((statement.namespace + "::") if statement.namespace else ""),
# statement.fn_name)
new_function = ET.SubElement(parent_elt,
"scope",
ilk="function",
name=statement.fn_name)
add_function_args(new_function, statement)
extract_blob_data(statement.stmts,
default_blob,
script_root,
local_namespace=new_function)
elif isinstance(statement, ObjectDeclNode):
# declare namespace
pass
# default behavior, this should be last, since many of the above are subclasses of ExprNode that we know how to handle better
elif isinstance(statement, (ExprNode, ReturnStmtNode)):
# recurse the tree for this expression, and extract any assignment expressions
handle_expr((statement if isinstance(statement, ExprNode) else
statement.expr), default_blob, script_root,
local_namespace)
def handle_expr(statement, default_blob, script_root, local_namespace):
if is_func(statement, "activatepackage"):
is_import = IMPORT_PACKAGE
elif is_func(statement, "exec"):
is_import = IMPORT_EXEC
else:
is_import = False
#log.info("-->Evaluating expression for variable declarations and type inferencing")
if is_import:
#log.info("-->It's an %s-type import", ("exec" if is_import == IMPORT_EXEC else "package"))
num_args = len(statement.args.stmts)
if not num_args:
#log.warn("--->With no arguments, skipping")
pack_name_node = None
elif num_args == 1:
package_name_node, = statement.args.stmts
elif is_import == IMPORT_EXEC and num_args > 1:
package_name_node = statement.args.stmts[0]
else:
#log.warn("--->Something was wrong with it")
package_name_node = None
if not package_name_node:
name_data = ""
elif isinstance(package_name_node, (ConstantNode, StrConstNode)):
name_data = package_name_node.value
elif isinstance(package_name_node, ExprNode):
#log.warn("--->This is a programmatically generated import, let's skip it")
name_name, name_data, name_type = handle_expr(
package_name_node, default_blob, script_root, local_namespace)
# this really needs to be processed more
else:
pass
#log.warn("--->We have an import expression of unknown provenance")
if name_data and is_import == IMPORT_EXEC:
name_data = find_exec_file(name_data)
if name_data:
do_import = ET.SubElement(default_blob,
"import",
name=str(name_data),
symbol="*")
expr_name = ""
expr_data = ""
expr_types = [
type_name for type_name in (BASE_TYPE, "IntType")
if type_name == BASE_TYPE or is_import == IMPORT_EXEC
]
else:
expr_name = ""
expr_data = ""
expr_types = [BASE_TYPE]
if isinstance(statement, (AssignExprNode, AssignOpExprNode)):
# it's a variable
# handle assignment
pass
elif isinstance(statement, (SlotAssignNode, SlotAssignOpNode)):
# it's an array
# handle assignment
pass
#nv_name, expr_data
return (expr_name, expr_data, " ".join(expr_types))
