def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language " "(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def main():
# More robust for determining the perforce script location
cix_filename = "javascript.cix"
scriptpath = os.path.dirname(__file__)
if not scriptpath:
scriptpath = "."
scriptpath = os.path.abspath(scriptpath)
cix_directory = scriptpath
# Get main codeintel directory, 3 up from this script location!
for i in range(3):
cix_directory = os.path.dirname(cix_directory)
cix_filename = os.path.join(
cix_directory, "lib", "codeintel2", "stdlibs", cix_filename)
# Generate the cix files
for filename in ("ecmaToCodeintel.py", "dom0_to_cix.py", "dom2_to_cix.py"):
p = os.popen("python %s" % (os.path.join(scriptpath, filename))).read()
# Combine the libraries
cixtree = ciElementTree.parse(os.path.join(scriptpath, "javascript.cix"))
cixscope = cixtree.find("file/scope")
# Note: XMLHttpRequest cix comes from the Mozilla implementation in:
# nsIXMLHttpRequest.idl
for domname in ("XMLHttpRequest", "dom0", "dom2"):
# cixscope.append(ciElementTree.Comment(" %s structure " % (domname)))
et = ciElementTree.parse("%s.cix" % (
os.path.join(scriptpath, domname)))
for scope in et.findall("//file/scope"):
for child in scope.getchildren():
# Ensure we remove from the dom tree first, otherwise
# we generate double elements
scope.remove(child)
cixscope.append(child)
pretty_tree_from_tree(cixtree.getroot())
update_documentation_sentences(cixtree)
p4update(cix_filename, ciElementTree.tostring(cixtree.getroot()))
# Update libraries
for dirname, filename in (("dojo", "dojo_json_to_cix.py"),
("MochiKit", "mochikit_to_cix.py"),
("prototype", "prototype_to_cix.py"),
("yui", "yui_to_cix.py")):
library_script_path = os.path.join(scriptpath, dirname, filename)
p = os.popen("python %s -u" % (library_script_path)).read()
def main():
# More robust for determining the perforce script location
cix_filename = "javascript.cix"
scriptpath = os.path.dirname(__file__)
if not scriptpath:
scriptpath = "."
scriptpath = os.path.abspath(scriptpath)
cix_directory = scriptpath
# Get main codeintel directory, 3 up from this script location!
for i in range(3):
cix_directory = os.path.dirname(cix_directory)
cix_filename = os.path.join(cix_directory, "lib", "codeintel2", "stdlibs",
cix_filename)
# Generate the cix files
for filename in ("ecmaToCodeintel.py", "dom0_to_cix.py", "dom2_to_cix.py"):
p = os.popen("python %s" % (os.path.join(scriptpath, filename))).read()
# Combine the libraries
cixtree = ciElementTree.parse(os.path.join(scriptpath, "javascript.cix"))
cixscope = cixtree.find("file/scope")
# Note: XMLHttpRequest cix comes from the Mozilla implementation in:
# nsIXMLHttpRequest.idl
for domname in ("XMLHttpRequest", "dom0", "dom2"):
# cixscope.append(ciElementTree.Comment(" %s structure " % (domname)))
et = ciElementTree.parse("%s.cix" %
(os.path.join(scriptpath, domname)))
for scope in et.findall("//file/scope"):
for child in scope.getchildren():
# Ensure we remove from the dom tree first, otherwise
# we generate double elements
scope.remove(child)
cixscope.append(child)
pretty_tree_from_tree(cixtree.getroot())
update_documentation_sentences(cixtree)
p4update(cix_filename, ciElementTree.tostring(cixtree.getroot()))
# Update libraries
for dirname, filename in (("dojo", "dojo_json_to_cix.py"),
("MochiKit", "mochikit_to_cix.py"),
("prototype", "prototype_to_cix.py"),
("yui", "yui_to_cix.py")):
library_script_path = os.path.join(scriptpath, dirname, filename)
p = os.popen("python %s -u" % (library_script_path)).read()
def update_cix_file(mgr, path):
log.info("convert `%s' to pretty CIX 2.0", path)
cix = open(path, 'r').read()
tree = tree_from_cix(cix) # converts to CIX 2.0
tree = pretty_tree_from_tree(tree)
new_cix = ET.tostring(tree)
_run("p4 edit %s" % path)
open(path, 'w').write(new_cix)
def update_cix_file(mgr, path):
log.info("convert `%s' to pretty CIX 2.0", path)
cix = open(path, 'r').read()
tree = tree_from_cix(cix) # converts to CIX 2.0
tree = pretty_tree_from_tree(tree)
new_cix = ET.tostring(tree)
_run("p4 edit %s" % path)
open(path, 'w').write(new_cix)
def mergeCixFiles(opts, filename1, filename2, outputfilename):
e1 = parse(filename1).getroot().getchildren()[0]
mergedcixroot = parse(filename2).getroot()
e2 = mergedcixroot.getchildren()[0]
elems1 = []
elems2 = []
if opts.lpath:
for lpath in opts.lpath:
elem1 = e1
elem2 = e2
lpath_split = lpath.split(".")
for i in range(len(lpath_split)):
name = lpath_split[i]
new_elem1 = elem1.names.get(name)
new_elem2 = elem2.names.get(name)
if new_elem1 is None and new_elem2 is None:
print "lpath not found in either cix file: %r (skipping it)" % (
lpath, )
break
elif new_elem2 is None:
answer = raw_input(
"lpath %r only found in first cix file, copy over? [Yn]"
% (lpath, ))
if answer.lower() not in ("n", "no"):
merge_missing(elem1, [name], lpath_split[:i - 1], e2)
break
# elif new_elem1 is None:
# print "lpath not found in either cix files: %r" % (lpath, )
# break
elem1 = new_elem1
elem2 = new_elem2
else:
elems1.append(elem1)
elems2.append(elem2)
else:
elems1 = [e1]
elems2 = [e2]
for e1, e2 in zip(elems1, elems2):
print "Diffing elements: %r, %r" % (e1, e2)
diffElements(opts, [], e1, e2)
pretty_tree_from_tree(mergedcixroot)
file(outputfilename, "w").write(cixtostring(mergedcixroot))
def mergeCixFiles(opts, filename1, filename2, outputfilename):
e1 = parse(filename1).getroot().getchildren()[0]
mergedcixroot = parse(filename2).getroot()
e2 = mergedcixroot.getchildren()[0]
elems1 = []
elems2 = []
if opts.lpath:
for lpath in opts.lpath:
elem1 = e1
elem2 = e2
lpath_split = lpath.split(".")
for i in range(len(lpath_split)):
name = lpath_split[i]
new_elem1 = elem1.names.get(name)
new_elem2 = elem2.names.get(name)
if new_elem1 is None and new_elem2 is None:
print "lpath not found in either cix file: %r (skipping it)" % (lpath, )
break
elif new_elem2 is None:
answer = raw_input(
"lpath %r only found in first cix file, copy over? [Yn]" % (lpath, ))
if answer.lower() not in ("n", "no"):
merge_missing(elem1, [name], lpath_split[:i-1], e2)
break
# elif new_elem1 is None:
# print "lpath not found in either cix files: %r" % (lpath, )
# break
elem1 = new_elem1
elem2 = new_elem2
else:
elems1.append(elem1)
elems2.append(elem2)
else:
elems1 = [e1]
elems2 = [e2]
for e1, e2 in zip(elems1, elems2):
print "Diffing elements: %r, %r" % (e1, e2)
diffElements(opts, [], e1, e2)
pretty_tree_from_tree(mergedcixroot)
file(outputfilename, "w").write(cixtostring(mergedcixroot))
def do_cix(self, subcmd, opts, *path_patterns):
"""Read in and print a CIX file (possibly converting to CIX 2.0
and prettifying in the process.
${cmd_usage}
${cmd_option_list}
"""
if opts.pretty_print:
opts.convert = True
for path in _paths_from_path_patterns(path_patterns):
tree = None
cix = open(path, 'r').read()
if opts.convert:
tree = tree_from_cix(cix)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
else:
sys.stdout.write(cix)
def do_cix(self, subcmd, opts, *path_patterns):
"""Read in and print a CIX file (possibly converting to CIX 2.0
and prettifying in the process.
${cmd_usage}
${cmd_option_list}
"""
if opts.pretty_print:
opts.convert = True
for path in _paths_from_path_patterns(path_patterns):
tree = None
cix = open(path, 'r').read()
if opts.convert:
tree = tree_from_cix(cix)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
else:
sys.stdout.write(cix)
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
extra_module_dirs = []
if koextlib.is_ext_dir() and exists("pylib"):
sys.path.append(abspath("pylib"))
extra_module_dirs = [sys.path[-1]]
mgr = Manager(extra_module_dirs=extra_module_dirs)
mgr.upgrade()
mgr.initialize()
try:
tree = None
for path in _paths_from_path_patterns(path_patterns):
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info(
"skip `%s': couldn't determine language "
"(use --lang option)", path)
continue
buf = mgr.buf_from_path(path, lang=opts.lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
buf.scan() # force a fresh scan
tree = buf.tree
for severity, msg in check_tree(tree):
dump = {"warning": log.warn, "error": log.error}[severity]
dump(msg)
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
ET.dump(tree)
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path+"\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
buf = mgr.buf_from_path(path, lang=lang)
if not isinstance(buf, CitadelBuffer):
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end-start))
finally:
mgr.finalize()
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
#mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(
ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) #XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
if t and t.get("ilk") in reg_ilks:
replace_node(origBlobStar, curr_tree, t, i)
i += 1
print "After doing yaml-nodes, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
i = 0
for t in bin_tree:
replace_node(origFile, bin_tree, t, i)
i += 1
print "After doing binary modules, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
# Finally update the YAML nodes, assuming that if we do a replacement
# we can replace the entire subtree.
i = 0
for t in yaml_stdlib_tree:
replace_node(origFile, yaml_stdlib_tree, t, i)
i += 1
print "After doing yaml modules, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
newFile = options.origfile + ".new"
fd = open(newFile, "w")
fd.write(ET.tostring(pretty_tree_from_tree(origTopLevel)))
fd.close()
print "Done writing to file %s" % newFile
def _testOneInputFile(self, fpath, tags=None):
_debug = False # Set to true to dump status info for each test run.
infile = os.path.join(gInputsDir, fpath) # input
outfile = os.path.join(gOutputsDir, fpath + '.cix') # expected output
tmpfile = os.path.join(gTmpDir, fpath + '.cix') # actual output
if not os.path.exists(os.path.dirname(tmpfile)):
os.makedirs(os.path.dirname(tmpfile))
errfile = os.path.join(gOutputsDir, fpath + '.error') # expected error
# An options file is a set of kwargs for the buf.scan()
# method call. One key-value pair per-line like this:
# key=value
# Whitespace is stripped off the value.
optsfile = os.path.join(gInputsDir, fpath + '.options') # input options
if _debug:
print()
print("*" * 50, "codeintel '%s'" % fpath)
# Set standard options:
opts = {"mtime": "42"}
# Determine input options to use, if any.
#XXX Not used. Drop it.
if os.path.exists(optsfile):
for line in open(optsfile, 'r').read().splitlines(0):
name, value = line.split('=', 1)
value = value.strip()
try: # allow value to be a type other than string
value = eval(value)
except Exception:
pass
opts[name] = value
if _debug:
print("*" * 50, "options")
pprint.pprint(opts)
# Scan the file, capturing stdout and stderr and any possible
# error.
# - To allow testing from different dirs (resulting in changing
# path strings, we normalize the <file path="..."> value and any
# <scope ilk="blob" src="..."> attributes).
oldStdout = sys.stdout
oldStderr = sys.stderr
sys.stdout = StringIO()
sys.stderr = StringIO()
try:
try:
lang = None
if tags and "python3" in tags:
lang = "Python3"
buf = self.mgr.buf_from_path(infile, lang=lang)
buf.scan(**opts)
tree = buf.tree
# Normalize paths.
relnorm_infile = infile[len(dirname(gInputsDir)) + 1:]
absnorm_infile = infile
relnorm_infile = relnorm_infile.replace('\\', '/')
absnorm_infile = absnorm_infile.replace('\\', '/')
for file_elem in tree:
file_elem.set("path", relnorm_infile)
for blob_elem in file_elem:
if blob_elem.get("ilk") != "blob": continue
norm_src = normpath(blob_elem.get("src"))
norm_src = norm_src.replace('\\', '/')
if norm_src in (relnorm_infile, absnorm_infile):
blob_elem.set("src", relnorm_infile)
tree = pretty_tree_from_tree(tree)
# Due to the dynamic nature of the ciler errors (which often
# includes the source code line numbers), it's difficult to check
# that the errors are identical, so we work around this by just
# taking the first 30 characters of the error.
cile_error = tree[0].get("error")
if cile_error and fpath.endswith(".js"):
tree[0].set(
"error",
len(cile_error) < 30 and cile_error
or (cile_error[:30] + "..."))
cix = ET.tostring(tree)
except CodeIntelError as ex:
error = traceback.format_exc()
else:
error = None
if isinstance(cix, six.text_type):
with io.open(tmpfile, mode="wt", encoding="utf-8") as fout:
fout.write(cix)
else:
with open(tmpfile, mode="wb") as fout:
fout.write(cix)
finally:
stdout = sys.stdout.getvalue()
stderr = sys.stderr.getvalue()
sys.stdout = oldStdout
sys.stderr = oldStderr
if _debug:
print("*" * 50, "stdout")
print(stdout)
print("*" * 50, "stderr")
print(stderr)
print("*" * 50, "error")
print(str(error))
print("*" * 50)
generateMissing = False
if not os.path.exists(outfile) and generateMissing:
with io.open(outfile, mode='wt', encoding='utf-8') as fout:
with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp:
fout.write(ftmp.read())
# Verify that the results are as expected.
if os.path.exists(outfile) and error:
self.fail("scanning '%s' raised an error but success was "
"expected:\n%s" % (_encode_for_stdout(fpath), indent(error)))
elif os.path.exists(outfile):
# Convert the <file path="..."/> to the native directory separator.
def to_native_sep(match):
path = match.group(2).replace("\\", os.sep).replace("/", os.sep)
return match.group(1) + path + match.group(3)
path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S)
# Note that we don't really care about line endings here, so we read
# both files in universal newlines mode (i.e. translate to \n)
# and normalize ' ', ' ' and '''
with io.open(outfile, mode='rt', encoding='utf-8') as fout:
expected = path_pat.sub(to_native_sep, fout.read())
expected = expected.replace('
', ' ').replace(
'
', ' ').replace(''', '\'')
with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp:
actual = path_pat.sub(to_native_sep, ftmp.read())
actual = actual.replace('
', ' ').replace(
'
', ' ').replace(''', '\'')
if expected != actual:
do_fail = True
# Useful temporary thing while XML output format is changing.
#if os.stat("../support/xmldiff.py"):
# rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile))
# if rc == 0:
# do_fail = False
if do_fail:
diff = list(
difflib.ndiff(expected.splitlines(1),
actual.splitlines(1)))
diff = _diffContext(diff, 2)
if diff:
error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \
% (outfile, tmpfile, outfile, tmpfile,
''.join(diff))
if gMaxDiffOutput > 0 and gMaxNumLines > 0:
if len(error_str) > gMaxDiffOutput:
error_lines = error_str.split("\n")
if len(error_lines) > gMaxNumLines:
error_lines = error_lines[:gMaxNumLines] + [
"..."
]
if gMaxLineLength > 0:
error_str = "\n".join([
len(x) > gMaxLineLength
and x[:gMaxLineLength] or x
for x in error_lines
])
else:
error_str = "\n".join(error_lines)
self.fail(_encode_for_stdout(error_str))
elif os.path.exists(errfile):
# There is no reference output file. This means that processing
# this file is expected to fail.
expectedError = open(errfile, 'r').read()
actualError = str(error)
self.failUnlessEqual(actualError.strip(), expectedError.strip())
else:
self.fail("No reference output file or error file for '%s'." % infile)
# Ensure next test file gets a clean codeintel.
toDelete = []
for modname in sys.modules:
if modname == "codeintel" or modname.startswith("codeintel."):
toDelete.append(modname)
for modname in toDelete:
del sys.modules[modname]
def scan(content, filename, md5sum=None, mtime=None, lang="Python"):
"""Scan the given Python content and return Code Intelligence data
conforming the the Code Intelligence XML format.
"content" is the Python content to scan. This should be an
encoded string: must be a string for `md5` and
`compiler.parse` -- see bug 73461.
"filename" is the source of the Python content (used in the
generated output).
"md5sum" (optional) if the MD5 hexdigest has already been calculated
for the content, it can be passed in here. Otherwise this
is calculated.
"mtime" (optional) is a modified time for the file (in seconds since
the "epoch"). If it is not specified the _current_ time is used.
Note that the default is not to stat() the file and use that
because the given content might not reflect the saved file state.
"lang" (optional) is the language of the given file content.
Typically this is "Python" (i.e. a pure Python file), but it
may also be "DjangoHTML" or similar for Python embedded in
other documents.
XXX Add an optional 'eoltype' so that it need not be
re-calculated if already known.
This can raise one of SyntaxError, PythonCILEError or parser.ParserError
if there was an error processing. Currently this implementation uses the
Python 'compiler' package for processing, therefore the given Python
content must be syntactically correct.
"""
log.info("scan '%s'", filename)
if md5sum is None:
md5sum = md5(content).hexdigest()
if mtime is None:
mtime = int(time.time())
# 'compiler' both (1) wants a newline at the end and (2) can fail on
# funky *whitespace* at the end of the file.
content = content.rstrip() + '\n'
if isinstance(filename, str):
filename = filename.encode('utf-8')
# The 'path' attribute must use normalized dir separators.
if sys.platform.startswith("win"):
path = filename.replace('\\', '/')
else:
path = filename
fileAttrs = {"language": "Python",
"generator": "Python",
"path": path}
try:
tree2 = pythoncile2(path, content).find('file').find('scope')
# print >> sys.stderr, ("******************", tree2, tree2.tag, tree2.attrib); sys.stderr.flush()
# assert False, ('===============', tree2.getchildren()[0].attrib)
if tree2.get('error'):
raise Exception(tree2.get('error'))
if _gClockIt:
sys.stdout.write(" (ast:%.3fs)" % (_gClock()-_gStartTime))
except Exception as ex:
fileAttrs["error"] = str(ex)
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
if tree2 is None:
# This happens, for example, with:
# foo(bar, baz=1, blam)
fileAttrs["error"] = "could not generate AST"
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
fileAttrs["md5"] = md5sum
fileAttrs["mtime"] = mtime
moduleName = os.path.splitext(os.path.basename(filename))[0]
if _gClockIt:
sys.stdout.write(" (walk:%.3fs)" % (_gClock()-_gStartTime))
if log.isEnabledFor(logging.INFO):
# Dump a repr of the gathering info for debugging
# - We only have to dump the module namespace because
# everything else should be linked from it.
for nspath, namespace in list(visitor.st.items()):
if len(nspath) == 0: # this is the module namespace
pprint.pprint(namespace)
file = ' <file%s>\n\n%s\n </file>'\
% (getAttrStr(fileAttrs), cix2)
if _gClockIt:
sys.stdout.write(" (getCIX:%.3fs)" % (_gClock()-_gStartTime))
cix = '''\
<?xml version="1.0" encoding="UTF-8"?>
<codeintel version="0.1">
%s
</codeintel>
''' % file
return cix
def main(argv=None):
if argv is None:
argv = sys.argv
if not logging.root.handlers:
logging.basicConfig()
usage = "usage: %prog [PATHS...]"
version = "%prog "+__version__
parser = optparse.OptionParser(prog="pythoncile2", usage=usage,
version=version, description=__doc__,
formatter=_NoReflowFormatter())
parser.add_option("-v", "--verbose", dest="log_level",
action="store_const", const=logging.DEBUG,
help="more verbose output")
parser.add_option("-q", "--quiet", dest="log_level",
action="store_const", const=logging.ERROR,
help="less verbose output (only show errors)")
parser.add_option("-c", "--compare", action="store_true",
help="run against pythoncile.py as well (for testing)")
parser.set_defaults(log_level=logging.INFO, compare=False)
opts, paths = parser.parse_args(argv)
#log.setLevel(opts.log_level)
assert len(paths) == 1, "usage: pythoncile2.py PATH"
path = paths[0]
try:
tree2 = pythoncile2(path)
except PythonCILEError as ex:
log.error(str(ex))
if log.isEnabledFor(logging.DEBUG):
print()
import traceback
traceback.print_exception(*sys.exc_info())
return 1
except KeyboardInterrupt:
log.debug("user abort")
return 1
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
if opts.compare:
cix1 = _pythoncile_cix_from_path(path)
if log.isEnabledFor(logging.DEBUG):
print("-- pythoncile1 %s" % path)
print(cix1)
print("-- pythoncile2 %s" % path)
print(cix2)
# Normalizing for comparison.
tree1 = ET.fromstring(cix1)
# - mtime slightly different
tree1[0].set("mtime", tree2[0].get("mtime"))
# - pythoncile.py seems to put <import>'s at the top of the scope.
# We'll just get tree2 to look like that.
for scope in tree2.getiterator("scope"):
if scope.get("ilk") not in ("blob", "function"):
continue
indeces = []
n = 0
for i, child in enumerate(scope[:]):
if child.tag == "import":
imp = scope[i]
del scope[i]
scope.insert(n, imp)
n += 1
# - pythoncile2.py set citdl=tuple for 'varargs' arguments, and
# citdl=dict for 'kwargs' arguments
for arg in tree1.getiterator("variable"):
if arg.get("ilk") != "argument":
continue
if arg.get("attributes") == "varargs":
arg.set("citdl", "tuple")
elif arg.get("attributes") == "kwargs":
arg.set("citdl", "dict")
# - pythoncile2.py will set citdl=None if for `foo(a=None)`, pythoncile.py
# does not
for arg in tree2.getiterator("variable"):
if arg.get("ilk") != "argument":
continue
if arg.get("citdl") == "None":
del arg.attrib["citdl"]
# - Just can't agree on 'lineend' value for functions and classes right now.
# TODO:XXX Need to eventually make sure that the new lineends
# make sense! The main current diff is that pythoncile2
# includes trailing comment lines, even it not indented.
# I think that is fine -- might even be helpful.
for scope in tree1.getiterator("scope"):
if scope.get("ilk") in ("function", "class") and scope.get("lineend"):
del scope.attrib["lineend"]
for scope in tree2.getiterator("scope"):
if scope.get("ilk") in ("function", "class") and scope.get("lineend"):
del scope.attrib["lineend"]
# - pythoncile.py incorrectly normalizes double-quotes in function signature
# argument default values to single quotes. This normalization will get it
# wrong for quotes in quotes.
for func in tree2.getiterator("scope"):
if func.get("ilk") == "function":
sig = func.get("signature")
if '"' in sig:
func.set("signature", sig.replace('"', "'"))
pretty_tree_from_tree(tree2)
norm_cix2 = ET.tostring(tree2, "utf-8")
pretty_tree_from_tree(tree1)
norm_cix1 = ET.tostring(tree1, "utf-8")
import difflib
diff = difflib.unified_diff(
norm_cix1.splitlines(1),
norm_cix2.splitlines(1),
"pythoncile %s (normalized)" % path,
"pythoncile2 %s (normalized)" % path)
diff = ''.join(list(diff))
if diff:
print(diff)
else:
sys.stdout.write(cix2)
#print >> sys.stderr, ("******************", tree2, tree2.tag, tree2.attrib); sys.stderr.flush()
#assert False, ('===============', tree2.getchildren()[0].attrib)
if tree2.get('error'):
raise Exception(tree2.get('error'))
if _gClockIt: sys.stdout.write(" (ast:%.3fs)" % (_gClock()-_gStartTime))
except Exception, ex:
fileAttrs["error"] = str(ex)
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
if tree2 is None:
# This happens, for example, with:
# foo(bar, baz=1, blam)
fileAttrs["error"] = "could not generate AST"
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
fileAttrs["md5"] = md5sum
fileAttrs["mtime"] = mtime
moduleName = os.path.splitext(os.path.basename(filename))[0]
if _gClockIt: sys.stdout.write(" (walk:%.3fs)" % (_gClock()-_gStartTime))
if log.isEnabledFor(logging.INFO):
# Dump a repr of the gathering info for debugging
# - We only have to dump the module namespace because
# everything else should be linked from it.
for nspath, namespace in visitor.st.items():
if len(nspath) == 0: # this is the module namespace
pprint.pprint(namespace)
file = ' <file%s>\n\n%s\n </file>'\
% (getAttrStr(fileAttrs), cix2)
def main(argv=None):
if argv is None:
argv = sys.argv
if not logging.root.handlers:
logging.basicConfig()
usage = "usage: %prog [PATHS...]"
version = "%prog "+__version__
parser = optparse.OptionParser(prog="pythoncile2", usage=usage,
version=version, description=__doc__,
formatter=_NoReflowFormatter())
parser.add_option("-v", "--verbose", dest="log_level",
action="store_const", const=logging.DEBUG,
help="more verbose output")
parser.add_option("-q", "--quiet", dest="log_level",
action="store_const", const=logging.ERROR,
help="less verbose output (only show errors)")
parser.add_option("-c", "--compare", action="store_true",
help="run against pythoncile.py as well (for testing)")
parser.set_defaults(log_level=logging.INFO, compare=False)
opts, paths = parser.parse_args(argv)
log.setLevel(opts.log_level)
assert len(paths) == 1, "usage: pythoncile2.py PATH"
path = paths[0]
try:
tree2 = pythoncile2(path)
except PythonCILEError as ex:
log.error(str(ex))
if log.isEnabledFor(logging.DEBUG):
print()
import traceback
traceback.print_exception(*sys.exc_info())
return 1
except KeyboardInterrupt:
log.debug("user abort")
return 1
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
if opts.compare:
cix1 = _pythoncile_cix_from_path(path)
if log.isEnabledFor(logging.DEBUG):
print("-- pythoncile1 %s" % path)
print(cix1)
print("-- pythoncile2 %s" % path)
print(cix2)
# Normalizing for comparison.
tree1 = ET.fromstring(cix1)
# - mtime slightly different
tree1[0].set("mtime", tree2[0].get("mtime"))
# - pythoncile.py seems to put <import>'s at the top of the scope.
# We'll just get tree2 to look like that.
for scope in tree2.getiterator("scope"):
if scope.get("ilk") not in ("blob", "function"):
continue
indeces = []
n = 0
for i, child in enumerate(scope[:]):
if child.tag == "import":
imp = scope[i]
del scope[i]
scope.insert(n, imp)
n += 1
# - pythoncile2.py set citdl=tuple for 'varargs' arguments, and
# citdl=dict for 'kwargs' arguments
for arg in tree1.getiterator("variable"):
if arg.get("ilk") != "argument":
continue
if arg.get("attributes") == "varargs":
arg.set("citdl", "tuple")
elif arg.get("attributes") == "kwargs":
arg.set("citdl", "dict")
# - pythoncile2.py will set citdl=None if for `foo(a=None)`, pythoncile.py
# does not
for arg in tree2.getiterator("variable"):
if arg.get("ilk") != "argument":
continue
if arg.get("citdl") == "None":
del arg.attrib["citdl"]
# - Just can't agree on 'lineend' value for functions and classes right now.
# TODO:XXX Need to eventually make sure that the new lineends
# make sense! The main current diff is that pythoncile2
# includes trailing comment lines, even it not indented.
# I think that is fine -- might even be helpful.
for scope in tree1.getiterator("scope"):
if scope.get("ilk") in ("function", "class") and scope.get("lineend"):
del scope.attrib["lineend"]
for scope in tree2.getiterator("scope"):
if scope.get("ilk") in ("function", "class") and scope.get("lineend"):
del scope.attrib["lineend"]
# - pythoncile.py incorrectly normalizes double-quotes in function signature
# argument default values to single quotes. This normalization will get it
# wrong for quotes in quotes.
for func in tree2.getiterator("scope"):
if func.get("ilk") == "function":
sig = func.get("signature")
if '"' in sig:
func.set("signature", sig.replace('"', "'"))
pretty_tree_from_tree(tree2)
norm_cix2 = ET.tostring(tree2, "utf-8")
pretty_tree_from_tree(tree1)
norm_cix1 = ET.tostring(tree1, "utf-8")
import difflib
diff = difflib.unified_diff(
norm_cix1.splitlines(1),
norm_cix2.splitlines(1),
"pythoncile %s (normalized)" % path,
"pythoncile2 %s (normalized)" % path)
diff = ''.join(list(diff))
if diff:
print(diff)
else:
sys.stdout.write(cix2)
def scan(content, filename, md5sum=None, mtime=None, lang="Python"):
"""Scan the given Python content and return Code Intelligence data
conforming the the Code Intelligence XML format.
"content" is the Python content to scan. This should be an
encoded string: must be a string for `md5` and
`compiler.parse` -- see bug 73461.
"filename" is the source of the Python content (used in the
generated output).
"md5sum" (optional) if the MD5 hexdigest has already been calculated
for the content, it can be passed in here. Otherwise this
is calculated.
"mtime" (optional) is a modified time for the file (in seconds since
the "epoch"). If it is not specified the _current_ time is used.
Note that the default is not to stat() the file and use that
because the given content might not reflect the saved file state.
"lang" (optional) is the language of the given file content.
Typically this is "Python" (i.e. a pure Python file), but it
may also be "DjangoHTML" or similar for Python embedded in
other documents.
XXX Add an optional 'eoltype' so that it need not be
re-calculated if already known.
This can raise one of SyntaxError, PythonCILEError or parser.ParserError
if there was an error processing. Currently this implementation uses the
Python 'compiler' package for processing, therefore the given Python
content must be syntactically correct.
"""
log.info("scan '%s'", filename)
if md5sum is None:
md5sum = md5(content).hexdigest()
if mtime is None:
mtime = int(time.time())
# 'compiler' both (1) wants a newline at the end and (2) can fail on
# funky *whitespace* at the end of the file.
content = content.rstrip() + '\n'
if isinstance(filename, str):
filename = filename.encode('utf-8')
# The 'path' attribute must use normalized dir separators.
if sys.platform.startswith("win"):
path = filename.replace('\\', '/')
else:
path = filename
fileAttrs = {"language": "Python",
"generator": "Python",
"path": path}
try:
tree2 = pythoncile2(path, content).find('file').find('scope')
# print >> sys.stderr, ("******************", tree2, tree2.tag, tree2.attrib); sys.stderr.flush()
# assert False, ('===============', tree2.getchildren()[0].attrib)
if tree2.get('error'):
raise Exception(tree2.get('error'))
if _gClockIt:
sys.stdout.write(" (ast:%.3fs)" % (_gClock()-_gStartTime))
except Exception as ex:
fileAttrs["error"] = str(ex)
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
if tree2 is None:
# This happens, for example, with:
# foo(bar, baz=1, blam)
fileAttrs["error"] = "could not generate AST"
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
fileAttrs["md5"] = md5sum
fileAttrs["mtime"] = mtime
moduleName = os.path.splitext(os.path.basename(filename))[0]
if _gClockIt:
sys.stdout.write(" (walk:%.3fs)" % (_gClock()-_gStartTime))
if log.isEnabledFor(logging.INFO):
# Dump a repr of the gathering info for debugging
# - We only have to dump the module namespace because
# everything else should be linked from it.
for nspath, namespace in list(visitor.st.items()):
if len(nspath) == 0: # this is the module namespace
pprint.pprint(namespace)
file = ' <file%s>\n\n%s\n </file>'\
% (getAttrStr(fileAttrs), cix2)
if _gClockIt:
sys.stdout.write(" (getCIX:%.3fs)" % (_gClock()-_gStartTime))
cix = '''\
<?xml version="1.0" encoding="UTF-8"?>
<codeintel version="0.1">
%s
</codeintel>
''' % file
return cix
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
import time
import ciElementTree as ET
from ci2 import _paths_from_path_patterns
from codeintel2.manager import Manager
from codeintel2.citadel import CitadelBuffer
from codeintel2.common import CodeIntelError
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.util import guess_lang_from_path
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
self.log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
self.log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
self.log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree, "file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
self.log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [child for child in function
if child.tag != 'variable']
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns):
"""Scan and print the CIX for the given path(s).
${cmd_usage}
${cmd_option_list}
"""
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
if opts.time_it:
start = time.time()
quiet = opts.quiet
if opts.time_it or opts.time_details:
opts.force = True
scan_count = 0
lang_warnings = set()
tree = None
for path in _paths_from_path_patterns(path_patterns,
recursive=opts.recursive,
includes=opts.includes):
if opts.time_it:
sys.stderr.write(path + "\n")
if opts.time_details:
start1 = time.time()
try:
lang = opts.lang or guess_lang_from_path(path)
except CodeIntelError:
log.info("skip `%s': couldn't determine language", path)
continue
try:
buf = mgr.buf_from_path(path, lang=lang)
except OSError as ex:
# Couldn't access the file.
if not opts.recursive:
raise
# Ignore files we don't really care about.
log.warn("%r - %r", ex, path)
continue
if not isinstance(buf, CitadelBuffer):
if opts.recursive:
# Ignore files that scanning isn't provided for.
continue
raise CodeIntelError("`%s' (%s) is not a language that "
"uses CIX" % (path, buf.lang))
scan_count += 1
if scan_count % 10 == 0:
log.info("%d scanning %r", scan_count, path)
try:
if opts.force:
buf.scan()
if tree is None:
tree = ET.Element("codeintel", version="2.0")
file_elem = ET.SubElement(tree,
"file",
lang=buf.lang,
mtime=str(int(time.time())),
path=os.path.basename(path))
for lang, blob in sorted(buf.blob_from_lang.items()):
blob = buf.blob_from_lang[lang]
file_elem.append(blob)
except KeyError as ex:
# Unknown cile language.
if not opts.recursive:
raise
message = str(ex)
if message not in lang_warnings:
lang_warnings.add(message)
log.warn("Skipping unhandled language %s", message)
if opts.time_details:
delta = time.time() - start1
sys.stderr.write("%.3f %s\n" % (delta, path))
sys.stderr.flush()
if tree is not None:
if opts.stripfuncvars:
# For stdlibs, we don't care about variables inside of
# functions and they take up a lot of space.
for function in tree.getiterator('scope'):
if function.get('ilk') == 'function':
function[:] = [
child for child in function
if child.tag != 'variable'
]
if opts.pretty_print:
tree = pretty_tree_from_tree(tree)
if not quiet:
sys.stdout.write(
'<?xml version="1.0" encoding="UTF-8"?>\n')
ET.dump(tree)
if opts.time_it:
end = time.time()
sys.stderr.write("scan took %.3fs\n" % (end - start))
finally:
mgr.finalize()
def _testOneInputFile(self, fpath, tags=None):
_debug = False # Set to true to dump status info for each test run.
infile = os.path.join(gInputsDir, fpath) # input
outfile = os.path.join(gOutputsDir, fpath + '.cix') # expected output
tmpfile = os.path.join(gTmpDir, fpath + '.cix') # actual output
if not os.path.exists(os.path.dirname(tmpfile)):
os.makedirs(os.path.dirname(tmpfile))
errfile = os.path.join(gOutputsDir, fpath + '.error') # expected error
# An options file is a set of kwargs for the buf.scan()
# method call. One key-value pair per-line like this:
# key=value
# Whitespace is stripped off the value.
optsfile = os.path.join(gInputsDir, fpath + '.options') # input options
if _debug:
print
print "*" * 50, "codeintel '%s'" % fpath
# Set standard options:
opts = {"mtime": "42"}
# Determine input options to use, if any.
# XXX Not used. Drop it.
if os.path.exists(optsfile):
for line in open(optsfile, 'r').read().splitlines(0):
name, value = line.split('=', 1)
value = value.strip()
try: # allow value to be a type other than string
value = eval(value)
except Exception:
pass
opts[name] = value
if _debug:
print "*" * 50, "options"
pprint.pprint(opts)
# Scan the file, capturing stdout and stderr and any possible
# error.
# - To allow testing from different dirs (resulting in changing
# path strings, we normalize the <file path="..."> value and any
# <scope ilk="blob" src="..."> attributes).
oldStdout = sys.stdout
oldStderr = sys.stderr
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
try:
try:
lang = None
if tags and "python3" in tags:
lang = "Python3"
buf = self.mgr.buf_from_path(infile, lang=lang)
buf.scan(**opts)
tree = buf.tree
# Normalize paths.
relnorm_infile = infile[len(dirname(gInputsDir)) + 1:]
absnorm_infile = infile
relnorm_infile = relnorm_infile.replace('\\', '/')
absnorm_infile = absnorm_infile.replace('\\', '/')
for file_elem in tree:
file_elem.set("path", relnorm_infile)
for blob_elem in file_elem:
if blob_elem.get("ilk") != "blob":
continue
norm_src = normpath(blob_elem.get("src"))
norm_src = norm_src.replace('\\', '/')
if norm_src in (relnorm_infile, absnorm_infile):
blob_elem.set("src", relnorm_infile)
tree = pretty_tree_from_tree(tree)
# Due to the dynamic nature of the ciler errors (which often
# includes the source code line numbers), it's difficult to check
# that the errors are identical, so we work around this by just
# taking the first 30 characters of the error.
cile_error = tree[0].get("error")
if cile_error and fpath.endswith(".js"):
tree[0].set(
"error",
len(cile_error) < 30 and cile_error
or (cile_error[:30] + "..."))
cix = ET.tostring(tree)
except CodeIntelError, ex:
error = traceback.format_exc()
else:
def _testOneInputFile(self, fpath, tags=None):
_debug = False # Set to true to dump status info for each test run.
infile = os.path.join(gInputsDir, fpath) # input
outfile = os.path.join(gOutputsDir, fpath+'.cix') # expected output
tmpfile = os.path.join(gTmpDir, fpath+'.cix') # actual output
if not os.path.exists(os.path.dirname(tmpfile)):
os.makedirs(os.path.dirname(tmpfile))
errfile = os.path.join(gOutputsDir, fpath+'.error') # expected error
# An options file is a set of kwargs for the buf.scan()
# method call. One key-value pair per-line like this:
# key=value
# Whitespace is stripped off the value.
optsfile = os.path.join(gInputsDir, fpath+'.options') # input options
if _debug:
print
print "*"*50, "codeintel '%s'" % fpath
# Set standard options:
opts = {"mtime": "42"}
# Determine input options to use, if any.
# XXX Not used. Drop it.
if os.path.exists(optsfile):
for line in open(optsfile, 'r').read().splitlines(0):
name, value = line.split('=', 1)
value = value.strip()
try: # allow value to be a type other than string
value = eval(value)
except Exception:
pass
opts[name] = value
if _debug:
print "*"*50, "options"
pprint.pprint(opts)
# Scan the file, capturing stdout and stderr and any possible
# error.
# - To allow testing from different dirs (resulting in changing
# path strings, we normalize the <file path="..."> value and any
# <scope ilk="blob" src="..."> attributes).
oldStdout = sys.stdout
oldStderr = sys.stderr
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
try:
try:
lang = None
if tags and "python3" in tags:
lang = "Python3"
buf = self.mgr.buf_from_path(infile, lang=lang)
buf.scan(**opts)
tree = buf.tree
# Normalize paths.
relnorm_infile = infile[len(dirname(gInputsDir))+1:]
absnorm_infile = infile
relnorm_infile = relnorm_infile.replace('\\', '/')
absnorm_infile = absnorm_infile.replace('\\', '/')
for file_elem in tree:
file_elem.set("path", relnorm_infile)
for blob_elem in file_elem:
if blob_elem.get("ilk") != "blob":
continue
norm_src = normpath(blob_elem.get("src"))
norm_src = norm_src.replace('\\', '/')
if norm_src in (relnorm_infile, absnorm_infile):
blob_elem.set("src", relnorm_infile)
tree = pretty_tree_from_tree(tree)
# Due to the dynamic nature of the ciler errors (which often
# includes the source code line numbers), it's difficult to check
# that the errors are identical, so we work around this by just
# taking the first 30 characters of the error.
cile_error = tree[0].get("error")
if cile_error and fpath.endswith(".js"):
tree[0].set("error", len(
cile_error) < 30 and cile_error or (cile_error[:30] + "..."))
cix = ET.tostring(tree)
except CodeIntelError, ex:
error = traceback.format_exc()
else:
# assert False, ('===============', tree2.getchildren()[0].attrib)
if tree2.get('error'):
raise Exception(tree2.get('error'))
if _gClockIt:
sys.stdout.write(" (ast:%.3fs)" % (_gClock() - _gStartTime))
except Exception, ex:
fileAttrs["error"] = str(ex)
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
if tree2 is None:
# This happens, for example, with:
# foo(bar, baz=1, blam)
fileAttrs["error"] = "could not generate AST"
file = ' <file%s/>' % getAttrStr(fileAttrs)
else:
pretty_tree_from_tree(tree2)
cix2 = ET.tostring(tree2, "utf-8")
fileAttrs["md5"] = md5sum
fileAttrs["mtime"] = mtime
moduleName = os.path.splitext(os.path.basename(filename))[0]
if _gClockIt:
sys.stdout.write(" (walk:%.3fs)" % (_gClock() - _gStartTime))
if log.isEnabledFor(logging.INFO):
# Dump a repr of the gathering info for debugging
# - We only have to dump the module namespace because
# everything else should be linked from it.
for nspath, namespace in visitor.st.items():
if len(nspath) == 0: # this is the module namespace
pprint.pprint(namespace)
file = ' <file%s>\n\n%s\n </file>'\
replace_node(origBlobStar, curr_tree, t, i)
i += 1
print "After doing yaml-nodes, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
i = 0
for t in bin_tree:
replace_node(origFile, bin_tree, t, i)
i += 1
print "After doing binary modules, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
# Finally update the YAML nodes, assuming that if we do a replacement
# we can replace the entire subtree.
i = 0
for t in yaml_stdlib_tree:
replace_node(origFile, yaml_stdlib_tree, t, i)
i += 1
print "After doing yaml modules, orig tree has %d nodes" % len(origBlobStar)
print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions)
newFile = options.origfile + ".new"
fd = open(newFile, "w")
fd.write(ET.tostring(pretty_tree_from_tree(origTopLevel)))
fd.close()
print "Done writing to file %s" % newFile
def do_play(self, subcmd, opts):
"""Run my current play/dev code.
${cmd_usage}
${cmd_option_list}
"""
import pprint
import random
import ciElementTree as ET
from codeintel2.manager import Manager
from codeintel2.tree import pretty_tree_from_tree
from codeintel2.common import LogEvalController, Error
from codeintel2.util import tree_from_cix, dedent, unmark_text, banner
from ci2 import _escaped_text_from_text
if False:
lang = "CSS"
markedup_content = dedent("""
/* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */
h1 {
border: 1px solid black;
font-weight /* hi */: <|> !important
}
""")
content, data = unmark_text(markedup_content)
pos = data["pos"]
mgr = Manager()
# mgr.upgrade() # Don't need it for just CSS usage.
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path="play.css")
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
completions = buf.cplns_from_trg(trg)
print("COMPLETIONS: %r" % completions)
finally:
mgr.finalize()
elif False:
lang = "Python"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""
import sys, os
class Foo:
def bar(self):
pass
sys.<|>path # should have path in completion list
f = Foo()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data["pos"]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
print(buf.cix)
trg = buf.trg_from_pos(pos)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(2) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
elif False:
lang = "Ruby"
path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100))
markedup_content = dedent("""\
r<1>equire 'net/http'
include Net
req = HTTPRequest.new
req.<2>get()
""")
content, data = unmark_text(markedup_content)
print(banner(path))
print(_escaped_text_from_text(content, "whitespace"))
pos = data[1]
mgr = Manager()
mgr.upgrade()
mgr.initialize()
try:
buf = mgr.buf_from_content(content, lang=lang, path=path)
print(banner("cix", '-'))
cix = buf.cix
print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix))))
trg = buf.trg_from_pos(pos, implicit=False)
if trg is None:
raise Error("unexpected trigger: %r" % trg)
print(banner("completions", '-'))
ctlr = LogEvalController(self.log)
buf.async_eval_at_trg(trg, ctlr)
ctlr.wait(30) # XXX
if not ctlr.is_done():
ctlr.abort()
raise Error("XXX async eval timed out")
pprint.pprint(ctlr.cplns)
print(banner(None))
finally:
mgr.finalize()
def _testOneInputFile(self, fpath, tags=None):
_debug = False # Set to true to dump status info for each test run.
infile = os.path.join(gInputsDir, fpath) # input
outfile = os.path.join(gOutputsDir, fpath+'.cix') # expected output
tmpfile = os.path.join(gTmpDir, fpath+'.cix') # actual output
if not os.path.exists(os.path.dirname(tmpfile)):
os.makedirs(os.path.dirname(tmpfile))
errfile = os.path.join(gOutputsDir, fpath+'.error') # expected error
# An options file is a set of kwargs for the buf.scan()
# method call. One key-value pair per-line like this:
# key=value
# Whitespace is stripped off the value.
optsfile = os.path.join(gInputsDir, fpath+'.options') # input options
if _debug:
print()
print("*"*50, "codeintel '%s'" % fpath)
# Set standard options:
opts = {"mtime": "42"}
# Determine input options to use, if any.
#XXX Not used. Drop it.
if os.path.exists(optsfile):
for line in open(optsfile, 'r').read().splitlines(0):
name, value = line.split('=', 1)
value = value.strip()
try: # allow value to be a type other than string
value = eval(value)
except Exception:
pass
opts[name] = value
if _debug:
print("*"*50, "options")
pprint.pprint(opts)
# Scan the file, capturing stdout and stderr and any possible
# error.
# - To allow testing from different dirs (resulting in changing
# path strings, we normalize the <file path="..."> value and any
# <scope ilk="blob" src="..."> attributes).
oldStdout = sys.stdout
oldStderr = sys.stderr
sys.stdout = StringIO()
sys.stderr = StringIO()
try:
try:
lang = None
if tags and "python3" in tags:
lang = "Python3"
buf = self.mgr.buf_from_path(infile, lang=lang)
buf.scan(**opts)
tree = buf.tree
# Normalize paths.
relnorm_infile = infile[len(dirname(gInputsDir))+1:]
absnorm_infile = infile
relnorm_infile = relnorm_infile.replace('\\', '/')
absnorm_infile = absnorm_infile.replace('\\', '/')
for file_elem in tree:
file_elem.set("path", relnorm_infile)
for blob_elem in file_elem:
if blob_elem.get("ilk") != "blob": continue
norm_src = normpath(blob_elem.get("src"))
norm_src = norm_src.replace('\\', '/')
if norm_src in (relnorm_infile, absnorm_infile):
blob_elem.set("src", relnorm_infile)
tree = pretty_tree_from_tree(tree)
# Due to the dynamic nature of the ciler errors (which often
# includes the source code line numbers), it's difficult to check
# that the errors are identical, so we work around this by just
# taking the first 30 characters of the error.
cile_error = tree[0].get("error")
if cile_error and fpath.endswith(".js"):
tree[0].set("error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "..."))
cix = ET.tostring(tree)
except CodeIntelError as ex:
error = traceback.format_exc()
else:
error = None
if isinstance(cix, six.text_type):
with io.open(tmpfile, mode="wt", encoding="utf-8") as fout:
fout.write(cix)
else:
with open(tmpfile, mode="wb") as fout:
fout.write(cix)
finally:
stdout = sys.stdout.getvalue()
stderr = sys.stderr.getvalue()
sys.stdout = oldStdout
sys.stderr = oldStderr
if _debug:
print("*"*50, "stdout")
print(stdout)
print("*"*50, "stderr")
print(stderr)
print("*"*50, "error")
print(str(error))
print("*" * 50)
generateMissing = False
if not os.path.exists(outfile) and generateMissing:
with io.open(outfile, mode='wt', encoding='utf-8') as fout:
with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp:
fout.write(ftmp.read())
# Verify that the results are as expected.
if os.path.exists(outfile) and error:
self.fail("scanning '%s' raised an error but success was "
"expected:\n%s" % (_encode_for_stdout(fpath), indent(error)))
elif os.path.exists(outfile):
# Convert the <file path="..."/> to the native directory separator.
def to_native_sep(match):
path = match.group(2).replace("\\", os.sep).replace("/", os.sep)
return match.group(1)+path+match.group(3)
path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S)
# Note that we don't really care about line endings here, so we read
# both files in universal newlines mode (i.e. translate to \n)
# and normalize ' ', ' ' and '''
with io.open(outfile, mode='rt', encoding='utf-8') as fout:
expected = path_pat.sub(to_native_sep, fout.read())
expected = expected.replace('
', ' ').replace('
', ' ').replace(''', '\'')
with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp:
actual = path_pat.sub(to_native_sep, ftmp.read())
actual = actual.replace('
', ' ').replace('
', ' ').replace(''', '\'')
if expected != actual:
do_fail = True
# Useful temporary thing while XML output format is changing.
#if os.stat("../support/xmldiff.py"):
# rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile))
# if rc == 0:
# do_fail = False
if do_fail:
diff = list(difflib.ndiff(expected.splitlines(1),
actual.splitlines(1)))
diff = _diffContext(diff, 2)
if diff:
error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \
% (outfile, tmpfile, outfile, tmpfile,
''.join(diff))
if gMaxDiffOutput > 0 and gMaxNumLines > 0:
if len(error_str) > gMaxDiffOutput:
error_lines = error_str.split("\n")
if len(error_lines) > gMaxNumLines:
error_lines = error_lines[:gMaxNumLines] + ["..."]
if gMaxLineLength > 0:
error_str = "\n".join([len(x) > gMaxLineLength and x[:gMaxLineLength] or x
for x in error_lines])
else:
error_str = "\n".join(error_lines)
self.fail(_encode_for_stdout(error_str))
elif os.path.exists(errfile):
# There is no reference output file. This means that processing
# this file is expected to fail.
expectedError = open(errfile, 'r').read()
actualError = str(error)
self.failUnlessEqual(actualError.strip(), expectedError.strip())
else:
self.fail("No reference output file or error file for '%s'." % infile)
# Ensure next test file gets a clean codeintel.
toDelete = []
for modname in sys.modules:
if modname == "codeintel" or modname.startswith("codeintel."):
toDelete.append(modname)
for modname in toDelete:
del sys.modules[modname]
