def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def main(): # More robust for determining the perforce script location cix_filename = "javascript.cix" scriptpath = os.path.dirname(__file__) if not scriptpath: scriptpath = "." scriptpath = os.path.abspath(scriptpath) cix_directory = scriptpath # Get main codeintel directory, 3 up from this script location! for i in range(3): cix_directory = os.path.dirname(cix_directory) cix_filename = os.path.join( cix_directory, "lib", "codeintel2", "stdlibs", cix_filename) # Generate the cix files for filename in ("ecmaToCodeintel.py", "dom0_to_cix.py", "dom2_to_cix.py"): p = os.popen("python %s" % (os.path.join(scriptpath, filename))).read() # Combine the libraries cixtree = ciElementTree.parse(os.path.join(scriptpath, "javascript.cix")) cixscope = cixtree.find("file/scope") # Note: XMLHttpRequest cix comes from the Mozilla implementation in: # nsIXMLHttpRequest.idl for domname in ("XMLHttpRequest", "dom0", "dom2"): # cixscope.append(ciElementTree.Comment(" %s structure " % (domname))) et = ciElementTree.parse("%s.cix" % ( os.path.join(scriptpath, domname))) for scope in et.findall("//file/scope"): for child in scope.getchildren(): # Ensure we remove from the dom tree first, otherwise # we generate double elements scope.remove(child) cixscope.append(child) pretty_tree_from_tree(cixtree.getroot()) update_documentation_sentences(cixtree) p4update(cix_filename, ciElementTree.tostring(cixtree.getroot())) # Update libraries for dirname, filename in (("dojo", "dojo_json_to_cix.py"), ("MochiKit", "mochikit_to_cix.py"), ("prototype", "prototype_to_cix.py"), ("yui", "yui_to_cix.py")): library_script_path = os.path.join(scriptpath, dirname, filename) p = os.popen("python %s -u" % (library_script_path)).read()
def main(): # More robust for determining the perforce script location cix_filename = "javascript.cix" scriptpath = os.path.dirname(__file__) if not scriptpath: scriptpath = "." scriptpath = os.path.abspath(scriptpath) cix_directory = scriptpath # Get main codeintel directory, 3 up from this script location! for i in range(3): cix_directory = os.path.dirname(cix_directory) cix_filename = os.path.join(cix_directory, "lib", "codeintel2", "stdlibs", cix_filename) # Generate the cix files for filename in ("ecmaToCodeintel.py", "dom0_to_cix.py", "dom2_to_cix.py"): p = os.popen("python %s" % (os.path.join(scriptpath, filename))).read() # Combine the libraries cixtree = ciElementTree.parse(os.path.join(scriptpath, "javascript.cix")) cixscope = cixtree.find("file/scope") # Note: XMLHttpRequest cix comes from the Mozilla implementation in: # nsIXMLHttpRequest.idl for domname in ("XMLHttpRequest", "dom0", "dom2"): # cixscope.append(ciElementTree.Comment(" %s structure " % (domname))) et = ciElementTree.parse("%s.cix" % (os.path.join(scriptpath, domname))) for scope in et.findall("//file/scope"): for child in scope.getchildren(): # Ensure we remove from the dom tree first, otherwise # we generate double elements scope.remove(child) cixscope.append(child) pretty_tree_from_tree(cixtree.getroot()) update_documentation_sentences(cixtree) p4update(cix_filename, ciElementTree.tostring(cixtree.getroot())) # Update libraries for dirname, filename in (("dojo", "dojo_json_to_cix.py"), ("MochiKit", "mochikit_to_cix.py"), ("prototype", "prototype_to_cix.py"), ("yui", "yui_to_cix.py")): library_script_path = os.path.join(scriptpath, dirname, filename) p = os.popen("python %s -u" % (library_script_path)).read()
def update_cix_file(mgr, path): log.info("convert `%s' to pretty CIX 2.0", path) cix = open(path, 'r').read() tree = tree_from_cix(cix) # converts to CIX 2.0 tree = pretty_tree_from_tree(tree) new_cix = ET.tostring(tree) _run("p4 edit %s" % path) open(path, 'w').write(new_cix)
def mergeCixFiles(opts, filename1, filename2, outputfilename): e1 = parse(filename1).getroot().getchildren()[0] mergedcixroot = parse(filename2).getroot() e2 = mergedcixroot.getchildren()[0] elems1 = [] elems2 = [] if opts.lpath: for lpath in opts.lpath: elem1 = e1 elem2 = e2 lpath_split = lpath.split(".") for i in range(len(lpath_split)): name = lpath_split[i] new_elem1 = elem1.names.get(name) new_elem2 = elem2.names.get(name) if new_elem1 is None and new_elem2 is None: print "lpath not found in either cix file: %r (skipping it)" % ( lpath, ) break elif new_elem2 is None: answer = raw_input( "lpath %r only found in first cix file, copy over? [Yn]" % (lpath, )) if answer.lower() not in ("n", "no"): merge_missing(elem1, [name], lpath_split[:i - 1], e2) break # elif new_elem1 is None: # print "lpath not found in either cix files: %r" % (lpath, ) # break elem1 = new_elem1 elem2 = new_elem2 else: elems1.append(elem1) elems2.append(elem2) else: elems1 = [e1] elems2 = [e2] for e1, e2 in zip(elems1, elems2): print "Diffing elements: %r, %r" % (e1, e2) diffElements(opts, [], e1, e2) pretty_tree_from_tree(mergedcixroot) file(outputfilename, "w").write(cixtostring(mergedcixroot))
def mergeCixFiles(opts, filename1, filename2, outputfilename): e1 = parse(filename1).getroot().getchildren()[0] mergedcixroot = parse(filename2).getroot() e2 = mergedcixroot.getchildren()[0] elems1 = [] elems2 = [] if opts.lpath: for lpath in opts.lpath: elem1 = e1 elem2 = e2 lpath_split = lpath.split(".") for i in range(len(lpath_split)): name = lpath_split[i] new_elem1 = elem1.names.get(name) new_elem2 = elem2.names.get(name) if new_elem1 is None and new_elem2 is None: print "lpath not found in either cix file: %r (skipping it)" % (lpath, ) break elif new_elem2 is None: answer = raw_input( "lpath %r only found in first cix file, copy over? [Yn]" % (lpath, )) if answer.lower() not in ("n", "no"): merge_missing(elem1, [name], lpath_split[:i-1], e2) break # elif new_elem1 is None: # print "lpath not found in either cix files: %r" % (lpath, ) # break elem1 = new_elem1 elem2 = new_elem2 else: elems1.append(elem1) elems2.append(elem2) else: elems1 = [e1] elems2 = [e2] for e1, e2 in zip(elems1, elems2): print "Diffing elements: %r, %r" % (e1, e2) diffElements(opts, [], e1, e2) pretty_tree_from_tree(mergedcixroot) file(outputfilename, "w").write(cixtostring(mergedcixroot))
def do_cix(self, subcmd, opts, *path_patterns): """Read in and print a CIX file (possibly converting to CIX 2.0 and prettifying in the process. ${cmd_usage} ${cmd_option_list} """ if opts.pretty_print: opts.convert = True for path in _paths_from_path_patterns(path_patterns): tree = None cix = open(path, 'r').read() if opts.convert: tree = tree_from_cix(cix) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) else: sys.stdout.write(cix)
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path+"\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue buf = mgr.buf_from_path(path, lang=lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end-start)) finally: mgr.finalize()
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() #mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print( ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
if t and t.get("ilk") in reg_ilks: replace_node(origBlobStar, curr_tree, t, i) i += 1 print "After doing yaml-nodes, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) i = 0 for t in bin_tree: replace_node(origFile, bin_tree, t, i) i += 1 print "After doing binary modules, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) # Finally update the YAML nodes, assuming that if we do a replacement # we can replace the entire subtree. i = 0 for t in yaml_stdlib_tree: replace_node(origFile, yaml_stdlib_tree, t, i) i += 1 print "After doing yaml modules, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) newFile = options.origfile + ".new" fd = open(newFile, "w") fd.write(ET.tostring(pretty_tree_from_tree(origTopLevel))) fd.close() print "Done writing to file %s" % newFile
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath + '.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath + '.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath + '.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath + '.options') # input options if _debug: print() print("*" * 50, "codeintel '%s'" % fpath) # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. #XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print("*" * 50, "options") pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir)) + 1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set( "error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError as ex: error = traceback.format_exc() else: error = None if isinstance(cix, six.text_type): with io.open(tmpfile, mode="wt", encoding="utf-8") as fout: fout.write(cix) else: with open(tmpfile, mode="wb") as fout: fout.write(cix) finally: stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print("*" * 50, "stdout") print(stdout) print("*" * 50, "stderr") print(stderr) print("*" * 50, "error") print(str(error)) print("*" * 50) generateMissing = False if not os.path.exists(outfile) and generateMissing: with io.open(outfile, mode='wt', encoding='utf-8') as fout: with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: fout.write(ftmp.read()) # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1) + path + match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) # and normalize ' ', ' ' and ''' with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) expected = expected.replace('
', ' ').replace( '
', ' ').replace(''', '\'') with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read()) actual = actual.replace('
', ' ').replace( '
', ' ').replace(''', '\'') if expected != actual: do_fail = True # Useful temporary thing while XML output format is changing. #if os.stat("../support/xmldiff.py"): # rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile)) # if rc == 0: # do_fail = False if do_fail: diff = list( difflib.ndiff(expected.splitlines(1), actual.splitlines(1))) diff = _diffContext(diff, 2) if diff: error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \ % (outfile, tmpfile, outfile, tmpfile, ''.join(diff)) if gMaxDiffOutput > 0 and gMaxNumLines > 0: if len(error_str) > gMaxDiffOutput: error_lines = error_str.split("\n") if len(error_lines) > gMaxNumLines: error_lines = error_lines[:gMaxNumLines] + [ "..." ] if gMaxLineLength > 0: error_str = "\n".join([ len(x) > gMaxLineLength and x[:gMaxLineLength] or x for x in error_lines ]) else: error_str = "\n".join(error_lines) self.fail(_encode_for_stdout(error_str)) elif os.path.exists(errfile): # There is no reference output file. This means that processing # this file is expected to fail. expectedError = open(errfile, 'r').read() actualError = str(error) self.failUnlessEqual(actualError.strip(), expectedError.strip()) else: self.fail("No reference output file or error file for '%s'." % infile) # Ensure next test file gets a clean codeintel. toDelete = [] for modname in sys.modules: if modname == "codeintel" or modname.startswith("codeintel."): toDelete.append(modname) for modname in toDelete: del sys.modules[modname]
def scan(content, filename, md5sum=None, mtime=None, lang="Python"): """Scan the given Python content and return Code Intelligence data conforming the the Code Intelligence XML format. "content" is the Python content to scan. This should be an encoded string: must be a string for `md5` and `compiler.parse` -- see bug 73461. "filename" is the source of the Python content (used in the generated output). "md5sum" (optional) if the MD5 hexdigest has already been calculated for the content, it can be passed in here. Otherwise this is calculated. "mtime" (optional) is a modified time for the file (in seconds since the "epoch"). If it is not specified the _current_ time is used. Note that the default is not to stat() the file and use that because the given content might not reflect the saved file state. "lang" (optional) is the language of the given file content. Typically this is "Python" (i.e. a pure Python file), but it may also be "DjangoHTML" or similar for Python embedded in other documents. XXX Add an optional 'eoltype' so that it need not be re-calculated if already known. This can raise one of SyntaxError, PythonCILEError or parser.ParserError if there was an error processing. Currently this implementation uses the Python 'compiler' package for processing, therefore the given Python content must be syntactically correct. """ log.info("scan '%s'", filename) if md5sum is None: md5sum = md5(content).hexdigest() if mtime is None: mtime = int(time.time()) # 'compiler' both (1) wants a newline at the end and (2) can fail on # funky *whitespace* at the end of the file. content = content.rstrip() + '\n' if isinstance(filename, str): filename = filename.encode('utf-8') # The 'path' attribute must use normalized dir separators. if sys.platform.startswith("win"): path = filename.replace('\\', '/') else: path = filename fileAttrs = {"language": "Python", "generator": "Python", "path": path} try: tree2 = pythoncile2(path, content).find('file').find('scope') # print >> sys.stderr, ("******************", tree2, tree2.tag, tree2.attrib); sys.stderr.flush() # assert False, ('===============', tree2.getchildren()[0].attrib) if tree2.get('error'): raise Exception(tree2.get('error')) if _gClockIt: sys.stdout.write(" (ast:%.3fs)" % (_gClock()-_gStartTime)) except Exception as ex: fileAttrs["error"] = str(ex) file = ' <file%s/>' % getAttrStr(fileAttrs) else: if tree2 is None: # This happens, for example, with: # foo(bar, baz=1, blam) fileAttrs["error"] = "could not generate AST" file = ' <file%s/>' % getAttrStr(fileAttrs) else: pretty_tree_from_tree(tree2) cix2 = ET.tostring(tree2, "utf-8") fileAttrs["md5"] = md5sum fileAttrs["mtime"] = mtime moduleName = os.path.splitext(os.path.basename(filename))[0] if _gClockIt: sys.stdout.write(" (walk:%.3fs)" % (_gClock()-_gStartTime)) if log.isEnabledFor(logging.INFO): # Dump a repr of the gathering info for debugging # - We only have to dump the module namespace because # everything else should be linked from it. for nspath, namespace in list(visitor.st.items()): if len(nspath) == 0: # this is the module namespace pprint.pprint(namespace) file = ' <file%s>\n\n%s\n </file>'\ % (getAttrStr(fileAttrs), cix2) if _gClockIt: sys.stdout.write(" (getCIX:%.3fs)" % (_gClock()-_gStartTime)) cix = '''\ <?xml version="1.0" encoding="UTF-8"?> <codeintel version="0.1"> %s </codeintel> ''' % file return cix
def main(argv=None): if argv is None: argv = sys.argv if not logging.root.handlers: logging.basicConfig() usage = "usage: %prog [PATHS...]" version = "%prog "+__version__ parser = optparse.OptionParser(prog="pythoncile2", usage=usage, version=version, description=__doc__, formatter=_NoReflowFormatter()) parser.add_option("-v", "--verbose", dest="log_level", action="store_const", const=logging.DEBUG, help="more verbose output") parser.add_option("-q", "--quiet", dest="log_level", action="store_const", const=logging.ERROR, help="less verbose output (only show errors)") parser.add_option("-c", "--compare", action="store_true", help="run against pythoncile.py as well (for testing)") parser.set_defaults(log_level=logging.INFO, compare=False) opts, paths = parser.parse_args(argv) #log.setLevel(opts.log_level) assert len(paths) == 1, "usage: pythoncile2.py PATH" path = paths[0] try: tree2 = pythoncile2(path) except PythonCILEError as ex: log.error(str(ex)) if log.isEnabledFor(logging.DEBUG): print() import traceback traceback.print_exception(*sys.exc_info()) return 1 except KeyboardInterrupt: log.debug("user abort") return 1 pretty_tree_from_tree(tree2) cix2 = ET.tostring(tree2, "utf-8") if opts.compare: cix1 = _pythoncile_cix_from_path(path) if log.isEnabledFor(logging.DEBUG): print("-- pythoncile1 %s" % path) print(cix1) print("-- pythoncile2 %s" % path) print(cix2) # Normalizing for comparison. tree1 = ET.fromstring(cix1) # - mtime slightly different tree1[0].set("mtime", tree2[0].get("mtime")) # - pythoncile.py seems to put <import>'s at the top of the scope. # We'll just get tree2 to look like that. for scope in tree2.getiterator("scope"): if scope.get("ilk") not in ("blob", "function"): continue indeces = [] n = 0 for i, child in enumerate(scope[:]): if child.tag == "import": imp = scope[i] del scope[i] scope.insert(n, imp) n += 1 # - pythoncile2.py set citdl=tuple for 'varargs' arguments, and # citdl=dict for 'kwargs' arguments for arg in tree1.getiterator("variable"): if arg.get("ilk") != "argument": continue if arg.get("attributes") == "varargs": arg.set("citdl", "tuple") elif arg.get("attributes") == "kwargs": arg.set("citdl", "dict") # - pythoncile2.py will set citdl=None if for `foo(a=None)`, pythoncile.py # does not for arg in tree2.getiterator("variable"): if arg.get("ilk") != "argument": continue if arg.get("citdl") == "None": del arg.attrib["citdl"] # - Just can't agree on 'lineend' value for functions and classes right now. # TODO:XXX Need to eventually make sure that the new lineends # make sense! The main current diff is that pythoncile2 # includes trailing comment lines, even it not indented. # I think that is fine -- might even be helpful. for scope in tree1.getiterator("scope"): if scope.get("ilk") in ("function", "class") and scope.get("lineend"): del scope.attrib["lineend"] for scope in tree2.getiterator("scope"): if scope.get("ilk") in ("function", "class") and scope.get("lineend"): del scope.attrib["lineend"] # - pythoncile.py incorrectly normalizes double-quotes in function signature # argument default values to single quotes. This normalization will get it # wrong for quotes in quotes. for func in tree2.getiterator("scope"): if func.get("ilk") == "function": sig = func.get("signature") if '"' in sig: func.set("signature", sig.replace('"', "'")) pretty_tree_from_tree(tree2) norm_cix2 = ET.tostring(tree2, "utf-8") pretty_tree_from_tree(tree1) norm_cix1 = ET.tostring(tree1, "utf-8") import difflib diff = difflib.unified_diff( norm_cix1.splitlines(1), norm_cix2.splitlines(1), "pythoncile %s (normalized)" % path, "pythoncile2 %s (normalized)" % path) diff = ''.join(list(diff)) if diff: print(diff) else: sys.stdout.write(cix2)
#print >> sys.stderr, ("******************", tree2, tree2.tag, tree2.attrib); sys.stderr.flush() #assert False, ('===============', tree2.getchildren()[0].attrib) if tree2.get('error'): raise Exception(tree2.get('error')) if _gClockIt: sys.stdout.write(" (ast:%.3fs)" % (_gClock()-_gStartTime)) except Exception, ex: fileAttrs["error"] = str(ex) file = ' <file%s/>' % getAttrStr(fileAttrs) else: if tree2 is None: # This happens, for example, with: # foo(bar, baz=1, blam) fileAttrs["error"] = "could not generate AST" file = ' <file%s/>' % getAttrStr(fileAttrs) else: pretty_tree_from_tree(tree2) cix2 = ET.tostring(tree2, "utf-8") fileAttrs["md5"] = md5sum fileAttrs["mtime"] = mtime moduleName = os.path.splitext(os.path.basename(filename))[0] if _gClockIt: sys.stdout.write(" (walk:%.3fs)" % (_gClock()-_gStartTime)) if log.isEnabledFor(logging.INFO): # Dump a repr of the gathering info for debugging # - We only have to dump the module namespace because # everything else should be linked from it. for nspath, namespace in visitor.st.items(): if len(nspath) == 0: # this is the module namespace pprint.pprint(namespace) file = ' <file%s>\n\n%s\n </file>'\ % (getAttrStr(fileAttrs), cix2)
def main(argv=None): if argv is None: argv = sys.argv if not logging.root.handlers: logging.basicConfig() usage = "usage: %prog [PATHS...]" version = "%prog "+__version__ parser = optparse.OptionParser(prog="pythoncile2", usage=usage, version=version, description=__doc__, formatter=_NoReflowFormatter()) parser.add_option("-v", "--verbose", dest="log_level", action="store_const", const=logging.DEBUG, help="more verbose output") parser.add_option("-q", "--quiet", dest="log_level", action="store_const", const=logging.ERROR, help="less verbose output (only show errors)") parser.add_option("-c", "--compare", action="store_true", help="run against pythoncile.py as well (for testing)") parser.set_defaults(log_level=logging.INFO, compare=False) opts, paths = parser.parse_args(argv) log.setLevel(opts.log_level) assert len(paths) == 1, "usage: pythoncile2.py PATH" path = paths[0] try: tree2 = pythoncile2(path) except PythonCILEError as ex: log.error(str(ex)) if log.isEnabledFor(logging.DEBUG): print() import traceback traceback.print_exception(*sys.exc_info()) return 1 except KeyboardInterrupt: log.debug("user abort") return 1 pretty_tree_from_tree(tree2) cix2 = ET.tostring(tree2, "utf-8") if opts.compare: cix1 = _pythoncile_cix_from_path(path) if log.isEnabledFor(logging.DEBUG): print("-- pythoncile1 %s" % path) print(cix1) print("-- pythoncile2 %s" % path) print(cix2) # Normalizing for comparison. tree1 = ET.fromstring(cix1) # - mtime slightly different tree1[0].set("mtime", tree2[0].get("mtime")) # - pythoncile.py seems to put <import>'s at the top of the scope. # We'll just get tree2 to look like that. for scope in tree2.getiterator("scope"): if scope.get("ilk") not in ("blob", "function"): continue indeces = [] n = 0 for i, child in enumerate(scope[:]): if child.tag == "import": imp = scope[i] del scope[i] scope.insert(n, imp) n += 1 # - pythoncile2.py set citdl=tuple for 'varargs' arguments, and # citdl=dict for 'kwargs' arguments for arg in tree1.getiterator("variable"): if arg.get("ilk") != "argument": continue if arg.get("attributes") == "varargs": arg.set("citdl", "tuple") elif arg.get("attributes") == "kwargs": arg.set("citdl", "dict") # - pythoncile2.py will set citdl=None if for `foo(a=None)`, pythoncile.py # does not for arg in tree2.getiterator("variable"): if arg.get("ilk") != "argument": continue if arg.get("citdl") == "None": del arg.attrib["citdl"] # - Just can't agree on 'lineend' value for functions and classes right now. # TODO:XXX Need to eventually make sure that the new lineends # make sense! The main current diff is that pythoncile2 # includes trailing comment lines, even it not indented. # I think that is fine -- might even be helpful. for scope in tree1.getiterator("scope"): if scope.get("ilk") in ("function", "class") and scope.get("lineend"): del scope.attrib["lineend"] for scope in tree2.getiterator("scope"): if scope.get("ilk") in ("function", "class") and scope.get("lineend"): del scope.attrib["lineend"] # - pythoncile.py incorrectly normalizes double-quotes in function signature # argument default values to single quotes. This normalization will get it # wrong for quotes in quotes. for func in tree2.getiterator("scope"): if func.get("ilk") == "function": sig = func.get("signature") if '"' in sig: func.set("signature", sig.replace('"', "'")) pretty_tree_from_tree(tree2) norm_cix2 = ET.tostring(tree2, "utf-8") pretty_tree_from_tree(tree1) norm_cix1 = ET.tostring(tree1, "utf-8") import difflib diff = difflib.unified_diff( norm_cix1.splitlines(1), norm_cix2.splitlines(1), "pythoncile %s (normalized)" % path, "pythoncile2 %s (normalized)" % path) diff = ''.join(list(diff)) if diff: print(diff) else: sys.stdout.write(cix2)
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ import time import ciElementTree as ET from ci2 import _paths_from_path_patterns from codeintel2.manager import Manager from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.tree import pretty_tree_from_tree from codeintel2.util import guess_lang_from_path mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: self.log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. self.log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: self.log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) self.log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [ child for child in function if child.tag != 'variable' ] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write( '<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath + '.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath + '.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath + '.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath + '.options') # input options if _debug: print print "*" * 50, "codeintel '%s'" % fpath # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. # XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print "*" * 50, "options" pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO.StringIO() sys.stderr = StringIO.StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir)) + 1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set( "error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError, ex: error = traceback.format_exc() else:
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath+'.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath+'.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath+'.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath+'.options') # input options if _debug: print print "*"*50, "codeintel '%s'" % fpath # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. # XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print "*"*50, "options" pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO.StringIO() sys.stderr = StringIO.StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir))+1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set("error", len( cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError, ex: error = traceback.format_exc() else:
# assert False, ('===============', tree2.getchildren()[0].attrib) if tree2.get('error'): raise Exception(tree2.get('error')) if _gClockIt: sys.stdout.write(" (ast:%.3fs)" % (_gClock() - _gStartTime)) except Exception, ex: fileAttrs["error"] = str(ex) file = ' <file%s/>' % getAttrStr(fileAttrs) else: if tree2 is None: # This happens, for example, with: # foo(bar, baz=1, blam) fileAttrs["error"] = "could not generate AST" file = ' <file%s/>' % getAttrStr(fileAttrs) else: pretty_tree_from_tree(tree2) cix2 = ET.tostring(tree2, "utf-8") fileAttrs["md5"] = md5sum fileAttrs["mtime"] = mtime moduleName = os.path.splitext(os.path.basename(filename))[0] if _gClockIt: sys.stdout.write(" (walk:%.3fs)" % (_gClock() - _gStartTime)) if log.isEnabledFor(logging.INFO): # Dump a repr of the gathering info for debugging # - We only have to dump the module namespace because # everything else should be linked from it. for nspath, namespace in visitor.st.items(): if len(nspath) == 0: # this is the module namespace pprint.pprint(namespace) file = ' <file%s>\n\n%s\n </file>'\
replace_node(origBlobStar, curr_tree, t, i) i += 1 print "After doing yaml-nodes, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) i = 0 for t in bin_tree: replace_node(origFile, bin_tree, t, i) i += 1 print "After doing binary modules, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) # Finally update the YAML nodes, assuming that if we do a replacement # we can replace the entire subtree. i = 0 for t in yaml_stdlib_tree: replace_node(origFile, yaml_stdlib_tree, t, i) i += 1 print "After doing yaml modules, orig tree has %d nodes" % len(origBlobStar) print "Builtins: %d replacements, %d additions" % (num_replacements, num_additions) newFile = options.origfile + ".new" fd = open(newFile, "w") fd.write(ET.tostring(pretty_tree_from_tree(origTopLevel))) fd.close() print "Done writing to file %s" % newFile
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ import pprint import random import ciElementTree as ET from codeintel2.manager import Manager from codeintel2.tree import pretty_tree_from_tree from codeintel2.common import LogEvalController, Error from codeintel2.util import tree_from_cix, dedent, unmark_text, banner from ci2 import _escaped_text_from_text if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() # mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(self.log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) # XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint.pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(self.log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) # XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint.pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath+'.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath+'.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath+'.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath+'.options') # input options if _debug: print() print("*"*50, "codeintel '%s'" % fpath) # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. #XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print("*"*50, "options") pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir))+1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set("error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError as ex: error = traceback.format_exc() else: error = None if isinstance(cix, six.text_type): with io.open(tmpfile, mode="wt", encoding="utf-8") as fout: fout.write(cix) else: with open(tmpfile, mode="wb") as fout: fout.write(cix) finally: stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print("*"*50, "stdout") print(stdout) print("*"*50, "stderr") print(stderr) print("*"*50, "error") print(str(error)) print("*" * 50) generateMissing = False if not os.path.exists(outfile) and generateMissing: with io.open(outfile, mode='wt', encoding='utf-8') as fout: with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: fout.write(ftmp.read()) # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1)+path+match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) # and normalize ' ', ' ' and ''' with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) expected = expected.replace('
', ' ').replace('
', ' ').replace(''', '\'') with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read()) actual = actual.replace('
', ' ').replace('
', ' ').replace(''', '\'') if expected != actual: do_fail = True # Useful temporary thing while XML output format is changing. #if os.stat("../support/xmldiff.py"): # rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile)) # if rc == 0: # do_fail = False if do_fail: diff = list(difflib.ndiff(expected.splitlines(1), actual.splitlines(1))) diff = _diffContext(diff, 2) if diff: error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \ % (outfile, tmpfile, outfile, tmpfile, ''.join(diff)) if gMaxDiffOutput > 0 and gMaxNumLines > 0: if len(error_str) > gMaxDiffOutput: error_lines = error_str.split("\n") if len(error_lines) > gMaxNumLines: error_lines = error_lines[:gMaxNumLines] + ["..."] if gMaxLineLength > 0: error_str = "\n".join([len(x) > gMaxLineLength and x[:gMaxLineLength] or x for x in error_lines]) else: error_str = "\n".join(error_lines) self.fail(_encode_for_stdout(error_str)) elif os.path.exists(errfile): # There is no reference output file. This means that processing # this file is expected to fail. expectedError = open(errfile, 'r').read() actualError = str(error) self.failUnlessEqual(actualError.strip(), expectedError.strip()) else: self.fail("No reference output file or error file for '%s'." % infile) # Ensure next test file gets a clean codeintel. toDelete = [] for modname in sys.modules: if modname == "codeintel" or modname.startswith("codeintel."): toDelete.append(modname) for modname in toDelete: del sys.modules[modname]