def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def worker(queue, lock, cix_dir, root="/"): """ worker procedure """ fix_module_path() import dxr.mime from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.manager import Manager from codeintel2.util import guess_lang_from_path logging.getLogger("codeintel").setLevel(logging.ERROR) log.info("starting indexing using %s", multiprocessing.current_process().name) mgr = Manager(db_base_dir=cix_dir) #mgr.upgrade() mgr.initialize() while True: file_path = queue.get() if file_path is None: # marker for end of list queue.put(None) # put it back so others can quit too break rel_path = os.path.relpath(file_path, root) try: lang = guess_lang_from_path(file_path) except CodeIntelError: log.info("%s: Cannot determine language, skipping", rel_path) continue # the file with open(file_path, "r") as source_file: data = source_file.read() # Discard non-text files if not dxr.mime.is_text(file_path, data): continue try: buf = mgr.buf_from_path(file_path, lang=lang) except CodeIntelError as ex: if ex.message.startswith("File too big."): log.info("%s: %s", file_path, ex.message) continue # Nothing we can do about that, and the user can't # fix this ever. raise if not isinstance(buf, CitadelBuffer): log.info("%s: language %s does not have CIX, skipping", rel_path, lang) continue log.info("%s: Using language %s", rel_path, lang) buf.scan() mgr.finalize()
def do_html(self, subcmd, opts, path): """Convert the given path to styled HTML. ${cmd_usage} ${cmd_option_list} The output includes trigger info and other stats. I.e. this is primarily a debugging tool. """ from codeintel2.manager import Manager from codeintel2.common import Error from ci2 import _url_from_local_path mgr = Manager() try: if opts.browse: htmls = [] buf = mgr.buf_from_path(path, lang=opts.lang) html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval) finally: mgr.finalize() if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = path + ".html" if os.path.exists(output_path): if opts.force: os.remove(output_path) else: raise Error( "`%s' exists: use -f|--force option to allow overwrite" % output_path) output_file = open(output_path, 'w') # else: # output_path = None # output_file = sys.stdout # #XXX Disable writing t # output_file = None if output_file: output_file.write(html) if output_path: output_file.close() if opts.browse: if not output_path: raise Error("cannot open in browser if stdout used " "for output") import webbrowser url = _url_from_local_path(output_path) webbrowser.open_new(url)
def do_html(self, subcmd, opts, path): """Convert the given path to styled HTML. ${cmd_usage} ${cmd_option_list} The generated HTML provides a good tool for seeing how Komodo's lexer lexes the given content. This is the same tokenization that you get from "buf.accessor.*" methods -- which you can use for writing the CILE, trigger handling, and completion evaluation for this language. Use the "-t" and "-e" option to also exercise the current trigger handling and completion evaluation (i.e. determining the appropriate completions and calltips at a given trigger point). """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) try: if opts.browse: htmls = [] buf = mgr.buf_from_path(path, lang=opts.lang) html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval) finally: mgr.finalize() if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = path + ".html" if exists(output_path): os.remove(output_path) output_file = open(output_path, 'w') if output_file: output_file.write(html) if output_path: output_file.close() if opts.browse: if not output_path: raise CodeIntelError("cannot open in browser if stdout " "used for output") import webbrowser url = _url_from_local_path(output_path) webbrowser.open_new(url)
def do_json(self, subcmd, opts, path): """Convert cix XML file into json format. ${cmd_usage} ${cmd_option_list} """ import json if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = splitext(path)[0]+".json" if exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to " "allow overwrite" % output_path) output_file = open(output_path, 'w') mgr = Manager() mgr.upgrade() mgr.initialize() try: if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree result = {} ci = result["codeintel"] = defaultdict(list) def _elemToDict(parent, elem): data = defaultdict(list) name = elem.get("name") if name is not None: data["name"] = name data["tag"] = elem.tag for attr_name, attr in elem.attrib.items(): data[attr_name] = attr parent["children"].append(data) for child in elem: _elemToDict(data, child) for child in tree: _elemToDict(ci, child) json.dump(result, output_file, indent=2) finally: mgr.finalize()
def do_json(self, subcmd, opts, path): """Convert cix XML file into json format. ${cmd_usage} ${cmd_option_list} """ import json if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = splitext(path)[0] + ".json" if exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to " "allow overwrite" % output_path) output_file = open(output_path, 'w') mgr = Manager() mgr.upgrade() mgr.initialize() try: if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree result = {} ci = result["codeintel"] = defaultdict(list) def _elemToDict(parent, elem): data = defaultdict(list) name = elem.get("name") if name is not None: data["name"] = name data["tag"] = elem.tag for attr_name, attr in elem.attrib.items(): data[attr_name] = attr parent["children"].append(data) for child in elem: _elemToDict(data, child) for child in tree: _elemToDict(ci, child) json.dump(result, output_file, indent=2) finally: mgr.finalize()
def do_html(self, subcmd, opts, path): """Convert the given path to styled HTML. ${cmd_usage} ${cmd_option_list} The output includes trigger info and other stats. I.e. this is primarily a debugging tool. """ from codeintel2.manager import Manager from codeintel2.common import Error from ci2 import _url_from_local_path mgr = Manager() try: if opts.browse: htmls = [] buf = mgr.buf_from_path(path, lang=opts.lang) html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval) finally: mgr.finalize() if opts.output == '-': output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = path + ".html" if os.path.exists(output_path): if opts.force: os.remove(output_path) else: raise Error("`%s' exists: use -f|--force option to allow overwrite" % output_path) output_file = open(output_path, 'w') # else: # output_path = None # output_file = sys.stdout # #XXX Disable writing t # output_file = None if output_file: output_file.write(html) if output_path: output_file.close() if opts.browse: if not output_path: raise Error("cannot open in browser if stdout used " "for output") import webbrowser url = _url_from_local_path(output_path) webbrowser.open_new(url)
def do_html(self, subcmd, opts, path): """Convert the given path to styled HTML. ${cmd_usage} ${cmd_option_list} The generated HTML provides a good tool for seeing how Komodo's lexer lexes the given content. This is the same tokenization that you get from "buf.accessor.*" methods -- which you can use for writing the CILE, trigger handling, and completion evaluation for this language. Use the "-t" and "-e" option to also exercise the current trigger handling and completion evaluation (i.e. determining the appropriate completions and calltips at a given trigger point). """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) try: if opts.browse: htmls = [] buf = mgr.buf_from_path(path, lang=opts.lang) html = buf.to_html(True, True, title=path, do_trg=opts.do_trg, do_eval=opts.do_eval) finally: mgr.finalize() if opts.output == "-": output_path = None output_file = sys.stdout else: if opts.output: output_path = opts.output else: output_path = path + ".html" if exists(output_path): os.remove(output_path) output_file = open(output_path, "w") if output_file: output_file.write(html) if output_path: output_file.close() if opts.browse: if not output_path: raise CodeIntelError("cannot open in browser if stdout " "used for output") import webbrowser url = _url_from_local_path(output_path) webbrowser.open_new(url)
def do_dbcheck(self, subcmd, opts): """Run an internal consistency check on the database. ${cmd_usage} ${cmd_option_list} Any errors will be printed. Returns the number of errors (i.e. exit value is 0 if there are no consistency problems). """ mgr = Manager(opts.db_base_dir) try: errors = mgr.db.check() finally: mgr.finalize() for error in errors: print(error) return len(errors)
def do_dbcheck(self, subcmd, opts): """Run an internal consistency check on the database. ${cmd_usage} ${cmd_option_list} Any errors will be printed. Returns the number of errors (i.e. exit value is 0 if there are no consistency problems). """ mgr = Manager(opts.db_base_dir) try: errors = mgr.db.check() finally: mgr.finalize() for error in errors: print error return len(errors)
def casper_tests(): from codeintel2.manager import Manager db_base_dir = join(dirname(__file__), "tmp", "db") mgr = Manager(db_base_dir) mgr.upgrade() mgr.initialize() try: for testpath in testpaths(): buf = mgr.buf_from_path(testpath, lang="JavaScript") # Ensure the test is up to date. if buf.scan_time < os.stat(testpath).st_mtime: buf.scan() for test in casper_tests_from_citree(buf.tree): yield test finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ extra_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_module_dirs = [sys.path[-1]] mgr = Manager(extra_module_dirs=extra_module_dirs) mgr.upgrade() mgr.initialize() try: tree = None for path in _paths_from_path_patterns(path_patterns): try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) continue buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) buf.scan() # force a fresh scan tree = buf.tree for severity, msg in check_tree(tree): dump = {"warning": log.warn, "error": log.error}[severity] dump(msg) if opts.pretty_print: tree = pretty_tree_from_tree(tree) ET.dump(tree) finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ import time import ciElementTree as ET from ci2 import _paths_from_path_patterns from codeintel2.manager import Manager from codeintel2.citadel import CitadelBuffer from codeintel2.common import CodeIntelError from codeintel2.tree import pretty_tree_from_tree from codeintel2.util import guess_lang_from_path mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: self.log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. self.log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: self.log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) self.log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Print code outline of the given file. You can specify a lookup path into the file code outline to display via URL-anchor syntax, e.g.: ci2 outline path/to/foo.py#AClass.amethod ${cmd_usage} ${cmd_option_list} """ import re from ci2 import _outline_ci_elem from codeintel2.manager import Manager from codeintel2.util import tree_from_cix mgr = Manager() mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) # buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: self.log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort) except IOError as ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True scan_count = 0 lang_warnings = set() tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path + "\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue try: buf = mgr.buf_from_path(path, lang=lang) except OSError as ex: # Couldn't access the file. if not opts.recursive: raise # Ignore files we don't really care about. log.warn("%r - %r", ex, path) continue if not isinstance(buf, CitadelBuffer): if opts.recursive: # Ignore files that scanning isn't provided for. continue raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) scan_count += 1 if scan_count % 10 == 0: log.info("%d scanning %r", scan_count, path) try: if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) except KeyError as ex: # Unknown cile language. if not opts.recursive: raise message = str(ex) if message not in lang_warnings: lang_warnings.add(message) log.warn("Skipping unhandled language %s", message) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [ child for child in function if child.tag != 'variable' ] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write( '<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end - start)) finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Scan and outline the structure of the given path. ${cmd_usage} ${cmd_option_list} """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None tree = None try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info( "skip `%s': couldn't determine language " "(use --lang option)", path) return if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(mgr, elem, quiet=opts.quiet) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of this # command: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Print code outline of the given file. You can specify a lookup path into the file code outline to display via URL-anchor syntax, e.g.: ci2 outline path/to/foo.py#AClass.amethod ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error( "could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort) except IOError as ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
def cix2html(opts, path): """Turn cix file into html API documentation. Example: cix2html path/to/foo.cix#AClass.amethod cix2html path/to/foo.cix -o file.html ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob if '#' in path: path, anchor = path.rsplit('#', 1) else: anchor = None if path.endswith(".cix"): tree = tree_from_cix(open(path, 'r').read()) #buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r'\.|::', anchor) for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: if elem.tag == "codeintel": _html_ci_elem(opts, elem.getchildren()[0]) else: _html_ci_elem(opts, elem) except IOError as ex: if ex.errno == 0: # Ignore this error from aborting 'less' of 'ci2 outline' # output: # IOError: (0, 'Error') pass else: raise except Exception as e: import traceback traceback.print_exc() finally: mgr.finalize()
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() #mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print( ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) #XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
class Driver(threading.Thread): """ Out-of-process codeintel2 driver This class implements the main loop of the codeintel worker process This should be a singleton object """ # static _instance = None _command_handler_map = collections.defaultdict(list) """Registered command handlers; the key is the command name (a str), and the value is a list of command handler instances or (for lazy handlers) a single-element tuple containing the callable to get the real handler.""" _default_handler = None """The default (core) command handler; instance of a CoreHandler""" _builtin_commands = {} """Built-in commands that cannot be overridden""" def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout): threading.Thread.__init__(self, name="CodeIntel OOP Driver") assert Driver._instance is None, "Driver should be a singleton" Driver._instance = self logging.root.addHandler(LoggingHandler(self)) self.daemon = True self.fd_in = fd_in self.fd_out = fd_out self.abort = None self.quit = False self.buffers = {} # path to Buffer objects self.next_buffer = 0 self.active_request = None self.send_queue = queue.Queue() self.send_thread = threading.Thread( name="Codeintel OOP Driver Send Thread", target=self._send_proc) self.send_thread.daemon = True self.send_thread.start() self.queue = collections.deque() self.queue_cv = threading.Condition() self.env = Environment(name="global", send_fn=functools.partial(self.send, request=None)) # Fill out the non-overridable build-in commands self._builtin_commands = {} for attr in dir(self): # Note that we check startswith first to avoid getters etc. if attr.startswith("do_") and callable(getattr(self, attr)): command = attr[len("do_"):].replace("_", "-") self._builtin_commands[command] = getattr(self, attr) from codeintel2.manager import Manager log.debug("using db base dir %s", db_base_dir) self.mgr = Manager(db_base_dir=db_base_dir, db_catalog_dirs=[], db_event_reporter=self._DBEventReporter(self), env=self.env, on_scan_complete=self._on_scan_complete) self.mgr.initialize() def _on_scan_complete(self, scan_request): if scan_request.status in ("changed", "skipped"): # Send unsolicited response about the completed scan buf = scan_request.buf self.send(request=None, path=buf.path, language=buf.lang, command="scan-complete") class _DBEventReporter(object): def __init__(self, driver): self.driver = driver self.log = log.getChild("DBEventReporter") self.debug = self.log.debug # directories being scanned (completed or not) # key is unicode path, value is number of times it's active self._dirs = collections.defaultdict(int) # directories which are complete (unicode path) self._completed_dirs = set() def send(self, **kwargs): self.driver.send(request=None, command="report-message", **kwargs) def __call__(self, message): """Old-style status messages before long-running jobs @param msg {str or None} The message to display """ if len(self._dirs): return # ignore old-style db events if we're doing a scan self.debug("db event: %s", message) self.send(message=message) def onScanStarted(self, description, dirs=set()): """Called when a directory scan is about to start @param description {unicode} A string suitable for showing the user about the upcoming operation @param dirs {set of unicode} The directories about to be scanned """ self.debug("scan started: %s (%s dirs)", description, len(dirs)) assert dirs, "onScanStarted expects non-empty directories" if not dirs: # empty set - we shouldn't have gotten here, but be nice return for dir_path in dirs: self._dirs[dir_path] += 1 self.send(type="scan-progress", message=description, completed=len(self._completed_dirs), total=len(self._dirs)) def onScanDirectory(self, description, dir_path, current=None, total=None): """Called when a directory is being scanned (out of possibly many) @param description {unicode} A string suitable for showing the user regarding the progress @param dir {unicode} The directory currently being scanned @param current {int} The current progress @param total {int} The total number of directories to scan in this request """ self.debug("scan directory: %s (%s %s/%s)", description, dir_path, current, total) assert dir_path, "onScanDirectory got no directory" if dir_path: self._completed_dirs.add(dir_path) self.send(type="scan-progress", message=description, completed=len(self._completed_dirs), total=len(self._dirs)) def onScanFile(self, description, file_path, current=None, total=None): """Called when a directory is being scanned (out of possibly many) @param description {unicode} A string suitable for showing the user regarding the progress @param dir {unicode} The directory currently being scanned @param current {int} The current progress @param total {int} The total number of directories to scan in this request """ self.debug("scan file: %s (%s %s/%s)", description, file_path, current, total) assert file_path, "onScanDirectory got no directory" self.send(type="scan-progress", message=description, completed=current, total=total) def onScanComplete(self, dirs=set(), scanned=set()): """Called when a scan operation is complete @param dirs {set of unicode} The directories that were intially requested to be scanned (as pass in onScanStarted) @param scanned {set of unicode} Directories which were successfully scanned. This may be a subset of dirs if the scan was aborted. """ self.debug("scan complete: scanned %r/%r dirs", len(scanned), len(dirs)) for dir_path in dirs: self._dirs[dir_path] -= 1 if not self._dirs[dir_path]: del self._dirs[dir_path] self._completed_dirs.discard(dir_path) self.send(type="scan-progress", completed=len(self._completed_dirs), total=len(self._dirs)) REQUEST_DEFAULT = object() def send(self, request=REQUEST_DEFAULT, **kwargs): """ Send a response """ data = dict(kwargs) if request is Driver.REQUEST_DEFAULT: request = self.active_request if request: data["req_id"] = request.id if "success" not in data: data["success"] = True elif data["success"] is None: del data["success"] buf = json.dumps(data, separators=(',', ':')) buf_len = u"%i" % len(buf) log.debug("sending: %s:[%s]", buf_len, buf) self.send_queue.put(buf.encode('utf-8')) def _send_proc(self): while True: buf = self.send_queue.get() try: buf_len = (u"%i" % len(buf)).encode('utf-8') self.fd_out.write(buf_len) self.fd_out.write(buf) finally: self.send_queue.task_done() def fail(self, request=REQUEST_DEFAULT, **kwargs): kwargs = kwargs.copy() if not "command" in kwargs and request: try: kwargs["command"] = request["command"] except KeyError: pass return self.send(request=request, success=False, **kwargs) def exception(self, request=REQUEST_DEFAULT, **kwargs): kwargs["command"] = "report-error" kwargs["type"] = "logging" kwargs["name"] = log.name kwargs["level"] = logging.ERROR return self.fail(request=request, stack=traceback.format_exc(), **kwargs) @staticmethod def normpath(path): """Routine to normalize the path used for codeintel buffers This is annoying because it needs to handle unsaved files, as well as urls. @note See also koCodeIntel.py::KoCodeIntelBuffer.normpath """ if path.startswith("<Unsaved>"): return path # Don't munge unsaved file paths scheme = path.split(":", 1)[0] if len(scheme) == len(path): # didn't find a scheme at all; assume this is a local path return os.path.normcase(path) if len(scheme) == 1: # single-character scheme; assume this is actually a drive letter # (for a Windows-style path) return os.path.normcase(path) scheme_chars = string.ascii_letters + string.digits + "-" try: scheme = scheme.encode("ascii") except UnicodeEncodeError: # scheme has a non-ascii character; assume local path return os.path.normcase(path) if scheme.translate(None, scheme_chars): # has a non-scheme character: this is not a valid scheme # assume this is a local file path return os.path.normcase(path) if scheme != "file": return path # non-file scheme path = path[len(scheme) + 1:] return os.path.normcase(path) def get_buffer(self, request=REQUEST_DEFAULT, path=None): if request is Driver.REQUEST_DEFAULT: request = self.active_request if path is None: if not "path" in request: raise RequestFailure(message="No path given to locate buffer") path = request.path path = self.normpath(path) try: buf = self.buffers[path] except KeyError: buf = None else: if "language" in request and buf.lang != request.language: buf = None # language changed, re-scan if not buf: # Need to construct a new buffer lang = request.get("language") env = Environment(request=request, name=os.path.basename(path)) if request.get("text") is not None: # pass no content; we'll reset it later buf = self.mgr.buf_from_content("", lang, path=path, env=env) else: # read from file try: buf = self.mgr.buf_from_path( path, lang, env=env, encoding=request.get("encoding")) except OSError: # Can't read the file buf = self.mgr.buf_from_content("", lang, path=path, env=env) assert not request.path.startswith("<"), \ "Can't create an unsaved buffer with no text" if request.get("text") is not None: # overwrite the buffer contents if we have new ones buf.accessor.reset_content(request.text) buf.encoding = "utf-8" try: env = request["env"] except KeyError: pass # no environment, use current else: buf._env.update(env) #log.debug("Got buffer %r: [%s]", buf, buf.accessor.content) log.debug("Got buffer %r", buf) self.buffers[path] = buf return buf def do_abort(self, request): try: req_id = request["id"] with self.queue_cv: for item in self.queue: if item.get("req_id") == req_id: self.queue.remove(item) self.send(request=request) break else: self.abort = req_id if self.active_request and self.active_request.id == req_id: # need to wait a bit... self.send(request=request) else: self.fail(request=request, message="Request %s not found" % (req_id, )) except RequestFailure as e: self.fail(request=request, **e.kwargs) except Exception as e: log.exception(e.message) self.exception(request=request, message=e.message) def do_add_dirs(self, request): catalog_dirs = request.get("catalog-dirs", None) if catalog_dirs is not None: self.mgr.db.catalog_dirs.extend(catalog_dirs) catalog_dirs = self.mgr.db.catalog_dirs lexer_dirs = request.get("lexer-dirs", []) codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs) module_dirs = request.get("module-dirs", []) if module_dirs: self.mgr._register_modules(module_dirs) if catalog_dirs is not None: self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs self.send(request=request) def do_load_extension(self, request): """Load an extension that, for example, might provide additional command handlers""" name = request.get("module-name", None) if not name: raise RequestFailure(msg="load-extension requires a module-name") path = request.get("module-path", None) names = name.split(".") if len(names) > 1: # It's inside a package - go look for the package(s). for package_name in names[:-1]: iinfo = imp.find_module(package_name, [path] if path else None) if not iinfo: raise RequestFailure(msg="load-extension could not find " "package %r for given name %r" % (package_name, name)) path = iinfo[1] name = names[-1] iinfo = imp.find_module(name, [path] if path else None) try: module = imp.load_module(name, *iinfo) finally: if iinfo and iinfo[0]: iinfo[0].close() callback = getattr(module, "registerExtension", None) if not callback: raise RequestFailure(msg="load-extension module %s should " "have a 'registerExtension' method " "taking no arguments" % (name, )) callback() self.send() # success, finally def do_quit(self, request): self.quit = True self.send(command="quit") def report_error(self, message): self.send(request=None, command="report-error", message=six.text_type(message)) def finalize(self, timeout=None): self.do_quit(None) self.mgr.finalize() def start(self): """Start reading from the socket and dump requests into the queue""" log.info("Running codeintel driver...") buf = b'' self.send(success=None) self.daemon = True threading.Thread.start(self) while not self.quit: try: ch = self.fd_in.read(1) except IOError: log.debug( "Failed to read frame length, assuming connection died") self.quit = True break if not ch: log.debug("Input was closed") self.quit = True break if ch == b'{': length = int(buf, 10) try: buf = ch while len(buf) < length: data = self.fd_in.read(length - len(buf)) if not data: # nothing read, EOF raise IOError("Failed to read frame from socket") buf += data except IOError: log.debug( "Failed to read frame data, assuming connection died") self.quit = True break try: data = json.loads(buf.decode('utf-8')) request = Request(data) except Exception as e: log.debug("Got codeintel request: %r" % buf) log.exception(e) self.exception(message=e.message, request=None) continue finally: buf = b'' if request.get("command") == "abort": self.do_abort(request=request) else: log.debug("queuing request %r", request) with self.queue_cv: self.queue.appendleft(request) self.queue_cv.notify() elif ch in b'0123456789': buf += ch elif ch in b' \t\r\n': pass else: raise ValueError("Invalid frame length character: %r" % ch) log.info("Codeintel driver ended") def run(self): """Evaluate and send results back""" log.info("Running codeintel eval thread...") log.debug("default supported commands: %s", ", ".join(self._default_handler.supportedCommands)) while not self.quit: with self.queue_cv: try: request = self.queue.pop() except IndexError: self.queue_cv.wait() continue log.debug("doing request %r", request) try: self.active_request = request command = request.command # First, check abort and quit; don't allow those to be overridden try: builtin = self._builtin_commands[command] except KeyError: pass else: builtin(request) continue handlers = self._command_handler_map.get(command, [])[:] if command in self._default_handler.supportedCommands: # The default handler can deal with this, put it at the end handlers.append(self._default_handler) for handler in handlers: if isinstance(handler, tuple): try: real_handler = handler[0]() except Exception as ex: log.exception("Failed to get lazy handler for %s", command) real_handler = None if real_handler is None: # Handler failed to instantiate, drop it try: self._command_handler_map[command].remove( handler) except ValueError: pass # ... shouldn't happen, but tolerate it continue for handlers in self._command_handler_map.values(): try: handlers[handlers.index( handler)] = real_handler except ValueError: pass # handler not in this list handler = real_handler if handler.canHandleRequest(request): handler.handleRequest(request, self) break else: self.fail(request=request, message="Don't know how to handle command %s" % (command, )) except RequestFailure as e: self.fail(request=request, **e.kwargs) except Exception as e: log.exception(e) e = getattr(e, 'message', repr(e)) self.exception(request=request, message=e) finally: self.active_request = None log.info("Codeintel eval thread ended") @classmethod def registerCommandHandler(cls, handlerInstance): """Register a command handler""" for command in handlerInstance.supportedCommands: cls._command_handler_map[command].append(handlerInstance) @classmethod def registerLazyCommandHandler(cls, supported_commands, constructor): """Register a lazy command handler @param supported_commands {iterable} The commands to handle; each element should be a str of the command name. @param constructor {callable} Function to be called to get the real command handler; it should take no arguments and return a command handler instance. It may return None if the command is not available; it will not be asked again. """ for command in supported_commands: cls._command_handler_map[command].append((constructor, )) @classmethod def getInstance(cls): """Get the singleton instance of the driver""" return Driver._instance
def do_scan(self, subcmd, opts, *path_patterns): """Scan and print the CIX for the given path(s). ${cmd_usage} ${cmd_option_list} """ mgr = Manager() mgr.upgrade() mgr.initialize() try: if opts.time_it: start = time.time() quiet = opts.quiet if opts.time_it or opts.time_details: opts.force = True tree = None for path in _paths_from_path_patterns(path_patterns, recursive=opts.recursive, includes=opts.includes): if opts.time_it: sys.stderr.write(path+"\n") if opts.time_details: start1 = time.time() try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language", path) continue buf = mgr.buf_from_path(path, lang=lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) if opts.force: buf.scan() if tree is None: tree = ET.Element("codeintel", version="2.0") file_elem = ET.SubElement(tree, "file", lang=buf.lang, mtime=str(int(time.time())), path=os.path.basename(path)) for lang, blob in sorted(buf.blob_from_lang.items()): blob = buf.blob_from_lang[lang] file_elem.append(blob) if opts.time_details: delta = time.time() - start1 sys.stderr.write("%.3f %s\n" % (delta, path)) sys.stderr.flush() if tree is not None: if opts.stripfuncvars: # For stdlibs, we don't care about variables inside of # functions and they take up a lot of space. for function in tree.getiterator('scope'): if function.get('ilk') == 'function': function[:] = [child for child in function if child.tag != 'variable'] if opts.pretty_print: tree = pretty_tree_from_tree(tree) if not quiet: sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n') ET.dump(tree) if opts.time_it: end = time.time() sys.stderr.write("scan took %.3fs\n" % (end-start)) finally: mgr.finalize()
def do_outline(self, subcmd, opts, path): """Scan and outline the structure of the given path. ${cmd_usage} ${cmd_option_list} """ extra_lang_module_dirs = [] if koextlib.is_ext_dir() and exists("pylib"): sys.path.append(abspath("pylib")) extra_lang_module_dirs = [sys.path[-1]] mgr = Manager(extra_lang_module_dirs=extra_lang_module_dirs) mgr.upgrade() mgr.initialize() try: if "#" in path: path, anchor = path.rsplit("#", 1) else: anchor = None tree = None try: lang = opts.lang or guess_lang_from_path(path) except CodeIntelError: log.info("skip `%s': couldn't determine language " "(use --lang option)", path) return if path.endswith(".cix"): tree = tree_from_cix(open(path, "r").read()) # buf = mgr.buf_from_content("", tree[0].get("lang"), path=path) else: buf = mgr.buf_from_path(path, lang=opts.lang) if not isinstance(buf, CitadelBuffer): raise CodeIntelError("`%s' (%s) is not a language that " "uses CIX" % (path, buf.lang)) tree = buf.tree if anchor is not None: # Lookup the anchor in the codeintel CIX tree. lpath = re.split(r"\.|::", anchor) def blobs_from_tree(tree): for file_elem in tree: for blob in file_elem: yield blob for elem in blobs_from_tree(tree): # Generally have 3 types of codeintel trees: # 1. single-lang file: one <file>, one <blob> # 2. multi-lang file: one <file>, one or two <blob>'s # 3. CIX stdlib/catalog file: possibly multiple # <file>'s, likely multiple <blob>'s # Allow the first token to be the blob name or lang. # (This can sometimes be weird, but seems the most # convenient solution.) if lpath[0] in (elem.get("name"), elem.get("lang")): remaining_lpath = lpath[1:] else: remaining_lpath = lpath for name in remaining_lpath: try: elem = elem.names[name] except KeyError: elem = None break # try next lang blob if elem is not None: break # found one else: log.error("could not find `%s' definition (or blob) in `%s'", anchor, path) return 1 else: elem = tree try: _outline_ci_elem(mgr, elem, quiet=opts.quiet) except IOError, ex: if ex.errno == 0: # Ignore this error from aborting 'less' of this # command: # IOError: (0, 'Error') pass else: raise finally: mgr.finalize()
class Driver(threading.Thread): """ Out-of-process codeintel2 driver This class implements the main loop of the codeintel worker process This should be a singleton object """ # static _instance = None _command_handler_map = collections.defaultdict(list) """Registered command handlers; the key is the command name (a str), and the value is a list of command handler instances or (for lazy handlers) a single-element tuple containing the callable to get the real handler.""" _default_handler = None """The default (core) command handler; instance of a CoreHandler""" _builtin_commands = {} """Built-in commands that cannot be overridden""" def __init__(self, db_base_dir=None, fd_in=sys.stdin, fd_out=sys.stdout): threading.Thread.__init__(self, name="CodeIntel OOP Driver") assert Driver._instance is None, "Driver should be a singleton" Driver._instance = self logging.root.addHandler(LoggingHandler(self)) self.daemon = True self.fd_in = fd_in self.fd_out = fd_out self.abort = None self.quit = False self.buffers = {} # path to Buffer objects self.next_buffer = 0 self.active_request = None self.send_queue = queue.Queue() self.send_thread = threading.Thread(name="Codeintel OOP Driver Send Thread", target=self._send_proc) self.send_thread.daemon = True self.send_thread.start() self.queue = collections.deque() self.queue_cv = threading.Condition() self.env = Environment(name="global", send_fn=functools.partial(self.send, request=None)) # Fill out the non-overridable build-in commands self._builtin_commands = {} for attr in dir(self): # Note that we check startswith first to avoid getters etc. if attr.startswith("do_") and callable(getattr(self, attr)): command = attr[len("do_"):].replace("_", "-") self._builtin_commands[command] = getattr(self, attr) from codeintel2.manager import Manager log.debug("using db base dir %s", db_base_dir) self.mgr = Manager(db_base_dir=db_base_dir, db_catalog_dirs=[], db_event_reporter=self._DBEventReporter(self), env=self.env, on_scan_complete=self._on_scan_complete) self.mgr.initialize() def _on_scan_complete(self, scan_request): if scan_request.status in ("changed", "skipped"): # Send unsolicited response about the completed scan buf = scan_request.buf self.send(request=None, path=buf.path, language=buf.lang, command="scan-complete") class _DBEventReporter(object): def __init__(self, driver): self.driver = driver self.log = log.getChild("DBEventReporter") self.debug = self.log.debug # directories being scanned (completed or not) # key is unicode path, value is number of times it's active self._dirs = collections.defaultdict(int) # directories which are complete (unicode path) self._completed_dirs = set() def send(self, **kwargs): self.driver.send(request=None, command="report-message", **kwargs) def __call__(self, message): """Old-style status messages before long-running jobs @param msg {str or None} The message to display """ if len(self._dirs): return # ignore old-style db events if we're doing a scan self.debug("db event: %s", message) self.send(message=message) def onScanStarted(self, description, dirs=set()): """Called when a directory scan is about to start @param description {unicode} A string suitable for showing the user about the upcoming operation @param dirs {set of unicode} The directories about to be scanned """ self.debug("scan started: %s (%s dirs)", description, len(dirs)) assert dirs, "onScanStarted expects non-empty directories" if not dirs: # empty set - we shouldn't have gotten here, but be nice return for dir_path in dirs: self._dirs[dir_path] += 1 self.send(type="scan-progress", message=description, completed=len(self._completed_dirs), total=len(self._dirs)) def onScanDirectory(self, description, dir_path, current=None, total=None): """Called when a directory is being scanned (out of possibly many) @param description {unicode} A string suitable for showing the user regarding the progress @param dir {unicode} The directory currently being scanned @param current {int} The current progress @param total {int} The total number of directories to scan in this request """ self.debug("scan directory: %s (%s %s/%s)", description, dir_path, current, total) assert dir_path, "onScanDirectory got no directory" if dir_path: self._completed_dirs.add(dir_path) self.send(type="scan-progress", message=description, completed=len(self._completed_dirs), total=len(self._dirs)) def onScanFile(self, description, file_path, current=None, total=None): """Called when a directory is being scanned (out of possibly many) @param description {unicode} A string suitable for showing the user regarding the progress @param dir {unicode} The directory currently being scanned @param current {int} The current progress @param total {int} The total number of directories to scan in this request """ self.debug("scan file: %s (%s %s/%s)", description, file_path, current, total) assert file_path, "onScanDirectory got no directory" self.send(type="scan-progress", message=description, completed=current, total=total) def onScanComplete(self, dirs=set(), scanned=set()): """Called when a scan operation is complete @param dirs {set of unicode} The directories that were intially requested to be scanned (as pass in onScanStarted) @param scanned {set of unicode} Directories which were successfully scanned. This may be a subset of dirs if the scan was aborted. """ self.debug("scan complete: scanned %r/%r dirs", len(scanned), len(dirs)) for dir_path in dirs: self._dirs[dir_path] -= 1 if not self._dirs[dir_path]: del self._dirs[dir_path] self._completed_dirs.discard(dir_path) self.send(type="scan-progress", completed=len(self._completed_dirs), total=len(self._dirs)) REQUEST_DEFAULT = object() def send(self, request=REQUEST_DEFAULT, **kwargs): """ Send a response """ data = dict(kwargs) if request is Driver.REQUEST_DEFAULT: request = self.active_request if request: data["req_id"] = request.id if "success" not in data: data["success"] = True elif data["success"] is None: del data["success"] buf = json.dumps(data, separators=(',', ':')) buf_len = u"%i" % len(buf) log.debug("sending: %s:[%s]", buf_len, buf) self.send_queue.put(buf.encode('utf-8')) def _send_proc(self): while True: buf = self.send_queue.get() try: buf_len = (u"%i" % len(buf)).encode('utf-8') self.fd_out.write(buf_len) self.fd_out.write(buf) finally: self.send_queue.task_done() def fail(self, request=REQUEST_DEFAULT, **kwargs): kwargs = kwargs.copy() if not "command" in kwargs and request: try: kwargs["command"] = request["command"] except KeyError: pass return self.send(request=request, success=False, **kwargs) def exception(self, request=REQUEST_DEFAULT, **kwargs): kwargs["command"] = "report-error" kwargs["type"] = "logging" kwargs["name"] = log.name kwargs["level"] = logging.ERROR return self.fail(request=request, stack=traceback.format_exc(), **kwargs) @staticmethod def normpath(path): """Routine to normalize the path used for codeintel buffers This is annoying because it needs to handle unsaved files, as well as urls. @note See also koCodeIntel.py::KoCodeIntelBuffer.normpath """ if path.startswith("<Unsaved>"): return path # Don't munge unsaved file paths scheme = path.split(":", 1)[0] if len(scheme) == len(path): # didn't find a scheme at all; assume this is a local path return os.path.normcase(path) if len(scheme) == 1: # single-character scheme; assume this is actually a drive letter # (for a Windows-style path) return os.path.normcase(path) scheme_chars = string.ascii_letters + string.digits + "-" try: scheme = scheme.encode("ascii") except UnicodeEncodeError: # scheme has a non-ascii character; assume local path return os.path.normcase(path) if scheme.translate(None, scheme_chars): # has a non-scheme character: this is not a valid scheme # assume this is a local file path return os.path.normcase(path) if scheme != "file": return path # non-file scheme path = path[len(scheme) + 1:] return os.path.normcase(path) def get_buffer(self, request=REQUEST_DEFAULT, path=None): if request is Driver.REQUEST_DEFAULT: request = self.active_request if path is None: if not "path" in request: raise RequestFailure(message="No path given to locate buffer") path = request.path path = self.normpath(path) try: buf = self.buffers[path] except KeyError: buf = None else: if "language" in request and buf.lang != request.language: buf = None # language changed, re-scan if not buf: # Need to construct a new buffer lang = request.get("language") env = Environment(request=request, name=os.path.basename(path)) if request.get("text") is not None: # pass no content; we'll reset it later buf = self.mgr.buf_from_content("", lang, path=path, env=env) else: # read from file try: buf = self.mgr.buf_from_path(path, lang, env=env, encoding=request.get("encoding")) except OSError: # Can't read the file buf = self.mgr.buf_from_content("", lang, path=path, env=env) assert not request.path.startswith("<"), \ "Can't create an unsaved buffer with no text" if request.get("text") is not None: # overwrite the buffer contents if we have new ones buf.accessor.reset_content(request.text) buf.encoding = "utf-8" try: env = request["env"] except KeyError: pass # no environment, use current else: buf._env.update(env) #log.debug("Got buffer %r: [%s]", buf, buf.accessor.content) log.debug("Got buffer %r", buf) self.buffers[path] = buf return buf def do_abort(self, request): try: req_id = request["id"] with self.queue_cv: for item in self.queue: if item.get("req_id") == req_id: self.queue.remove(item) self.send(request=request) break else: self.abort = req_id if self.active_request and self.active_request.id == req_id: # need to wait a bit... self.send(request=request) else: self.fail(request=request, message="Request %s not found" % (req_id,)) except RequestFailure as e: self.fail(request=request, **e.kwargs) except Exception as e: log.exception(e.message) self.exception(request=request, message=e.message) def do_add_dirs(self, request): catalog_dirs = request.get("catalog-dirs", None) if catalog_dirs is not None: self.mgr.db.catalog_dirs.extend(catalog_dirs) catalog_dirs = self.mgr.db.catalog_dirs lexer_dirs = request.get("lexer-dirs", []) codeintel2.udl.UDLLexer.add_extra_lexer_dirs(lexer_dirs) module_dirs = request.get("module-dirs", []) if module_dirs: self.mgr._register_modules(module_dirs) if catalog_dirs is not None: self.mgr.db.get_catalogs_zone().catalog_dirs = catalog_dirs self.send(request=request) def do_load_extension(self, request): """Load an extension that, for example, might provide additional command handlers""" name = request.get("module-name", None) if not name: raise RequestFailure(msg="load-extension requires a module-name") path = request.get("module-path", None) names = name.split(".") if len(names) > 1: # It's inside a package - go look for the package(s). for package_name in names[:-1]: iinfo = imp.find_module(package_name, [path] if path else None) if not iinfo: raise RequestFailure(msg="load-extension could not find " "package %r for given name %r" % (package_name, name)) path = iinfo[1] name = names[-1] iinfo = imp.find_module(name, [path] if path else None) try: module = imp.load_module(name, *iinfo) finally: if iinfo and iinfo[0]: iinfo[0].close() callback = getattr(module, "registerExtension", None) if not callback: raise RequestFailure(msg="load-extension module %s should " "have a 'registerExtension' method " "taking no arguments" % (name,)) callback() self.send() # success, finally def do_quit(self, request): self.quit = True self.send(command="quit") def report_error(self, message): self.send(request=None, command="report-error", message=six.text_type(message)) def finalize(self, timeout=None): self.do_quit(None) self.mgr.finalize() def start(self): """Start reading from the socket and dump requests into the queue""" log.info("Running codeintel driver...") buf = b'' self.send(success=None) self.daemon = True threading.Thread.start(self) while not self.quit: try: ch = self.fd_in.read(1) except IOError: log.debug("Failed to read frame length, assuming connection died") self.quit = True break if not ch: log.debug("Input was closed") self.quit = True break if ch == b'{': length = int(buf, 10) try: buf = ch while len(buf) < length: data = self.fd_in.read(length - len(buf)) if not data: # nothing read, EOF raise IOError("Failed to read frame from socket") buf += data except IOError: log.debug("Failed to read frame data, assuming connection died") self.quit = True break try: data = json.loads(buf.decode('utf-8')) request = Request(data) except Exception as e: log.debug("Got codeintel request: %r" % buf) log.exception(e) self.exception(message=e.message, request=None) continue finally: buf = b'' if request.get("command") == "abort": self.do_abort(request=request) else: log.debug("queuing request %r", request) with self.queue_cv: self.queue.appendleft(request) self.queue_cv.notify() elif ch in b'0123456789': buf += ch elif ch in b' \t\r\n': pass else: raise ValueError("Invalid frame length character: %r" % ch) log.info("Codeintel driver ended") def run(self): """Evaluate and send results back""" log.info("Running codeintel eval thread...") log.debug("default supported commands: %s", ", ".join(self._default_handler.supportedCommands)) while not self.quit: with self.queue_cv: try: request = self.queue.pop() except IndexError: self.queue_cv.wait() continue log.debug("doing request %r", request) try: self.active_request = request command = request.command # First, check abort and quit; don't allow those to be overridden try: builtin = self._builtin_commands[command] except KeyError: pass else: builtin(request) continue handlers = self._command_handler_map.get(command, [])[:] if command in self._default_handler.supportedCommands: # The default handler can deal with this, put it at the end handlers.append(self._default_handler) for handler in handlers: if isinstance(handler, tuple): try: real_handler = handler[0]() except Exception as ex: log.exception("Failed to get lazy handler for %s", command) real_handler = None if real_handler is None: # Handler failed to instantiate, drop it try: self._command_handler_map[command].remove(handler) except ValueError: pass # ... shouldn't happen, but tolerate it continue for handlers in self._command_handler_map.values(): try: handlers[handlers.index(handler)] = real_handler except ValueError: pass # handler not in this list handler = real_handler if handler.canHandleRequest(request): handler.handleRequest(request, self) break else: self.fail(request=request, message="Don't know how to handle command %s" % (command,)) except RequestFailure as e: self.fail(request=request, **e.kwargs) except Exception as e: log.exception(e) e = getattr(e, 'message', repr(e)) self.exception(request=request, message=e) finally: self.active_request = None log.info("Codeintel eval thread ended") @classmethod def registerCommandHandler(cls, handlerInstance): """Register a command handler""" for command in handlerInstance.supportedCommands: cls._command_handler_map[command].append(handlerInstance) @classmethod def registerLazyCommandHandler(cls, supported_commands, constructor): """Register a lazy command handler @param supported_commands {iterable} The commands to handle; each element should be a str of the command name. @param constructor {callable} Function to be called to get the real command handler; it should take no arguments and return a command handler instance. It may return None if the command is not available; it will not be asked again. """ for command in supported_commands: cls._command_handler_map[command].append((constructor,)) @classmethod def getInstance(cls): """Get the singleton instance of the driver""" return Driver._instance
class CodeIntelTestCase(unittest.TestCase): # Subclasses can override this to have setUp pass these settings to the # codeintel.Manager. _ci_db_base_dir_ = None _ci_db_catalog_dirs_ = [] _ci_db_import_everything_langs = None _ci_env_prefs_ = None # A test case can set this to false to have the setUp/tearDown *not* # create a `self.mgr'. _ci_test_setup_mgr_ = True _ci_extra_module_dirs_ = None def setUp(self): if _xpcom_: # The tests are run outside of Komodo. If run with PyXPCOM up # parts codeintel will try to use the nsIDirectoryService and # will query dirs only provided by nsXREDirProvider -- which # isn't registered outside of Komodo (XRE_main() isn't called). # The KoTestService provides a backup. koTestSvc = components.classes["@activestate.com/koTestService;1"] \ .getService(components.interfaces.koITestService) koTestSvc.init() if self._ci_test_setup_mgr_: env = None if self._ci_env_prefs_ is not None: env = SimplePrefsEnvironment(**self._ci_env_prefs_) self.mgr = Manager( extra_module_dirs=self._ci_extra_module_dirs_, db_base_dir=self._ci_db_base_dir_ or test_db_base_dir, db_catalog_dirs=self._ci_db_catalog_dirs_, db_import_everything_langs=self._ci_db_import_everything_langs, env=env) self.mgr.upgrade() self.mgr.initialize() def tearDown(self): if self._ci_test_setup_mgr_: self.mgr.finalize() self.mgr = None def adjust_content(self, content): """A hook for subclasses to modify markedup_content before use in test cases. This is useful for sharing test cases between pure- and multi-lang uses of a given lang. """ return content def adjust_pos(self, pos): """A accompanying hook for `adjust_content' to adjust trigger pos values accordingly. """ return pos def _get_buf_and_data(self, markedup_content, lang, path=None, env=None): if path is None: # Try to ensure no accidental re-use of the same buffer name # across the whole test suite. Also try to keep the buffer # names relatively short (keeps tracebacks cleaner). name = "buf-" + md5(markedup_content).hexdigest()[:16] path = os.path.join("<Unsaved>", name) content, data = unmark_text(self.adjust_content(markedup_content)) #print banner(path) #sys.stdout.write(content) #print banner(None) buf = self.mgr.buf_from_content(content, lang=lang, path=path, env=env) return buf, data def _get_buf_and_trg(self, markedup_content, lang, path=None, implicit=True, env=None): buf, data = self._get_buf_and_data(markedup_content, lang, path, env) trg = buf.trg_from_pos(data["pos"], implicit=implicit) return buf, trg def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the CITDL expression at the current position is as expected. This uses buf.citdl_expr_under_pos() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). Note: This API is a mess right now. C.f. bug 65776. The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text(self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, length=0, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: #actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"]) trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual( actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr under pos:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if prefix_filter is not None: XXX #TODO: compare prefix_filter to given value def _assertDefnMatches(self, buf, pos, lang=None, **fields): ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) defns = buf.defns_from_trg(trg, ctlr=ctlr) if not defns: self.fail( "unexpectedly did not find a definition in %r at pos %d\n" " eval log\n%s\n" " buffer:\n%s" % (buf, pos, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) defn = defns[0] for name, value in fields.items(): try: actual_value = getattr(defn, name) except AttributeError: actual_value = None self.assertEqual( actual_value, value, "%s definition, unexpected value for field %r\n" " defn: %r\n" " expected: %r\n" " got: %r\n" " eval log\n%s\n" " buffer:\n%s" % (buf.lang, name, defn, value, actual_value, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text))) def assertDefnMatches(self, markedup_content, lang=None, **fields): if lang is None: lang = self.lang buf, data = self._get_buf_and_data(markedup_content, lang) self._assertDefnMatches(buf, data["pos"], lang, **fields) def assertDefnMatches2(self, buf, pos, lang=None, **fields): self._assertDefnMatches(buf, pos, lang, **fields) def assertNoDuplicateDefns(self, markedup_content, lang=None): if lang is None: lang = self.lang buf, data = self._get_buf_and_data(markedup_content, lang) self.assertNoDuplicateDefns2(buf, data["pos"]) def assertNoDuplicateDefns2(self, buf, pos): markedup_content = markup_text(buf.accessor.text, pos=pos) ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) actual_defns = buf.defns_from_trg(trg, ctlr=ctlr) if not actual_defns: self.fail("%s trigger resulted in no definitions when expecting " "to check for duplicate definitions:\n%s" % (buf.lang, indent(markedup_content))) count_from_defn_repr = {} for defn_repr in (repr(d) for d in actual_defns): if defn_repr not in count_from_defn_repr: count_from_defn_repr[defn_repr] = 0 count_from_defn_repr[defn_repr] += 1 defn_dupes = [(count, defn_repr) for defn_repr, count in count_from_defn_repr.items() if count > 1] self.failIf( defn_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %s' % d for d in defn_dupes)), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertTriggerMatches(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertEqual( actual_value, value, "unexpected %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) # Used when a position generates a trigger, but it's not the one specified def assertTriggerDoesNotMatch(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: # No trigger is as good as return if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertNotEqual( actual_value, value, "unexpected %s trigger '%s' value: expected not %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) def assertNoTrigger(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is not None: self.fail("unexpectedly found a %s trigger %r when didn't expect " "one, buffer:\n%s" % (lang, trg, indent(markedup_content))) def assertPrecedingTriggerMatches(self, markedup_content, lang=None, **fields): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is None: self.fail("unexpectedly did not find a preceding %s trigger, " "buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): actual_value = getattr(trg, name) self.assertEqual( actual_value, value, "unexpected preceding %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) def assertNoPrecedingTrigger(self, markedup_content, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is not None: self.fail("unexpectedly found a preceding %s trigger '%s' when " "didn't expect one, buffer:\n%s" % (lang, trg.name, indent(markedup_content))) def assertScopeLpathIs(self, markedup_content, lpath, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) buf.scan(skip_scan_time_check=True) actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"]) self.failUnlessEqual( lpath, actual_lpath, "unexpected %s scope lookup path (lpath) at the given position\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (self.lang, lpath, actual_lpath, indent(markedup_content))) def assertCompletionsAre2(self, buf, pos, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang markedup_content = markup_text(buf.accessor.text, pos=pos) trg = buf.trg_from_pos(pos, implicit=implicit) self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit) def assertCompletionsAre(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit) def _assertCompletionsAre(self, markedup_content, buf, trg, completions, lang, implicit): if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to be %r:\n%s" % (lang, completions, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) self.assertEqual( completions, actual_completions, "unexpected %s completions at the given position\n" " expected: %r\n" " got: %r\n" " extra: %r\n" " missing: %r\n" " eval log\n%s\n" " buffer:\n%s" % (lang, completions, actual_completions, list(set(actual_completions or []).difference(completions or [])), list(set(completions or []).difference(actual_completions or [])), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertNoDuplicateCompletions(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected there to be completions:\n%s" % (lang, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) if actual_completions is None: self.fail("%s trigger resulted in no completions when expecting " "to check for duplicate completions:\n%s" % (lang, indent(markedup_content))) count_from_cpln = {} for cpln in actual_completions: if cpln not in count_from_cpln: count_from_cpln[cpln] = 0 count_from_cpln[cpln] += 1 cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items() if count > 1] self.failIf( cpln_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %r' % d for d in cpln_dupes)), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def _assertCompletionsInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) missing_completions = [ c for c in completions if c not in (actual_completions or []) ] self.failIf( missing_completions, "%s completions at the given position did not " "include all expected values\n" " missing: %r\n" " expected all of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, missing_completions, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertCompletionsInclude(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to include %r:\n%s" % (lang, completions, indent(markedup_content))) self._assertCompletionsInclude(buf, trg, completions) def assertCompletionsInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to include %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsInclude(buf, trg, completions) def _assertCompletionsDoNotInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) completions_that_shouldnt_be_there = [ c for c in (actual_completions or []) if c in completions ] self.failIf( completions_that_shouldnt_be_there, "%s completions at the given position included " "some unexpected values\n" " shouldn't have had these: %r\n" " expected none of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertCompletionsDoNotInclude(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions) def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsDoNotInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions) def assertCalltipIs2(self, buf, pos, calltip, implicit=True): """A variant of assertCalltipIs() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) markedup_content = markup_text(buf.accessor.text, pos=trg.pos) self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang) def assertCalltipIs(self, markedup_content, calltip, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCalltipIs(buf, trg, markedup_content, calltip, lang) def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang): if trg is None: self.fail("given position is not a %s trigger point, " "expected the following calltip:\n" " calltip:\n%s\n" " buffer:\n%s" % (lang, indent(calltip), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertEqual( calltip, actual_calltip, "unexpected %s calltip at the given position\n" " expected:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(calltip and calltip or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertCalltipMatches(self, markedup_content, calltip, lang=None, implicit=True, env=None, flags=0): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang, flags) def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags): if trg is None: self.fail("given position is not a %s trigger point, " "expected the calltip to match the following:\n" " exression:\n%s\n" " buffer:\n%s" % (lang, indent(expr), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertNotEquals( re.search(expr, actual_calltip, flags), None, "unexpected %s calltip at the given position\n" " expression:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(expr and expr or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content))) def assertCurrCalltipArgRange(self, markedup_content, calltip, expected_range, lang=None, implicit=True): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) pos = data["pos"] buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit) actual_range = buf.curr_calltip_arg_range(trg.pos, calltip, curr_pos=data["pos"]) self.assertEqual( actual_range, expected_range, "unexpected current calltip arg range\n" " expected: %s\n" " got: %s\n" " calltip:\n%s\n" " buffer:\n%s" % (expected_range, actual_range, indent(calltip), indent(markedup_content))) #def assertCompletionRaises(self, markedup_content, exception, lang=None, # **kwargs): # """Assert that the given completion raises the given exception. # # You may also specify either of the "exc_args" or "exc_pattern" # keyword args to match the exception's "args" attribute or match # the stringified exception against a regex pattern. # # c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970 # """ # if lang is None: # lang = self.lang # if "exc_args" in kwargs: # exc_args = kwargs["exc_args"] # del kwargs["exc_args"] # else: # exc_args = None # if "exc_pattern" in kwargs: # exc_pattern = kwargs["exc_pattern"] # del kwargs["exc_pattern"] # else: # exc_pattern = None # # callsig = "the given %s completion" % lang # try: # buf, trg = self._get_buf_and_trg(markedup_content, lang) # if trg is None: # self.fail("given position is not a %s trigger point, " # "no completion can be done to see if it raises" # % self.lang) # callsig = "completion at %s" % trg # cplns = buf.cplns_from_trg(trg) # except exception, exc: # if exc_args is not None: # self.failIf(exc.args != exc_args, # "%s raised %s with unexpected args: "\ # "expected=%r, actual=%r"\ # % (callsig, exc.__class__, exc_args, exc.args)) # if exc_pattern is not None: # self.failUnless(exc_pattern.search(str(exc)), # "%s raised %s, but the exception "\ # "does not match '%s': %r"\ # % (callsig, exc.__class__, exc_pattern.pattern, # str(exc))) # except: # exc_info = sys.exc_info() # print exc_info # self.fail("%s raised an unexpected exception type: "\ # "expected=%s, actual=%s"\ # % (callsig, exception, exc_info[0])) # else: # self.fail("%s did not raise %s" % (callsig, exception)) def assertEvalError(self, markedup_content, log_pattern, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "no completion can be done to see if errors" % self.lang) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test class TestEvalController(EvalController): """A completion controller that captures all eval logging.""" def __init__(self): EvalController.__init__(self) self.log = [] def debug(self, msg, *args): self.log.append(("debug", msg % args)) def info(self, msg, *args): self.log.append(("info", msg % args)) def warn(self, msg, *args): self.log.append(("warn", msg % args)) def error(self, msg, *args): self.log.append(("error", msg % args)) ctlr = TestEvalController() buf.async_eval_at_trg(trg, ctlr=ctlr) ctlr.wait() if not ctlr.is_done(): self.fail("evaluation is not 'done': didn't expect that") if trg.form == TRG_FORM_CPLN and ctlr.cplns: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) if log_pattern: #pprint(ctlr.log) matching_logs = [(level, msg) for level, msg in ctlr.log if log_pattern.search(msg)] self.failUnless( matching_logs, "the given completion failed but no logs matched the given pattern:\n" " log_pattern: /%s/\n" " log messages:\n%s\n" " buffer:\n%s" % (log_pattern.pattern, indent('\n'.join(['%s: %s' % lg for lg in ctlr.log ])), indent(markedup_content))) #pprint(matching_logs) def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the preceding CITDL expression at the current position is as expected. This uses buf.citdl_expr_from_trg() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text(self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": # Bit of a hack to fake the trigger length. if content[data["pos"] - 1] in ('>', ':'): # '->' or '::' triggers length = 2 else: length = 1 trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, length=length, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual( actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if lang == "Perl" and prefix_filter is not None: self.assertEqual( actual_prefix_filter, prefix_filter, "unexpected actual %s variable prefix filter " "preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter, indent(markedup_content))) def _unmark_lex_text(self, markedup_text): from SilverCity import ScintillaConstants tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)') tokens = [] text = '' while markedup_text: match = tokenizer.search(markedup_text) if match: text += markedup_text[:match.start()] token = { 'end_index': len(text) + len(match.group(3)) - 1, 'start_index': len(text), 'style': getattr(ScintillaConstants, match.group(2)), 'text': match.group(3), #'end_column': ???, #'end_line': ???, #'start_column': ???, #'start_line': ???, } tokens.append(token) text += match.group(3) markedup_text = markedup_text[match.end():] else: text += markedup_text markedup_text = '' return text, tokens def assertLex(self, markedup_content, lang=None): """Lex the given content and assert that the lexed tokens are as expected. What is "expected" is given via pseudo-xml markup like this: fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear This example expects that "wuzza" will be a token with style SCE_UDL_SSL_COMMENTBLOCK. """ from codeintel2.accessor import SilverCityAccessor if lang is None: lang = self.lang content, tokens = self._unmark_lex_text(markedup_content) # Do lexing of this content via the codeintel Buffer's, because # they already handle all the SilverCity lexer hookup. path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) assert isinstance(buf.accessor, SilverCityAccessor) actual_tokens = buf.accessor.tokens # cheating for actual_token in actual_tokens: # There are a few SilverCity token dict keys that we # don't bother checking. del actual_token["end_column"] del actual_token["end_line"] del actual_token["start_column"] del actual_token["start_line"] unmatched_tokens = [t for t in tokens if t not in actual_tokens] if unmatched_tokens: self.fail("not all expected %s lex tokens were found in the " "actual lexer output:\n" " buffer:\n%s\n" " actual lexer tokens:\n%s\n" " unmatched tokens:\n%s\n" % (lang, indent(content), indent(pformat(actual_tokens)), indent(pformat(unmatched_tokens))))
def do_play(self, subcmd, opts): """Run my current play/dev code. ${cmd_usage} ${cmd_option_list} """ import pprint import random import ciElementTree as ET from codeintel2.manager import Manager from codeintel2.tree import pretty_tree_from_tree from codeintel2.common import LogEvalController, Error from codeintel2.util import tree_from_cix, dedent, unmark_text, banner from ci2 import _escaped_text_from_text if False: lang = "CSS" markedup_content = dedent(""" /* http://www.w3.org/TR/REC-CSS2/fonts.html#propdef-font-weight */ h1 { border: 1px solid black; font-weight /* hi */: <|> !important } """) content, data = unmark_text(markedup_content) pos = data["pos"] mgr = Manager() # mgr.upgrade() # Don't need it for just CSS usage. mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path="play.css") trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) completions = buf.cplns_from_trg(trg) print("COMPLETIONS: %r" % completions) finally: mgr.finalize() elif False: lang = "Python" path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent(""" import sys, os class Foo: def bar(self): pass sys.<|>path # should have path in completion list f = Foo() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data["pos"] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) print(buf.cix) trg = buf.trg_from_pos(pos) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(self.log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(2) # XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint.pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize() elif False: lang = "Ruby" path = os.path.join("<Unsaved>", "rand%d.py" % random.randint(0, 100)) markedup_content = dedent("""\ r<1>equire 'net/http' include Net req = HTTPRequest.new req.<2>get() """) content, data = unmark_text(markedup_content) print(banner(path)) print(_escaped_text_from_text(content, "whitespace")) pos = data[1] mgr = Manager() mgr.upgrade() mgr.initialize() try: buf = mgr.buf_from_content(content, lang=lang, path=path) print(banner("cix", '-')) cix = buf.cix print(ET.tostring(pretty_tree_from_tree(tree_from_cix(cix)))) trg = buf.trg_from_pos(pos, implicit=False) if trg is None: raise Error("unexpected trigger: %r" % trg) print(banner("completions", '-')) ctlr = LogEvalController(self.log) buf.async_eval_at_trg(trg, ctlr) ctlr.wait(30) # XXX if not ctlr.is_done(): ctlr.abort() raise Error("XXX async eval timed out") pprint.pprint(ctlr.cplns) print(banner(None)) finally: mgr.finalize()
class CodeIntelTestCase(unittest.TestCase): # Subclasses can override this to have setUp pass these settings to the # codeintel.Manager. _ci_db_base_dir_ = None _ci_db_catalog_dirs_ = [] _ci_db_import_everything_langs = None _ci_env_prefs_ = None # A test case can set this to false to have the setUp/tearDown *not* # create a `self.mgr'. _ci_test_setup_mgr_ = True _ci_extra_module_dirs_ = None def setUp(self): if _xpcom_: # The tests are run outside of Komodo. If run with PyXPCOM up # parts codeintel will try to use the nsIDirectoryService and # will query dirs only provided by nsXREDirProvider -- which # isn't registered outside of Komodo (XRE_main() isn't called). # The KoTestService provides a backup. koTestSvc = components.classes["@activestate.com/koTestService;1"] \ .getService(components.interfaces.koITestService) koTestSvc.init() if self._ci_test_setup_mgr_: env = None if self._ci_env_prefs_ is not None: env = SimplePrefsEnvironment(**self._ci_env_prefs_) self.mgr = Manager( extra_module_dirs=self._ci_extra_module_dirs_, db_base_dir=self._ci_db_base_dir_ or test_db_base_dir, db_catalog_dirs=self._ci_db_catalog_dirs_, db_import_everything_langs=self._ci_db_import_everything_langs, env=env) self.mgr.upgrade() self.mgr.initialize() def tearDown(self): if self._ci_test_setup_mgr_: self.mgr.finalize() self.mgr = None def adjust_content(self, content): """A hook for subclasses to modify markedup_content before use in test cases. This is useful for sharing test cases between pure- and multi-lang uses of a given lang. """ return content def adjust_pos(self, pos): """A accompanying hook for `adjust_content' to adjust trigger pos values accordingly. """ return pos def _get_buf_and_data(self, markedup_content, lang, path=None, env=None): if path is None: # Try to ensure no accidental re-use of the same buffer name # across the whole test suite. Also try to keep the buffer # names relatively short (keeps tracebacks cleaner). name = "buf-" + md5(markedup_content).hexdigest()[:16] path = os.path.join("<Unsaved>", name) content, data = unmark_text(self.adjust_content(markedup_content)) #print banner(path) #sys.stdout.write(content) #print banner(None) buf = self.mgr.buf_from_content(content, lang=lang, path=path, env=env) return buf, data def _get_buf_and_trg(self, markedup_content, lang, path=None, implicit=True, env=None): buf, data = self._get_buf_and_data(markedup_content, lang, path, env) trg = buf.trg_from_pos(data["pos"], implicit=implicit) return buf, trg def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the CITDL expression at the current position is as expected. This uses buf.citdl_expr_under_pos() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). Note: This API is a mess right now. C.f. bug 65776. The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text( self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, length=0, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: #actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"]) trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual(actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr under pos:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if prefix_filter is not None: XXX #TODO: compare prefix_filter to given value def _assertDefnMatches(self, buf, pos, lang=None, **fields): ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) defns = buf.defns_from_trg(trg, ctlr=ctlr) if not defns: self.fail("unexpectedly did not find a definition in %r at pos %d\n" " eval log\n%s\n" " buffer:\n%s" % (buf, pos, indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(buf.accessor.text))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) defn = defns[0] for name, value in fields.items(): try: actual_value = getattr(defn, name) except AttributeError: actual_value = None self.assertEqual(actual_value, value, "%s definition, unexpected value for field %r\n" " defn: %r\n" " expected: %r\n" " got: %r\n" " eval log\n%s\n" " buffer:\n%s" % (buf.lang, name, defn, value, actual_value, indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(buf.accessor.text))) def assertDefnMatches(self, markedup_content, lang=None, **fields): if lang is None: lang = self.lang buf, data = self._get_buf_and_data(markedup_content, lang) self._assertDefnMatches(buf, data["pos"], lang, **fields) def assertDefnMatches2(self, buf, pos, lang=None, **fields): self._assertDefnMatches(buf, pos, lang, **fields) def assertNoDuplicateDefns(self, markedup_content, lang=None): if lang is None: lang = self.lang buf, data = self._get_buf_and_data(markedup_content, lang) self.assertNoDuplicateDefns2(buf, data["pos"]) def assertNoDuplicateDefns2(self, buf, pos): markedup_content = markup_text(buf.accessor.text, pos=pos) ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) actual_defns = buf.defns_from_trg(trg, ctlr=ctlr) if not actual_defns: self.fail("%s trigger resulted in no definitions when expecting " "to check for duplicate definitions:\n%s" % (buf.lang, indent(markedup_content))) count_from_defn_repr = {} for defn_repr in (repr(d) for d in actual_defns): if defn_repr not in count_from_defn_repr: count_from_defn_repr[defn_repr] = 0 count_from_defn_repr[defn_repr] += 1 defn_dupes = [(count, defn_repr) for defn_repr, count in count_from_defn_repr.items() if count > 1] self.failIf(defn_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %s' % d for d in defn_dupes)), indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertTriggerMatches(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertEqual(actual_value, value, "unexpected %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) # Used when a position generates a trigger, but it's not the one specified def assertTriggerDoesNotMatch(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: # No trigger is as good as return if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertNotEqual(actual_value, value, "unexpected %s trigger '%s' value: expected not %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) def assertNoTrigger(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is not None: self.fail("unexpectedly found a %s trigger %r when didn't expect " "one, buffer:\n%s" % (lang, trg, indent(markedup_content))) def assertPrecedingTriggerMatches(self, markedup_content, lang=None, **fields): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is None: self.fail("unexpectedly did not find a preceding %s trigger, " "buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): actual_value = getattr(trg, name) self.assertEqual(actual_value, value, "unexpected preceding %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content))) def assertNoPrecedingTrigger(self, markedup_content, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is not None: self.fail("unexpectedly found a preceding %s trigger '%s' when " "didn't expect one, buffer:\n%s" % (lang, trg.name, indent(markedup_content))) def assertScopeLpathIs(self, markedup_content, lpath, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) buf.scan(skip_scan_time_check=True) actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"]) self.failUnlessEqual(lpath, actual_lpath, "unexpected %s scope lookup path (lpath) at the given position\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (self.lang, lpath, actual_lpath, indent(markedup_content))) def assertCompletionsAre2(self, buf, pos, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang markedup_content = markup_text(buf.accessor.text, pos=pos) trg = buf.trg_from_pos(pos, implicit=implicit) self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit) def assertCompletionsAre(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCompletionsAre(markedup_content, buf, trg, completions, lang, implicit) def _assertCompletionsAre(self, markedup_content, buf, trg, completions, lang, implicit): if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to be %r:\n%s" % (lang, completions, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) self.assertEqual(completions, actual_completions, "unexpected %s completions at the given position\n" " expected: %r\n" " got: %r\n" " extra: %r\n" " missing: %r\n" " eval log\n%s\n" " buffer:\n%s" % (lang, completions, actual_completions, list(set(actual_completions or []).difference(completions or [])), list(set(completions or []).difference(actual_completions or [])), indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertNoDuplicateCompletions(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected there to be completions:\n%s" % (lang, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) if actual_completions is None: self.fail("%s trigger resulted in no completions when expecting " "to check for duplicate completions:\n%s" % (lang, indent(markedup_content))) count_from_cpln = {} for cpln in actual_completions: if cpln not in count_from_cpln: count_from_cpln[cpln] = 0 count_from_cpln[cpln] += 1 cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items() if count > 1] self.failIf(cpln_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %r' % d for d in cpln_dupes)), indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def _assertCompletionsInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) missing_completions = [c for c in completions if c not in (actual_completions or [])] self.failIf(missing_completions, "%s completions at the given position did not " "include all expected values\n" " missing: %r\n" " expected all of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, missing_completions, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertCompletionsInclude(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to include %r:\n%s" % (lang, completions, indent(markedup_content))) self._assertCompletionsInclude(buf, trg, completions) def assertCompletionsInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to include %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsInclude(buf, trg, completions) def _assertCompletionsDoNotInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) completions_that_shouldnt_be_there = [ c for c in (actual_completions or []) if c in completions ] self.failIf(completions_that_shouldnt_be_there, "%s completions at the given position included " "some unexpected values\n" " shouldn't have had these: %r\n" " expected none of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertCompletionsDoNotInclude(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions) def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsDoNotInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions) def assertCalltipIs2(self, buf, pos, calltip, implicit=True): """A variant of assertCalltipIs() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) markedup_content = markup_text(buf.accessor.text, pos=trg.pos) self._assertCalltipIs(buf, trg, markedup_content, calltip, self.lang) def assertCalltipIs(self, markedup_content, calltip, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCalltipIs(buf, trg, markedup_content, calltip, lang) def _assertCalltipIs(self, buf, trg, markedup_content, calltip, lang): if trg is None: self.fail("given position is not a %s trigger point, " "expected the following calltip:\n" " calltip:\n%s\n" " buffer:\n%s" % (lang, indent(calltip), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertEqual(calltip, actual_calltip, "unexpected %s calltip at the given position\n" " expected:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(calltip and calltip or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertCalltipMatches(self, markedup_content, calltip, lang=None, implicit=True, env=None, flags=0): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) self._assertCalltipMatches(buf, trg, markedup_content, calltip, lang, flags) def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags): if trg is None: self.fail("given position is not a %s trigger point, " "expected the calltip to match the following:\n" " exression:\n%s\n" " buffer:\n%s" % (lang, indent(expr), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertNotEquals(re.search(expr, actual_calltip, flags), None, "unexpected %s calltip at the given position\n" " expression:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(expr and expr or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl,m) for lvl,m in ctlr.log)), indent(markedup_content))) def assertCurrCalltipArgRange(self, markedup_content, calltip, expected_range, lang=None, implicit=True): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) pos = data["pos"] buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit) actual_range = buf.curr_calltip_arg_range(trg.pos, calltip, curr_pos=data["pos"]) self.assertEqual(actual_range, expected_range, "unexpected current calltip arg range\n" " expected: %s\n" " got: %s\n" " calltip:\n%s\n" " buffer:\n%s" % (expected_range, actual_range, indent(calltip), indent(markedup_content))) #def assertCompletionRaises(self, markedup_content, exception, lang=None, # **kwargs): # """Assert that the given completion raises the given exception. # # You may also specify either of the "exc_args" or "exc_pattern" # keyword args to match the exception's "args" attribute or match # the stringified exception against a regex pattern. # # c.f. http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/307970 # """ # if lang is None: # lang = self.lang # if "exc_args" in kwargs: # exc_args = kwargs["exc_args"] # del kwargs["exc_args"] # else: # exc_args = None # if "exc_pattern" in kwargs: # exc_pattern = kwargs["exc_pattern"] # del kwargs["exc_pattern"] # else: # exc_pattern = None # # callsig = "the given %s completion" % lang # try: # buf, trg = self._get_buf_and_trg(markedup_content, lang) # if trg is None: # self.fail("given position is not a %s trigger point, " # "no completion can be done to see if it raises" # % self.lang) # callsig = "completion at %s" % trg # cplns = buf.cplns_from_trg(trg) # except exception, exc: # if exc_args is not None: # self.failIf(exc.args != exc_args, # "%s raised %s with unexpected args: "\ # "expected=%r, actual=%r"\ # % (callsig, exc.__class__, exc_args, exc.args)) # if exc_pattern is not None: # self.failUnless(exc_pattern.search(str(exc)), # "%s raised %s, but the exception "\ # "does not match '%s': %r"\ # % (callsig, exc.__class__, exc_pattern.pattern, # str(exc))) # except: # exc_info = sys.exc_info() # print exc_info # self.fail("%s raised an unexpected exception type: "\ # "expected=%s, actual=%s"\ # % (callsig, exception, exc_info[0])) # else: # self.fail("%s did not raise %s" % (callsig, exception)) def assertEvalError(self, markedup_content, log_pattern, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "no completion can be done to see if errors" % self.lang) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test class TestEvalController(EvalController): """A completion controller that captures all eval logging.""" def __init__(self): EvalController.__init__(self) self.log = [] def debug(self, msg, *args): self.log.append(("debug", msg % args)) def info(self, msg, *args): self.log.append(("info", msg % args)) def warn(self, msg, *args): self.log.append(("warn", msg % args)) def error(self, msg, *args): self.log.append(("error", msg % args)) ctlr = TestEvalController() buf.async_eval_at_trg(trg, ctlr=ctlr) ctlr.wait() if not ctlr.is_done(): self.fail("evaluation is not 'done': didn't expect that") if trg.form == TRG_FORM_CPLN and ctlr.cplns: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) if log_pattern: #pprint(ctlr.log) matching_logs = [(level, msg) for level, msg in ctlr.log if log_pattern.search(msg)] self.failUnless(matching_logs, "the given completion failed but no logs matched the given pattern:\n" " log_pattern: /%s/\n" " log messages:\n%s\n" " buffer:\n%s" % (log_pattern.pattern, indent('\n'.join(['%s: %s' % lg for lg in ctlr.log])), indent(markedup_content))) #pprint(matching_logs) def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the preceding CITDL expression at the current position is as expected. This uses buf.citdl_expr_from_trg() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text( self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": # Bit of a hack to fake the trigger length. if content[data["pos"]-1] in ('>', ':'): # '->' or '::' triggers length = 2 else: length = 1 trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, length=length, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual(actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if lang == "Perl" and prefix_filter is not None: self.assertEqual(actual_prefix_filter, prefix_filter, "unexpected actual %s variable prefix filter " "preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter, indent(markedup_content))) def _unmark_lex_text(self, markedup_text): from SilverCity import ScintillaConstants tokenizer = re.compile(r'(<(SCE_\w+)>(.*?)</\2>)') tokens = [] text = '' while markedup_text: match = tokenizer.search(markedup_text) if match: text += markedup_text[:match.start()] token = { 'end_index': len(text) + len(match.group(3)) - 1, 'start_index': len(text), 'style': getattr(ScintillaConstants, match.group(2)), 'text': match.group(3), #'end_column': ???, #'end_line': ???, #'start_column': ???, #'start_line': ???, } tokens.append(token) text += match.group(3) markedup_text = markedup_text[match.end():] else: text += markedup_text markedup_text = '' return text, tokens def assertLex(self, markedup_content, lang=None): """Lex the given content and assert that the lexed tokens are as expected. What is "expected" is given via pseudo-xml markup like this: fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear This example expects that "wuzza" will be a token with style SCE_UDL_SSL_COMMENTBLOCK. """ from codeintel2.accessor import SilverCityAccessor if lang is None: lang = self.lang content, tokens = self._unmark_lex_text(markedup_content) # Do lexing of this content via the codeintel Buffer's, because # they already handle all the SilverCity lexer hookup. path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) assert isinstance(buf.accessor, SilverCityAccessor) actual_tokens = buf.accessor.tokens # cheating for actual_token in actual_tokens: # There are a few SilverCity token dict keys that we # don't bother checking. del actual_token["end_column"] del actual_token["end_line"] del actual_token["start_column"] del actual_token["start_line"] unmatched_tokens = [t for t in tokens if t not in actual_tokens] if unmatched_tokens: self.fail("not all expected %s lex tokens were found in the " "actual lexer output:\n" " buffer:\n%s\n" " actual lexer tokens:\n%s\n" " unmatched tokens:\n%s\n" % (lang, indent(content), indent(pformat(actual_tokens)), indent(pformat(unmatched_tokens))))