def _assertCompletionsAre(self, markedup_content, buf, trg, completions, lang, implicit): if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to be %r:\n%s" % (lang, completions, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) self.assertEqual(completions, actual_completions, "unexpected %s completions at the given position\n" " expected: %r\n" " got: %r\n" " extra: %r\n" " missing: %r\n" " eval log\n%s\n" " buffer:\n%s" % (lang, completions, actual_completions, list(set(actual_completions or []).difference( completions or [])), list(set(completions or []).difference( actual_completions or [])), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def assertNoDuplicateDefns2(self, buf, pos): markedup_content = markup_text(buf.accessor.text, pos=pos) ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) actual_defns = buf.defns_from_trg(trg, ctlr=ctlr) if not actual_defns: self.fail("%s trigger resulted in no definitions when expecting " "to check for duplicate definitions:\n%s" % (buf.lang, indent(markedup_content))) count_from_defn_repr = {} for defn_repr in (repr(d) for d in actual_defns): if defn_repr not in count_from_defn_repr: count_from_defn_repr[defn_repr] = 0 count_from_defn_repr[defn_repr] += 1 defn_dupes = [(count, defn_repr) for defn_repr, count in count_from_defn_repr.items() if count > 1] self.failIf(defn_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %s' % d for d in defn_dupes)), indent('\n'.join('%5s: %s' % ( lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def assertNoDuplicateCompletions(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected there to be completions:\n%s" % (lang, indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) if actual_completions is None: self.fail("%s trigger resulted in no completions when expecting " "to check for duplicate completions:\n%s" % (lang, indent(markedup_content))) count_from_cpln = {} for cpln in actual_completions: if cpln not in count_from_cpln: count_from_cpln[cpln] = 0 count_from_cpln[cpln] += 1 cpln_dupes = [(count, cpln) for cpln, count in count_from_cpln.items() if count > 1] self.failIf(cpln_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %r' % d for d in cpln_dupes)), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def _assertDefnMatches(self, buf, pos, lang=None, **fields): ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) defns = buf.defns_from_trg(trg, ctlr=ctlr) if not defns: self.fail( "unexpectedly did not find a definition in %r at pos %d\n" " eval log\n%s\n" " buffer:\n%s" % (buf, pos, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) defn = defns[0] for name, value in fields.items(): try: actual_value = getattr(defn, name) except AttributeError: actual_value = None self.assertEqual( actual_value, value, "%s definition, unexpected value for field %r\n" " defn: %r\n" " expected: %r\n" " got: %r\n" " eval log\n%s\n" " buffer:\n%s" % (buf.lang, name, defn, value, actual_value, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text)))
def _assertDefnMatches(self, buf, pos, lang=None, **fields): ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) defns = buf.defns_from_trg(trg, ctlr=ctlr) if not defns: self.fail("unexpectedly did not find a definition in %r at pos %d\n" " eval log\n%s\n" " buffer:\n%s" % (buf, pos, indent('\n'.join('%5s: %s' % ( lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) defn = defns[0] for name, value in fields.items(): try: actual_value = getattr(defn, name) except AttributeError: actual_value = None self.assertEqual(actual_value, value, "%s definition, unexpected value for field %r\n" " defn: %r\n" " expected: %r\n" " got: %r\n" " eval log\n%s\n" " buffer:\n%s" % (buf.lang, name, defn, value, actual_value, indent('\n'.join('%5s: %s' % ( lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text)))
def assertDefnIncludes(self, buf, pos, lang=None, **fields): """Check that a definition is found within one of the results for a given position. Note that this is not very useful; we normally only use the first definition. It can however be used to ensure we don't regress things before they get fixed correctly. """ ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) defns = buf.defns_from_trg(trg, ctlr=ctlr) if not defns: self.fail("unexpectedly did not find a definition in %r at pos %d\n" " eval log\n%s\n" " buffer:\n%s" % (buf, pos, indent('\n'.join('%5s: %s' % ( lvl, m) for lvl, m in ctlr.log)), indent(buf.accessor.text))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) # Copy defns over to filter for keys we care about filtered_defns = [] for defn in defns: filtered_defn = dict((key, getattr(defn, key, None)) for key in fields.keys()) filtered_defns.append(filtered_defn) self.assertIn(fields, filtered_defns)
def assertNoDuplicateDefns2(self, buf, pos): markedup_content = markup_text(buf.accessor.text, pos=pos) ctlr = _CaptureEvalController() trg = buf.defn_trg_from_pos(pos) actual_defns = buf.defns_from_trg(trg, ctlr=ctlr) if not actual_defns: self.fail("%s trigger resulted in no definitions when expecting " "to check for duplicate definitions:\n%s" % (buf.lang, indent(markedup_content))) count_from_defn_repr = {} for defn_repr in (repr(d) for d in actual_defns): if defn_repr not in count_from_defn_repr: count_from_defn_repr[defn_repr] = 0 count_from_defn_repr[defn_repr] += 1 defn_dupes = [(count, defn_repr) for defn_repr, count in count_from_defn_repr.items() if count > 1] self.failIf( defn_dupes, "unexpectedly got duplicate completions at the given position\n" " duplicates:\n%s\n" " eval log\n%s\n" " buffer:\n%s" % (indent('\n'.join('%d of %s' % d for d in defn_dupes)), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def assertTriggerMatches(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertEqual( actual_value, value, "unexpected %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content)))
def assertEvalError(self, markedup_content, log_pattern, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "no completion can be done to see if errors" % self.lang) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test class TestEvalController(EvalController): """A completion controller that captures all eval logging.""" def __init__(self): EvalController.__init__(self) self.log = [] def debug(self, msg, *args): self.log.append(("debug", msg % args)) def info(self, msg, *args): self.log.append(("info", msg % args)) def warn(self, msg, *args): self.log.append(("warn", msg % args)) def error(self, msg, *args): self.log.append(("error", msg % args)) ctlr = TestEvalController() buf.async_eval_at_trg(trg, ctlr=ctlr) ctlr.wait() if not ctlr.is_done(): self.fail("evaluation is not 'done': didn't expect that") if trg.form == TRG_FORM_CPLN and ctlr.cplns: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) elif trg.form == TRG_FORM_CALLTIP and ctlr.calltips: self.fail("evalution had results: didn't expect that: %r" % ctlr.cplns) if log_pattern: # pprint(ctlr.log) matching_logs = [(level, msg) for level, msg in ctlr.log if log_pattern.search(msg)] self.failUnless(matching_logs, "the given completion failed but no logs matched the given pattern:\n" " log_pattern: /%s/\n" " log messages:\n%s\n" " buffer:\n%s" % (log_pattern.pattern, indent('\n'.join(['%s: %s' % lg for lg in ctlr.log])), indent(markedup_content)))
def assertCITDLExprIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the preceding CITDL expression at the current position is as expected. This uses buf.citdl_expr_from_trg() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text( self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": # Bit of a hack to fake the trigger length. if content[data["pos"]-1] in ('>', ':'): # '->' or '::' triggers length = 2 else: length = 1 trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, length=length, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: trg = Trigger(lang, TRG_FORM_CPLN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual(actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if lang == "Perl" and prefix_filter is not None: self.assertEqual(actual_prefix_filter, prefix_filter, "unexpected actual %s variable prefix filter " "preceding trigger:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, prefix_filter, actual_prefix_filter, indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions, implicit=True): """Check that the given completions are NOT found @param markedup_content Content with a <|> marking @param completions List of expected completions; each item is a tuple of (type, completion string) @param implicit Whether the trigger should be implicit """ actual_completions = self._doEval(markedup_content, implicit=implicit).completions extra_completions = set() for typ, cpln in completions: if UIHandler.AutoCompleteInfo(cpln, typ) in actual_completions: extra_completions.add((typ, cpln)) self.failIf(extra_completions and True, "%s completions at the given position included " "some unexpected values\n" " shouldn't have had these: %r\n" " expected none of: %r\n" " got: %r\n" " buffer:\n%s" % (self.language, list(extra_completions), completions, [(cpln.type, cpln.completion) for cpln in actual_completions], indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions, implicit=True): """Check that the given completions are NOT found @param markedup_content Content with a <|> marking @param completions List of expected completions; each item is a tuple of (type, completion string) @param implicit Whether the trigger should be implicit """ actual_completions = self._doEval(markedup_content, implicit=implicit).completions extra_completions = set() for typ, cpln in completions: if UIHandler.AutoCompleteInfo(cpln, typ) in actual_completions: extra_completions.add((typ, cpln)) self.failIf( extra_completions and True, "%s completions at the given position included " "some unexpected values\n" " shouldn't have had these: %r\n" " expected none of: %r\n" " got: %r\n" " buffer:\n%s" % (self.language, list(extra_completions), completions, [ (cpln.type, cpln.completion) for cpln in actual_completions ], indent(markedup_content)))
def assertCalltipIs(self, markedup_content, calltip, implicit=True): """Check that the calltip is as expected @param markedup_content Content with a <|> marking @param calltip The calltip string @param implicit Whether the trigger should be implicit """ handler = self._doEval(markedup_content, implicit=implicit) actual_calltip = getattr(handler, "calltip", None) self.assertEqual(calltip, actual_calltip, "unexpected calltip at the given position\n" " expected:\n%s\n" " got:\n%s\n" " buffer:\n%s" % (indent(calltip if calltip else "(none)"), indent(actual_calltip if actual_calltip else "(none)"), indent(markedup_content)))
def assertCompletionsInclude(self, markedup_content, completions, implicit=True): """Check that the given completions are found @param markedup_content Content with a <|> marking @param completions List of expected completions; each item is a tuple of (type, completion string) @param implicit Whether the trigger should be implicit """ actual_completions = self._doEval(markedup_content, implicit=implicit).completions missing_completions = set() for typ, cpln in completions: if UIHandler.AutoCompleteInfo(cpln, typ) not in actual_completions: missing_completions.add((typ, cpln)) self.failIf(missing_completions and True, "%s completions at the given position did not " "include all expected values\n" " missing: %r\n" " expected all of: %r\n" " got: %r\n" " buffer:\n%s" % (self.language, list(missing_completions), completions, [(cpln.type, cpln.completion) for cpln in actual_completions], indent(markedup_content)))
def assertCompletionsInclude(self, markedup_content, completions, implicit=True): """Check that the given completions are found @param markedup_content Content with a <|> marking @param completions List of expected completions; each item is a tuple of (type, completion string) @param implicit Whether the trigger should be implicit """ actual_completions = self._doEval(markedup_content, implicit=implicit).completions missing_completions = set() for typ, cpln in completions: if UIHandler.AutoCompleteInfo(cpln, typ) not in actual_completions: missing_completions.add((typ, cpln)) self.failIf( missing_completions and True, "%s completions at the given position did not " "include all expected values\n" " missing: %r\n" " expected all of: %r\n" " got: %r\n" " buffer:\n%s" % (self.language, list(missing_completions), completions, [ (cpln.type, cpln.completion) for cpln in actual_completions ], indent(markedup_content)))
def assertCalltipIs(self, markedup_content, calltip, implicit=True): """Check that the calltip is as expected @param markedup_content Content with a <|> marking @param calltip The calltip string @param implicit Whether the trigger should be implicit """ handler = self._doEval(markedup_content, implicit=implicit) actual_calltip = getattr(handler, "calltip", None) self.assertEqual( calltip, actual_calltip, "unexpected calltip at the given position\n" " expected:\n%s\n" " got:\n%s\n" " buffer:\n%s" % (indent(calltip if calltip else "(none)"), indent(actual_calltip if actual_calltip else "(none)"), indent(markedup_content)))
def assertNoTrigger(self, markedup_content, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is not None: self.fail("unexpectedly found a %s trigger %r when didn't expect " "one, buffer:\n%s" % (lang, trg, indent(markedup_content)))
def assertCITDLExprUnderPosIs(self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the CITDL expression at the current position is as expected. This uses buf.citdl_expr_under_pos() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). Note: This API is a mess right now. C.f. bug 65776. The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text(self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, length=0, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: #actual_citdl_expr = langintel.citdl_expr_under_pos(buf, data["pos"]) trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual( actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr under pos:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if prefix_filter is not None: XXX #TODO: compare prefix_filter to given value
def assertLex(self, markedup_content, lang=None): """Lex the given content and assert that the lexed tokens are as expected. What is "expected" is given via pseudo-xml markup like this: fuzzy wuzzy <SCE_UDL_SSL_COMMENTBLOCK>wuzza</SCE_UDL_SSL_COMMENTBLOCK> bear This example expects that "wuzza" will be a token with style SCE_UDL_SSL_COMMENTBLOCK. """ from codeintel2.accessor import SilverCityAccessor if lang is None: lang = self.lang content, tokens = self._unmark_lex_text(markedup_content) # Do lexing of this content via the codeintel Buffer's, because # they already handle all the SilverCity lexer hookup. path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) assert isinstance(buf.accessor, SilverCityAccessor) actual_tokens = buf.accessor.tokens # cheating for actual_token in actual_tokens: # There are a few SilverCity token dict keys that we # don't bother checking. del actual_token["end_column"] del actual_token["end_line"] del actual_token["start_column"] del actual_token["start_line"] unmatched_tokens = [t for t in tokens if t not in actual_tokens] if unmatched_tokens: self.fail("not all expected %s lex tokens were found in the " "actual lexer output:\n" " buffer:\n%s\n" " actual lexer tokens:\n%s\n" " unmatched tokens:\n%s\n" % (lang, indent(content), indent(pformat(actual_tokens)), indent(pformat(unmatched_tokens))))
def _assertCompletionsInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) missing_completions = [c for c in completions if c not in (actual_completions or [])] self.failIf(missing_completions, "%s completions at the given position did not " "include all expected values\n" " missing: %r\n" " expected all of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, missing_completions, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def assertCompletionsDoNotInclude(self, markedup_content, completions, lang=None, implicit=True, env=None): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions)
def assertCurrCalltipArgRange(self, markedup_content, calltip, expected_range, lang=None, implicit=True): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) pos = data["pos"] buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.trg_from_pos(data["trg_pos"], implicit=implicit) actual_range = buf.curr_calltip_arg_range(trg.pos, calltip, curr_pos=data["pos"]) self.assertEqual(actual_range, expected_range, "unexpected current calltip arg range\n" " expected: %s\n" " got: %s\n" " calltip:\n%s\n" " buffer:\n%s" % (expected_range, actual_range, indent(calltip), indent(markedup_content)))
def assertNoPrecedingTrigger(self, markedup_content, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is not None: self.fail("unexpectedly found a preceding %s trigger '%s' when " "didn't expect one, buffer:\n%s" % (lang, trg.name, indent(markedup_content)))
def _python_info_from_python(self, python, env): """Call the given Python and return: (<version>, <sys.prefix>, <lib-dir>, <site-lib-dir>, <sys.path>) TODO: Unicode path issues? """ import process argv = [python, "-c", self.info_cmd] log.debug("run `%s -c ...'", python) p = process.ProcessOpen(argv, env=env.get_all_envvars(), stdin=None) stdout, stderr = p.communicate() stdout_lines = stdout.splitlines(0) retval = p.returncode if retval: log.warn( "failed to determine Python info:\n" " path: %s\n" " retval: %s\n" " stdout:\n%s\n" " stderr:\n%s\n", python, retval, indent("\n".join(stdout_lines)), indent(stderr), ) # We are only to rely on the first 2 digits being in the form x.y. ver_match = re.search("([0-9]+.[0-9]+)", stdout_lines[0]) if ver_match: ver = ver_match.group(1) else: ver = None prefix = stdout_lines[1] if sys.platform == "win32": libdir = join(prefix, "Lib") else: libdir = join(prefix, "lib", "python" + ver) sitelibdir = join(libdir, "site-packages") sys_path = stdout_lines[2:] return ver, prefix, libdir, sitelibdir, sys_path
def assertTriggerMatches(self, markedup_content, lang=None, implicit=True, env=None, **fields): if lang is None: lang = self.lang buf, trg = self._get_buf_and_trg(markedup_content, lang, implicit=implicit, env=env) if trg is None: self.fail("unexpectedly did not find a %s trigger, buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): try: actual_value = getattr(trg, name) except AttributeError: actual_value = trg.extra[name] self.assertEqual(actual_value, value, "unexpected %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content)))
def assertPrecedingTriggerMatches(self, markedup_content, lang=None, **fields): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is None: self.fail("unexpectedly did not find a preceding %s trigger, " "buffer:\n%s" % (lang, indent(markedup_content))) if "pos" in fields: fields["pos"] = self.adjust_pos(fields["pos"]) for name, value in fields.items(): actual_value = getattr(trg, name) self.assertEqual(actual_value, value, "unexpected preceding %s trigger '%s' value: expected %r, " "got %r, buffer:\n%s" % (lang, name, value, actual_value, indent(markedup_content)))
def assertCompletionsInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to include %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsInclude(buf, trg, completions)
def assertNoPrecedingTrigger(self, markedup_content, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) trg = buf.preceding_trg_from_pos(data["start_pos"], data["pos"]) if trg is not None: self.fail("unexpectedly found a preceding %s trigger '%s' when " "didn't expect one, buffer:\n%s" % (lang, trg.name, indent(markedup_content)))
def _assertCompletionsDoNotInclude(self, buf, trg, completions): markedup_content = markup_text(buf.accessor.text, pos=trg.pos) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_completions = buf.cplns_from_trg(trg, ctlr=ctlr) completions_that_shouldnt_be_there = [ c for c in (actual_completions or []) if c in completions ] self.failIf(completions_that_shouldnt_be_there, "%s completions at the given position included " "some unexpected values\n" " shouldn't have had these: %r\n" " expected none of: %r\n" " got: %r\n" " eval log:\n%s\n" " buffer:\n%s" % (buf.lang, completions_that_shouldnt_be_there, completions, actual_completions, indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def assertCompletionsDoNotInclude2(self, buf, pos, completions, implicit=True): """A version of assertCompletionsDoNotInclude() where you pass in a Buffer instance instead of marked up content. Sometimes this is more convenient. """ trg = buf.trg_from_pos(pos, implicit=implicit) if trg is None: markedup_content = markup_text(buf.accessor.text, pos=pos) self.fail("given position is not a %s trigger point, " "expected completions to exclude %r:\n%s" % (buf.lang, completions, indent(markedup_content))) self._assertCompletionsDoNotInclude(buf, trg, completions)
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags): if trg is None: self.fail("given position is not a %s trigger point, " "expected the calltip to match the following:\n" " exression:\n%s\n" " buffer:\n%s" % (lang, indent(expr), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertNotEquals(re.search(expr, actual_calltip, flags), None, "unexpected %s calltip at the given position\n" " expression:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(expr and expr or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def _assertCalltipMatches(self, buf, trg, markedup_content, expr, lang, flags): if trg is None: self.fail("given position is not a %s trigger point, " "expected the calltip to match the following:\n" " exression:\n%s\n" " buffer:\n%s" % (lang, indent(expr), indent(markedup_content))) if isinstance(buf, CitadelBuffer): buf.unload() # remove any entry from CIDB to ensure clean test ctlr = _CaptureEvalController() actual_calltips = buf.calltips_from_trg(trg, ctlr=ctlr) if actual_calltips and actual_calltips[0]: actual_calltip = actual_calltips[0] else: actual_calltip = None self.assertNotEquals( re.search(expr, actual_calltip, flags), None, "unexpected %s calltip at the given position\n" " expression:\n%s\n" " got:\n%s\n" " eval log:\n%s\n" " buffer:\n%s" % (trg.name, indent(expr and expr or "(none)"), indent(actual_calltip and actual_calltip or "(none)"), indent('\n'.join('%5s: %s' % (lvl, m) for lvl, m in ctlr.log)), indent(markedup_content)))
def _python_info_from_python(self, python, env): """Call the given Python and return: (<version>, <sys.prefix>, <lib-dir>, <site-lib-dir>, <sys.path>) TODO: Unicode path issues? """ import process argv = [python, "-c", self.info_cmd] log.debug("run `%s -c ...'", python) p = process.ProcessOpen(argv, env=env.get_all_envvars(), stdin=None) stdout, stderr = p.communicate() stdout_lines = stdout.splitlines(0) retval = p.returncode if retval: log.warn( "failed to determine Python info:\n" " path: %s\n" " retval: %s\n" " stdout:\n%s\n" " stderr:\n%s\n", python, retval, indent('\n'.join(stdout_lines)), indent(stderr)) # We are only to rely on the first 2 digits being in the form x.y. ver_match = re.search("([0-9]+.[0-9]+)", stdout_lines[0]) if ver_match: ver = ver_match.group(1) else: ver = None prefix = stdout_lines[1] if sys.platform == "win32": libdir = join(prefix, "Lib") else: libdir = join(prefix, "lib", "python" + ver) sitelibdir = join(libdir, "site-packages") sys_path = stdout_lines[2:] return ver, prefix, libdir, sitelibdir, sys_path
def assertScopeLpathIs(self, markedup_content, lpath, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text(self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) buf.scan(skip_scan_time_check=True) actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"]) self.failUnlessEqual( lpath, actual_lpath, "unexpected %s scope lookup path (lpath) at the given position\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (self.lang, lpath, actual_lpath, indent(markedup_content)))
def assertScopeLpathIs(self, markedup_content, lpath, lang=None): if lang is None: lang = self.lang path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) content, data = unmark_text( self.adjust_content(markedup_content)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) buf.scan(skip_scan_time_check=True) actual_blob, actual_lpath = buf.scoperef_from_pos(data["pos"]) self.failUnlessEqual(lpath, actual_lpath, "unexpected %s scope lookup path (lpath) at the given position\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (self.lang, lpath, actual_lpath, indent(markedup_content)))
def assertCITDLExprUnderPosIs( self, markedup_content, citdl_expr, lang=None, prefix_filter=None, implicit=True, trigger_name=None, **fields): """Assert that the CITDL expression at the current position is as expected. This uses buf.citdl_expr_under_pos() -- or, for Perl, buf.citdl_expr_and_prefix_filter_from_trg(). Note: This API is a mess right now. C.f. bug 65776. The "prefix_filter" optional argument can be used for Perl to test the value its relevant function returns. """ if lang is None: lang = self.lang content, data = unmark_text( self.adjust_content(markedup_content)) path = os.path.join("<Unsaved>", "rand%d" % random.randint(0, 100)) buf = self.mgr.buf_from_content(content, lang=lang, path=path) langintel = self.mgr.langintel_from_lang(lang) if trigger_name is None: trigger_name = "fakey-completion-type" if lang == "Perl": trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, length=0, **fields) actual_citdl_expr, actual_prefix_filter \ = langintel.citdl_expr_and_prefix_filter_from_trg(buf, trg) else: # actual_citdl_expr = langintel.citdl_expr_under_pos(buf, # data["pos"]) trg = Trigger(lang, TRG_FORM_DEFN, trigger_name, data["pos"], implicit=implicit, **fields) actual_citdl_expr = langintel.citdl_expr_from_trg(buf, trg) self.assertEqual(actual_citdl_expr, citdl_expr, "unexpected actual %s CITDL expr under pos:\n" " expected: %r\n" " got: %r\n" " buffer:\n%s" % (lang, citdl_expr, actual_citdl_expr, indent(markedup_content))) if prefix_filter is not None: XXX # TODO: compare prefix_filter to given value
def preceding_trg_from_pos(self, buf, pos, curr_pos, preceding_trg_terminators=None, DEBUG=False): accessor = buf.accessor if preceding_trg_terminators is None: preceding_trg_terminators = self.preceding_trg_terminators if DEBUG: print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)" % (pos, curr_pos)) print indent( markup_text(accessor.text, pos=curr_pos, start_pos=pos)) print banner(None, '-') # Skip over comments and strings in our checking, unless we are # in one of these styles for the whole range. This is so an explicit # trigger in a comment (or, e.g., a doc string) will work, but # the appearance of small comments or strings in code will not mess # things up. comment_and_string_styles = dict( (s, True) for s in buf.comment_styles() + buf.string_styles()) skip_styles = {} start_style = accessor.style_at_pos(pos - 1) EOL_CHARS = tuple("\n\r") # Limiting simplification: Only backtrack a max of 200 chars. # Can increase that if necessary. The problem is detecting a # statement boundary backwards in langs like Python and Ruby # where you can't rely on ';' (actually # `preceding_trg_terminators'). limit = max(1, pos - 200) # First stage. We only consider autocomplete trigger (i.e. # trg.form==TRG_FORM_COMPLETION) if within range of the # curr_pos. Here "within range" means you don't have to more the # cursor to show the autocomplete UI. first_stage_limit = curr_pos for (char, style) in accessor.gen_char_and_style_back( curr_pos - 1, limit - 1): if not isident(char): break first_stage_limit -= 1 if DEBUG: print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\ % (first_stage_limit, (first_stage_limit > 0 and accessor.char_at_pos(first_stage_limit-1) or None)) p = pos if p >= first_stage_limit: for (prev_ch, prev_style) in accessor.gen_char_and_style_back( p - 1, first_stage_limit - 2): if (not skip_styles and prev_style != start_style # EOLs in comments seem to always be style 0. Don't count # them. and prev_ch not in EOL_CHARS): if DEBUG: print "[stage 1] have seen a style change (%d -> %d), " \ "now skipping strings and comments" \ % (start_style, prev_style) skip_styles = comment_and_string_styles if DEBUG: print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\ % (p, prev_ch, prev_style), if prev_style in skip_styles: if DEBUG: print "comment or string, skip it" elif self._is_terminating_char(prev_ch, prev_style, preceding_trg_terminators): if DEBUG: print "in `preceding_trg_terminators': break" return None elif prev_ch in self.trg_chars: if DEBUG: print "trigger char, try it" trg = buf.trg_from_pos(p, implicit=False) if trg: if DEBUG: print "[stage 1] %s" % trg return trg p -= 1 break elif DEBUG: print "not a trigger char, skip it" p -= 1 if DEBUG: print "[stage 1] end of possible autocomplete trigger range" # Second stage. We only consider calltip triggers now # (self.calltip_trg_chars). # # As well, ignore enclosed paren sections to make sure we are # in-range. For example, we shouldn't trigger on "bar(" here: # foo(bar("skip", "this", "arg", "list"), <|>) close_paren_count = 0 for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p - 1, limit - 2): if (not skip_styles and prev_style != start_style # EOLs in comments seem to always be style 0. Don't count # them. and prev_ch not in EOL_CHARS): if DEBUG: print "[stage 2] seen a style change (%d -> %d), now " \ "skipping strings and comments" \ % (start_style, prev_style) skip_styles = comment_and_string_styles if DEBUG: print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\ % (p, prev_ch, prev_style), if prev_style in skip_styles: if DEBUG: print "comment or string, skip it" elif prev_ch == ')': close_paren_count += 1 if DEBUG: print "close paren: count=%d" % close_paren_count elif close_paren_count and prev_ch == '(': close_paren_count -= 1 if DEBUG: print "open paren: count=%d" % close_paren_count elif self._is_terminating_char(prev_ch, prev_style, preceding_trg_terminators): if DEBUG: print "in `preceding_trg_terminators': break" return None elif prev_ch in self.calltip_trg_chars: if DEBUG: print "trigger char, try it" trg = buf.trg_from_pos(p, implicit=False) if trg: if DEBUG: print "[stage 2] %s" % trg return trg elif DEBUG: print "not a trigger char, skip it" p -= 1 return None
def curr_calltip_arg_range(self, buf, trg_pos, calltip, curr_pos, DEBUG=False): """Return that range in the calltip of the "current" arg. I.e. what argument is currently being entered. "buf" is the buffer object on which this is being done. "trg_pos" is the trigger position. "calltip" is the full calltip text. "curr_pos" is the current position in the buffer. Returns a range: (start, end) Set `start == -1` to cancel the calltip, i.e. if the entered text has closed the call region. The default implementation uses: self.calltip_region_terminators to handle languages with calltip signatures with the following characteristics: - uses '(' and ')' to bound the argument list (though because of support for ';' statement termination, this isn't absolutely required) - uses a comma to separate arguments - basic block delimiters are {}, (), and [] For example: foo() blam(a, b) range([start,] stop[, step]) -> list of integers bar(arg1, *args, **kwargs) flash(boom, bang=42) """ # Dev Notes: # - Eventually should pass in the trigger to aid in processing. # - TODO figure out dependence on buf.comment_styles() and # buf.string_styles() accessor = buf.accessor if DEBUG: print banner("curr_calltip_arg_range") print "calltip:\n%s" % indent(calltip) print "buffer:\n%s" % indent( markup_text(accessor.text, trg_pos=trg_pos, pos=curr_pos)) # Start from the trigger position and walk forward to the current # pos: counting args and looking for termination of the calltip # region. skip_styles = dict( (s, True) for s in buf.comment_styles() + buf.string_styles()) if accessor.style_at_pos(trg_pos - 1) in skip_styles: skip_styles = {} comma_count = 0 blocks = { # Map a block start token to its block end token. '(': ')', '[': ']', '{': '}', } block_stack = [] p = trg_pos for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos): if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style), if style in skip_styles: if DEBUG: print "skip" elif ch in blocks: if DEBUG: print "open block" block_stack.append(blocks[ch]) elif block_stack: if ch == block_stack[-1]: if DEBUG: print "close block" block_stack.pop() elif ch in self.calltip_region_terminators: if DEBUG: print "end of call region: (-1, -1)" return (-1, -1) elif DEBUG: print "ignore (in block)" elif ch == ',': if DEBUG: print "next arg" comma_count += 1 elif ch in self.calltip_region_terminators and \ self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos): if DEBUG: print "end of call region: (-1, -1)" return (-1, -1) elif DEBUG: print "ignore" p += 1 # Parse the signature from the calltip. If there is no signature # then we default to not indicating any arg range. if self._parsed_calltip_cache[0] == calltip: parsed = self._parsed_calltip_cache[1] else: parsed = _parse_calltip(calltip, DEBUG) self._parsed_calltip_cache = (calltip, parsed) if parsed is None: if DEBUG: print "couldn't parse any calltip: (0, 0)" return (0, 0) signature, name, args = parsed if DEBUG: print "parsed calltip:\n signature:\n%s\n name:\n%s\n args:\n%s"\ % (indent(signature), indent(name), indent(pformat(args))) if not args: if DEBUG: print "no args in signature: (0, 0)" return (0, 0) elif comma_count >= len(args): #XXX ellipsis if DEBUG: print "more commas than args: ellipsis?" span = args[-1].span # default to last arg else: span = args[comma_count].span if DEBUG: print "curr calltip range (%s, %s):" % (span[0], span[1]) print indent(signature) print " %s%s" % (' ' * span[0], '-' * (span[1] - span[0])) return span
def preceding_trg_from_pos(self, buf, pos, curr_pos, preceding_trg_terminators=None, DEBUG=False): accessor = buf.accessor if preceding_trg_terminators is None: preceding_trg_terminators = self.preceding_trg_terminators if DEBUG: print banner("preceding_trg_from_pos(pos=%r, curr_pos=%r)" % (pos, curr_pos)) print indent(markup_text(accessor.text, pos=curr_pos, start_pos=pos)) print banner(None, '-') # Skip over comments and strings in our checking, unless we are # in one of these styles for the whole range. This is so an explicit # trigger in a comment (or, e.g., a doc string) will work, but # the appearance of small comments or strings in code will not mess # things up. comment_and_string_styles = dict( (s, True) for s in buf.comment_styles() + buf.string_styles()) skip_styles = {} start_style = accessor.style_at_pos(pos-1) EOL_CHARS = tuple("\n\r") # Limiting simplification: Only backtrack a max of 200 chars. # Can increase that if necessary. The problem is detecting a # statement boundary backwards in langs like Python and Ruby # where you can't rely on ';' (actually # `preceding_trg_terminators'). limit = max(1, pos - 200) # First stage. We only consider autocomplete trigger (i.e. # trg.form==TRG_FORM_COMPLETION) if within range of the # curr_pos. Here "within range" means you don't have to more the # cursor to show the autocomplete UI. first_stage_limit = curr_pos for (char, style) in accessor.gen_char_and_style_back(curr_pos-1, limit-1): if not isident(char): break first_stage_limit -= 1 if DEBUG: print "[stage 1] first_stage_limit=%d (prev_ch=%r)"\ % (first_stage_limit, (first_stage_limit > 0 and accessor.char_at_pos(first_stage_limit-1) or None)) p = pos if p >= first_stage_limit: for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1, first_stage_limit-2): if (not skip_styles and prev_style != start_style # EOLs in comments seem to always be style 0. Don't count # them. and prev_ch not in EOL_CHARS): if DEBUG: print "[stage 1] have seen a style change (%d -> %d), " \ "now skipping strings and comments" \ % (start_style, prev_style) skip_styles = comment_and_string_styles if DEBUG: print "[stage 1] consider pos %2d: prev_ch=%r (%d) --"\ % (p, prev_ch, prev_style), if prev_style in skip_styles: if DEBUG: print "comment or string, skip it" elif self._is_terminating_char(prev_ch, prev_style, preceding_trg_terminators): if DEBUG: print "in `preceding_trg_terminators': break" return None elif prev_ch in self.trg_chars: if DEBUG: print "trigger char, try it" trg = buf.trg_from_pos(p, implicit=False) if trg: if DEBUG: print "[stage 1] %s" % trg return trg p -= 1 break elif DEBUG: print "not a trigger char, skip it" p -= 1 if DEBUG: print "[stage 1] end of possible autocomplete trigger range" # Second stage. We only consider calltip triggers now # (self.calltip_trg_chars). # # As well, ignore enclosed paren sections to make sure we are # in-range. For example, we shouldn't trigger on "bar(" here: # foo(bar("skip", "this", "arg", "list"), <|>) close_paren_count = 0 for (prev_ch, prev_style) in accessor.gen_char_and_style_back(p-1, limit-2): if (not skip_styles and prev_style != start_style # EOLs in comments seem to always be style 0. Don't count # them. and prev_ch not in EOL_CHARS): if DEBUG: print "[stage 2] seen a style change (%d -> %d), now " \ "skipping strings and comments" \ % (start_style, prev_style) skip_styles = comment_and_string_styles if DEBUG: print "[stage 2] consider pos %2d: prev_ch=%r (%d) --"\ % (p, prev_ch, prev_style), if prev_style in skip_styles: if DEBUG: print "comment or string, skip it" elif prev_ch == ')': close_paren_count += 1 if DEBUG: print "close paren: count=%d" % close_paren_count elif close_paren_count and prev_ch == '(': close_paren_count -= 1 if DEBUG: print "open paren: count=%d" % close_paren_count elif self._is_terminating_char(prev_ch, prev_style, preceding_trg_terminators): if DEBUG: print "in `preceding_trg_terminators': break" return None elif prev_ch in self.calltip_trg_chars: if DEBUG: print "trigger char, try it" trg = buf.trg_from_pos(p, implicit=False) if trg: if DEBUG: print "[stage 2] %s" % trg return trg elif DEBUG: print "not a trigger char, skip it" p -= 1 return None
def curr_calltip_arg_range(self, buf, trg_pos, calltip, curr_pos, DEBUG=False): """Return that range in the calltip of the "current" arg. I.e. what argument is currently being entered. "buf" is the buffer object on which this is being done. "trg_pos" is the trigger position. "calltip" is the full calltip text. "curr_pos" is the current position in the buffer. Returns a range: (start, end) Set `start == -1` to cancel the calltip, i.e. if the entered text has closed the call region. The default implementation uses: self.calltip_region_terminators to handle languages with calltip signatures with the following characteristics: - uses '(' and ')' to bound the argument list (though because of support for ';' statement termination, this isn't absolutely required) - uses a comma to separate arguments - basic block delimiters are {}, (), and [] For example: foo() blam(a, b) range([start,] stop[, step]) -> list of integers bar(arg1, *args, **kwargs) flash(boom, bang=42) """ # Dev Notes: # - Eventually should pass in the trigger to aid in processing. # - TODO figure out dependence on buf.comment_styles() and # buf.string_styles() accessor = buf.accessor if DEBUG: print banner("curr_calltip_arg_range") print "calltip:\n%s" % indent(calltip) print "buffer:\n%s" % indent(markup_text(accessor.text, trg_pos=trg_pos, pos=curr_pos)) # Start from the trigger position and walk forward to the current # pos: counting args and looking for termination of the calltip # region. skip_styles = dict( (s, True) for s in buf.comment_styles() + buf.string_styles()) if accessor.style_at_pos(trg_pos-1) in skip_styles: skip_styles = {} comma_count = 0 blocks = { # Map a block start token to its block end token. '(': ')', '[': ']', '{': '}', } block_stack = [] p = trg_pos for ch, style in accessor.gen_char_and_style(trg_pos, curr_pos): if DEBUG: print "pos %2d: %r (%2s) --" % (p, ch, style), if style in skip_styles: if DEBUG: print "skip" elif ch in blocks: if DEBUG: print "open block" block_stack.append(blocks[ch]) elif block_stack: if ch == block_stack[-1]: if DEBUG: print "close block" block_stack.pop() elif ch in self.calltip_region_terminators: if DEBUG: print "end of call region: (-1, -1)" return (-1, -1) elif DEBUG: print "ignore (in block)" elif ch == ',': if DEBUG: print "next arg" comma_count += 1 elif ch in self.calltip_region_terminators and \ self.calltip_verify_termination(accessor, ch, trg_pos, curr_pos): if DEBUG: print "end of call region: (-1, -1)" return (-1, -1) elif DEBUG: print "ignore" p += 1 # Parse the signature from the calltip. If there is no signature # then we default to not indicating any arg range. if self._parsed_calltip_cache[0] == calltip: parsed = self._parsed_calltip_cache[1] else: parsed = _parse_calltip(calltip, DEBUG) self._parsed_calltip_cache = (calltip, parsed) if parsed is None: if DEBUG: print "couldn't parse any calltip: (0, 0)" return (0, 0) signature, name, args = parsed if DEBUG: print "parsed calltip:\n signature:\n%s\n name:\n%s\n args:\n%s"\ % (indent(signature), indent(name), indent(pformat(args))) if not args: if DEBUG: print "no args in signature: (0, 0)" return (0, 0) elif comma_count >= len(args): #XXX ellipsis if DEBUG: print "more commas than args: ellipsis?" span = args[-1].span # default to last arg else: span = args[comma_count].span if DEBUG: print "curr calltip range (%s, %s):" % (span[0], span[1]) print indent(signature) print " %s%s" % (' '*span[0], '-'*(span[1]-span[0])) return span
def _buf_indep_libs_from_env(self, env): """Create the buffer-independent list of libs.""" cache_key = self.lang + "-libs" libs = env.cache.get(cache_key) if libs is None: env.add_pref_observer(self.interpreterPrefName, self._invalidate_cache) env.add_pref_observer(self.extraPathsPrefName, self._invalidate_cache_and_rescan_extra_dirs) env.add_pref_observer("codeintel_selected_catalogs", self._invalidate_cache) db = self.mgr.db ver, prefix, libdir, sitelibdir, sys_path \ = self.python_info_from_env(env) libs = [] # - extradirslib extra_dirs = self._extra_dirs_from_env(env) if extra_dirs: libs.append( db.get_lang_lib(self.lang, "extradirslib", extra_dirs) ) # Figure out which sys.path dirs belong to which lib. paths_from_libname = {"sitelib": [], "envlib": [], "stdlib": []} canon_sitelibdir = sitelibdir and normcase(sitelibdir) or None canon_prefix = prefix and normcase(prefix) or None canon_libdir = libdir and normcase(libdir) or "" canon_libdir_plat_prefix = normcase(join(libdir, "plat-")) canon_libdir_lib_prefix = normcase(join(libdir, "lib-")) for dir in sys_path: STATE = "envlib" canon_dir = normcase(dir) if dir == "": # -> curdirlib (already handled) continue elif canon_dir.endswith(".zip") and isfile(dir): log.warn("`%s': not handling .zip file on Python sys.path", dir) continue elif canon_dir.endswith(".egg") and isfile(dir): #log.warn("`%s': not handling .egg file on Python sys.path", # dir) continue elif canon_dir.startswith(canon_sitelibdir): STATE = "sitelib" # Check against the known list of standard library locations. elif canon_dir == canon_libdir or \ canon_dir.startswith(canon_libdir_plat_prefix) or \ canon_dir.startswith(canon_libdir_lib_prefix): STATE = "stdlib" if not exists(dir): continue paths_from_libname[STATE].append(dir) log.debug("Python %s paths for each lib:\n%s", ver, indent(pformat(paths_from_libname))) # - envlib, sitelib, cataloglib, stdlib if paths_from_libname["envlib"]: libs.append( db.get_lang_lib(self.lang, "envlib", paths_from_libname["envlib"]) ) if paths_from_libname["sitelib"]: libs.append( db.get_lang_lib(self.lang, "sitelib", paths_from_libname["sitelib"]) ) catalog_selections = env.get_pref("codeintel_selected_catalogs") libs += [ db.get_catalog_lib(self.lang, catalog_selections), db.get_stdlib(self.lang, ver) ] env.cache[cache_key] = libs return libs
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath + '.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath + '.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath + '.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath + '.options') # input options if _debug: print() print("*" * 50, "codeintel '%s'" % fpath) # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. #XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print("*" * 50, "options") pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir)) + 1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set( "error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError as ex: error = traceback.format_exc() else: error = None if isinstance(cix, six.text_type): with io.open(tmpfile, mode="wt", encoding="utf-8") as fout: fout.write(cix) else: with open(tmpfile, mode="wb") as fout: fout.write(cix) finally: stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print("*" * 50, "stdout") print(stdout) print("*" * 50, "stderr") print(stderr) print("*" * 50, "error") print(str(error)) print("*" * 50) generateMissing = False if not os.path.exists(outfile) and generateMissing: with io.open(outfile, mode='wt', encoding='utf-8') as fout: with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: fout.write(ftmp.read()) # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1) + path + match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) # and normalize ' ', ' ' and ''' with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) expected = expected.replace('
', ' ').replace( '
', ' ').replace(''', '\'') with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read()) actual = actual.replace('
', ' ').replace( '
', ' ').replace(''', '\'') if expected != actual: do_fail = True # Useful temporary thing while XML output format is changing. #if os.stat("../support/xmldiff.py"): # rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile)) # if rc == 0: # do_fail = False if do_fail: diff = list( difflib.ndiff(expected.splitlines(1), actual.splitlines(1))) diff = _diffContext(diff, 2) if diff: error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \ % (outfile, tmpfile, outfile, tmpfile, ''.join(diff)) if gMaxDiffOutput > 0 and gMaxNumLines > 0: if len(error_str) > gMaxDiffOutput: error_lines = error_str.split("\n") if len(error_lines) > gMaxNumLines: error_lines = error_lines[:gMaxNumLines] + [ "..." ] if gMaxLineLength > 0: error_str = "\n".join([ len(x) > gMaxLineLength and x[:gMaxLineLength] or x for x in error_lines ]) else: error_str = "\n".join(error_lines) self.fail(_encode_for_stdout(error_str)) elif os.path.exists(errfile): # There is no reference output file. This means that processing # this file is expected to fail. expectedError = open(errfile, 'r').read() actualError = str(error) self.failUnlessEqual(actualError.strip(), expectedError.strip()) else: self.fail("No reference output file or error file for '%s'." % infile) # Ensure next test file gets a clean codeintel. toDelete = [] for modname in sys.modules: if modname == "codeintel" or modname.startswith("codeintel."): toDelete.append(modname) for modname in toDelete: del sys.modules[modname]
def _buf_indep_libs_from_env(self, env): """Create the buffer-independent list of libs.""" cache_key = self.lang + "-libs" libs = env.cache.get(cache_key) if libs is None: env.add_pref_observer(self.interpreterPrefName, self._invalidate_cache) env.add_pref_observer(self.extraPathsPrefName, self._invalidate_cache_and_rescan_extra_dirs) env.add_pref_observer("codeintel_selected_catalogs", self._invalidate_cache) db = self.mgr.db ver, prefix, libdir, sitelibdir, sys_path \ = self.python_info_from_env(env) libs = [] # - extradirslib extra_dirs = self._extra_dirs_from_env(env) if extra_dirs: libs.append( db.get_lang_lib(self.lang, "extradirslib", extra_dirs)) # Figure out which sys.path dirs belong to which lib. paths_from_libname = {"sitelib": [], "envlib": [], "stdlib": []} canon_sitelibdir = sitelibdir and normcase(sitelibdir) or None canon_prefix = prefix and normcase(prefix) or None canon_libdir = normcase(libdir) canon_libdir_plat_prefix = normcase(join(libdir, "plat-")) canon_libdir_lib_prefix = normcase(join(libdir, "lib-")) for dir in sys_path: STATE = "envlib" canon_dir = normcase(dir) if dir == "": # -> curdirlib (already handled) continue elif canon_dir.endswith(".zip") and isfile(dir): log.warn("`%s': not handling .zip file on Python sys.path", dir) continue elif canon_dir.endswith(".egg") and isfile(dir): #log.warn("`%s': not handling .egg file on Python sys.path", # dir) continue elif canon_dir.startswith(canon_sitelibdir): STATE = "sitelib" # Check against the known list of standard library locations. elif canon_dir == canon_libdir or \ canon_dir.startswith(canon_libdir_plat_prefix) or \ canon_dir.startswith(canon_libdir_lib_prefix): STATE = "stdlib" if not exists(dir): continue paths_from_libname[STATE].append(dir) log.debug("Python %s paths for each lib:\n%s", ver, indent(pformat(paths_from_libname))) # - envlib, sitelib, cataloglib, stdlib if paths_from_libname["envlib"]: libs.append( db.get_lang_lib(self.lang, "envlib", paths_from_libname["envlib"])) if paths_from_libname["sitelib"]: libs.append( db.get_lang_lib(self.lang, "sitelib", paths_from_libname["sitelib"])) catalog_selections = env.get_pref("codeintel_selected_catalogs") libs += [ db.get_catalog_lib(self.lang, catalog_selections), db.get_stdlib(self.lang, ver) ] env.cache[cache_key] = libs return libs
stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print "*" * 50, "stdout" print stdout print "*" * 50, "stderr" print stderr print "*" * 50, "error" print str(error) print "*" * 50 # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1) + path + match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read())
stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print "*"*50, "stdout" print stdout print "*"*50, "stderr" print stderr print "*"*50, "error" print str(error) print "*" * 50 # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1)+path+match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read()) if expected != actual:
def _testOneInputFile(self, fpath, tags=None): _debug = False # Set to true to dump status info for each test run. infile = os.path.join(gInputsDir, fpath) # input outfile = os.path.join(gOutputsDir, fpath+'.cix') # expected output tmpfile = os.path.join(gTmpDir, fpath+'.cix') # actual output if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) errfile = os.path.join(gOutputsDir, fpath+'.error') # expected error # An options file is a set of kwargs for the buf.scan() # method call. One key-value pair per-line like this: # key=value # Whitespace is stripped off the value. optsfile = os.path.join(gInputsDir, fpath+'.options') # input options if _debug: print() print("*"*50, "codeintel '%s'" % fpath) # Set standard options: opts = {"mtime": "42"} # Determine input options to use, if any. #XXX Not used. Drop it. if os.path.exists(optsfile): for line in open(optsfile, 'r').read().splitlines(0): name, value = line.split('=', 1) value = value.strip() try: # allow value to be a type other than string value = eval(value) except Exception: pass opts[name] = value if _debug: print("*"*50, "options") pprint.pprint(opts) # Scan the file, capturing stdout and stderr and any possible # error. # - To allow testing from different dirs (resulting in changing # path strings, we normalize the <file path="..."> value and any # <scope ilk="blob" src="..."> attributes). oldStdout = sys.stdout oldStderr = sys.stderr sys.stdout = StringIO() sys.stderr = StringIO() try: try: lang = None if tags and "python3" in tags: lang = "Python3" buf = self.mgr.buf_from_path(infile, lang=lang) buf.scan(**opts) tree = buf.tree # Normalize paths. relnorm_infile = infile[len(dirname(gInputsDir))+1:] absnorm_infile = infile relnorm_infile = relnorm_infile.replace('\\', '/') absnorm_infile = absnorm_infile.replace('\\', '/') for file_elem in tree: file_elem.set("path", relnorm_infile) for blob_elem in file_elem: if blob_elem.get("ilk") != "blob": continue norm_src = normpath(blob_elem.get("src")) norm_src = norm_src.replace('\\', '/') if norm_src in (relnorm_infile, absnorm_infile): blob_elem.set("src", relnorm_infile) tree = pretty_tree_from_tree(tree) # Due to the dynamic nature of the ciler errors (which often # includes the source code line numbers), it's difficult to check # that the errors are identical, so we work around this by just # taking the first 30 characters of the error. cile_error = tree[0].get("error") if cile_error and fpath.endswith(".js"): tree[0].set("error", len(cile_error) < 30 and cile_error or (cile_error[:30] + "...")) cix = ET.tostring(tree) except CodeIntelError as ex: error = traceback.format_exc() else: error = None if isinstance(cix, six.text_type): with io.open(tmpfile, mode="wt", encoding="utf-8") as fout: fout.write(cix) else: with open(tmpfile, mode="wb") as fout: fout.write(cix) finally: stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() sys.stdout = oldStdout sys.stderr = oldStderr if _debug: print("*"*50, "stdout") print(stdout) print("*"*50, "stderr") print(stderr) print("*"*50, "error") print(str(error)) print("*" * 50) generateMissing = False if not os.path.exists(outfile) and generateMissing: with io.open(outfile, mode='wt', encoding='utf-8') as fout: with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: fout.write(ftmp.read()) # Verify that the results are as expected. if os.path.exists(outfile) and error: self.fail("scanning '%s' raised an error but success was " "expected:\n%s" % (_encode_for_stdout(fpath), indent(error))) elif os.path.exists(outfile): # Convert the <file path="..."/> to the native directory separator. def to_native_sep(match): path = match.group(2).replace("\\", os.sep).replace("/", os.sep) return match.group(1)+path+match.group(3) path_pat = re.compile(r'(<file .*?path=")(.*?)(".*?>)', re.S) # Note that we don't really care about line endings here, so we read # both files in universal newlines mode (i.e. translate to \n) # and normalize ' ', ' ' and ''' with io.open(outfile, mode='rt', encoding='utf-8') as fout: expected = path_pat.sub(to_native_sep, fout.read()) expected = expected.replace('
', ' ').replace('
', ' ').replace(''', '\'') with io.open(tmpfile, mode='rt', encoding='utf-8') as ftmp: actual = path_pat.sub(to_native_sep, ftmp.read()) actual = actual.replace('
', ' ').replace('
', ' ').replace(''', '\'') if expected != actual: do_fail = True # Useful temporary thing while XML output format is changing. #if os.stat("../support/xmldiff.py"): # rc = os.system('python ../support/xmldiff.py "%s" "%s"' % (outfile, tmpfile)) # if rc == 0: # do_fail = False if do_fail: diff = list(difflib.ndiff(expected.splitlines(1), actual.splitlines(1))) diff = _diffContext(diff, 2) if diff: error_str = "%r != %r:\n --- %s\n +++ %s\n%s" \ % (outfile, tmpfile, outfile, tmpfile, ''.join(diff)) if gMaxDiffOutput > 0 and gMaxNumLines > 0: if len(error_str) > gMaxDiffOutput: error_lines = error_str.split("\n") if len(error_lines) > gMaxNumLines: error_lines = error_lines[:gMaxNumLines] + ["..."] if gMaxLineLength > 0: error_str = "\n".join([len(x) > gMaxLineLength and x[:gMaxLineLength] or x for x in error_lines]) else: error_str = "\n".join(error_lines) self.fail(_encode_for_stdout(error_str)) elif os.path.exists(errfile): # There is no reference output file. This means that processing # this file is expected to fail. expectedError = open(errfile, 'r').read() actualError = str(error) self.failUnlessEqual(actualError.strip(), expectedError.strip()) else: self.fail("No reference output file or error file for '%s'." % infile) # Ensure next test file gets a clean codeintel. toDelete = [] for modname in sys.modules: if modname == "codeintel" or modname.startswith("codeintel."): toDelete.append(modname) for modname in toDelete: del sys.modules[modname]