def get_stack_at_position(grammar, code_lines, module_node, pos): """ Returns the possible node names (e.g. import_from, xor_test or yield_stmt). """ class EndMarkerReached(Exception): pass def tokenize_without_endmarker(code): # TODO This is for now not an official parso API that exists purely # for Jedi. tokens = grammar._tokenize(code) for token_ in tokens: if token_.string == safeword: raise EndMarkerReached() elif token_.prefix.endswith(safeword): # This happens with comments. raise EndMarkerReached() else: yield token_ # The code might be indedented, just remove it. code = dedent(_get_code_for_stack(code_lines, module_node, pos)) # We use a word to tell Jedi when we have reached the start of the # completion. # Use Z as a prefix because it's not part of a number suffix. safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI' code = code + ' ' + safeword p = Parser(grammar._pgen_grammar, error_recovery=True) try: p.parse(tokens=tokenize_without_endmarker(code)) except EndMarkerReached: return p.stack raise SystemError("This really shouldn't happen. There's a bug in Jedi.")
def _try_parse_part(self, until_line): """ Sets up a normal parser that uses a spezialized tokenizer to only parse until a certain position (or a bit longer if the statement hasn't ended. """ self._parser_count += 1 # TODO speed up, shouldn't copy the whole list all the time. # memoryview? parsed_until_line = self._nodes_stack.parsed_until_line lines_after = self._parser_lines_new[parsed_until_line:] tokens = self._diff_tokenize(lines_after, until_line, line_offset=parsed_until_line) self._active_parser = Parser(self._pgen_grammar, error_recovery=True) return self._active_parser.parse(tokens=tokens)
def update(self, old_lines, new_lines): ''' The algorithm works as follows: Equal: - Assure that the start is a newline, otherwise parse until we get one. - Copy from parsed_until_line + 1 to max(i2 + 1) - Make sure that the indentation is correct (e.g. add DEDENT) - Add old and change positions Insert: - Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not much more. Returns the new module node. ''' LOG.debug('diff parser start') # Reset the used names cache so they get regenerated. self._module._used_names = None self._parser_lines_new = new_lines self._reset() line_length = len(new_lines) sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new) opcodes = sm.get_opcodes() LOG.debug('line_lengths old: %s; new: %s' % (len(old_lines), line_length)) for operation, i1, i2, j1, j2 in opcodes: LOG.debug('-> code[%s] old[%s:%s] new[%s:%s]', operation, i1 + 1, i2, j1 + 1, j2) if j2 == line_length and new_lines[-1] == '': # The empty part after the last newline is not relevant. j2 -= 1 if operation == 'equal': line_offset = j1 - i1 self._copy_from_old_parser(line_offset, i1 + 1, i2, j2) elif operation == 'replace': self._parse(until_line=j2) elif operation == 'insert': self._parse(until_line=j2) else: assert operation == 'delete' # With this action all change will finally be applied and we have a # changed module. self._nodes_tree.close() if DEBUG_DIFF_PARSER: # If there is reasonable suspicion that the diff parser is not # behaving well, this should be enabled. try: code = ''.join(new_lines) assert self._module.get_code() == code _assert_valid_graph(self._module) without_diff_parser_module = Parser( self._pgen_grammar, error_recovery=True).parse(self._tokenizer(new_lines)) _assert_nodes_are_equal(self._module, without_diff_parser_module) except AssertionError: print( _get_debug_error_message(self._module, old_lines, new_lines)) raise last_pos = self._module.end_pos[0] if last_pos != line_length: raise Exception( ('(%s != %s) ' % (last_pos, line_length)) + _get_debug_error_message(self._module, old_lines, new_lines)) LOG.debug('diff parser end') return self._module