def ComputeCandidatesInner_BeforeUnicode_test( completer, execute_command ):
# Col 8 corresponds to cursor at log.Pr^int("Line 7 ...
completer.ComputeCandidatesInner( BuildRequest( 7, 8 ) )
execute_command.assert_called_once_with(
[ DUMMY_BINARY, '-sock', 'tcp', '-addr', completer._gocode_host,
'-f=json', 'autocomplete', PATH_TO_TEST_FILE, '119' ],
contents = ToBytes( ReadFile( PATH_TO_TEST_FILE ) ) )
def LineAndColumnNumbersClamped(line_num, column_num):
line_num = max(min(line_num, len(vim.current.buffer)), 1)
# Vim buffers are a list of Unicode objects on Python 3.
max_column = len(ToBytes(vim.current.buffer[line_num - 1]))
return line_num, min(column_num, max_column)
def _BuildGetDocResponse(doc_data):
"""Builds a "DetailedInfoResponse" for a GetDoc request. doc_data is a
DocumentationData object returned from the ClangCompleter"""
# Parse the XML, as this is the only way to get the declaration text out of
# libclang. It seems quite wasteful, but while the contents of the XML
# provide fully parsed doxygen documentation tree, we actually don't want to
# ever lose any information from the comment, so we just want display
# the stripped comment. Arguably we could skip all of this XML generation and
# parsing, but having the raw declaration text is likely one of the most
# useful pieces of documentation available to the developer. Perhaps in
# future, we can use this XML for more interesting things.
try:
# Only python2 actually requires bytes here.
# Doing the same on python3 makes the code simpler,
# but introduces unnecessary, though quite acceptable overhead
# (compared to XML processing).
root = xml.etree.ElementTree.fromstring(ToBytes(doc_data.comment_xml))
except XmlParseError:
raise ValueError(NO_DOCUMENTATION_MESSAGE)
# Note: declaration is False-y if it has no child elements, hence the below
# (wordy) if not declaration is None
declaration = root.find("Declaration")
return responses.BuildDetailedInfoResponse(
'{0}\n{1}\nType: {2}\nName: {3}\n---\n{4}'.format(
ToUnicode(declaration.text) if declaration is not None else "",
ToUnicode(doc_data.brief_comment),
ToUnicode(doc_data.canonical_type),
ToUnicode(doc_data.display_name),
ToUnicode(_FormatRawComment(doc_data.raw_comment))))
def _CallExtraConfFlagsForFile(module, filename, client_data):
# We want to ensure we pass a native py2 `str` on py2 and a native py3 `str`
# (unicode) object on py3. That's the API we provide.
# In a vacuum, always passing a unicode object (`unicode` on py2 and `str` on
# py3) would be better, but we can't do that because that would break all the
# ycm_extra_conf files already out there that expect a py2 `str` object on
# py2, and WE DO NOT BREAK BACKWARDS COMPATIBILITY.
# Hindsight is 20/20.
if PY2:
filename = native(ToBytes(filename))
else:
filename = native(ToUnicode(filename))
if hasattr(module, 'Settings'):
results = module.Settings(language='cfamily',
filename=filename,
client_data=client_data)
# For the sake of backwards compatibility, we need to first check whether the
# FlagsForFile function in the extra conf module even allows keyword args.
elif inspect.getargspec(module.FlagsForFile).keywords:
results = module.FlagsForFile(filename, client_data=client_data)
else:
results = module.FlagsForFile(filename)
if not isinstance(results, dict) or 'flags' not in results:
return EMPTY_FLAGS
results['flags'] = _MakeRelativePathsInFlagsAbsolute(
results['flags'], results.get('include_paths_relative_to_dir'))
return results
def ReplaceChunk_MultipleLinesToSingleLine_test():
result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCaaaa" ) ]
start, end = _BuildLocations( 2, 2, 3, 2 )
( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'E',
0, 0, result_buffer )
expected_buffer = [ ToBytes( "aAa" ), ToBytes( "aECaaaa" ) ]
eq_( expected_buffer, result_buffer )
eq_( line_offset, -1 )
eq_( char_offset, 1 )
# make another modification applying offsets
start, end = _BuildLocations( 3, 3, 3, 4 )
( new_line_offset, new_char_offset ) = vimsupport.ReplaceChunk(
start,
end,
'cccc',
line_offset,
char_offset,
result_buffer )
line_offset += new_line_offset
char_offset += new_char_offset
eq_( [ ToBytes( "aAa" ),
ToBytes( "aECccccaaa" ) ], result_buffer )
eq_( line_offset, -1 )
eq_( char_offset, 4 )
# and another, for luck
start, end = _BuildLocations( 3, 4, 3, 5 )
( new_line_offset, new_char_offset ) = vimsupport.ReplaceChunk(
start,
end,
'dd\ndd',
line_offset,
char_offset,
result_buffer )
line_offset += new_line_offset
char_offset += new_char_offset
eq_( [ ToBytes( "aAa" ),
ToBytes( "aECccccdd" ),
ToBytes( "ddaa" ) ],
result_buffer )
eq_( line_offset, 0 )
eq_( char_offset, -2 )
def ComputeCandidatesInner_AfterUnicode_test(completer, execute_command):
# Col 9 corresponds to cursor at log.Pri^nt("Line 7 ...
completer.ComputeCandidatesInner(BuildRequest(9, 9))
execute_command.assert_called_once_with([
DUMMY_BINARY, '-addr', completer._gocode_address, '-f=json',
'autocomplete', PATH_TO_TEST_FILE, '212'
],
contents=ToBytes(
ReadFile(PATH_TO_TEST_FILE)))
def SetupOptions( options_file ): options = user_options_store.DefaultOptions() user_options = json.loads( ReadFile( options_file ) ) options.update( user_options ) utils.RemoveIfExists( options_file ) hmac_secret = ToBytes( base64.b64decode( options[ 'hmac_secret' ] ) ) del options[ 'hmac_secret' ] user_options_store.SetAll( options ) return options, hmac_secret
def _GetResponse(self, handler, request_data={}):
"""POST JSON data to JediHTTP server and return JSON response."""
handler = ToBytes(handler)
url = urljoin(self._jedihttp_host, handler)
parameters = self._TranslateRequestForJediHTTP(request_data)
body = ToBytes(json.dumps(parameters)) if parameters else bytes()
extra_headers = self._ExtraHeaders(handler, body)
self._logger.debug('Making JediHTTP request: %s %s %s %s', 'POST', url,
extra_headers, body)
response = requests.request(native(bytes(b'POST')),
native(url),
data=body,
headers=extra_headers)
response.raise_for_status()
return response.json()
def ToBytesOnPY2(data):
# To test the omnifunc, etc. returning strings, which can be of different
# types depending on python version, we use ToBytes on PY2 and just the native
# str on python3. This roughly matches what happens between py2 and py3
# versions within Vim
if PY2:
return ToBytes(data)
return data
def ReplaceChunk(start, end, replacement_text, vim_buffer):
# ycmd's results are all 1-based, but vim's/python's are all 0-based
# (so we do -1 on all of the values)
start_line = start['line_num'] - 1
end_line = end['line_num'] - 1
start_column = start['column_num'] - 1
end_column = end['column_num'] - 1
# When sending a request to the server, a newline is added to the buffer
# contents to match what gets saved to disk. If the server generates a chunk
# containing that newline, this chunk goes past the Vim buffer contents since
# there is actually no new line. When this happens, recompute the end position
# of where the chunk is applied and remove all trailing characters in the
# chunk.
if end_line >= len(vim_buffer):
end_column = len(ToBytes(vim_buffer[-1]))
end_line = len(vim_buffer) - 1
replacement_text = replacement_text.rstrip()
# NOTE: replacement_text is unicode, but all our offsets are byte offsets,
# so we convert to bytes
replacement_lines = SplitLines(ToBytes(replacement_text))
# NOTE: Vim buffers are a list of byte objects on Python 2 but unicode
# objects on Python 3.
start_existing_text = ToBytes(vim_buffer[start_line])[:start_column]
end_existing_text = ToBytes(vim_buffer[end_line])[end_column:]
replacement_lines[0] = start_existing_text + replacement_lines[0]
replacement_lines[-1] = replacement_lines[-1] + end_existing_text
vim_buffer[start_line:end_line + 1] = replacement_lines[:]
return {
'bufnr': vim_buffer.number,
'filename': vim_buffer.name,
# line and column numbers are 1-based in qflist
'lnum': start_line + 1,
'col': start_column + 1,
'text': replacement_text,
'type': 'F',
}
def GetUnsavedAndCurrentBufferData_EncodedUnicodeCharsInBuffers_test( *args ):
mock_buffer = MagicMock()
mock_buffer.name = os.path.realpath( 'filename' )
mock_buffer.number = 1
mock_buffer.__iter__.return_value = [ u'abc', ToBytes( u'fДa' ) ]
with patch( 'vim.buffers', [ mock_buffer ] ):
assert_that( vimsupport.GetUnsavedAndCurrentBufferData(),
has_entry( mock_buffer.name,
has_entry( u'contents', u'abc\nfДa\n' ) ) )
def CanComplete():
"""Returns whether it's appropriate to provide any completion at the current
line and column."""
try:
line, column = LineAndColumnAfterLastNonWhitespace()
except TypeError:
return False
if (line, column) == CurrentLineAndColumn():
return True
return (ToBytes(vim.current.buffer[line][column - 1])
in potential_hint_triggers)
def __init__( self, request, validate = True ):
if validate:
EnsureRequestValid( request )
self._request = request
# Maps the keys returned by this objects __getitem__ to a # tuple of
# ( getter_method, setter_method ). Values computed by getter_method (or set
# by setter_method) are cached in _cached_computed. setter_method may be
# None for read-only items.
self._computed_key = {
# Unicode string representation of the current line. If the line requested
# is not in the file, returns ''.
'line_value': ( self._CurrentLine, None ),
# The calculated start column, as a codepoint offset into the
# unicode string line_value
'start_codepoint': ( self._GetCompletionStartCodepoint,
self._SetCompletionStartCodepoint ),
# The 'column_num' as a unicode codepoint offset
'column_codepoint': ( lambda: ByteOffsetToCodepointOffset(
self[ 'line_bytes' ],
self[ 'column_num' ] ),
None ),
# Bytes string representation of the current line
'line_bytes': ( lambda: ToBytes( self[ 'line_value' ] ),
None ),
# The calculated start column, as a byte offset into the UTF-8 encoded
# bytes returned by line_bytes
'start_column': ( self._GetCompletionStartColumn,
self._SetCompletionStartColumn ),
# Note: column_num is the byte offset into the UTF-8 encoded bytes
# returned by line_bytes
# unicode string representation of the 'query' after the beginning
# of the identifier to be completed
'query': ( self._Query, None ),
# Unicode string representation of the line value up to the character
# before the start of 'query'
'prefix': ( self._Prefix, None ),
'filetypes': ( self._Filetypes, None ),
'first_filetype': ( self._FirstFiletype, None ),
'force_semantic': ( self._GetForceSemantic, None ),
'lines': ( self._CurrentLines, None )
}
self._cached_computed = {}
def StartColumn_UnicodeNotIdentifier_test():
contents = "var x = '†es†ing'."
# † not considered an identifier character
for i in range( 13, 15 ):
print( ToBytes( contents )[ i - 1 : i ] )
eq_( 13,
RequestWrap( PrepareJson( column_num = i,
contents = contents ) )[ 'start_column' ] )
eq_( 13,
RequestWrap( PrepareJson( column_num = 15,
contents = contents ) )[ 'start_column' ] )
for i in range( 18, 20 ):
print( ToBytes( contents )[ i - 1 : i ] )
eq_( 18,
RequestWrap( PrepareJson( column_num = i,
contents = contents ) )[ 'start_column' ] )
def _ExtraHeaders(self, method, handler, body):
if not body:
body = bytes()
hmac = hmac_utils.CreateRequestHmac(method, handler, body,
self._hmac_secret)
final_hmac_value = native(ToBytes(binascii.hexlify(hmac)))
extra_headers = {'content-type': 'application/json'}
extra_headers[RACERD_HMAC_HEADER] = final_hmac_value
return extra_headers
def SnappedLineAndColumn():
"""Will return CurrentLineAndColumn(), except when there's solely whitespace
between caret and a potential hint trigger, where it "snaps to trigger",
returning hint trigger's line and column instead."""
try:
line, column = LineAndColumnAfterLastNonWhitespace()
except TypeError:
return CurrentLineAndColumn()
if (ToBytes(vim.current.buffer[line][column - 1])
in potential_hint_triggers):
return (line, column)
return CurrentLineAndColumn()
def ConvertCompletionDataToVimData(completion_data):
# see :h complete-items for a description of the dictionary fields
vim_data = {
'word': '',
'dup': 1,
'empty': 1,
}
if ('extra_data' in completion_data
and 'doc_string' in completion_data['extra_data']):
doc_string = ToBytes(completion_data['extra_data']['doc_string'])
else:
doc_string = ""
if 'insertion_text' in completion_data:
vim_data['word'] = ToBytes(completion_data['insertion_text'])
if 'menu_text' in completion_data:
vim_data['abbr'] = ToBytes(completion_data['menu_text'])
if 'extra_menu_info' in completion_data:
vim_data['menu'] = ToBytes(completion_data['extra_menu_info'])
if 'kind' in completion_data:
kind = ToUnicode(completion_data['kind'])
if kind:
vim_data['kind'] = ToBytes(kind[0].lower())
if 'detailed_info' in completion_data:
vim_data['info'] = ToBytes(completion_data['detailed_info'])
if doc_string:
vim_data['info'] += '\n' + doc_string
elif doc_string:
vim_data['info'] = doc_string
return vim_data
def _StartServer(self):
with self._server_state_lock:
port = utils.GetUnusedLocalhostPort()
self._hmac_secret = self._CreateHmacSecret()
# racerd will delete the secret_file after it's done reading it
with tempfile.NamedTemporaryFile(delete=False) as secret_file:
secret_file.write(self._hmac_secret)
args = [
self._racerd, 'serve', '--port',
str(port), '-l', '--secret-file', secret_file.name
]
# Enable logging of crashes
env = os.environ.copy()
SetEnviron(env, 'RUST_BACKTRACE', '1')
if self._rust_source_path:
args.extend(['--rust-src-path', self._rust_source_path])
filename_format = p.join(utils.PathToCreatedTempDir(),
'racerd_{port}_{std}.log')
self._server_stdout = filename_format.format(port=port,
std='stdout')
self._server_stderr = filename_format.format(port=port,
std='stderr')
with open(self._server_stderr, 'w') as fstderr:
with open(self._server_stdout, 'w') as fstdout:
self._racerd_phandle = utils.SafePopen(args,
stdout=fstdout,
stderr=fstderr,
env=env)
self._racerd_host = ToBytes('http://127.0.0.1:{0}'.format(port))
if not self.ServerIsRunning():
raise RuntimeError('Failed to start racerd!')
_logger.info(ToBytes(b'Racerd started on: ') + self._racerd_host)
def ReplaceChunk_MultipleLinesToSameMultipleLines_test():
result_buffer = [ ToBytes( "aAa" ),
ToBytes( "aBa" ),
ToBytes( "aCa" ),
ToBytes( "aDe" ) ]
start, end = _BuildLocations( 2, 2, 3, 2 )
( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Eb\nbF',
0, 0, result_buffer )
expected_buffer = [ ToBytes( "aAa" ),
ToBytes( "aEb" ),
ToBytes( "bFCa" ),
ToBytes( "aDe" ) ]
eq_( expected_buffer, result_buffer )
eq_( line_offset, 0 )
eq_( char_offset, 1 )
def ReplaceChunk(start,
end,
replacement_text,
line_delta,
char_delta,
vim_buffer,
locations=None):
# ycmd's results are all 1-based, but vim's/python's are all 0-based
# (so we do -1 on all of the values)
start_line = start['line_num'] - 1 + line_delta
end_line = end['line_num'] - 1 + line_delta
source_lines_count = end_line - start_line + 1
start_column = start['column_num'] - 1 + char_delta
end_column = end['column_num'] - 1
if source_lines_count == 1:
end_column += char_delta
# NOTE: replacement_text is unicode, but all our offsets are byte offsets,
# so we convert to bytes
replacement_lines = ToBytes(replacement_text).splitlines(False)
if not replacement_lines:
replacement_lines = [bytes(b'')]
replacement_lines_count = len(replacement_lines)
end_existing_text = vim_buffer[end_line][end_column:]
start_existing_text = vim_buffer[start_line][:start_column]
new_char_delta = (len(replacement_lines[-1]) - (end_column - start_column))
if replacement_lines_count > 1:
new_char_delta -= start_column
replacement_lines[0] = start_existing_text + replacement_lines[0]
replacement_lines[-1] = replacement_lines[-1] + end_existing_text
vim_buffer[start_line:end_line + 1] = replacement_lines[:]
if locations is not None:
locations.append({
'bufnr': vim_buffer.number,
'filename': vim_buffer.name,
# line and column numbers are 1-based in qflist
'lnum': start_line + 1,
'col': start_column + 1,
'text': replacement_text,
'type': 'F',
})
new_line_delta = replacement_lines_count - source_lines_count
return (new_line_delta, new_char_delta)
def _GetResponse( self, handler, parameters = {}, timeout = None ):
""" Handle communication with server """
target = urljoin( self._ServerLocation(), handler )
LOGGER.debug( 'TX (%s): %s', handler, parameters )
try:
response = urllib.request.urlopen(
target,
data = ToBytes( json.dumps( parameters ) ),
timeout = timeout )
response = json.loads( response.read() )
except urllib.error.HTTPError as response:
response = json.loads( response.fp.read() )
LOGGER.debug( 'RX: %s', response )
return response
def ComputeCandidatesInner_test( completer, execute_command ):
# Col 40 corresponds to cursor at ..., log.Prefi^x ...
result = completer.ComputeCandidatesInner( BuildRequest( 10, 40 ) )
execute_command.assert_called_once_with(
[ DUMMY_BINARY, '-sock', 'tcp', '-addr', completer._gocode_address,
'-f=json', 'autocomplete', PATH_TO_TEST_FILE, '287' ],
contents = ToBytes( ReadFile( PATH_TO_TEST_FILE ) ) )
eq_( result, [ {
'menu_text': u'Prefix',
'insertion_text': u'Prefix',
'extra_menu_info': u'func() string',
'detailed_info': u'Prefix func() string func',
'kind': u'func'
} ] )
def ToBytesOnPY2(data):
# To test the omnifunc, etc. returning strings, which can be of different
# types depending on python version, we use ToBytes on PY2 and just the native
# str on python3. This roughly matches what happens between py2 and py3
# versions within Vim.
if not PY2:
return data
if isinstance(data, list):
return [ToBytesOnPY2(item) for item in data]
if isinstance(data, dict):
for item in data:
data[item] = ToBytesOnPY2(data[item])
return data
return ToBytes(data)
def ReplaceChunk_SingleToMultipleLinesReplace_test():
result_buffer = [ ToBytes( "aAa" ), ToBytes( "aBa" ), ToBytes( "aCa" ) ]
start, end = _BuildLocations( 1, 2, 1, 4 )
( line_offset, char_offset ) = vimsupport.ReplaceChunk( start,
end,
'Eb\nbFb\nbGb',
0,
0,
result_buffer )
expected_buffer = [ ToBytes( "aEb" ),
ToBytes( "bFb" ),
ToBytes( "bGb" ),
ToBytes( "aBa" ),
ToBytes( "aCa" ) ]
eq_( expected_buffer, result_buffer )
eq_( line_offset, 2 )
eq_( char_offset, 0 )
def wrapper( *args, **kwargs ):
if not HostHeaderCorrect( request ):
self._logger.info( 'Dropping request with bad Host header.' )
abort( http.client.UNAUTHORIZED, 'Unauthorized, received bad Host header.' )
return
body = ToBytes( request.body.read() )
if not RequestAuthenticated( request.method, request.path, body,
self._hmac_secret ):
self._logger.info( 'Dropping request with bad HMAC.' )
abort( http.client.UNAUTHORIZED, 'Unauthorized, received bad HMAC.' )
return
body = callback( *args, **kwargs )
SetHmacHeader( body, self._hmac_secret )
return body
def _ComputeOffset( contents, line, col ):
contents = ToBytes( contents )
curline = 1
curcol = 1
newline = bytes( b'\n' )[ 0 ]
for i, byte in enumerate( contents ):
if curline == line and curcol == col:
return i
curcol += 1
if byte == newline:
curline += 1
curcol = 1
_logger.error( 'GoCode completer - could not compute byte offset ' +
'corresponding to L%i C%i', line, col )
return -1
def LineAndColumnNumbersClamped(line_num, column_num):
new_line_num = line_num
new_column_num = column_num
max_line = len(vim.current.buffer)
if line_num and line_num > max_line:
new_line_num = max_line
# Vim buffers are a list of byte objects on Python 2 but Unicode objects on
# Python 3.
max_column = len(ToBytes(vim.current.buffer[new_line_num - 1]))
if column_num and column_num > max_column:
new_column_num = max_column
return new_line_num, new_column_num
def ReplaceChunk_SingleLineToMultipleLinesOffsetWorks_test():
result_buffer = [ ToBytes( "aAa" ),
ToBytes( "aBa" ),
ToBytes( "aCa" ) ]
start, end = _BuildLocations( 1, 1, 1, 2 )
( line_offset, char_offset ) = vimsupport.ReplaceChunk( start, end, 'Db\nE',
1, 1, result_buffer )
expected_buffer = [ ToBytes( "aAa" ),
ToBytes( "aDb" ),
ToBytes( "Ea" ),
ToBytes( "aCa" ) ]
eq_( expected_buffer, result_buffer )
eq_( line_offset, 1 )
eq_( char_offset, -1 )
def _ComputeOffset(contents, line, column):
"""Compute the byte offset in the file given the line and column."""
contents = ToBytes(contents)
current_line = 1
current_column = 1
newline = bytes(b'\n')[0]
for i, byte in enumerate(contents):
if current_line == line and current_column == column:
return i
current_column += 1
if byte == newline:
current_line += 1
current_column = 1
message = COMPUTE_OFFSET_ERROR_MESSAGE.format(line=line, column=column)
_logger.error(message)
raise RuntimeError(message)
def wrapper( *args, **kwargs ):
if not HostHeaderCorrect( request ):
LOGGER.info( 'Dropping request with bad Host header' )
abort( requests.codes.unauthorized,
'Unauthorized, received bad Host header.' )
return
body = ToBytes( request.body.read() )
if not RequestAuthenticated( request.method, request.path, body,
self._hmac_secret ):
LOGGER.info( 'Dropping request with bad HMAC' )
abort( requests.codes.unauthorized, 'Unauthorized, received bad HMAC.' )
return
body = callback( *args, **kwargs )
SetHmacHeader( body, self._hmac_secret )
return body
