def __init__(
self,
yacc_optimize=True,
yacc_table="xonsh.completion_parser_table",
debug=False,
outputdir=None,
):
self.cursor = 0
self.current_input = ""
self.line_indices = ()
self.paren_counts = defaultdict(int)
self.error = None
self.debug = debug
self.lexer = Lexer(tolerant=True)
self.tokens = tuple(self.used_tokens | self.artificial_tokens)
yacc_kwargs = dict(
module=self,
debug=debug,
optimize=yacc_optimize,
tabmodule=yacc_table,
)
if not debug:
yacc_kwargs["errorlog"] = yacc.NullLogger()
if outputdir is None:
outputdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
yacc_kwargs["outputdir"] = outputdir
# create parser on main thread, it's small and should be fast
self.parser = yacc.yacc(**yacc_kwargs)
def check_token(inp, exp):
l = Lexer()
l.input(inp)
obs = list(l)
if len(obs) != 1:
msg = "The observed sequence does not have length-1: {0!r} != 1\n"
msg += "# obs\n{1}"
pytest.fail(msg.format(len(obs), pformat(obs)))
return assert_token_equal(exp, obs[0])
def check_token(inp, exp):
l = Lexer()
l.input(inp)
obs = list(l)
if len(obs) != 1:
msg = 'The observed sequence does not have length-1: {0!r} != 1\n'
msg += '# obs\n{1}'
pytest.fail(msg.format(len(obs), pformat(obs)))
return assert_token_equal(exp, obs[0])
def check_token(input, exp):
l = Lexer()
l.input(input)
obs = list(l)
if len(obs) != 1:
msg = 'The observed sequence does not have length-1: {0!r} != 1\n'
msg += '# obs\n{1}'
raise AssertionError(msg.format(len(obs), pformat(obs)))
assert_token_equal(exp, obs[0])
def check_token(inp, exp):
l = Lexer()
l.input(inp)
obs = list(l)
if len(obs) != 1:
msg = 'The observed sequence does not have length-1: {0!r} != 1\n'
msg += '# obs\n{1}'
raise AssertionError(msg.format(len(obs), pformat(obs)))
return assert_token_equal(exp, obs[0])
def test_redir_whitespace(case):
inp = "![{}/path/to/file]".format(case)
l = Lexer()
l.input(inp)
obs = list(l)
assert obs[2].type == "WS"
def check_tokens_subproc(inp, exp):
l = Lexer()
l.input('$[{}]'.format(inp))
obs = list(l)[1:-1]
return assert_tokens_equal(exp, obs)
def check_tokens(inp, exp):
l = Lexer()
l.input(inp)
obs = list(l)
return assert_tokens_equal(exp, obs)
class CompletionContextParser:
"""A parser to construct a completion context."""
used_tokens = {
"STRING",
}
paren_pairs = (
("DOLLAR_LPAREN", "RPAREN"), # $()
("BANG_LPAREN", "RPAREN"), # !()
("ATDOLLAR_LPAREN", "RPAREN"), # @$()
("DOLLAR_LBRACKET", "RBRACKET"), # $[]
("BANG_LBRACKET", "RBRACKET"), # ![]
# python sub-expression:
("AT_LPAREN", "RPAREN"), # @()
)
r_parens = {right for _, right in paren_pairs}
l_to_r_parens = {left: right for left, right in paren_pairs}
used_tokens.update(left for left, _ in paren_pairs)
used_tokens.update(right for _, right in paren_pairs)
multi_tokens = {
# multiple commands
"SEMI", # ;
"NEWLINE",
"PIPE",
"AND",
"OR",
}
used_tokens |= multi_tokens
artificial_tokens = {"ANY"}
ignored_tokens = {"INDENT", "DEDENT", "WS"}
def __init__(
self,
yacc_optimize=True,
yacc_table="xonsh.completion_parser_table",
debug=False,
outputdir=None,
):
self.cursor = 0
self.current_input = ""
self.line_indices = ()
self.paren_counts = defaultdict(int)
self.error = None
self.debug = debug
self.lexer = Lexer(tolerant=True)
self.tokens = tuple(self.used_tokens | self.artificial_tokens)
yacc_kwargs = dict(
module=self,
debug=debug,
optimize=yacc_optimize,
tabmodule=yacc_table,
)
if not debug:
yacc_kwargs["errorlog"] = yacc.NullLogger()
if outputdir is None:
outputdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
yacc_kwargs["outputdir"] = outputdir
# create parser on main thread, it's small and should be fast
self.parser = yacc.yacc(**yacc_kwargs)
def parse(
self,
multiline_text: str,
cursor_index: int,
ctx: Optional[Dict[str, Any]] = None,
) -> Optional[CompletionContext]:
"""Returns a CompletionContext from a command line.
Parameters
----------
multiline_text : str
The complete multiline text.
cursor_index : int
The current cursor's index in the multiline text.
"""
self.cursor = cursor_index
self.current_input = multiline_text
self.line_indices = (0,) + tuple(
match.start() + 1 for match in NEWLINE_RE.finditer(multiline_text)
)
self.paren_counts.clear()
self.error = None
try:
assert self.cursor_in_span(
slice(0, len(multiline_text))
), f"Bad cursor index: {cursor_index}"
context: Optional[CompletionContext] = self.parser.parse(
input=multiline_text, lexer=self, debug=1 if self.debug else 0
)
except (SyntaxError, AssertionError):
if self.debug:
raise
context = None
if self.debug and self.error is not None:
raise self.error
if context and ctx is not None:
context = context.with_ctx(ctx)
return context
# Tokenizer:
def input(self, s):
return self.lexer.input(s)
def token(self):
"""Simulate some lexer properties for the parser:
* skip tokens from ``ignored_tokens``.
* make ``lexpos`` absolute instead of per line.
* set tokens that aren't in ``used_tokens`` to type ``ANY``.
* handle a weird lexer behavior with ``AND``/``OR``.
* set multi_tokens with cursor to type ``ANY``.
* set mismatched closing parens to type ``ANY``.
The paren checking is needed since accepting both matched and unmatched parenthesis isn't possible with an LALR(1) parser.
See https://stackoverflow.com/questions/8496065/why-is-this-lr1-grammar-not-lalr1
"""
while True:
tok = self.lexer.token()
if tok is None:
return tok
if tok.type in self.ignored_tokens:
continue
lineno = tok.lineno - 1 # tok.lineno is 1-indexed
assert lineno < len(
self.line_indices
), f"Invalid lexer state for token {tok} - bad lineno"
tok.lexpos = lexpos = self.line_indices[lineno] + tok.lexpos
if tok.type in self.multi_tokens:
# for some reason the lexer simulates ``and`` / ``or`` values for ``&&` / ``||``
if (
tok.type == "AND"
and self.current_input[lexpos : lexpos + 2] == "&&"
):
tok.value = "&&"
elif (
tok.type == "OR" and self.current_input[lexpos : lexpos + 2] == "||"
):
tok.value = "||"
# if the cursor is inside this token, set it to type ``ANY``
outer_span = slice(lexpos, lexpos + len(tok.value))
inner_span = slice(outer_span.start + 1, outer_span.stop)
if self.cursor_in_span(inner_span) or (
# the cursor is in a space-separated multi keyword.
# even if the cursor's at the edge, the keyword should be considered as a normal arg
tok.value in ("and", "or")
and self.cursor_in_span(outer_span)
):
tok.type = "ANY"
# parentheses handling
elif tok.type in self.l_to_r_parens:
self.paren_counts[self.l_to_r_parens[tok.type]] += 1
elif self.paren_counts.get(tok.type):
self.paren_counts[tok.type] -= 1
elif tok.type in self.r_parens:
# tok.type is not in self.paren_counts, meaning this right paren is unmatched
tok.type = "ANY"
if tok.type in self.used_tokens:
return tok
tok.type = "ANY"
return tok
# Grammar:
def p_context_command(self, p):
"""context : command
| commands
"""
spanned: Union[Spanned[CommandContext], Commands] = p[1]
# expand the commands to the complete input
complete_span = slice(0, len(self.current_input))
spanned = self.try_expand_span(spanned, complete_span) or spanned
context = spanned.cursor_context
if isinstance(context, CommandContext):
# if the context is the main command, it might be python code
context_is_main_command = False
if isinstance(spanned.value, list):
for command in spanned.value:
if context is command.value:
# TODO: False for connecting keywords other than '\n' and ';'
context_is_main_command = True
break
else:
if context is spanned.value:
context_is_main_command = True
if context_is_main_command:
python_context = PythonContext(
multiline_code=self.current_input,
cursor_index=self.cursor,
)
p[0] = CompletionContext(command=context, python=python_context)
else:
p[0] = CompletionContext(command=context)
elif isinstance(context, PythonContext):
# the cursor is in a python sub expression `@()`, so it must be python
p[0] = CompletionContext(python=context)
else:
if self.debug:
self.error = SyntaxError(f"Failed to find cursor context in {spanned}")
p[0] = None
def p_command(self, p):
"""command : args
|
"""
if len(p) == 2:
spanned_args: List[Spanned[CommandArg]] = p[1]
span = slice(spanned_args[0].span.start, spanned_args[-1].span.stop)
else:
# empty command
spanned_args = []
span = EMPTY_SPAN # this will be expanded in expand_command_span
args = tuple(arg.value for arg in spanned_args)
cursor_context: Optional[Union[CommandContext, PythonContext]] = None
context = CommandContext(args, arg_index=-1)
if self.cursor_in_span(span):
for arg_index, arg in enumerate(spanned_args):
if self.cursor < arg.span.start:
# an empty arg that will be inserted into arg_index
context = CommandContext(args, arg_index)
break
if self.cursor_in_span(arg.span):
context, cursor_context = self.handle_command_arg(arg)
context = context._replace(
args=args[:arg_index] + args[arg_index + 1 :],
arg_index=arg_index,
)
break
if cursor_context is None and context.arg_index != -1:
cursor_context = context
p[0] = Spanned(
context,
span,
cursor_context,
expansion_obj=spanned_args[-1] if spanned_args else None,
)
@staticmethod
def p_multiple_commands_first(p):
"""commands : command"""
command: Spanned[CommandContext] = p[1]
p[0] = Spanned(
[command],
command.span,
cursor_context=command.cursor_context,
)
@with_docstr(
f"""commands : {RULES_SEP.join(f"commands {kwd} command" for kwd in multi_tokens)}"""
)
def p_multiple_commands_many(self, p):
# commands KWD command
commands: Commands = p[1]
kwd_index = 2
command: Spanned[CommandContext] = p[3]
# expand commands span
kwd_start = p.lexpos(kwd_index)
commands = self.try_expand_right(commands, kwd_start) or commands
# expand command
kwd_stop = kwd_start + len(p[kwd_index])
command = self.try_expand_left(command, kwd_stop) or command
commands.value.append(command)
expansion_obj = command
if command.cursor_context is not None:
cursor_context = command.cursor_context
else:
cursor_context = commands.cursor_context
commands = commands.replace(
span=slice(commands.span.start, expansion_obj.span.stop),
cursor_context=cursor_context,
expansion_obj=expansion_obj,
)
p[0] = commands
@with_docstr(
f"""sub_expression : {RULES_SEP.join(f"{l} commands {r}" for l, r in paren_pairs)}
| {RULES_SEP.join(f"{l} commands" for l, _ in paren_pairs)}
"""
)
def p_sub_expression(self, p):
sub_expr_opening = p[1]
outer_start = p.lexpos(1)
inner_start = outer_start + len(sub_expr_opening)
commands: Commands
if len(p) == 4:
# LPAREN commands RPAREN
commands = p[2]
inner_stop = p.lexpos(3)
outer_stop = inner_stop + len(p[3])
closed_parens = True
else:
# LPAREN commands
commands = p[2]
if commands.span is EMPTY_SPAN: # an empty command without a location
inner_stop = outer_stop = inner_start
else:
inner_stop = outer_stop = commands.span.stop
closed_parens = False
inner_span = slice(inner_start, inner_stop)
outer_span = slice(outer_start, outer_stop)
commands = self.try_expand_span(commands, inner_span) or commands
if len(commands.value) == 1:
# if this is a single command, set it's subcmd_opening attribute
single_command = commands.value[0]
new_value: CommandContext = single_command.value._replace(
subcmd_opening=sub_expr_opening
)
if commands.cursor_context is single_command.value:
single_command = single_command.replace(cursor_context=new_value)
commands = commands.replace(cursor_context=new_value)
commands.value[0] = single_command.replace(value=new_value)
if sub_expr_opening == "@(":
# python sub-expression
python_context = PythonContext(
self.current_input[inner_span],
self.cursor - inner_span.start,
is_sub_expression=True,
)
if commands.cursor_context is not None and not any(
command.value == commands.cursor_context for command in commands.value
):
# the cursor is inside an inner arg
cursor_context = commands.cursor_context
elif self.cursor_in_span(inner_span):
# the cursor is in the python expression
cursor_context = python_context
else:
cursor_context = None
if (
len(commands.value)
and commands.value[-1].expansion_obj is not None
and self.is_command_or_commands(
commands.value[-1].expansion_obj.expansion_obj
)
):
# the last arg (in the last command) is a subcommand, e.g. `@(a; x = $(echo `
expansion_obj = commands.value[-1].expansion_obj.expansion_obj
else:
expansion_obj = None
p[0] = Spanned(python_context, outer_span, cursor_context, expansion_obj)
else:
p[0] = commands.replace(span=outer_span)
if closed_parens:
p[0] = p[0].replace(expansion_obj=ExpansionOperation.NEVER_EXPAND)
def p_sub_expression_arg(self, p):
"""arg : sub_expression"""
p[0] = self.sub_expression_arg(p[1])
@with_docstr(
f"""arg : {RULES_SEP.join({"ANY"} | used_tokens - multi_tokens - r_parens)}"""
)
def p_any_token_arg(self, p):
raw_arg: str = p[1]
start = p.lexpos(1)
stop = start + len(raw_arg)
span = slice(start, stop)
# handle line continuations
raw_arg, relative_cursor = self.process_string_segment(raw_arg, span)
arg = CompletionContextParser.try_parse_string_literal(raw_arg)
if arg is None:
arg = CommandArg(raw_arg)
p[0] = Spanned(arg, span, cursor_context=relative_cursor)
@staticmethod
def p_args_first(p):
"""args : arg"""
p[0] = [p[1]]
def p_args_many(self, p):
"""args : args arg"""
args: List[Spanned[CommandArg]] = p[1]
new_arg: Spanned[CommandArg] = p[2]
last_arg: Spanned[CommandArg] = args[-1]
in_between_span = slice(last_arg.span.stop, new_arg.span.start)
in_between = self.current_input[in_between_span]
# handle line continuations between these args
in_between, relative_cursor = self.process_string_segment(
in_between, in_between_span
)
joined_raw = f"{last_arg.value.raw_value}{in_between}{new_arg.value.raw_value}"
string_literal = self.try_parse_string_literal(joined_raw)
if string_literal is not None or not in_between:
if string_literal is not None:
# we're appending to a partial string, e.g. `"a b`
arg = string_literal
else:
# these args are adjacent and didn't match other rules, e.g. `a"b"`
arg = CommandArg(joined_raw)
# select which context to preserve
cursor_context = None
if relative_cursor is not None:
# the cursor is in between
cursor_context = len(last_arg.value.raw_value) + relative_cursor
elif last_arg.cursor_context is not None:
# the cursor is in the last arg
cursor_context = last_arg.cursor_context
elif new_arg.cursor_context is not None:
# the cursor is in the new arg
if isinstance(new_arg.cursor_context, int):
# the context is a relative cursor
cursor_context = (
len(last_arg.value.raw_value)
+ len(in_between)
+ new_arg.cursor_context
)
else:
cursor_context = new_arg.cursor_context
args[-1] = Spanned(
value=arg,
span=slice(last_arg.span.start, new_arg.span.stop),
cursor_context=cursor_context,
)
else:
args.append(new_arg)
p[0] = args
def p_error(self, p):
if p is None:
raise_parse_error("no further code")
raise_parse_error(
"code: {0}".format(p.value),
Location("input", p.lineno, p.lexpos - self.line_indices[p.lineno - 1]),
self.current_input,
self.current_input.splitlines(keepends=True),
)
# Utils:
def try_expand_right(self, obj: Exp, new_right: int) -> Optional[Exp]:
if obj.span is EMPTY_SPAN:
new_span = slice(new_right, new_right)
else:
new_span = slice(obj.span.start, new_right)
return self.try_expand_span(obj, new_span)
def try_expand_left(self, obj: Exp, new_left: int) -> Optional[Exp]:
if obj.span is EMPTY_SPAN:
new_span = slice(new_left, new_left)
else:
new_span = slice(new_left, obj.span.stop)
return self.try_expand_span(obj, new_span)
def try_expand_span(self, obj: Exp, new_span: slice) -> Optional[Exp]:
if obj.span.start <= new_span.start and new_span.stop <= obj.span.stop:
# the new span doesn't expand the old one
if obj.span is not EMPTY_SPAN:
# EMPTY_SPAN is a special value for an empty element that isn't yet located anywhere
return obj
if obj.expansion_obj is ExpansionOperation.NEVER_EXPAND:
return None
elif isinstance(obj.value, CommandArg):
return self.try_expand_arg_span(obj, new_span)
elif isinstance(obj.value, CommandContext):
return self.expand_command_span(obj, new_span)
elif isinstance(obj.value, list):
# obj is multiple commands
return self.expand_commands_span(cast(Commands, obj), new_span)
elif isinstance(obj.value, PythonContext):
return self.try_expand_python_context(obj, new_span)
return None
def expand_command_span(
self, command: Spanned[CommandContext], new_span: slice
) -> Spanned[CommandContext]:
"""This is used when we know the command's real span is larger
For example, only when we're done parsing ` echo hi`, we know the head whitespace is also part of the command.
"""
is_empty_command = (
command.span is EMPTY_SPAN
) # special span for an empty command in an unknown location
new_arg_index = None
if command.cursor_context is None and self.cursor_in_span(new_span):
# the cursor is in the expanded span
if is_empty_command or self.cursor < command.span.start:
new_arg_index = 0
elif self.cursor > command.span.stop:
new_arg_index = len(command.value.args)
if command.expansion_obj is not None:
# this command has a last argument that we should try to expand
assert isinstance(command.expansion_obj, Spanned) and isinstance(
command.expansion_obj.value, CommandArg
)
last_arg = cast(Spanned[CommandArg], command.expansion_obj)
expanded_arg = self.try_expand_right(last_arg, new_span.stop)
if expanded_arg is not None:
# arg was expanded successfully!
new_context, new_cursor_context = self.handle_command_arg(
expanded_arg
)
old_args = command.value.args
new_context = new_context._replace(
args=old_args[:-1],
arg_index=new_arg_index - 1,
subcmd_opening=command.value.subcmd_opening,
)
if new_cursor_context is None:
new_cursor_context = new_context
return Spanned(
value=new_context,
span=new_span,
cursor_context=new_cursor_context,
expansion_obj=expanded_arg,
)
# if the arg can't be expanded, the cursor just adds a new empty arg
if new_arg_index is not None:
new_context = command.value._replace(arg_index=new_arg_index)
return Spanned(value=new_context, span=new_span, cursor_context=new_context)
return command.replace(span=new_span)
def expand_commands_span(self, commands: Commands, new_span: slice) -> Commands:
"""Like expand_command_span, but for multiple commands - expands the first command and the last command."""
cursor_context = commands.cursor_context
is_empty_command = commands.span is EMPTY_SPAN
if is_empty_command or new_span.start < commands.span.start:
# expand first command
first_command: Spanned[CommandContext] = commands.value[0]
commands.value[0] = first_command = (
self.try_expand_left(first_command, new_span.start) or first_command
)
if first_command.cursor_context is not None:
cursor_context = first_command.cursor_context
if is_empty_command or new_span.stop > commands.span.stop:
# expand last command
last_command: Spanned[CommandContext] = commands.value[-1]
commands.value[-1] = last_command = (
self.try_expand_right(last_command, new_span.stop) or last_command
)
if last_command.cursor_context is not None:
cursor_context = last_command.cursor_context
return commands.replace(span=new_span, cursor_context=cursor_context)
def try_expand_arg_span(
self, arg: Spanned[CommandArg], new_span: slice
) -> Optional[Spanned[CommandArg]]:
"""Try to expand the arg to a new span. This will return None if the arg can't be expanded to the new span.
For example, expanding `"hi ` will work since the added whitespace is part of the arg, but `"hi" ` won't work.
Similarly, `$(hi ` can be expanded but `$(nice) ` can't.
"""
if arg.expansion_obj is ExpansionOperation.SIMPLE_ARG_EXPANSION.value:
# this is a simple textual arg
added_span = slice(arg.span.stop, new_span.stop)
added_text = self.current_input[added_span]
# handle line continuations between these args
added_text, relative_cursor = self.process_string_segment(
added_text, added_span
)
joined_raw = arg.value.raw_value + added_text
string_literal = self.try_parse_string_literal(joined_raw)
if string_literal is None:
return None
cursor_context = None
if arg.cursor_context is not None:
cursor_context = arg.cursor_context
elif relative_cursor is not None:
# the cursor is in the whitespace
cursor_context = len(arg.value.raw_value) + relative_cursor
return Spanned(string_literal, new_span, cursor_context)
elif isinstance(arg.expansion_obj, Spanned):
assert self.is_command_or_commands(arg.expansion_obj) or self.is_python(
arg.expansion_obj
)
sub_expr = cast(ArgContext, arg.expansion_obj)
# this arg is a subcommand or multiple subcommands, e.g. `$(a && b)`
expanded_obj: Optional[ArgContext] = self.try_expand_span(sub_expr, new_span) # type: ignore
if expanded_obj is None:
return None
return self.sub_expression_arg(expanded_obj)
else:
# this shouldn't happen
return None
def try_expand_python_context(
self, python_context: Spanned[PythonContext], new_span: slice
) -> Optional[Spanned[PythonContext]]:
added_span = slice(python_context.span.stop, new_span.stop)
added_code = self.current_input[added_span]
new_code = python_context.value.multiline_code + added_code
if python_context.cursor_context is None and self.cursor_in_span(added_span):
new_cursor_index = (
len(python_context.value.multiline_code)
+ self.cursor
- added_span.start
)
else:
new_cursor_index = python_context.value.cursor_index
new_python_context = python_context.value._replace(
multiline_code=new_code, cursor_index=new_cursor_index
)
if python_context.expansion_obj is not None:
# the last command is expandable
# if it were an `ExpansionOperation`, `try_expand` would caught it instead
expandable = cast(ExpandableObject, python_context.expansion_obj)
expanded_command: Optional[ExpandableObject] = self.try_expand_right(expandable, new_span.stop) # type: ignore
if (
expanded_command is not None
and expanded_command.cursor_context is not None
):
return python_context.replace(
value=new_python_context,
span=new_span,
cursor_context=expanded_command.cursor_context,
expansion_obj=expanded_command,
)
# the last command can't be expanded, but the python code is still valid
new_cursor_context: Optional[PythonContext] = None
if self.cursor_in_span(new_span):
new_cursor_context = new_python_context
return python_context.replace(
value=new_python_context, span=new_span, cursor_context=new_cursor_context
)
def handle_command_arg(
self, arg: Spanned[CommandArg]
) -> Tuple[CommandContext, Optional[Union[CommandContext, PythonContext]]]:
"""Create a command context from an arg which contains the cursor.
Also return the internal cursor context if it exists.
`args`, `arg_index`, and `subcmd_opening` aren't set by this function
and need to be set by the caller via `_replace`.
"""
assert self.cursor_in_span(arg.span)
prefix = suffix = opening_quote = closing_quote = ""
cursor_context = None
is_after_closing_quote = False
if self.cursor == arg.span.stop:
# cursor is at the end of this arg
if arg.cursor_context is not None and not isinstance(
arg.cursor_context, int
):
# this arg is already a context (e.g. a sub expression)
cursor_context = arg.cursor_context
elif arg.value.closing_quote:
# appending to a quoted string, e.g. `ls "C:\\Wind"`
is_after_closing_quote = True
opening_quote = arg.value.opening_quote
prefix = arg.value.value
closing_quote = arg.value.closing_quote
else:
# appending to a partial string, e.g. `ls "C:\\Wind`
prefix = arg.value.value
opening_quote = arg.value.opening_quote
elif self.cursor_in_span(arg.span):
if arg.cursor_context is not None and not isinstance(
arg.cursor_context, int
):
# this arg is already a context (e.g. a sub expression)
cursor_context = arg.cursor_context
else:
if arg.cursor_context is not None:
# this arg provides a relative cursor location
relative_location = arg.cursor_context
else:
relative_location = self.cursor - arg.span.start
raw_value = arg.value.raw_value
if relative_location < len(arg.value.opening_quote):
# the cursor is inside the opening quote
prefix = arg.value.opening_quote[:relative_location]
suffix = raw_value[relative_location:]
elif (
relative_location
>= len(arg.value.opening_quote) + len(arg.value.value) + 1
):
# the cursor is inside the closing quote
prefix = raw_value[:relative_location]
suffix = raw_value[relative_location:]
else:
# the cursor is inside the string
opening_quote = arg.value.opening_quote
closing_quote = arg.value.closing_quote
location_in_value = relative_location - len(opening_quote)
prefix = arg.value.value[:location_in_value]
suffix = arg.value.value[location_in_value:]
return (
CommandContext(
args=(),
arg_index=-1, # the caller needs to fill these
prefix=prefix,
suffix=suffix,
opening_quote=opening_quote,
closing_quote=closing_quote,
is_after_closing_quote=is_after_closing_quote,
),
cursor_context,
)
def sub_expression_arg(self, sub_expression: ArgContext) -> Spanned[CommandArg]:
value = self.current_input[sub_expression.span]
arg = sub_expression.replace(
value=CommandArg(value),
expansion_obj=sub_expression,
) # preserves the cursor_context and expansion_obj if it they exist
return arg
@staticmethod
def try_parse_string_literal(raw_arg: str) -> Optional[CommandArg]:
"""Try to parse this as a single string literal. can be partial
For example:
"wow"
"a b
'''a b 'c' "d"
"""
startix, endix, quote = check_for_partial_string(raw_arg)
if startix != 0 or endix not in (
None, # the arg doesn't start with a string literal
len(raw_arg), # the string literal ends in the middle of the arg
):
# xonsh won't treat it as a string literal
return None
else:
if endix is None:
# no closing quote
return CommandArg(raw_arg[len(quote) : endix], opening_quote=quote)
else:
closing_quote_len = quote.count('"') + quote.count("'")
return CommandArg(
value=raw_arg[len(quote) : -closing_quote_len],
closing_quote=raw_arg[-closing_quote_len:],
opening_quote=quote,
)
def process_string_segment(
self, string: str, span: slice
) -> Tuple[str, Optional[int]]:
"""Process a string segment:
1. Return a relative_cursor if it's inside the span (for ``Spanned.cursor_context``).
2. Handle line continuations in the string.
"""
relative_cursor = None
line_cont, replacement, diff = LINE_CONT_REPLACEMENT_DIFF
if self.cursor_in_span(span):
relative_cursor = self.cursor - span.start
relative_cursor += string.count(line_cont, 0, relative_cursor) * diff
string = string.replace(line_cont, replacement)
return string, relative_cursor
@staticmethod
def is_command_or_commands(obj: Any) -> bool:
if isinstance(obj, Spanned):
if isinstance(obj.value, CommandContext):
return True
if isinstance(obj.value, list) and len(obj.value):
first_element = obj.value[0]
if isinstance(first_element, Spanned) and isinstance(
first_element.value, CommandContext
):
return True
return False
@staticmethod
def is_python(obj: Any) -> bool:
return isinstance(obj, Spanned) and isinstance(obj.value, PythonContext)
def cursor_in_span(self, span: slice) -> bool:
"""Returns whether the cursor is in the span.
The edge is included (if `self.cursor`` == ``stop``).
"""
return span.start <= self.cursor <= span.stop
def check_tokens_subproc(inp, exp, stop=-1):
l = Lexer()
l.input("$[{}]".format(inp))
obs = list(l)[1:stop]
return assert_tokens_equal(exp, obs)
swap_values,
get_line_continuation,
get_logical_line,
replace_logical_line,
check_quotes,
deprecated,
is_writable_file,
balanced_parens,
iglobpath,
all_permutations,
)
from xonsh.environ import Env
from tools import skip_if_on_windows
LEXER = Lexer()
LEXER.build()
INDENT = " "
TOOLS_ENV = {"EXPAND_ENV_VARS": True, "XONSH_ENCODING_ERRORS": "strict"}
ENCODE_ENV_ONLY = {"XONSH_ENCODING_ERRORS": "strict"}
PATHEXT_ENV = {"PATHEXT": [".COM", ".EXE", ".BAT"]}
def test_subproc_toks_x():
exp = "![x]"
obs = subproc_toks("x", lexer=LEXER, returnline=True)
assert exp == obs
def check_tokens(input, exp):
l = Lexer()
l.input(input)
obs = list(l)
assert_tokens_equal(exp, obs)
def check_tokens(inp, exp):
l = Lexer()
l.input(inp)
obs = list(l)
return assert_tokens_equal(exp, obs)
def test_lexer_split(s, exp):
lexer = Lexer()
obs = lexer.split(s)
assert exp == obs
def test_redir_whitespace(case):
inp = '![{}/path/to/file]'.format(case)
l = Lexer()
l.input(inp)
obs = list(l)
assert obs[2].type == 'WS'
def lex_input(inp: str):
lex = Lexer()
lex.input(inp)
return list(lex)
def test_tolerant_lexer(s):
lexer = Lexer(tolerant=True)
lexer.input(s)
error_tokens = list(tok for tok in lexer if tok.type == "ERRORTOKEN")
assert all(tok.value in s for tok in error_tokens) # no error messages
def check_tokens_subproc(inp, exp):
l = Lexer()
l.input('$[{}]'.format(inp))
obs = list(l)[1:-1]
return assert_tokens_equal(exp, obs)
def test_lexer_split(s, exp):
lexer = Lexer()
obs = lexer.split(s)
assert exp == obs
# -*- coding: utf-8 -*-
"""Tests the xonsh lexer."""
from __future__ import unicode_literals, print_function
import os
import nose
from nose.tools import assert_equal, assert_true, assert_false
from xonsh.lexer import Lexer
from xonsh.tools import subproc_toks, subexpr_from_unbalanced, is_int, \
always_true, always_false, ensure_string, is_env_path, str_to_env_path, \
env_path_to_str, escape_windows_title_string, is_bool, to_bool, bool_to_str, \
ensure_int_or_slice, is_float, is_string, check_for_partial_string
LEXER = Lexer()
LEXER.build()
INDENT = ' '
def test_subproc_toks_x():
exp = '$[x]'
obs = subproc_toks('x', lexer=LEXER, returnline=True)
assert_equal(exp, obs)
def test_subproc_toks_ls_l():
exp = '$[ls -l]'
obs = subproc_toks('ls -l', lexer=LEXER, returnline=True)
assert_equal(exp, obs)
def test_subproc_toks_git():
s = 'git commit -am "hello doc"'
def check_tokens_subproc(inp, exp, stop=-1):
l = Lexer()
l.input("$[{}]".format(inp))
obs = list(l)[1:stop]
return assert_tokens_equal(exp, obs)
