def test_replacer(self):
value = object()
t1 = textbuilder.Text("To be or not\n to be?\n", value)
patches = make_regexp_patches(
t1.get_text(), re.compile(r'be|to', re.I), lambda m:
(m.group() + m.group()).upper())
t2 = textbuilder.Replacer(t1, patches)
self.assertEqual(t2.get_text(), "TOTO BEBE or not\n TOTO BEBE?\n")
self.assertEqual(
t2.map_back_patch(make_patch(t2.get_text(), 0, 4, "xxx")),
(t1.get_text(), value, Patch(0, 2, "To", "xxx")))
self.assertEqual(
t2.map_back_patch(make_patch(t2.get_text(), 5, 9, "xxx")),
(t1.get_text(), value, Patch(3, 5, "be", "xxx")))
self.assertEqual(
t2.map_back_patch(make_patch(t2.get_text(), 18, 23, "xxx")),
(t1.get_text(), value, Patch(14, 17, " to", "xxx")))
# Match the entire second line
self.assertEqual(
t2.map_back_patch(make_patch(t2.get_text(), 17, 29, "xxx")),
(t1.get_text(), value, Patch(13, 21, " to be?", "xxx")))
def test_combiner(self):
valueA, valueB = object(), object()
t1 = textbuilder.Text("To be or not\n to be?\n", valueA)
patches = make_regexp_patches(
t1.get_text(), re.compile(r'be|to', re.I), lambda m:
(m.group() + m.group()).upper())
t2 = textbuilder.Replacer(t1, patches)
t3 = textbuilder.Text("That is the question", valueB)
t4 = textbuilder.Combiner(["[", t2, t3, "]"])
self.assertEqual(
t4.get_text(),
"[TOTO BEBE or not\n TOTO BEBE?\nThat is the question]")
self.assertEqual(
t4.map_back_patch(make_patch(t4.get_text(), 1, 5, "xxx")),
(t1.get_text(), valueA, Patch(0, 2, "To", "xxx")))
self.assertEqual(
t4.map_back_patch(make_patch(t4.get_text(), 18, 30, "xxx")),
(t1.get_text(), valueA, Patch(13, 21, " to be?", "xxx")))
self.assertEqual(
t4.map_back_patch(make_patch(t4.get_text(), 0, 1, "xxx")), None)
self.assertEqual(
t4.map_back_patch(make_patch(t4.get_text(), 31, 38, "xxx")),
(t3.get_text(), valueB, Patch(0, 7, "That is", "xxx")))
def make_formula_body(formula, default_value, assoc_value=None):
"""
Given a formula, returns a textbuilder.Builder object suitable to be the body of a function,
with the formula transformed to replace `$foo` with `rec.foo`, and to insert `return` if
appropriate. Assoc_value is associated with textbuilder.Text() to be returned by map_back_patch.
"""
if isinstance(formula, six.binary_type):
formula = formula.decode('utf8')
if not formula.strip():
return textbuilder.Text('return ' + repr(default_value), assoc_value)
formula_builder_text = textbuilder.Text(formula, assoc_value)
# Start with a temporary builder, since we need to translate "$" before we can parse the code at
# all (namely, we turn '$foo' into 'DOLLARfoo' first). Once we can parse the code, we'll create
# a proper set of patches. Note that we initially translate into 'DOLLARfoo' rather than
# 'rec.foo', so that the translated entity is a single token: this makes for more precisely
# reported errors if there are any.
tmp_patches = textbuilder.make_regexp_patches(formula, DOLLAR_REGEX, 'DOLLAR')
tmp_formula = textbuilder.Replacer(formula_builder_text, tmp_patches)
# Parse the formula into an abstract syntax tree (AST), catching syntax errors.
try:
atok = asttokens.ASTTokens(tmp_formula.get_text(), parse=True)
except SyntaxError as e:
return textbuilder.Text(_create_syntax_error_code(tmp_formula, formula, e))
# Parse formula and generate error code on assignment to rec
with use_inferences(InferRecAssignment):
try:
astroid.parse(tmp_formula.get_text())
except SyntaxError as e:
return textbuilder.Text(_create_syntax_error_code(tmp_formula, formula, e))
# Once we have a tree, go through it and create a subset of the dollar patches that are actually
# relevant. E.g. this is where we'll skip the "$foo" patches that appear in strings or comments.
patches = []
for node in ast.walk(atok.tree):
if isinstance(node, ast.Name) and node.id.startswith('DOLLAR'):
input_pos = tmp_formula.map_back_offset(node.first_token.startpos)
m = DOLLAR_REGEX.match(formula, input_pos)
# If there is no match, then we must have had a "DOLLARblah" identifier that didn't come
# from translating a "$" prefix.
if m:
patches.append(textbuilder.make_patch(formula, m.start(0), m.end(0), 'rec.'))
# Wrap arguments to the top-level "IF()" function into lambdas, for lazy evaluation. This is
# to ensure it's not affected by an exception in the unused value, to match Excel behavior.
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
lazy_args_slice = LAZY_ARG_FUNCTIONS.get(node.func.id)
if lazy_args_slice:
for arg in node.args[lazy_args_slice]:
start, end = map(tmp_formula.map_back_offset, atok.get_text_range(arg))
patches.append(textbuilder.make_patch(formula, start, start, 'lambda: ('))
patches.append(textbuilder.make_patch(formula, end, end, ')'))
# If the last statement is an expression that has its result unused (an ast.Expr node),
# then insert a "return" keyword.
last_statement = atok.tree.body[-1] if atok.tree.body else None
if isinstance(last_statement, ast.Expr):
input_pos = tmp_formula.map_back_offset(last_statement.first_token.startpos)
patches.append(textbuilder.make_patch(formula, input_pos, input_pos, "return "))
elif last_statement is None:
# If we have an empty body (e.g. just a comment), add a 'pass' at the end.
patches.append(textbuilder.make_patch(formula, len(formula), len(formula), '\npass'))
# Apply the new set of patches to the original formula to get the real output.
final_formula = textbuilder.Replacer(formula_builder_text, patches)
# Try parsing again before returning it just in case we have new syntax errors. These are
# possible in cases when a single token ('DOLLARfoo') is valid but an expression ('rec.foo') is
# not, e.g. `foo($bar=1)` or `def $foo()`.
try:
atok = asttokens.ASTTokens(final_formula.get_text(), parse=True)
except SyntaxError as e:
return textbuilder.Text(_create_syntax_error_code(final_formula, formula, e))
# We return the text-builder object whose .get_text() is the final formula.
return final_formula
def indent(body, levels=1): """Indents all lines in body (which should be a textbuilder.Builder), except empty ones.""" patches = textbuilder.make_regexp_patches(body.get_text(), indent_line_re, indent_str * levels) return textbuilder.Replacer(body, patches)