def handle_from_mod_tuple(node):
"""Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string.
Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)`
and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"`
Args:
node (ast.BinOp): The node to convert to a f-string
Returns ast.JoinedStr (f-string)
"""
format_str = node.left.s
matches = VAR_KEY_PATTERN.findall(format_str)
if len(node.right.elts) != len(matches):
raise ValueError("string formatting length mismatch")
str_vars = list(map(lambda x: x, node.right.elts))
# build result node
result_node = ast.JoinedStr()
result_node.values = []
str_vars.reverse()
blocks = VAR_KEY_PATTERN.split(format_str)
for block in blocks:
if VAR_KEY_PATTERN.match(block):
fv = ast.FormattedValue(value=str_vars.pop(),
conversion=-1,
format_spec=None)
result_node.values.append(fv)
else:
result_node.values.append(ast.Str(s=block))
return result_node
def _create_endpoint_fstring(
value: str) -> typing.Union[ast.Str, ast.JoinedStr]:
"""Create the value for the `endpoint=..` parameter in the client.
If the parameter doesn't contain a var a regular string is returned,
otherwise an f-string is created and returned.
Note that this method assumes that the parameters in the endpoint are kept
'intact'. E.g. the raml specifices `/{container}/{key}` so we assume that
those variables are available in the scope when we create the f-string.
"""
parts = []
last = 0
value = value.lstrip("/")
for m in re.finditer("{[^}]+}", value):
parts.append(ast.Constant(value=value[last:m.start()]))
identifier = snakeit(value[m.start() + 1:m.end() - 1])
parts.append(
ast.FormattedValue(value=ast.Name(identifier),
conversion=-1,
format_spec=None))
last = m.end()
if last != len(value):
parts.append(ast.Constant(value=value[last:len(value)]))
# If no values are in the f-string we can just generate a regular string
if len(parts) == 1:
return ast.Str(s=parts[0].value, kind=None)
return ast.JoinedStr(values=parts)
def visit_BinOp(self, node: ast.BinOp):
"""
Transforms a string concat to an f-string
"""
if is_string_concat(node):
self.counter += 1
left, right = node.left, node.right
left = self.visit(left)
right = self.visit(right)
if not check_sns_depth(left) or not check_sns_depth(right):
node.left = left
node.right = right
return node
parts = []
for p in [left, right]:
if isinstance(p, ast.JoinedStr):
parts += p.values
else:
parts.append(p)
segments = []
for p in parts:
if isinstance(p, ast.Constant):
segments.append(ast_string_node(p.value))
else:
segments.append(ast_formatted_value(p))
return ast.JoinedStr(segments)
else:
return self.generic_visit(node)
def transform_dict(node):
"""Convert a `BinOp` `%` formatted str with a name representing a Dict on the right to an f-string.
Takes an ast.BinOp representing `"1. %(key1)s 2. %(key2)s" % mydict`
and converted it to a ast.JoinedStr representing `f"1. {mydict['key1']} 2. {mydict['key2']}"`
Args:
node (ast.BinOp): The node to convert to a f-string
Returns ast.JoinedStr (f-string)
"""
format_str = node.left.s
matches = DICT_PATTERN.findall(format_str)
if len(matches) != len(ANY_DICT.findall(format_str)):
raise FlyntException("Some locations have unknown format modifiers.")
spec = []
for idx, m in enumerate(matches):
_, var_key, prefix, fmt_str, _ = SPLIT_DICT_PATTERN.split(m)
if not var_key:
raise FlyntException("could not find dict key")
spec.append((prefix, var_key, fmt_str))
# build result node
segments = []
spec.reverse()
blocks = DICT_PATTERN.split(format_str)
mapping = {}
if isinstance(node.right, ast.Dict):
for k, v in zip(node.right.keys, node.right.values):
mapping[str(ast.literal_eval(k))] = v
def make_fv(key: str):
return mapping.pop(key)
else:
def make_fv(key: str):
return ast.Subscript(value=node.right,
slice=ast.Index(value=ast.Str(s=key)))
for block in blocks:
# if this block matches a %(arg)s pattern then inject f-string instead
if DICT_PATTERN.match(block):
prefix, var_key, fmt_str = spec.pop()
fv = formatted_value(prefix, fmt_str, make_fv(var_key))
segments.append(fv)
else:
# no match means it's just a literal string
segments.append(ast.Str(s=block.replace("%%", "%")))
if mapping:
raise FlyntException(
"Not all keys were matched - either a flynt error or original code error."
)
return ast.JoinedStr(segments)
def joined_string(fmt_call: ast.Call) -> ast.JoinedStr:
""" Transform a "...".format() call node into a f-string node. """
string = fmt_call.func.value.s
var_map = {kw.arg: kw.value for kw in fmt_call.keywords}
for i, val in enumerate(fmt_call.args):
var_map[i] = val
splits = deque(stdlib_parse(string))
seq_ctr = 0
new_segments = []
manual_field_ordering = False
for raw, var_name, fmt_str, conversion in splits:
if raw:
new_segments.append(ast_string_node(raw))
if var_name is None:
continue
if "[" in var_name:
raise FlyntException(
f"Skipping f-stringify of a fmt call with indexed name {var_name}"
)
suffix = ""
if "." in var_name:
idx = var_name.find(".")
var_name, suffix = var_name[:idx], var_name[idx + 1:]
if var_name.isdigit():
manual_field_ordering = True
identifier = int(var_name)
elif len(var_name) == 0:
assert not manual_field_ordering
identifier = seq_ctr
seq_ctr += 1
else:
identifier = var_name
try:
ast_name = var_map.pop(identifier)
except KeyError as e:
raise ConversionRefused(
f"A variable {identifier} is used multiple times - better not to replace it."
) from e
if suffix:
ast_name = ast.Attribute(value=ast_name, attr=suffix)
new_segments.append(ast_formatted_value(ast_name, fmt_str, conversion))
if var_map:
raise FlyntException(
f"Some variables were never used: {var_map} - skipping conversion, it's a risk of bug."
)
return ast.JoinedStr(new_segments)
def test_invalid_fstring_value(self):
self.check_invalid(
ast.JoinedStr(
values=[
ast.Name(id="test"),
ast.Constant(value="test")
]
)
)
def generate_joined_string(max_depth=None):
length = random.randrange(MAX_LIST_LENGTH)
values = []
for _ in range(length):
if 0 == random.randrange(2):
string_part = generate_string(max_depth=max_depth)
else:
string_part = generate_formatted_value(max_depth=max_depth)
values.append(string_part)
values.append(ast.Str(" "))
return ast.JoinedStr(values)
def maybe_replace_with_fstring(fs: PercentFormatString,
args_node: ast.AST) -> Optional[ast.AST]:
"""If appropriate, emits an error to replace this % format with an f-string."""
# otherwise there are no f-strings
if sys.version_info < (3, 6):
return None
# there are no bytes f-strings
if isinstance(fs.pattern, bytes):
return None
# if there is a { in the string, we will need escaping in order to use an f-string, which might
# make the code worse
if any("{" in piece or "}" in piece for piece in fs.raw_pieces):
return None
# special conversion specifiers are rare and more difficult to replace, so just ignore them for
# now
if any(
any([
cs.mapping_key,
cs.conversion_flags,
cs.field_width,
cs.precision,
cs.length_modifier,
]) for cs in fs.specifiers):
return None
# don't attempt fancy conversion types
if any(cs.conversion_type not in ("d", "s") for cs in fs.specifiers):
return None
# only proceed if all the arguments are simple (currently, names or attribute accesses)
if isinstance(args_node, ast.Tuple):
if any(not _is_simple_enough(elt) for elt in args_node.elts):
return None
substitutions = args_node.elts
elif len(fs.specifiers) == 1:
if not _is_simple_enough(args_node):
return None
substitutions = [args_node]
else:
return None
# the linter should have given an error in this case
if len(substitutions) != len(fs.specifiers) != len(fs.raw_pieces) - 1:
return None
parts = []
for raw_piece, substitution in zip(fs.raw_pieces, substitutions):
if raw_piece:
parts.append(ast.Str(s=raw_piece))
parts.append(
ast.FormattedValue(value=substitution,
conversion=-1,
format_spec=None))
if fs.raw_pieces[-1]:
parts.append(ast.Str(s=fs.raw_pieces[-1]))
return ast.JoinedStr(values=parts)
def get_kernel_embed():
"""A list of kernel embed nodes
Returns:
nodes (list): AST nodes which form the following code.
```
import os
pid = os.fork()
if os.fork() == 0:
open(f'{os.environ["HOME"]}/.pynt', 'a').close()
import IPython
IPython.start_kernel(user_ns={**locals(), **globals(), **vars()})
os.waitpid(pid, 0)
```
This is a purely functional method which always return the same thing.
"""
return [
ast.Import(names=[ast.alias(name='os', asname=None),]),
ast.Assign(targets=[ast.Name(id='pid', ctx=ast.Store()),], value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='fork', ctx=ast.Load()), args=[], keywords=[])),
ast.If(
test=ast.Compare(left=ast.Name(id='pid', ctx=ast.Load()), ops=[ast.Eq(),], comparators=[ast.Num(n=0),]),
body=[
ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Call(func=ast.Name(id='open', ctx=ast.Load()), args=[
ast.JoinedStr(values=[
ast.FormattedValue(value=ast.Subscript(value=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='environ', ctx=ast.Load()), slice=ast.Index(value=ast.Str(s='HOME')), ctx=ast.Load()), conversion=-1, format_spec=None),
ast.Str(s='/.pynt'),
]),
ast.Str(s='a'),
], keywords=[]), attr='close', ctx=ast.Load()), args=[], keywords=[])),
ast.Import(names=[
ast.alias(name='IPython', asname=None),
]),
ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='IPython', ctx=ast.Load()), attr='start_kernel', ctx=ast.Load()), args=[], keywords=[
ast.keyword(arg='user_ns', value=ast.Dict(keys=[
None,
None,
None,
], values=[
ast.Call(func=ast.Name(id='locals', ctx=ast.Load()), args=[], keywords=[]),
ast.Call(func=ast.Name(id='globals', ctx=ast.Load()), args=[], keywords=[]),
ast.Call(func=ast.Name(id='vars', ctx=ast.Load()), args=[], keywords=[]),
])),
])),
], orelse=[]),
ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='waitpid', ctx=ast.Load()), args=[
ast.Name(id='pid', ctx=ast.Load()),
ast.Num(n=0),
], keywords=[])),
]
def ast_formatted_value(val,
fmt_str: str = None,
conversion=None) -> ast.FormattedValue:
if astor.to_source(val)[0] == "{":
raise FlyntException(
"values starting with '{' are better left not transformed.")
format_spec = ast.JoinedStr([ast_string_node(fmt_str)
]) if fmt_str else None
conversion = -1 if conversion is None else ord(conversion.replace("!", ""))
return ast.FormattedValue(value=val,
conversion=conversion,
format_spec=format_spec)
def handle_from_mod_tuple(node):
"""Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string.
Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)`
and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"`
Args:
node (ast.BinOp): The node to convert to a f-string
Returns ast.JoinedStr (f-string)
"""
format_str = node.left.s
matches = VAR_KEY_PATTERN.findall(format_str)
if len(node.right.elts) != len(matches):
raise FlyntException("string formatting length mismatch")
str_vars = deque(node.right.elts)
# build result node
result_node = ast.JoinedStr()
result_node.values = []
blocks = deque(VAR_KEY_PATTERN.split(format_str))
result_node.values.append(ast_string_node(blocks.popleft()))
while len(blocks) > 0:
fmt_prefix = blocks.popleft()
fmt_spec = blocks.popleft()
for c in obsolete_specifiers:
fmt_spec = fmt_spec.replace(c, '')
if fmt_spec in conversion_methods:
if fmt_prefix:
raise FlyntException(
"Default text alignment has changed between percent fmt and fstrings. "
"Proceeding would result in changed code behaviour.")
fv = ast_formatted_value(str_vars.popleft(),
fmt_str=fmt_prefix,
conversion=conversion_methods[fmt_spec])
else:
fmt_spec = translate_conversion_types.get(fmt_spec, fmt_spec)
fv = ast_formatted_value(str_vars.popleft(),
fmt_str=fmt_prefix + fmt_spec)
result_node.values.append(fv)
result_node.values.append(ast_string_node(blocks.popleft()))
return result_node
def visit_BinOp(self, node: ast.BinOp):
"""
Transforms a string concat to an f-string
"""
self.counter += 1
pieces = unpack_binop(node)
segments = []
for p in pieces:
if isinstance(p, ast.Constant):
segments.append(ast_string_node(p.value))
else:
segments.append(ast_formatted_value(p))
return ast.JoinedStr(segments)
def make(self) -> ast.JoinedStr:
text = self.element.text
if isinstance(text, str):
text = text.strip()
base = [ast.Str(text)] if text else []
children = map(
partial(ast.FormattedValue, conversion=-1, format_spec=None),
map(operator.attrgetter("value"), self.expr.children),
)
texts = map(
ast.Str,
map(str.strip, map(operator.attrgetter("tail"), self.element)))
base.extend(chain.from_iterable(zip(children, texts)))
return ast.JoinedStr(base)
def handle_from_mod_dict_name(node):
"""Convert a `BinOp` `%` formatted str with a name representing a Dict on the right to an f-string.
Takes an ast.BinOp representing `"1. %(key1)s 2. %(key2)s" % mydict`
and converted it to a ast.JoinedStr representing `f"1. {mydict['key1']} 2. {mydict['key2']}"`
Args:
node (ast.BinOp): The node to convert to a f-string
Returns ast.JoinedStr (f-string)
"""
# raise ValueError("blah")
format_str = node.left.s
matches = MOD_KEY_PATTERN.findall(format_str)
var_keys = []
for idx, m in enumerate(matches):
var_key = MOD_KEY_NAME_PATTERN.match(m)
if not var_key:
raise ValueError("could not find dict key")
var_keys.append(var_key[1])
# build result node
result_node = ast.JoinedStr()
result_node.values = []
var_keys.reverse()
blocks = MOD_KEY_PATTERN.split(format_str)
# loop through the blocks of a string to build up dateh JoinStr.values
for block in blocks:
# if this block matches a %(arg)s pattern then inject f-string instead
if MOD_KEY_PATTERN.match(block):
fv = ast.FormattedValue(
value=ast.Subscript(
value=node.right,
slice=ast.Index(value=ast.Str(s=var_keys.pop()))),
conversion=-1,
format_spec=None,
)
result_node.values.append(fv)
else:
# no match means it's just a literal string
result_node.values.append(ast.Str(s=block))
return result_node
def test_fstring(self):
xml = Parser(self.get_xml("dict.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Assign([ast.Name('name', ast.Store())], ast.Str('Batuhan')),
ast.Assign([ast.Name('age', ast.Store())], ast.Num(15)),
ast.Expr(
ast.Call(ast.Name('print', ast.Load()), [
ast.JoinedStr(values=[
ast.Str(s='Benim adım'),
ast.FormattedValue(
value=ast.Name(id='name', ctx=ast.Load()),
conversion=-1,
format_spec=None,
),
ast.Str(s=', yaşım'),
ast.FormattedValue(
value=ast.Name(id='age', ctx=ast.Load()),
conversion=-1,
format_spec=None,
),
ast.Str(s='. Hakkımda geri kalan bilgi:'),
ast.FormattedValue(
value=ast.Call(
func=ast.Name(id='str', ctx=ast.Load()),
args=[ast.Num(n=235)],
keywords=[],
),
conversion=-1,
format_spec=None,
),
ast.Str(s=''),
])
], []))
])
ast.fix_missing_locations(mymodule)
pprint(module)
mycode = compile(mymodule, "<ast>", "exec")
self.assertEqual(code, mycode)
def _transform_string(self, node: ast.JoinedStr) -> ast.Call:
htm_strings: List[str] = []
exp_nodes: List[ast.AST] = []
for inner_node in node.values:
if isinstance(inner_node, ast.Str):
htm_strings.append(inner_node.s)
elif isinstance(inner_node, ast.FormattedValue):
if len(htm_strings) == len(exp_nodes):
htm_strings.append("")
if inner_node.conversion != -1 or inner_node.format_spec:
exp_nodes.append(ast.JoinedStr([inner_node]))
else:
exp_nodes.append(inner_node.value)
call_stack = _HtmlCallStack()
for op_type, *data in htm.htm_parse(htm_strings):
getattr(self,
f"_transform_htm_{op_type.lower()}")(exp_nodes, call_stack,
*data)
return call_stack.finish()
def visit_BinOp(self, node):
# transform the left and right arguments of the binary operation:
node.left = pythonTransformer().visit(node.left)
node.right = pythonTransformer().visit(node.right)
# formatted strings with %
# note: we have extended the pythong syntax slightly, to accommodate both tuples and lists
# so both '%_%' % (1,2) and '%_%' % [1,2] are successfully transpiled
if isinstance(node.op, ast.Mod) and isinstance(node.left, ast.Str):
# transform the node into an f-string node:
stringFormat = node.left.value
stringTuple = node.right.elts if (
isinstance(node.right, ast.Tuple) or isinstance(node.right, ast.List))\
else [node.right]
values = []
tupleIndex = 0
while True:
# TODO deal with more complicated formats, such as %.3f
match = re.search(r'%.', stringFormat)
if match is None:
break
values.append(ast.Constant(value=stringFormat[0:match.span(0)[0]], kind=None))
values.append(
self.visit_FormattedValue(
ast.FormattedValue(
value=stringTuple[tupleIndex],
conversion=-1,
format_spec=None
)
)
)
stringFormat = stringFormat[match.span(0)[1]:]
tupleIndex += 1
return ast.JoinedStr(values)
return node
def transform_tuple(node):
"""Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string.
Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)`
and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"`
Args:
node (ast.BinOp): The node to convert to a f-string
Returns ast.JoinedStr (f-string)
"""
format_str = node.left.s
matches = VAR_KEY_PATTERN.findall(format_str)
if len(node.right.elts) != len(matches):
raise ConversionRefused("This expression involves tuple unpacking.")
str_vars = deque(node.right.elts)
segments = []
blocks = deque(VAR_KEY_PATTERN.split(format_str))
segments.append(ast_string_node(blocks.popleft().replace("%%", "%")))
while len(blocks) > 0:
fmt_prefix = blocks.popleft()
fmt_spec = blocks.popleft()
val = str_vars.popleft()
fv = formatted_value(fmt_prefix, fmt_spec, val)
segments.append(fv)
segments.append(ast_string_node(blocks.popleft().replace("%%", "%")))
return ast.JoinedStr(segments)
def str_maker(*strs: Tokenizer):
head = strs[0]
return ast.JoinedStr(**(loc @ head), values=list(map(_parse_expr, strs)))
def test_invalid_fstring_constant(self):
self.check_invalid(ast.JoinedStr(values=[ast.Constant(value=100)]))
def visit_JoinedStr(self, node: JoinedStr, *args, **kwargs) -> C.JoinedStr:
values = self.visit(node.values, *args, **kwargs)
return C.JoinedStr(values=values, )
