def handle_from_mod_tuple(node): """Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string. Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)` and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"` Args: node (ast.BinOp): The node to convert to a f-string Returns ast.JoinedStr (f-string) """ format_str = node.left.s matches = VAR_KEY_PATTERN.findall(format_str) if len(node.right.elts) != len(matches): raise ValueError("string formatting length mismatch") str_vars = list(map(lambda x: x, node.right.elts)) # build result node result_node = ast.JoinedStr() result_node.values = [] str_vars.reverse() blocks = VAR_KEY_PATTERN.split(format_str) for block in blocks: if VAR_KEY_PATTERN.match(block): fv = ast.FormattedValue(value=str_vars.pop(), conversion=-1, format_spec=None) result_node.values.append(fv) else: result_node.values.append(ast.Str(s=block)) return result_node
def _create_endpoint_fstring( value: str) -> typing.Union[ast.Str, ast.JoinedStr]: """Create the value for the `endpoint=..` parameter in the client. If the parameter doesn't contain a var a regular string is returned, otherwise an f-string is created and returned. Note that this method assumes that the parameters in the endpoint are kept 'intact'. E.g. the raml specifices `/{container}/{key}` so we assume that those variables are available in the scope when we create the f-string. """ parts = [] last = 0 value = value.lstrip("/") for m in re.finditer("{[^}]+}", value): parts.append(ast.Constant(value=value[last:m.start()])) identifier = snakeit(value[m.start() + 1:m.end() - 1]) parts.append( ast.FormattedValue(value=ast.Name(identifier), conversion=-1, format_spec=None)) last = m.end() if last != len(value): parts.append(ast.Constant(value=value[last:len(value)])) # If no values are in the f-string we can just generate a regular string if len(parts) == 1: return ast.Str(s=parts[0].value, kind=None) return ast.JoinedStr(values=parts)
def visit_BinOp(self, node: ast.BinOp): """ Transforms a string concat to an f-string """ if is_string_concat(node): self.counter += 1 left, right = node.left, node.right left = self.visit(left) right = self.visit(right) if not check_sns_depth(left) or not check_sns_depth(right): node.left = left node.right = right return node parts = [] for p in [left, right]: if isinstance(p, ast.JoinedStr): parts += p.values else: parts.append(p) segments = [] for p in parts: if isinstance(p, ast.Constant): segments.append(ast_string_node(p.value)) else: segments.append(ast_formatted_value(p)) return ast.JoinedStr(segments) else: return self.generic_visit(node)
def transform_dict(node): """Convert a `BinOp` `%` formatted str with a name representing a Dict on the right to an f-string. Takes an ast.BinOp representing `"1. %(key1)s 2. %(key2)s" % mydict` and converted it to a ast.JoinedStr representing `f"1. {mydict['key1']} 2. {mydict['key2']}"` Args: node (ast.BinOp): The node to convert to a f-string Returns ast.JoinedStr (f-string) """ format_str = node.left.s matches = DICT_PATTERN.findall(format_str) if len(matches) != len(ANY_DICT.findall(format_str)): raise FlyntException("Some locations have unknown format modifiers.") spec = [] for idx, m in enumerate(matches): _, var_key, prefix, fmt_str, _ = SPLIT_DICT_PATTERN.split(m) if not var_key: raise FlyntException("could not find dict key") spec.append((prefix, var_key, fmt_str)) # build result node segments = [] spec.reverse() blocks = DICT_PATTERN.split(format_str) mapping = {} if isinstance(node.right, ast.Dict): for k, v in zip(node.right.keys, node.right.values): mapping[str(ast.literal_eval(k))] = v def make_fv(key: str): return mapping.pop(key) else: def make_fv(key: str): return ast.Subscript(value=node.right, slice=ast.Index(value=ast.Str(s=key))) for block in blocks: # if this block matches a %(arg)s pattern then inject f-string instead if DICT_PATTERN.match(block): prefix, var_key, fmt_str = spec.pop() fv = formatted_value(prefix, fmt_str, make_fv(var_key)) segments.append(fv) else: # no match means it's just a literal string segments.append(ast.Str(s=block.replace("%%", "%"))) if mapping: raise FlyntException( "Not all keys were matched - either a flynt error or original code error." ) return ast.JoinedStr(segments)
def joined_string(fmt_call: ast.Call) -> ast.JoinedStr: """ Transform a "...".format() call node into a f-string node. """ string = fmt_call.func.value.s var_map = {kw.arg: kw.value for kw in fmt_call.keywords} for i, val in enumerate(fmt_call.args): var_map[i] = val splits = deque(stdlib_parse(string)) seq_ctr = 0 new_segments = [] manual_field_ordering = False for raw, var_name, fmt_str, conversion in splits: if raw: new_segments.append(ast_string_node(raw)) if var_name is None: continue if "[" in var_name: raise FlyntException( f"Skipping f-stringify of a fmt call with indexed name {var_name}" ) suffix = "" if "." in var_name: idx = var_name.find(".") var_name, suffix = var_name[:idx], var_name[idx + 1:] if var_name.isdigit(): manual_field_ordering = True identifier = int(var_name) elif len(var_name) == 0: assert not manual_field_ordering identifier = seq_ctr seq_ctr += 1 else: identifier = var_name try: ast_name = var_map.pop(identifier) except KeyError as e: raise ConversionRefused( f"A variable {identifier} is used multiple times - better not to replace it." ) from e if suffix: ast_name = ast.Attribute(value=ast_name, attr=suffix) new_segments.append(ast_formatted_value(ast_name, fmt_str, conversion)) if var_map: raise FlyntException( f"Some variables were never used: {var_map} - skipping conversion, it's a risk of bug." ) return ast.JoinedStr(new_segments)
def test_invalid_fstring_value(self): self.check_invalid( ast.JoinedStr( values=[ ast.Name(id="test"), ast.Constant(value="test") ] ) )
def generate_joined_string(max_depth=None): length = random.randrange(MAX_LIST_LENGTH) values = [] for _ in range(length): if 0 == random.randrange(2): string_part = generate_string(max_depth=max_depth) else: string_part = generate_formatted_value(max_depth=max_depth) values.append(string_part) values.append(ast.Str(" ")) return ast.JoinedStr(values)
def maybe_replace_with_fstring(fs: PercentFormatString, args_node: ast.AST) -> Optional[ast.AST]: """If appropriate, emits an error to replace this % format with an f-string.""" # otherwise there are no f-strings if sys.version_info < (3, 6): return None # there are no bytes f-strings if isinstance(fs.pattern, bytes): return None # if there is a { in the string, we will need escaping in order to use an f-string, which might # make the code worse if any("{" in piece or "}" in piece for piece in fs.raw_pieces): return None # special conversion specifiers are rare and more difficult to replace, so just ignore them for # now if any( any([ cs.mapping_key, cs.conversion_flags, cs.field_width, cs.precision, cs.length_modifier, ]) for cs in fs.specifiers): return None # don't attempt fancy conversion types if any(cs.conversion_type not in ("d", "s") for cs in fs.specifiers): return None # only proceed if all the arguments are simple (currently, names or attribute accesses) if isinstance(args_node, ast.Tuple): if any(not _is_simple_enough(elt) for elt in args_node.elts): return None substitutions = args_node.elts elif len(fs.specifiers) == 1: if not _is_simple_enough(args_node): return None substitutions = [args_node] else: return None # the linter should have given an error in this case if len(substitutions) != len(fs.specifiers) != len(fs.raw_pieces) - 1: return None parts = [] for raw_piece, substitution in zip(fs.raw_pieces, substitutions): if raw_piece: parts.append(ast.Str(s=raw_piece)) parts.append( ast.FormattedValue(value=substitution, conversion=-1, format_spec=None)) if fs.raw_pieces[-1]: parts.append(ast.Str(s=fs.raw_pieces[-1])) return ast.JoinedStr(values=parts)
def get_kernel_embed(): """A list of kernel embed nodes Returns: nodes (list): AST nodes which form the following code. ``` import os pid = os.fork() if os.fork() == 0: open(f'{os.environ["HOME"]}/.pynt', 'a').close() import IPython IPython.start_kernel(user_ns={**locals(), **globals(), **vars()}) os.waitpid(pid, 0) ``` This is a purely functional method which always return the same thing. """ return [ ast.Import(names=[ast.alias(name='os', asname=None),]), ast.Assign(targets=[ast.Name(id='pid', ctx=ast.Store()),], value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='fork', ctx=ast.Load()), args=[], keywords=[])), ast.If( test=ast.Compare(left=ast.Name(id='pid', ctx=ast.Load()), ops=[ast.Eq(),], comparators=[ast.Num(n=0),]), body=[ ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Call(func=ast.Name(id='open', ctx=ast.Load()), args=[ ast.JoinedStr(values=[ ast.FormattedValue(value=ast.Subscript(value=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='environ', ctx=ast.Load()), slice=ast.Index(value=ast.Str(s='HOME')), ctx=ast.Load()), conversion=-1, format_spec=None), ast.Str(s='/.pynt'), ]), ast.Str(s='a'), ], keywords=[]), attr='close', ctx=ast.Load()), args=[], keywords=[])), ast.Import(names=[ ast.alias(name='IPython', asname=None), ]), ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='IPython', ctx=ast.Load()), attr='start_kernel', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='user_ns', value=ast.Dict(keys=[ None, None, None, ], values=[ ast.Call(func=ast.Name(id='locals', ctx=ast.Load()), args=[], keywords=[]), ast.Call(func=ast.Name(id='globals', ctx=ast.Load()), args=[], keywords=[]), ast.Call(func=ast.Name(id='vars', ctx=ast.Load()), args=[], keywords=[]), ])), ])), ], orelse=[]), ast.Expr(value=ast.Call(func=ast.Attribute(value=ast.Name(id='os', ctx=ast.Load()), attr='waitpid', ctx=ast.Load()), args=[ ast.Name(id='pid', ctx=ast.Load()), ast.Num(n=0), ], keywords=[])), ]
def ast_formatted_value(val, fmt_str: str = None, conversion=None) -> ast.FormattedValue: if astor.to_source(val)[0] == "{": raise FlyntException( "values starting with '{' are better left not transformed.") format_spec = ast.JoinedStr([ast_string_node(fmt_str) ]) if fmt_str else None conversion = -1 if conversion is None else ord(conversion.replace("!", "")) return ast.FormattedValue(value=val, conversion=conversion, format_spec=format_spec)
def handle_from_mod_tuple(node): """Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string. Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)` and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"` Args: node (ast.BinOp): The node to convert to a f-string Returns ast.JoinedStr (f-string) """ format_str = node.left.s matches = VAR_KEY_PATTERN.findall(format_str) if len(node.right.elts) != len(matches): raise FlyntException("string formatting length mismatch") str_vars = deque(node.right.elts) # build result node result_node = ast.JoinedStr() result_node.values = [] blocks = deque(VAR_KEY_PATTERN.split(format_str)) result_node.values.append(ast_string_node(blocks.popleft())) while len(blocks) > 0: fmt_prefix = blocks.popleft() fmt_spec = blocks.popleft() for c in obsolete_specifiers: fmt_spec = fmt_spec.replace(c, '') if fmt_spec in conversion_methods: if fmt_prefix: raise FlyntException( "Default text alignment has changed between percent fmt and fstrings. " "Proceeding would result in changed code behaviour.") fv = ast_formatted_value(str_vars.popleft(), fmt_str=fmt_prefix, conversion=conversion_methods[fmt_spec]) else: fmt_spec = translate_conversion_types.get(fmt_spec, fmt_spec) fv = ast_formatted_value(str_vars.popleft(), fmt_str=fmt_prefix + fmt_spec) result_node.values.append(fv) result_node.values.append(ast_string_node(blocks.popleft())) return result_node
def visit_BinOp(self, node: ast.BinOp): """ Transforms a string concat to an f-string """ self.counter += 1 pieces = unpack_binop(node) segments = [] for p in pieces: if isinstance(p, ast.Constant): segments.append(ast_string_node(p.value)) else: segments.append(ast_formatted_value(p)) return ast.JoinedStr(segments)
def make(self) -> ast.JoinedStr: text = self.element.text if isinstance(text, str): text = text.strip() base = [ast.Str(text)] if text else [] children = map( partial(ast.FormattedValue, conversion=-1, format_spec=None), map(operator.attrgetter("value"), self.expr.children), ) texts = map( ast.Str, map(str.strip, map(operator.attrgetter("tail"), self.element))) base.extend(chain.from_iterable(zip(children, texts))) return ast.JoinedStr(base)
def handle_from_mod_dict_name(node): """Convert a `BinOp` `%` formatted str with a name representing a Dict on the right to an f-string. Takes an ast.BinOp representing `"1. %(key1)s 2. %(key2)s" % mydict` and converted it to a ast.JoinedStr representing `f"1. {mydict['key1']} 2. {mydict['key2']}"` Args: node (ast.BinOp): The node to convert to a f-string Returns ast.JoinedStr (f-string) """ # raise ValueError("blah") format_str = node.left.s matches = MOD_KEY_PATTERN.findall(format_str) var_keys = [] for idx, m in enumerate(matches): var_key = MOD_KEY_NAME_PATTERN.match(m) if not var_key: raise ValueError("could not find dict key") var_keys.append(var_key[1]) # build result node result_node = ast.JoinedStr() result_node.values = [] var_keys.reverse() blocks = MOD_KEY_PATTERN.split(format_str) # loop through the blocks of a string to build up dateh JoinStr.values for block in blocks: # if this block matches a %(arg)s pattern then inject f-string instead if MOD_KEY_PATTERN.match(block): fv = ast.FormattedValue( value=ast.Subscript( value=node.right, slice=ast.Index(value=ast.Str(s=var_keys.pop()))), conversion=-1, format_spec=None, ) result_node.values.append(fv) else: # no match means it's just a literal string result_node.values.append(ast.Str(s=block)) return result_node
def test_fstring(self): xml = Parser(self.get_xml("dict.xml")) module = xml.parse() code = compile(module, "<ast>", "exec") mymodule = ast.Module([ ast.Assign([ast.Name('name', ast.Store())], ast.Str('Batuhan')), ast.Assign([ast.Name('age', ast.Store())], ast.Num(15)), ast.Expr( ast.Call(ast.Name('print', ast.Load()), [ ast.JoinedStr(values=[ ast.Str(s='Benim adım'), ast.FormattedValue( value=ast.Name(id='name', ctx=ast.Load()), conversion=-1, format_spec=None, ), ast.Str(s=', yaşım'), ast.FormattedValue( value=ast.Name(id='age', ctx=ast.Load()), conversion=-1, format_spec=None, ), ast.Str(s='. Hakkımda geri kalan bilgi:'), ast.FormattedValue( value=ast.Call( func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Num(n=235)], keywords=[], ), conversion=-1, format_spec=None, ), ast.Str(s=''), ]) ], [])) ]) ast.fix_missing_locations(mymodule) pprint(module) mycode = compile(mymodule, "<ast>", "exec") self.assertEqual(code, mycode)
def _transform_string(self, node: ast.JoinedStr) -> ast.Call: htm_strings: List[str] = [] exp_nodes: List[ast.AST] = [] for inner_node in node.values: if isinstance(inner_node, ast.Str): htm_strings.append(inner_node.s) elif isinstance(inner_node, ast.FormattedValue): if len(htm_strings) == len(exp_nodes): htm_strings.append("") if inner_node.conversion != -1 or inner_node.format_spec: exp_nodes.append(ast.JoinedStr([inner_node])) else: exp_nodes.append(inner_node.value) call_stack = _HtmlCallStack() for op_type, *data in htm.htm_parse(htm_strings): getattr(self, f"_transform_htm_{op_type.lower()}")(exp_nodes, call_stack, *data) return call_stack.finish()
def visit_BinOp(self, node): # transform the left and right arguments of the binary operation: node.left = pythonTransformer().visit(node.left) node.right = pythonTransformer().visit(node.right) # formatted strings with % # note: we have extended the pythong syntax slightly, to accommodate both tuples and lists # so both '%_%' % (1,2) and '%_%' % [1,2] are successfully transpiled if isinstance(node.op, ast.Mod) and isinstance(node.left, ast.Str): # transform the node into an f-string node: stringFormat = node.left.value stringTuple = node.right.elts if ( isinstance(node.right, ast.Tuple) or isinstance(node.right, ast.List))\ else [node.right] values = [] tupleIndex = 0 while True: # TODO deal with more complicated formats, such as %.3f match = re.search(r'%.', stringFormat) if match is None: break values.append(ast.Constant(value=stringFormat[0:match.span(0)[0]], kind=None)) values.append( self.visit_FormattedValue( ast.FormattedValue( value=stringTuple[tupleIndex], conversion=-1, format_spec=None ) ) ) stringFormat = stringFormat[match.span(0)[1]:] tupleIndex += 1 return ast.JoinedStr(values) return node
def transform_tuple(node): """Convert a `BinOp` `%` formatted str with a tuple on the right to an f-string. Takes an ast.BinOp representing `"1. %s 2. %s" % (a, b)` and converted it to a ast.JoinedStr representing `f"1. {a} 2. {b}"` Args: node (ast.BinOp): The node to convert to a f-string Returns ast.JoinedStr (f-string) """ format_str = node.left.s matches = VAR_KEY_PATTERN.findall(format_str) if len(node.right.elts) != len(matches): raise ConversionRefused("This expression involves tuple unpacking.") str_vars = deque(node.right.elts) segments = [] blocks = deque(VAR_KEY_PATTERN.split(format_str)) segments.append(ast_string_node(blocks.popleft().replace("%%", "%"))) while len(blocks) > 0: fmt_prefix = blocks.popleft() fmt_spec = blocks.popleft() val = str_vars.popleft() fv = formatted_value(fmt_prefix, fmt_spec, val) segments.append(fv) segments.append(ast_string_node(blocks.popleft().replace("%%", "%"))) return ast.JoinedStr(segments)
def str_maker(*strs: Tokenizer): head = strs[0] return ast.JoinedStr(**(loc @ head), values=list(map(_parse_expr, strs)))
def test_invalid_fstring_constant(self): self.check_invalid(ast.JoinedStr(values=[ast.Constant(value=100)]))
def visit_JoinedStr(self, node: JoinedStr, *args, **kwargs) -> C.JoinedStr: values = self.visit(node.values, *args, **kwargs) return C.JoinedStr(values=values, )