def _field_access(self, field_name, format_spec, conversion):
first, rest = _string.formatter_field_name_split(field_name)
funcs = []
for is_attr, key in rest:
if is_attr:
func = operator.attrgetter
elif ":" in key:
func = self._slicegetter
else:
func = operator.itemgetter
funcs.append(func(key))
if conversion:
funcs.append(self.CONVERSIONS[conversion])
if format_spec:
if format_spec[0] == "?":
func = self._format_optional
elif format_spec[0] == "L":
func = self._format_maxlen
elif format_spec[0] == "J":
func = self._format_join
elif format_spec[0] == "R":
func = self._format_replace
else:
func = self._format_default
fmt = func(format_spec)
else:
fmt = str
if funcs:
return self._apply(first, funcs, fmt)
return self._apply_simple(first, fmt)
def get_field(self, field_name, args, kwargs):
try:
val = super(PartialFormatter, self).get_field(field_name, args, kwargs)
except (IndexError, KeyError, AttributeError):
first, _ = formatter_field_name_split(field_name)
val = '{' + field_name + '}', first
return val
def get_field(self, field_name, args, kwargs):
if appier.legacy.PYTHON_3:
import _string
first, rest = _string.formatter_field_name_split(field_name)
else:
first, rest = field_name._formatter_field_name_split()
try:
obj = self.get_value(first, args, kwargs)
except Exception:
return self.fallback, first
for is_attr, key in rest:
if is_attr:
try:
obj = getattr(obj, key)
except Exception:
obj = self.fallback
break
else:
try:
obj = obj[key]
except Exception:
obj = self.fallback
break
return obj, first
def get_field(self, field_name, args, kwargs):
if appier.legacy.PYTHON_3:
import _string
first, rest = _string.formatter_field_name_split(field_name)
else:
first, rest = field_name._formatter_field_name_split()
try:
obj = self.get_value(first, args, kwargs)
except Exception:
return self.fallback, first
for is_attr, key in rest:
if is_attr:
try:
obj = getattr(obj, key)
except Exception:
obj = self.fallback
break
else:
try:
obj = obj[key]
except Exception:
obj = self.fallback
break
return obj, first
def get_field(self, field_name, args, kwargs):
"""Find the object `field_name` references.
:field_name: the field being looked up
:args: as passed in to vformat
:kwargs: as passed in to vformat
:returns: the referenced object and the used field
"""
# Parse the file_name. first is the keyword, rest contains the
# attributes and indices, if any.
first, rest = _string.formatter_field_name_split(field_name)
# Get the python object referenced by first.
obj = self.get_value(first, args, kwargs)
# Loop through the attributes and indices to get the referenced object.
for is_attr, i in rest:
if is_attr:
# i is the name of an attribute.
if isinstance(obj, list):
obj = [getattr(o, i) for o in obj]
else:
obj = getattr(obj, i)
elif ':' in str(i):
# i is an index in slice notation.
obj = obj[slice(*[int(j) if j else None
for j in i.split(':')])]
else:
# i is a simple index.
obj = obj[i]
return obj, first
def _field_access(self, field_name, format_spec, conversion):
first, rest = _string.formatter_field_name_split(field_name)
funcs = []
for is_attr, key in rest:
if is_attr:
func = operator.attrgetter
elif ":" in key:
func = self._slicegetter
else:
func = operator.itemgetter
funcs.append(func(key))
if conversion:
funcs.append(self.CONVERSIONS[conversion])
if format_spec:
if format_spec[0] == "?":
func = self._format_optional
elif format_spec[0] == "L":
func = self._format_maxlen
elif format_spec[0] == "J":
func = self._format_join
else:
func = self._format_default
fmt = func(format_spec)
else:
fmt = str
if funcs:
return self._apply(first, funcs, fmt)
return self._apply_simple(first, fmt)
def get_field(self, field_name, args, kwargs):
# To illustrate, the example '{mass[element]}' is used with
# the kwargs {"element":"Pr", "mass":{"Pr":128}}
# Split the field_name into the field and an iterator
# ex. mass <fieldnameiterator object at 0x105308840>
try:
# Python 2.7
first, rest = field_name._formatter_field_name_split()
except:
# Python 3 (Only tested on 3.5)
first, rest = _string.formatter_field_name_split(field_name)
# print("First:", first)
# print("Kwargs:", kwargs)
# obj = kwargs[field_name] or obj = '' if KeyError
# ex. obj = {"Pr":128}
obj = self.get_value(first, args, kwargs)
# Often, "rest" is only one deep
# is_attr is a bool. I think it is true if something.keyword exists
# keyword is just a keyword, like something[keyword] or something.keyword
for is_attr, keyword in rest:
# This is the juciy stuff. If the keyword is in kwargs, return the
# value in obj
# ex. obj = {"Pr":128}["Pr"] = 128
if keyword in kwargs:
#print(obj)
obj = obj[kwargs.get(keyword)]
# ex. 128
return obj, first
def get_field(self, field_name, args, kwargs):
first, rest = _string.formatter_field_name_split(field_name)
try:
obj = self.get_value(first, args, kwargs)
except (IndexError, KeyError):
return "<Invalid>", first
# loop through the rest of the field_name, doing
# getattr or getitem as needed
# stops when reaches the depth of 2 or starts with _.
try:
for n, (is_attr, i) in enumerate(rest):
if n >= 2:
break
if is_attr:
if str(i).startswith("_"):
break
obj = getattr(obj, i)
else:
obj = obj[i]
else:
return obj, first
except (IndexError, KeyError):
pass
return "<Invalid>", first
def test_formatter_field_name_split(self):
import _string
first, rest = _string.formatter_field_name_split('')
assert first == ''
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('31')
assert first == 31
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('foo')
assert first == 'foo'
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('foo.bar')
assert first == 'foo'
assert list(rest) == [(True, 'bar')]
#
first, rest = _string.formatter_field_name_split('foo[123]')
assert first == 'foo'
assert list(rest) == [(False, 123)]
#
first, rest = _string.formatter_field_name_split('foo.baz[123].bok')
assert first == 'foo'
assert list(rest) == [(True, 'baz'), (False, 123), (True, 'bok')]
#
first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
assert first == 'foo'
assert list(rest) == [(True, 'baz'), (False, 'hi'), (True, 'bok')]
def test_formatter_field_name_split(self):
import _string
first, rest = _string.formatter_field_name_split('')
assert first == ''
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('31')
assert first == 31
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('foo')
assert first == 'foo'
assert list(rest) == []
#
first, rest = _string.formatter_field_name_split('foo.bar')
assert first == 'foo'
assert list(rest) == [(True, 'bar')]
#
first, rest = _string.formatter_field_name_split('foo[123]')
assert first == 'foo'
assert list(rest) == [(False, 123)]
#
first, rest = _string.formatter_field_name_split('foo.baz[123].bok')
assert first == 'foo'
assert list(rest) == [(True, 'baz'), (False, 123), (True, 'bok')]
#
first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
assert first == 'foo'
assert list(rest) == [(True, 'baz'), (False, 'hi'), (True, 'bok')]
def get_field(self, field_name, args, kwargs):
(first, rest) = _string.formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for (is_attr, i) in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return (obj, first)
def get_field(self, field_name, args, kwargs):
first, rest = formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if is_attr:
obj = self._env.getattr(obj, i)
else:
obj = self._env.getitem(obj, i)
return obj, first
def get_field(self, field_name, args, kwargs):
first, rest = formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if is_attr:
obj = self._env.getattr(obj, i)
else:
obj = self._env.getitem(obj, i)
return obj, first
def test_u_formatter_field_name_split(self):
import _string
first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
l = list(rest)
assert first == 'foo'
assert l == [(True, 'baz'), (False, 'hi'), (True, 'bok')]
assert isinstance(first, str)
for x, y in l:
assert isinstance(y, str)
def get_field(self, field_name, args, kwargs):
first, rest = _string.formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj, first
def test_u_formatter_field_name_split(self):
import _string
first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
l = list(rest)
assert first == 'foo'
assert l == [(True, 'baz'), (False, 'hi'), (True, 'bok')]
assert isinstance(first, str)
for x, y in l:
assert isinstance(y, str)
def get_field(self, field_name: str, args: t.Sequence[t.Any],
kwargs: t.Mapping[str, t.Any]) -> t.Tuple[t.Any, str]:
first, rest = formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if is_attr:
obj = self._env.getattr(obj, i)
else:
obj = self._env.getitem(obj, i)
return obj, first
def __init__(self, format_string):
'''Perform minimal validation of names used in format fields.'''
for _, name, _, _ in Formatter().parse(format_string):
if name is None:
continue
name, _ = _string.formatter_field_name_split(name)
if isinstance(name, int) or not name:
raise ValueError('positional format fields are not supported;'
' use named format fields only')
if name[:1].isdigit():
raise ValueError('invalid format field name: {}'.format(name))
def __init__(self, format_string):
'''Perform minimal validation of names used in format fields.'''
for _, name, _, _ in Formatter().parse(format_string):
if name is None:
continue
name, _ = _string.formatter_field_name_split(name)
if isinstance(name, int) or not name:
raise ValueError('positional format fields are not supported;'
' use named format fields only')
if name[:1].isdigit():
raise ValueError('invalid format field name: {}'.format(name))
def test():
for _ in range(9999):
try:
list(_string.formatter_field_name_split(fstr())[1])
except ValueError:
pass
try:
list(_string.formatter_parser(fstr()))
except ValueError:
pass
def get_field(self, field_name, args, kwargs):
try:
first, rest = formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if type(i) == int:
obj = obj[str(i)]
else:
obj = obj[i]
return obj, first
except:
return '{%s}' % field_name, first
def get_field(field_name, kwargs):
"""Return value called 'field_name' from 'kwargs'"""
first, rest = _string.formatter_field_name_split(field_name)
obj = kwargs[first]
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj
def get_field(self, field_name, args, kwargs):
try:
first, rest = formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for _, i in rest:
if isinstance(i, int):
obj = obj[str(i)]
else:
obj = obj[i]
return obj, first
except:
return '{%s}' % field_name, first
def get_field(field_name, args, kwargs):
# Corresponds to string.Formatter.get_field in the stdlib.
first, rest = _string.formatter_field_name_split(field_name)
obj = args[first] if isinstance(first, int) else kwargs[first]
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj
def get_field(self, field_name, args, kwargs):
first, rest = _string.formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
# loop through the rest of the field_name, doing
# getattr or getitem as needed
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj, first
def get_field(self, field_name, args, kwargs):
first, rest = _string.formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
# loop through the rest of the field_name, doing
# getattr or getitem as needed
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj, first
def get_field(self, field_name, kwargs):
"""Return value with key 'field_name' from 'kwargs'"""
first, rest = _string.formatter_field_name_split(field_name)
if first not in kwargs:
return self.kwdefault
obj = kwargs[first]
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj
def get_field(self, field_name, kwargs):
"""Return value with key 'field_name' from 'kwargs'"""
first, rest = _string.formatter_field_name_split(field_name)
if first not in kwargs:
return self.kwdefault
obj = kwargs[first]
for is_attr, i in rest:
if is_attr:
obj = getattr(obj, i)
elif ":" in i:
start, _, stop = i.partition(":")
start = int(start) if start else 0
return obj[start:int(stop)] if stop else obj[start:]
else:
obj = obj[i]
return obj
def text_keywords(text, caller, log_args):
"""
extract keyword arguments from format text
and evaluate them in caller scope.
"""
log_msg = compiled_log_msg_cache.get(text)
# we need to compile the message and add it to the cache
if log_msg is None:
raw_keywords = {
formatter_field_name_split(kw)[0]
for _, kw, _, _ in string_formatter.parse(text, silent=True) if kw
}
keywords = {
kw: kw.translate(valid_chars_transtable)
for kw in raw_keywords if kw not in log_args
}
arguments = {k for k in six.iterkeys(log_args) if k in raw_keywords}
# create a valid log message (some characters aren't allowed)
# and create the code that extracts keyword statements
valid_text = text
code = ["uber_kw = {}"]
for kw, valid_kw in six.iteritems(keywords):
code.append('uber_kw["{vfn}"] = {fn}'.format(vfn=valid_kw, fn=kw))
valid_text = valid_text.replace(kw, valid_kw)
log_msg = CompiledLogMessage(text=valid_text,
keywords=keywords,
arguments=arguments,
code=compile("\n".join(code), '<string>',
'exec'))
compiled_log_msg_cache[text] = log_msg
elif not log_msg.cached:
log_msg.cached = True
compiled_log_msg_cache.capacity += 1
# execute the compiled code in caller context
exec(log_msg.code, caller.f_globals, caller.f_locals)
return log_msg.text, log_msg.keyword_keys, caller.f_locals.pop("uber_kw")
def compile(cls, text, args):
"""
create a UberLogRecord instance of the given text and args
"""
raw_keywords = {formatter_field_name_split(kw)[0] for _, kw, _, _
in string_formatter.parse(text, silent=True) if kw}
keywords = {kw: kw.translate(valid_chars_transtable) for kw
in raw_keywords if kw not in args}
arguments = {k for k in six.iterkeys(args) if k in raw_keywords}
# create a valid log message (some characters aren't allowed)
# and create the code that extracts keyword statements
valid_text = text
code = ["uber_kw = {}"]
for kw, valid_kw in six.iteritems(keywords):
vfn = valid_kw.replace('"', r'\"')
code.append('uber_kw["{vfn}"] = {fn}'.format(vfn=vfn,
fn=kw))
valid_text = valid_text.replace(kw, valid_kw)
keyword_keys = set(six.itervalues(keywords)).union(arguments)
# check if the varnames used for logging are reserved.
# That can be the case if either:
# 1. parsed keywords contain reserved varnames
# 2. kwargs contain reserved varnames
# 3. extra contains reserved varnames
varnames = keyword_keys.union(args.keys())
used_reserved_names = cls.reserved_varnames.intersection(varnames)
if used_reserved_names:
random_reserved_name = choice(list(used_reserved_names))
m = "{fn}() got multiple values for keyword argument: '{reserved}'"
raise TypeError(m.format(fn="log_message",
reserved=random_reserved_name))
return cls(text=valid_text,
keyword_keys=keyword_keys,
code=compile("\n".join(code),
'<string>',
'exec'))
def get_field(self, field_name, args, kwargs):
first, rest = _string.formatter_field_name_split(field_name)
obj = self.get_value(first, args, kwargs)
for is_attr, i in rest:
if is_attr:
# hide private fields
if i.startswith('_'):
obj = ''
else:
obj = getattr(obj, i)
else:
mslice = re.match('^([0-9]+):([0-9]+)$', i)
msuffixgreedy = re.match('^%%(.+)$', i)
msuffix = re.match('^%(.+)$', i) # to test after greedy
msed = re.match('^//([^/]+)/(.*)$', i)
# Note: we could also define
# mprefixgreedy = re.match('^##(.+)$', i)
# mprefix = re.match('^#(.+)$', i)
if mslice:
a, b = map(int, mslice.groups())
obj = obj[a:b]
elif msuffixgreedy:
suffix = msuffixgreedy.groups()[0]
prefix = translate_prefix(reverse(suffix), True)
obj = reverse(re.sub(prefix, '', reverse(obj), count=1))
elif msuffix:
suffix = msuffix.groups()[0]
prefix = translate_prefix(reverse(suffix), False)
obj = reverse(re.sub(prefix, '', reverse(obj), count=1))
elif msed:
glob, dest = msed.groups()
regexp = translate(glob, True)
obj = re.sub(regexp, dest, obj, count=0)
else:
obj = obj[i]
return obj, first
def get_field(self, field_name, args, kwargs):
"""This is just a copy of the base implementation, re-implementing the portion we need"""
if sys.version_info[0] < 3:
first, rest = field_name._formatter_field_name_split()
else:
first, rest = _string.formatter_field_name_split(field_name)
# end py3
try:
obj = self.get_value(first, args, kwargs)
except Exception:
raise AttributeError("Couldn't find value named '%s'" % first)
# end be a bit better here
# loop through the rest of the field_name, doing
# getattr or getitem as needed
prev_attr = None
for is_attr, attr in rest:
if is_attr:
if attr.startswith('as_'):
obj = self._type_by_name(attr[3:])(obj)
else:
try:
obj = getattr(obj, attr)
except AttributeError:
if prev_attr not in self._custom_types:
raise
# end re-raise if key is unknown
# this may re-raise, but in that case we don't have to care
obj = getattr(self._custom_types[prev_attr](obj), attr)
# end try special values
# end handle attribute
else:
obj = obj[attr]
# end handle is_attr
prev_attr = attr
# end for each attribute
return obj, first
def _parse_field_name(field_name):
first, rest = _string.formatter_field_name_split(field_name)
funcs = []
for is_attr, key in rest:
if is_attr:
func = operator.attrgetter
else:
func = operator.itemgetter
try:
if ":" in key:
start, _, stop = key.partition(":")
stop, _, step = stop.partition(":")
start = int(start) if start else None
stop = int(stop) if stop else None
step = int(step) if step else None
key = slice(start, stop, step)
except TypeError:
pass # key is an integer
funcs.append(func(key))
return first, funcs
def get_field(self, field_name: str, args: Sequence[Any],
kwargs: Mapping[str, Any]) -> Any:
first, rest = _string.formatter_field_name_split(field_name.strip())
if (isinstance(first, int) and first < len(args)) or first in kwargs:
obj = self.get_value(first, args, kwargs)
# loop through the rest of the field_name, doing
# getattr or getitem as needed
for is_attr, i in rest:
if is_attr:
if '(' in i:
fct, args = i.split('(')
fct = getattr(obj, fct)
obj = fct(args.strip("\")'"))
else:
obj = getattr(obj, i)
else:
obj = obj[i]
return obj, first
else:
return "{" + field_name + "}", field_name
def arg_of_field_name(self, field_name, args):
x = int(_string.formatter_field_name_split(field_name)[0])
return x if x >= 0 else len(args) + x
def split_format_field_names(format_string) -> Tuple[str, Iterable[Tuple[bool, str]]]:
try:
return _string.formatter_field_name_split(format_string)
except ValueError:
raise IncompleteFormatString()
NoneType = type(None)
NotImplementedType = type(NotImplemented)
QuitterType = type(exit)
WrapperDescriptorType = type(type.__call__)
if PY2:
StrFormatterIteratorType = type(str()._formatter_parser())
StrFieldNameIteratorType = type(str()._formatter_field_name_split()[1])
UnicodeFormatterIteratorType = type(unicode()._formatter_parser())
UnicodeFieldNameIteratorType = type(unicode()._formatter_field_name_split()[1])
if PY3:
import _string
StrFormatterIteratorType = type(_string.formatter_parser(str()))
StrFieldNameIteratorType = type(_string.formatter_field_name_split(str())[1])
if PY3:
ModuleDefType = only(stubtool.name.search_for_type('builtins.moduledef', object))
StdPrinterType = only(stubtool.name.search_for_type('builtins.stderrprinter', object))
if PY33:
ManagedBufferType = type(only(gc.get_referents(memoryview(b''))))
if PY35:
exec('async def func(): pass')
coro = func()
wrapper = coro.__await__()
coro.close()
CoroutineWrapperType = type(wrapper)
del func, coro, wrapper
def split_format_field_names(
format_string) -> Tuple[str, Iterable[Tuple[bool, str]]]:
try:
return _string.formatter_field_name_split(format_string)
except ValueError:
raise IncompleteFormatString()
def formatter_field_name_split(field_name):
if six.PY3:
return _string.formatter_field_name_split(field_name)
else:
return field_name._formatter_field_name_split()
input = formatted
return formatted
def get_field(self, field_name_, args, kwargs):
if not (m := re.match(r'(.+?)(\??[=+]|\?$)(.*)', field_name_,
re.DOTALL)):
return super().get_field(field_name_, args, kwargs)
field_name, operation, text = m.groups()
try:
obj, used_key = super().get_field(field_name, args, kwargs)
except KeyError:
if not operation.startswith('?'):
raise
# consider it used for check_unused_args(), as it's intentionally optional
used_key = _string.formatter_field_name_split(field_name)[0]
return '', used_key
finished_with_basic = operation == '?'
if obj is None or finished_with_basic:
return obj, used_key
# obj exists, now do something special
replaced = text.replace('__value__', obj)
if operation.endswith('='):
return replaced, used_key
if operation.endswith('+'):
return obj + replaced, used_key
raise AssertionError
def convert_field(self, value, conversion):
def split_format_field_names(format_string):
try:
return _string.formatter_field_name_split(format_string)
except ValueError:
raise utils.IncompleteFormatString()
def formatter_field_name_split(field_name):
if six.PY3:
return _string.formatter_field_name_split(field_name)
else:
return field_name._formatter_field_name_split()
def split_format_field_names(format_string):
return _string.formatter_field_name_split(format_string)
def split_format_field_names(format_string):
try:
return _string.formatter_field_name_split(format_string)
except ValueError:
raise utils.IncompleteFormatString()
def split_format_field_names(format_string):
return _string.formatter_field_name_split(format_string)
