def get_format_args(fstr):
# TODO: memoize
formatter = Formatter()
fargs, fkwargs, _dedup = [], [], set()
def _add_arg(argname, type_char='s'):
if argname not in _dedup:
_dedup.add(argname)
argtype = _TYPE_MAP.get(type_char, str) # TODO: unicode
try:
fargs.append((int(argname), argtype))
except ValueError:
fkwargs.append((argname, argtype))
for lit, fname, fspec, conv in formatter.parse(fstr):
if fname is not None:
type_char = fspec[-1:]
fname_list = re.split('[.[]', fname)
if len(fname_list) > 1:
raise ValueError('encountered compound format arg: %r' % fname)
try:
base_fname = fname_list[0]
assert base_fname
except (IndexError, AssertionError):
raise ValueError('encountered anonymous positional argument')
_add_arg(fname, type_char)
for sublit, subfname, _, _ in formatter.parse(fspec):
# TODO: positional and anon args not allowed here.
if subfname is not None:
_add_arg(subfname)
return fargs, fkwargs
def formatMentions(cls, text, *args, **kwargs):
"""Like `str.format`, but takes tuples with a thread id and text instead.
Returns a `Message` object, with the formatted string and relevant mentions.
```
>>> Message.formatMentions("Hey {!r}! My name is {}", ("1234", "Peter"), ("4321", "Michael"))
<Message (None): "Hey 'Peter'! My name is Michael", mentions=[<Mention 1234: offset=4 length=7>, <Mention 4321: offset=24 length=7>] emoji_size=None attachments=[]>
>>> Message.formatMentions("Hey {p}! My name is {}", ("1234", "Michael"), p=("4321", "Peter"))
<Message (None): 'Hey Peter! My name is Michael', mentions=[<Mention 4321: offset=4 length=5>, <Mention 1234: offset=22 length=7>] emoji_size=None attachments=[]>
```
"""
result = ""
mentions = list()
offset = 0
f = Formatter()
field_names = [field_name[1] for field_name in f.parse(text)]
automatic = "" in field_names
i = 0
for (literal_text, field_name, format_spec,
conversion) in f.parse(text):
offset += len(literal_text)
result += literal_text
if field_name is None:
continue
if field_name == "":
field_name = str(i)
i += 1
elif automatic and field_name.isdigit():
raise ValueError(
"cannot switch from automatic field numbering to manual field specification"
)
thread_id, name = f.get_field(field_name, args, kwargs)[0]
if format_spec:
name = f.format_field(name, format_spec)
if conversion:
name = f.convert_field(name, conversion)
result += name
mentions.append(
Mention(thread_id=thread_id, offset=offset, length=len(name)))
offset += len(name)
message = cls(text=result, mentions=mentions)
return message
def format_mentions(text, *args, **kwargs):
"""Like `str.format`, but takes tuples with a thread id and text instead.
Return a tuple, with the formatted string and relevant mentions.
>>> Message.format_mentions("Hey {!r}! My name is {}", ("1234", "Peter"), ("4321", "Michael"))
("Hey 'Peter'! My name is Michael", [Mention(thread_id=1234, offset=4, length=7), Mention(thread_id=4321, offset=24, length=7)])
>>> Message.format_mentions("Hey {p}! My name is {}", ("1234", "Michael"), p=("4321", "Peter"))
('Hey Peter! My name is Michael', [Mention(thread_id=4321, offset=4, length=5), Mention(thread_id=1234, offset=22, length=7)])
"""
result = ""
mentions = list()
offset = 0
f = Formatter()
field_names = [field_name[1] for field_name in f.parse(text)]
automatic = "" in field_names
i = 0
for (literal_text, field_name, format_spec,
conversion) in f.parse(text):
offset += len(literal_text)
result += literal_text
if field_name is None:
continue
if field_name == "":
field_name = str(i)
i += 1
elif automatic and field_name.isdigit():
raise ValueError(
"cannot switch from automatic field numbering to manual field specification"
)
thread_id, name = f.get_field(field_name, args, kwargs)[0]
if format_spec:
name = f.format_field(name, format_spec)
if conversion:
name = f.convert_field(name, conversion)
result += name
mentions.append(
Mention(thread_id=thread_id, offset=offset, length=len(name)))
offset += len(name)
return result, mentions
def template_has_field(template: str, field: str) -> bool:
q = StringFormatter()
for (literal_text, field_name, format_spec,
conversion) in q.parse(template):
if field_name and field in field_name.split():
return True
return False
def SafeStringParse(formatter, s, keys):
"""
A "safe" version :func:`string.Formatter.parse` that will
only parse the input keys specified in ``keys``
Parameters
----------
formatter : string.Formatter
the string formatter class instance
s : str
the string we are formatting
keys : list of str
list of the keys to accept as valid
"""
# the default list of keys
l = list(Formatter.parse(formatter, s))
toret = []
for x in l:
if x[1] in keys:
toret.append(x)
else:
val = x[0]
if x[1] is not None:
fmt = "" if not x[2] else ":%s" % x[2]
val += "{%s%s}" % (x[1], fmt)
toret.append((val, None, None, None))
return iter(toret)
def template_opts(self):
"""Returns the template key/values, if a key from the template
does not exist raises an exception.
"""
template_opts = {}
formatter = Formatter()
keys = [
opt[1] for opt in formatter.parse(self.template)
if opt[1] not in ['sequence']
]
template_opts.update(
sequence=str(self.sequence_number).rjust(self.padding, '0'))
for key in keys:
try:
value = getattr(self, key)
except AttributeError:
raise IdentifierMissingTemplateValue(
f'Required option not provided. Got \'{key}\'.')
else:
if value:
template_opts.update({key: value})
else:
raise IdentifierMissingTemplateValue(
f'Required option cannot be None. Got \'{key}\'.')
return template_opts
def parse_pattern(format_string, env, wrapper=lambda x, y: y):
""" Parse the format_string and return prepared data according to the env.
Pick each field found in the format_string from the env(ironment), apply
the wrapper on each data and return a mapping between field-to-replace and
values for each.
"""
formatter = Formatter()
fields = [x[1] for x in formatter.parse(format_string) if x[1] is not None]
prepared_env = {}
# Create a prepared environment with only used fields, all as list:
for field in fields:
# Search for a movie attribute for each alternative field separated
# by a pipe sign:
for field_alt in (x.strip() for x in field.split('|')):
# Handle default values (enclosed by quotes):
if field_alt[0] in '\'"' and field_alt[-1] in '\'"':
field_values = field_alt[1:-1]
else:
field_values = env.get(field_alt)
if field_values is not None:
break
else:
field_values = []
if not isinstance(field_values, list):
field_values = [field_values]
prepared_env[field] = wrapper(field_alt, field_values)
return prepared_env
def strfdelta(tdelta,
fmt='{D:02}d {H:02}h {M:02}m {S:02}s',
inputtype='timedelta'):
# convert time to our format
if inputtype == 'timedelta':
remainder = int(tdelta.total_seconds())
elif inputtype in ['s', 'seconds']:
remainder = int(tdelta)
elif inputtype in ['m', 'minutes']:
remainder = int(tdelta) * 60
elif inputtype in ['h', 'hours']:
remainder = int(tdelta) * 3600
elif inputtype in ['d', 'days']:
remainder = int(tdelta) * 86400
elif inputtype in ['w', 'weeks']:
remainder = int(tdelta) * 604800
f = Formatter()
desired_fields = [field_tuple[1] for field_tuple in f.parse(fmt)]
possible_fields = ('W', 'D', 'H', 'M', 'S')
constants = {'W': 604800, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
values = {}
for field in possible_fields:
if field in desired_fields and field in constants:
values[field], remainder = divmod(remainder, constants[field])
return f.format(fmt, **values)
def __str__(self):
'Return: \"Format string.\"'
try:
fmt = Formatter()
buff = []
#
for (dl, item, _, _) in fmt.parse(self.msg_format):
if dl: buff.append(dl)
#
if item == 'chat_id':
buff.append(str(Request.get_chat_id(self._msg)))
#
elif item == 'message_id':
buff.append(str(Request.get_message_id(self._msg)))
#
elif item == 'username':
buff.append(Request.get_username(self._msg))
#
elif item == 'chat_type':
buff.append(Request.get_chat_type(self._msg))
#
elif item == 'msg_date':
date = Request.get_msg_date(self._msg)
buff.append(date.strftime(self.dt_format))
#
elif item == 'chat_text':
buff.append(Request.get_chat_text(self._msg))
#
return ''.join(buff)
#
except KeyError:
return
def _open(self, id=None, mode='rb', compression='default', **kwargs):
formatter = Formatter()
fields = [f[1] for f in formatter.parse(self.url)]
assert len(
set(fields).intersection(['id', 'stubbypath', 'pairtreepath'])) > 0
if compression == 'default':
compression = self.compression
if mode == 'wb':
raise NotImplementedError("Mode is not defined")
stubbypath, pairtreepath = None, None
if 'stubbypath' in fields:
stubbypath = id_to_stubbytree(id,
format=self.format,
compression=compression)
if 'pairtreepath' in fields:
pairtreepath = id_to_pairtree(id,
format=self.format,
compression=compression)
path_or_url = self.url.format(id=id,
stubbypath=stubbypath,
pairtreepath=pairtreepath)
try:
byt = _urlopen(path_or_url).read()
req = BytesIO(byt)
except HTTPError:
logging.exception("HTTP Error accessing %s" % path_or_url)
raise
return req
class DescriptionMapMethod(MapMethod):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.formatter = Formatter()
self.translator = Translator()
def map(self, *args, **kwargs):
try:
template = self.additional_args["template"]
locale = kwargs["raw_ad"].feed_in.locale
# We take all keywords (no positional) and we translate them and then we create a dict
# eg:
# template = "That is an example {SALARY}: $ {0} in {COMPANY}: {1}"
# locale = "es_ar"
# translated_words = {"SALARY": "Salario", "COMPANY": "Compañía"}
#
translated_words = {
parse[1]: self.translator.translate(locale, parse[1],
"addesc_label")
for parse in self.formatter.parse(template)
if type(parse[1]) == str and not parse[1].isdigit()
}
addesc = self.formatter.format(template, *args, **translated_words)
return {"addesc": addesc}
except KeyError:
type_id = kwargs["raw_ad"].feed_in.feed_type.id if kwargs.get(
"raw_ad", None) else ""
msg = "El parámetro 'template' no se encuentra definido para " + type_id
raise FeedMappingException(msg)
def SafeStringParse(formatter, s, keys):
"""
A "safe" version :func:`string.Formatter.parse` that will
only parse the input keys specified in ``keys``
Parameters
----------
formatter : string.Formatter
the string formatter class instance
s : str
the string we are formatting
keys : list of str
list of the keys to accept as valid
"""
# the default list of keys
l = list(Formatter.parse(formatter, s))
toret = []
for x in l:
if x[1] in keys:
toret.append(x)
else:
val = x[0]
if x[1] is not None:
fmt = "" if not x[2] else ":%s" %x[2]
val += "{%s%s}" %(x[1], fmt)
toret.append((val, None, None, None))
return iter(toret)
def _has_format(filename):
if not isinstance(filename, str):
return False
formatter = Formatter()
return any(
any(t.isalnum() for t in fields)
for _, fields, *__ in formatter.parse(filename) if fields)
def parse_route_path(path):
fmt = Formatter()
rule = turn_noncapturing(path)
arguments = OrderedDict()
pattern = ''
for literal_text, field_name, format_spec, conversion in fmt.parse(rule):
pattern += literal_text
if field_name is None:
continue
format_spec = format_spec.lower()
subpattern = _route_rule_types.get(format_spec, None)
if subpattern is None:
subpattern = _route_rule_types.get('str')
if field_name in arguments:
err = "The argument \{%s:%s\} are already defined in %s"
err %= (field_name, format_spec, path)
raise SyntaxError(err)
arguments[field_name] = format_spec
pattern += '(?P<' + field_name + '>' + subpattern + ')'
return pattern, arguments
def info(cli):
formatter = Formatter()
for name, attr in cli.cfg.tasks.items():
kind = 'remote'
if attr.python:
kind = 'python'
elif attr.local:
kind = 'local'
elif attr.multi:
kind = 'multi'
elif attr.send:
kind = 'send file'
print(f'{name} [{kind}]:\n\tDescription: {attr.desc}')
values = []
for v in attr.values():
if isinstance(v, list):
values.extend(v)
elif isinstance(v, dict):
values.extend(v.values())
else:
values.append(v)
values = filter(lambda x: isinstance(x, str), values)
fmt_fields = [i[1] for v in values for i in formatter.parse(v) if i[1]]
if fmt_fields:
variables = ', '.join(sorted(set(fmt_fields)))
else:
variables = None
if variables:
print(f'\tVariables: {variables}')
def parse_route_path(path):
fmt = Formatter()
rule = turn_noncapturing(path)
arguments = OrderedDict()
pattern = ''
for literal_text, field_name, format_spec, conversion in fmt.parse(rule):
pattern += literal_text
if field_name is None:
continue
format_spec = format_spec.lower()
subpattern = _route_rule_types.get(format_spec, None)
if subpattern is None:
subpattern = _route_rule_types.get('str')
if field_name in arguments:
err = "The argument \{%s:%s\} are already defined in %s"
err %= (field_name, format_spec, path)
raise SyntaxError(err)
arguments[field_name] = format_spec
pattern += '(?P<' + field_name + '>' + subpattern + ')'
return pattern, arguments
def __init__(self, route_path):
route_path = six.ensure_text(route_path)
assert route_path.startswith('/')
self.org_route_path = route_path
# route_path = route_path[1:]
self.route_path = route_path[:-1] if route_path.endswith(
'/') else route_path
self.key_set = set()
re_segs = []
fmt = Formatter()
for prefix, key, fmt_spec, conversion in fmt.parse(self.route_path):
if key is None:
re_segs.append(prefix)
continue
assert key and not fmt_spec and not conversion, (
'fail parsing parameter from path: ' + self.route_path)
assert key not in self.key_set, 'duplicated key {}'.format(key)
self.key_set.add(key)
re_segs.append('{prefix}(?P<{key}>[^/]+)'.format(prefix=prefix,
key=key))
# print('regex: ' + ''.join(re_segs))
self.regex = re.compile('^' + ''.join(re_segs) + '$')
def parse(self, fmt_string):
for literal_txt, field_name, format_spec, conversion in Formatter.parse(
self, fmt_string):
# Find $foo patterns in the literal text.
continue_from = 0
txt = ""
for m in self._dollar_pattern_ignore_single_quote.finditer(
literal_txt):
new_txt, new_field = m.group(1, 2)
# $$foo --> $foo
if new_field.startswith("$"):
txt += new_txt + new_field
else:
yield (txt + new_txt, new_field, "", None)
txt = ""
continue_from = m.end()
# Re-yield the {foo} style pattern
yield (
txt + literal_txt[continue_from:],
field_name,
format_spec,
conversion,
)
def get_format_names(self, text: str) -> Set[str]:
"""Return the names of any variables mentioned in the templates of `text`"""
formatter = Formatter()
return set(template_name
for _, template_name, _, _ in formatter.parse(text)
if template_name is not None)
def _extract_url_params(self):
from string import Formatter
formatter = Formatter()
params = [
format_tuple[1] for format_tuple in formatter.parse(self.url)
]
return params
def __get_field_list(self, text):
formatter = Formatter()
fields = []
for literal_text, field_name, format_spec, conversion in formatter.parse(
text):
if field_name is not None:
fields.append(field_name)
return fields
def get_format_args(fstr):
"""
Turn a format string into two lists of arguments referenced by the
format string. One is positional arguments, and the other is named
arguments. Each element of the list includes the name and the
nominal type of the field.
# >>> get_format_args("{noun} is {1:d} years old{punct}")
# ([(1, <type 'int'>)], [('noun', <type 'str'>), ('punct', <type 'str'>)])
# XXX: Py3k
>>> get_format_args("{noun} is {1:d} years old{punct}") == \
([(1, int)], [('noun', str), ('punct', str)])
True
"""
# TODO: memoize
formatter = Formatter()
fargs, fkwargs, _dedup = [], [], set()
def _add_arg(argname, type_char="s"):
if argname not in _dedup:
_dedup.add(argname)
argtype = _TYPE_MAP.get(type_char, str) # TODO: unicode
try:
fargs.append((int(argname), argtype))
except ValueError:
fkwargs.append((argname, argtype))
for lit, fname, fspec, conv in formatter.parse(fstr):
if fname is not None:
type_char = fspec[-1:]
fname_list = re.split("[.[]", fname)
if len(fname_list) > 1:
raise ValueError("encountered compound format arg: %r" % fname)
try:
base_fname = fname_list[0]
assert base_fname
except (IndexError, AssertionError):
raise ValueError("encountered anonymous positional argument")
_add_arg(fname, type_char)
for sublit, subfname, _, _ in formatter.parse(fspec):
# TODO: positional and anon args not allowed here.
if subfname is not None:
_add_arg(subfname)
return fargs, fkwargs
def get_format_args(fstr):
"""
Turn a format string into two lists of arguments referenced by the
format string. One is positional arguments, and the other is named
arguments. Each element of the list includes the name and the
nominal type of the field.
# >>> get_format_args("{noun} is {1:d} years old{punct}")
# ([(1, <type 'int'>)], [('noun', <type 'str'>), ('punct', <type 'str'>)])
# XXX: Py3k
>>> get_format_args("{noun} is {1:d} years old{punct}") == \
([(1, int)], [('noun', str), ('punct', str)])
True
"""
# TODO: memoize
formatter = Formatter()
fargs, fkwargs, _dedup = [], [], set()
def _add_arg(argname, type_char='s'):
if argname not in _dedup:
_dedup.add(argname)
argtype = _TYPE_MAP.get(type_char, str) # TODO: unicode
try:
fargs.append((int(argname), argtype))
except ValueError:
fkwargs.append((argname, argtype))
for lit, fname, fspec, conv in formatter.parse(fstr):
if fname is not None:
type_char = fspec[-1:]
fname_list = re.split('[.[]', fname)
if len(fname_list) > 1:
raise ValueError('encountered compound format arg: %r' % fname)
try:
base_fname = fname_list[0]
assert base_fname
except (IndexError, AssertionError):
raise ValueError('encountered anonymous positional argument')
_add_arg(fname, type_char)
for sublit, subfname, _, _ in formatter.parse(fspec):
# TODO: positional and anon args not allowed here.
if subfname is not None:
_add_arg(subfname)
return fargs, fkwargs
def select(self,type='VAV',subtype = ['RM STPT DIAL','ROOM TEMP'], floor = 0,nexp='', pattern ='{i}',cond = 'cmax(x,7)',maxn = 10, dtfilter = ''):
env = self._genv()
env.init()
l = env.getSensorsByType(type)
lname = []
for it in l:
k = it.split(':')
if floor != 0:
fl = int(k[0].split('.')[1])
if fl != floor:
continue
if nexp != '':
sname = k[0].split('.')[3]
if not fnmatch.fnmatch(sname,nexp):
continue
if k[1] in subtype:
lname.append((it,env.getSensorId(it)))
ltemp = sorted(lname,key=itemgetter(0))[:maxn]
filt = ''
i = 0
f = Formatter()
l = "{{%s}}"
f.format(l)
ns = ''
exp = []
i,m =0,len(ltemp)
loopi=0
while i < m:
h = f.parse(pattern)
for a,b,c,d in h :
ns += a
if b is not None :
ns += '{' + str(eval(b)) + '}'
i = eval(b)
loopi = max(i,loopi)
if cond != '':
ns += '.apply(lambda x:' + cond + ')'
if loopi < m-1:
ns += '; '
i = loopi + 1
cs = self.getSeries(ltemp,dtfilter)
re = self.getExpression(ltemp,ns, cs)
series = []
for name ,c in re:
dataserie = defaultdict()
json_s = c[['stime','svalue']].to_json(orient='values')
dataserie['name']=name
dataserie['data']=json_s
series.append(dataserie)
return series
def iter_format_fields(strings, split=False):
formatter = Formatter()
for string in strings:
for _, field, _, _ in formatter.parse(string):
if field is None:
continue
if split:
field = field.partition('.')[0]
yield field
def parse(self, text, silent=False):
try:
for field in StringFormatter.parse(self, text):
yield field
except ValueError:
# If string can't be formatted
# We silence the error and return nothing
if not silent:
raise
def strfdelta(tdelta, fmt):
f = Formatter()
d = {}
lst = {'H': 3600, 'M': 60, 'S': 1}
k = map(lambda x: x[1], list(f.parse(fmt)))
rem = int(tdelta.total_seconds())
for i in ('H', 'M', 'S'):
if i in k and i in lst.keys():
d[i], rem = divmod(rem, lst[i])
return f.format(fmt, **d)
def get_zoned_dates_for_format_string(fmt_string, date):
import pytz
from string import Formatter
sf = Formatter()
variables = {}
for literal_text, field_name, format_spec, conversion in sf.parse(
fmt_string):
if field_name in pytz.common_timezones:
variables[field_name] = date.astimezone(pytz.timezone(field_name))
return variables
def parse_files(sending_list_file, message_file):
text_message = open(message_file).read()
formatter = Formatter()
number_of_fields = len(
[a for a in formatter.parse(text_message) if a[1] is not None])
sending_list = []
line_num = 0
for line in open(sending_list_file).read().splitlines():
line_num = 1
receiver = line.split(",")
if len(receiver) is not number_of_fields + 1:
print(list(formatter.parse(text_message)))
raise Exception(
"The number of fields must be {}. (line {})".format(
number_of_fields + 1, line_num))
sending_list.append(receiver)
return {"message": text_message, "sending_list": sending_list}
def iter_format_fields(strings, split=False):
formatter = Formatter()
for string in strings:
for _, field, _, _ in formatter.parse(string):
if field is None:
continue
field = [
f for f in field.split('.') if '(' not in f and ')' not in f
]
yield field if split else '.'.join(field)
def _parse_with_formatting(string,
args,
kwargs,
*,
recursion_depth=2,
auto_arg_index=0,
recursive=False):
# This function re-implements Formatter._vformat()
if recursion_depth < 0:
raise ValueError("Max string recursion exceeded")
formatter = Formatter()
parser = AnsiParser()
for literal_text, field_name, format_spec, conversion in formatter.parse(
string):
parser.feed(literal_text, raw=recursive)
if field_name is not None:
if field_name == "":
if auto_arg_index is False:
raise ValueError("cannot switch from manual field "
"specification to automatic field "
"numbering")
field_name = str(auto_arg_index)
auto_arg_index += 1
elif field_name.isdigit():
if auto_arg_index:
raise ValueError("cannot switch from manual field "
"specification to automatic field "
"numbering")
auto_arg_index = False
obj, _ = formatter.get_field(field_name, args, kwargs)
obj = formatter.convert_field(obj, conversion)
format_spec, auto_arg_index = Colorizer._parse_with_formatting(
format_spec,
args,
kwargs,
recursion_depth=recursion_depth - 1,
auto_arg_index=auto_arg_index,
recursive=True,
)
formatted = formatter.format_field(obj, format_spec)
parser.feed(formatted, raw=True)
tokens = parser.done()
if recursive:
return AnsiParser.strip(tokens), auto_arg_index
return tokens
def strfdelta(tdelta, fmt):
""" Get a string from a timedelta.
"""
f, d = Formatter(), {}
l = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
k = list(map(lambda x: x[1], list(f.parse(fmt))))
rem = int(tdelta.total_seconds())
for i in ('D', 'H', 'M', 'S'):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(fmt, **d)
def strfdelta(tdelta, fmt):
""" Get a string from a timedelta.
"""
f, d = Formatter(), {}
l = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
k = list(map(lambda x: x[1], list(f.parse(fmt))))
rem = int(tdelta.total_seconds())
for i in ('D', 'H', 'M', 'S'):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(fmt, **d)
def strfdelta(tdelta, fmt):
""" Get a string from a timedelta.
"""
f, d = Formatter(), {}
l = {"D": 86400, "H": 3600, "M": 60, "S": 1}
k = list(map(lambda x: x[1], list(f.parse(fmt))))
rem = int(tdelta.total_seconds())
for i in ("D", "H", "M", "S"):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(fmt, **d)
def tokenize_format_str(fstr, resolve_pos=True):
ret = []
if resolve_pos:
fstr = infer_positional_format_args(fstr)
formatter = Formatter()
for lit, fname, fspec, conv in formatter.parse(fstr):
if lit:
ret.append(lit)
if fname is None:
continue
ret.append(BaseFormatField(fname, fspec, conv))
return ret
def strfdelta(tdelta,
fmt='{D:02}d {H:02}h {M:02}m {S:02}s',
inputtype='timedelta'):
"""Convert a datetime.timedelta object or a regular number to a custom-
formatted string, just like the stftime() method does for datetime.datetime
objects.
The fmt argument allows custom formatting to be specified. Fields can
include seconds, minutes, hours, days, and weeks. Each field is optional.
Some examples:
'{D:02}d {H:02}h {M:02}m {S:02}s' --> '05d 08h 04m 02s' (default)
'{W}w {D}d {H}:{M:02}:{S:02}' --> '4w 5d 8:04:02'
'{D:2}d {H:2}:{M:02}:{S:02}' --> ' 5d 8:04:02'
'{H}h {S}s' --> '72h 800s'
The inputtype argument allows tdelta to be a regular number instead of the
default, which is a datetime.timedelta object. Valid inputtype strings:
's', 'seconds',
'm', 'minutes',
'h', 'hours',
'd', 'days',
'w', 'weeks'
:param tdelta: time (in datetime or integer)
:param fmt: the desired format
:param inputtype: type of input
:return: formatted time
"""
# Convert tdelta to integer seconds.
if inputtype == 'timedelta':
remainder = int(tdelta.total_seconds())
elif inputtype in ['s', 'seconds']:
remainder = int(tdelta)
elif inputtype in ['m', 'minutes']:
remainder = int(tdelta) * 60
elif inputtype in ['h', 'hours']:
remainder = int(tdelta) * 3600
elif inputtype in ['d', 'days']:
remainder = int(tdelta) * 86400
elif inputtype in ['w', 'weeks']:
remainder = int(tdelta) * 604800
f = Formatter()
desired_fields = [field_tuple[1] for field_tuple in f.parse(fmt)]
possible_fields = ('W', 'D', 'H', 'M', 'S')
constants = {'W': 604800, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
values = {}
for field in possible_fields:
if field in desired_fields and field in constants:
values[field], remainder = divmod(remainder, constants[field])
return f.format(fmt, **values)
def getTimeFromTdelta(tdelta, fmt):
f = Formatter()
d = {}
l = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
k = map(lambda x: x[1], list(f.parse(fmt)))
rem = int(tdelta.total_seconds())
for i in ('D', 'H', 'M', 'S'):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(fmt, **d)
def strfdelta(tdelta: datetime.timedelta, fmt: str) -> str:
f = Formatter()
d = {}
l = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
k = list(map(lambda x: x[1], list(f.parse(fmt))))
rem = int(tdelta.total_seconds())
for i in ('D', 'H', 'M', 'S'):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(fmt, **d)
def parse(self, fmt_string):
for literal_txt, field_name, format_spec, conversion \
in Formatter.parse(self, fmt_string):
# Find $foo patterns in the literal text.
continue_from = 0
for m in self._dollar_pattern.finditer(literal_txt):
new_txt, new_field = m.group(1,2)
yield (new_txt, new_field, "", None)
continue_from = m.end()
# Re-yield the {foo} style pattern
yield (literal_txt[continue_from:], field_name, format_spec, conversion)
def save_params(self):
f = Formatter()
tokens = f.parse(self.command_string)
params = []
for (_ , param_name, _ , _) in tokens:
if param_name is not None:
if param_name in self.inputdict:
param = self.inputdict[param_name]
else:
param = iobjs.Input(name=param_name)
if param_name in self._defaults:
param.default = self._defaults[param_name]
params += [param]
self.inputs = params
def strfdelta(tdelta, fmt='{D:02}d {H:02}h {M:02}m {S:02}s', inputtype='timedelta'):
"""Convert a datetime.timedelta object or a regular number to a custom-
formatted string, just like the stftime() method does for datetime.datetime
objects.
The fmt argument allows custom formatting to be specified. Fields can
include seconds, minutes, hours, days, and weeks. Each field is optional.
Some examples:
'{D:02}d {H:02}h {M:02}m {S:02}s' --> '05d 08h 04m 02s' (default)
'{W}w {D}d {H}:{M:02}:{S:02}' --> '4w 5d 8:04:02'
'{D:2}d {H:2}:{M:02}:{S:02}' --> ' 5d 8:04:02'
'{H}h {S}s' --> '72h 800s'
The inputtype argument allows tdelta to be a regular number instead of the
default, which is a datetime.timedelta object. Valid inputtype strings:
's', 'seconds',
'm', 'minutes',
'h', 'hours',
'd', 'days',
'w', 'weeks'
"""
# Convert tdelta to integer seconds.
if inputtype == 'timedelta':
remainder = int(tdelta.total_seconds())
elif inputtype in ['s', 'seconds']:
remainder = int(tdelta)
elif inputtype in ['m', 'minutes']:
remainder = int(tdelta)*60
elif inputtype in ['h', 'hours']:
remainder = int(tdelta)*3600
elif inputtype in ['d', 'days']:
remainder = int(tdelta)*86400
elif inputtype in ['w', 'weeks']:
remainder = int(tdelta)*604800
f = Formatter()
desired_fields = [field_tuple[1] for field_tuple in f.parse(fmt)]
possible_fields = ('W', 'D', 'H', 'M', 'S')
constants = {'W': 604800, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
values = {}
for field in possible_fields:
if field in desired_fields and field in constants:
values[field], remainder = divmod(remainder, constants[field])
return f.format(fmt, **values)
def _wrapped(pseudo_type, string, **kwargs):
text = []
formatter = Formatter()
for (literal_text, field_name, format_spec, conversion) in formatter.parse(string):
if literal_text:
literal_text = next_splitter_or_func(
literal_text, self.splitters, func, pseudo_type)
literal_text = literal_text.replace('{', '{{').replace('}', '}}')
text.append(literal_text)
if field_name is not None:
fmt = field_name
if conversion is not None:
fmt += '!' + conversion
if format_spec:
fmt += ':' + format_spec
text.append('{%s}' % (fmt, ))
return "".join(text)
def strfdelta(tdelta, fmt=None):
from string import Formatter
if not fmt:
# The standard, most human readable format.
fmt = "{D} days {H:02} hours {M:02} minutes {S:02} seconds"
if tdelta == timedelta():
return "0 minutes"
formatter = Formatter()
return_map = {}
div_by_map = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
keys = map(lambda x: x[1], list(formatter.parse(fmt)))
remainder = int(tdelta.total_seconds())
for unit in ('D', 'H', 'M', 'S'):
if unit in keys and unit in div_by_map.keys():
return_map[unit], remainder = divmod(remainder, div_by_map[unit])
return formatter.format(fmt, **return_map)
def get_params(self):
formatter = Formatter()
format_iterator = formatter.parse(self.get_cleaned_content())
params = {
'args': list(),
'kwargs': set()
}
for _tuple in format_iterator:
field_name = _tuple[1]
if field_name is not None:
if field_name == '':
params['args'].append(field_name)
elif field_name.isdigit():
if field_name not in params['args']:
params['args'].append(field_name)
else:
params['kwargs'].add(field_name)
return params
def parse(self, fmt_string):
for literal_txt, field_name, format_spec, conversion in Formatter.parse(self, fmt_string):
# Find $foo patterns in the literal text.
continue_from = 0
txt = ""
for m in self._dollar_pattern.finditer(literal_txt):
new_txt, new_field = m.group(1, 2)
# $$foo --> $foo
if new_field.startswith("$"):
txt += new_txt + new_field
else:
yield (txt + new_txt, new_field, "", None)
txt = ""
continue_from = m.end()
# Re-yield the {foo} style pattern
yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
def tokenize_format_str(fstr, resolve_pos=True):
"""Takes a format string, turns it into a list of alternating string
literals and :class:`BaseFormatField` tokens. By default, also
infers anonymous positional references into explict, numbered
positional references. To disable this behavior set *resolve_pos*
to ``False``.
"""
ret = []
if resolve_pos:
fstr = infer_positional_format_args(fstr)
formatter = Formatter()
for lit, fname, fspec, conv in formatter.parse(fstr):
if lit:
ret.append(lit)
if fname is None:
continue
ret.append(BaseFormatField(fname, fspec, conv))
return ret
class BaseManager(object):
"""Base class for Endpoint Manager objects."""
url = ''
def __init__(self, api_client):
self.api_client = api_client
self._formatter = StringFormatter()
def _get_format_kwargs(self, **kwargs):
it = self._formatter.parse(self.url)
output = {i[1]: '' for i in it}
for key in output.keys():
if kwargs.get(key):
output[key] = kwargs[key]
if 'endpoint' in output.keys():
output.pop('endpoint')
return output
def get_url(self,
endpoint,
kwargs,
authorized_args=[]):
"""Returns the required url for a request against CloudKitty's API.
:param endpoint: The endpoint on which the request should be done
:type endpoint: str
:param kwargs: kwargs that will be used to build the query (part after
'?' in the url) and to format the url.
:type kwargs: dict
:param authorized_args: The arguments that are authorized in url
parameters
:type authorized_args: list
"""
query_kwargs = {
key: kwargs[key] for key in authorized_args
if kwargs.get(key, None)
}
kwargs = self._get_format_kwargs(**kwargs)
url = self.url.format(endpoint=endpoint, **kwargs)
query = urlencode(query_kwargs)
if query:
url += '?' + query
return url
def strfdelta(tsec, format_str="P", format_no_day="PT{H}H{M}M{S}S", format_zero="PT0S"):
"""Formatting the time duration.
Duration ISO8601 format (PnYnMnDTnHnMnS): http://en.wikipedia.org/wiki/ISO_8601
Choosing the format P[nD]TnHnMnS where days is the total number of days (if not 0), 0 values may be omitted,
0 duration is PT0S
:param tsec: float, number of seconds
:param format_str: Format string, ISO 8601 is "P{D}DT{H}H{M}M{S}S". Default is a format string "P": will
use ISO 8601 but skip elements that have 0 value, e.g. P1H7S instead of P1H0M7S
:param format_no_day: Format string, ISO 8601, default "PT{H}H{M}M{S}S"
:param format_zero: format for when tsec is 0 or None, default "PT0S"
:return: Formatted time duration
"""
if not tsec:
# 0 or None
return format_zero
f = Formatter()
d = {}
l = {'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
rem = long(tsec)
if format_str == "P":
# variable format
if 0 < tsec < 86400:
format_str = "PT"
for i in ('D', 'H', 'M', 'S'):
if i in l.keys():
d[i], rem = divmod(rem, l[i])
if d[i] != 0:
format_str = "%s{%s}%s" % (format_str, i, i)
else:
if 0 < tsec < 86400:
format_str = format_no_day
k = map(lambda x: x[1], list(f.parse(format_str)))
for i in ('D', 'H', 'M', 'S'):
if i in k and i in l.keys():
d[i], rem = divmod(rem, l[i])
return f.format(format_str, **d)
def resolve_conf(unresolved):
"""take a config with values using string formatting; apply string formatting until they all don't;
returns (dict of things that could be resolved, dict of things that couldn't be resolved)"""
f = Formatter()
resolved = {}
while unresolved:
changed_something, missing_defs = False, []
for k, v in unresolved.items():
if isinstance(v, basestring) and v and tuple(f.parse(v))[0][1] is not None:
try:
unresolved[k] = f.vformat(v, [], resolved)
changed_something = True
except KeyError, e: # missing a definition; but could in the next pass if we changed something
missing_defs.append(e.args[0])
else:
# non strings are, by definition, resolved; no recursion into complex data structures here
# items without format specifiers are also resolved
del unresolved[k]
resolved[k] = v
changed_something = True
if not changed_something:
break
def template_opts(self):
"""Returns the template key/values, if a key from the template
does not exist raises an exception.
"""
template_opts = {}
formatter = Formatter()
keys = [opt[1] for opt in formatter.parse(
self.template) if opt[1] not in ['sequence']]
template_opts.update(
sequence=str(self.sequence_number).rjust(self.padding, '0'))
for key in keys:
try:
value = getattr(self, key)
except AttributeError:
raise IdentifierMissingTemplateValue(
f'Required option not provided. Got \'{key}\'.')
else:
if value:
template_opts.update({key: value})
else:
raise IdentifierMissingTemplateValue(
f'Required option cannot be None. Got \'{key}\'.')
return template_opts
def main(argv):
out = sys.stdout
if len(argv) < 2:
sys.stderr.write('''
usage: %(prog)s format
'''.lstrip() % {
'prog': argv[0]
})
return 1
if argv[1] == '--tools':
out.write('\n'.join([
tool.name for tool in TOOLS
]) + '\n')
out.flush()
return 0
format = argv[1]
formatter = Formatter()
try:
tool = get_tool()
except NoToolError:
return 0
values = {}
for null, field_name, null, null in formatter.parse(format):
if not field_name:
continue
values[field_name] = getattr(tool, field_name)
out.write(format.format(**values))
out.flush()
return 0
def compute_one_task(self, itask, task):
"""
Run the algorithm once, using the parameters specified by `task`,
which is the `itask` iteration
Parameters
----------
itask : int
the integer index of this task
task : tuple
a tuple of values representing this task value
"""
# if you are the pool's root, write out the temporary parameter file
this_config = None
if self.workers.subcomm.rank == 0:
# initialize a temporary file
with tempfile.NamedTemporaryFile(delete=False) as ff:
this_config = ff.name
logger.debug("creating temporary file: %s" %this_config)
# key/values for this task
if len(self.task_dims) == 1:
possible_kwargs = {self.task_dims[0] : task}
else:
possible_kwargs = dict(zip(self.task_dims, task))
# any extra key/value pairs for this tasks
if self.extras is not None:
for k in self.extras:
possible_kwargs[k] = self.extras[k][itask]
# use custom formatter that only formats the possible keys, ignoring other
# occurences of curly brackets
formatter = Formatter()
formatter.parse = lambda l: SafeStringParse(formatter, l, list(possible_kwargs))
kwargs = [kw for _, kw, _, _ in formatter.parse(self.template) if kw]
# do the string formatting if the key is present in template
valid = {k:possible_kwargs[k] for k in possible_kwargs if k in kwargs}
ff.write(formatter.format(self.template, **valid).encode())
# bcast the file name to all in the worker pool
this_config = self.workers.subcomm.bcast(this_config, root=0)
# configuration file passed via -c
params, extra = ReadConfigFile(open(this_config, 'r').read(), self.algorithm_class.schema)
# output is required
output = getattr(extra, 'output', None)
if output is None:
raise ValueError("argument `output` is required in config file")
# initialize the algorithm and run
alg = self.algorithm_class(**vars(params))
result = alg.run()
alg.save(output, result)
# remove temporary files
if self.workers.subcomm.rank == 0:
if os.path.exists(this_config):
logger.debug("removing temporary file: %s" %this_config)
os.remove(this_config)
return 0
class RelayProtocol(LineOnlyReceiver):
MAX_LENGTH = 65536
def __init__(self, factory):
self.factory = factory
self._ids = set()
self._format = Formatter()
def error(self, s):
return self.reply({
'id': 0,
'state': 'error',
'message': s
})
def reply(self, msg={}):
try:
self.transport.write(json.dumps(msg) + self.delimiter)
return True
except (ValueError, TypeError):
return self.error('unable to serialise response (internal error)')
def parse(self, line):
try:
deserialised = json.loads(line)
except (ValueError, TypeError):
return self.error('unable to deserialise request (client error)')
if 'request' not in deserialised:
return self.error('missing required parameter: request')
if 'id' not in deserialised:
return self.error('missing required parameter: id')
id = deserialised.get('id', '')
request = deserialised.get('request', '')
params = deserialised.get('params', {})
data = deserialised.get('data', {})
request_config = self.factory.get_request(request)
if not request_config:
return self.error('not a valid request: {}'.format(request))
if len(request_config) != 5:
return self.error('request tuple does not have three elements: {}'.format(request))
request_method, request_url, expected_status, request_auth, parser = request_config
if not request_auth:
request_auth = NullAuth()
if not parser:
parser = NullParser()
for (lit, field, spec, conv) in self._format.parse(request_url):
if field and field not in params:
return self.error('missing required parameter: {}'.format(field))
try:
parsed_url = request_url.format(**params)
except KeyError:
return self.error('unable to parse request url with provided paramters')
self.send_request(
id=id,
method=request_method,
url=parsed_url,
auth=request_auth,
data=data,
expected=expected_status,
parser=parser
)
def send_request(self, id='', method='GET', url='', auth=None, data='', expected=200, parser=None):
request = HttpRequest(
id=id,
method=method,
url=url,
expected=expected,
parser=parser
)
if data:
request.set_body(data)
request = auth.apply_auth(request)
HttpTransport(request).go(self.got_response)
self.reply({
'id': str(request.id),
'status': 'dispatched',
})
def got_response(self, response):
self.reply({
'id': response.request.id,
'status': 'ok' if response.ok() else 'error',
'response': response.data(),
'time': round(time.time() - response.request.start_time, 5),
})
def lineReceived(self, line):
self.parse(line)
def extract_used_fields(text):
fmtr = Formatter()
return [i[1] for i in fmtr.parse(text) if i[1]]
import polib
from string import Formatter
import glob
filelist = glob.glob("*.po")
pos = {filename: polib.pofile(filename) for filename in filelist}
formatter = Formatter()
for name, po in pos.iteritems():
print "Testing", name
for entry in po:
if len(entry.msgstr) > 0:
try:
ids = [field_name
for literal_text, field_name, format_spec, conversion
in formatter.parse(entry.msgid)]
tids = [field_name
for literal_text, field_name, format_spec, conversion
in formatter.parse(entry.msgstr)]
except Exception as e:
print "Got exception!", e, "for entry", entry.msgid
else:
if tids is not None:
missing = [name for name in tids
if name is not None and name not in ids]
if len(missing) > 0:
print "Missing parameters", missing, \
"in translation of", entry.msgid
def createScriptFile(script_fp,
xmlfile,
nlogofile,
experiment,
combination_nr,
script_template,
csv_output_dir = "./"
):
"""
Create a script file from a template string.
Parameters
----------
script_fp : file pointer
File opened for writing.
xmlfile : string
File name and path of the xml experiment file.
This string will be accessible through the key {setup}
in the script_template string.
nlogofile : string
File name and path of the ,nlogo model file.
This string will be accessible through the key {model}
in the script_template string.
experiment : string
Name of the experiment.
This string will be accessible through the key {experiment}
in the script_template string.
combination_nr : int
The experiment combination number.
This value will be accessible through the key {combination}
in the script_template string.
script_template : str
The script template string. This string will be cloned for each script
but the following keys can be used and will have individual values.
{job} - Name of the job. Will be the name of the xml-file (minus extension).
{combination} - The value of the parameter combination_nr.
{experiment} - The value of the parameter experiment.
{csv} - File name, including full path, of a experiment-unique csv-file.
{setup} - The value of the parameter csvfile.
{model} - The value of the parameter nlogofile.
{csvfname} - Only the file name part of the {csv} key.
{csvfpath} - Only the path part of the {csv} key.
csv_output_dir : str, optional
Path to the directory used when constructing the {csv} and {csvfpath}
keys.
Returns
-------
file_name : str
Name of the file name used for the script.
"""
jobname = os.path.splitext(os.path.basename(xmlfile))[0]
fname = jobname + ".csv"
csvfile = os.path.join(csv_output_dir, fname)
strformatter = Formatter()
formatmap = {
"job" : jobname,
"combination" : combination_nr,
"experiment" : experiment,
"csv" : csvfile,
"setup" : xmlfile,
"model" : nlogofile,
"csvfname" : fname,
"csvfpath" : csv_output_dir
}
# Use string formatter to go through the script template and
# look for unknown keys. Do not replace them, but print warning.
for lt, fn, fs, co in strformatter.parse(script_template):
if fn != None and fn not in formatmap.keys():
print("Warning: Unsupported key '{{{0}}}' in script template. Ignoring."\
.format(fn))
formatmap[fn] = "{" + fn + "}"
script_fp.write(script_template.format(**formatmap))
