"""Returns (drive, parts) of the given filepath"""
parts = []
drive, _ = os.path.splitdrive(filepath)
while True:
newpath, tail = os.path.split(filepath)
if newpath == filepath:
assert not tail
if filepath: parts.append(filepath)
break
parts.append(tail)
filepath = newpath
parts.reverse()
return drive, parts
def block_indent(string, spaces=4):
"""Returns the given multiline string indented by the number of spaces given"""
return "\n".join((spaces * " ") + i for i in string.splitlines())
class SafeDict(dict):
def __missing__(self, key):
return '{' + key + '}'
FORMATTER = string.Formatter()
def safe_format(string, *args, **kwargs):
return FORMATTER.vformat(string, args, SafeDict(**kwargs))
funcname = method_spec.get('method_name', None)
if funcname:
request_func.__name__ = funcname
docstring = method_spec.get('docstring', None)
if docstring:
request_func.__doc__ = docstring
assert callable(
output_trans
), f'output_trans {output_trans} is not callable, try again'
return request_func
DFLT_METHOD_FUNC_FROM_METHOD_SPEC = mk_request_function
str_formatter = string.Formatter()
class Py2Request(object):
""" Make a class that has methods that offer a python interface to web requests """
def __init__(
self,
method_specs=None,
method_func_from_method_spec=DFLT_METHOD_FUNC_FROM_METHOD_SPEC,
**kwargs,
):
"""
Initialize the object with web request calling methods.
You can also just make an empty Py2Request object, and inject methods later on, one by one.
from pyupgrade._data import Settings
from pyupgrade._data import Version
from pyupgrade._data import visit
from pyupgrade._string_helpers import is_ascii
from pyupgrade._string_helpers import is_codec
from pyupgrade._token_helpers import CLOSING
from pyupgrade._token_helpers import KEYWORDS
from pyupgrade._token_helpers import OPENING
from pyupgrade._token_helpers import parse_call_args
from pyupgrade._token_helpers import remove_brace
DotFormatPart = Tuple[str, Optional[str], Optional[str], Optional[str]]
FUNC_TYPES = (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef)
_stdlib_parse_format = string.Formatter().parse
def parse_format(s: str) -> Tuple[DotFormatPart, ...]:
"""Makes the empty string not a special case. In the stdlib, there's
loss of information (the type) on the empty string.
"""
parsed = tuple(_stdlib_parse_format(s))
if not parsed:
return ((s, None, None, None),)
else:
return parsed
def unparse_parsed_string(parsed: Sequence[DotFormatPart]) -> str:
def _convert_tup(tup: DotFormatPart) -> str:
def _replace_variables(data, variables):
"""Replace the format variables in all items of data."""
formatter = string.Formatter()
return [formatter.vformat(item, [], variables) for item in data]
def _get_path_params_from_url(self, url):
formatter = string.Formatter()
path_params = [item[1] for item in formatter.parse(url)]
return filter(None, path_params)
def parse_template_fields(str_template):
return {
field
for (_, field, _, _) in string.Formatter().parse(str_template)
}
# Copyright (c) 2017-2020, Lawrence Livermore National Security, LLC and
# other Shroud Project Developers.
# See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (BSD-3-Clause)
from __future__ import print_function
from __future__ import absolute_import
import collections
import os
import string
from . import metadata
fmt = string.Formatter()
def wformat(template, dct):
# shorthand, wrap fmt.vformat
assert template is not None
try:
return fmt.vformat(template, None, dct)
except AttributeError:
# raise # uncomment for detailed backtrace
# use %r to avoid expanding tabs
raise SystemExit("Error with template: " + "%r" % template)
def append_format(lstout, template, fmt):
"""Format template and append to lstout.
def extract_string_vars(template_path):
with open(template_path, "r") as template_file:
template_str = template_file.read()
return string.Formatter().parse(template_str)
class SQL(Composable):
"""
A `Composable` representing a snippet of SQL statement.
`!SQL` exposes `join()` and `format()` methods useful to create a template
where to merge variable parts of a query (for instance field or table
names).
The *string* doesn't undergo any form of escaping, so it is not suitable to
represent variable identifiers or values: you should only use it to pass
constant strings representing templates or snippets of SQL statements; use
other objects such as `Identifier` or `Literal` to represent variable
parts.
Example::
>>> query = sql.SQL("SELECT {0} FROM {1}").format(
... sql.SQL(', ').join([sql.Identifier('foo'), sql.Identifier('bar')]),
... sql.Identifier('table'))
>>> print(query.as_string(conn))
SELECT "foo", "bar" FROM "table"
"""
_obj: str
_formatter = string.Formatter()
def __init__(self, obj: str):
super().__init__(obj)
if not isinstance(obj, str):
raise TypeError(f"SQL values must be strings, got {obj!r} instead")
def as_string(self, context: Optional[AdaptContext]) -> str:
return self._obj
def as_bytes(self, context: Optional[AdaptContext]) -> bytes:
enc = "utf-8"
if context:
conn = context.connection
if conn:
enc = pgconn_encoding(conn.pgconn)
return self._obj.encode(enc)
def format(self, *args: Any, **kwargs: Any) -> Composed:
"""
Merge `Composable` objects into a template.
:param args: parameters to replace to numbered (``{0}``, ``{1}``) or
auto-numbered (``{}``) placeholders
:param kwargs: parameters to replace to named (``{name}``) placeholders
:return: the union of the `!SQL` string with placeholders replaced
:rtype: `Composed`
The method is similar to the Python `str.format()` method: the string
template supports auto-numbered (``{}``), numbered (``{0}``,
``{1}``...), and named placeholders (``{name}``), with positional
arguments replacing the numbered placeholders and keywords replacing
the named ones. However placeholder modifiers (``{0!r}``, ``{0:<10}``)
are not supported.
If a `!Composable` objects is passed to the template it will be merged
according to its `as_string()` method. If any other Python object is
passed, it will be wrapped in a `Literal` object and so escaped
according to SQL rules.
Example::
>>> print(sql.SQL("SELECT * FROM {} WHERE {} = %s")
... .format(sql.Identifier('people'), sql.Identifier('id'))
... .as_string(conn))
SELECT * FROM "people" WHERE "id" = %s
>>> print(sql.SQL("SELECT * FROM {tbl} WHERE name = {name}")
... .format(tbl=sql.Identifier('people'), name="O'Rourke"))
... .as_string(conn))
SELECT * FROM "people" WHERE name = 'O''Rourke'
"""
rv: List[Composable] = []
autonum: Optional[int] = 0
for pre, name, spec, conv in self._formatter.parse(self._obj):
if spec:
raise ValueError("no format specification supported by SQL")
if conv:
raise ValueError("no format conversion supported by SQL")
if pre:
rv.append(SQL(pre))
if name is None:
continue
if name.isdigit():
if autonum:
raise ValueError(
"cannot switch from automatic field numbering to manual"
)
rv.append(args[int(name)])
autonum = None
elif not name:
if autonum is None:
raise ValueError(
"cannot switch from manual field numbering to automatic"
)
rv.append(args[autonum])
autonum += 1
else:
rv.append(kwargs[name])
return Composed(rv)
def join(self, seq: Iterable[Composable]) -> Composed:
"""
Join a sequence of `Composable`.
:param seq: the elements to join.
:type seq: iterable of `!Composable`
Use the `!SQL` object's *string* to separate the elements in *seq*.
Note that `Composed` objects are iterable too, so they can be used as
argument for this method.
Example::
>>> snip = sql.SQL(', ').join(
... sql.Identifier(n) for n in ['foo', 'bar', 'baz'])
>>> print(snip.as_string(conn))
"foo", "bar", "baz"
"""
rv = []
it = iter(seq)
try:
rv.append(next(it))
except StopIteration:
pass
else:
for i in it:
rv.append(self)
rv.append(i)
return Composed(rv)
def format_config(s, *d):
dd = merge_dicts(*d)
return string.Formatter().vformat(s, [], SafeFormat(dd))
def create_variables(ws,
expression,
nbins=None,
bins=None,
values=None,
ranges=None,
index_names=None):
if nbins is not None:
nbins = np.atleast_1d(nbins)
if bins is not None:
if not isinstance(bins[0], list) and not isinstance(bins[0], tuple):
bins = [bins]
if values is not None:
values = np.atleast_1d(values)
if ranges is not None:
ranges = np.atleast_1d(ranges)
if np.iterable(values) and nbins is None:
nbins = np.shape(values)
if bins is not None and nbins is None:
nbins = [len(b) for b in bins]
if nbins is not None and bins is None:
bins = [string_range(nb) for nb in nbins]
if nbins is not None and bins is not None:
if [len(b) for b in bins] != list(nbins):
raise ValueError('nbins=%s and bins=%s does not match' %
(nbins, bins))
if nbins is None or nbins == (1, ):
return create_scalar(ws, expression,
None if values is None else values[0], ranges)
if values is not None and values.shape != nbins:
raise ValueError('values has wrong shape, should be %s, it is %s' %
(nbins, values.shape))
index_names = np.atleast_1d(index_names)
logging.debug(
'after normalizations inputs are: expression=%s, nbins=%s, bins=%s, values=%s, ranges=%s, index_names=%s',
expression, nbins, bins, values, ranges, index_names)
if np.sum([b != 1 for b in nbins]) != np.sum(
[idx is not None for idx in index_names]):
logging.debug('trying to determine indexes in expression %s',
expression)
possible_indexes = set([
tup[1] for tup in string.Formatter().parse(expression)
if tup[1] is not None
])
if values is not None:
if 'value' in possible_indexes:
possible_indexes.remove('value')
if len(possible_indexes) == 0:
raise ValueError('there is no index in expression %s' % expression)
if len(possible_indexes) != 1:
raise ValueError(
'cannot determine which index to use in expression %s, possible indexes: %s'
% (expression, possible_indexes))
index_names = list(possible_indexes)
logging.debug('index_names = %s', index_names)
if len(nbins) == 1:
results = []
if values is None:
for b in bins[0]:
results.append(
create_variables(ws,
expression.format(**{index_names[0]: b})))
else:
for v, b in zip(values, bins[0]):
results.append(
create_variables(ws,
expression.format(**{
index_names[0]: b,
'value': '{value}'
}),
values=v,
ranges=ranges))
return results
else:
if len(index_names) != len(nbins):
raise ValueError('cannot find all the index')
results = []
if values is None:
for ib, b in enumerate(bins[0]):
logging.debug('calling with expression=%s, bins=%s',
expression.replace('{%s}' % index_names[0], b),
bins[1:])
results.append(
create_variables(ws,
expression.replace(
'{%s}' % index_names[0], b),
bins=bins[1:]))
else:
for ib, b in enumerate(bins[0]):
logging.debug('calling with expression=%s, values=%s, bins=%s',
expression.replace('{%s}' % index_names[0], b),
values[ib, :], bins[1:])
results.append(
create_variables(ws,
expression.replace(
'{%s}' % index_names[0], b),
values=values[ib, :],
bins=bins[1:]))
return results
def format_partial(fmt, *args, **kwargs):
class SafeDict(dict):
def __missing__(self, key):
return '{' + key + '}'
return string.Formatter().vformat(fmt, args, SafeDict(**kwargs))
def _get_api_args(uri):
return [api_arg
for _, api_arg, _, _ in string.Formatter().parse(uri)]
def format(self, ref: DatasetRef) -> str:
"""Format a template string into a full path.
Parameters
----------
ref : `DatasetRef`
The dataset to be formatted.
Returns
-------
path : `str`
Expanded path.
Raises
------
KeyError
Raised if the requested field is not defined and the field is
not optional. Or, `component` is specified but "component" was
not part of the template.
"""
# Extract defined non-None dimensions from the dataId.
# This guards against Nones being explicitly present in the data ID
# (which can happen if, say, an exposure has no filter), as well as
# the case where only required dimensions are present (which in this
# context should only happen in unit tests; in general we need all
# dimensions to fill out templates).
fields = {
k: ref.dataId.get(k)
for k in ref.datasetType.dimensions.names
if ref.dataId.get(k) is not None
}
# Extra information that can be included using . syntax
extras = {}
if isinstance(ref.dataId, DataCoordinate):
if ref.dataId.hasRecords():
extras = ref.dataId.records.byName()
skypix_alias = self._determine_skypix_alias(ref)
if skypix_alias is not None:
fields["skypix"] = fields[skypix_alias]
if extras:
extras["skypix"] = extras[skypix_alias]
datasetType = ref.datasetType
fields["datasetType"], component = datasetType.nameAndComponent()
usedComponent = False
if component is not None:
fields["component"] = component
usedRun = False
fields["run"] = ref.run
fmt = string.Formatter()
parts = fmt.parse(self.template)
output = ""
for literal, field_name, format_spec, conversion in parts:
if field_name == "component":
usedComponent = True
if format_spec is None:
output = output + literal
continue
# Should only happen if format_spec is None
if field_name is None:
raise RuntimeError(
f"Unexpected blank field_name encountered in {self.template} [{literal}]"
)
if "?" in format_spec:
optional = True
# Remove the non-standard character from the spec
format_spec = format_spec.replace("?", "")
else:
optional = False
if field_name == "run":
usedRun = True
if field_name == "collection":
raise KeyError("'collection' is no longer supported as a "
"file template placeholder; use 'run' instead.")
# Check for request for additional information from the dataId
if "." in field_name:
primary, secondary = field_name.split(".")
if primary in extras:
record = extras[primary]
# Only fill in the fields if we have a value, the
# KeyError will trigger below if the attribute is missing.
if hasattr(record, secondary):
fields[field_name] = getattr(record, secondary)
if field_name in fields:
value = fields[field_name]
elif optional:
# If this is optional ignore the format spec
# and do not include the literal text prior to the optional
# field unless it contains a "/" path separator
format_spec = ""
value = ""
if "/" not in literal:
literal = ""
else:
raise KeyError(
f"'{field_name}' requested in template via '{self.template}' "
"but not defined and not optional")
# Handle "/" in values since we do not want to be surprised by
# unexpected directories turning up
replace_slash = True
if "/" in format_spec:
# Remove the non-standard character from the spec
format_spec = format_spec.replace("/", "")
replace_slash = False
if isinstance(value, str):
# Replace spaces with underscores for more friendly file paths
value = value.replace(" ", "_")
if replace_slash:
value = value.replace("/", "_")
# Now use standard formatting
output = output + literal + format(value, format_spec)
# Replace periods with underscores in the non-directory part to
# prevent file extension confusion. Also replace # in the non-dir
# part to avoid confusion with URI fragments
head, tail = os.path.split(output)
tail = tail.replace(".", "_")
tail = tail.replace("#", "HASH")
output = os.path.join(head, tail)
# Complain if we were meant to use a component
if component is not None and not usedComponent:
raise KeyError(
"Component '{}' specified but template {} did not use it".
format(component, self.template))
# Complain if there's no run
if not usedRun:
raise KeyError("Template does not include 'run'.")
# Since this is known to be a path, normalize it in case some double
# slashes have crept in
path = os.path.normpath(output)
# It should not be an absolute path (may happen with optionals)
if os.path.isabs(path):
path = os.path.relpath(path, start="/")
return path
import re
from options import Options, OptionsContext, Transient, Prohibited
from options.chainstuf import chainstuf
from options.methset import *
import six
import ansiwrap
from ansiwrap import ansilen
from say.util import *
from say.util import _PY2
from say.styling import *
from say.vertical import Vertical, vertical
from say.version import __version__
### Workhorse functions
sformatter = string.Formatter() # piggyback Python's format() template parser
QUOTES = ("'", '"')
def populate_style(style_name, styles):
if style_name.startswith(QUOTES):
style_name = style_name.strip(style_name[0])
if style_name not in styles:
styles[style_name] = autostyle(style_name)
def _sprintf(arg, caller, styles=None, override=None):
"""
Format the template string (arg) with the values seen in the context of the
caller. If override is defined, it is a Mapping providing additional values
def __init__(self,
url_formatter=None,
quantity=None,
searchterms_str=None,
searchterms_fn=None,
site_name=None,
product_url=None,
products_url=None,
retailer_id=None,
user_agent=None,
scrape_variants_with_extra_requests=True,
summary=None,
*args,
**kwargs):
self.proxy_service = None
self.proxy_config = ""
self.proxy_config_filename = ""
if user_agent is None or user_agent not in self.USER_AGENTS.keys():
self.log(
"Not available user agent type or it wasn't set."
" Default user agent will be used.", INFO)
user_agent = 'default'
if user_agent:
self.user_agent = self.USER_AGENTS[user_agent]
self.user_agent_key = user_agent
if scrape_variants_with_extra_requests in (0, '0', 'false', 'False',
False, None):
self.scrape_variants_with_extra_requests = False
else:
self.scrape_variants_with_extra_requests = True
if product_url is None: # searchterms mode
self.scrape_variants_with_extra_requests = False
try:
self.server_ip = socket.gethostbyname(socket.gethostname())
except Exception as e:
self.log("Failed to get server IP", ERROR)
super(BaseProductsSpider, self).__init__(*args, **kwargs)
if site_name is None:
assert len(self.allowed_domains) == 1, \
"A single allowed domain is required to auto-detect site name."
self.site_name = self.allowed_domains[0]
else:
self.site_name = site_name
if url_formatter is None:
self.url_formatter = string.Formatter()
else:
self.url_formatter = url_formatter
if quantity is None:
self.log("No quantity specified. Will retrieve all products.",
INFO)
import sys
self.quantity = sys.maxint
else:
self.quantity = int(quantity)
self.product_url = product_url
self.products_url = products_url
self.retailer_id = retailer_id
self.searchterms = []
if searchterms_str is not None:
self.searchterms = searchterms_str.decode('utf-8').split(',')
elif searchterms_fn is not None:
with open(searchterms_fn, encoding='utf-8') as f:
self.searchterms = f.readlines()
self.log(
"Created for %s with %d search terms." %
(self.site_name, len(self.searchterms)), INFO)
def parse_format_fields(format_str):
formatter = string.Formatter()
format_fields = [(field, (spec, conv))
for _, field, spec, conv in formatter.parse(format_str)
if field is not None]
return format_fields
def test_formatter(self):
fmt = string.Formatter()
self.assertEqual(fmt.format("foo"), "foo")
self.assertEqual(fmt.format("foo{0}", "bar"), "foobar")
self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6")
self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-")
# override get_value ############################################
class NamespaceFormatter(string.Formatter):
def __init__(self, namespace={}):
string.Formatter.__init__(self)
self.namespace = namespace
def get_value(self, key, args, kwds):
if isinstance(key, str):
try:
# Check explicitly passed arguments first
return kwds[key]
except KeyError:
return self.namespace[key]
else:
string.Formatter.get_value(key, args, kwds)
fmt = NamespaceFormatter({'greeting': 'hello'})
self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!')
# override format_field #########################################
class CallFormatter(string.Formatter):
def format_field(self, value, format_spec):
return format(value(), format_spec)
fmt = CallFormatter()
self.assertEqual(fmt.format('*{0}*', lambda: 'result'), '*result*')
# override convert_field ########################################
class XFormatter(string.Formatter):
def convert_field(self, value, conversion):
if conversion == 'x':
return None
return super(XFormatter, self).convert_field(value, conversion)
fmt = XFormatter()
self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None")
# override parse ################################################
class BarFormatter(string.Formatter):
# returns an iterable that contains tuples of the form:
# (literal_text, field_name, format_spec, conversion)
def parse(self, format_string):
for field in format_string.split('|'):
if field[0] == '+':
# it's markup
field_name, _, format_spec = field[1:].partition(':')
yield '', field_name, format_spec, None
else:
yield field, None, None, None
fmt = BarFormatter()
self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *')
# test all parameters used
class CheckAllUsedFormatter(string.Formatter):
def check_unused_args(self, used_args, args, kwargs):
# Track which arguments actuallly got used
unused_args = set(kwargs.keys())
unused_args.update(range(0, len(args)))
for arg in used_args:
unused_args.remove(arg)
if unused_args:
raise ValueError("unused arguments")
fmt = CheckAllUsedFormatter()
self.assertEqual(fmt.format("{0}", 10), "10")
self.assertEqual(fmt.format("{0}{i}", 10, i=100), "10100")
self.assertEqual(fmt.format("{0}{i}{1}", 10, 20, i=100), "1010020")
self.assertRaises(ValueError,
fmt.format,
"{0}{i}{1}",
10,
20,
i=100,
j=0)
self.assertRaises(ValueError, fmt.format, "{0}", 10, 20)
self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100)
self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100)
def labels_for_format(serialize_format):
return (label
for _, label, _, _ in string.Formatter().parse(serialize_format)
if label)
def ensure_common_structure(
self,
internal_type,
internal_name,
parameter,
remainder,
common_initializers,
meta_initializer,
setter,
):
name = self.common_structure_names.get(parameter.uuid)
if name is None:
if len(self.h_code) > 0:
self.h_code.append("")
if len(self.c_code) > 0:
self.c_code.append("")
formatted_uuid = str(parameter.uuid).replace("-", "_")
name = f"InterfaceItem_table_common_{internal_type}_{formatted_uuid}"
if remainder is None:
sizes = {}
full_base_variable = "NULL"
else:
nested_array = self.array_nests[remainder]
layers = []
for layer in nested_array.array_layers:
(layer_format_name,) = [
list(field)[0][1]
for field in [string.Formatter().parse(layer[1])]
]
layers.append(layer_format_name)
variable_base = nested_array.index(
indexes={layer: 0 for layer in layers},
)
sizes = {
layer: nested_array.sizeof(layers[: i + 1])
for i, layer in enumerate(layers)
}
full_base_variable_name = f"{variable_base}.{remainder}"
full_base_variable = f"&{full_base_variable_name}"
if internal_type == "PackedString":
meta_entry = []
variable_base_length_entry = [
f".variable_base_length = sizeof({full_base_variable_name}),",
]
else:
meta_entry = [
f".meta_values = {{",
meta_initializer,
f"}},",
]
variable_base_length_entry = []
self.common_structure_names[parameter.uuid] = name
self.h_code.append(
f"extern InterfaceItem_table_common_{internal_name} {name};",
)
self.c_code.extend(
[
f'#pragma DATA_SECTION({name}, "Interface")',
f"// {node_path_string(parameter)}",
f"// {parameter.uuid}",
f"InterfaceItem_table_common_{internal_type} const {name} = {{",
[
f".common = {{",
common_initializers,
f"}},",
f".variable_base = {full_base_variable},",
*variable_base_length_entry,
f'.setter = {"NULL" if setter is None else setter},',
f'.zone_size = {sizes.get("curve_type", 0)},',
f'.curve_size = {sizes.get("curve_index", 0)},',
f'.point_size = {sizes.get("point_index", 0)},',
*meta_entry,
],
f"}};",
]
)
return name
def expand(*args, **wildcards):
"""
Expand wildcards in given filepatterns.
Arguments
*args -- first arg: filepatterns as list or one single filepattern,
second arg (optional): a function to combine wildcard values
(itertools.product per default)
**wildcards -- the wildcards as keyword arguments
with their values as lists. If allow_missing=True is included
wildcards in filepattern without values will stay unformatted.
"""
filepatterns = args[0]
if len(args) == 1:
combinator = product
elif len(args) == 2:
combinator = args[1]
if isinstance(filepatterns, str):
filepatterns = [filepatterns]
def path_to_str(f):
if isinstance(f, Path):
return str(f)
return f
filepatterns = list(map(path_to_str, filepatterns))
if any(map(lambda f: getattr(f, "flags", {}), filepatterns)):
raise WorkflowError(
"Flags in file patterns given to expand() are invalid. "
"Flags (e.g. temp(), directory()) have to be applied outside "
"of expand (e.g. 'temp(expand(\"plots/{sample}.pdf\", sample=SAMPLES))')."
)
# check if remove missing is provided
format_dict = dict
if "allow_missing" in wildcards and wildcards["allow_missing"] is True:
class FormatDict(dict):
def __missing__(self, key):
return "{" + key + "}"
format_dict = FormatDict
# check that remove missing is not a wildcard in the filepatterns
for filepattern in filepatterns:
if "allow_missing" in re.findall(r"{([^}\.[!:]+)", filepattern):
format_dict = dict
break
# remove unused wildcards to avoid duplicate filepatterns
wildcards = {
filepattern: {
k: v
for k, v in wildcards.items()
if k in re.findall(r"{([^}\.[!:]+)", filepattern)
}
for filepattern in filepatterns
}
def flatten(wildcards):
for wildcard, values in wildcards.items():
if isinstance(values, str) or not isinstance(
values, collections.abc.Iterable
):
values = [values]
yield [(wildcard, value) for value in values]
formatter = string.Formatter()
try:
return [
formatter.vformat(filepattern, (), comb)
for filepattern in filepatterns
for comb in map(format_dict, combinator(*flatten(wildcards[filepattern])))
]
except KeyError as e:
raise WildcardError("No values given for wildcard {}.".format(e))
def description(self, widget):
return string.Formatter().vformat(self._fmt, (), self._tags)
def __init__(self, pod):
self.pod = pod
self.formatter = string.Formatter()
def analyze(self, event):
for t in self._fileTemplate:
fname = string.Formatter().vformat(t, [], event)
self._canvasMaker.canvas().Print(fname)
print fname
def format_map(format_string, mapping, _format=string.Formatter().vformat):
return _format(format_string, None, mapping)
def format(self, ref):
"""Format a template string into a full path.
Parameters
----------
ref : `DatasetRef`
The dataset to be formatted.
Returns
-------
path : `str`
Expanded path.
Raises
------
KeyError
Raised if the requested field is not defined and the field is
not optional. Or, `component` is specified but "component" was
not part of the template.
"""
# Extract defined non-None units from the dataId
fields = {k: v for k, v in ref.dataId.items() if v is not None}
datasetType = ref.datasetType
fields["datasetType"], component = datasetType.nameAndComponent()
usedComponent = False
if component is not None:
fields["component"] = component
usedRunOrCollection = False
fields["collection"] = ref.run.collection
fields["run"] = ref.run.id
fmt = string.Formatter()
parts = fmt.parse(self.template)
output = ""
for literal, field_name, format_spec, conversion in parts:
if field_name == "component":
usedComponent = True
if format_spec is None:
output = output + literal
continue
if "?" in format_spec:
optional = True
# Remove the non-standard character from the spec
format_spec = format_spec.replace("?", "")
else:
optional = False
if field_name in ("run", "collection"):
usedRunOrCollection = True
if field_name in fields:
value = fields[field_name]
elif optional:
# If this is optional ignore the format spec
# and do not include the literal text prior to the optional
# field unless it contains a "/" path separator
format_spec = ""
value = ""
if "/" not in literal:
literal = ""
else:
raise KeyError(
f"'{field_name}' requested in template via '{self.template}' "
"but not defined and not optional")
# Now use standard formatting
output = output + literal + format(value, format_spec)
# Complain if we were meant to use a component
if component is not None and not usedComponent:
raise KeyError(
"Component '{}' specified but template {} did not use it".
format(component, self.template))
# Complain if there's no run or collection
if not usedRunOrCollection:
raise KeyError("Template does not include 'run' or 'collection'.")
# Since this is known to be a path, normalize it in case some double
# slashes have crept in
path = os.path.normpath(output)
# It should not be an absolute path (may happen with optionals)
if os.path.isabs(path):
path = os.path.relpath(path, start="/")
return path
def construct_searchstring_from_format(format_str, wildcard=False):
"""
Parses format file string and returns string formatted for searching.
Parameters
----------
format_str : string with python format codes
Provides the naming pattern of the instrument files and the
locations of date information so an ordered list may be produced.
Supports 'year', 'month', 'day', 'hour', 'minute', 'second', 'version',
and 'revision'
Ex: 'cnofs_vefi_bfield_1sec_{year:04d}{month:02d}{day:02d}_v05.cdf'
wildcard : bool
if True, replaces the ? sequence with a * . This option may be well
suited when dealing with delimited filenames.
Returns
-------
dict
'search_string' format_str with data to be parsed replaced with ?
'keys' keys for data to be parsed
'lengths' string length for data to be parsed
'string_blocks' the filenames are broken down into fixed width
segments and '' strings are placed in locations where data will
eventually be parsed from a list of filenames. A standards
compliant filename can be constructed by starting with
string_blocks, adding keys in order, and replacing the '' locations
with data of length length.
Note
----
The '?' may be used to indicate a set number of spaces for a variable
part of the name that need not be extracted.
'cnofs_cindi_ivm_500ms_{year:4d}{month:02d}{day:02d}_v??.cdf'
"""
if format_str is None:
raise ValueError("Must supply a filename template (format_str).")
# parse format string to figure out how to construct the search string
# to identify files in the filesystem
search_str = ''
form = string.Formatter()
# stores the keywords extracted from format_string
keys = []
# and length of string
snips = []
lengths = []
for snip in form.parse(format_str):
# collect all of the format keywords
# replace them in the string with the '?' wildcard
# numnber of ?s goes with length of data to be parsed
# length grabbed from format keywords so we know
# later on where to parse information out from
search_str += snip[0]
snips.append(snip[0])
if snip[1] is not None:
keys.append(snip[1])
# try and determine formatting width
temp = re.findall(r'\d+', snip[2])
if temp:
# there are items, try and grab width
for i in temp:
# make sure there is truly something there
if i != 0:
# store length and add to the search string
lengths.append(int(i))
if not wildcard:
search_str += '?' * int(i)
else:
search_str += '*'
break
else:
raise ValueError("Couldn't determine formatting width")
return {
'search_string': search_str,
'keys': keys,
'lengths': lengths,
'string_blocks': snips
}
def get_fmt_names(format_string):
"""Yield field names in `format_string`.
"""
for _, name, _, _ in string.Formatter().parse(format_string):
if name:
yield name
def report(manager, fileobj, sev_level, conf_level, lines=-1, template=None):
"""Prints issues in custom format
:param manager: the bandit manager object
:param fileobj: The output file object, which may be sys.stdout
:param sev_level: Filtering severity level
:param conf_level: Filtering confidence level
:param lines: Number of lines to report, -1 for all
:param template: Output template with non-terminal tags <N>
(default: '{abspath}:{line}:
{test_id}[bandit]: {severity}: {msg}')
"""
machine_output = {'results': [], 'errors': []}
for (fname, reason) in manager.get_skipped():
machine_output['errors'].append({'filename': fname,
'reason': reason})
results = manager.get_issue_list(sev_level=sev_level,
conf_level=conf_level)
msg_template = template
if template is None:
msg_template = "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
# Dictionary of non-terminal tags that will be expanded
tag_mapper = {
'abspath': lambda issue: os.path.abspath(issue.fname),
'relpath': lambda issue: os.path.relpath(issue.fname),
'line': lambda issue: issue.lineno,
'test_id': lambda issue: issue.test_id,
'severity': lambda issue: issue.severity,
'msg': lambda issue: issue.text,
'confidence': lambda issue: issue.confidence,
'range': lambda issue: issue.linerange
}
# Create dictionary with tag sets to speed up search for similar tags
tag_sim_dict = dict(
[(tag, set(tag)) for tag, _ in tag_mapper.items()]
)
# Parse the format_string template and check the validity of tags
try:
parsed_template_orig = list(string.Formatter().parse(msg_template))
# of type (literal_text, field_name, fmt_spec, conversion)
# Check the format validity only, ignore keys
string.Formatter().vformat(msg_template, (), SafeMapper(line=0))
except ValueError as e:
LOG.error("Template is not in valid format: %s", e.args[0])
sys.exit(2)
tag_set = {t[1] for t in parsed_template_orig if t[1] is not None}
if not tag_set:
LOG.error("No tags were found in the template. Are you missing '{}'?")
sys.exit(2)
def get_similar_tag(tag):
similarity_list = [(len(set(tag) & t_set), t)
for t, t_set in tag_sim_dict.items()]
return sorted(similarity_list)[-1][1]
tag_blacklist = []
for tag in tag_set:
# check if the tag is in dictionary
if tag not in tag_mapper:
similar_tag = get_similar_tag(tag)
LOG.warning(
"Tag '%s' was not recognized and will be skipped, "
"did you mean to use '%s'?", tag, similar_tag
)
tag_blacklist += [tag]
# Compose the message template back with the valid values only
msg_parsed_template_list = []
for literal_text, field_name, fmt_spec, conversion in parsed_template_orig:
if literal_text:
# if there is '{' or '}', double it to prevent expansion
literal_text = re.sub('{', '{{', literal_text)
literal_text = re.sub('}', '}}', literal_text)
msg_parsed_template_list.append(literal_text)
if field_name is not None:
if field_name in tag_blacklist:
msg_parsed_template_list.append(field_name)
continue
# Append the fmt_spec part
params = [field_name, fmt_spec, conversion]
markers = ['', ':', '!']
msg_parsed_template_list.append(
['{'] +
["%s" % (m + p) if p else ''
for m, p in zip(markers, params)] +
['}']
)
msg_parsed_template = "".join([item for lst in msg_parsed_template_list
for item in lst]) + "\n"
limit = lines if lines > 0 else None
with fileobj:
for defect in results[:limit]:
evaluated_tags = SafeMapper(
(k, v(defect)) for k, v in tag_mapper.items()
)
output = msg_parsed_template.format(**evaluated_tags)
fileobj.write(output)
if fileobj.name != sys.stdout.name:
LOG.info("Result written to file: %s", fileobj.name)
def get_format_string_arguments(format_string: str) -> typing.List[str]:
return [
t[1] for t in string.Formatter().parse(format_string)
if t[1] is not None
]
