def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def g(a=1, b=10, c=0):
return a + b + c
cg = curry(g, b=2)
assert cg() == 3
assert cg(b=3) == 4
assert cg(a=0) == 2
assert cg(a=0, b=1) == 1
assert cg(0) == 2 # pass "a" as arg, not kwarg
assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg
def h(x, func=int):
return func(x)
# __init__ must not pick func as positional arg
assert curry(h)(0.0) == 0
assert curry(h)(func=str)(0.0) == '0.0'
assert curry(h, func=str)(0.0) == '0.0'
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def g(a=1, b=10, c=0):
return a + b + c
cg = curry(g, b=2)
assert cg() == 3
assert cg(b=3) == 4
assert cg(a=0) == 2
assert cg(a=0, b=1) == 1
assert cg(0) == 2 # pass "a" as arg, not kwarg
assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg
def h(x, func=int):
return func(x)
if platform.python_implementation() != 'PyPy'\
or platform.python_version_tuple()[0] != '3': # Bug on PyPy3<2.5
# __init__ must not pick func as positional arg
assert curry(h)(0.0) == 0
assert curry(h)(func=str)(0.0) == '0.0'
assert curry(h, func=str)(0.0) == '0.0'
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def g(a=1, b=10, c=0):
return a + b + c
cg = curry(g, b=2)
assert cg() == 3
assert cg(b=3) == 4
assert cg(a=0) == 2
assert cg(a=0, b=1) == 1
assert cg(0) == 2 # pass "a" as arg, not kwarg
assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg
def h(x, func=int):
return func(x)
if platform.python_implementation() != 'PyPy'\
or platform.python_version_tuple()[0] != '3': # Bug on PyPy3<2.5
# __init__ must not pick func as positional arg
assert curry(h)(0.0) == 0
assert curry(h)(func=str)(0.0) == '0.0'
assert curry(h, func=str)(0.0) == '0.0'
def __init__(self, inputs, calls, outputs, origin=None):
'''
A Function represents a function in the computational sense. Function objects are the
intermediary between fitted estimators and generated code. Adapters return Function
objects, and sklearn2code converts Function objects into working code. A Function object
is composed of Expression objects (including Variable objects) and other Function objects.
It knows its inputs (Variable objects), its internal calls (made up of Variable objects
and other Function objects), and its outputs (general Expression objects).
Parameters
----------
inputs : tuple of Variables
The input variables for this function.
calls : tuple of pairs with (tuples of Variables, pairs of Function objects and tuples of their inputs)
The values are other function calls made by this function. The keys are
variables to which the outputs are assigned. The number of output variables in the
key must match the number of outputs in the Function. The length of the tuple of inputs must match the
number of inputs for the function. Also, no two keys may contain
the same variable. These constraints are checked.
outputs : tuple of expressions
The actual calculations made by this Function. The return values of the function
are the results of the computations expressed by the expressions.
'''
self.inputs = tuple(map(safe_symbol, tupify(inputs)))
self.calls = tupsmap(
1, tupfun(identity, compose(tuple,
curry(map)(safe_symbol))),
tupsmap(0, compose(tuple,
curry(map)(safe_symbol)), calls))
self.outputs = tupify(outputs)
self._validate()
def summary(session: Session,
user: int,
start_date: Optional[date] = None,
end_date: Optional[date] = None,
country: Optional[int] = None):
results = {}
def metric_query(session, metric_name):
op = compose(
curry(country_filtered)(country),
curry(date_filtered)(start_date, end_date),
curry(user_only)(user),
lambda q: q.join(models.Shape),
lambda q: q.filter(models.Metric.name == metric_name),
lambda q: q.query(models.Metric.value),
)
return np.array([*op(session).all()])
metric_query = curry(metric_query, session)
def maybe_metric_query(metric, operation):
metrics = metric_query(metric)
if len(metrics) > 0:
return getattr(metrics, operation)()
else:
return None
operations = {
"predicted_area":
lambda: maybe_metric_query("square_km_area", "sum"),
"conflict_coverage":
lambda: maybe_metric_query("conflict_coverage", "mean"),
"accuracy":
lambda: maybe_metric_query("correct", "mean"),
}
shapes = compose(
curry(country_filtered)(country),
curry(date_filtered)(start_date, end_date),
curry(user_only)(user),
lambda q: q.query(models.Shape.shape))(session).all()
shapes = [row["shape"]["geometry"] for row in shapes]
shapes = [*shapes]
results["shapes"] = len(shapes)
ops_if_shapes = ("conflict_coverage", "predicted_area", "accuracy")
if results["shapes"] > 0:
for k in ops_if_shapes:
results[k] = operations[k]()
else:
for k in ops_if_shapes:
results[k] = None
results = {k: v for k, v in results.items() if not v is None}
return results
def test_curry_simple():
cmul = curry(mul)
double = cmul(2)
assert callable(double)
assert double(10) == 20
cmap = curry(map)
assert list(cmap(inc)([1, 2, 3])) == [2, 3, 4]
def test_curry_simple():
cmul = curry(mul)
double = cmul(2)
assert callable(double)
assert double(10) == 20
assert repr(cmul) == repr(mul)
cmap = curry(map)
assert list(cmap(inc)([1, 2, 3])) == [2, 3, 4]
def metric_query(session, metric_name):
op = compose(
curry(country_filtered)(country),
curry(date_filtered)(start_date, end_date),
curry(user_only)(user),
lambda q: q.join(models.Shape),
lambda q: q.filter(models.Metric.name == metric_name),
lambda q: q.query(models.Metric.value),
)
return np.array([*op(session).all()])
def test_curry_simple():
cmul = curry(mul)
double = cmul(2)
assert callable(double)
assert double(10) == 20
assert repr(cmul) == repr(mul)
cmap = curry(map)
assert list(cmap(inc)([1, 2, 3])) == [2, 3, 4]
assert raises(TypeError, lambda: curry({1: 2}))
def metric_countries_in_span(session: Session,
user: int,
start_date: date = None,
end_date: date = None):
query = compose(
curry(date_filtered)(start_date, end_date),
curry(user_only)(user),
lambda q: q.distinct(),
lambda q: q.join(models.Metric),
lambda q: q.query(models.Shape.country_id),
)(session)
return [dict(r) for r in query.all()]
def test_curry_is_idempotent():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
g = curry(f)
assert isinstance(f, curry)
assert isinstance(g, curry)
assert not isinstance(g.func, curry)
assert not hasattr(g.func, 'func')
assert f.func == g.func
assert f.args == g.args
assert f.keywords == g.keywords
def sym_predict_random_forest_regressor(estimator):
inputs = syms(estimator)
Var = VariableFactory(existing=inputs)
subs = tuple(map(sym_predict, estimator.estimators_))
calls = tuple((tuple(Var() for _ in range(len(sub.outputs))), (sub, inputs)) for sub in subs)
outputs = tuple(map(flip(__truediv__)(RealNumber(len(subs))), map(curry(reduce)(__add__), zip(*map(flip(getitem)(0), calls)))))
return Function(inputs, calls, outputs)
def split_cf_messages(format_message,
var_length_key,
event,
separator=', ',
max_length=255):
"""
Try to split cloud feed log events out into multiple events if the message
is too long (the variable-length variable would cause the message to be
too long.)
:param str format_message: The format string to use to format the event
:param str var_length_key: The key in the event dictionary that contains
the variable-length part of the formatted message.
:param dict event: The event dictionary
:param str separator: The separator to use to join the various elements
that should be varied. (e.g. if the elements in "var_length_key" are
["1", "2", "3"] and the separator is "; ", "var_length_key" will be
represented as "1; 2; 3")
:param int max_length: The maximum length of the formatted message.
:return: `list` of event dictionaries with the formatted message and
the split event field.
"""
def length_calc(e):
return len(format_message.format(**e))
render = compose(assoc(event, var_length_key), separator.join,
curry(map, str))
if length_calc(event) <= max_length:
return [(render(event[var_length_key]), format_message)]
events = split(render, event[var_length_key], max_length, length_calc)
return [(e, format_message) for e in events]
def split_cf_messages(format_message, var_length_key, event, separator=', ',
max_length=255):
"""
Try to split cloud feed log events out into multiple events if the message
is too long (the variable-length variable would cause the message to be
too long.)
:param str format_message: The format string to use to format the event
:param str var_length_key: The key in the event dictionary that contains
the variable-length part of the formatted message.
:param dict event: The event dictionary
:param str separator: The separator to use to join the various elements
that should be varied. (e.g. if the elements in "var_length_key" are
["1", "2", "3"] and the separator is "; ", "var_length_key" will be
represented as "1; 2; 3")
:param int max_length: The maximum length of the formatted message.
:return: `list` of event dictionaries with the formatted message and
the split event field.
"""
def length_calc(e):
return len(format_message.format(**e))
render = compose(assoc(event, var_length_key), separator.join,
curry(map, str))
if length_calc(event) <= max_length:
return [(render(event[var_length_key]), format_message)]
events = split(render, event[var_length_key], max_length, length_calc)
return [(e, format_message) for e in events]
def __call__(self, losses, **kwargs):
if len(losses) <= self.n:
return False
return all(
map(
curry(__lt__)(-self.threshold),
starmap(self.stat, sliding_window(2, losses[-(self.n + 1):]))))
def reset_defaults(load,save):
with_defaults = [curry(with_key_value,k,v) for k,v in DEFAULT_SETTINGS.items()]
compose(
save,
compose(*with_defaults),
load,
)()
def test_curry_attributes_readonly():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
assert raises(AttributeError, lambda: setattr(f, 'args', (2,)))
assert raises(AttributeError, lambda: setattr(f, 'keywords', {'c': 3}))
assert raises(AttributeError, lambda: setattr(f, 'func', f))
def test_curry_attributes_readonly():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
assert raises(AttributeError, lambda: setattr(f, "args", (2,)))
assert raises(AttributeError, lambda: setattr(f, "keywords", {"c": 3}))
assert raises(AttributeError, lambda: setattr(f, "func", f))
def test_curry_docstring():
def f(x, y):
""" A docstring """
return x
g = curry(f)
assert g.__doc__ == f.__doc__
assert str(g) == str(f)
def test_curry_docstring():
def f(x, y):
""" A docstring """
return x
g = curry(f)
assert g.__doc__ == f.__doc__
assert str(g) == str(f)
def test_curry_comparable():
def foo(a, b, c=1):
return a + b + c
f1 = curry(foo, 1, c=2)
f2 = curry(foo, 1, c=2)
g1 = curry(foo, 1, c=3)
h1 = curry(foo, c=2)
h2 = h1(c=2)
h3 = h1()
assert f1 == f2
assert not (f1 != f2)
assert f1 != g1
assert not (f1 == g1)
assert f1 != h1
assert h1 == h2
assert h1 == h3
# test function comparison works
def bar(a, b, c=1):
return a + b + c
b1 = curry(bar, 1, c=2)
assert b1 != f1
assert set([f1, f2, g1, h1, h2, h3, b1, b1()]) == set([f1, g1, h1, b1])
# test unhashable input
unhash1 = curry(foo, [])
assert raises(TypeError, lambda: hash(unhash1))
unhash2 = curry(foo, c=[])
assert raises(TypeError, lambda: hash(unhash2))
def test_curry_doesnot_transmogrify():
# Early versions of `curry` transmogrified to `partial` objects if
# only one positional argument remained even if keyword arguments
# were present. Now, `curry` should always remain `curry`.
def f(x, y=0):
return x + y
cf = curry(f)
assert cf(y=1)(y=2)(y=3)(1) == f(1, 3)
def map_call_symbols(self, symbol_map):
safe_sub_map = itemmap(tupfun(safe_symbol, safe_symbol), symbol_map)
symbol_map_ = fallback(safe_sub_map.__getitem__,
identity,
exception_type=KeyError)
symbol_tup_map = compose(tuple, curry(map)(symbol_map_))
return tuple(
map(tupfun(symbol_tup_map, tupfun(identity, symbol_tup_map)),
self.calls))
def test_curry_doesnot_transmogrify():
# Early versions of `curry` transmogrified to `partial` objects if
# only one positional argument remained even if keyword arguments
# were present. Now, `curry` should always remain `curry`.
def f(x, y=0):
return x + y
cf = curry(f)
assert cf(y=1)(y=2)(y=3)(1) == f(1, 3)
def test_curry_wrapped():
def foo(a):
"""
Docstring
"""
pass
curried_foo = curry(foo)
assert curried_foo.__wrapped__ is foo
def test_curry_wrapped():
def foo(a):
"""
Docstring
"""
pass
curried_foo = curry(foo)
assert curried_foo.__wrapped__ is foo
def free_symbols(self):
return reduce(
__or__,
map(
compose(
curry(reduce)(__or__),
tupfun(
flip(getattr)('free_symbols'),
flip(getattr)('free_symbols'))),
self.mapping.items())) | self.arg.free_symbols
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def test_curry_is_like_partial():
def foo(a, b, c=1):
return a + b + c
p, c = partial(foo, 1, c=2), curry(foo)(1, c=2)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(3) == c(3)
p, c = partial(foo, 1), curry(foo)(1)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(3) == c(3)
assert p(3, c=2) == c(3, c=2)
p, c = partial(foo, c=1), curry(foo)(c=1)
assert p.keywords == c.keywords
assert p.args == c.args
assert p(1, 2) == c(1, 2)
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def __op__(self, other):
if isinstance(other, Function):
self.ensure_same_inputs(other)
local_vars = self.local_vars()
Var = VariableFactory(existing=(other.vars() | local_vars))
other = other.map_symbols({var: Var() for var in local_vars})
self.ensure_same_output_length(other)
calls, symbol_map = self._merge_calls(other)
outputs = tuple(
starmap(
op, zip(self.outputs,
other.map_output_symbols(symbol_map))))
else:
calls = self.calls
outputs = tuple(
map((curry(tzflip(op)) if not flip else curry(op))(other),
self.outputs))
return Function(inputs=self.inputs, calls=calls, outputs=outputs)
def check_curry(func, args, kwargs, incomplete=True):
try:
curry(func)(*args, **kwargs)
curry(func, *args)(**kwargs)
curry(func, **kwargs)(*args)
curry(func, *args, **kwargs)()
if not isinstance(func, type(lambda: None)):
return None
return incomplete
except ValueError:
return True
except TypeError:
return False
def test_curry_attributes_writable():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
f.__name__ = "newname"
f.__doc__ = "newdoc"
assert f.__name__ == "newname"
assert f.__doc__ == "newdoc"
if hasattr(f, "func_name"):
assert f.__name__ == f.func_name
def test_curry_attributes_writable():
def foo(a, b, c=1):
return a + b + c
f = curry(foo, 1, c=2)
f.__name__ = 'newname'
f.__doc__ = 'newdoc'
assert f.__name__ == 'newname'
assert f.__doc__ == 'newdoc'
if hasattr(f, 'func_name'):
assert f.__name__ == f.func_name
def check_curry(func, args, kwargs, incomplete=True):
try:
curry(func)(*args, **kwargs)
curry(func, *args)(**kwargs)
curry(func, **kwargs)(*args)
curry(func, *args, **kwargs)()
if not isinstance(func, type(lambda: None)):
return None
return incomplete
except ValueError:
return True
except TypeError:
return False
def combine_argument_formatters(*formatters):
warnings.warn(
DeprecationWarning(
"combine_argument_formatters(formatter1, formatter2)([item1, item2])"
"has been deprecated and will be removed in a subsequent major version "
"release of the eth-utils library. Update your calls to use "
"apply_formatters_to_sequence([formatter1, formatter2], [item1, item2]) "
"instead."))
_formatter_at_index = curry(apply_formatter_at_index)
return compose(*(_formatter_at_index(formatter, index)
for index, formatter in enumerate(formatters)))
def test_curry_kwargs():
def f(a, b, c=10):
return (a + b) * c
f = curry(f)
assert f(1, 2, 3) == 9
assert f(1)(2, 3) == 9
assert f(1, 2) == 30
assert f(1, c=3)(2) == 9
assert f(c=3)(1, 2) == 9
def g(a=1, b=10, c=0):
return a + b + c
cg = curry(g, b=2)
assert cg() == 3
assert cg(b=3) == 4
assert cg(a=0) == 2
assert cg(a=0, b=1) == 1
assert cg(0) == 2 # pass "a" as arg, not kwarg
assert raises(TypeError, lambda: cg(1, 2)) # pass "b" as arg AND kwarg
async def fetch_set(base_url: str, queryset: models.Queryset)-> Either:
async with aiohttp.ClientSession() as session:
get_data = compose(
deserialize,
curry(get,session),
)
results = await asyncio.gather(
*map(get_data, make_urls(base_url, queryset))
)
unpack_errors = compose(
curry(filter,lambda x: x is not None),
curry(map,lambda e: e.either(identity, lambda _: None))
)
errors = [*unpack_errors(results)]
if len(errors)>0:
return Left(errors)
else:
return Right(fast_views.inner_join([res.value for res in results]))
def check_geojson_ownership(geojson, user):
check = curry(_check_feature_ownership, user)
if geojson["type"] == "FeatureCollection":
if len(geojson["features"]) == 0:
return True
else:
return reduce(lambda a, b: a and b,
[check(f) for f in geojson["features"]])
elif geojson["type"] == "Feature":
return check(geojson)
else:
raise ValueError("Not feature or featurecollection")
def fit(self, X, y=None, exposure=None, xlabels=None):
if xlabels is not None:
self.xlabels_ = xlabels
else:
self.xlabels_ = safe_column_names(X)
self.clean_transformations_ = keymap(clean_column_name(self.xlabels_),
self.transformations)
if not self.strict:
input_variables = set(self.xlabels_)
self.clean_transformations_ = valfilter(
compose(
curry(__ge__)(input_variables), methodcaller('inputs')),
self.clean_transformations_)
return self
def test_curry_attributes_writable():
def foo(a, b, c=1):
return a + b + c
foo.__qualname__ = 'this.is.foo'
f = curry(foo, 1, c=2)
assert f.__qualname__ == 'this.is.foo'
f.__name__ = 'newname'
f.__doc__ = 'newdoc'
f.__module__ = 'newmodule'
f.__qualname__ = 'newqualname'
assert f.__name__ == 'newname'
assert f.__doc__ == 'newdoc'
assert f.__module__ == 'newmodule'
assert f.__qualname__ == 'newqualname'
if hasattr(f, 'func_name'):
assert f.__name__ == f.func_name
from toolz.functoolz import curry, known_numargs
from .include import get_include
from ._coiter import coiter
from ._comap import comap
from ._cozip import cozip
from ._emptycoroutine import emptycoroutine
__all__ = [
'coiter',
'comap',
'cozip',
'emptycoroutine',
'get_include',
]
known_numargs[comap] = 2
comap = curry(comap)
del curry, known_numargs
from __future__ import absolute_import
import operator
from toolz.functoolz import curry, num_required_args, has_keywords
def should_curry(f):
num = num_required_args(f)
return num is None or num > 1 or num == 1 and has_keywords(f) is not False
locals().update(
dict((name, curry(f) if should_curry(f) else f)
for name, f in vars(operator).items() if callable(f)),
)
# Clean up the namespace.
del curry
del num_required_args
del has_keywords
del operator
del should_curry
def test_curry_bad_types():
assert raises(TypeError, lambda: curry(1))
def test_curry_simple():
cmul = curry(mul)
double = cmul(2)
assert callable(double)
assert double(10) == 20
"""
Format logs based on specification
"""
import json
import math
from toolz.curried import assoc
from toolz.dicttoolz import keyfilter
from toolz.functoolz import compose, curry
from twisted.python.failure import Failure
from otter.log.formatters import LoggingEncoder
_json_len = compose(len, curry(json.dumps, cls=LoggingEncoder))
# Maximum length of entire JSON-formatted event dictionary
event_max_length = 50000
def split_execute_convergence(event, max_length=event_max_length):
"""
Try to split execute-convergence event out into multiple events if there
are too many CLB nodes, too many servers, or too many steps.
The problem is mainly the servers, since they take up the most space.
Experimentally determined that probably logs cut off at around 75k,
characters - we're going to limit it to 50k.
