def call_func_ignoring_excess(func, **kwargs):
"""Call func, sourcing the arguments from kwargs and ignoring the excess arguments.
Also works if func has some position only arguments.
"""
s = Sig(func)
args, kwargs = s.args_and_kwargs_from_kwargs(s.source_kwargs(**kwargs))
return func(*args, **kwargs)
def compose(func1, func2):
sig1 = Sig(func1)
sig2 = Sig(func2)
@sig1
def composed_funcs(*args, **kwargs):
return func2(func1(*args, **kwargs))
composed_funcs.__return_annotation__ = sig2.return_annotation
return composed_funcs
def __post_init__(self):
self.func_nodes = tuple(_mk_func_nodes(self.func_nodes))
self.graph = _func_nodes_to_graph_dict(self.func_nodes)
self.nodes = topological_sort(self.graph)
# reorder the nodes to fit topological order
self.func_nodes, self.var_nodes = _separate_func_nodes_and_var_nodes(
self.nodes)
# figure out the roots and leaves
self.roots = set(root_nodes(self.graph))
self.leafs = set(leaf_nodes(self.graph))
# self.sig = Sig(dict(extract_items(sig.parameters, 'xz')))
self.sig = Sig(sort_params(self.src_name_params(self.roots)))
self.sig(self) # to put the signature on the callable DAG
self.last_scope = None
def register_cli_method(
openapi_spec: dict,
client_method: Callable,
expected_auth_kwargs: Iterable[str] = None,
config_filename: str = DFLT_CONFIG_FILENAME,
profile: str = '',
):
"""Creates a CLI-friendly function to instantiate an HttpClient with appropriate authentication
arguments and call a particular method of the client instance
:param openapi_spec: The OpenAPI spec used to make the client
:param client_method: The instance method to wrap
:param expected_auth_kwargs: A list of authentication kwargs that the CLI should ask for
(defaults to empty list)
:param config_filename: The path to a JSON file to read for authentication kwargs
(defaults to ~/.http2py/credentials.json)
:param profile: The named profile to find in the config file
"""
methodname = client_method.__name__
method_sig = Sig(client_method)
if not expected_auth_kwargs:
expected_auth_kwargs = []
method_sig = method_sig.merge_with_sig(
[
*[
{'name': kwarg, 'kind': KO, 'default': ''}
for kwarg in expected_auth_kwargs
],
{'name': 'config', 'kind': KO, 'default': config_filename},
]
)
def cli_method(*args, **kwargs):
config_filename = kwargs.pop('config')
auth_kwargs = {key: kwargs.pop(key) for key in expected_auth_kwargs}
auth_kwargs = mk_auth(
auth_kwargs, expected_auth_kwargs, config_filename, profile
)
http_client = HttpClient(openapi_spec, **auth_kwargs)
return getattr(http_client, methodname)(**kwargs)
method_sig.wrap(cli_method)
cli_method.__name__ = methodname
return mk_argparse_friendly(cli_method)
def main(*args, **kwargs):
max_samples = kwargs.pop(
'max_samples',
None) # this argument will be hidden (use Sig to add it to signature?)
kwargs = Sig(mk_pipeline).extract_kwargs(*args, **kwargs)
if 'chunker' in kwargs and str.isnumeric(kwargs['chunker']):
kwargs['chunker'] = int(kwargs['chunker'])
pipeline = mk_pipeline(**kwargs)
return launch(pipeline, max_samples=max_samples)
def __post_init__(self):
self.name, self.output_name = self.names_maker(self.func, self.name,
self.output_name)
self.sig = Sig(self.func)
# complete src_names with the argnames of the signature
_complete_dict_with_iterable_of_required_keys(self.src_names,
self.sig.names)
self.extractor = partial(_mapped_extraction, key_map=self.src_names)
def mk_argparse_friendly(func):
"""Wraps a function to expose a signature that is compatible with argparse by stripping leading
underscores from all keyword arguments, but does not mutate the signature of the original function"""
orig_sig = Sig(func)
new_params = dict(mk_sig_argparse_friendly(orig_sig))
new_sig = orig_sig.replace(
parameters=new_params, return_annotation=orig_sig.return_annotation
)
@wraps(func)
def _func(*args, **kwargs):
mapped_kwargs = {}
for argname, argvalue in kwargs.items():
if argname not in [param.name for param in orig_sig.params]:
mapped_argname = '_' + argname[:-1]
else:
mapped_argname = argname
mapped_kwargs[mapped_argname] = argvalue
# convert strings into correct JSON types
io_trans = JSONAnnotAndDfltIoTrans()
return io_trans(func)(*args, **mapped_kwargs)
return new_sig(_func)
def arg_names(func, func_name, exclude_names=()):
names = Sig(func).names
def gen():
_exclude_names = exclude_names
for name in names:
if name not in _exclude_names:
yield name
else:
found_name = find_first_free_name(f'{func_name}__{name}',
_exclude_names)
yield found_name
_exclude_names = _exclude_names + (found_name, )
return list(gen())
def funcs_that_need_args(funcs, func_to_kwargs=None, self_name=None):
"""
>>> from sklearn.utils import all_estimators
>>> estimator_classes = [obj for name, obj in all_estimators()]
>>> estimator_names = list(map(lambda x: x.__name__, funcs_that_need_args(all_estimator_classes)))
>>> expected = [
... 'ClassifierChain', 'ColumnTransformer', 'FeatureUnion', 'GridSearchCV', 'MultiOutputClassifier',
... 'MultiOutputRegressor', 'OneVsOneClassifier', 'OneVsRestClassifier', 'OutputCodeClassifier',
... 'Pipeline', 'RFE', 'RFECV', 'RandomizedSearchCV', 'RegressorChain', 'SelectFromModel',
... 'SparseCoder', 'StackingClassifier', 'StackingRegressor', 'VotingClassifier', 'VotingRegressor']
>>>
>>> import sklearn
>>> if sklearn.__version__ == '0.23.1':
... assert estimator_names == expected
"""
for func in funcs:
required_args = set(
Sig(func).without_defaults.parameters) - {self_name}
if func_to_kwargs is not None:
required_args -= func_to_kwargs(func).keys()
if required_args:
yield func
def _handle_exclude_nodes(func):
sig = Sig(func)
@wraps(func)
def _func(*args, **kwargs):
kwargs = sig.kwargs_from_args_and_kwargs(args,
kwargs,
apply_defaults=True)
try:
_exclude_nodes = kwargs["_exclude_nodes"]
except KeyError:
raise RuntimeError(
f"{func} doesn't have a _exclude_nodes argument")
if _exclude_nodes is None:
_exclude_nodes = set()
elif not isinstance(_exclude_nodes, set):
_exclude_nodes = set(_exclude_nodes)
kwargs["_exclude_nodes"] = _exclude_nodes
args, kwargs = sig.args_and_kwargs_from_kwargs(kwargs)
return func(*args, **kwargs)
return _func
def mk_flat(cls,
method,
*,
func_name: str = 'flat_func',
cls_cache_key: str = None):
"""
Flatten a simple cls->instance->method call pipeline into one function.
That is, a function mk_flat(cls, method) that returns a "flat function" such that
```
cls(**init_kwargs).method(**method_kwargs) == flat_func(**init_kwargs, **method_kwargs)
```
So, instead of this:
```graphviz
label="NESTED: result = cls(**init_kwargs).method(**method_kwargs)"
cls, init_kwargs -> instance
instance, method, method_kwargs -> result
```
you get a function `flat_func` that you can use like this:
```graphviz
label="FLAT: result = flat_func(**init_kwargs, **method_kwargs)"
flat_func, init_kwargs, method_kwargs -> result
```
:param cls: A class
:param method: A method of this class
:param func_name: The name of the function (will be "flat_func" by default)
:param cls_cache_key: The name of the kwarg used to manage cache. If not None, the
same instance of ``cls`` will be used for all the flattened method called with the same
value for this kwarg.
:return:
>>> class MultiplierClass:
... def __init__(self, x):
... self.x = x
... def multiply(self, y: float = 1) -> float:
... return self.x * y
... def subtract(self, z):
... return self.x - z
...
>>> MultiplierClass(6).multiply(7)
42
>>> MultiplierClass(3.14).multiply()
3.14
>>> MultiplierClass(3).subtract(1)
2
>>> f = mk_flat(MultiplierClass, 'multiply', func_name='my_special_func')
>>> help(f) # doctest: +SKIP
Help on function my_special_func in module ...
<BLANKLINE>
my_special_func(x, y: float = 1) -> float
<BLANKLINE>
>>> f = mk_flat(MultiplierClass, MultiplierClass.subtract)
>>> help(f) # doctest: +SKIP
Help on function flat_func in in module ...
<BLANKLINE>
flat_func(x, z)
<BLANKLINE>
"""
# sig1 = signature(cls)
# if isinstance(method, str):
# method = getattr(cls, method)
# sig2 = signature(method)
# parameters = list(sig1.parameters.values()) + list(sig2.parameters.values())[1:]
# parameters.sort(key=lambda x: x.kind) # sort by kind
# duplicates = [x for x, count in collections.Counter(parameters).items() if count > 1]
# for d in duplicates:
# if d.kind != Parameter.VAR_POSITIONAL and d.kind != Parameter.VAR_KEYWORD:
# raise TypeError(
# f"Cannot flatten {method.__name__}! Duplicate argument found: {d.name} is in both {cls.__name__} class' and {method.__name__} method's signatures.")
# parameters = list(dict.fromkeys(parameters)) # remove args and kwargs duplicates
if isinstance(method, str):
method = getattr(cls, method)
sig_cls = Sig(cls)
sig_method = Sig(method)
sig_flat = sig_cls + sig_method
if cls_cache_key:
param = dict(name=cls_cache_key, kind=KO, default=None)
sig_flat = sig_flat.add_params([param])
sig_flat = sig_flat.remove_names(['self'])
sig_flat = sig_flat.replace(return_annotation=sig_method.return_annotation)
def flat_func(**kwargs):
if (len([
p for p in sig_cls.parameters.values()
if p.kind == Parameter.VAR_KEYWORD
]) == 1):
cls_params = kwargs
else:
cls_params = {
k: kwargs[k]
for k in kwargs if k in sig_cls.parameters
}
if (len([
p for p in sig_method.parameters.values()
if p.kind == Parameter.VAR_KEYWORD
]) == 1):
method_params = kwargs
else:
method_params = {
k: kwargs[k]
for k in kwargs if k in sig_method.parameters
}
cls_cache_id = next(
iter(v for k, v in kwargs.items() if k == cls_cache_key), None)
if cls_cache_id is not None:
instance = _instantiate(cls, cache_id=cls_cache_id, **cls_params)
else:
instance = cls(**cls_params)
return getattr(instance, method.__name__)(**method_params)
flat_func.__dict__ = method.__dict__.copy() # to copy attributes of method
flat_func.__signature__ = sig_flat
flat_func.__name__ = func_name
flat_func.__doc__ = method.__doc__
return flat_func
from pathlib import Path
from i2.signatures import Sig
from oui.color_util import color_hex_from, add_alpha, dec_to_hex
HTML('<script>var exports = {"__esModule": true};</script>')
pkg_dir = os.path.dirname(__file__)
dflt_filename = 'splatter_defaults.json'
pjoin = lambda *p: os.path.join(pkg_dir, *p)
dflts_filepath = pjoin(dflt_filename)
dflts = json.load(open(dflts_filepath, 'r'))
splatter_dflts = dict(dflts['options'], **dflts['tsneOptions'])
_splatter_raw_sig = Sig.from_objs('pts', splatter_dflts.items())
# splatter_kwargs_dflts = {k: dflts[k] for k in dflts if k not in }
# _splatter_sig = Sig.from_objs('pts', splatter_dflts.items())
def assert_jsonizable(d):
_ = json.dumps(d)
return True
# @Sig.from_objs('pts', dflts.items(), assert_same_sized_fvs=True)
# def mysplatter():
# pass
class DAG:
"""
>>> from meshed.dag import DAG, Sig
>>>
>>> def this(a, b=1):
... return a + b
>>> def that(x, b=1):
... return x * b
>>> def combine(this, that):
... return (this, that)
>>>
>>> dag = DAG((this, that, combine))
>>> print(dag.synopsis_string())
x,b -> that_ -> that
a,b -> this_ -> this
this,that -> combine_ -> combine
But what does it do?
It's a callable, with a signature:
>>> Sig(dag)
<Sig (x, a, b=1)>
And when you call it, it executes the dag from the root values you give it and
returns the leaf output values.
>>> dag(1, 2, 3) # (a+b,x*b) == (2+3,1*3) == (5, 3)
(5, 3)
>>> dag(1, 2) # (a+b,x*b) == (2+1,1*1) == (3, 1)
(3, 1)
The above DAG was created straight from the functions, using only the names of the
functions and their arguments to define how to hook the network up.
But if you didn't write those functions specifically for that purpose, or you want
to use someone else's functions, we got you covered.
You can define the name of the node (the `name` argument), the name of the output
(the `output_name` argument) and a mapping from the function's arguments names to
"network names" (through the `src_names` argument).
The edges of the DAG are defined by matching `output_name` TO `src_names`.
"""
func_nodes: Iterable[FuncNode]
cache_last_scope: bool = True
parameter_merge: ParameterMerger = conservative_parameter_merge
def __post_init__(self):
self.func_nodes = tuple(_mk_func_nodes(self.func_nodes))
self.graph = _func_nodes_to_graph_dict(self.func_nodes)
self.nodes = topological_sort(self.graph)
# reorder the nodes to fit topological order
self.func_nodes, self.var_nodes = _separate_func_nodes_and_var_nodes(
self.nodes)
# figure out the roots and leaves
self.roots = set(root_nodes(self.graph))
self.leafs = set(leaf_nodes(self.graph))
# self.sig = Sig(dict(extract_items(sig.parameters, 'xz')))
self.sig = Sig(sort_params(self.src_name_params(self.roots)))
self.sig(self) # to put the signature on the callable DAG
self.last_scope = None
def __call__(self, *args, **kwargs):
scope = self.sig.kwargs_from_args_and_kwargs(args, kwargs)
self.call_on_scope(scope)
tup = tuple(extract_values(scope, self.leafs))
if len(tup) > 1:
return tup
elif len(tup) == 1:
return tup[0]
else:
return None
def call_on_scope(self, scope=None):
"""Calls the func_nodes using scope (a dict or MutableMapping) both to
source it's arguments and write it's results.
Note: This method is only meant to be used as a backend to __call__, not as
an actual interface method. Additional control/constraints on read and writes
can be implemented by providing a custom scope for that. For example, one could
log read and/or writes to specific keys, or disallow overwriting to an existing
key (useful for pipeline sanity), etc.
"""
if scope is None:
scope = dict() # fresh new scope
if self.cache_last_scope:
self.last_scope = scope # just to remember it, for debugging purposes ONLY!
for func_node in self.func_nodes:
func_node.call_on_scope(scope)
def __getitem__(self, item):
return self._getitem(item)
def _getitem(self, item):
input_names, output_names = item
# ------------ utils --------------------------------------------------------------
def src_name_params(self, src_names: Optional[Iterable[str]] = None):
d = defaultdict(list)
for node in self.func_nodes:
for arg_name, src_name in node.src_names.items():
d[src_name].append(node.sig.parameters[arg_name])
if src_names is None:
src_names = set(d)
for src_name in filter(src_names.__contains__, d):
params = d[src_name]
if len(params) == 1:
yield params[0].replace(name=src_name)
else:
yield self.parameter_merge(params).replace(name=src_name)
# ------------ display --------------------------------------------------------------
def synopsis_string(self):
return '\n'.join(func_node.synopsis_string()
for func_node in self.func_nodes)
# TODO: Give more control (merge with lined)
def dot_digraph_body(self):
yield from dot_lines_of_func_nodes(self.func_nodes)
@wraps(dot_digraph_body)
def dot_digraph_ascii(self, *args, **kwargs):
"""Get an ascii art string that represents the pipeline"""
from lined.util import dot_to_ascii
return dot_to_ascii('\n'.join(self.dot_digraph_body(*args, **kwargs)))
@wraps(dot_digraph_body)
def dot_digraph(self, *args, **kwargs):
try:
import graphviz
except (ModuleNotFoundError, ImportError) as e:
raise ModuleNotFoundError(
f'{e}\nYou may not have graphviz installed. '
f'See https://pypi.org/project/graphviz/.')
body = list(self.dot_digraph_body(*args, **kwargs))
return graphviz.Digraph(body=body)
def call_func(func, **kwargs):
args, kwargs = Sig(func).args_and_kwargs_from_kwargs(kwargs)
return func(*args, **kwargs)
def missing_args_func_(func):
missing_args = set(Sig(func).without_defaults.parameters) - set(ignore or ())
if func_to_kwargs is not None:
missing_args -= func_to_kwargs(func).keys()
return missing_args
def func_to_kwargs(func):
return {
k: val_for_argname[k]
for k in val_for_argname.keys() & set(Sig(func).without_defaults)
}
def call_func(func, kwargs):
kwargs = {k.__name__: v for k, v in kwargs.items()}
return Sig(func).source_kwargs(kwargs)