def sort_by_run(fs):
"""
oneliner:
1. sort `fs` from first to last to run
2. based on their attrs such as `toward_end`, `run_after`, `run_before`
purpose:
1. `_is_instance`, `_is_first` help us figure out whether `f` is first among `fs`
2. but in the end we want `fs` to be sorted by execution order
3. `sort_by_run` does it for us
3.1 'toward_end' help us find the last func in `fs`, put it at the end of `inp`, keep the rest funcs before it.
3.2 loop through the rest funcs `inp`, use `_is_first` to find the actual first func to run, and pop it out of `inp`, and into `res`
3.3 again loop through the remaining funcs `inp`, and do the previous step, until `inp` is empty
4. return res
"""
end = L(getattr(f, 'toward_end', False) for f in fs)
inp, res = L(fs)[~end] + L(fs)[end], []
while len(inp) > 0:
for i, o in enumerate(inp):
if _is_first(o, inp):
res.append(inp.pop(i))
break
else:
raise Exception("Impossible to sort")
return res
def _is_first(f, gs):
"""
oneliner:
1. whether `f` is the first func to run among `gs` (a list of funcs)
2. based on `run_after` and `run_before` of `gs`
purpose:
1. among many tranform funcs, it is important to figure out which run first
2. each func may have attributes like `run_after`, `run_before` to order
3. how can we figure out which func is first with their attributes above?
3.1 if `f` run_after `o`, and `o` is among `gs`, then `f` can't be first
3.2 say `g` run before `o`, and `f` _is_instance to `o`,
then `f` can't be first.
3.3 otherwise, yes, `f` is first
"""
for o in L(getattr(f, 'run_after', None)):
if _is_instance(o, gs): return False
for g in gs:
if _is_instance(f, L(getattr(g, 'run_before', None))): return False
return True
def compose(*funcs: Callable, order=None):
"""
purpose:
1. sometimes, we got a number functions to run in series, and
2. these funcs are even taking same args
3. so, we will refactor the code and make life easier with `compose`
3.1 all functions are organized into a L
3.2 all functions are sorted by `order` using `L.sorted`
4. then we create a `_inner()`:
4.1 we loop through every function
4.2 run the function with *args, **kwargs
Note: *args, **kwargs are given by user from outside, see examples below
"""
funcs = L(funcs)
if order is not None: funcs = funcs.sorted(order)
def _inner(x, *args, **kwargs):
for f in L(funcs):
x = f(x, *args, **kwargs)
return x
return _inner
if k in self._xtra:
return getattr(self.default, k)
raise AttributeError(k)
def __dir__(self):
return custom_dir(self, self._xtra)
############################################
# how _C borrow str.lower to use
class _C(GetAttr):
default, _xtra = 'Hi', ['lower']
t = _C()
t.default
t._xtra
t.__getattr__('lower')
t.lower
t.lower()
test_fail(lambda: t.upper(), contains='upper')
##########################################
# how L borrow list.sort to use
x = [1, 1, 0, 5, 0, 3]
x = list(OrderedDict.fromkeys(x).keys())
x
x = L(8, 9) + x
x.sort()
x
def mask2idxs(mask):
"Convert bool mask or index list to index `L`"
return L(_mask2idxs(mask))
from local.test import *
from local.imports import *
from local.notebook.showdoc import show_doc
from local.core import L, tensor
def mapper(f):
"""
purpose:
1. outside L, it would be nice to have sth similar to `L.mapped`
2. it would be nice if we can do `mapper(f)(data)`
2.1 `f` is the func we want to apply to every item of 'data'
2.2 `data` is the data including many items
3. `mapper(f)` return a lambda func, inside the func, it can
3.1 loop through every item `o_` from `o` (or `data`)
3.2 apply `f` on each `o_`
3.3 put each output into a single list
4. the lambda func returned from `mapper(f)` will take `(data)` to run
So, this above is the logic of `mapper(f)(data)`
"""
return lambda o: [f(o_) for o_ in o]
mapper(lambda o: o * 2)(range(3))
mapper(lambda o: o * 2)(L(1, 2, 3))
mapper(lambda o: o * 2)(array([1, 2, 3]))
mapper(lambda o: o * 2)(tensor([1, 2, 3]))
from local.test import * from local.imports import * from local.notebook.showdoc import show_doc from local.core import L, tensor L(None) tuplify(None, use_list=False, match=None) t = L([1, 2, 3]) t t = tuplify(t, use_list=False, match=None) t t = tuplify(t, use_list=True, match=None) t t = tuplify([1], use_list=True, match=[1, 2, 3]) t
from local.test import *
from local.imports import *
from local.notebook.showdoc import show_doc
################
from local.core import is_listy, tensor, L
def apply(func, x, *args, **kwargs):
"""
purpose:
1. although `map(f, x)` can apply f to every element of x on one level
2. but `map` can't dive deep recursively when element itself is iterable
3. we can do three things with this func
a. exactly like `map(f, x)` with `args` and `kwargs`
b. when `x` is list-like, do map recursively on x, level by level
c. when `x` is dict-like, do map recursively on x, level by level
"""
if is_listy(x): return [apply(func, o, *args, **kwargs) for o in x]
if isinstance(x, dict):
return {k: apply(func, v, *args, **kwargs) for k, v in x.items()}
return func(x, *args, **kwargs)
list(map(str, L([1, 2, 3])))
apply(str, [1, 2, 3])
apply(str, [1, [[2, [3]], [[[9]]]]])
apply(str, {'a': {'a1': {'aa1': 1}}, 'b': 3})
def _inner(x, *args, **kwargs):
for f in L(funcs):
x = f(x, *args, **kwargs)
return x
3. `idx` allows us to pick which branch/top_level of the data tree, and
4. after top_level, `idx` remains 0, so `item_find(x, idx=0)` gives us
the lowest/final_level leaf/item of the tree/data
note: it is like `x[idx][0]...[0]`
5. also, `x` can be listy(tuple, list, slice, L) or dict
"""
if is_listy(x): return item_find(x[idx])
if isinstance(x, dict):
key = list(x.keys())[idx] if isinstance(idx, int) else idx
return item_find(x[key])
return x
x = [[1, 2, 3], [2, 3, 4], [5, 6, 7]]
item_find(x=x, idx=1)
x = L(([1, 2, 3], [2, 3, 4], [5, 6, 7]))
item_find(x=x, idx=2)
x = ((1, 2, 3), (2, 3, 4), (5, 6, 7))
item_find(x=x, idx=0)
x = {'a': {"m": 5, "n": 7, "p": 9}, 'b': {"m": 15, "n": 17, "p": 19}}
item_find(x=x, idx=0)
item_find(x=x, idx=1)
item_find(x=x, idx='b')
###############################################################################
def find_device(b):
"""
purpose:
from local.test import * from local.imports import * from local.notebook.showdoc import show_doc from local.core import L assert is_listy([1]) assert is_listy(L([1])) assert is_listy(slice(2)) # when and how to use slice? assert not is_listy(torch.tensor([[1, 3], [2, 4]]))
def mk_class(nm, *fld_names, sup=None, doc=None, funcs=None, **flds):
"""
"Dynamically add a class containing `fld_names` to the caller's module"
purpose:
1. how do we use a func to create a class
2. prepare a class name as `nm`
3. prepare/add a list of names as `fld_names` to be `flds.keys()`,
- None to be these `flds.values()`
4. prepare/add a list of `funcs` whose names to be `flds.keys()`,
- `funcs` themselves as `flds.values()`
5. make super class a tuple of super class
6. get the module where this class should be defined
7. create _init method:
7.1 assign `*args` to `flds[fld_names[idx]]` by args' position/idx
7.2 assign `**kwargs` to `flds[key] = value`
8. create _repr method:
8.1 to print out all non-hidden and non-methods attributes with
their names and values
9. turn `_init` and `_repr` to `flds.__init__` and `flds.__repr__`
10. create a type/class:
10.1 its super class is `sup`
10.2 its class name is `nm`
10.3 its attribute content dict is `flds`
return `res`
11. add `docs` to the class `res`
12. add class `res` to the module `mod` under the name `nm`
"""
# fill the dict `flds` with keys as `fld_names` and values as 'None'
for f in fld_names:
flds[f] = None
# if `funcs` are provided, feed each `f.__name__` and `f` to `flds` the dict
for f in L(funcs):
flds[f.__name__] = f
# get super class ready,
sup = ifnone(sup, ())
# if super is not a tuple, make it one
if not isinstance(sup, tuple): sup = (sup, )
# to get the model where the class is defined
stk = inspect.stack()[1]
mod = ifnone(inspect.getmodule(stk[0]), sys.modules['__main__'])
# _init is to add funcs to match with fld_names as methods using both *args and **kwargs
def _init(self, *args, **kwargs):
for i, v in enumerate(args):
setattr(self, fld_names[i], v)
for k, v in kwargs.items():
setattr(self, k, v)
# to print out all non-hidden and non-method attributes
def _repr(self):
return '\n'.join(
f'{o}: {getattr(self,o)}' for o in set(dir(self))
if not o.startswith('_')
and not isinstance(getattr(self, o), types.MethodType))
# get __repr__ and __init__ assigned
flds['__repr__'] = _repr
flds['__init__'] = _init
# create this new type of class
res = type(nm, sup, flds)
# add docs to the class
if doc is not None: res.__doc__ = doc
# assign `res` to `mod.nm` to put this new class in the module
setattr(mod, nm, res)
def _inner(f):
mk_class(nm, *fld_names, sup=sup, doc=doc, funcs=L(funcs) + f, **flds)
return f