def _set(store: typing.Callable, key: str, data: typing.Any) -> str:
"""Wraps redis.set command.
"""
store.set(key, json.dumps(encoder.encode(data), indent=4))
return key
def _set_one(store: typing.Callable, item: Item) -> str:
"""Set item under a key.
"""
store.set(item.key, item.data_as_json, ex=item.expiration)
return item.key
def _decorator(fn: typing.Callable) -> typing.Callable:
fn.task = settings.TASKHAWK_TASK_CLASS(fn, priority)
fn.dispatch = fn.task.dispatch
fn.with_headers = fn.task.with_headers
fn.with_priority = fn.task.with_priority
_ALL_TASKS[fn.task.name] = fn.task
return fn
def generate_networks(args, tasks, masker: typing.Callable):
"""Generate splitted per-task neural networks.
Parameters
----------
args: argparse.Namespace
argparse.ArgumentParser().parse() return value. User provided arguments.
tasks: int
How many tasks were done within original network.
masker:
Object creating and applying masks to neural network layers
Yields
-------
torch.nn.Module
Module with per-task masks applied.
"""
path = pathlib.Path(args.save)
path.mkdir(parents=True, exist_ok=True)
masks = get_masks(args, tasks, masker)
for task in range(tasks):
model = get_model(args)
layer_idx = 0
for module in model.modules():
if nn.layers.spatial(module):
print(masks[layer_idx].shape, module)
if layer_idx in args.where:
masker.apply(module.weight.data, masks[layer_idx], task)
layer_idx += 1
torch.save(model, path / f"{task}.pt")
def add_route(
self,
path: str,
endpoint: typing.Callable,
methods: typing.List[str] = None,
name: str = None,
include_in_schema: bool = True,
status_code: int = 200,
response_schema: marshmallow.Schema = None,
request_schemas: typing.Dict[str, marshmallow.Schema] = None,
):
# If @schemas is used, it has precedence
# i.e.:
# @app.route('/', arg1: FooSchema)
# @app.schemas('/', arg2: BarSchema, response_schema=FooBarSchema)
# def endpoint(arg1, arg2):
# pass
if getattr(endpoint, '_request_schemas', None):
merged_schemas = request_schemas.copy()
merged_schemas.update(endpoint._request_schemas)
endpoint._request_schemas = merged_schemas
else:
endpoint._request_schemas = request_schemas
if getattr(endpoint, '_response_schema', None) is None:
endpoint._response_schema = response_schema
self.routes.append(
Route(path,
endpoint=endpoint,
methods=methods,
name=name,
include_in_schema=include_in_schema,
router=self,
status_code=status_code))
def _get_all(store: typing.Callable, search_key: str) -> typing.List[typing.Any]:
"""Wraps redis.mget command.
"""
CHUNK_SIZE = 5000
_, keys = store.scan(match=search_key, count=CHUNK_SIZE)
return [_decode_item(i) for i in store.mget(keys)] if keys else []
def _flush(store: typing.Callable, ns_keys: str):
"""Flushes data from cache.
"""
CHUNK_SIZE = 1000
cursor = '0'
while cursor != 0:
cursor, keys = store.scan(cursor=cursor, match=ns_keys, count=CHUNK_SIZE)
if keys:
store.delete(*keys)
def _set_one_singleton(store: typing.Callable,
item: Item) -> typing.Tuple[str, bool]:
"""Sets item under a key if not already cached.
"""
key, was_cached = item.key, bool(store.setnx(item.key, item.data_as_json))
if was_cached and item.expiration:
store.expire(key, item.expiration)
return key, was_cached
def _delete_many(store: typing.Callable, search_key: SearchKey):
"""Deletes items under matching keys.
"""
chunk_size = 1000
cursor = '0'
while cursor != 0:
cursor, keys = store.scan(cursor=cursor,
match=search_key.key,
count=chunk_size)
if keys:
store.delete(*keys)
def _get_one_from_many(store: typing.Callable,
item_key: ItemKey) -> typing.Any:
"""Returns item under first matched key.
"""
chunk_size = 1000
cursor = '0'
while cursor != 0:
cursor, keys = store.scan(cursor=cursor,
match=item_key.key,
count=chunk_size)
if keys:
return _decode_item(store.get(keys[0]))
def script(func: t.Callable = None, help: HelpDict = None):
if not func:
if help:
return functools.partial(script, help=help)
else:
return Script()
scrpt = Script(func, help=help)
print(scrpt)
func.run = scrpt.run
func.subcommand = scrpt.subcommand
func._script = scrpt
return func
def _get_many(store: typing.Callable,
search_key: SearchKey) -> typing.List[typing.Any]:
"""Returns collection cached under all matched keys.
"""
keys = []
chunk_size = 2000
cursor = '0'
while cursor != 0:
cursor, keys_ = store.scan(cursor=cursor,
match=search_key.key,
count=chunk_size)
keys += keys_
return [_decode_item(i) for i in store.mget(keys)] if keys else []
def create_Callable(args: Iterable[type], rtype: type, class_poly_vars: Set[type] = None) -> Callable:
"""Initialize and return Callable with given parameters, return types, and polymorphic type variables."""
poly_vars = set(class_poly_vars) if class_poly_vars else set()
c = Callable.copy_with(tuple([*args, rtype]))
poly_vars.update(_get_poly_vars(c))
c.__polymorphic_tvars__ = frozenset(poly_vars)
return c
def apply_enforcer(
func: typing.Callable,
generic: bool = False,
settings=None,
parent_root: typing.Optional[Validator] = None,
instance_of: typing.Optional[GenericProxy] = None,
) -> typing.Callable:
"""
Adds an Enforcer instance to the passed function/generic if it doesn't yet exist
or if it is not an instance of Enforcer
Such instance is added as '__enforcer__'
"""
if not hasattr(func, "__enforcer__") or not isinstance(
func.__enforcer__, Enforcer):
# if not hasattr(func, '__enforcer__'):
# func = EnforceProxy(func)
# if not isinstance(func.__enforcer__, Enforcer):
# Replaces 'incorrect' enforcers
func.__enforcer__ = generate_new_enforcer(func, generic, parent_root,
instance_of, settings)
func.__enforcer__.reference = func
return func
def __call__(self, func:typing.Callable):
string = func.__doc__
pos = string.index(cn.EXPAND)
if pos < 0:
return func
# Find the indentation for the line
indent = pos
for idx in range(1, pos):
if string[pos - idx] != " ":
indent = idx - 1
break
indent_str = self._getIndentStr(indent)
# Construct the expansion
if len(self.header) > 0:
expansion = self._indentText(self.header, indent_str)
else:
expansion = ""
keywords = list(self.keywords)
keywords.sort()
for idx, keyword in enumerate(keywords):
self.kwarg_dct[keyword].setIndent(self._indent)
expansion += str(self.kwarg_dct[keyword])
expansion += self._indentText(self.trailer, indent_str)
# Replace the docstring
replace_str = "%s%s" % (indent_str, cn.EXPAND)
func.__doc__ = string.replace(replace_str, expansion)
return func
def func(f: t.Callable):
name = f.__name__
#args = f.__arguments__
annotations = f.__annotations__
module = f.__module__
doc = f.__doc__
qname = f.__qualname__
for opt in sorted(option.options):
f = cmd_option(opt, name_prefix)(f)
f.__name__ = name[0:-2] if name.endswith("_") else name
f.__qualname__ = qname[0:-2] if qname.endswith("_") else qname
#f.__args__ = args
f.__annotations__ = annotations
f.__module__ = module
f.__doc__ = doc
return f
def _get_counter_many(
store: typing.Callable, search_key: SearchKey
) -> typing.Tuple[typing.List[str], typing.List[int]]:
"""Returns counts under matched keys.
"""
keys = []
chunk_size = 1000
cursor = '0'
while cursor != 0:
cursor, keys_ = store.scan(cursor=cursor,
match=search_key.key,
count=chunk_size)
keys += keys_
return [i.decode('utf8') for i in keys], [int(i) for i in store.mget(keys)]
def _get(store: typing.Callable, key: str) -> typing.Any:
"""Wraps redis.get command.
"""
obj = store.get(key)
if obj is not None:
return _decode_item(obj)
def reqrep(wrapped: t.Callable,
meth: t.Callable,
channel: str = "shell") -> t.Callable:
wrapped = wrapped(meth, channel)
if not meth.__doc__:
# python -OO removes docstrings,
# so don't bother building the wrapped docstring
return wrapped
basedoc, _ = meth.__doc__.split("Returns\n", 1)
parts = [basedoc.strip()]
if "Parameters" not in basedoc:
parts.append("""
Parameters
----------
""")
parts.append("""
reply: bool (default: False)
Whether to wait for and return reply
timeout: float or None (default: None)
Timeout to use when waiting for a reply
Returns
-------
msg_id: str
The msg_id of the request sent, if reply=False (default)
reply: dict
The reply message for this request, if reply=True
""")
wrapped.__doc__ = "\n".join(parts)
return wrapped
def event_source(method: t.Callable, name: t.Optional[str] = None):
"""A decorator which makes the function act as a source of before and after call events.
You can later subscribe to these event with :py:func:`before` and :py:func`after` decorators.
:param method: Target class method
:param: Name of for the join point. If not given use the function name.
"""
# We must use function name instead of function pointer for the registry, because function object changes with unbound vs. bound Python class methods
if not name:
name = method.__name__
@functools.wraps(method)
def _inner(*args, **kwargs):
_self = args[0]
fire_advisor_event(_self, name, AdvisorRole.before)
retval = method(*args, **kwargs)
fire_advisor_event(_self, name, AdvisorRole.after)
return retval
assert name not in _event_source_hooks, "There already exist event_source with same name"
_event_source_hooks.append(name)
method._event_source_name = name
return _inner
def create_Callable(args: Iterable[type], rtype: type, class_poly_vars: Set[type] = None) -> Callable:
"""Initialize and return Callable with given parameters, return types, and polymorphic type variables."""
poly_vars = set(class_poly_vars) if class_poly_vars else set()
c = Callable.copy_with(tuple([*args, rtype]))
poly_vars.update(_get_poly_vars(c))
c.__polymorphic_tvars__ = frozenset(poly_vars)
return c
def _get_counter_one(store: typing.Callable, item_key: ItemKey) -> int:
"""Returns count under exactly matched key.
"""
count = store.get(item_key.key)
return 0 if count is None else int(count)
def add_websocket_route(self, path: str, route: typing.Callable) -> None:
if not inspect.isclass(route):
route = asgi_from_websocket(route, self.injector)
else:
route.injector = self.injector
instance = Path(path, route, protocol="websocket")
self.router.routes.append(instance)
def parser(generator_func: typing.Callable) -> typing.Callable:
"decorator function to wrap a generator"
def create_parser(*args, **kwargs) -> Parser:
return Parser(generator_func(*args, **kwargs))
generator_func.parser = create_parser
return generator_func
def override_in_channel(func: typing.Callable) -> typing.Callable:
"""
Set command callback attribute for detection in `in_channel_check`.
This decorator has to go before (below) below the `command` decorator.
"""
func.in_channel_override = True
return func
def wrapper(func: typing.Callable) -> typing.Callable:
func._meta = ResourceMethodMeta(
path=path,
methods=methods if methods is not None else ["GET"],
name=name,
kwargs=kwargs)
return func
def _get_count(store: typing.Callable, search_key: str) -> int:
"""Wraps redis.mget command.
"""
CHUNK_SIZE = 5000
_, keys = store.scan(match=search_key, count=CHUNK_SIZE)
return len(keys)
def func(f: t.Callable):
name = f.__name__
#args = f.__arguments__
annotations = f.__annotations__
module = f.__module__
doc = f.__doc__
qname = f.__qualname__
for i, opt in enumerate(sorted(option.options)):
validate = None
if isinstance(opt, CmdOption) and i == 0:
validate = True
f = cmd_option(opt, name_prefix, validate=validate)(f)
f.__name__ = name[0:-2] if name.endswith("_") else name
f.__qualname__ = qname[0:-2] if qname.endswith("_") else qname
#f.__args__ = args
f.__annotations__ = annotations
f.__module__ = module
f.__doc__ = doc
return f
def test_cannot_instantiate(self):
with self.assertRaises(TypeError):
Callable()
with self.assertRaises(TypeError):
type(Callable)()
c = Callable[[int], str]
with self.assertRaises(TypeError):
c()
with self.assertRaises(TypeError):
type(c)()
def register_event(self, event: str, handler: typing.Callable) -> None:
"""Register the given function as a handler of the given event."""
# Set function attribute for future reference
handler.event = event
# Create default delegate if none exist
if event not in self.delegates:
self.delegates[event] = Delegate(event, self)
self.delegates[event].handlers.append(handler)
def metric(func: typing.Callable) -> typing.Callable:
"""Decorator function that should be used within a problem implementation to define metrics as functions.
Arguments:
func {typing.Callable} -- a function that calculates a metric
Returns:
typing.Callable -- the decorated function
"""
func._metric = func.__name__
return func
def _wrap_generic_meta(t: _GenericAlias, args: List[type]) -> TypeResult:
if t.__origin__ is tuple:
tuple_args = tuple(args)
# Handle the special case when t1 or t2 are empty tuples; TODO: investigate this
if tuple_args == ((),):
tuple_args = ()
return TypeInfo(Tuple[tuple_args])
elif is_callable(t):
c = Callable.copy_with(tuple(args))
c.__polymorphic_tvars__ = getattr(t, '__polymorphic_tvars__', frozenset())
return TypeInfo(c)
else:
return TypeInfo(t.copy_with(tuple(args)))
def __call__(self, func: typing.Callable) -> typing.Callable:
funcname = func.__name__
if self._argname not in inspect.signature(func).parameters:
raise ValueError("{} has no argument {}!".format(funcname,
self._argname))
if not hasattr(func, 'qute_args'):
func.qute_args = {} # type: ignore
elif func.qute_args is None: # type: ignore
raise ValueError("@cmdutils.argument got called above (after) "
"@cmdutils.register for {}!".format(funcname))
arginfo = command.ArgInfo(**self._kwargs)
func.qute_args[self._argname] = arginfo # type: ignore
return func
def apply_enforcer(func: typing.Callable,
generic: bool=False,
settings = None,
parent_root: typing.Optional[Validator]=None,
instance_of: typing.Optional[GenericProxy]=None) -> typing.Callable:
"""
Adds an Enforcer instance to the passed function/generic if it doesn't yet exist
or if it is not an instance of Enforcer
Such instance is added as '__enforcer__'
"""
if not hasattr(func, '__enforcer__') or not isinstance(func.__enforcer__, Enforcer):
# Replaces 'incorrect' enforcers
func.__enforcer__ = generate_new_enforcer(func, generic, parent_root, instance_of, settings)
func.__enforcer__.reference = func
return func