def info(calculators, gsims, views, exports, extracts, report, input_file=''): """ Give information. You can pass the name of an available calculator, a job.ini file, or a zip archive with the input files. """ logging.basicConfig(level=logging.INFO) if calculators: for calc in sorted(base.calculators): print(calc) if gsims: for gs in gsim.get_available_gsims(): print(gs) if views: for name in sorted(view): print(name) if exports: dic = groupby(export, operator.itemgetter(0), lambda group: [r[1] for r in group]) n = 0 for exporter, formats in dic.items(): print(exporter, formats) n += len(formats) print('There are %d exporters defined.' % n) if extracts: for key in extract: func = extract[key] if hasattr(func, '__wrapped__'): fm = FunctionMaker(func.__wrapped__) else: fm = FunctionMaker(func) print('%s(%s)%s' % (fm.name, fm.signature, fm.doc)) if os.path.isdir(input_file) and report: with Monitor('info', measuremem=True) as mon: with mock.patch.object(logging.root, 'info'): # reduce logging do_build_reports(input_file) print(mon) elif input_file.endswith('.xml'): node = nrml.read(input_file) if node[0].tag.endswith('sourceModel'): if node['xmlns'].endswith('nrml/0.4'): raise InvalidFile( '%s is in NRML 0.4 format, please run the following ' 'command:\noq upgrade_nrml %s' % (input_file, os.path.dirname(input_file) or '.')) print(source_model_info(node[0])) else: print(node.to_str()) elif input_file.endswith(('.ini', '.zip')): with Monitor('info', measuremem=True) as mon: if report: print('Generated', reportwriter.build_report(input_file)) else: print_csm_info(input_file) if mon.duration > 1: print(mon) elif input_file: print("No info for '%s'" % input_file)
def make_transformer_class(func=identity): """Make an improved FunctionTransformer class from a function.""" signature = inspect.signature(func) # args = [ # name for name, param in signature.parameters.items() # if param.default is inspect._empty # ] kwargs_defaults = [ (name, param.default) for name, param in signature.parameters.items() if param.default is not inspect._empty ] if kwargs_defaults: kwargs, defaults = zip(*kwargs_defaults) else: kwargs, defaults = [], [] # all_args = list(args) + list(kwargs) init_signature = '__init__(self, {args})'.format(args=', '.join(kwargs)) init_kwarg_string = '\n'.join( ['self.{kwarg}={kwarg}'.format(kwarg=kwarg) for kwarg in kwargs] ) init_body = 'self.func = func\n{init_kwarg_string}'.format( init_kwarg_string=init_kwarg_string ) proto__init = FunctionMaker.create( init_signature, init_body, {'func': func}, defaults=defaults ) proto_fit = FunctionMaker.create('fit(self, x)', 'return self', {}) kwarg_string = ', '.join( ['{kwarg}=self.{kwarg}'.format(kwarg=kwarg) for kwarg in kwargs] ) transform_body = 'return self.func(x, {kwarg_string})'.format( kwarg_string=kwarg_string ) proto_transform = FunctionMaker.create( 'transform(self, x)', transform_body, {} ) proto_dict = { '__init__': proto__init, '__doc__': func.__doc__, 'fit': proto_fit, 'transform': proto_transform } new_class = type( 'FunctionTransformer_' + func.__name__, (BaseEstimator, TransformerMixin), proto_dict ) return new_class
def litetask_futures(func): """ Add monitoring support to the decorated function. The last argument must be a monitor object. """ def wrapper(*args): monitor = args[-1] check_mem_usage(monitor) # check if too much memory is used monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) # NB: flush must not be called in the workers - they must not # have access to the datastore - so we remove it rec_delattr(monitor, 'flush') return result # NB: the returned function must have the same signature of func; # we need pickle=True because celery is using the worst possible # protocol; once we remove celery we can try to remove pickle=True return FunctionMaker.create( func, 'return _s_(_w_, (%(shortsignature)s,), pickle=True)', dict(_s_=safely_call, _w_=wrapper), task_func=func)
def generate_ui_command(self, display_name, name=None, argspec=None, method=None, function=None): if name is None: name = display_name if argspec.args[0] == "self": argspec.args.pop(0) signature = "%s(self, %s)" % (name, inspect.formatargspec(formatvalue=lambda val: "", *argspec)[1:-1]) # logger.error('sig=%s', signature) def func(self, *args, **kwargs): command = func.__command__ return self(_meth=command, *args, **kwargs) func = FunctionMaker.create( signature, # "return _func_(self, command, %(shortsignature)s)", "return _func_(%(shortsignature)s)", dict(_func_=func), __command__=name, ) func = types.MethodType(func, self) setattr(self, name, func) self._generated_ui_commands[name] = func
def _decorator_apply(dec, func): return FunctionMaker.create( func, "return decfunc(%(shortsignature)s)", dict(decfunc=dec(func)), __wrapped__=func, )
def generic(func): typemap = {(object,) * len(dispatch_args): func} def when_type(*types): "Decorator to add a method that will be called for the given types" if len(types) != len(dispatch_args): raise TypeError('Length mismatch: expected %d types, got %d' % (len(dispatch_args), len(types))) def dec(f): typemap[types] = f return f return dec def dispatch(dispatch_args, *args, **kw): types = tuple(type(arg) for arg in dispatch_args) try: # fast path return typemap[types](*args, **kw) except KeyError: pass _gettypemap = typemap.get for types in itertools.product(*(t.__mro__ for t in types)): f = _gettypemap(types) if f is not None: return f(*args, **kw) # else call the default implementation return func(*args, **kw) return FunctionMaker.create( func, 'return _f_({}, %(shortsignature)s)'.format(dispatch_str), dict(_f_=dispatch), when_type=when_type, default=func, typemap=typemap)
def decorator_apply(dec, func): """ Decorate a function by preserving the signature even if dec is not a signature-preserving decorator. """ return FunctionMaker.create( func, 'return decfunc(%(signature)s)', dict(decfunc=dec(func)), __wrapped__=func)
def decorator_apply(dec, func): """Decorate a function by preserving the signature even if dec is not a signature-preserving decorator. This recipe is derived from http://micheles.googlecode.com/hg/decorator/documentation.html#id14 """ return FunctionMaker.create( func, 'return decorated(%(signature)s)', dict(decorated=dec(func)), __wrapped__=func)
def replace_funct(fnc, body): """Returns a function with a new body that replaces the given function. The signature of the original function is preserved in the new function. """ fname = fnc.__name__ spec = getargspec(fnc) sig = formatargspec(*spec) return FunctionMaker.create('%s%s' % (fname,sig), body, {}, defaults=spec[3], doc=fnc.__doc__)
def decorator_apply(dec, func): """ Decorate a function by preserving the signature even if dec is not a signature-preserving decorator. """ return FunctionMaker.create(func, 'return decfunc(%(signature)s)', dict(decfunc=dec(func)), __wrapped__=func)
def _rpc(func): global _rpc_next_id, _rpc_map # Store function in the call map func_id = _rpc_next_id _rpc_next_id += 1 _rpc_map[func_id] = func # For local usage of the function, append a _ in the name #globals()['_' + func.__name__] = func # Handle special parameter 'websocket' func_meta = inspect.getargspec(func) try: ws_idx = func_meta.args.index('ws') except ValueError: ws_idx = -1 defaults = func_meta.defaults def ws_proxy_call(*args, **kwar): if len(args) > ws_idx: (ws_fd, ws_uid) = _ws_serialize(args[ws_idx]) args = list(args) args[ws_idx] = ws_uid ws_locator = ws_idx else: try: ws = kwar['ws'] except KeyError: ws = defaults[ws_idx] (ws_fd, ws_uid) = _ws_serialize(ws) kwar['ws'] = ws_uid ws_locator = None msg = pickle.dumps((func_id, args, kwar, ws_locator), pickle.HIGHEST_PROTOCOL) ret = fd_trick.send_with_fd(_call_endpoint, msg, ws_fd) if ret != len(msg): # TODO: Weird! Can this ever happen? Maybe if message is too big. # Do something about it... pass def proxy_call(*args, **kwar): msg = pickle.dumps((func_id, args, kwar), pickle.HIGHEST_PROTOCOL) ret = _call_endpoint.send(msg) if ret != len(msg): # TODO: Weird! Can this ever happen? Maybe if message is too big. # Do something about it... pass # Have no ideia of how this works, just copied from decorator.py: evaldict = func.func_globals.copy() evaldict['_call_'] = ws_proxy_call if ws_idx >= 0 else proxy_call return FunctionMaker.create(func, "return _call_(%(shortsignature)s)", evaldict)
def forwarder(cls, fnc, delegatename): """Returns a method that forwards calls on the scoping object to calls on the delegate object. The signature of the delegate method is preserved in the forwarding method. """ fname = fnc.__name__ spec = getargspec(fnc) sig = formatargspec(*spec) body = "return self.%s.%s(%s)" % (delegatename, fname, ",".join(spec[0][1:])) f = FunctionMaker.create("%s%s" % (fname, sig), body, {}, defaults=spec[3], doc=fnc.__doc__) return types.MethodType(f, None, cls)
def decorator_apply(dec, func): """ Decorate a function by preserving the signature even if dec is not a signature-preserving decorator. """ if (type(func) == types.BuiltinFunctionType) or (type(func) == types.BuiltinMethodType): return builtin_decorator_apply(dec, func) # FunctionMaker doesn't seem to work for built-ins (i.e. compiled code, it should though). return FunctionMaker.create( func, 'return decorated(%(signature)s)', dict(decorated=dec(func)), undecorated=func)
def forwarder(cls, fnc, delegatename): """Returns a method that forwards calls on the scoping object to calls on the delegate object. The signature of the delegate method is preserved in the forwarding method. """ fname = fnc.__name__ spec = getargspec(fnc) sig = formatargspec(*spec) body = 'return self.%s.%s(%s)' % (delegatename, fname, ','.join(spec[0][1:])) f = FunctionMaker.create('%s%s' % (fname,sig), body, {}, defaults=spec[3], doc=fnc.__doc__) return types.MethodType(f, None, cls)
def deco(func): return FunctionMaker.create( "w_wrapped_func(processes{})".format(pt_args), """ return exec_with_process( processes, func, args_list, kwargs_list, expected_results, timeout{}) """.format(pt_args), dict(args_list=args_list, kwargs_list=kwargs_list, expected_results=expected_results, timeout=timeout, func=func, exec_with_process=exec_with_process))
def factory(name, params, templ, rx=re.compile(r'{([^}]+)\}')): "Return a factory of Query objects" def _makesql(namespace, kind, as_, globs): def evaluator(match): return str(eval(match.group(1), globs or {}, namespace)) evaluated_templ = rx.sub(evaluator, templ) return Query(evaluated_templ, kind, as_) signature = "{}({}, kind, as_, globs)".format(name, ', '.join(params)) body = 'return _makesql(dict({}), kind, as_, globs)'.format( ', '.join('%s=%s' % (p, p) for p in params)) makesql = FunctionMaker.create( signature, body, dict(_makesql=_makesql), defaults=('all', None, None)) makesql.__doc__ = 'Build query objects from the template\n' + templ makesql.templ = templ return makesql
def litetask(func): """ Add monitoring support to the decorated function. The last argument must be a monitor object. """ def wrapper(*args): monitor = args[-1] monitor.flush = noflush with monitor('total ' + func.__name__, measuremem=True): result = func(*args) delattr(monitor, 'flush') return result # NB: we need pickle=True because celery is using the worst possible # protocol; once we remove celery we can try to remove pickle=True return FunctionMaker.create( func, 'return _s_(_w_, (%(shortsignature)s,), pickle=True)', dict(_s_=safely_call, _w_=wrapper), task_func=func)
def _decorate_polyfill(func, caller): """ decorate(func, caller) decorates a function using a caller. """ try: from decorator import decorate return decorate(func, caller) except ImportError: from decorator import FunctionMaker evaldict = dict(_call_=caller, _func_=func) fun = FunctionMaker.create(func, "return _call_(_func_, %(shortsignature)s)", evaldict, __wrapped__=func) if hasattr(func, '__qualname__'): fun.__qualname__ = func.__qualname__ return fun
def init_wrapper(cls, delegate_class_list): """Returns a function that calls the wrapped class' __init__ function and then creates an instance of each delegate class in delegate_class_list. """ fnc = cls.__init__ spec = getargspec(fnc) sig = formatargspec(*spec) template = ["if not hasattr(self, '_delegates_'): self._delegates_ = {}"] for name, delegate in delegate_class_list: template.append('self.%s = %s(self)' % (name, delegate.__name__)) template.append("self._delegates_['%s'] = self.%s" % (name,name)) template.append('old_init__(%s)' % ','.join(spec[0])) f = FunctionMaker.create('__init__%s' % sig, '\n'.join(template), dict([(c.__name__,c) for n,c in delegate_class_list]+ [('old_init__',fnc)]), doc=fnc.__doc__, defaults=spec[3]) return types.MethodType(f, None, cls)
def wrapped(func): return FunctionMaker.create( f"w_wrapped_func(processes{pt_args})", f""" return exec_with_process( processes, func, args_list, kwargs_list, expected_results, timeout{pt_args}) """, dict( args_list=args_list, kwargs_list=kwargs_list, expected_results=expected_results, timeout=timeout, func=func, exec_with_process=exec_with_process, ), )
def init_wrapper(cls, delegate_class_list): """Returns a function that calls the wrapped class' __init__ function and then creates an instance of each delegate class in delegate_class_list. """ fnc = cls.__init__ spec = getargspec(fnc) sig = formatargspec(*spec) template = [ 'old_init__(%s)' % ','.join(spec[0]), ] for name, delegate in delegate_class_list: template.append('self.%s = %s(self)' % (name, delegate.__name__)) template.append("if not hasattr(self, '_delegates_'): self._delegates_ = {}") template.append("self._delegates_['%s'] = self.%s" % (name,name)) f = FunctionMaker.create('__init__%s' % sig, '\n'.join(template), dict([(c.__name__,c) for n,c in delegate_class_list]+ [('old_init__',fnc)]), doc=fnc.__doc__, defaults=spec[3]) return types.MethodType(f, None, cls)
def litetask_futures(func): """ Add monitoring support to the decorated function. The last argument must be a monitor object. """ def wrapper(*args): monitor = args[-1] check_mem_usage(monitor) # check if too much memory is used monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) rec_delattr(monitor, 'flush') return result # NB: we need pickle=True because celery is using the worst possible # protocol; once we remove celery we can try to remove pickle=True return FunctionMaker.create( func, 'return _s_(_w_, (%(shortsignature)s,), pickle=True)', dict(_s_=safely_call, _w_=wrapper), task_func=func)
def decorator(f): # collect our argspec sig = signature(f) @wraps(f) def _(*args, **kwargs): bvals = sig.bind(*args, **kwargs) # replace with instance if desired for varname, val in bvals.arguments.items(): anno = sig.parameters[varname].annotation if anno in classes or (len(classes) == 0 and anno != _empty): bvals.arguments[varname] = anno(val) return f(*bvals.args, **bvals.kwargs) # create another layer by wrapping in a FunctionMaker. this is done # to preserve the original signature return FunctionMaker.create( f, 'return _(%(signature)s)', dict(_=_, __wrapped__=f) )
def main(what, report=False): """ Give information about the passed keyword or filename """ if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' if what == 'calculators': for calc in sorted(base.calculators): print(calc) elif what == 'gsims': for gs in gsim.get_available_gsims(): print(gs) elif what == 'imts': for im in gen_subclasses(imt.IMT): print(im.__name__) elif what == 'views': for name in sorted(view): print(name) elif what == 'exports': dic = groupby(export, operator.itemgetter(0), lambda group: [r[1] for r in group]) n = 0 for exporter, formats in dic.items(): print(exporter, formats) n += len(formats) print('There are %d exporters defined.' % n) elif what == 'extracts': for key in extract: func = extract[key] if hasattr(func, '__wrapped__'): fm = FunctionMaker(func.__wrapped__) elif hasattr(func, 'func'): # for partial objects fm = FunctionMaker(func.func) else: fm = FunctionMaker(func) print('%s(%s)%s' % (fm.name, fm.signature, fm.doc)) elif what == 'parameters': params = [] for val in vars(OqParam).values(): if hasattr(val, 'name'): params.append(val) params.sort(key=lambda x: x.name) for param in params: print(param.name) elif what == 'mfds': for cls in gen_subclasses(BaseMFD): print(cls.__name__) elif what == 'sources': for cls in gen_subclasses(BaseSeismicSource): print(cls.__name__) elif os.path.isdir(what) and report: with Monitor('info', measuremem=True) as mon: with mock.patch.object(logging.root, 'info'): # reduce logging do_build_reports(what) print(mon) elif what.endswith('.xml'): node = nrml.read(what) if node[0].tag.endswith('sourceModel'): print(source_model_info([node])) elif node[0].tag.endswith('logicTree'): sm_nodes = [] for smpath in logictree.collect_info(what).smpaths: sm_nodes.append(nrml.read(smpath)) print(source_model_info(sm_nodes)) else: print(node.to_str()) elif what.endswith(('.ini', '.zip')): with Monitor('info', measuremem=True) as mon: if report: print('Generated', reportwriter.build_report(what)) else: print(readinput.get_oqparam(what).json()) if mon.duration > 1: print(mon) elif what: print("No info for '%s'" % what)
def main(what, report=False): """ Give information about the passed keyword or filename """ if os.environ.get('OQ_DISTRIBUTE') not in ('no', 'processpool'): os.environ['OQ_DISTRIBUTE'] = 'processpool' if what == 'calculators': for calc in sorted(base.calculators): print(calc) elif what == 'gsims': for gs in gsim.get_available_gsims(): print(gs) elif what == 'portable_gsims': for gs in gsim.get_portable_gsims(): print(gs) elif what == 'imts': for im in vars(imt).values(): if inspect.isfunction(im) and is_upper(im): print(im.__name__) elif what == 'views': for name in sorted(view): print(name) elif what == 'exports': dic = groupby(export, operator.itemgetter(0), lambda group: [r[1] for r in group]) items = [(DISPLAY_NAME.get(exporter, '?'), exporter, formats) for exporter, formats in dic.items()] n = 0 for dispname, exporter, formats in sorted(items): print(dispname, '"%s"' % exporter, formats) n += len(formats) print('There are %d exporters defined.' % n) elif what == 'extracts': for key in extract: func = extract[key] if hasattr(func, '__wrapped__'): fm = FunctionMaker(func.__wrapped__) elif hasattr(func, 'func'): # for partial objects fm = FunctionMaker(func.func) else: fm = FunctionMaker(func) print('%s(%s)%s' % (fm.name, fm.signature, fm.doc)) elif what == 'parameters': docs = OqParam.docs() names = set() for val in vars(OqParam).values(): if hasattr(val, 'name'): names.add(val.name) params = sorted(names) for param in params: print(param) print(docs[param]) elif what == 'mfds': for cls in gen_subclasses(BaseMFD): print(cls.__name__) elif what == 'venv': print(sys.prefix) elif what == 'sources': for cls in gen_subclasses(BaseSeismicSource): print(cls.__name__) elif what == 'consequences': known = scientific.KNOWN_CONSEQUENCES print('The following %d consequences are implemented:' % len(known)) for cons in known: print(cons) elif os.path.isdir(what) and report: with Monitor('info', measuremem=True) as mon: with mock.patch.object(logging.root, 'info'): # reduce logging do_build_reports(what) print(mon) elif what.endswith('.xml'): node = nrml.read(what) if node[0].tag.endswith('sourceModel'): print(source_model_info([node])) elif node[0].tag.endswith('logicTree'): bset = node[0][0] if bset.tag.endswith("logicTreeBranchingLevel"): bset = bset[0] if bset.attrib['uncertaintyType'] == 'sourceModel': sm_nodes = [] for smpath in logictree.collect_info(what).smpaths: sm_nodes.append(nrml.read(smpath)) print(source_model_info(sm_nodes)) elif bset.attrib['uncertaintyType'] == 'gmpeModel': print(logictree.GsimLogicTree(what)) else: print(node.to_str()) elif what.endswith(('.ini', '.zip')): with Monitor('info', measuremem=True) as mon: if report: print('Generated', reportwriter.build_report(what)) else: print(readinput.get_oqparam(what).json()) if mon.duration > 1: print(mon) elif what: print("No info for '%s'" % what)
def decorator_apply(dec, func): return FunctionMaker.create(func, 'return decorated(%(signature)s)', dict(decorated=dec(func)), __wrapped__=func)
def dec(f): check_is_argument(conn_arg, f) return FunctionMaker.create(f, '''\ with {}.opencopy() as {}, {}.begin(): return _f_(%(shortsignature)s) '''.format(conn_arg, conn_arg, conn_arg), dict(_f_=f))
def dec(f): check_is_argument(arg, f) return FunctionMaker.create(f, '''\ with {}: return _f_(%(shortsignature)s) '''.format(arg), dict(_f_=f))
def decorator_apply(dec, func): return FunctionMaker.create( func, 'return decorated(%(signature)s)', dict(decorated=dec(func)), __wrapped__=func)
def verbose(function: _FuncT) -> _FuncT: """Verbose decorator to allow functions to override log-level. Parameters ---------- function : callable Function to be decorated by setting the verbosity level. Returns ------- dec : callable The decorated function. See Also -------- set_log_level set_config Notes ----- This decorator is used to set the verbose level during a function or method call, such as :func:`mne.compute_covariance`. The `verbose` keyword argument can be 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', True (an alias for 'INFO'), or False (an alias for 'WARNING'). To set the global verbosity level for all functions, use :func:`mne.set_log_level`. This function also serves as a docstring filler. Examples -------- You can use the ``verbose`` argument to set the verbose level on the fly:: >>> import mne >>> cov = mne.compute_raw_covariance(raw, verbose='WARNING') # doctest: +SKIP >>> cov = mne.compute_raw_covariance(raw, verbose='INFO') # doctest: +SKIP Using up to 49 segments Number of samples used : 5880 [done] """ # noqa: E501 # See https://decorator.readthedocs.io/en/latest/tests.documentation.html # #dealing-with-third-party-decorators try: fill_doc(function) except TypeError: # nothing to add pass # Anything using verbose should have `verbose=None` in the signature. # This code path will raise an error if this is not the case. body = """\ def %(name)s(%(signature)s):\n try: do_level_change = verbose is not None except (NameError, UnboundLocalError): raise RuntimeError('Function/method %%s does not accept verbose ' 'parameter' %% (_function_,)) from None if do_level_change: with _use_log_level_(verbose): return _function_(%(shortsignature)s) else: return _function_(%(shortsignature)s)""" evaldict = dict(_use_log_level_=use_log_level, _function_=function) fm = FunctionMaker(function, None, None, None, None, function.__module__) attrs = dict(__wrapped__=function, __qualname__=function.__qualname__, __globals__=function.__globals__) return fm.make(body, evaldict, addsource=True, **attrs)
def __call__(self, func): return FunctionMaker.create( func, 'with _self_: return _f_(%(shortsignature)s)', dict(_f_=func, _self_=self))