# Workers are created as daemon threads. This is done to allow the interpreter # to exit when there are still idle threads in a ThreadPoolExecutor's thread # pool (i.e. shutdown() was not called). However, allowing workers to die with # the interpreter has two undesirable properties: # - The workers would still be running during interpreter shutdown, # meaning that they would fail in unpredictable ways. # - The workers could be killed while evaluating a work item, which could # be bad if the callable being evaluated has external side-effects e.g. # writing to a file. # # To work around this problem, an exit handler is installed which tells the # workers to exit when their work queues are empty and then waits until the # threads finish. _threads_queues: weakref.WeakKeyDictionary[ threading.Thread, queue.Queue] = weakref.WeakKeyDictionary() _shutdown = False def _python_exit(): global _shutdown _shutdown = True items = list(_threads_queues.items()) for t, q in items: q.put(None) for t, q in items: t.join() atexit.register(_python_exit)
def __init__(self, pymux): self.pymux = pymux self._bodies_for_clis = weakref.WeakKeyDictionary( ) # Maps CLI to (hash, Container)
class Callback(param.Parameterized): """ A Callback defines some callback to be triggered when a property changes on the source object. A Callback can execute arbitrary Javascript code and will make all objects referenced in the args available in the JS namespace. """ args = param.Dict(default={}, allow_None=True, doc=""" A mapping of names to Python objects. These objects are made available to the callback's code snippet as the values of named parameters to the callback.""") code = param.Dict(default=None, doc=""" A dictionary mapping from a source specication to a JS code snippet to be executed if the source property changes.""") # Mapping from a source id to a Link instance registry = weakref.WeakKeyDictionary() # Mapping to define callbacks by backend and Link type. # e.g. Callback._callbacks[Link] = Callback _callbacks = {} # Whether the link requires a target _requires_target = False def __init__(self, source, target=None, **params): if source is None: raise ValueError('%s must define a source' % type(self).__name__) # Source is stored as a weakref to allow it to be garbage collected self._source = None if source is None else weakref.ref(source) super(Callback, self).__init__(**params) self.init() def init(self): """ Registers the Callback """ if self.source in self.registry: links = self.registry[self.source] params = { k: v for k, v in self.param.get_param_values() if k != 'name' } for link in links: link_params = { k: v for k, v in link.param.get_param_values() if k != 'name' } if not hasattr(link, 'target'): pass elif (type(link) is type(self) and link.source is self.source and link.target is self.target and params == link_params): return self.registry[self.source].append(self) else: self.registry[self.source] = [self] @classmethod def register_callback(cls, callback): """ Register a LinkCallback providing the implementation for the Link for a particular backend. """ cls._callbacks[cls] = callback @property def source(self): return self._source() if self._source else None @classmethod def _process_callbacks(cls, root_view, root_model): if not root_model: return linkable = root_view.select(Viewable) linkable += root_model.select({'type': BkModel}) if not linkable: return found = [(link, src, getattr(link, 'target', None)) for src in linkable for link in cls.registry.get(src, []) if not link._requires_target or link.target in linkable] arg_overrides = {} if 'holoviews' in sys.modules: from .pane.holoviews import HoloViews, generate_panel_bokeh_map hv_views = root_view.select(HoloViews) map_hve_bk = generate_panel_bokeh_map(root_model, hv_views) for src in linkable: for link in cls.registry.get(src, []): if hasattr(link, 'target'): for tgt in map_hve_bk.get(link.target, []): found.append((link, src, tgt)) arg_overrides[id(link)] = {} for k, v in link.args.items(): # Not all args are hashable try: hv_objs = map_hve_bk.get(v, []) except Exception: continue for tgt in hv_objs: arg_overrides[id(link)][k] = tgt ref = root_model.ref['id'] callbacks = [] for link, src, tgt in found: cb = cls._callbacks[type(link)] if ((src is None or ref not in getattr(src, '_models', [ref])) or (getattr(link, '_requires_target', False) and tgt is None) or (tgt is not None and ref not in getattr(tgt, '_models', [ref]))): continue overrides = arg_overrides.get(id(link), {}) callbacks.append( cb(root_model, link, src, tgt, arg_overrides=overrides)) return callbacks
import tool_shed.repository_types.util as rt_util from galaxy import util from galaxy.model.orm.now import now from galaxy.security.validate_user_input import validate_password_str from galaxy.util import unique_id from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable from galaxy.util.hash_util import new_secure_hash from tool_shed.dependencies.repository import relation_builder from tool_shed.util import (hg_util, metadata_util) from tool_shed.util.hgweb_config import hgweb_config_manager log = logging.getLogger(__name__) WEAK_HG_REPO_CACHE: Mapping['Repository', Any] = weakref.WeakKeyDictionary() if TYPE_CHECKING: from sqlalchemy.schema import Table class _HasTable: table: Table else: _HasTable = object class APIKeys(_HasTable): pass class User(Dictifiable, _HasTable):
def make_weak_keyed_dict(self): dict = weakref.WeakKeyDictionary() objects = map(Object, range(self.COUNT)) for o in objects: dict[o] = o.arg return dict, objects
class CloudPickler(Pickler): dispatch = Pickler.dispatch.copy() def __init__(self, file, protocol=None): if protocol is None: protocol = DEFAULT_PROTOCOL Pickler.__init__(self, file, protocol=protocol) # set of modules to unpickle self.modules = set() # map ids to dictionary. used to ensure that functions can share global env self.globals_ref = {} def dump(self, obj): self.inject_addons() try: return Pickler.dump(self, obj) except RuntimeError as e: if 'recursion' in e.args[0]: msg = """Could not pickle object as excessively deep recursion required.""" raise pickle.PicklingError(msg) def save_memoryview(self, obj): self.save(obj.tobytes()) dispatch[memoryview] = save_memoryview if not PY3: def save_buffer(self, obj): self.save(str(obj)) dispatch[buffer] = save_buffer def save_unsupported(self, obj): raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj)) dispatch[types.GeneratorType] = save_unsupported # itertools objects do not pickle! for v in itertools.__dict__.values(): if type(v) is type: dispatch[v] = save_unsupported def save_module(self, obj): """ Save a module as an import """ mod_name = obj.__name__ # If module is successfully found then it is not a dynamically created module if hasattr(obj, '__file__'): is_dynamic = False else: try: _find_module(mod_name) is_dynamic = False except ImportError: is_dynamic = True self.modules.add(obj) if is_dynamic: self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), obj=obj) else: self.save_reduce(subimport, (obj.__name__, ), obj=obj) dispatch[types.ModuleType] = save_module def save_codeobject(self, obj): """ Save a code object """ if PY3: args = (obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars) else: args = (obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars) self.save_reduce(types.CodeType, args, obj=obj) dispatch[types.CodeType] = save_codeobject def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ if obj in _BUILTIN_TYPE_CONSTRUCTORS: # We keep a special-cased cache of built-in type constructors at # global scope, because these functions are structured very # differently in different python versions and implementations (for # example, they're instances of types.BuiltinFunctionType in # CPython, but they're ordinary types.FunctionType instances in # PyPy). # # If the function we've received is in that cache, we just # serialize it as a lookup into the cache. return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj) write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None try: lookedup_by_name = getattr(themodule, name, None) except Exception: lookedup_by_name = None if themodule: self.modules.add(themodule) if lookedup_by_name is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if PY3: rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '<stdin>' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) dispatch[types.FunctionType] = save_function def _save_subimports(self, code, top_level_dependencies): """ Ensure de-pickler imports any package child-modules that are needed by the function """ # check if any known dependency is an imported package for x in top_level_dependencies: if isinstance(x, types.ModuleType) and hasattr( x, '__package__') and x.__package__: # check if the package has any currently loaded sub-imports prefix = x.__name__ + '.' for name, module in sys.modules.items(): # Older versions of pytest will add a "None" module to sys.modules. if name is not None and name.startswith(prefix): # check whether the function can address the sub-module tokens = set(name[len(prefix):].split('.')) if not tokens - set(code.co_names): # ensure unpickler executes this import self.save(module) # then discards the reference to it self.write(pickle.POP) def save_dynamic_class(self, obj): """ Save a class that can't be stored as module global. This method is used to serialize classes that are defined inside functions, or that otherwise can't be serialized as attribute lookups from global modules. """ clsdict = dict(obj.__dict__) # copy dict proxy to a dict clsdict.pop('__weakref__', None) # On PyPy, __doc__ is a readonly attribute, so we need to include it in # the initial skeleton class. This is safe because we know that the # doc can't participate in a cycle with the original class. type_kwargs = {'__doc__': clsdict.pop('__doc__', None)} # If type overrides __dict__ as a property, include it in the type kwargs. # In Python 2, we can't set this attribute after construction. __dict__ = clsdict.pop('__dict__', None) if isinstance(__dict__, property): type_kwargs['__dict__'] = __dict__ save = self.save write = self.write # We write pickle instructions explicitly here to handle the # possibility that the type object participates in a cycle with its own # __dict__. We first write an empty "skeleton" version of the class and # memoize it before writing the class' __dict__ itself. We then write # instructions to "rehydrate" the skeleton class by restoring the # attributes from the __dict__. # # A type can appear in a cycle with its __dict__ if an instance of the # type appears in the type's __dict__ (which happens for the stdlib # Enum class), or if the type defines methods that close over the name # of the type, (which is utils for Python 2-style super() calls). # Push the rehydration function. save(_rehydrate_skeleton_class) # Mark the start of the args tuple for the rehydration function. write(pickle.MARK) # Create and memoize an skeleton class with obj's name and bases. tp = type(obj) self.save_reduce(tp, (obj.__name__, obj.__bases__, type_kwargs), obj=obj) # Now save the rest of obj's __dict__. Any references to obj # encountered while saving will point to the skeleton class. save(clsdict) # Write a tuple of (skeleton_class, clsdict). write(pickle.TUPLE) # Call _rehydrate_skeleton_class(skeleton_class, clsdict) write(pickle.REDUCE) def save_function_tuple(self, func): """ Pickles an actual func object. A func comprises: code, globals, defaults, closure, and dict. We extract and save these, injecting reducing functions at certain points to recreate the func object. Keep in mind that some of these pieces can contain a ref to the func itself. Thus, a naive save on these pieces could trigger an infinite loop of save's. To get around that, we first create a skeleton func object using just the code (this is safe, since this won't contain a ref to the func), and memoize it as soon as it's created. The other stuff can then be filled in later. """ if is_tornado_coroutine(func): self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__, ), obj=func) return save = self.save write = self.write code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data( func) save(_fill_function) # skeleton function updater write(pickle.MARK) # beginning of tuple that _fill_function expects self._save_subimports( code, itertools.chain(f_globals.values(), closure_values or ()), ) # create a skeleton function object and memoize it save(_make_skel_func) save(( code, len(closure_values) if closure_values is not None else -1, base_globals, )) write(pickle.REDUCE) self.memoize(func) # save the rest of the func data needed by _fill_function state = { 'globals': f_globals, 'defaults': defaults, 'dict': dct, 'module': func.__module__, 'closure_values': closure_values, } if hasattr(func, '__qualname__'): state['qualname'] = func.__qualname__ save(state) write(pickle.TUPLE) write(pickle.REDUCE) # applies _fill_function on the tuple _extract_code_globals_cache = (weakref.WeakKeyDictionary() if not hasattr(sys, "pypy_version_info") else {}) @classmethod def extract_code_globals(cls, co): """ Find all globals names read or written to by codeblock co """ out_names = cls._extract_code_globals_cache.get(co) if out_names is None: try: names = co.co_names except AttributeError: # PyPy "builtin-code" object out_names = set() else: out_names = set(names[oparg] for op, oparg in _walk_global_ops(co)) # see if nested function have any global refs if co.co_consts: for const in co.co_consts: if type(const) is types.CodeType: out_names |= cls.extract_code_globals(const) cls._extract_code_globals_cache[co] = out_names return out_names def extract_func_data(self, func): """ Turn the function into a tuple of data necessary to recreate it: code, globals, defaults, closure_values, dict """ code = func.__code__ # extract all global ref's func_global_refs = self.extract_code_globals(code) # process all variables referenced by global environment f_globals = {} for var in func_global_refs: if var in func.__globals__: f_globals[var] = func.__globals__[var] # defaults requires no processing defaults = func.__defaults__ # process closure closure = (list(map(_get_cell_contents, func.__closure__)) if func.__closure__ is not None else None) # save the dict dct = func.__dict__ base_globals = self.globals_ref.get(id(func.__globals__), {}) self.globals_ref[id(func.__globals__)] = base_globals return (code, f_globals, defaults, closure, dct, base_globals) def save_builtin_function(self, obj): if obj.__module__ == "__builtin__": return self.save_global(obj) return self.save_function(obj) dispatch[types.BuiltinFunctionType] = save_builtin_function def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__main__": return self.save_dynamic_class(obj) try: return Pickler.save_global(self, obj, name=name) except Exception: if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): return self.save_dynamic_class(obj) raise dispatch[type] = save_global dispatch[types.ClassType] = save_global def save_instancemethod(self, obj): # Memoization rarely is ever useful due to python bounding if obj.__self__ is None: self.save_reduce(getattr, (obj.im_class, obj.__name__)) else: if PY3: self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) else: self.save_reduce( types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__), obj=obj) dispatch[types.MethodType] = save_instancemethod def save_inst(self, obj): """Inner logic to save instance. Based off pickle.save_inst""" cls = obj.__class__ # Try the dispatch table (pickle module doesn't do it) f = self.dispatch.get(cls) if f: f(self, obj) # Call unbound method with explicit self return memo = self.memo write = self.write save = self.save if hasattr(obj, '__getinitargs__'): args = obj.__getinitargs__() len(args) # XXX Assert it's a sequence pickle._keep_alive(args, memo) else: args = () write(pickle.MARK) if self.bin: save(cls) for arg in args: save(arg) write(pickle.OBJ) else: for arg in args: save(arg) write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n') self.memoize(obj) try: getstate = obj.__getstate__ except AttributeError: stuff = obj.__dict__ else: stuff = getstate() pickle._keep_alive(stuff, memo) save(stuff) write(pickle.BUILD) if not PY3: dispatch[types.InstanceType] = save_inst def save_property(self, obj): # properties not correctly saved in python self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) dispatch[property] = save_property def save_classmethod(self, obj): orig_func = obj.__func__ self.save_reduce(type(obj), (orig_func, ), obj=obj) dispatch[classmethod] = save_classmethod dispatch[staticmethod] = save_classmethod def save_itemgetter(self, obj): """itemgetter serializer (needed for namedtuple support)""" class Dummy: def __getitem__(self, item): return item items = obj(Dummy()) if not isinstance(items, tuple): items = (items, ) return self.save_reduce(operator.itemgetter, items) if type(operator.itemgetter) is type: dispatch[operator.itemgetter] = save_itemgetter def save_attrgetter(self, obj): """attrgetter serializer""" class Dummy(object): def __init__(self, attrs, index=None): self.attrs = attrs self.index = index def __getattribute__(self, item): attrs = object.__getattribute__(self, "attrs") index = object.__getattribute__(self, "index") if index is None: index = len(attrs) attrs.append(item) else: attrs[index] = ".".join([attrs[index], item]) return type(self)(attrs, index) attrs = [] obj(Dummy(attrs)) return self.save_reduce(operator.attrgetter, tuple(attrs)) if type(operator.attrgetter) is type: dispatch[operator.attrgetter] = save_attrgetter def save_file(self, obj): """Save a file""" try: import StringIO as pystringIO # we can't use cStringIO as it lacks the name attribute except ImportError: import io as pystringIO if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): raise pickle.PicklingError( "Cannot pickle files that do not map to an actual file") if obj is sys.stdout: return self.save_reduce(getattr, (sys, 'stdout'), obj=obj) if obj is sys.stderr: return self.save_reduce(getattr, (sys, 'stderr'), obj=obj) if obj is sys.stdin: raise pickle.PicklingError("Cannot pickle standard input") if obj.closed: raise pickle.PicklingError("Cannot pickle closed files") if hasattr(obj, 'isatty') and obj.isatty(): raise pickle.PicklingError( "Cannot pickle files that map to tty objects") if 'r' not in obj.mode and '+' not in obj.mode: raise pickle.PicklingError( "Cannot pickle files that are not opened for reading: %s" % obj.mode) name = obj.name retval = pystringIO.StringIO() try: # Read the whole file curloc = obj.tell() obj.seek(0) contents = obj.read() obj.seek(curloc) except IOError: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be read" % name) retval.write(contents) retval.seek(curloc) retval.name = name self.save(retval) self.memoize(obj) def save_ellipsis(self, obj): self.save_reduce(_gen_ellipsis, ()) def save_not_implemented(self, obj): self.save_reduce(_gen_not_implemented, ()) if PY3: dispatch[io.TextIOWrapper] = save_file else: dispatch[file] = save_file dispatch[type(Ellipsis)] = save_ellipsis dispatch[type(NotImplemented)] = save_not_implemented def save_weakset(self, obj): self.save_reduce(weakref.WeakSet, (list(obj), )) dispatch[weakref.WeakSet] = save_weakset def save_logger(self, obj): self.save_reduce(logging.getLogger, (obj.name, ), obj=obj) dispatch[logging.Logger] = save_logger """Special functions for Add-on libraries""" def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass
struct timespec st_mtim; ...; }; int fstat(int fd, struct stat *buf); int futimens(int fd, const struct timespec times[2]); ''') lib = ffi.verify(''' #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> ''', ext_package='bedup') _stat_ownership = weakref.WeakKeyDictionary() def fstat_ns(fd): stat = ffi.new('struct stat *') if lib.fstat(fd, stat) != 0: raise IOError(ffi.errno, os.strerror(ffi.errno), fd) # The nested structs seem to be recreated at every member access. atime, mtime = stat.st_atim, stat.st_mtim assert 0 <= atime.tv_nsec < 1e9 assert 0 <= mtime.tv_nsec < 1e9 _stat_ownership[atime] = _stat_ownership[mtime] = stat return atime, mtime def futimens(fd, ns):
class CloudPickler(Pickler): dispatch = Pickler.dispatch.copy() def __init__(self, file, protocol=None): Pickler.__init__(self, file, protocol) # set of modules to unpickle self.modules = set() # map ids to dictionary. used to ensure that functions can share global env self.globals_ref = {} def dump(self, obj): self.inject_addons() try: return Pickler.dump(self, obj) except RuntimeError as e: if 'recursion' in e.args[0]: msg = """Could not pickle object as excessively deep recursion required.""" raise pickle.PicklingError(msg) except pickle.PickleError: raise except Exception as e: if "'i' format requires" in e.message: msg = "Object too large to serialize: " + e.message else: msg = "Could not serialize object: " + e.__class__.__name__ + ": " + e.message print_exec(sys.stderr) raise pickle.PicklingError(msg) def save_memoryview(self, obj): """Fallback to save_string""" Pickler.save_string(self, str(obj)) def save_buffer(self, obj): """Fallback to save_string""" Pickler.save_string(self, str(obj)) if PY3: dispatch[memoryview] = save_memoryview else: dispatch[buffer] = save_buffer def save_unsupported(self, obj): raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj)) dispatch[types.GeneratorType] = save_unsupported # itertools objects do not pickle! for v in itertools.__dict__.values(): if type(v) is type: dispatch[v] = save_unsupported def save_module(self, obj): """ Save a module as an import """ self.modules.add(obj) self.save_reduce(subimport, (obj.__name__, ), obj=obj) dispatch[types.ModuleType] = save_module def save_codeobject(self, obj): """ Save a code object """ if PY3: args = (obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars) else: args = (obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars) self.save_reduce(types.CodeType, args, obj=obj) dispatch[types.CodeType] = save_codeobject def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None if themodule: self.modules.add(themodule) if getattr(themodule, name, None) is obj: return self.save_global(obj, name) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if islambda( obj ) or obj.__code__.co_filename == '<stdin>' or themodule is None: #print("save global", islambda(obj), obj.__code__.co_filename, modname, themodule) self.save_function_tuple(obj) return else: # func is nested klass = getattr(themodule, name, None) if klass is None or klass is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) dispatch[types.FunctionType] = save_function def save_function_tuple(self, func): """ Pickles an actual func object. A func comprises: code, globals, defaults, closure, and dict. We extract and save these, injecting reducing functions at certain points to recreate the func object. Keep in mind that some of these pieces can contain a ref to the func itself. Thus, a naive save on these pieces could trigger an infinite loop of save's. To get around that, we first create a skeleton func object using just the code (this is safe, since this won't contain a ref to the func), and memoize it as soon as it's created. The other stuff can then be filled in later. """ save = self.save write = self.write code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data( func) save(_fill_function) # skeleton function updater write(pickle.MARK) # beginning of tuple that _fill_function expects # create a skeleton function object and memoize it save(_make_skel_func) save((code, closure, base_globals)) write(pickle.REDUCE) self.memoize(func) # save the rest of the func data needed by _fill_function save(f_globals) save(defaults) save(dct) save(func.__module__) write(pickle.TUPLE) write(pickle.REDUCE) # applies _fill_function on the tuple _extract_code_globals_cache = ( weakref.WeakKeyDictionary() if sys.version_info >= (2, 7) and not hasattr(sys, "pypy_version_info") else {}) @classmethod def extract_code_globals(cls, co): """ Find all globals names read or written to by codeblock co """ out_names = cls._extract_code_globals_cache.get(co) if out_names is None: try: names = co.co_names except AttributeError: # PyPy "builtin-code" object out_names = set() else: out_names = set(names[oparg] for op, oparg in _walk_global_ops(co)) # see if nested function have any global refs if co.co_consts: for const in co.co_consts: if type(const) is types.CodeType: out_names |= cls.extract_code_globals(const) cls._extract_code_globals_cache[co] = out_names return out_names def extract_func_data(self, func): """ Turn the function into a tuple of data necessary to recreate it: code, globals, defaults, closure, dict """ code = func.__code__ # extract all global ref's func_global_refs = self.extract_code_globals(code) # process all variables referenced by global environment f_globals = {} for var in func_global_refs: if var in func.__globals__: f_globals[var] = func.__globals__[var] # defaults requires no processing defaults = func.__defaults__ # process closure closure = [c.cell_contents for c in func.__closure__] if func.__closure__ else [] # save the dict dct = func.__dict__ base_globals = self.globals_ref.get(id(func.__globals__), {}) self.globals_ref[id(func.__globals__)] = base_globals return (code, f_globals, defaults, closure, dct, base_globals) def save_builtin_function(self, obj): if obj.__module__ is "__builtin__": return self.save_global(obj) return self.save_function(obj) dispatch[types.BuiltinFunctionType] = save_builtin_function def save_global(self, obj, name=None, pack=struct.pack): if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj], ), obj=obj) if name is None: name = obj.__name__ modname = getattr(obj, "__module__", None) if modname is None: try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = '__main__' if modname == '__main__': themodule = None else: __import__(modname) themodule = sys.modules[modname] self.modules.add(themodule) if hasattr(themodule, name) and getattr(themodule, name) is obj: return Pickler.save_global(self, obj, name) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): d = dict(obj.__dict__) # copy dict proxy to a dict if not isinstance(d.get('__dict__', None), property): # don't extract dict that are properties d.pop('__dict__', None) d.pop('__weakref__', None) # hack as __new__ is stored differently in the __dict__ new_override = d.get('__new__', None) if new_override: d['__new__'] = obj.__new__ # workaround for namedtuple (hijacked by PySpark) if getattr(obj, '_is_namedtuple_', False): self.save_reduce(_load_namedtuple, (obj.__name__, obj._fields)) return self.save(_load_class) self.save_reduce(typ, (obj.__name__, obj.__bases__, { "__doc__": obj.__doc__ }), obj=obj) d.pop('__doc__', None) # handle property and staticmethod dd = {} for k, v in d.items(): if isinstance(v, property): k = ('property', k) v = (v.fget, v.fset, v.fdel, v.__doc__) elif isinstance(v, staticmethod) and hasattr(v, '__func__'): k = ('staticmethod', k) v = v.__func__ elif isinstance(v, classmethod) and hasattr(v, '__func__'): k = ('classmethod', k) v = v.__func__ dd[k] = v self.save(dd) self.write(pickle.TUPLE2) self.write(pickle.REDUCE) else: raise pickle.PicklingError("Can't pickle %r" % obj) dispatch[type] = save_global dispatch[types.ClassType] = save_global def save_instancemethod(self, obj): # Memoization rarely is ever useful due to python bounding if PY3: self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) else: self.save_reduce( types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__), obj=obj) dispatch[types.MethodType] = save_instancemethod def save_inst(self, obj): """Inner logic to save instance. Based off pickle.save_inst Supports __transient__""" cls = obj.__class__ memo = self.memo write = self.write save = self.save if hasattr(obj, '__getinitargs__'): args = obj.__getinitargs__() len(args) # XXX Assert it's a sequence pickle._keep_alive(args, memo) else: args = () write(pickle.MARK) if self.bin: save(cls) for arg in args: save(arg) write(pickle.OBJ) else: for arg in args: save(arg) write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n') self.memoize(obj) try: getstate = obj.__getstate__ except AttributeError: stuff = obj.__dict__ #remove items if transient if hasattr(obj, '__transient__'): transient = obj.__transient__ stuff = stuff.copy() for k in list(stuff.keys()): if k in transient: del stuff[k] else: stuff = getstate() pickle._keep_alive(stuff, memo) save(stuff) write(pickle.BUILD) if not PY3: dispatch[types.InstanceType] = save_inst def save_property(self, obj): # properties not correctly saved in python self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) dispatch[property] = save_property def save_itemgetter(self, obj): """itemgetter serializer (needed for namedtuple support)""" class Dummy: def __getitem__(self, item): return item items = obj(Dummy()) if not isinstance(items, tuple): items = (items, ) return self.save_reduce(operator.itemgetter, items) if type(operator.itemgetter) is type: dispatch[operator.itemgetter] = save_itemgetter def save_attrgetter(self, obj): """attrgetter serializer""" class Dummy(object): def __init__(self, attrs, index=None): self.attrs = attrs self.index = index def __getattribute__(self, item): attrs = object.__getattribute__(self, "attrs") index = object.__getattribute__(self, "index") if index is None: index = len(attrs) attrs.append(item) else: attrs[index] = ".".join([attrs[index], item]) return type(self)(attrs, index) attrs = [] obj(Dummy(attrs)) return self.save_reduce(operator.attrgetter, tuple(attrs)) if type(operator.attrgetter) is type: dispatch[operator.attrgetter] = save_attrgetter def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, obj=None): """Modified to support __transient__ on new objects Change only affects protocol level 2 (which is always used by PiCloud""" # Assert that args is a tuple or None if not isinstance(args, tuple): raise pickle.PicklingError("args from reduce() should be a tuple") # Assert that func is callable if not hasattr(func, '__call__'): raise pickle.PicklingError("func from reduce should be callable") save = self.save write = self.write # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__": #Added fix to allow transient cls = args[0] if not hasattr(cls, "__new__"): raise pickle.PicklingError( "args[0] from __newobj__ args has no __new__") if obj is not None and cls is not obj.__class__: raise pickle.PicklingError( "args[0] from __newobj__ args has the wrong class") args = args[1:] save(cls) #Don't pickle transient entries if hasattr(obj, '__transient__'): transient = obj.__transient__ state = state.copy() for k in list(state.keys()): if k in transient: del state[k] save(args) write(pickle.NEWOBJ) else: save(func) save(args) write(pickle.REDUCE) if obj is not None: self.memoize(obj) # More new special cases (that work with older protocols as # well): when __reduce__ returns a tuple with 4 or 5 items, # the 4th and 5th item should be iterators that provide list # items and dict items (as (key, value) tuples), or None. if listitems is not None: self._batch_appends(listitems) if dictitems is not None: self._batch_setitems(dictitems) if state is not None: save(state) write(pickle.BUILD) def save_partial(self, obj): """Partial objects do not serialize correctly in python2.x -- this fixes the bugs""" self.save_reduce(_genpartial, (obj.func, obj.args, obj.keywords)) if sys.version_info < (2, 7): # 2.7 supports partial pickling dispatch[partial] = save_partial def save_file(self, obj): """Save a file""" try: import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute except ImportError: import io as pystringIO if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): raise pickle.PicklingError( "Cannot pickle files that do not map to an actual file") if obj is sys.stdout: return self.save_reduce(getattr, (sys, 'stdout'), obj=obj) if obj is sys.stderr: return self.save_reduce(getattr, (sys, 'stderr'), obj=obj) if obj is sys.stdin: raise pickle.PicklingError("Cannot pickle standard input") if hasattr(obj, 'isatty') and obj.isatty(): raise pickle.PicklingError( "Cannot pickle files that map to tty objects") if 'r' not in obj.mode: raise pickle.PicklingError( "Cannot pickle files that are not opened for reading") name = obj.name try: fsize = os.stat(name).st_size except OSError: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be stat" % name) if obj.closed: #create an empty closed string io retval = pystringIO.StringIO("") retval.close() elif not fsize: #empty file retval = pystringIO.StringIO("") try: tmpfile = file(name) tst = tmpfile.read(1) except IOError: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be read" % name) tmpfile.close() if tst != '': raise pickle.PicklingError( "Cannot pickle file %s as it does not appear to map to a physical, real file" % name) else: try: tmpfile = file(name) contents = tmpfile.read() tmpfile.close() except IOError: raise pickle.PicklingError( "Cannot pickle file %s as it cannot be read" % name) retval = pystringIO.StringIO(contents) curloc = obj.tell() retval.seek(curloc) retval.name = name self.save(retval) self.memoize(obj) if PY3: dispatch[io.TextIOWrapper] = save_file else: dispatch[file] = save_file """Special functions for Add-on libraries""" def inject_numpy(self): numpy = sys.modules.get('numpy') if not numpy or not hasattr(numpy, 'ufunc'): return self.dispatch[numpy.ufunc] = self.__class__.save_ufunc def save_ufunc(self, obj): """Hack function for saving numpy ufunc objects""" name = obj.__name__ numpy_tst_mods = ['numpy', 'scipy.special'] for tst_mod_name in numpy_tst_mods: tst_mod = sys.modules.get(tst_mod_name, None) if tst_mod and name in tst_mod.__dict__: return self.save_reduce(_getobject, (tst_mod_name, name)) raise pickle.PicklingError( 'cannot save %s. Cannot resolve what module it is defined in' % str(obj)) def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" self.inject_numpy()
def __init__(self, container_strategy, tpu_cluster_resolver=None, steps_per_run=None, device_assignment=None): super(TPUExtended, self).__init__(container_strategy) if tpu_cluster_resolver is None: tpu_cluster_resolver = TPUClusterResolver("") if steps_per_run is None: # TODO(frankchn): Warn when we are being used by DS/Keras and this is # not specified. steps_per_run = 1 self._tpu_function_cache = weakref.WeakKeyDictionary() self._tpu_cluster_resolver = tpu_cluster_resolver self._tpu_metadata = get_tpu_system_metadata( self._tpu_cluster_resolver) self._device_assignment = device_assignment # Device assignment is currently only supported for 1 core case. if self._device_assignment: assert isinstance(self._device_assignment, device_assignment_lib.DeviceAssignment) if self._device_assignment.num_replicas != 1: raise ValueError( "Device assignment is only supported for a single " "core single replica case currently.") if self._device_assignment.num_cores_per_replica != 1: raise ValueError( "Device assignment is only supported for a single " "core single replica case currently.") if not all(self._device_assignment.core_assignment[0][0] == [0, 0, 0]): raise ValueError( "Device assignment is only supported for a single " "core single replica case currently.") # TODO(jhseu): Switch to DeviceAssignment to support pods and model # parallelism. self._tpu_devices = [ d.name for d in self._tpu_metadata.devices if "device:TPU:" in d.name ] self._host_device = device_util.get_host_for_device( self._tpu_devices[0]) # Only create variables for the number of replicas we're running. self._tpu_devices = self._tpu_devices[:self._num_replicas_in_sync] self._device_map = values.ReplicaDeviceMap(self._tpu_devices) # Preload the data onto the TPUs. input_worker_devices = collections.OrderedDict() for tpu_device in self._tpu_devices: host_device = device_util.get_host_for_device(tpu_device) input_worker_devices.setdefault(host_device, []) input_worker_devices[host_device].append(tpu_device) self._input_workers = input_lib.InputWorkers( self._device_map, tuple(input_worker_devices.items())) # TODO(sourabhbajaj): Remove this once performance of running one step # at a time is comparable to multiple steps. self.steps_per_run = steps_per_run self._require_static_shapes = True self.experimental_enable_get_next_as_optional = True
#coding:utf-8 import logging import time from scrapy.dupefilters import BaseDupeFilter from scrapy.utils.request import request_fingerprint from scrapy.utils.python import to_bytes from . import defaults from .connection import get_redis_from_settings from .BloomFilter import BloomFilter from scrapy.utils.httpobj import urlparse_cached from urlparse import urlparse, parse_qs from hashlib import sha1 import weakref _fingerprint_cache = weakref.WeakKeyDictionary() logger = logging.getLogger(__name__) # TODO: Rename class to RedisDupeFilter. class RFPDupeFilter(BaseDupeFilter): """Redis-based request duplicates filter. This class can also be used with default Scrapy's scheduler. """ logger = logger def __init__(self, server, key, debug=False): """Initialize the duplicates filter.
def __init__(self): self.weak_key_dict = weakref.WeakKeyDictionary()
def __init__(self, constraint): self.constraint = constraint self.data = weakref.WeakKeyDictionary()
def _object_cache(self): try: return self._tls.object_cache except AttributeError: self._tls.object_cache = weakref.WeakKeyDictionary() return self._tls.object_cache
def __init__(self, parent=None, name=None, labels=None, title=None, viewBox=None, axisItems=None, enableMenu=True, **kargs): """ Create a new PlotItem. All arguments are optional. Any extra keyword arguments are passed to :func:`PlotItem.plot() <pyqtgraph.PlotItem.plot>`. ============== ========================================================================================== **Arguments:** *title* Title to display at the top of the item. Html is allowed. *labels* A dictionary specifying the axis labels to display:: {'left': (args), 'bottom': (args), ...} The name of each axis and the corresponding arguments are passed to :func:`PlotItem.setLabel() <pyqtgraph.PlotItem.setLabel>` Optionally, PlotItem my also be initialized with the keyword arguments left, right, top, or bottom to achieve the same effect. *name* Registers a name for this view so that others may link to it *viewBox* If specified, the PlotItem will be constructed with this as its ViewBox. *axisItems* Optional dictionary instructing the PlotItem to use pre-constructed items for its axes. The dict keys must be axis names ('left', 'bottom', 'right', 'top') and the values must be instances of AxisItem (or at least compatible with AxisItem). ============== ========================================================================================== """ GraphicsWidget.__init__(self, parent) self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding) ## Set up control buttons path = os.path.dirname(__file__) self.autoBtn = ButtonItem(pixmaps.getPixmap('auto'), 14, self) self.autoBtn.mode = 'auto' self.autoBtn.clicked.connect(self.autoBtnClicked) self.buttonsHidden = False ## whether the user has requested buttons to be hidden self.mouseHovering = False self.layout = QtGui.QGraphicsGridLayout() self.layout.setContentsMargins(1, 1, 1, 1) self.setLayout(self.layout) self.layout.setHorizontalSpacing(0) self.layout.setVerticalSpacing(0) if viewBox is None: viewBox = ViewBox(parent=self) self.vb = viewBox self.vb.sigStateChanged.connect(self.viewStateChanged) self.setMenuEnabled( enableMenu, enableMenu) ## en/disable plotitem and viewbox menus if name is not None: self.vb.register(name) self.vb.sigRangeChanged.connect(self.sigRangeChanged) self.vb.sigXRangeChanged.connect(self.sigXRangeChanged) self.vb.sigYRangeChanged.connect(self.sigYRangeChanged) self.layout.addItem(self.vb, 2, 1) self.alpha = 1.0 self.autoAlpha = True self.spectrumMode = False self.legend = None # Initialize axis items self.axes = {} self.setAxisItems(axisItems) self.titleLabel = LabelItem('', size='11pt', parent=self) self.layout.addItem(self.titleLabel, 0, 1) self.setTitle(None) ## hide for i in range(4): self.layout.setRowPreferredHeight(i, 0) self.layout.setRowMinimumHeight(i, 0) self.layout.setRowSpacing(i, 0) self.layout.setRowStretchFactor(i, 1) for i in range(3): self.layout.setColumnPreferredWidth(i, 0) self.layout.setColumnMinimumWidth(i, 0) self.layout.setColumnSpacing(i, 0) self.layout.setColumnStretchFactor(i, 1) self.layout.setRowStretchFactor(2, 100) self.layout.setColumnStretchFactor(1, 100) self.items = [] self.curves = [] self.itemMeta = weakref.WeakKeyDictionary() self.dataItems = [] self.paramList = {} self.avgCurves = {} ### Set up context menu w = QtGui.QWidget() self.ctrl = c = Ui_Form() c.setupUi(w) dv = QtGui.QDoubleValidator(self) menuItems = [ ('Transforms', c.transformGroup), ('Downsample', c.decimateGroup), ('Average', c.averageGroup), ('Alpha', c.alphaGroup), ('Grid', c.gridGroup), ('Points', c.pointsGroup), ] self.ctrlMenu = QtGui.QMenu() self.ctrlMenu.setTitle('Plot Options') self.subMenus = [] for name, grp in menuItems: sm = QtGui.QMenu(name) act = QtGui.QWidgetAction(self) act.setDefaultWidget(grp) sm.addAction(act) self.subMenus.append(sm) self.ctrlMenu.addMenu(sm) self.stateGroup = WidgetGroup() for name, w in menuItems: self.stateGroup.autoAdd(w) self.fileDialog = None c.alphaGroup.toggled.connect(self.updateAlpha) c.alphaSlider.valueChanged.connect(self.updateAlpha) c.autoAlphaCheck.toggled.connect(self.updateAlpha) c.xGridCheck.toggled.connect(self.updateGrid) c.yGridCheck.toggled.connect(self.updateGrid) c.gridAlphaSlider.valueChanged.connect(self.updateGrid) c.fftCheck.toggled.connect(self.updateSpectrumMode) c.logXCheck.toggled.connect(self.updateLogMode) c.logYCheck.toggled.connect(self.updateLogMode) c.derivativeCheck.toggled.connect(self.updateDerivativeMode) c.phasemapCheck.toggled.connect(self.updatePhasemapMode) c.downsampleSpin.valueChanged.connect(self.updateDownsampling) c.downsampleCheck.toggled.connect(self.updateDownsampling) c.autoDownsampleCheck.toggled.connect(self.updateDownsampling) c.subsampleRadio.toggled.connect(self.updateDownsampling) c.meanRadio.toggled.connect(self.updateDownsampling) c.clipToViewCheck.toggled.connect(self.updateDownsampling) self.ctrl.avgParamList.itemClicked.connect(self.avgParamListClicked) self.ctrl.averageGroup.toggled.connect(self.avgToggled) self.ctrl.maxTracesCheck.toggled.connect(self.updateDecimation) self.ctrl.maxTracesSpin.valueChanged.connect(self.updateDecimation) if labels is None: labels = {} for label in list(self.axes.keys()): if label in kargs: labels[label] = kargs[label] del kargs[label] for k in labels: if isinstance(labels[k], basestring): labels[k] = (labels[k], ) self.setLabel(k, *labels[k]) if title is not None: self.setTitle(title) if len(kargs) > 0: self.plot(**kargs)
def __init__(self): self.proxy_refs = weakref.WeakKeyDictionary() self.testing_engines = collections.defaultdict(set) self.dbapi_connections = set()
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # See http://www.gnu.org/licenses/ for more information. """ Manages locking access (across threads) to Poppler.Document instances. """ import threading import weakref _locks = weakref.WeakKeyDictionary() _lock = threading.RLock() def lock(document): """Returns a threading.RLock instance for the given Poppler.Document. Use: with lock(document): do_something """ with _lock: try: return _locks[document]
"""Find/replace widget.""" import re import tkinter as tk import weakref import porcupine from porcupine import utils, tabs find_widgets = weakref.WeakKeyDictionary() class Finder(tk.Frame): """A widget for finding and replacing text. Use the pack geometry manager with this widget. """ def __init__(self, parent, textwidget, **kwargs): super().__init__(parent, **kwargs) self._last_pattern = None self._matches = None self.grid_columnconfigure(1, weight=1) self._textwidget = textwidget entrygrid = tk.Frame(self) entrygrid.grid(row=0, column=0) self._find_entry = self._add_entry(entrygrid, 0, "Find:", self.find) self._replace_entry = self._add_entry(entrygrid, 1, "Replace with:") buttonframe = tk.Frame(self)
def __init__(self, http_port=None, distrib_port=None, allowForce=None, public_html="public_html", site=None, numbuilds=20, num_events=200, num_events_max=None, auth=None, order_console_by_time=False, changecommentlink=None, revlink=None, projects=None, repositories=None, authz=None, logRotateLength=None, maxRotatedFiles=None, change_hook_dialects={}, provide_feeds=None): """Run a web server that provides Buildbot status. @type http_port: int or L{twisted.application.strports} string @param http_port: a strports specification describing which port the buildbot should use for its web server, with the Waterfall display as the root page. For backwards compatibility this can also be an int. Use 'tcp:8000' to listen on that port, or 'tcp:12345:interface=127.0.0.1' if you only want local processes to connect to it (perhaps because you are using an HTTP reverse proxy to make the buildbot available to the outside world, and do not want to make the raw port visible). @type distrib_port: int or L{twisted.application.strports} string @param distrib_port: Use this if you want to publish the Waterfall page using web.distrib instead. The most common case is to provide a string that is an absolute pathname to the unix socket on which the publisher should listen (C{os.path.expanduser(~/.twistd-web-pb)} will match the default settings of a standard twisted.web 'personal web server'). Another possibility is to pass an integer, which means the publisher should listen on a TCP socket, allowing the web server to be on a different machine entirely. Both forms are provided for backwards compatibility; the preferred form is a strports specification like 'unix:/home/buildbot/.twistd-web-pb'. Providing a non-absolute pathname will probably confuse the strports parser. @param allowForce: deprecated; use authz instead @param auth: deprecated; use with authz @param authz: a buildbot.status.web.authz.Authz instance giving the authorization parameters for this view @param public_html: the path to the public_html directory for this display, either absolute or relative to the basedir. The default is 'public_html', which selects BASEDIR/public_html. @type site: None or L{twisted.web.server.Site} @param site: Use this if you want to define your own object instead of using the default.` @type numbuilds: int @param numbuilds: Default number of entries in lists at the /one_line_per_build and /builders/FOO URLs. This default can be overriden both programatically --- by passing the equally named argument to constructors of OneLinePerBuildOneBuilder and OneLinePerBuild --- and via the UI, by tacking ?numbuilds=xy onto the URL. @type num_events: int @param num_events: Default number of events to show in the waterfall. @type num_events_max: int @param num_events_max: The maximum number of events that are allowed to be shown in the waterfall. The default value of C{None} will disable this check @type auth: a L{status.web.auth.IAuth} or C{None} @param auth: an object that performs authentication to restrict access to the C{allowForce} features. Ignored if C{allowForce} is not C{True}. If C{auth} is C{None}, people can force or stop builds without auth. @type order_console_by_time: bool @param order_console_by_time: Whether to order changes (commits) in the console view according to the time they were created (for VCS like Git) or according to their integer revision numbers (for VCS like SVN). @type changecommentlink: callable, dict, tuple (2 or 3 strings) or C{None} @param changecommentlink: adds links to ticket/bug ids in change comments, see buildbot.status.web.base.changecommentlink for details @type revlink: callable, dict, string or C{None} @param revlink: decorations revision ids with links to a web-view, see buildbot.status.web.base.revlink for details @type projects: callable, dict or c{None} @param projects: maps project identifiers to URLs, so that any project listed is automatically decorated with a link to it's front page. see buildbot.status.web.base.dictlink for details @type repositories: callable, dict or c{None} @param repositories: maps repository identifiers to URLs, so that any project listed is automatically decorated with a link to it's web view. see buildbot.status.web.base.dictlink for details @type logRotateLength: None or int @param logRotateLength: file size at which the http.log is rotated/reset. If not set, the value set in the buildbot.tac will be used, falling back to the BuildMaster's default value (1 Mb). @type maxRotatedFiles: None or int @param maxRotatedFiles: number of old http.log files to keep during log rotation. If not set, the value set in the buildbot.tac will be used, falling back to the BuildMaster's default value (10 files). @type change_hook_dialects: None or dict @param change_hook_dialects: If empty, disables change_hook support, otherwise whitelists valid dialects. In the format of {"dialect1": "Option1", "dialect2", None} Where the values are options that will be passed to the dialect To enable the DEFAULT handler, use a key of DEFAULT @type provide_feeds: None or list @param provide_feeds: If empty, provides atom, json, and rss feeds. Otherwise, a dictionary of strings of the type of feeds provided. Current possibilities are "atom", "json", and "rss" """ service.MultiService.__init__(self) if type(http_port) is int: http_port = "tcp:%d" % http_port self.http_port = http_port if distrib_port is not None: if type(distrib_port) is int: distrib_port = "tcp:%d" % distrib_port if distrib_port[0] in "/~.": # pathnames distrib_port = "unix:%s" % distrib_port self.distrib_port = distrib_port self.num_events = num_events if num_events_max: assert num_events_max >= num_events self.num_events_max = num_events_max self.public_html = public_html # make up an authz if allowForce was given if authz: if allowForce is not None: raise ValueError( "cannot use both allowForce and authz parameters") if auth: raise ValueError( "cannot use both auth and authz parameters (pass " "auth as an Authz parameter)") else: # invent an authz if allowForce and auth: authz = Authz(auth=auth, default_action="auth") elif allowForce: authz = Authz(default_action=True) else: if auth: log.msg( "Warning: Ignoring authentication. Search for 'authorization'" " in the manual") authz = Authz() # no authorization for anything self.authz = authz self.orderConsoleByTime = order_console_by_time # If we were given a site object, go ahead and use it. (if not, we add one later) self.site = site # store the log settings until we create the site object self.logRotateLength = logRotateLength self.maxRotatedFiles = maxRotatedFiles # create the web site page structure self.childrenToBeAdded = {} self.setupUsualPages(numbuilds=numbuilds, num_events=num_events, num_events_max=num_events_max) # Set up the jinja templating engine. self.templates = createJinjaEnv(revlink, changecommentlink, repositories, projects) # keep track of cached connections so we can break them when we shut # down. See ticket #102 for more details. self.channels = weakref.WeakKeyDictionary() # do we want to allow change_hook self.change_hook_dialects = {} if change_hook_dialects: self.change_hook_dialects = change_hook_dialects self.putChild( "change_hook", ChangeHookResource(dialects=self.change_hook_dialects)) # Set default feeds if provide_feeds is None: self.provide_feeds = ["atom", "json", "rss"] else: self.provide_feeds = provide_feeds
if sys.version_info[:2] > (3, 7): from typing import Final else: from typing_extensions import Final LockType: Type try: import _thread LockType = _thread.LockType except ImportError: import _dummy_thread LockType = _dummy_thread.LockType # Wrapper functions that can call either of 2 functions depending on a boolean # argument boolean_dispatched: 'weakref.WeakKeyDictionary[Callable, Dict[str, Callable]]' = weakref.WeakKeyDictionary( ) # noqa: T484 def createResolutionCallbackFromEnv(lookup_base): """ Creates a resolution callback that will look up qualified names in an environment, starting with `lookup_base` for the base of any qualified names, then proceeding down the lookup chain with the resolved object. You should not use this directly, it should only be used from the other createResolutionCallbackFrom* functions. """ def lookupInModule(qualified_name, module): if '.' in qualified_name: parts = qualified_name.split('.') base = parts[0]
@convert.register(ufl.WithMapping) def convert_withmapping(element, **kwargs): return _create_element(element.wrapee, **kwargs) @convert.register(ufl.RestrictedElement) def convert_restrictedelement(element, **kwargs): finat_elem, deps = _create_element(element._element, **kwargs) return finat.RestrictedElement(finat_elem, element.restriction_domain()), deps hexahedron_tpc = ufl.TensorProductCell(ufl.quadrilateral, ufl.interval) quadrilateral_tpc = ufl.TensorProductCell(ufl.interval, ufl.interval) _cache = weakref.WeakKeyDictionary() def create_element(ufl_element, shape_innermost=True, shift_axes=0, restriction=None): """Create a FInAT element (suitable for tabulating with) given a UFL element. :arg ufl_element: The UFL element to create a FInAT element from. :arg shape_innermost: Vector/tensor indices come after basis function indices :arg restriction: cell restriction in interior facet integrals (only for runtime tabulated elements) """ finat_element, deps = _create_element(ufl_element, shape_innermost=shape_innermost,
def create_metadata_env(): return weakref.WeakKeyDictionary()
if not to_tile.is_coarse(): return to_tile dispatched = self.dispatch(to_tile.op) self._assign_to([d.data for d in dispatched], to_tile.op.outputs) return to_tile @classmethod def tiles(cls, to_tile): to_tile.build_graph(tiled=True, compose=False) return get_tiled(to_tile) handler = OperandTilesHandler() register = OperandTilesHandler.register _tileable_data_to_tiled = weakref.WeakKeyDictionary() _op_to_copied = weakref.WeakKeyDictionary() @enter_mode(build=True) def get_tiled(tileable, mapping=None, raise_err_if_not_tiled=True): tileable_data = tileable.data if hasattr(tileable, 'data') else tileable if mapping: tileable_data = mapping.get(tileable_data, tileable_data) if raise_err_if_not_tiled: return _tileable_data_to_tiled[tileable_data] else: return _tileable_data_to_tiled.get(tileable_data) class ChunkGraphBuilder(GraphBuilder):
__author__ = 'Brian Quinlan ([email protected])' import os from concurrent.futures import _base import queue import multiprocessing as mp import multiprocessing.connection from multiprocessing.queues import Queue import threading import weakref from functools import partial import itertools import sys import traceback _threads_wakeups = weakref.WeakKeyDictionary() _global_shutdown = False class _ThreadWakeup: def __init__(self): self._closed = False self._reader, self._writer = mp.Pipe(duplex=False) def close(self): if not self._closed: self._closed = True self._writer.close() self._reader.close() def wakeup(self):
def singledispatch(func): """Single-dispatch generic function decorator. Transforms a function into a generic function, which can have different behaviours depending upon the type of its first argument. The decorated function acts as the default implementation, and additional implementations can be registered using the register() attribute of the generic function. """ # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. import types, weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() cache_token = None def dispatch(cls): """generic_func.dispatch(cls) -> <function implementation> Runs the dispatch algorithm to return the best available implementation for the given *cls* registered on *generic_func*. """ nonlocal cache_token if cache_token is not None: current_token = get_cache_token() if cache_token != current_token: dispatch_cache.clear() cache_token = current_token try: impl = dispatch_cache[cls] except KeyError: try: impl = registry[cls] except KeyError: impl = _find_impl(cls, registry) dispatch_cache[cls] = impl return impl def _is_union_type(cls): from typing import get_origin, Union return get_origin(cls) in {Union, types.UnionType} def _is_valid_dispatch_type(cls): if isinstance(cls, type) and not isinstance(cls, GenericAlias): return True from typing import get_args return (_is_union_type(cls) and all( isinstance(arg, type) and not isinstance(arg, GenericAlias) for arg in get_args(cls))) def register(cls, func=None): """generic_func.register(cls, func) -> func Registers a new implementation for the given *cls* on a *generic_func*. """ nonlocal cache_token if _is_valid_dispatch_type(cls): if func is None: return lambda f: register(cls, f) else: if func is not None: raise TypeError(f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type.") ann = getattr(cls, '__annotations__', {}) if not ann: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " f"on an annotated function.") func = cls # only import typing if annotation parsing is necessary from typing import get_type_hints argname, cls = next(iter(get_type_hints(func).items())) if not _is_valid_dispatch_type(cls): if _is_union_type(cls): raise TypeError(f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes.") else: raise TypeError(f"Invalid annotation for {argname!r}. " f"{cls!r} is not a class.") if _is_union_type(cls): from typing import get_args for arg in get_args(cls): registry[arg] = func else: registry[cls] = func if cache_token is None and hasattr(cls, '__abstractmethods__'): cache_token = get_cache_token() dispatch_cache.clear() return func def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') registry[object] = func wrapper.register = register wrapper.dispatch = dispatch wrapper.registry = types.MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper
class AsyncAuth(object): ''' Set up an Async object to maintain authentication with the salt master ''' # This class is only a singleton per minion/master pair # mapping of io_loop -> {key -> auth} instance_map = weakref.WeakKeyDictionary() # mapping of key -> creds creds_map = {} def __new__(cls, opts, io_loop=None): ''' Only create one instance of SAuth per __key() ''' # do we have any mapping for this io_loop io_loop = io_loop or tornado.ioloop.IOLoop.current() if io_loop not in AsyncAuth.instance_map: AsyncAuth.instance_map[io_loop] = weakref.WeakValueDictionary() loop_instance_map = AsyncAuth.instance_map[io_loop] key = cls.__key(opts) if key not in loop_instance_map: log.debug('Initializing new SAuth for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller new_auth = object.__new__(cls) new_auth.__singleton_init__(opts, io_loop=io_loop) loop_instance_map[key] = new_auth else: log.debug('Re-using SAuth for {0}'.format(key)) return loop_instance_map[key] @classmethod def __key(cls, opts, io_loop=None): return (opts['pki_dir'], # where the keys are stored opts['id'], # minion ID opts['master_uri'], # master ID ) # has to remain empty for singletons, since __init__ will *always* be called def __init__(self, opts, io_loop=None): pass # an init for the singleton instance to call def __singleton_init__(self, opts, io_loop=None): ''' Init an Auth instance :param dict opts: Options for this server :return: Auth instance :rtype: Auth ''' self.opts = opts self.token = Crypticle.generate_key_string() self.serial = salt.payload.Serial(self.opts) self.pub_path = os.path.join(self.opts['pki_dir'], 'minion.pub') self.rsa_path = os.path.join(self.opts['pki_dir'], 'minion.pem') if 'syndic_master' in self.opts: self.mpub = 'syndic_master.pub' elif 'alert_master' in self.opts: self.mpub = 'monitor_master.pub' else: self.mpub = 'minion_master.pub' if not os.path.isfile(self.pub_path): self.get_keys() self.io_loop = io_loop or tornado.ioloop.IOLoop.current() salt.utils.reinit_crypto() key = self.__key(self.opts) # TODO: if we already have creds for this key, lets just re-use if key in AsyncAuth.creds_map: creds = AsyncAuth.creds_map[key] self._creds = creds self._crypticle = Crypticle(self.opts, creds['aes']) self._authenticate_future = tornado.concurrent.Future() self._authenticate_future.set_result(True) else: self.authenticate() def __deepcopy__(self, memo): cls = self.__class__ result = cls.__new__(cls, copy.deepcopy(self.opts, memo), io_loop=None) memo[id(self)] = result for key in self.__dict__: if key in ('io_loop',): # The io_loop has a thread Lock which will fail to be deep # copied. Skip it because it will just be recreated on the # new copy. continue setattr(result, key, copy.deepcopy(self.__dict__[key], memo)) return result @property def creds(self): return self._creds @property def crypticle(self): return self._crypticle @property def authenticated(self): return hasattr(self, '_authenticate_future') and \ self._authenticate_future.done() and \ self._authenticate_future.exception() is None def invalidate(self): if self.authenticated: del self._authenticate_future key = self.__key(self.opts) if key in AsyncAuth.creds_map: del AsyncAuth.creds_map[key] def authenticate(self, callback=None): ''' Ask for this client to reconnect to the origin This function will de-dupe all calls here and return a *single* future for the sign-in-- whis way callers can all assume there aren't others ''' # if an auth is in flight-- and not done-- just pass that back as the future to wait on if hasattr(self, '_authenticate_future') and not self._authenticate_future.done(): future = self._authenticate_future else: future = tornado.concurrent.Future() self._authenticate_future = future self.io_loop.add_callback(self._authenticate) if callback is not None: def handle_future(future): response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) return future @tornado.gen.coroutine def _authenticate(self): ''' Authenticate with the master, this method breaks the functional paradigm, it will update the master information from a fresh sign in, signing in can occur as often as needed to keep up with the revolving master AES key. :rtype: Crypticle :returns: A crypticle used for encryption operations ''' acceptance_wait_time = self.opts['acceptance_wait_time'] acceptance_wait_time_max = self.opts['acceptance_wait_time_max'] if not acceptance_wait_time_max: acceptance_wait_time_max = acceptance_wait_time creds = None while True: try: creds = yield self.sign_in() except SaltClientError: break if creds == 'retry': if self.opts.get('caller'): print('Minion failed to authenticate with the master, ' 'has the minion key been accepted?') sys.exit(2) if acceptance_wait_time: log.info('Waiting {0} seconds before retry.'.format(acceptance_wait_time)) yield tornado.gen.sleep(acceptance_wait_time) if acceptance_wait_time < acceptance_wait_time_max: acceptance_wait_time += acceptance_wait_time log.debug('Authentication wait time is {0}'.format(acceptance_wait_time)) continue break if not isinstance(creds, dict) or 'aes' not in creds: try: del AsyncAuth.creds_map[self.__key(self.opts)] except KeyError: pass self._authenticate_future.set_exception( SaltClientError('Attempt to authenticate with the salt master failed') ) else: AsyncAuth.creds_map[self.__key(self.opts)] = creds self._creds = creds self._crypticle = Crypticle(self.opts, creds['aes']) self._authenticate_future.set_result(True) # mark the sign-in as complete @tornado.gen.coroutine def sign_in(self, timeout=60, safe=True, tries=1): ''' Send a sign in request to the master, sets the key information and returns a dict containing the master publish interface to bind to and the decrypted aes key for transport decryption. :param int timeout: Number of seconds to wait before timing out the sign-in request :param bool safe: If True, do not raise an exception on timeout. Retry instead. :param int tries: The number of times to try to authenticate before giving up. :raises SaltReqTimeoutError: If the sign-in request has timed out and :param safe: is not set :return: Return a string on failure indicating the reason for failure. On success, return a dictionary with the publication port and the shared AES key. ''' auth = {} auth_timeout = self.opts.get('auth_timeout', None) if auth_timeout is not None: timeout = auth_timeout auth_safemode = self.opts.get('auth_safemode', None) if auth_safemode is not None: safe = auth_safemode auth_tries = self.opts.get('auth_tries', None) if auth_tries is not None: tries = auth_tries m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) auth['master_uri'] = self.opts['master_uri'] channel = salt.transport.client.AsyncReqChannel.factory(self.opts, crypt='clear', io_loop=self.io_loop) try: payload = yield channel.send( self.minion_sign_in_payload(), tries=tries, timeout=timeout ) except SaltReqTimeoutError as e: if safe: log.warning('SaltReqTimeoutError: {0}'.format(e)) raise tornado.gen.Return('retry') raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error') if 'load' in payload: if 'ret' in payload['load']: if not payload['load']['ret']: if self.opts['rejected_retry']: log.error( 'The Salt Master has rejected this minion\'s public ' 'key.\nTo repair this issue, delete the public key ' 'for this minion on the Salt Master.\nThe Salt ' 'Minion will attempt to to re-authenicate.' ) raise tornado.gen.Return('retry') else: log.critical( 'The Salt Master has rejected this minion\'s public ' 'key!\nTo repair this issue, delete the public key ' 'for this minion on the Salt Master and restart this ' 'minion.\nOr restart the Salt Master in open mode to ' 'clean out the keys. The Salt Minion will now exit.' ) sys.exit(salt.defaults.exitcodes.EX_OK) # has the master returned that its maxed out with minions? elif payload['load']['ret'] == 'full': raise tornado.gen.Return('full') else: log.error( 'The Salt Master has cached the public key for this ' 'node, this salt minion will wait for {0} seconds ' 'before attempting to re-authenticate'.format( self.opts['acceptance_wait_time'] ) ) raise tornado.gen.Return('retry') auth['aes'] = self.verify_master(payload) if not auth['aes']: log.critical( 'The Salt Master server\'s public key did not authenticate!\n' 'The master may need to be updated if it is a version of Salt ' 'lower than {0}, or\n' 'If you are confident that you are connecting to a valid Salt ' 'Master, then remove the master public key and restart the ' 'Salt Minion.\nThe master public key can be found ' 'at:\n{1}'.format(salt.version.__version__, m_pub_fn) ) raise SaltSystemExit('Invalid master key') if self.opts.get('syndic_master', False): # Is syndic syndic_finger = self.opts.get('syndic_finger', self.opts.get('master_finger', False)) if syndic_finger: if salt.utils.pem_finger(m_pub_fn) != syndic_finger: self._finger_fail(syndic_finger, m_pub_fn) else: if self.opts.get('master_finger', False): if salt.utils.pem_finger(m_pub_fn) != self.opts['master_finger']: self._finger_fail(self.opts['master_finger'], m_pub_fn) auth['publish_port'] = payload['publish_port'] raise tornado.gen.Return(auth) def get_keys(self): ''' Return keypair object for the minion. :rtype: Crypto.PublicKey.RSA._RSAobj :return: The RSA keypair ''' # Make sure all key parent directories are accessible user = self.opts.get('user', 'root') salt.utils.verify.check_path_traversal(self.opts['pki_dir'], user) if os.path.exists(self.rsa_path): with salt.utils.fopen(self.rsa_path) as f: key = RSA.importKey(f.read()) log.debug('Loaded minion key: {0}'.format(self.rsa_path)) else: log.info('Generating keys: {0}'.format(self.opts['pki_dir'])) gen_keys(self.opts['pki_dir'], 'minion', self.opts['keysize'], self.opts.get('user')) with salt.utils.fopen(self.rsa_path) as f: key = RSA.importKey(f.read()) return key def gen_token(self, clear_tok): ''' Encrypt a string with the minion private key to verify identity with the master. :param str clear_tok: A plaintext token to encrypt :return: Encrypted token :rtype: str ''' return private_encrypt(self.get_keys(), clear_tok) def minion_sign_in_payload(self): ''' Generates the payload used to authenticate with the master server. This payload consists of the passed in id_ and the ssh public key to encrypt the AES key sent back from the master. :return: Payload dictionary :rtype: dict ''' payload = {} payload['cmd'] = '_auth' payload['id'] = self.opts['id'] try: pubkey_path = os.path.join(self.opts['pki_dir'], self.mpub) with salt.utils.fopen(pubkey_path) as f: pub = RSA.importKey(f.read()) cipher = PKCS1_OAEP.new(pub) payload['token'] = cipher.encrypt(self.token) except Exception: pass with salt.utils.fopen(self.pub_path) as f: payload['pub'] = f.read() return payload def decrypt_aes(self, payload, master_pub=True): ''' This function is used to decrypt the AES seed phrase returned from the master server. The seed phrase is decrypted with the SSH RSA host key. Pass in the encrypted AES key. Returns the decrypted AES seed key, a string :param dict payload: The incoming payload. This is a dictionary which may have the following keys: 'aes': The shared AES key 'enc': The format of the message. ('clear', 'pub', etc) 'sig': The message signature 'publish_port': The TCP port which published the message 'token': The encrypted token used to verify the message. 'pub_key': The public key of the sender. :rtype: str :return: The decrypted token that was provided, with padding. :rtype: str :return: The decrypted AES seed key ''' if self.opts.get('auth_trb', False): log.warning( 'Auth Called: {0}'.format( ''.join(traceback.format_stack()) ) ) else: log.debug('Decrypting the current master AES key') key = self.get_keys() cipher = PKCS1_OAEP.new(key) key_str = cipher.decrypt(payload['aes']) if 'sig' in payload: m_path = os.path.join(self.opts['pki_dir'], self.mpub) if os.path.exists(m_path): try: with salt.utils.fopen(m_path) as f: mkey = RSA.importKey(f.read()) except Exception: return '', '' digest = hashlib.sha256(key_str).hexdigest() m_digest = public_decrypt(mkey.publickey(), payload['sig']) if m_digest != digest: return '', '' else: return '', '' if '_|-' in key_str: return key_str.split('_|-') else: if 'token' in payload: token = cipher.decrypt(payload['token']) return key_str, token elif not master_pub: return key_str, '' return '', '' def verify_pubkey_sig(self, message, sig): ''' Wraps the verify_signature method so we have additional checks. :rtype: bool :return: Success or failure of public key verification ''' if self.opts['master_sign_key_name']: path = os.path.join(self.opts['pki_dir'], self.opts['master_sign_key_name'] + '.pub') if os.path.isfile(path): res = verify_signature(path, message, binascii.a2b_base64(sig)) else: log.error('Verification public key {0} does not exist. You ' 'need to copy it from the master to the minions ' 'pki directory'.format(os.path.basename(path))) return False if res: log.debug('Successfully verified signature of master ' 'public key with verification public key ' '{0}'.format(self.opts['master_sign_key_name'] + '.pub')) return True else: log.debug('Failed to verify signature of public key') return False else: log.error('Failed to verify the signature of the message because ' 'the verification key-pairs name is not defined. Please ' 'make sure that master_sign_key_name is defined.') return False def verify_signing_master(self, payload): try: if self.verify_pubkey_sig(payload['pub_key'], payload['pub_sig']): log.info('Received signed and verified master pubkey ' 'from master {0}'.format(self.opts['master'])) m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) uid = salt.utils.get_uid(self.opts.get('user', None)) with salt.utils.fpopen(m_pub_fn, 'wb+', uid=uid) as wfh: wfh.write(payload['pub_key']) return True else: log.error('Received signed public-key from master {0} ' 'but signature verification failed!'.format(self.opts['master'])) return False except Exception as sign_exc: log.error('There was an error while verifying the masters public-key signature') raise Exception(sign_exc) def check_auth_deps(self, payload): ''' Checks if both master and minion either sign (master) and verify (minion). If one side does not, it should fail. :param dict payload: The incoming payload. This is a dictionary which may have the following keys: 'aes': The shared AES key 'enc': The format of the message. ('clear', 'pub', 'aes') 'publish_port': The TCP port which published the message 'token': The encrypted token used to verify the message. 'pub_key': The RSA public key of the sender. ''' # master and minion sign and verify if 'pub_sig' in payload and self.opts['verify_master_pubkey_sign']: return True # master and minion do NOT sign and do NOT verify elif 'pub_sig' not in payload and not self.opts['verify_master_pubkey_sign']: return True # master signs, but minion does NOT verify elif 'pub_sig' in payload and not self.opts['verify_master_pubkey_sign']: log.error('The masters sent its public-key signature, but signature ' 'verification is not enabled on the minion. Either enable ' 'signature verification on the minion or disable signing ' 'the public key on the master!') return False # master does NOT sign but minion wants to verify elif 'pub_sig' not in payload and self.opts['verify_master_pubkey_sign']: log.error('The master did not send its public-key signature, but ' 'signature verification is enabled on the minion. Either ' 'disable signature verification on the minion or enable ' 'signing the public on the master!') return False def extract_aes(self, payload, master_pub=True): ''' Return the AES key received from the master after the minion has been successfully authenticated. :param dict payload: The incoming payload. This is a dictionary which may have the following keys: 'aes': The shared AES key 'enc': The format of the message. ('clear', 'pub', etc) 'publish_port': The TCP port which published the message 'token': The encrypted token used to verify the message. 'pub_key': The RSA public key of the sender. :rtype: str :return: The shared AES key received from the master. ''' if master_pub: try: aes, token = self.decrypt_aes(payload, master_pub) if token != self.token: log.error( 'The master failed to decrypt the random minion token' ) return '' except Exception: log.error( 'The master failed to decrypt the random minion token' ) return '' return aes else: aes, token = self.decrypt_aes(payload, master_pub) return aes def verify_master(self, payload): ''' Verify that the master is the same one that was previously accepted. :param dict payload: The incoming payload. This is a dictionary which may have the following keys: 'aes': The shared AES key 'enc': The format of the message. ('clear', 'pub', etc) 'publish_port': The TCP port which published the message 'token': The encrypted token used to verify the message. 'pub_key': The RSA public key of the sender. :rtype: str :return: An empty string on verification failure. On success, the decrypted AES message in the payload. ''' m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) if os.path.isfile(m_pub_fn) and not self.opts['open_mode']: local_master_pub = salt.utils.fopen(m_pub_fn).read() if payload['pub_key'].replace('\n', '').replace('\r', '') != \ local_master_pub.replace('\n', '').replace('\r', ''): if not self.check_auth_deps(payload): return '' if self.opts['verify_master_pubkey_sign']: if self.verify_signing_master(payload): return self.extract_aes(payload, master_pub=False) else: return '' else: # This is not the last master we connected to log.error('The master key has changed, the salt master could ' 'have been subverted, verify salt master\'s public ' 'key') return '' else: if not self.check_auth_deps(payload): return '' # verify the signature of the pubkey even if it has # not changed compared with the one we already have if self.opts['always_verify_signature']: if self.verify_signing_master(payload): return self.extract_aes(payload) else: log.error('The masters public could not be verified. Is the ' 'verification pubkey {0} up to date?' ''.format(self.opts['master_sign_key_name'] + '.pub')) return '' else: return self.extract_aes(payload) else: if not self.check_auth_deps(payload): return '' # verify the masters pubkey signature if the minion # has not received any masters pubkey before if self.opts['verify_master_pubkey_sign']: if self.verify_signing_master(payload): return self.extract_aes(payload, master_pub=False) else: return '' # the minion has not received any masters pubkey yet, write # the newly received pubkey to minion_master.pub else: salt.utils.fopen(m_pub_fn, 'wb+').write(payload['pub_key']) return self.extract_aes(payload, master_pub=False)
def __init__(self): self._registry = weakref.WeakKeyDictionary()
def get(self, request): params = self.get_params(request) if params['aware']: context = self.get_context(request) else: context = DataContext() # Get all published app/model pairs to produce counts for. model_names = DataField.objects.published()\ .values_list('app_name', 'model_name')\ .order_by('model_name').distinct() results = [] data = [] models = set() QueryProcessor = pipeline.query_processors[params['processor']] # Workaround for a Python bug for versions 2.7.5 and below # http://bugs.python.org/issue10015 if not hasattr(threading.current_thread(), '_children'): threading.current_thread()._children = weakref.WeakKeyDictionary() # Pool of threads to execute the counts in parallel pool = ThreadPool() for app_name, model_name in model_names: # DataField used here to resolve foreign key-based fields. model = DataField(app_name=app_name, model_name=model_name).model # No redundant counts if model in models: continue models.add(model) opts = model._meta # Format is called to resolve Django's internal proxy wrapper. verbose_name = opts.verbose_name.format() verbose_name_plural = opts.verbose_name_plural.format() # Assume no custom verbose_name as been set in Meta class, so # apply a minimal title-case. if verbose_name.islower(): verbose_name = verbose_name.title() if verbose_name_plural.islower(): verbose_name_plural = verbose_name_plural.title() # Placeholder with the model name. The count will be replaced if # successful. data.append({ 'count': None, 'app_name': app_name, 'model_name': model_name, 'verbose_name': verbose_name, 'verbose_name_plural': verbose_name_plural, }) # Asynchronously execute the count result = pool.apply_async(get_count, args=(request, model, params['refresh'], QueryProcessor, context)) results.append(result) pool.close() for i, r in enumerate(results): try: count = r.get(timeout=serrano_settings.STATS_COUNT_TIMEOUT) data[i]['count'] = count except Exception: pass return data
def initializeGL(self, gls): # Build trace vertex VBO and associated vertex data dtype = [("vertex", numpy.float32, 2), ("ptid", numpy.uint32)] self.working_array = numpy.zeros(NUM_ENDCAP_SEGMENTS * 2 + 2, dtype=dtype) self.trace_vbo = VBO(self.working_array, GL.GL_DYNAMIC_DRAW) # Generate geometry for trace and endcaps # ptid is a variable with value 0 or 1 that indicates which endpoint the geometry is associated with self.__build_trace() self.__attribute_shader_vao = VAO() self.__attribute_shader = gls.shader_cache.get( "line_vertex_shader", "frag1", defines={"INPUT_TYPE": "in"}) # Now we build an index buffer that allows us to render filled geometry from the same # VBO. arr = [] for i in range(NUM_ENDCAP_SEGMENTS - 1): arr.append(0) arr.append(i + 2) arr.append(i + 3) for i in range(NUM_ENDCAP_SEGMENTS - 1): arr.append(1) arr.append(i + NUM_ENDCAP_SEGMENTS + 2) arr.append(i + NUM_ENDCAP_SEGMENTS + 3) arr.append(2) arr.append(2 + NUM_ENDCAP_SEGMENTS - 1) arr.append(2 + NUM_ENDCAP_SEGMENTS) arr.append(2 + NUM_ENDCAP_SEGMENTS) arr.append(2 + NUM_ENDCAP_SEGMENTS * 2 - 1) arr.append(2) arr = numpy.array(arr, dtype=numpy.uint32) self.index_vbo = VBO(arr, target=GL.GL_ELEMENT_ARRAY_BUFFER) self.instance_dtype = numpy.dtype([ ("pos_a", numpy.float32, 2), ("pos_b", numpy.float32, 2), ("thickness", numpy.float32, 1), #("color", numpy.float32, 4) ]) # Use a fake array to get a zero-length VBO for initial binding instance_array = numpy.ndarray(0, dtype=self.instance_dtype) self.instance_vbo = VBO(instance_array) with self.__attribute_shader_vao, self.trace_vbo: vbobind(self.__attribute_shader, self.trace_vbo.dtype, "vertex").assign() vbobind(self.__attribute_shader, self.trace_vbo.dtype, "ptid").assign() with self.__attribute_shader_vao, self.instance_vbo: self.__bind_pos_a = vbobind(self.__attribute_shader, self.instance_dtype, "pos_a", div=1) self.__bind_pos_b = vbobind(self.__attribute_shader, self.instance_dtype, "pos_b", div=1) self.__bind_thickness = vbobind(self.__attribute_shader, self.instance_dtype, "thickness", div=1) #vbobind(self.__attribute_shader, self.instance_dtype, "color", div=1).assign() self.__base_rebind(0) self.index_vbo.bind() self.__initialize_uniform(gls) self.__last_prepared = weakref.WeakKeyDictionary()
lines = [max_content_width(context, child)] # The first text line goes on the current line current_line += lines[0] if len(lines) > 1: # Forced line break yield current_line if len(lines) > 2: for line in lines[1:-1]: yield line current_line = lines[-1] is_line_start = lines[-1] == 0 skip_stack = None yield current_line TABLE_CACHE = weakref.WeakKeyDictionary() def _percentage_contribution(box): """Return the percentage contribution of a cell, column or column group. http://dbaron.org/css/intrinsic/#pct-contrib """ min_width = ( box.style.min_width.value if box.style.min_width != 'auto' and box.style.min_width.unit == '%' else 0) max_width = ( box.style.max_width.value if box.style.max_width != 'auto' and box.style.max_width.unit == '%' else float('inf')) width = (
class _PossiblyDisallowedChildren: """A set of possibly disallowed types contained within a type. These kinds of types may be banned in one or more contexts, so `_possibly_disallowed_members` and its cache record which of these type kinds appears inside a type, allowing for quick well-formedness checks. """ federated: Optional[Type] function: Optional[Type] sequence: Optional[Type] # Manual cache used rather than `cachetools.cached` due to incompatibility # with `WeakKeyDictionary`. We want to use a `WeakKeyDictionary` so that # cache entries are destroyed once the types they index no longer exist. _possibly_disallowed_children_cache = weakref.WeakKeyDictionary({}) def _possibly_disallowed_children( type_signature: Type, ) -> _PossiblyDisallowedChildren: """Returns possibly disallowed child types appearing in `type_signature`.""" cached = _possibly_disallowed_children_cache.get(type_signature, None) if cached: return cached disallowed = _PossiblyDisallowedChildren(None, None, None) for child_type in type_signature.children(): if child_type.is_federated(): disallowed = attr.evolve(disallowed, federated=child_type) elif child_type.is_function(): disallowed = attr.evolve(disallowed, function=child_type) elif child_type.is_sequence():