class Showcase(SessionSingleton): ''' This class enables other classes to translate names previously registered to actual objects ''' def __init__(self): self._objects = WeakValueDictionary() self._cases = WeakValueDictionary() def get(self, oid): ''' :param str oid: the oid registered in the NameAuthority ''' return self._objects.get(oid) def put(self, instance): ''' :param INamed oid: the exposed object. ''' self._objects[instance.oid] = instance def get_case(self, tag, default=None): ''' :param str tag: The tag that the returned case should have ''' return self._cases.get(tag, default)
class Monitor(QObject): """File monitor This monitor can be used to track single files """ def __init__(self, **kwargs): super(Monitor, self).__init__(**kwargs) self.watched = WeakValueDictionary() self.delMapper = QSignalMapper(self) self.delMapper.mapped[str].connect(self.unmonitorFile) self.watcher = MonitorWithRename(parent=self) self.watcher.fileChanged.connect(self._onFileChanged) def monitorFile(self, path): """Monitor a file and return an object that tracks only `path` :rtype: SingleFileWatcher :return: an object tracking `path`, the same object is returned if the method is called with the same path. """ path = os.path.abspath(path) self.watcher.addPath(path) proxy = self.watched.get(path) if not proxy: proxy = SingleFileWatcher(path) proxy.destroyed.connect(self.delMapper.map) self.delMapper.setMapping(proxy, path) self.watched[path] = proxy return proxy @Slot(str) def unmonitorFile(self, path): """Stop monitoring a file Since there is only one :any:`SingleFileWatcher` object per path, all objects monitoring `path` will not receive notifications anymore. To let only one object stop monitoring the file, simply disconnect its `modified` signal. When the :any:`SingleFileWatcher` object returned by method :any:`monitorFile` is destroyed, the file is automatically un-monitored. """ path = os.path.abspath(path) self.watcher.removePath(path) self.watched.pop(path, None) @Slot(str) def _onFileChanged(self, path): proxy = self.watched.get(path) if proxy: proxy.modified.emit()
class Reader: def __init__(self, document: Document): self.standalone_instances = WeakValueDictionary() self.tables = {} self.registry = TypeRegistry.instance self.document = document @property def file(self): return self.document.file def get_type_by_id(self, type_id): return self.registry.get_by_id(type_id) def find_instances(self, cls): return self.document.find_instances(cls, context=self) def add_instance(self, instance): if instance.__vo_id__ is not None: self.standalone_instances[instance.__vo_id__] = instance def get_instance_by_id(self, instance_id): return self.standalone_instances.get(instance_id, None) def add_table(self, table_id, table): self.tables[table_id] = table def get_table_by_id(self, table_id): return self.tables.get(table_id, None)
class _TransformExecutorServices(object): """Schedules and completes TransformExecutors. Controls the concurrency as appropriate for the applied transform the executor exists for. """ def __init__(self, executor_service): self._executor_service = executor_service self._scheduled = set() self._parallel = _ParallelEvaluationState( self._executor_service, self._scheduled) self._serial_cache = WeakValueDictionary() def parallel(self): return self._parallel def serial(self, step): cached = self._serial_cache.get(step) if not cached: cached = _SerialEvaluationState(self._executor_service, self._scheduled) self._serial_cache[step] = cached return cached @property def executors(self): return frozenset(self._scheduled)
class Root(Directory): cls_dir_class = Directory def __init__(self, dbname=':memory:', dirname='', threaded=True): super(Root,self).__init__(None,dirname) if threaded: self.con = hscommon.sqlite.ThreadedConn(dbname, False) else: self.con = sqlite.connect(dbname) self.id = 0 self._attrs_to_read = None self._id_cache = WeakValueDictionary() self._id_cache[0] = self try: cur = self.con.execute("select * from nodes where 1=2") except sqlite.OperationalError: self.__create_tables() self._load_from_db() def __create_tables(self): sqls = [ 'create table nodes(parent INT,type INT,name TEXT);', 'create table attrs(parent INT,name TEXT,type INT,value TEXT);', 'create index idx_node_parent on nodes(parent);', 'create index idx_node_name on nodes(name);', 'create index idx_attr_parent on attrs(parent);', 'create index idx_attr_name on attrs(name);', 'create unique index idx_node_unique on nodes(parent,name);', 'create unique index idx_attr_unique on attrs(parent,name);' ] for sql in sqls: self.con.execute(sql) def find_node_of_id(self,id): return self._id_cache.get(id)
class TransformExecutorServices(object): """Schedules and completes TransformExecutors. Controls the concurrency as appropriate for the applied transform the executor exists for. """ def __init__(self, executor_service): self._executor_service = executor_service self._scheduled = set() self._parallel = ParallelEvaluationState( self._executor_service, self._scheduled) self._serial_cache = WeakValueDictionary() def parallel(self): return self._parallel def serial(self, step): cached = self._serial_cache.get(step) if not cached: cached = SerialEvaluationState(self._executor_service, self._scheduled) self._serial_cache[step] = cached return cached @property def executors(self): return frozenset(self._scheduled)
class HasWeaks(metaclass=HasWeaksMeta): """A mix-in to allow objects to store weak references to other objects. Each individual class or instance of a class that subclasses this will have a separate weak reference dict, there is no inheritance. """ def __init__(self): super().__init__() self._get_weak_wr = WeakMethod(self._inst_get_weak) self._get_weak = lambda k: self._get_weak_wr()(k) self._set_weak_wr = WeakMethod(self._inst_set_weak) self._set_weak = lambda k, o: self._set_weak_wr()(k, o) self._del_weak_wr = WeakMethod(self._inst_del_weak) self._del_weak = lambda k: self._del_weak_wr()(k) self._weak_refs = WeakValueDictionary() def _inst_get_weak(self, name): return self._weak_refs.get(name) def _inst_set_weak(self, name, obj): if obj is None: self._del_weak(name) else: self._weak_refs[name] = obj def _inst_del_weak(self, name): if name in self._weak_refs: del self._weak_refs[name]
class XKeyListener(Thread): def __init__(self, display, root): super().__init__() self.daemon = True self.display = display self.root = root self.listenFor = WeakValueDictionary() self.start() #enddef def run(self): self.root.xinput_select_events([(xinput.AllDevices, xinput.KeyPressMask)]) while True: event = self.display.next_event() if event.type == X.KeyPress: cb = self.listenFor.get((event.detail, event. state), None) if cb: cb.trigger() #endif #endwhile #enddef def register(self, key, mods, sk): self.listenFor[(key, mods)] = sk #enddef def unregister(self, key, mods): del self.listenFor[(key, mods)]
class LRUCache: def __init__(self, max_size): self.LRU = [Node(time(), "none%s"%i) for i in range(max_size)] self.search = WeakValueDictionary() for i in self.LRU: self.search[i.name] = i def __setitem__(self, name, value): q = self.search.get(name, None) if q: q.data = value q.time = time() else: lru = self.LRU[0] self.search.pop(lru.name) lru.data = value lru.time = time() lru.name = name self.search[lru.name] = lru self.LRU.sort() def get(self, name, default=None): pos = None try: pos = self.search.__getitem__(name) pos.time = time() return pos.data except KeyError: if default is not None: return default else: raise
class Root(Directory): cls_dir_class = Directory def __init__(self, dbname=':memory:', dirname='', threaded=True): super(Root, self).__init__(None, dirname) if threaded: self.con = hscommon.sqlite.ThreadedConn(dbname, False) else: self.con = sqlite.connect(dbname) self.id = 0 self._attrs_to_read = None self._id_cache = WeakValueDictionary() self._id_cache[0] = self try: cur = self.con.execute("select * from nodes where 1=2") except sqlite.OperationalError: self.__create_tables() self._load_from_db() def __create_tables(self): sqls = [ 'create table nodes(parent INT,type INT,name TEXT);', 'create table attrs(parent INT,name TEXT,type INT,value TEXT);', 'create index idx_node_parent on nodes(parent);', 'create index idx_node_name on nodes(name);', 'create index idx_attr_parent on attrs(parent);', 'create index idx_attr_name on attrs(name);', 'create unique index idx_node_unique on nodes(parent,name);', 'create unique index idx_attr_unique on attrs(parent,name);' ] for sql in sqls: self.con.execute(sql) def find_node_of_id(self, id): return self._id_cache.get(id)
class TimerWnd(object): def __init__(self): win = ui.CreateFrame() win.CreateWindow(None, "", 0, (0, 0, 0, 0)) win.AttachObject(self) self._win = win self.tasks = WeakValueDictionary() def schedule(self, task): self.cancel(task) event_id = id(task) timer_id = self._win.SetTimer(event_id, task._milliseconds) if not timer_id: raise ValueError("Out of timers") task._win_timer_id = timer_id self.tasks[event_id] = task def cancel(self, task): timer_id = task._win_timer_id if timer_id: self._win.KillTimer(timer_id) task._win_timer_id = None def OnTimer(self, event_id): #print "TimerWnd.OnTimer:", event_id task = self.tasks.get(event_id) if task: if not task._repeat: self.cancel(task) task._proc() # We do this so that the application can't get starved of idle time # by a repeatedly-firing Task: application()._win_idle()
class _TransformExecutorServices(object): """Schedules and completes TransformExecutors. Controls the concurrency as appropriate for the applied transform the executor exists for. """ def __init__(self, executor_service): # type: (_ExecutorService) -> None self._executor_service = executor_service self._scheduled = set() # type: Set[TransformExecutor] self._parallel = _ParallelEvaluationState(self._executor_service, self._scheduled) self._serial_cache = WeakValueDictionary( ) # type: WeakValueDictionary[Any, _SerialEvaluationState] def parallel(self): # type: () -> _ParallelEvaluationState return self._parallel def serial(self, step): # type: (Any) -> _SerialEvaluationState cached = self._serial_cache.get(step) if not cached: cached = _SerialEvaluationState(self._executor_service, self._scheduled) self._serial_cache[step] = cached return cached @property def executors(self): # type: () -> FrozenSet[TransformExecutor] return frozenset(self._scheduled)
class Cache: __slots__ = ['delay', 'rate', '_page', '_commission'] def __init__(self, delay=0.25, rate=sqrt(2)): self.delay = delay self.rate = rate self._page = WeakValueDictionary() self._commission = WeakValueDictionary() def _backoff(self): return accumulate(chain([self.delay], repeat(self.rate)), mul) async def _download(self, session, url): for delay in self._backoff(): try: res = await session.get(url, timeout=60, connection_timeout=15) except AsksException: pass except BrokenResourceError: pass except ConnectionError: pass except GAIError: pass else: # The server raises 404(!) on parameter error if res.status_code // 100 == 2: break await sleep(delay) encoding = (res.encoding.lower() if 'charset' in res.headers.get('content-type') else None) # FIXME unreliable (change in asks?) return res.content, encoding async def page(self, session, url): page = self._page.get(url) if page is None: content, encoding = await self._download(session, url) page = self._page[url] = BeautifulSoup( content, _PARSER, from_encoding=encoding) return page def commission(self, parent, url): comm = self._commission.get(url) if comm is None: comm = self._commission[url] = Commission(parent, url) assert comm.parent == parent return comm
class RecentFiles(object): numberOfFiles = 8 separator = u";" def __init__(self, fileManager): self.fileManager = fileManager fileManager.onRead.connect(self.onReadOrWrite) fileManager.onWrite.connect(self.onReadOrWrite) self.files = self.retrieveList() self.menu = JMenu("Open Recent Program") self.menuActions = WeakValueDictionary() self.disabledItem = JMenuItem("(No recent programs)") self.disabledItem.enabled = False self.fillMenu() ### Maintaining the menu def onReadOrWrite(self, fileManager, filename, **_): try: self.files.remove(filename) except ValueError: if len(self.files) >= self.numberOfFiles: # Remove enough files that there's a spare slot. self.files = self.files[:self.numberOfFiles - 1] self.files.insert(0, filename) self.storeList(self.files) self.fillMenu() def fillMenu(self): self.menu.removeAll() if self.files: for filename in self.files: action = self.menuActions.get(filename) if action is None: action = PythonAction(self.fileManager.readAction, filename, name=filename) self.menuActions[filename] = action self.menu.add(action) else: self.menu.add(self.disabledItem) ### Writing filenames def retrieveList(self): joined = JESConfig.getInstance().getStringProperty(JESConfig.CONFIG_RECENT_FILES) if not joined: return [] else: return [fn for fn in joined.split(self.separator) if os.path.isfile(fn)] def storeList(self, files): joined = self.separator.join(files) JESConfig.getInstance().setStringProperty(JESConfig.CONFIG_RECENT_FILES, joined)
class ObjectPool(object): """ This class allows to fetch mvc model objects using their UUID. This requires to model to have a property called "uuid". All class inheriting from the base 'Model' class will have this. If implementing a custom model, the UUID property is responsible for the removal and addition to the pool when it changes values. Also see the UUIDProperty descriptor for an example implementation. We can use this to store complex relations between objects where references to each other can be replaced with the UUID. For a multi-threaded version see ThreadedObjectPool. """ def __init__(self, *args, **kwargs): object.__init__(self) self._objects = WeakValueDictionary() def add_or_get_object(self, obj): try: self.add_object(obj, force=False, silent=False) return obj except KeyError: return self.get_object(obj.uuid) def add_object(self, obj, force=False, fail_on_duplicate=False): if not obj.uuid in self._objects or force: self._objects[obj.uuid] = obj elif fail_on_duplicate: raise KeyError( "UUID %s is already taken by another object %s, cannot add object %s" % (obj.uuid, self._objects[obj.uuid], obj)) else: # Just change the objects uuid, will break refs, but # it prevents issues with inherited properties etc. logger.warning( "A duplicate UUID was passed to an ObjectPool for a %s object." % obj) obj.uuid = get_new_uuid() def change_all_uuids(self): # first get a copy off all uuids & objects: items = list(self._objects.items()) for uuid, obj in items: # @UnusedVariable obj.uuid = get_new_uuid() def remove_object(self, obj): if obj.uuid in self._objects and self._objects[obj.uuid] == obj: del self._objects[obj.uuid] def get_object(self, uuid): obj = self._objects.get(uuid, None) return obj def clear(self): self._objects.clear()
class ObjectRedis(object): """ Extended Redis client, which additionally provides object-oriented operations and object caching. Objects are represented as hashes in the Redis database. The translation from a hash to an object is carried out by a given `decode` function. When `caching` is enabled, objects loaded from the Redis database are cached and subsequently retrieved from the cache. An object stays in the cache as long as there is a reference to it and it is automatically removed when the Python interpreter destroys it. Thus, it is guaranteed that getting the same key multiple times will yield the identical object. Attributes: * `r`: Underlying Redis client. Read-Only. * `decode`: function, which decodes an object from a Redis hash. It is called with the hash (a `dict`) as single argument. Read-Only. * `caching`: switch to enable / disable object caching. """ # TODO: add oset and omset def __init__(self, r, decode, caching=True): self.r = r self.decode = decode self.caching = caching self._cache = WeakValueDictionary() def oget(self, key): """ Get the object for `key`. """ object = self._cache.get(key) if self.caching else None if not object: hash = self.hgetall(key) if hash: object = self.decode(hash) if self.caching: self._cache[key] = object return object def omget(self, keys): """ Get the objects for all specified `keys`. """ # TODO: make atomic return [self.oget(k) for k in keys] def __getattr__(self, name): return getattr(self.r, name)
class ObjectPool(object): """ This class allows to fetch mvc model objects using their UUID. This requires to model to have a property called "uuid". All class inheriting from the base 'Model' class will have this. If implementing a custom model, the UUID property is responsible for the removal and addition to the pool when it changes values. Also see the UUIDPropIntel class for an example implementation. We can use this to store complex relations between objects where references to each other can be replaced with the UUID. For a multi-threaded version see ThreadedObjectPool. """ def __init__(self, *args, **kwargs): object.__init__(self) self._objects = WeakValueDictionary() def add_or_get_object(self, obj): try: self.add_object(obj, force=False, silent=False) return obj except KeyError: return self.get_object(obj.uuid) def add_object(self, obj, force=False, fail_on_duplicate=False): if not obj.uuid in self._objects or force: self._objects[obj.uuid] = obj elif fail_on_duplicate: raise KeyError, "UUID %s is already taken by another object %s, cannot add object %s" % (obj.uuid, self._objects[obj.uuid], obj) else: # Just change the objects uuid, will break refs, but # it prevents issues with inherited properties etc. logger.warning("A duplicate UUID was passed to an ObjectPool for a %s object." % obj) obj.uuid = get_new_uuid() def change_all_uuids(self): # first get a copy off all uuids & objects: items = self._objects.items() for uuid, obj in items: # @UnusedVariable obj.uuid = get_new_uuid() def remove_object(self, obj): if obj.uuid in self._objects and self._objects[obj.uuid] == obj: del self._objects[obj.uuid] def get_object(self, uuid): obj = self._objects.get(uuid, None) return obj def clear(self): self._objects.clear()
class factory: """package generator does weakref caching per repository :cvar child_class: callable to generate packages """ child_class = package def __init__(self, parent_repo): self._parent_repo = parent_repo self._cached_instances = WeakValueDictionary() def new_package(self, *args): """generate a new package instance""" inst = self._cached_instances.get(args) if inst is None: inst = self._cached_instances[args] = self.child_class(self, *args) return inst def __call__(self, *args, **kwds): return self.new_package(*args, **kwds) def clear(self): """wipe the weakref cache of packages instances""" self._cached_instances.clear() def _get_metadata(self, *args): """Pulls metadata from the repo/cache/wherever. Must be overridden in derivatives. """ raise NotImplementedError def _update_metadata(self, *args): """Updates metadata in the repo/cache/wherever. Must be overridden in derivatives. """ raise NotImplementedError def __getstate__(self): d = self.__dict__.copy() del d['_cached_instances'] return d def __setstate__(self, state): self.__dict__ = state.copy() self.__dict__['_cached_instances'] = WeakValueDictionary()
class DialogManager: """ Opens and tracks Qt dialogs. """ def __init__(self, parent: QWidget) -> None: self._parent = parent # All dialogs will be children of this widget. self._dialogs = WeakValueDictionary( ) # Tracks a single instance of each dialog by its opener. def attach(self, dialog: QDialog) -> None: """ Attach <dialog> to our parent, preserving its window flags. """ flags = dialog.windowFlags() dialog.setParent(self._parent) dialog.setWindowFlags(flags) def open_unique(self, opener: DialogOpener) -> None: """ If a previous dialog is open, set focus on it, otherwise open a new one using <opener>. """ key = getattr(opener, '__qualname__', id(opener)) dialog = self._dialogs.get(key) if dialog is not None and not dialog.isHidden(): dialog.activateWindow() else: dialog = self._dialogs[key] = opener() self.attach(dialog) dialog.show() def yes_or_no(self, title: str, message: str) -> bool: """ Present a modal yes/no dialog and return the user's response as True/False. """ yes, no = QMessageBox.Yes, QMessageBox.No button = QMessageBox.question(self._parent, title, message, yes | no) return button == yes def open_file(self, title="Open File", file_exts="", start_dir=".") -> str: """ Present a modal dialog for the user to select a file to open. Return the selected filename, or an empty string on cancel. """ return QFileDialog.getOpenFileName(self._parent, title, start_dir, _filter_str(file_exts))[0] def open_files(self, title="Open Files", file_exts="", start_dir=".") -> List[str]: """ Present a modal dialog for the user to select multiple files to open. Return a list of selected filenames, or an empty list on cancel. """ return QFileDialog.getOpenFileNames(self._parent, title, start_dir, _filter_str(file_exts))[0] def save_file(self, title="Save File", file_exts="", start_dir=".") -> str: """ Present a modal dialog for the user to select a file to save. Return the selected filename, or an empty string on cancel. """ return QFileDialog.getSaveFileName(self._parent, title, start_dir, _filter_str(file_exts))[0]
class WindowRegistry(object): def __init__(self, configs: GlobalConfigs, documents: Any, session_starter: Callable, sublime: Any, handler_dispatcher: LanguageHandlerListener) -> None: self._windows = WeakValueDictionary( ) # type: WeakValueDictionary[int, WindowManager] self._configs = configs self._documents = documents self._session_starter = session_starter self._sublime = sublime self._handler_dispatcher = handler_dispatcher self._diagnostics_ui_class = None # type: Optional[Callable] self._server_panel_factory = None # type: Optional[Callable] self._settings = None # type: Optional[Settings] def set_diagnostics_ui(self, ui_class: Any) -> None: self._diagnostics_ui_class = ui_class def set_server_panel_factory(self, factory: Callable) -> None: self._server_panel_factory = factory def set_settings_factory(self, settings: Settings) -> None: self._settings = settings def lookup(self, window: Any) -> WindowManager: state = self._windows.get(window.id()) if state is None: if not self._settings: raise RuntimeError("no settings") workspace = ProjectFolders(window) window_configs = self._configs.for_window(window) window_documents = self._documents.for_window( window, workspace, window_configs) diagnostics_ui = self._diagnostics_ui_class( window, window_documents) if self._diagnostics_ui_class else None state = WindowManager( window=window, workspace=workspace, settings=self._settings, configs=window_configs, documents=window_documents, diagnostics=DiagnosticsStorage(diagnostics_ui), session_starter=self._session_starter, sublime=self._sublime, handler_dispatcher=self._handler_dispatcher, server_panel_factory=self._server_panel_factory) self._windows[window.id()] = state return state
class NodeManager: _all: WeakNodeSet _by_id: WeakNodeDictionary _all_num: int _delegate_num: int def __init__(self): super().__init__() self._all = WeakSet() self._by_id = WeakValueDictionary() self._all_num = 0 self._delegate_num = 0 def register(self, node: BaseNode): assert node not in self._all, f'duplicate node {node!r}' assert node not in self._by_id, f'duplicate node ID {node.id!r}' self._all.add(node) self._by_id[node.id] = node self._all_num += 1 if node.is_delegate: self._delegate_num += 1 def get_node(self, id: int, default: Optional[BaseNode] = None) -> Optional[BaseNode]: return self._by_id.get(id, default=default) def get_delegates(self): return {node for node in self._all if node.is_delegate} def block(self, id: int): node = self.get_node(id) if node: node.is_blacked = True def nodes(self) -> Set[BaseNode]: return set(self._all) @property def delegate_num(self) -> int: return self._delegate_num @property def all_num(self) -> int: return self._all_num def __len__(self) -> int: return self._all_num
class DialogManager: """ Tracks standard dialogs and opens common modal dialogs and returns their results. """ _DEFAULT_DIALOG_FLAGS = Qt.CustomizeWindowHint | Qt.Dialog | Qt.WindowTitleHint | Qt.WindowCloseButtonHint def __init__(self, parent: QWidget) -> None: self._parent = parent # All dialogs will be children of this widget. self._dialogs_by_cls = WeakValueDictionary( ) # Contains the current instance of each dialog class. def load(self, dlg_cls: Type[QDialog], flags=_DEFAULT_DIALOG_FLAGS) -> Optional[QDialog]: """ If a previous <dlg_cls> is open, set focus on it and return None, otherwise return a new one. """ dialog = self._dialogs_by_cls.get(dlg_cls) if dialog is not None and not dialog.isHidden(): dialog.show() dialog.activateWindow() return None dialog = self._dialogs_by_cls[dlg_cls] = dlg_cls(self._parent, flags) return dialog def yes_or_no(self, title: str, message: str) -> bool: """ Present a yes/no dialog and return the user's response as a bool. """ yes, no = QMessageBox.Yes, QMessageBox.No button = QMessageBox.question(self._parent, title, message, yes | no) return button == yes def open_file(self, title="Open File", file_exts="", start_dir=".") -> str: """ Present a modal dialog for the user to select a file to open. Return the selected filename, or an empty string on cancel. """ return QFileDialog.getOpenFileName(self._parent, title, start_dir, _filter_str(file_exts))[0] def open_files(self, title="Open Files", file_exts="", start_dir=".") -> List[str]: """ Present a modal dialog for the user to select multiple files to open. Return a list of selected filenames, or an empty list on cancel. """ return QFileDialog.getOpenFileNames(self._parent, title, start_dir, _filter_str(file_exts))[0] def save_file(self, title="Save File", file_exts="", start_dir=".") -> str: """ Present a modal dialog for the user to select a file to save. Return the selected filename, or an empty string on cancel. """ return QFileDialog.getSaveFileName(self._parent, title, start_dir, _filter_str(file_exts))[0]
class BaseContextualizable(object): def __init__(self, *args, **kwargs): super(BaseContextualizable, self).__init__(*args, **kwargs) if not hasattr(self, '_contexts'): self._contexts = WeakValueDictionary() @property def context(self): return None def contextualize(self, context): """ Return an object with the given context. If the provided ``context`` is `None`, then `self` MUST be returned unmodified. It is generally not correct to set a field on the object and return the same object as this would change the context for other users of the object. Also, returning a copy of the object is usually inappropriate for mutable objects. Immutable objects may maintain a 'context' property and return a copy of themselves with that property set to the provided ``context`` argument. """ ctxd = self._contexts.get(context) if ctxd is not None: return ctxd ctxd = self.contextualize_augment(context) self._contexts[context] = ctxd return ctxd def decontextualize(self): """ Return the object with all contexts removed """ return self def add_contextualization(self, context, contextualization): try: self._contexts[context] = contextualization except AttributeError: self._contexts = WeakValueDictionary() self._contexts[context] = contextualization def contextualize_augment(self, context): return self
class ThreadLocalEntityCache(local): def __init__(self): self.lock = Lock() self._dict = WeakValueDictionary() def __contains__(self, key): return key in self._dict def __getitem__(self, key): return self._dict[key] def get(self, key, default=None): return self._dict.get(key, default) def clear(self): self._dict.clear() def keys(self): return self._dict.keys() def update(self, key, value): """ Extract, insert or remove a value for a given key. """ with self.lock: if value is None: # remove try: del self._dict[key] except KeyError: pass else: return None elif callable(value): try: # extract return self._dict[key] except KeyError: # construct and insert new_value = value() self._dict[key] = new_value return new_value else: # insert or replace self._dict[key] = value return value
class Cache(object): def __init__(self, config): self._entries = dict() # indexed by (domain, user) tuple self._entry_by_domain = WeakValueDictionary() self._config = config def set(self, service, username, password, timeout=None): expire = None if timeout: expire = int(time.time()) + timeout entry = CacheEntry(username, password, expire) self._entries[service] = entry domain = self._config.get_domain_for_service(service) if domain is not None: logging.debug("updating password for domain,user = %s,%s", domain, username) self._entry_by_domain[(domain, username)] = entry def get(self, service): entry = self._entries.get(service) if entry is None: domain = self._config.get_domain_for_service(service) username = self._config.get_user_for_service(service) if domain and username: logging.debug("searching in domain %s", domain) entry = self._entry_by_domain.get((domain, username)) if entry is None: return None now = int(time.time()) if entry.expire and entry.expire <= now: del self._entries[service] # no need to delete from domain dict since is a weakref values return None return entry
class FileServer(MessageDispatcher): """Serve files via an http server.""" FILE_CONTENT = "/file/" def __init__(self, port=8001): """Create a new file server.""" self.app = Flask(self.__class__.__name__) self.files = WeakValueDictionary() self.app.route(self.FILE_CONTENT + "<id>")(self.serve_file_content_by_id) self.port = port def serve_file_content_by_id(self, id): """Serve a file by an id.""" file = self.files.get(id, NoFile()) return file.flask_send_file() def add_file(self, file): """Serve a file until noone needs it.""" self.files[str(file.get_id())] = file file.served_by(self) def receive_new_image(self, message): """React to new images.""" image = message["image"] self.add_file(image) def get_file_content_path(self, file): """Return the path component for the file.""" return self.FILE_CONTENT + str(file.get_id()) def get_port(self): """Return the port used to serve the files.""" return self.port def run(self): """Run the server.""" self.app.run(host="0.0.0.0", port=self.port) def run_in_parallel(self): """Run the server in parellel.""" thread = threading.Thread(target=self.run, daemon=True) thread.start()
class SharedSinkProvider(SinkProviderBase): def __init__(self, key_selector): self._key_selector = key_selector self._cache = WeakValueDictionary() super(SharedSinkProvider, self).__init__() def CreateSink(self, properties): key = self._key_selector(properties) if key: sink = self._cache.get(key) if not sink: new_sink = self.next_provider.CreateSink(properties) sink = RefCountedSink(new_sink) self._cache[key] = sink return sink else: return self.next_provider.CreateSink(properties) @property def sink_class(self): return self.next_provider.sink_class
class WeakLocks(object): """ A cache of DeferredLocks that gets garbage collected after the lock has been utilized """ def __init__(self): self._locks = WeakValueDictionary() def get_lock(self, key): """ Get lock based on key. If no lock exists, create one. :param key: Some arbitrary key :return: :class:`DeferredLock` """ lock = self._locks.get(key) if not lock: lock = DeferredLock() self._locks[key] = lock return lock
class Index2Source(IPackSource): @property def index(self): return self._index @property def pack(self): return self._pack def __init__(self, pack, index): self._pack = pack self._index = index self._files = WeakValueDictionary() self._file_path_map = {} def file_exists(self, path_or_hash): if isinstance(path_or_hash, str): path_or_hash = _compute_hash(path_or_hash) return path_or_hash in self.index.files def get_file(self, path_or_hash): def from_hash(hash): file = self._files.get(hash, None) if file is not None: return file index = self.index.files[hash] file = FileFactory.get(self.pack, index) self._files[hash] = file return file if isinstance(path_or_hash, str): f = from_hash(_compute_hash(path_or_hash)) f.path = path_or_hash return f return from_hash(path_or_hash) def __iter__(self): for file in self.index.files.values(): yield self.get_file(file.file_key)
class StoreCache(object): ''' Cache of stores previously cached by a `BDS <BundleDependencyStore>`. We don't want to keep hold of a store if there's no BDS using it, so we only reference the stores weakly. ''' def __init__(self): self._cache = WeakValueDictionary() self._refcounts = dict() ''' Counts for references to cached BDS stores. Needed so we know when we can do clean-up. ''' def check_out(self, key): self._refcounts[key] = self._refcounts.get(key, 0) + 1 def check_in(self, key): rc = self._refcounts[key] if rc == 1: del self._refcounts[key] else: self._refcounts[key] = rc - 1 return rc - 1 def refcount(self, key): return self._refcounts.get(key, 0) def get(self, key, default=None): return self._cache.get(key, default) def __getitem__(self, key): return self._cache[key] def __setitem__(self, key, val): if key in self._cache: raise Exception(f"There's already an entry for {key}") self._cache[key] = val
class Layout(object): def __init__(self): self.solver = Solver() self.introns = WeakValueDictionary() def update(self, element): self.solver.autosolve = False for constraint in element.constraints[1]: self.solver.remove_constraint(constraint) for constraint in element.constraints[0]: self.solver.add_constraint(constraint) element.constraints = [], element.constraints[0] self.solver.autosolve = True def add_intron(self, source, intron): self.introns[source] = intron def pop_intron(self, source): return self.introns.pop(source) def get_intron(self, source, otherwise=None): return self.introns.get(source, otherwise)
class FileYAMLReader(AbstractYAMLReader): """Implementation of the YAML reader.""" def __init__(self, config_files, filter_parameters=None, filter_filenames=True, **kwargs): super(FileYAMLReader, self).__init__(config_files) self.file_handlers = {} self.filter_filenames = self.info.get('filter_filenames', filter_filenames) self.filter_parameters = filter_parameters or {} if kwargs: logger.warning( "Unrecognized/unused reader keyword argument(s) '{}'".format( kwargs)) self.coords_cache = WeakValueDictionary() @property def sensor_names(self): if not self.file_handlers: return self.info['sensors'] file_handlers = (handlers[0] for handlers in self.file_handlers.values()) sensor_names = set() for fh in file_handlers: try: sensor_names.update(fh.sensor_names) except NotImplementedError: continue if not sensor_names: return self.info['sensors'] return sorted(sensor_names) @property def available_dataset_ids(self): for ds_id in self.all_dataset_ids: fts = self.ids[ds_id]["file_type"] if isinstance(fts, str) and fts in self.file_handlers: yield ds_id elif any(ft in self.file_handlers for ft in fts): yield ds_id @property def start_time(self): if not self.file_handlers: raise RuntimeError("Start time unknown until files are selected") return min(x[0].start_time for x in self.file_handlers.values()) @property def end_time(self): if not self.file_handlers: raise RuntimeError("End time unknown until files are selected") return max(x[-1].end_time for x in self.file_handlers.values()) @staticmethod def check_file_covers_area(file_handler, check_area): """Checks if the file covers the current area. If the file doesn't provide any bounding box information or 'area' was not provided in `filter_parameters`, the check returns True. """ try: gbb = Boundary(*file_handler.get_bounding_box()) except NotImplementedError as err: logger.debug("Bounding box computation not implemented: %s", str(err)) else: abb = AreaDefBoundary(get_area_def(check_area), frequency=1000) intersection = gbb.contour_poly.intersection(abb.contour_poly) if not intersection: return False return True def find_required_filehandlers(self, requirements, filename_info): """Find the necessary fhs for the current filehandler. We assume here requirements are available. """ req_fh = [] if requirements: for requirement in requirements: for fhd in self.file_handlers[requirement]: # FIXME: Isn't this super wasteful? filename_info.items() # every iteration? if (all(item in filename_info.items() for item in fhd.filename_info.items())): req_fh.append(fhd) break else: raise RuntimeError('No matching file in ' + requirement) # break everything and continue to next # filetype! return req_fh def sorted_filetype_items(self): """Sort the instance's filetypes in using order.""" processed_types = [] file_type_items = deque(self.config['file_types'].items()) while len(file_type_items): filetype, filetype_info = file_type_items.popleft() requirements = filetype_info.get('requires') if requirements is not None: # requirements have not been processed yet -> wait missing = [ req for req in requirements if req not in processed_types ] if missing: file_type_items.append((filetype, filetype_info)) continue processed_types.append(filetype) yield filetype, filetype_info @staticmethod def filename_items_for_filetype(filenames, filetype_info): """Iterator over the filenames matching *filetype_info*.""" matched_files = [] for pattern in filetype_info['file_patterns']: for filename in match_filenames(filenames, pattern): if filename in matched_files: continue try: filename_info = parse(pattern, get_filebase(filename, pattern)) except ValueError: logger.debug("Can't parse %s with %s.", filename, pattern) continue matched_files.append(filename) yield filename, filename_info def new_filehandler_instances(self, filetype_info, filename_items): """Generate new filehandler instances.""" requirements = filetype_info.get('requires') filetype_cls = filetype_info['file_reader'] for filename, filename_info in filename_items: try: req_fh = self.find_required_filehandlers( requirements, filename_info) except RuntimeError: logger.warning("Can't find requirements for %s", filename) continue except KeyError: logger.warning("Missing requirements for %s", filename) continue yield filetype_cls(filename, filename_info, filetype_info, *req_fh) def time_matches(self, fstart, fend): start_time = self.filter_parameters.get('start_time') end_time = self.filter_parameters.get('end_time') fend = fend or fstart if start_time and fend and fend < start_time: return False if end_time and fstart and fstart > end_time: return False return True def metadata_matches(self, sample_dict, file_handler=None): # special handling of start/end times if not self.time_matches(sample_dict.get('start_time'), sample_dict.get('end_time')): return False for key, val in self.filter_parameters.items(): if key != 'area' and key not in sample_dict: continue if key in ['start_time', 'end_time']: continue elif key == 'area' and file_handler: if not self.check_file_covers_area(file_handler, val): logger.info('Filtering out %s based on area', file_handler.filename) break elif key in sample_dict and val != sample_dict[key]: # don't use this file break else: # all the metadata keys are equal return True return False def filter_filenames_by_info(self, filename_items): """Filter out file using metadata from the filenames. Currently only uses start and end time. If only start time is available from the filename, keep all the filename that have a start time before the requested end time. """ for filename, filename_info in filename_items: fend = filename_info.get('end_time') fstart = filename_info.setdefault('start_time', fend) if fend and fend < fstart: # correct for filenames with 1 date and 2 times fend = fend.replace(year=fstart.year, month=fstart.month, day=fstart.day) filename_info['end_time'] = fend if self.metadata_matches(filename_info): yield filename, filename_info def filter_fh_by_metadata(self, filehandlers): """Filter out filehandlers using provide filter parameters.""" for filehandler in filehandlers: filehandler.metadata['start_time'] = filehandler.start_time filehandler.metadata['end_time'] = filehandler.end_time if self.metadata_matches(filehandler.metadata, filehandler): yield filehandler def filter_selected_filenames(self, filenames): for filetype, filetype_info in self.sorted_filetype_items(): filename_iter = self.filename_items_for_filetype( filenames, filetype_info) if self.filter_filenames: filename_iter = self.filter_filenames_by_info(filename_iter) for fn, _ in filename_iter: yield fn def new_filehandlers_for_filetype(self, filetype_info, filenames): """Create filehandlers for a given filetype.""" filename_iter = self.filename_items_for_filetype( filenames, filetype_info) if self.filter_filenames: # preliminary filter of filenames based on start/end time # to reduce the number of files to open filename_iter = self.filter_filenames_by_info(filename_iter) filehandler_iter = self.new_filehandler_instances( filetype_info, filename_iter) filtered_iter = self.filter_fh_by_metadata(filehandler_iter) return list(filtered_iter) def create_filehandlers(self, filenames): """Organize the filenames into file types and create file handlers.""" filenames = list(OrderedDict.fromkeys(filenames)) logger.debug("Assigning to %s: %s", self.info['name'], filenames) self.info.setdefault('filenames', []).extend(filenames) filename_set = set(filenames) # load files that we know about by creating the file handlers for filetype, filetype_info in self.sorted_filetype_items(): filehandlers = self.new_filehandlers_for_filetype( filetype_info, filename_set) filename_set -= set([fhd.filename for fhd in filehandlers]) if filehandlers: self.file_handlers[filetype] = sorted( filehandlers, key=lambda fhd: (fhd.start_time, fhd.filename)) # update existing dataset IDs with information from the file handler self.update_ds_ids_from_file_handlers() # load any additional dataset IDs determined dynamically from the file self.add_ds_ids_from_files() def update_ds_ids_from_file_handlers(self): """Update DatasetIDs with information from loaded files. This is useful, for example, if dataset resolution may change depending on what files were loaded. """ for file_handlers in self.file_handlers.values(): fh = file_handlers[0] # update resolution in the dataset IDs for this files resolution res = getattr(fh, 'resolution', None) if res is None: continue for ds_id, ds_info in list(self.ids.items()): if fh.filetype_info['file_type'] != ds_info['file_type']: continue if ds_id.resolution is not None: continue ds_info['resolution'] = res new_id = DatasetID.from_dict(ds_info) self.ids[new_id] = ds_info del self.ids[ds_id] def add_ds_ids_from_files(self): """Check files for more dynamically discovered datasets.""" for file_handlers in self.file_handlers.values(): try: fh = file_handlers[0] avail_ids = fh.available_datasets() except NotImplementedError: continue # dynamically discover other available datasets for ds_id, ds_info in avail_ids: # don't overwrite an existing dataset # especially from the yaml config coordinates = ds_info.get('coordinates') if isinstance(coordinates, list): # xarray doesn't like concatenating attributes that are # lists: https://github.com/pydata/xarray/issues/2060 ds_info['coordinates'] = tuple(ds_info['coordinates']) self.ids.setdefault(ds_id, ds_info) @staticmethod def _load_dataset(dsid, ds_info, file_handlers, dim='y'): """Load only a piece of the dataset.""" slice_list = [] failure = True for fh in file_handlers: try: projectable = fh.get_dataset(dsid, ds_info) if projectable is not None: slice_list.append(projectable) failure = False except KeyError: logger.warning("Failed to load {} from {}".format(dsid, fh), exc_info=True) if failure: raise KeyError( "Could not load {} from any provided files".format(dsid)) if dim not in slice_list[0].dims: return slice_list[0] res = xr.concat(slice_list, dim=dim) combined_info = file_handlers[0].combine_info( [p.attrs for p in slice_list]) res.attrs = combined_info return res def _load_dataset_data(self, file_handlers, dsid, xslice=slice(None), yslice=slice(None)): ds_info = self.ids[dsid] proj = self._load_dataset(dsid, ds_info, file_handlers) # FIXME: areas could be concatenated here # Update the metadata proj.attrs['start_time'] = file_handlers[0].start_time proj.attrs['end_time'] = file_handlers[-1].end_time return proj def _preferred_filetype(self, filetypes): """Get the preferred filetype out of the *filetypes* list. At the moment, it just returns the first filetype that has been loaded. """ if not isinstance(filetypes, list): filetypes = [filetypes] # look through the file types and use the first one that we have loaded for filetype in filetypes: if filetype in self.file_handlers: return filetype return None def _load_area_def(self, dsid, file_handlers): """Load the area definition of *dsid*.""" area_defs = [fh.get_area_def(dsid) for fh in file_handlers] area_defs = [ area_def for area_def in area_defs if area_def is not None ] final_area = StackedAreaDefinition(*area_defs) return final_area.squeeze() def _get_coordinates_for_dataset_key(self, dsid): """Get the coordinate dataset keys for *dsid*.""" ds_info = self.ids[dsid] cids = [] for cinfo in ds_info.get('coordinates', []): if not isinstance(cinfo, dict): cinfo = {'name': cinfo} cinfo['resolution'] = ds_info['resolution'] if 'polarization' in ds_info: cinfo['polarization'] = ds_info['polarization'] cid = DatasetID(**cinfo) cids.append(self.get_dataset_key(cid)) return cids def _get_coordinates_for_dataset_keys(self, dsids): """Get all coordinates.""" coordinates = {} for dsid in dsids: cids = self._get_coordinates_for_dataset_key(dsid) coordinates.setdefault(dsid, []).extend(cids) return coordinates def _get_file_handlers(self, dsid): """Get the file handler to load this dataset.""" ds_info = self.ids[dsid] filetype = self._preferred_filetype(ds_info['file_type']) if filetype is None: logger.warning( "Required file type '%s' not found or loaded for " "'%s'", ds_info['file_type'], dsid.name) else: return self.file_handlers[filetype] def _make_area_from_coords(self, coords): """Create an appropriate area with the given *coords*.""" if len(coords) == 2: lon_sn = coords[0].attrs.get('standard_name') lat_sn = coords[1].attrs.get('standard_name') if lon_sn == 'longitude' and lat_sn == 'latitude': key = None try: key = (coords[0].data.name, coords[1].data.name) sdef = self.coords_cache.get(key) except AttributeError: sdef = None if sdef is None: sdef = SwathDefinition(*coords) if key is not None: self.coords_cache[key] = sdef sensor_str = '_'.join(self.info['sensors']) shape_str = '_'.join(map(str, coords[0].shape)) sdef.name = "{}_{}_{}_{}".format(sensor_str, shape_str, coords[0].attrs['name'], coords[1].attrs['name']) return sdef else: raise ValueError( 'Coordinates info object missing standard_name key: ' + str(coords)) elif len(coords) != 0: raise NameError("Don't know what to do with coordinates " + str(coords)) def _load_dataset_area(self, dsid, file_handlers, coords): """Get the area for *dsid*.""" try: return self._load_area_def(dsid, file_handlers) except NotImplementedError: if any(x is None for x in coords): logger.warning( "Failed to load coordinates for '{}'".format(dsid)) return None area = self._make_area_from_coords(coords) if area is None: logger.debug("No coordinates found for %s", str(dsid)) return area # TODO: move this out of here. def _get_slices(self, area): """Get the slices of raw data covering area. Args: area: the area to slice. Returns: slice_kwargs: kwargs to pass on to loading giving the span of the data to load. area: the trimmed area corresponding to the slices. """ slice_kwargs = {} if area is not None and self.filter_parameters.get('area') is not None: try: slices = get_area_slices(area, self.filter_parameters['area']) area = get_sub_area(area, *slices) slice_kwargs['xslice'], slice_kwargs['yslice'] = slices except (NotImplementedError, AttributeError): logger.info("Cannot compute specific slice of data to load.") return slice_kwargs, area def _load_dataset_with_area(self, dsid, coords): """Loads *dsid* and it's area if available.""" file_handlers = self._get_file_handlers(dsid) if not file_handlers: return area = self._load_dataset_area(dsid, file_handlers, coords) slice_kwargs, area = self._get_slices(area) try: ds = self._load_dataset_data(file_handlers, dsid, **slice_kwargs) except (KeyError, ValueError) as err: logger.exception("Could not load dataset '%s': %s", dsid, str(err)) return None if area is not None: ds.attrs['area'] = area if (('x' not in ds.coords) or('y' not in ds.coords)) and \ hasattr(area, 'get_proj_vectors_dask'): ds['x'], ds['y'] = area.get_proj_vectors_dask(CHUNK_SIZE) return ds def _load_ancillary_variables(self, datasets): """Load the ancillary variables of `datasets`.""" all_av_ids = set() for dataset in datasets.values(): ancillary_variables = dataset.attrs.get('ancillary_variables', []) if not isinstance(ancillary_variables, (list, tuple, set)): ancillary_variables = ancillary_variables.split(' ') av_ids = [] for key in ancillary_variables: try: av_ids.append(self.get_dataset_key(key)) except KeyError: logger.warning("Can't load ancillary dataset %s", str(key)) all_av_ids |= set(av_ids) dataset.attrs['ancillary_variables'] = av_ids loadable_av_ids = [ av_id for av_id in all_av_ids if av_id not in datasets ] if not all_av_ids: return if loadable_av_ids: self.load(loadable_av_ids, previous_datasets=datasets) for dataset in datasets.values(): new_vars = [] for av_id in dataset.attrs.get('ancillary_variables', []): if isinstance(av_id, DatasetID): new_vars.append(datasets[av_id]) else: new_vars.append(av_id) dataset.attrs['ancillary_variables'] = new_vars def load(self, dataset_keys, previous_datasets=None): """Load `dataset_keys`. If `previous_datasets` is provided, do not reload those.""" all_datasets = previous_datasets or DatasetDict() datasets = DatasetDict() # Include coordinates in the list of datasets to load dsids = [self.get_dataset_key(ds_key) for ds_key in dataset_keys] coordinates = self._get_coordinates_for_dataset_keys(dsids) all_dsids = list(set().union(*coordinates.values())) + dsids for dsid in all_dsids: if dsid in all_datasets: continue coords = [ all_datasets.get(cid, None) for cid in coordinates.get(dsid, []) ] ds = self._load_dataset_with_area(dsid, coords) if ds is not None: all_datasets[dsid] = ds if dsid in dsids: datasets[dsid] = ds self._load_ancillary_variables(all_datasets) return datasets
class User(object): EVENT_CHAT_MESSAGE = 'chat_message' """On new private chat message :param user: steam user :type user: :class:`.SteamUser` :param message: message text :type message: str """ persona_state = EPersonaState.Online #: current persona state user = None #: :class:`.SteamUser` instance once logged on current_games_played = [] #: :class:`list` of app ids currently being played def __init__(self, *args, **kwargs): super(User, self).__init__(*args, **kwargs) self._user_cache = WeakValueDictionary() self.on(self.EVENT_DISCONNECTED, self.__handle_disconnect) self.on(self.EVENT_LOGGED_ON, self.__handle_set_persona) self.on(EMsg.ClientPersonaState, self.__handle_persona_state) self.on(EMsg.ClientFriendMsgIncoming, self.__handle_message_incoming) def __handle_message_incoming(self, msg): if msg.body.chat_entry_type == EChatEntryType.ChatMsg: user = self.get_user(msg.body.steamid_from) self.emit("chat_message", user, msg.body.message.decode('utf-8')) def __handle_disconnect(self): self.user = None self.current_games_played = [] def __handle_set_persona(self): self.user = self.get_user(self.steam_id, False) if self.persona_state != EPersonaState.Offline: self.change_status(persona_state=self.persona_state) def __handle_persona_state(self, message): for friend in message.body.friends: steamid = friend.friendid if steamid in self._user_cache: suser = self._user_cache[steamid] suser._pstate = friend suser._pstate_ready.set() def change_status(self, **kwargs): """ Set name, persona state, flags .. note:: Changing persona state will also change :attr:`persona_state` :param persona_state: persona state (Online/Offlane/Away/etc) :type persona_state: :class:`.EPersonaState` :param player_name: profile name :type player_name: :class:`str` :param persona_state_flags: persona state flags :type persona_state_flags: :class:`.EPersonaStateFlag` """ if not kwargs: return self.persona_state = kwargs.get('persona_state', self.persona_state) message = MsgProto(EMsg.ClientChangeStatus) proto_fill_from_dict(message.body, kwargs) self.send(message) def request_persona_state(self, steam_ids, state_flags=863): """Request persona state data :param steam_ids: list of steam ids :type steam_ids: :class:`list` :param state_flags: client state flags :type state_flags: :class:`.EClientPersonaStateFlag` """ m = MsgProto(EMsg.ClientRequestFriendData) m.body.persona_state_requested = state_flags m.body.friends.extend(steam_ids) self.send(m) def get_user(self, steam_id, fetch_persona_state=True): """Get :class:`.SteamUser` instance for ``steam id`` :param steam_id: steam id :type steam_id: :class:`int`, :class:`.SteamID` :param fetch_persona_state: whether to request person state when necessary :type fetch_persona_state: :class:`bool` :return: SteamUser instance :rtype: :class:`.SteamUser` """ steam_id = int(steam_id) suser = self._user_cache.get(steam_id, None) if suser is None: suser = SteamUser(steam_id, self) self._user_cache[steam_id] = suser if fetch_persona_state: self.request_persona_state([steam_id]) return suser def games_played(self, app_ids): """ Set the apps being played by the user :param app_ids: a list of application ids :type app_ids: :class:`list` These app ids will be recorded in :attr:`current_games_played`. """ if not isinstance(app_ids, list): raise ValueError("Expected app_ids to be of type list") self.current_games_played = app_ids = list(map(int, app_ids)) self.send(MsgProto(EMsg.ClientGamesPlayed), {'games_played': [{'game_id': app_id} for app_id in app_ids]} )
class FileYAMLReader(AbstractYAMLReader): """Primary reader base class that is configured by a YAML file. This class uses the idea of per-file "file handler" objects to read file contents and determine what is available in the file. This differs from the base :class:`AbstractYAMLReader` which does not depend on individual file handler objects. In almost all cases this class should be used over its base class and can be used as a reader by itself and requires no subclassing. """ def __init__(self, config_files, filter_parameters=None, filter_filenames=True, **kwargs): """Set up initial internal storage for loading file data.""" super(FileYAMLReader, self).__init__(config_files) self.file_handlers = {} self.available_ids = {} self.filter_filenames = self.info.get('filter_filenames', filter_filenames) self.filter_parameters = filter_parameters or {} self.coords_cache = WeakValueDictionary() @property def sensor_names(self): """Names of sensors whose data is being loaded by this reader.""" if not self.file_handlers: return self.info['sensors'] file_handlers = (handlers[0] for handlers in self.file_handlers.values()) sensor_names = set() for fh in file_handlers: try: sensor_names.update(fh.sensor_names) except NotImplementedError: continue if not sensor_names: return self.info['sensors'] return sorted(sensor_names) @property def available_dataset_ids(self): """Get DataIDs that are loadable by this reader.""" return self.available_ids.keys() @property def start_time(self): """Start time of the earlier file used by this reader.""" if not self.file_handlers: raise RuntimeError("Start time unknown until files are selected") return min(x[0].start_time for x in self.file_handlers.values()) @property def end_time(self): """End time of the latest file used by this reader.""" if not self.file_handlers: raise RuntimeError("End time unknown until files are selected") return max(x[-1].end_time for x in self.file_handlers.values()) @staticmethod def check_file_covers_area(file_handler, check_area): """Check if the file covers the current area. If the file doesn't provide any bounding box information or 'area' was not provided in `filter_parameters`, the check returns True. """ try: gbb = Boundary(*file_handler.get_bounding_box()) except NotImplementedError as err: logger.debug("Bounding box computation not implemented: %s", str(err)) else: abb = AreaDefBoundary(get_area_def(check_area), frequency=1000) intersection = gbb.contour_poly.intersection(abb.contour_poly) if not intersection: return False return True def find_required_filehandlers(self, requirements, filename_info): """Find the necessary file handlers for the given requirements. We assume here requirements are available. Raises: KeyError, if no handler for the given requirements is available. RuntimeError, if there is a handler for the given requirements, but it doesn't match the filename info. """ req_fh = [] filename_info = set(filename_info.items()) if requirements: for requirement in requirements: for fhd in self.file_handlers[requirement]: if set(fhd.filename_info.items()).issubset(filename_info): req_fh.append(fhd) break else: raise RuntimeError("No matching requirement file of type " "{}".format(requirement)) # break everything and continue to next # filetype! return req_fh def sorted_filetype_items(self): """Sort the instance's filetypes in using order.""" processed_types = [] file_type_items = deque(self.config['file_types'].items()) while len(file_type_items): filetype, filetype_info = file_type_items.popleft() requirements = filetype_info.get('requires') if requirements is not None: # requirements have not been processed yet -> wait missing = [req for req in requirements if req not in processed_types] if missing: file_type_items.append((filetype, filetype_info)) continue processed_types.append(filetype) yield filetype, filetype_info @staticmethod def filename_items_for_filetype(filenames, filetype_info): """Iterate over the filenames matching *filetype_info*.""" if not isinstance(filenames, set): # we perform set operations later on to improve performance filenames = set(filenames) for pattern in filetype_info['file_patterns']: matched_files = set() matches = _match_filenames(filenames, pattern) for filename in matches: try: filename_info = parse( pattern, _get_filebase(filename, pattern)) except ValueError: logger.debug("Can't parse %s with %s.", filename, pattern) continue matched_files.add(filename) yield filename, filename_info filenames -= matched_files def _new_filehandler_instances(self, filetype_info, filename_items, fh_kwargs=None): """Generate new filehandler instances.""" requirements = filetype_info.get('requires') filetype_cls = filetype_info['file_reader'] if fh_kwargs is None: fh_kwargs = {} for filename, filename_info in filename_items: try: req_fh = self.find_required_filehandlers(requirements, filename_info) except KeyError as req: msg = "No handler for reading requirement {} for {}".format( req, filename) warnings.warn(msg) continue except RuntimeError as err: warnings.warn(str(err) + ' for {}'.format(filename)) continue yield filetype_cls(filename, filename_info, filetype_info, *req_fh, **fh_kwargs) def time_matches(self, fstart, fend): """Check that a file's start and end time mtach filter_parameters of this reader.""" start_time = self.filter_parameters.get('start_time') end_time = self.filter_parameters.get('end_time') fend = fend or fstart if start_time and fend and fend < start_time: return False if end_time and fstart and fstart > end_time: return False return True def metadata_matches(self, sample_dict, file_handler=None): """Check that file metadata matches filter_parameters of this reader.""" # special handling of start/end times if not self.time_matches( sample_dict.get('start_time'), sample_dict.get('end_time')): return False for key, val in self.filter_parameters.items(): if key != 'area' and key not in sample_dict: continue if key in ['start_time', 'end_time']: continue elif key == 'area' and file_handler: if not self.check_file_covers_area(file_handler, val): logger.info('Filtering out %s based on area', file_handler.filename) break elif key in sample_dict and val != sample_dict[key]: # don't use this file break else: # all the metadata keys are equal return True return False def filter_filenames_by_info(self, filename_items): """Filter out file using metadata from the filenames. Currently only uses start and end time. If only start time is available from the filename, keep all the filename that have a start time before the requested end time. """ for filename, filename_info in filename_items: fend = filename_info.get('end_time') fstart = filename_info.setdefault('start_time', fend) if fend and fend < fstart: # correct for filenames with 1 date and 2 times fend = fend.replace(year=fstart.year, month=fstart.month, day=fstart.day) filename_info['end_time'] = fend if self.metadata_matches(filename_info): yield filename, filename_info def filter_fh_by_metadata(self, filehandlers): """Filter out filehandlers using provide filter parameters.""" for filehandler in filehandlers: filehandler.metadata['start_time'] = filehandler.start_time filehandler.metadata['end_time'] = filehandler.end_time if self.metadata_matches(filehandler.metadata, filehandler): yield filehandler def filter_selected_filenames(self, filenames): """Filter provided files based on metadata in the filename.""" if not isinstance(filenames, set): # we perform set operations later on to improve performance filenames = set(filenames) for _, filetype_info in self.sorted_filetype_items(): filename_iter = self.filename_items_for_filetype(filenames, filetype_info) if self.filter_filenames: filename_iter = self.filter_filenames_by_info(filename_iter) for fn, _ in filename_iter: yield fn def _new_filehandlers_for_filetype(self, filetype_info, filenames, fh_kwargs=None): """Create filehandlers for a given filetype.""" filename_iter = self.filename_items_for_filetype(filenames, filetype_info) if self.filter_filenames: # preliminary filter of filenames based on start/end time # to reduce the number of files to open filename_iter = self.filter_filenames_by_info(filename_iter) filehandler_iter = self._new_filehandler_instances(filetype_info, filename_iter, fh_kwargs=fh_kwargs) filtered_iter = self.filter_fh_by_metadata(filehandler_iter) return list(filtered_iter) def create_filehandlers(self, filenames, fh_kwargs=None): """Organize the filenames into file types and create file handlers.""" filenames = list(OrderedDict.fromkeys(filenames)) logger.debug("Assigning to %s: %s", self.info['name'], filenames) self.info.setdefault('filenames', []).extend(filenames) filename_set = set(filenames) created_fhs = {} # load files that we know about by creating the file handlers for filetype, filetype_info in self.sorted_filetype_items(): filehandlers = self._new_filehandlers_for_filetype(filetype_info, filename_set, fh_kwargs=fh_kwargs) if filehandlers: created_fhs[filetype] = filehandlers self.file_handlers[filetype] = sorted( self.file_handlers.get(filetype, []) + filehandlers, key=lambda fhd: (fhd.start_time, fhd.filename)) # load any additional dataset IDs determined dynamically from the file # and update any missing metadata that only the file knows self.update_ds_ids_from_file_handlers() return created_fhs def _file_handlers_available_datasets(self): """Generate a series of available dataset information. This is done by chaining file handler's :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets` together. See that method's documentation for more information. Returns: Generator of (bool, dict) where the boolean tells whether the current dataset is available from any of the file handlers. The boolean can also be None in the case where no loaded file handler is configured to load the dataset. The dictionary is the metadata provided either by the YAML configuration files or by the file handler itself if it is a new dataset. The file handler may have also supplemented or modified the information. """ # flatten all file handlers in to one list flat_fhs = (fh for fhs in self.file_handlers.values() for fh in fhs) id_values = list(self.all_ids.values()) configured_datasets = ((None, ds_info) for ds_info in id_values) for fh in flat_fhs: # chain the 'available_datasets' methods together by calling the # current file handler's method with the previous ones result configured_datasets = fh.available_datasets(configured_datasets=configured_datasets) return configured_datasets def update_ds_ids_from_file_handlers(self): """Add or modify available dataset information. Each file handler is consulted on whether or not it can load the dataset with the provided information dictionary. See :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets` for more information. """ avail_datasets = self._file_handlers_available_datasets() new_ids = {} for is_avail, ds_info in avail_datasets: # especially from the yaml config coordinates = ds_info.get('coordinates') if isinstance(coordinates, list): # xarray doesn't like concatenating attributes that are # lists: https://github.com/pydata/xarray/issues/2060 ds_info['coordinates'] = tuple(ds_info['coordinates']) ds_info.setdefault('modifiers', tuple()) # default to no mods # Create DataID for this dataset ds_id = DataID(self._id_keys, **ds_info) # all datasets new_ids[ds_id] = ds_info # available datasets # False == we have the file type but it doesn't have this dataset # None == we don't have the file type object to ask if is_avail: self.available_ids[ds_id] = ds_info self.all_ids = new_ids @staticmethod def _load_dataset(dsid, ds_info, file_handlers, dim='y', **kwargs): """Load only a piece of the dataset.""" slice_list = [] failure = True for fh in file_handlers: try: projectable = fh.get_dataset(dsid, ds_info) if projectable is not None: slice_list.append(projectable) failure = False except KeyError: logger.warning("Failed to load {} from {}".format(dsid, fh), exc_info=True) if failure: raise KeyError( "Could not load {} from any provided files".format(dsid)) if dim not in slice_list[0].dims: return slice_list[0] res = xr.concat(slice_list, dim=dim) combined_info = file_handlers[0].combine_info( [p.attrs for p in slice_list]) res.attrs = combined_info return res def _load_dataset_data(self, file_handlers, dsid, **kwargs): ds_info = self.all_ids[dsid] proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs) # FIXME: areas could be concatenated here # Update the metadata proj.attrs['start_time'] = file_handlers[0].start_time proj.attrs['end_time'] = file_handlers[-1].end_time return proj def _preferred_filetype(self, filetypes): """Get the preferred filetype out of the *filetypes* list. At the moment, it just returns the first filetype that has been loaded. """ if not isinstance(filetypes, list): filetypes = [filetypes] # look through the file types and use the first one that we have loaded for filetype in filetypes: if filetype in self.file_handlers: return filetype return None def _load_area_def(self, dsid, file_handlers, **kwargs): """Load the area definition of *dsid*.""" return _load_area_def(dsid, file_handlers) def _get_coordinates_for_dataset_key(self, dsid): """Get the coordinate dataset keys for *dsid*.""" ds_info = self.all_ids[dsid] cids = [] for cinfo in ds_info.get('coordinates', []): if not isinstance(cinfo, dict): cinfo = {'name': cinfo} for key in self._co_keys: if key == 'name': continue if key in ds_info: if ds_info[key] is not None: cinfo[key] = ds_info[key] cid = DataQuery.from_dict(cinfo) cids.append(self.get_dataset_key(cid)) return cids def _get_coordinates_for_dataset_keys(self, dsids): """Get all coordinates.""" coordinates = {} for dsid in dsids: cids = self._get_coordinates_for_dataset_key(dsid) coordinates.setdefault(dsid, []).extend(cids) return coordinates def _get_file_handlers(self, dsid): """Get the file handler to load this dataset.""" ds_info = self.all_ids[dsid] filetype = self._preferred_filetype(ds_info['file_type']) if filetype is None: logger.warning("Required file type '%s' not found or loaded for " "'%s'", ds_info['file_type'], dsid['name']) else: return self.file_handlers[filetype] def _make_area_from_coords(self, coords): """Create an appropriate area with the given *coords*.""" if len(coords) == 2: lon_sn = coords[0].attrs.get('standard_name') lat_sn = coords[1].attrs.get('standard_name') if lon_sn == 'longitude' and lat_sn == 'latitude': key = None try: key = (coords[0].data.name, coords[1].data.name) sdef = self.coords_cache.get(key) except AttributeError: sdef = None if sdef is None: sdef = SwathDefinition(*coords) sensor_str = '_'.join(self.info['sensors']) shape_str = '_'.join(map(str, coords[0].shape)) sdef.name = "{}_{}_{}_{}".format(sensor_str, shape_str, coords[0].attrs['name'], coords[1].attrs['name']) if key is not None: self.coords_cache[key] = sdef return sdef else: raise ValueError( 'Coordinates info object missing standard_name key: ' + str(coords)) elif len(coords) != 0: raise NameError("Don't know what to do with coordinates " + str( coords)) def _load_dataset_area(self, dsid, file_handlers, coords, **kwargs): """Get the area for *dsid*.""" try: return self._load_area_def(dsid, file_handlers, **kwargs) except NotImplementedError: if any(x is None for x in coords): logger.warning( "Failed to load coordinates for '{}'".format(dsid)) return None area = self._make_area_from_coords(coords) if area is None: logger.debug("No coordinates found for %s", str(dsid)) return area def _load_dataset_with_area(self, dsid, coords, **kwargs): """Load *dsid* and its area if available.""" file_handlers = self._get_file_handlers(dsid) if not file_handlers: return area = self._load_dataset_area(dsid, file_handlers, coords, **kwargs) try: ds = self._load_dataset_data(file_handlers, dsid, **kwargs) except (KeyError, ValueError) as err: logger.exception("Could not load dataset '%s': %s", dsid, str(err)) return None if area is not None: ds.attrs['area'] = area ds = add_crs_xy_coords(ds, area) return ds def _load_ancillary_variables(self, datasets, **kwargs): """Load the ancillary variables of `datasets`.""" all_av_ids = set() for dataset in datasets.values(): ancillary_variables = dataset.attrs.get('ancillary_variables', []) if not isinstance(ancillary_variables, (list, tuple, set)): ancillary_variables = ancillary_variables.split(' ') av_ids = [] for key in ancillary_variables: try: av_ids.append(self.get_dataset_key(key)) except KeyError: logger.warning("Can't load ancillary dataset %s", str(key)) all_av_ids |= set(av_ids) dataset.attrs['ancillary_variables'] = av_ids loadable_av_ids = [av_id for av_id in all_av_ids if av_id not in datasets] if not all_av_ids: return if loadable_av_ids: self.load(loadable_av_ids, previous_datasets=datasets, **kwargs) for dataset in datasets.values(): new_vars = [] for av_id in dataset.attrs.get('ancillary_variables', []): if isinstance(av_id, DataID): new_vars.append(datasets[av_id]) else: new_vars.append(av_id) dataset.attrs['ancillary_variables'] = new_vars def get_dataset_key(self, key, available_only=False, **kwargs): """Get the fully qualified `DataID` matching `key`. This will first search through available DataIDs, datasets that should be possible to load, and fallback to "known" datasets, those that are configured but aren't loadable from the provided files. Providing ``available_only=True`` will stop this fallback behavior and raise a ``KeyError`` exception if no available dataset is found. Args: key (str, float, DataID, DataQuery): Key to search for in this reader. available_only (bool): Search only loadable datasets for the provided key. Loadable datasets are always searched first, but if ``available_only=False`` (default) then all known datasets will be searched. kwargs: See :func:`satpy.readers.get_key` for more information about kwargs. Returns: Best matching DataID to the provided ``key``. Raises: KeyError: if no key match is found. """ try: return get_key(key, self.available_ids.keys(), **kwargs) except KeyError: if available_only: raise return get_key(key, self.all_ids.keys(), **kwargs) def load(self, dataset_keys, previous_datasets=None, **kwargs): """Load `dataset_keys`. If `previous_datasets` is provided, do not reload those. """ all_datasets = previous_datasets or DatasetDict() datasets = DatasetDict() # Include coordinates in the list of datasets to load dsids = [self.get_dataset_key(ds_key) for ds_key in dataset_keys] coordinates = self._get_coordinates_for_dataset_keys(dsids) all_dsids = list(set().union(*coordinates.values())) + dsids for dsid in all_dsids: if dsid in all_datasets: continue coords = [all_datasets.get(cid, None) for cid in coordinates.get(dsid, [])] ds = self._load_dataset_with_area(dsid, coords, **kwargs) if ds is not None: all_datasets[dsid] = ds if dsid in dsids: datasets[dsid] = ds self._load_ancillary_variables(all_datasets, **kwargs) return datasets
class PresenceService(ExportedGObject): __gtype_name__ = "PresenceService" def _create_owner(self): # Overridden by TestPresenceService return ShellOwner(self, self._session_bus) def __init__(self): self._next_object_id = 0 # all Buddy objects # identifier -> Buddy, GC'd when no more refs exist self._buddies = WeakValueDictionary() # the online buddies for whom we know the full public key # base64 public key -> Buddy self._buddies_by_pubkey = {} # The online buddies (those who're available via some CM) # TP plugin -> (handle -> Buddy) self._handles_buddies = {} # activity id -> Activity self._activities_by_id = {} #: Tp plugin -> (handle -> Activity) self._activities_by_handle = {} #: Connection -> list of SignalMatch self._conn_matches = {} self._session_bus = dbus.SessionBus() self._session_bus.add_signal_receiver(self._connection_disconnected_cb, signal_name="Disconnected", dbus_interface="org.freedesktop.DBus") # Create the Owner object self._owner = self._create_owner() key = self._owner.props.key keyid = pubkey_to_keyid(key) self._buddies['keyid/' + keyid] = self._owner self._buddies_by_pubkey[key] = self._owner self._registry = ManagerRegistry() self._registry.LoadManagers() # Set up the Telepathy plugins self._plugins = [] debug_flags = set(environ.get('PRESENCE_SERVICE_DEBUG', '').split(',')) _logger.debug('Debug flags: %r', debug_flags) if 'disable-gabble' in debug_flags: self._server_plugin = None else: server = self._owner.get_server() if server and len(server): self._server_plugin = ServerPlugin(self._registry, self._owner) self._plugins.append(self._server_plugin) else: self._server_plugin = None if 'disable-salut' in debug_flags: self._ll_plugin = None else: self._ll_plugin = LinkLocalPlugin(self._registry, self._owner) self._plugins.append(self._ll_plugin) self._connected_plugins = set() for tp in self._plugins: self._handles_buddies[tp] = {} self._activities_by_handle[tp] = {} tp.connect('status', self._tp_status_cb) tp.connect('contacts-online', self._contacts_online) tp.connect('contacts-offline', self._contacts_offline) tp.connect('activity-invitation', self._activity_invitation) tp.connect('private-invitation', self._private_invitation) tp.connect('want-to-connect', self._want_to_connect) connection = tp.get_connection() if connection is not None: status = connection.GetStatus() self._tp_status_cb(tp, status, CONNECTION_STATUS_REASON_NONE_SPECIFIED) self._contacts_online_queue = [] ExportedGObject.__init__(self, self._session_bus, _PRESENCE_PATH) # for activation to work in a race-free way, we should really # export the bus name only after we export our initial object; # so this comes after the parent __init__ self._bus_name = dbus.service.BusName(_PRESENCE_SERVICE, bus=self._session_bus) @property def owner(self): return self._owner def _connection_disconnected_cb(self, data=None): """Log event when D-Bus kicks us off the bus for some reason""" _logger.debug("Disconnected from session bus!!!") def _tp_status_cb(self, plugin, status, reason): if status == CONNECTION_STATUS_CONNECTED: self._tp_connected(plugin) else: self._tp_disconnected(plugin) def _tp_connected(self, tp): self._connected_plugins.add(tp) self._handles_buddies[tp][tp.self_handle] = self._owner self._owner.add_telepathy_handle(tp, tp.self_handle, tp.self_identifier) conn = tp.get_connection() self._conn_matches[conn] = [] if CONN_INTERFACE_ACTIVITY_PROPERTIES in conn: def activity_properties_changed(room, properties): self._activity_properties_changed(tp, room, properties) m = conn[CONN_INTERFACE_ACTIVITY_PROPERTIES].connect_to_signal( 'ActivityPropertiesChanged', activity_properties_changed) self._conn_matches[conn].append(m) else: _logger.warning('Connection %s does not support OLPC activity ' 'properties', conn.object_path) if CONN_INTERFACE_BUDDY_INFO in conn: def buddy_activities_changed(contact, activities): _logger.debug('ActivitiesChanged on %s: (%u, %r)', tp, contact, activities) self._buddy_activities_changed(tp, contact, activities) m = conn[CONN_INTERFACE_BUDDY_INFO].connect_to_signal( 'ActivitiesChanged', buddy_activities_changed) self._conn_matches[conn].append(m) def buddy_properties_changed(contact, properties): buddy = self._handles_buddies[tp].get(contact) if buddy is not None and buddy is not self._owner: buddy.update_buddy_properties(tp, properties) m = conn[CONN_INTERFACE_BUDDY_INFO].connect_to_signal( 'PropertiesChanged', buddy_properties_changed, byte_arrays=True) self._conn_matches[conn].append(m) def buddy_curact_changed(contact, act_id, room): if (act_id == '' or not util.validate_activity_id(act_id) or room == 0): act_id = '' room = 0 buddy = self._handles_buddies[tp].get(contact) if buddy is not None and buddy is not self._owner: buddy.update_current_activity(tp, act_id) # FIXME: do something useful with the room handle? m = conn[CONN_INTERFACE_BUDDY_INFO].connect_to_signal( 'CurrentActivityChanged', buddy_curact_changed) self._conn_matches[conn].append(m) else: _logger.warning('Connection %s does not support OLPC buddy info', conn.object_path) if 1: # FIXME: Avatars have been disabled for Trial-2 due to performance # issues in the avatar cache. Revisit this afterwards pass elif CONN_INTERFACE_AVATARS in conn: def avatar_retrieved(contact, avatar_token, avatar, mime_type): self._avatar_updated(tp, contact, avatar_token, avatar, mime_type) m = conn[CONN_INTERFACE_AVATARS].connect_to_signal( 'AvatarRetrieved', avatar_retrieved) self._conn_matches[conn].append(m) def avatar_updated(contact, avatar_token): self._avatar_updated(tp, contact, avatar_token) m = conn[CONN_INTERFACE_AVATARS].connect_to_signal('AvatarUpdated', avatar_updated) self._conn_matches[conn].append(m) else: _logger.warning('Connection %s does not support avatars', conn.object_path) if CONN_INTERFACE_ALIASING in conn: def aliases_changed(aliases): for contact, alias in aliases: buddy = self._handles_buddies[tp].get(contact) if buddy is not None and buddy is not self._owner: buddy.update_alias(tp, alias) m = conn[CONN_INTERFACE_ALIASING].connect_to_signal( 'AliasesChanged', aliases_changed) self._conn_matches[conn].append(m) else: _logger.warning('Connection %s does not support aliasing', conn.object_path) def _tp_disconnected(self, tp): self._connected_plugins.discard(tp) if tp.self_handle is not None: self._handles_buddies.setdefault(tp, {}).pop( tp.self_handle, None) self._owner.remove_telepathy_handle(tp) conn = tp.get_connection() matches = self._conn_matches.get(conn) try: del self._conn_matches[conn] except KeyError: pass if matches is not None: for match in matches: match.remove() def get_buddy_by_path(self, path): """Get the Buddy object corresponding to an object-path, or None. :Parameters: path : dbus.ObjectPath The object-path of a buddy :Returns: a Buddy object or None """ if not path.startswith(BUDDY_PATH): return None return self._buddies.get(path[len(BUDDY_PATH):]) def get_buddy(self, objid): buddy = self._buddies.get(objid) if buddy is None: _logger.debug('Creating new buddy at .../%s', objid) # we don't know yet this buddy buddy = Buddy(self._session_bus, objid) buddy.connect("validity-changed", self._buddy_validity_changed_cb) buddy.connect("disappeared", self._buddy_disappeared_cb) self._buddies[objid] = buddy return buddy def _contacts_online(self, tp, objids, handles, identifiers): # we'll iterate over handles many times, so make sure that will # work if not isinstance(handles, (list, tuple)): handles = tuple(handles) for objid, handle, identifier in izip(objids, handles, identifiers): _logger.debug('Handle %u, .../%s is now online', handle, objid) buddy = self.get_buddy(objid) self._handles_buddies[tp][handle] = buddy # Store the handle of the buddy for this CM. This doesn't # fetch anything over D-Bus, to avoid reaching the pending-call # limit. buddy.add_telepathy_handle(tp, handle, identifier) conn = tp.get_connection() if not self._contacts_online_queue: gobject.idle_add(self._run_contacts_online_queue) def handle_error(e, when): gobject.idle_add(self._run_contacts_online_queue) _logger.warning('Error %s: %s', when, e) if CONN_INTERFACE_ALIASING in conn: def got_aliases(aliases): gobject.idle_add(self._run_contacts_online_queue) for contact, alias in izip(handles, aliases): buddy = self._handles_buddies[tp].get(contact) if buddy is not None and buddy is not self._owner: buddy.update_alias(tp, alias) def request_aliases(): try: conn[CONN_INTERFACE_ALIASING].RequestAliases(handles, reply_handler=got_aliases, error_handler=lambda e: handle_error(e, 'fetching aliases')) except Exception, e: gobject.idle_add(self._run_contacts_online_queue) handle_error(e, 'fetching aliases') self._contacts_online_queue.append(request_aliases) for handle in handles: self._queue_contact_online(tp, handle) if 1: # FIXME: Avatars have been disabled for Trial-2 due to performance # issues in the avatar cache. Revisit this afterwards pass elif CONN_INTERFACE_AVATARS in conn: def got_avatar_tokens(tokens): gobject.idle_add(self._run_contacts_online_queue) for contact, token in izip(handles, tokens): self._avatar_updated(tp, contact, token) def get_avatar_tokens(): try: conn[CONN_INTERFACE_AVATARS].GetAvatarTokens(handles, reply_handler=got_avatar_tokens, error_handler=lambda e: handle_error(e, 'fetching avatar tokens')) except Exception, e: gobject.idle_add(self._run_contacts_online_queue) handle_error(e, 'fetching avatar tokens') self._contacts_online_queue.append(get_avatar_tokens)
class Service(object): """I manage a set of related :class:`ILocalCore`'s. All the resources in a service are stored in the same `rdflib.store.Store`:class:. :param classes: a list of classes to be used by this service (see below) :param service_config: kTBS configuration :param init_with: a callable to initialize the store if necessary (i.e. at least populate the root resource); it will be passed this service as its sole argument. root_uri (str), the URI of the root resource of this service store (rdflib.store.Store), the RDF store containing the data of this service init_with, a callable to initialize the store if necessary (i.e. at least populate the root resource); it will be passed this service as its sole argument. The classes passed to this service should all be subclasses of :class:`ILocalCore`, and all have an attribute `RDF_MAIN_TYPE` indicating the RDF type they implement. """ # too few public methods (1/2) #pylint: disable=R0903 def __init__(self, classes, service_config=None, init_with=None): """I create a local RDF-REST service around the given store. """ if service_config is None: service_config = get_service_configuration() self.config = service_config root_uri = build_service_root_uri(service_config) assert urisplit(root_uri)[3:] == (None, None), \ "Invalid URI <%s>" % root_uri self.root_uri = coerce_to_uri(root_uri) apply_logging_config(service_config) init_repo = False repository = service_config.get('rdf_database', 'repository', 1) if not repository: init_repo = True repository = ":IOMemory:" elif repository[0] != ":": init_repo = not exists(repository) repository = ":Sleepycat:%s" % repository # Whether we should force data repository initialization if service_config.getboolean('rdf_database', 'force-init'): init_repo = True _, store_type, config_str = repository.split(":", 2) store = rdflib_plugin.get(store_type, Store)(config_str) self.store = store self.class_map = class_map = {} for cls in classes: assert issubclass(cls, ILocalCore) assert cls.RDF_MAIN_TYPE not in class_map, \ "duplicate RDF_MAIN_TYPE <%s>" % cls.RDF_MAIN_TYPE class_map[cls.RDF_MAIN_TYPE] = cls # about self._resource_cache: this is not per se a cache, # but ensures that we will not generate multiple instances for the # same resource. self._resource_cache = WeakValueDictionary() self._context_level = 0 metadata_graph = self.get_metadata_graph(root_uri) initialized = list(metadata_graph.triples((self.root_uri, NS.hasImplementation, None))) if not initialized and init_repo: assert init_with, \ "Store is not initialized, and no initializer was provided" init_with(self) assert (list(metadata_graph.triples((self.root_uri, NS.hasImplementation, None)))) # correctly init'ed register_service(self) def __del__(self): try: unregister_service(self) except BaseException: pass @HostedCore.handle_fragments def get(self, uri, rdf_types=None, _no_spawn=False): """Get a resource from this service. :param uri: the URI of the resource :type uri: :class:`~rdflib.URIRef` :param rdf_types: if provided, a list of expected RDF types of the resource :type rdf_types: list of :class:`rdflib.term.URIRef` :param _no_spawn: if True, only *pre-existing* python objects will be returned :type _no_spawn: bool :return: the resource, or None :rtype: :class:`ILocalCore` or :class:`~.cores.hosted.HostedCore` TODO NOW: if no resource is found, try to get it from parent resource NB: if uri contains a fragment-id, the returned resource will be a `~rdfrest.cores.hosted.HostedCore`:class: hosted by a resource from this service. When using this function, it is a good practice to indicate the expected return type, either informally (with a comment) or formally, with a statement of the form:: assert isinstance(returned_object, expected_class) """ assert isinstance(uri, URIRef) assert rdf_types is None or isinstance(rdf_types, list) querystr, fragid = urisplit(uri)[3:] if querystr is not None or fragid is not None: # fragid is managed by the decorator HostedCore.handle_fragment return None resource = self._resource_cache.get(uri) if resource is None and not _no_spawn: # find base rdf:type metadata = self.get_metadata_graph(uri) if len(metadata) == 0: return None types = list( metadata.objects(uri, NS.hasImplementation)) assert len(types) == 1, types typ = types[0] # find base python class and wrap it py_class = self.class_map.get(typ) if py_class is None: raise ValueError("No implementation for type <%s> of <%s>" % (types[0], uri)) if rdf_types: py_class = get_wrapped(py_class, rdf_types) # make resource and store it in "cache" resource = py_class(self, uri) self._resource_cache[uri] = resource return resource def get_metadata_graph(self, uri): """Return the metadata graph for the resource identified by uri :param uri: the URI of the resou :return: the metadata graph :rtype: :class:`rdflib.graph.Graph` """ return Graph(self.store, URIRef(uri + '#metadata')) def __enter__(self): """Start to modifiy this service. The role of using a service as a context is to ensure that data is correctly handled by the underlying RDF store: * on normal exit, :meth:`self.store.commit <rdflib.store.Store.commit>` will be called. * if an exception is raised, :meth:`self.store.rollback <rdflib.store.Store.rollback>` will be called. * if several contexts are embeded (e.g. by calling a function that itself uses the service context), the commit/rollback will only occur when exiting the *outermost* context, ensuring that only globally consistent states are commited. Note that the implementations provided in this module already take care of using the service context, so implementors relying them should not have to worry about it. It may be necessary to explicitly call the service context, though, to make a set of resource modifications aromic. .. warning:: For the moment (2012-07), most implementations of :class:`rdflib.store.Store` do not support rollback (which simply does nothing). So unless you know for sure that the store you are using does support rollback, you should assume that the store is corrupted when exiting abnormally from the service context. """ if self._context_level == 0 and self.store.transaction_aware: self.store.transaction() self._context_level += 1 def __exit__(self, typ, _value, _traceback): """Ends modifications to this service. """ level = self._context_level - 1 self._context_level = level if level == 0: if typ is None: self.store.commit() else: self.store.rollback() # we rollback *in case* the store supports it, # to try to restore it in a consistent state. # However there is no guarantee that this work, # as not all stores support rollback. # This is therefore a best-effort to limit damages, # rather than a safe handling of the exception # (at least, until all stores support rollback). return False
class base: """ Maintains the cache information about eclasses available to an ebuild. """ def __init__(self, location=None, eclassdir=None): self._eclass_data_inst_cache = WeakValueDictionary() # generate this. # self.eclasses = {} # {"Name": ("location", "_mtime_")} self.location = location self.eclassdir = eclassdir def get_eclass_data(self, inherits): """Return the cachable entries from a list of inherited eclasses. Only make get_eclass_data calls for data you know came from this eclass_cache, otherwise be ready to catch a KeyError exception for any eclass that was requested, but not known to this cache. """ keys = OrderedFrozenSet(inherits) o = self._eclass_data_inst_cache.get(keys) if o is None: o = ImmutableDict((k, self.eclasses[k]) for k in keys) self._eclass_data_inst_cache[keys] = o return o def get_eclass(self, eclass): o = self.eclasses.get(eclass) if o is None: return None return local_source(o.path) eclasses = jit_attr_ext_method("_load_eclasses", "_eclasses") def rebuild_cache_entry(self, entry_eclasses): """Check if eclass data is still valid. Given a dict as returned by get_eclass_data, walk it comparing it to internal eclass view. :return: a boolean representing whether that eclass data is still up to date, or not """ ec = self.eclasses d = {} for eclass, chksums in entry_eclasses: data = ec.get(eclass) if any(val != getattr(data, chf, None) for chf, val in chksums): return None d[eclass] = data return d def __getstate__(self): d = self.__dict__.copy() del d['_eclass_data_inst_cache'] return d def __setstate__(self, state): self.__dict__ = state.copy() self.__dict__['_eclass_data_inst_cache'] = WeakValueDictionary()
class _WeakValueDictionary(object): # Maps from OID -> Persistent object, but # only weakly references the Persistent object. This is similar # to ``weakref.WeakValueDictionary``, but is customized depending on the # platform. On PyPy, all objects can cheaply use a WeakRef, so that's # what we actually use. On CPython, though, ``PersistentPy`` cannot be weakly # referenced, so we rely on the fact that the ``id()`` of an object is its # memory location, and we use ``ctypes`` to cast that integer back to # the object. # # To remove stale addresses, we rely on the ``ffi.gc()`` object with the exact # same lifetime as the ``PersistentPy`` object. It calls us, we get the ``id`` # back out of the CData, and clean up. if PYPY: # pragma: no cover def __init__(self): self._data = WeakValueDictionary() def _from_addr(self, addr): return addr def _save_addr(self, oid, obj): return obj cleanup_hook = None else: def __init__(self): # careful not to require ctypes at import time; most likely the # C implementation is in use. import ctypes self._data = {} self._addr_to_oid = {} self._cast = ctypes.cast self._py_object = ctypes.py_object def _save_addr(self, oid, obj): i = id(obj) self._addr_to_oid[i] = oid return i def _from_addr(self, addr): return self._cast(addr, self._py_object).value def cleanup_hook(self, cdata): # This is called during GC, possibly at interpreter shutdown # when the __dict__ of this object may have already been cleared. try: addr_to_oid = self._addr_to_oid except AttributeError: return oid = addr_to_oid.pop(cdata.pobj_id, None) self._data.pop(oid, None) def __contains__(self, oid): return oid in self._data def __len__(self): return len(self._data) def __setitem__(self, key, value): addr = self._save_addr(key, value) self._data[key] = addr def pop(self, oid): return self._from_addr(self._data.pop(oid)) def items(self): from_addr = self._from_addr for oid, addr in self._data.items(): yield oid, from_addr(addr) def get(self, oid, default=None): addr = self._data.get(oid, self) if addr is self: return default return self._from_addr(addr) def __getitem__(self, oid): addr = self._data[oid] return self._from_addr(addr)
class CacheBlock (MutableMapping,HtmlPlugin): r""" Instances of this class implements blocks of cells sharing the same functor. :param db: specification of the cache repository where the block resides :param functor: functor of the block :type functor: :class:`Functor` :param cacheonly: if :const:`True`, cell creation is disallowed :type cacheonly: :class:`bool` A :class:`CacheBlock` instance is callable, and calls take a single argument. Method :meth:`__call__` implements the cross-process cacheing mechanism which produces and reuses cache cells. It also implements a weak cache for local calls (within its process). Furthermore, a :class:`CacheBlock` instance acts as a mapping where the keys are cell identifiers (:class:`int`) and values are tuples of meta-information about the cells (i.e. not the values of the cells: these are only accessible through calling). Finally, :class:`CacheBlock` instances have an HTML ipython display. Attributes: .. attribute:: db the :class:`CacheDB` instance this block belongs to .. attribute:: functor the functor for this block (field ``functor`` in the ``Block`` table of the index is the functor's pickle) .. attribute:: block the :class:`int` identifier of this block (field ``oid`` in the ``Block`` table of the index) .. attribute:: cacheonly whether cell creation is disabled .. attribute:: memory a :class:`weakref.WeakValueDictionary` implementing a local cache of calls within the current process Methods: .. automethod:: __call__ """ #================================================================================================== def __init__(self,db=None,functor=None,block=None,cacheonly=False): self.db = db = CacheDB(db) self.functor = functor self.block = db.getblock(functor) if block is None else block self.cacheonly = cacheonly self.memory = WeakValueDictionary() def __hash__(self): return hash((self.db,self.block)) def __eq__(self,other): return isinstance(other,CacheBlock) and self.db is other.db and self.block == other.block def clear_error(self,dry_run=False): r""" Clears all the cells from this block which cache an exception. """ with self.db.connect() as conn: if dry_run: return [cell for cell, in conn.execute('SELECT oid FROM Cell WHERE block=? AND size<0',(self.block,))] conn.execute('DELETE FROM Cell WHERE block=? AND size<0',(self.block,)) deleted = conn.total_changes if deleted>0: logger.info('%s DELETED(%s)',self,deleted) return deleted def clear_overflow(self,n,dry_run=False): r""" Clears all the cells from this block except the *n* most recent (lru policy). """ assert isinstance(n,int) and n>=1 with self.db.connect() as conn: if dry_run: return [cell for cell, in conn.execute('SELECT oid FROM Cell WHERE block=? AND size>0 ORDER BY tstamp DESC, oid DESC LIMIT -1 OFFSET ?',(self.block,n))] conn.execute('DELETE FROM Cell WHERE oid IN (SELECT oid FROM Cell WHERE block=? AND size>0 ORDER BY tstamp DESC, oid DESC LIMIT -1 OFFSET ?)',(self.block,n)) deleted = conn.total_changes if deleted>0: logger.info('%s DELETED(%s)',self,deleted) return deleted def info(self,typ=namedtuple('BlockInfo',('hits','ncell','ncell_error','ncell_pending'))): r""" Returns information about this block. Available attributes: :attr:`hits`, :attr:`ncell`, :attr:`ncell_error`, :attr:`ncell_pending` """ with self.db.connect(detect_types=sqlite3.PARSE_DECLTYPES) as conn: ncell = dict(conn.execute('SELECT CASE WHEN size ISNULL THEN \'pending\' WHEN size<0 THEN \'error\' ELSE \'\' END AS status, count(*) FROM Cell WHERE block=? GROUP BY status',(self.block,))) hits, = conn.execute('SELECT sum(hits) FROM Cell WHERE block=?',(self.block,)).fetchone() return typ((hits or 0),sum(ncell.values()),*(ncell.get(k,0) for k in ('error','pending'))) #-------------------------------------------------------------------------------------------------- def __call__(self,arg): """ :param arg: argument of the call Implements cacheing as follows: - Method :meth:`getkey` of the functor is invoked with argument *arg* to obtain a ``ckey``. - If that ``ckey`` is present in the (local) memory mapping of this block, its associated value is returned. - Otherwise, a transaction is begun on the index database. - If there already exists a cell with the same ``ckey``, method :meth:`lookup` of the storage is invoked to obtain a getter for that cell, then the transaction is terminated and the result is extracted, using the obtained getter. The cell's hit count is incremented. - If there does not exist a cell with the same ``ckey``, a cell with that ``ckey`` is created, and method :meth:`insert` of the storage is invoked to obtain a setter for that cell, then the transaction is terminated. Then, method :meth:`getval` of the functor is invoked with argument *arg* and its result is stored, even if it is an exception, using the obtained setter. - If the result is an exception, it is raised. - Otherwise, the memory mapping of this block is updated at key ``ckey`` with the result (if possible), and the result is returned. """ #-------------------------------------------------------------------------------------------------- ckey = self.functor.getkey(arg) cval = self.memory.get(ckey) if cval is not None: return cval with self.db.connect() as conn: conn.execute('BEGIN IMMEDIATE TRANSACTION') row = conn.execute('SELECT oid,size FROM Cell WHERE block=? AND ckey=?',(self.block,ckey)).fetchone() if row is None: if self.cacheonly: raise Exception('Cache cell creation disallowed') cell = conn.execute('INSERT INTO Cell (block,ckey) VALUES (?,?)',(self.block,ckey)).lastrowid setval = self.db.storage.insert(cell) else: cell,size = row getval = self.db.storage.lookup(cell,size==0) if row is None: logger.info('%s MISS(%s)',self,cell) tm = process_time(),perf_counter() try: cval = self.functor.getval(arg) except BaseException as e: cval = e; size = -1 else: size = 1 tm = process_time()-tm[0],perf_counter()-tm[1] try: size *= setval(cval) except: with self.db.connect() as conn: conn.execute('DELETE FROM Cell WHERE oid=?',(cell,)) raise with self.db.connect() as conn: conn.execute('UPDATE Cell SET size=?, tprc=?, ttot=?, tstamp=datetime(\'now\') WHERE oid=?',(size,tm[0],tm[1],cell)) if not conn.total_changes: logger.info('%s LOST(%s)',self,cell) if size<0: raise cval else: if size==0: logger.info('%s WAIT(%s)',self,cell) cval = getval() logger.info('%s HIT(%s)',self,cell) with self.db.connect() as conn: conn.execute('UPDATE Cell SET hits=hits+1, tstamp=datetime(\'now\') WHERE oid=?',(cell,)) if isinstance(cval,BaseException): raise cval try: self.memory[ckey] = cval except: pass return cval #-------------------------------------------------------------------------------------------------- # CacheBlock as Mapping #-------------------------------------------------------------------------------------------------- def __getitem__(self,cell): with self.db.connect() as conn: r = conn.execute('SELECT ckey, tstamp, hits, size, tprc, ttot FROM Cell WHERE oid=?',(cell,)).fetchone() if r is None: raise KeyError(cell) return r def __delitem__(self,cell): with self.db.connect() as conn: conn.execute('DELETE FROM Cell WHERE oid=?',(cell,)) if not conn.total_changes: raise KeyError(cell) def __setitem__(self,cell,v): raise Exception('Direct create/update not permitted on Cell') def __iter__(self): with self.db.connect() as conn: for cell, in conn.execute('SELECT oid FROM Cell WHERE block=?',(self.block,)): yield cell def __len__(self): with self.db.connect() as conn: return conn.execute('SELECT count(*) FROM Cell WHERE block=?',(self.block,)).fetchone()[0] def items(self): with self.db.connect() as conn: for row in conn.execute('SELECT oid, ckey, tstamp, hits, size, tprc, ttot FROM Cell WHERE block=?',(self.block,)): yield row[0],row[1:] def clear(self): with self.db.connect() as conn: conn.execute('DELETE FROM Cell WHERE block=?',(self.block,)) #-------------------------------------------------------------------------------------------------- # Display #-------------------------------------------------------------------------------------------------- def as_html(self,incontext,size_fmt_=(lambda sz: '*'+size_fmt(-sz) if sz<0 else size_fmt(sz)),time_fmt_=(lambda t: '' if t is None else time_fmt(t))): n = len(self)-self._html_limit L = self.items(); closing = None if n>0: L = islice(L,self._html_limit); closing = '{} more'.format(n) return html_table(sorted(L),hdrs=('ckey','tstamp','hits','size','tprc','ttot'),fmts=((lambda ckey,h=self.functor.html: h(ckey,incontext)),str,str,size_fmt_,time_fmt_,time_fmt_),opening='{}: {}'.format(self.block,self.functor),closing=closing) def __repr__(self): return 'Cache<{}:{}>'.format(self.db.path,self.functor)
class LargeCache: '''File-backed cache for large objects (particularly images). Uses a WeakValueDictionary for in-memory storage, and a page-based format for on-disk.''' def __init__(self, data_file, index_file): # Dict storing currently loaded values self.cache = WeakValueDictionary() # Data file, and index mapping key to data offset self.data_file = data_file self.data_total = os.fstat(data_file.fileno()).st_size self.index_file = index_file self.index = {} self.used = set() # Read index data into dict self.index_file.seek(0) for line in self.index_file.readlines(): parts = line.strip().split() if len(parts) != 2: continue offset_str, key_str = parts offset = int(offset_str) key = pickle.loads(base64.decodebytes(key_str.encode('ascii'))) self.index[key] = offset # Seek both to EOF self.data_file.seek(0, os.SEEK_END) self.index_file.seek(0, os.SEEK_END) def contains(self, key): return key in self.cache or key in self.index def get(self, key): # Record the key use here, regardless of the outcome. This assumes the # caller runs `add` only on keys for which it first ran `get`. self.used.add(key) # Try to fetch from in-memory cache value = self.cache.get(key) if value is not None: return value # Try to load from file if key in self.index: offset = self.index[key] self.data_file.seek(offset) value = _safe_load(self.data_file) self.cache[key] = value return value # No cached copy of this image return None def add(self, key, value): # Add to in-memory cache self.cache[key] = value # Write pickled value to next available page page = CACHE_PAGE offset = (self.data_total + page - 1) & ~(page - 1) self.data_file.seek(offset) _safe_dump(value, self.data_file) self.data_total = self.data_file.tell() # Write index line self.index[key] = offset key_str = base64.encodebytes(pickle.dumps(key)).decode('ascii') self.index_file.write('%d %s\n' % (offset, key_str.replace('\n', ''))) def size(self): return len(self.index) def save(self): pass
class TaskQueue(AbstractTaskQueue): """Simple in-memory task queue implementation""" @classmethod def factory(cls, url, name=const.DEFAULT, *args, **kw): obj = _REFS.get((url, name)) if obj is None: obj = _REFS[(url, name)] = cls(url, name, *args, **kw) return obj def __init__(self, *args, **kw): super(TaskQueue, self).__init__(*args, **kw) self.queue = Queue() self.results = WeakValueDictionary() self.results_lock = Lock() def _init_result(self, result, status, message): with self.results_lock: if result.id in self.results: return False self.results[result.id] = result result.__status = status result.__value = Queue() result.__task = message result.__args = {} result.__lock = Lock() result.__for = None return True def enqueue_task(self, result, message): if self._init_result(result, const.ENQUEUED, message): self.queue.put(result) return True return False def defer_task(self, result, message, args): if self._init_result(result, const.PENDING, message): results = self.results # keep references to results to prevent GC result.__refs = [results.get(arg) for arg in args] return True return False def undefer_task(self, task_id): result = self.results[task_id] self.queue.put(result) def get(self, timeout=None): try: result = self.queue.get(timeout=timeout) except Empty: return None result.__status = const.PROCESSING return result.id, result.__task def size(self): return len(self.results) def discard_pending(self): with self.results_lock: while True: try: self.queue.get_nowait() except Empty: break self.results.clear() def reserve_argument(self, argument_id, deferred_id): result = self.results.get(argument_id) if result is None: return (False, None) with result.__lock: if result.__for is not None: return (False, None) result.__for = deferred_id try: message = result.__value.get_nowait() except Empty: message = None if message is not None: with self.results_lock: self.results.pop(argument_id, None) return (True, message) def set_argument(self, task_id, argument_id, message): result = self.results[task_id] with self.results_lock: self.results.pop(argument_id, None) with result.__lock: result.__args[argument_id] = message return len(result.__args) == len(result.__refs) def get_arguments(self, task_id): try: return self.results[task_id].__args except KeyError: return {} def set_task_timeout(self, task_id, timeout): pass def get_status(self, task_id): result = self.results.get(task_id) return None if result is None else result.__status def set_result(self, task_id, message, timeout): result = self.results.get(task_id) if result is not None: with result.__lock: result.__value.put(message) return result.__for def pop_result(self, task_id, timeout): result = self.results.get(task_id) if result is None: return const.TASK_EXPIRED # with result.__lock: # if result.__for is not None: # raise NotImplementedError # #return const.RESERVED # result.__for = task_id try: if timeout == 0: value = result.__value.get_nowait() else: value = result.__value.get(timeout=timeout) except Empty: value = None else: self.results.pop(task_id, None) return value def discard_result(self, task_id, task_expired_token): result = self.results.pop(task_id) if result is not None: result.__value.put(task_expired_token)
class TaurusFactory(object): """The base class for valid Factories in Taurus.""" schemes = () # reimplement in derived classes to declare supported schemes caseSensitive = True # reimplement if your scheme is case insensitive elementTypesMap = None # reimplement in derived classes to profit from # generic implementations of getAuthority, # getDevice, getAttribute, findObjectClass, etc. # see findObjectClass for more details DefaultPollingPeriod = 3000 def __init__(self): atexit.register(self.cleanUp) self._polling_period = self.DefaultPollingPeriod self.polling_timers = {} self._polling_enabled = True self._attrs = WeakValueDictionary() self._devs = WeakValueDictionary() self._auths = WeakValueDictionary() from . import taurusmanager manager = taurusmanager.TaurusManager() self._serialization_mode = manager.getSerializationMode() #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # API for cleanUp at exit #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def cleanUp(self): """Reimplement if you need to execute code on program execution exit. Default implementation does nothing. """ pass #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # API for serialization #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def setSerializationMode(self, mode): """Sets the serialization mode for the system. :param mode: (TaurusSerializationMode) the new serialization mode """ self._serialization_mode = mode def getSerializationMode(self): """Gives the serialization operation mode. :return: (TaurusSerializationMode) the current serialization mode """ return self._serialization_mode #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # API to get objects. Generic implementation. You may want to reimplement # it in your scheme factory #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def getAuthority(self, name=None): """Obtain the model object corresponding to the given authority name. If the corresponding authority already exists, the existing instance is returned. Otherwise a new instance is stored and returned. :param name: (str) authority name :return: a taurus.core.taurusauthority.TaurusAuthority object :raises: :TaurusException: if the given name is invalid. """ v = self.getAuthorityNameValidator() if not v.isValid(name): msg = "Invalid {scheme} authority name '{name}'".format( scheme=self.schemes[0], name=name) raise TaurusException(msg) fullname, _, _ = v.getNames(name) auth = self._auths.get(fullname) if auth is not None: return auth cls = self.elementTypesMap[TaurusElementType.Authority] auth = cls(name=fullname) self._auths[fullname] = auth return auth def getDevice(self, name, **kw): """Obtain the model object corresponding to the given device name. If the corresponding device already exists, the existing instance is returned. Otherwise a new instance is stored and returned. :param name: (str) device name :return: a taurus.core.taurusdevice.TaurusDevice object :raises: :TaurusException: if the given name is invalid. """ v = self.getDeviceNameValidator() if not v.isValid(name): msg = "Invalid {scheme} device name '{name}'".format( scheme=self.schemes[0], name=name) raise TaurusException(msg) fullname, _, _ = v.getNames(name) dev = self._devs.get(fullname) if dev is not None: return dev try: # this works if the authority name is present in the dev full name # (which in principle should always be the case) authname = v.getUriGroups(fullname)['authority'] auth = self.getAuthority(authname) except: self.debug('Cannot get device parent from name "%s"', fullname) auth = None cls = self.elementTypesMap[TaurusElementType.Device] dev = cls(name=fullname, parent=auth) self._devs[fullname] = dev return dev def getAttribute(self, name): """ Obtain the model object corresponding to the given attribute name. If the corresponding attribute already exists, the existing instance is returned. Otherwise a new instance is stored and returned. :param name: (str) attribute name :return: a taurus.core.taurusattribute.TaurusAttribute object :raises: :TaurusException: if the given name is invalid. """ v = self.getAttributeNameValidator() if not v.isValid(name): msg = "Invalid {scheme} attribute name '{name}'".format( scheme=self.schemes[0], name=name) raise TaurusException(msg) fullname, _, _ = v.getNames(name) attr = self._attrs.get(fullname) if attr is not None: return attr try: # this works only if the devname is present in the attr full name # (not all schemes are constructed in this way) devname = v.getUriGroups(fullname)['devname'] dev = self.getDevice(devname) except: self.debug('Cannot get attribute parent from name "%s"', fullname) dev = None cls = self.elementTypesMap[TaurusElementType.Attribute] attr = cls(name=fullname, parent=dev) self._attrs[fullname] = attr return attr #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # Methods that must be implemented by the specific Factory #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def getAuthorityNameValidator(self): raise NotImplementedError("getAuthorityNameValidator cannot be called" " for abstract TaurusFactory") def getDeviceNameValidator(self): raise NotImplementedError("getDeviceNameValidator cannot be called" " for abstract TaurusFactory") def getAttributeNameValidator(self): raise NotImplementedError("getAttributeNameValidator cannot be called" " for abstract TaurusFactory") #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # Factory extension API # Override the following methods if you need to provide special classes for # special object types #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def registerAttributeClass(self, attr_name, attr_klass): pass def unregisterAttributeClass(self, attr_name): pass def registerDeviceClass(self, dev_klass_name, dev_klass): pass def unregisterDeviceClass(self, dev_klass_name): pass #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- # Generic methods #-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- def supportsScheme(self, scheme): """Returns whether the given scheme is supported by this factory :param scheme: (str) the name of the schem to be checked :return: (bool) True if the scheme is supported (False otherwise) """ return scheme in self.schemes def findObject(self, absolute_name): """ Must give an absolute name""" if not absolute_name: return None obj = None cls = self.findObjectClass(absolute_name) if cls: obj = self.getObject(cls, absolute_name) return obj def getObject(self, cls, name): t4_msg = 'The TaurusConfiguration classes are deprecated in tep14' if issubclass(cls, TaurusAuthority): return self.getAuthority(name) elif issubclass(cls, TaurusDevice): return self.getDevice(name) elif issubclass(cls, TaurusAttribute): return self.getAttribute(name) # For backward compatibility elif issubclass(cls, TaurusConfiguration): self.deprecated(dep='TaurusConfiguration', alt='TaurusAttribute', rel='4.0', dbg_msg=t4_msg) return self.getAttribute(name) elif issubclass(cls, TaurusConfigurationProxy): self.deprecated(dep='TaurusConfigurationProxy', alt='TaurusAttribute', rel='4.0', dbg_msg=t4_msg) return self.getAttribute(name) else: return None def changeDefaultPollingPeriod(self, period): if period > 0: self._polling_period = period def getDefaultPollingPeriod(self): return self._polling_period def isPollingEnabled(self): """Tells if the Taurus polling is enabled :return: (bool) whether or not the polling is enabled """ return self._polling_enabled def disablePolling(self): """Disable the application tango polling""" if not self.isPollingEnabled(): return self._polling_enabled = False for period, timer in self.polling_timers.items(): timer.stop() def enablePolling(self): """Enable the application tango polling""" if self.isPollingEnabled(): return for period, timer in self.polling_timers.items(): timer.start() self._polling_enabled = True def addAttributeToPolling(self, attribute, period, unsubscribe_evts=False): """Activates the polling (client side) for the given attribute with the given period (seconds). :param attribute: (taurus.core.tango.TangoAttribute) attribute name. :param period: (float) polling period (in seconds) :param unsubscribe_evts: (bool) whether or not to unsubscribe from events """ tmr = self.polling_timers.get(period, TaurusPollingTimer(period)) self.polling_timers[period] = tmr tmr.addAttribute(attribute, self.isPollingEnabled()) def removeAttributeFromPolling(self, attribute): """Deactivate the polling (client side) for the given attribute. If the polling of the attribute was not previously enabled, nothing happens. :param attribute: (str) attribute name. """ p = None for period, timer in self.polling_timers.items(): if timer.containsAttribute(attribute): timer.removeAttribute(attribute) if timer.getAttributeCount() == 0: p = period break if p: del self.polling_timers[period] def __str__(self): return '{0}()'.format(self.__class__.__name__) def __repr__(self): return '{0}(schemes={1})'.format(self.__class__.__name__, ", ".join(self.schemes)) def getValidTypesForName(self, name, strict=None): ''' Returns a list of all Taurus element types for which `name` is a valid model name (while in many cases a name may only be valid for one element type, this is not necessarily true in general) In this base implementation, name is checked first for Attribute, then for Device and finally for Authority, and the return value is sorted in that same order. If a given schema requires a different ordering, reimplement this method :param name: (str) taurus model name :return: (list<TaurusElementType.element>) where element can be one of: `Attribute`, `Device` or `Authority` ''' ret = [] if self.getAttributeNameValidator().isValid(name, strict=strict): ret.append(TaurusElementType.Attribute) if self.getDeviceNameValidator().isValid(name, strict=strict): ret.append(TaurusElementType.Device) if self.getAuthorityNameValidator().isValid(name, strict=strict): ret.append(TaurusElementType.Authority) return ret def getValidatorFromName(self, name): """ Obtain the validator object corresponding to the given model name. If the model name is not valid for any TaurusModel class, it returns None """ modeltypes = self.getValidTypesForName(name) if not modeltypes: return None return self.elementTypesMap[modeltypes[0]].getNameValidator() def findObjectClass(self, absolute_name): """ Obtain the class object corresponding to the given name. Note, this generic implementation expects that derived classes provide a an attribute called elementTypesMap consisting in a dictionary whose keys are TaurusElementTypes and whose values are the corresponding specific object classes. e.g., the FooFactory should provide:: class FooFactory(TaurusFactory): elementTypesMap = {TaurusElementType.Authority: FooAuthority, TaurusElementType.Device: FooDevice, TaurusElementType.Attribute: FooAttribute, } (...) :param absolute_name: (str) the object absolute name string :return: (taurus.core.taurusmodel.TaurusModel or None) a TaurusModel class derived type or None if the name is not valid """ try: elementTypesMap = self.elementTypesMap except AttributeError: msg = ('generic findObjectClass called but %s does ' + 'not define elementTypesMap.') % self.__class__.__name__ raise RuntimeError(msg) for t in self.getValidTypesForName(absolute_name): ret = elementTypesMap.get(t, None) if ret is not None: return ret return None
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] Let's test the throttling mechanism. Configure a 2 seconds delay on channel A, end enqueue two jobs. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4:throttle=2') >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') We have only one job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A1>] >>> cm.get_wakeup_time() 102 We have no job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.get_wakeup_time() 102 2 seconds later, we can run the other job (even though the first one is still running, because we have enough capacity). >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob A2>] >>> cm.get_wakeup_time() 104 Let's test throttling in combination with a queue reaching full capacity. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,T:2:throttle=2') >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob T1>] >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob T2>] Channel is now full, so no job to run even though throttling delay is over. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.get_wakeup_time() # no wakeup time, since queue is full 0 >>> pp(list(cm.get_jobs_to_run(now=104))) [] >>> cm.get_wakeup_time() # queue is still full 0 >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob T3>] >>> cm.get_wakeup_time() # queue is full 0 >>> cm.notify(db, 'T', 'T2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 107 Test wakeup time behaviour in presence of eta. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,E:1') >>> cm.notify(db, 'E', 'E1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob E1>] >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'E', 'E1', 1, 0, 10, 105, 'pending') >>> cm.get_wakeup_time() # wakeup at eta 105 >>> pp(list(cm.get_jobs_to_run(now=102))) # but there is capacity [<ChannelJob E2>] >>> pp(list(cm.get_jobs_to_run(now=106))) # no capacity anymore [] >>> cm.get_wakeup_time() # no timed wakeup because no capacity 0 >>> cm.notify(db, 'E', 'E2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=107))) # no capacity anymore [<ChannelJob E1>] >>> cm.get_wakeup_time() 0 Test wakeup time behaviour in a sequential queue. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,S:1:sequential') >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob S1>] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'failed') >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'pending') >>> pp(list(cm.get_jobs_to_run(now=102))) [] No wakeup time because due to eta, because the sequential queue is waiting for a failed job. >>> cm.get_wakeup_time() 0 >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob S1>] >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'done') At this stage, we have S2 with an eta of 105 and since the queue is sequential, we wait for it. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S2>] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S3>] >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...] If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] It ignores whitespace around values, and drops empty entries which would be generated by trailing commas, or commented lines on the Odoo config file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4, ... , ... foo bar:1: k=va lue, ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}] It's also possible to replace commas with line breaks, which is more readable if the channel configuration comes from the odoo config file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4 ... foo bar:1: k=va lue ... baz ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}, {'capacity': 1, 'name': 'baz'}] """ res = [] config_string = config_string.replace("\n", ",") for channel_config_string in split_strip(config_string, ','): if not channel_config_string: # ignore empty entries (commented lines, trailing commas) continue config = {} config_items = split_strip(channel_config_string, ':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = split_strip(config_item, '=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError('Invalid channel config %s: ' 'incorrect config item %s' % (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' % (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2,seq:1:sequential') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root').sequential False >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('root.autosub.sub').sequential False >>> cm.get_channel_by_name('autosub.sub').capacity 2 >>> cm.get_channel_by_name('seq').capacity 1 >>> cm.get_channel_by_name('seq').sequential True """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) _logger.info("Configured channel: %s", channel) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning( 'unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now) def get_wakeup_time(self): return self._root_channel.get_wakeup_time()
class ValueCache: """Checksum-to-value cache. Every value is refered to by a expression (or more than one); non-expression values are cached elsewhere. Memory intensive. Like any other cache, does not persist unless offloaded. Cache comes in two flavors: buffer cache and object cache. Buffer cache items are directly serializable, and can be shared over the network, or offloaded to Redis. Keys are straightforward buffer checksums. Each item has two refcounts, authoritative and non-authoritative, corresponding to the authority of refering expression(s). Trees referring to structured-cells-with-partial-authority count as authoritative. Object cache items contain Python objects. They are strictly local. Keys are *semantic keys*: consisting of a *semantic checksum* (i.e. different from the buffer checksum in case of CSON and Python code expressions), an access mode, and a content type. They are maintained as a WeakValueDictionary, i.e. they get auto-cleaned-up if Python holds no reference to them. A worker that accesses them gets a Python reference. cell.value gets a Python reference too, for non-structured cells. Structured cells (.value, .data, .handle) do NOT get a Python reference; instead, the underlying Monitor gets backed up by an API that retrieves the value from object cache when needed (and does not store it) There are two ways an item can be requested: 1. Explicitly from object cache or buffer cache, using a semantic key resp. a buffer checksum 2. Items can be requested using a expression, from which the buffer checksum can be extracted, and a semantic key be generated or read from cache. Whenever a item is requested from object cache that is a partial miss (a miss from object cache but a hit from buffer cache), the item gets generated from buffer cache. In this case, and also if a new object item is added explicitly, a temporary reference to the item is added, that expires after 20 seconds For cells, there is normally only one buffer cache item, because all expressions of the same cell._storage_type are mapped to it. In contrast, every expression typically has its own semantic checksum (reflecting both cell type and subpath), and therefore its own object cache item. """ def __init__(self, manager): value_caches.add(self) self.manager = weakref.ref(manager) if manager is not None else lambda: None self._buffer_cache = {} #buffer-checksum-to-(refcount, refcount, value) self._object_cache = WeakValueDictionary() #semantic-key-to-value # TODO: caches for annotation purposes: reverse caches, ... def incref(self, buffer_checksum, buffer, *, has_auth): """Increase refcount for buffer checksum Add an extra non-auth refcount that expires after 20 secs (unless smaller than 100k). Returns True if the full buffer has been successfully inserted If buffer is not None, this is always the case If buffer is None: - If the checksum is unknown, a dummy item is inserted (returns False) - If a dummy item is found, it is incref'ed (returns False) - If a full item (with non-None buffer) is found, it is incref'ed (returns True) """ #print("INCREF", buffer_checksum.hex(), buffer) item = self._buffer_cache.get(buffer_checksum) if item is None: if has_auth: item = 1, 1, buffer else: item = 0, 2, buffer else: if item[2] is not None: buffer = item[2] # *should* be equal, # if buffer is not None, # and everyone is honest # (i.e. checksum is not spoofed) if has_auth: item = item[0] + 1, item[1] + 1, buffer else: item = item[0], item[1] + 2, buffer success = (buffer is not None) mgr = self.manager() if mgr is not None: tempref = functools.partial(self.decref, buffer_checksum, has_auth=False) mgr.temprefmanager.add_ref(tempref, 20.0) self._buffer_cache[buffer_checksum] = item redis_sinks.set_value(buffer_checksum, buffer) return success def decref(self, buffer_checksum, *, has_auth): item = self._buffer_cache[buffer_checksum] refcount_auth, refcount_nauth, buffer = item if (buffer is None or len(buffer) <= NO_EXPIRE_SIZE_LIMIT): return if has_auth: assert refcount_auth > 0 if refcount_auth == 1 and refcount_nauth == 0: self._buffer_cache.pop(buffer_checksum) return item = refcount_auth - 1, refcount_nauth, buffer self._buffer_cache[buffer_checksum] = item else: assert refcount_nauth > 0 if refcount_auth == 0 and refcount_nauth == 1: self._buffer_cache.pop(buffer_checksum) return item = refcount_auth, refcount_nauth - 1, buffer self._buffer_cache[buffer_checksum] = item def add_semantic_key(self, semantic_key, value): assert isinstance(semantic_key, SemanticKey) try: self._object_cache[semantic_key] = value hash(value) except TypeError: value = WeakrefableWrapper(value) self._object_cache[semantic_key] = value self.manager().temprefmanager.add_ref(value, 20.0) def get_object(self, semantic_key): assert isinstance(semantic_key, SemanticKey) item = self._object_cache.get(semantic_key) if isinstance(item, WeakrefableWrapper): return item.value else: return item def get_buffer(self, checksum): if checksum is None: return None item = self._buffer_cache.get(checksum) if item is None or item[2] is None: item = None value = redis_caches.get_value(checksum) if value is not None: item = 1, 1, value return item def value_check(self, checksum): """For the communionserver...""" if checksum in self._buffer_cache: return True return redis_caches.has_value(checksum)
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)?,... If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] """ res = [] for channel_config_string in config_string.split(','): config = {} config_items = channel_config_string.split(':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = config_item.split('=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError('Invalid channel config %s: ', 'incorrect config item %s' (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('autosub.sub').capacity 2 """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning('unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now)
class JSONRedis: """Extended :class:`Redis` client for convenient use with JSON objects. Objects are stored as JSON-encoded strings in the Redis database and en-/decoding is handled transparently. The translation from an arbitrary object to a JSON-serializable form is carried out by a given ``encode(object)`` function. A JSON-serializable object is one that only cosists of the types given in https://docs.python.org/3/library/json.html#py-to-json-table . *encode* is passed as *default* argument to :func:`json.dumps()`. The reverse translation is done by a given ``decode(json)`` function. *decode* is passed as *object_hook* argument to :func:`json.loads()`. When *caching* is enabled, objects loaded from the Redis database are cached and subsequently retrieved from the cache. An object stays in the cache as long as there is a reference to it and it is automatically removed when the Python interpreter destroys it. Thus, it is guaranteed that getting the same key multiple times will yield the identical object. .. attribute:: r Underlying :class:`Redis` client. .. attribute:: encode Function to encode an object to a JSON-serializable form. .. attribute:: decode Function to decode an object from a JSON-serializable form. .. attribute:: caching Switch to enable / disable object caching. """ def __init__(self, r, encode=None, decode=None, caching=True): self.r = r self.encode = encode self.decode = decode self.caching = caching self._cache = WeakValueDictionary() def oget(self, key): """Return the object at *key*.""" object = self._cache.get(key) if self.caching else None if not object: value = self.get(key) if value: try: object = json.loads(value.decode(), object_hook=self.decode) except ValueError: raise ResponseError() if self.caching: self._cache[key] = object return object def oset(self, key, object): """Set *key* to hold *object*.""" if self.caching: self._cache[key] = object self.set(key, json.dumps(object, default=self.encode)) def omget(self, keys): """Return a list of objects for the given *keys*.""" # TODO: make atomic return [self.oget(k) for k in keys] def omset(self, mapping): """Set each key in *mapping* to its corresponding object.""" # TODO: make atomic for key, object in mapping.items(): self.oset(key, object) def __getattr__(self, name): # proxy return getattr(self.r, name)
class Manager: def __init__(self): self.listeners = {} self.cell_aliases = {} self.cell_rev_aliases = {} self.macro_listeners = {} self.observers = {} self.registrar_listeners = WeakKeyDictionary() self.rev_registrar_listeners = WeakKeyDictionary() self.pin_to_cells = {} self.cells = WeakValueDictionary() self.cell_to_output_pin = WeakKeyDictionary() self._childids = WeakValueDictionary() self.registrar_items = [] self.unstable_workers = WeakSet() super().__init__() def set_stable(self, worker, value): assert value in (True, False), value if not value: #print("UNSTABLE", worker) self.unstable_workers.add(worker) else: #print("STABLE", worker) self.unstable_workers.discard(worker) def add_cell_alias(self, source, target): from .cell import Cell assert isinstance(source, Cell) assert isinstance(target, Cell) assert source is not target cell_id = self.get_cell_id(source) target_ref = weakref.ref(target) try: aliases = self.cell_aliases[cell_id] if target_ref not in aliases: aliases.append(target_ref) except KeyError: self.cell_aliases[cell_id] = [target_ref] if cell_id not in self.cells: self.cells[cell_id] = source #reverse alias cell_id = self.get_cell_id(target) source_ref = weakref.ref(source) try: rev_aliases = self.cell_rev_aliases[cell_id] if source_ref not in rev_aliases: rev_aliases.append(source_ref) except KeyError: self.cell_rev_aliases[cell_id] = [source_ref] if cell_id not in self.cells: self.cells[cell_id] = target def add_registrar_item(self, registrar_name, dtype, data, data_name): item = registrar_name, dtype, data, data_name for curr_item in self.registrar_items: if data_name is None: exists = (curr_item[:3] == item[:3]) else: exists = (curr_item[:2] == item[:2]) and \ curr_item[3] == data_name if exists: raise ValueError("Registrar item already exists") self.registrar_items.append(item) def remove_registrar_item(self, registrar_name, dtype, data, data_name): item = registrar_name, dtype, data, data_name self.registrar_items.remove(item) def add_listener(self, cell, input_pin): cell_id = self.get_cell_id(cell) pin_ref = weakref.ref(input_pin) try: listeners = self.listeners[cell_id] assert pin_ref not in listeners # TODO: tolerate (silently ignore) a connection that exists already? listeners.append(pin_ref) except KeyError: self.listeners[cell_id] = [pin_ref] try: curr_pin_to_cells = self.pin_to_cells[input_pin.get_pin_id()] assert cell_id not in curr_pin_to_cells # TODO: tolerate (append) multiple inputs? curr_pin_to_cells.append(cell_id) except KeyError: self.pin_to_cells[input_pin.get_pin_id()] = [cell_id] if cell_id not in self.cells: self.cells[cell_id] = cell def _remove_listener(self, cell_id, input_pin, worker): input_pin_id = input_pin.get_pin_id() l = self.listeners[cell_id] l[:] = [ref for ref in l if ref().get_pin_id() != input_pin_id] if not len(l): self.listeners.pop(cell_id) cell = self.cells.get(cell_id, None) if cell is not None: cell._on_disconnect(input_pin, worker, False) def remove_listener(self, cell, input_pin): worker = input_pin.worker_ref() input_pin_id = input_pin.get_pin_id() cell_ids = self.pin_to_cells.pop(input_pin_id, []) cell_id = self.get_cell_id(cell) self._remove_listener(cell_id, input_pin, worker) def remove_listeners_pin(self, input_pin): worker = input_pin.worker_ref() cell_ids = self.pin_to_cells.pop(input_pin.get_pin_id(), []) for cell_id in cell_ids: self._remove_listener(cell_id, input_pin, worker) def remove_aliases(self, cell): cell_id = self.get_cell_id(cell) cell_ref = weakref.ref(cell) targets = self.cell_aliases.pop(cell_id, []) for target_ref in targets: target = target_ref() if target is None: continue target._on_disconnect(cell, None, incoming=True) target_id = self.get_cell_id(target) r = self.cell_rev_aliases[target_id] r[:] = [rr for rr in r if rr is not cell_ref] if not len(r): self.cell_rev_aliases.pop(target_id) #rev_aliases targets = self.cell_rev_aliases.pop(cell_id, []) for target_ref in targets: target = target_ref() if target is None: continue target_id = self.get_cell_id(target) r = self.cell_aliases[target_id] r[:] = [rr for rr in r if rr is not cell_ref] if not len(r): self.cell_aliases.pop(target_id) def remove_listeners_cell(self, cell): cell_id = self.get_cell_id(cell) listeners = self.listeners.pop(cell_id, []) for listener in listeners: pin = listener() if pin is None: continue pin_id = pin.get_pin_id() if pin_id not in self.pin_to_cells: continue self.pin_to_cells[pin_id][:] = \ [c for c in self.pin_to_cells[pin_id] if c != cell_id ] def add_macro_listener(self, cell, macro_object, macro_arg): cell_id = self.get_cell_id(cell) m = (macro_object, macro_arg) try: macro_listeners = self.macro_listeners[cell_id] assert m not in macro_listeners macro_listeners.append(m) except KeyError: self.macro_listeners[cell_id] = [m] if cell_id not in self.cells: self.cells[cell_id] = cell def remove_macro_listener(self, cell, macro_object, macro_arg): cell_id = self.get_cell_id(cell) m = (macro_object, macro_arg) if cell_id in self.macro_listeners: l = self.macro_listeners[cell_id] if m in l: l.remove(m) def remove_macro_listeners_cell(self, cell): cell_id = self.get_cell_id(cell) listeners = self.macro_listeners.pop(cell_id, []) def add_registrar_listener(self, registrar, key, target, namespace_name): if registrar not in self.registrar_listeners: self.registrar_listeners[registrar] = {} d = self.registrar_listeners[registrar] if key not in d: d[key] = [] d[key].append((weakref.ref(target), namespace_name)) if target not in self.rev_registrar_listeners: self.rev_registrar_listeners[target] = {} r = self.rev_registrar_listeners[target] if key not in r: r[key] = [] r[key].append(weakref.ref(registrar)) def remove_registrar_listeners(self, target): if target not in self.rev_registrar_listeners: return rev = self.rev_registrar_listeners.pop(target) for key in rev: for registrar_ref in rev[key]: registrar = registrar_ref() if registrar not in self.registrar_listeners: continue r = self.registrar_listeners[registrar] t = r[key] t[:] = [tt for tt in t if tt[0]() is not None and tt[0]() is not target] if not len(t): r.pop(key) if not len(r): self.registrar_listeners.pop(registrar) def add_observer(self, cell, observer): cell_id = self.get_cell_id(cell) obs_ref = weakref.ref(observer) try: observers = self.observers[cell_id] assert obs_ref not in observers observers.append(obs_ref) except KeyError: self.observers[cell_id] = [obs_ref] if cell_id not in self.cells: self.cells[cell_id] = cell def remove_observer(self, cell, observer): cell_id = self.get_cell_id(cell) obs_ref = weakref.ref(observer) if cell_id in self.observers: l = self.observers[cell_id] if obs_ref in l: l.remove(obs_ref) def remove_observers_cell(self, cell): cell_id = self.get_cell_id(cell) listeners = self.observers.pop(cell_id, []) def _update(self, cell, dtype, value, *, worker=None, only_last=False): import threading assert threading.current_thread() is threading.main_thread() from .cell import Signal cell_id = self.get_cell_id(cell) observers = self.observers.get(cell_id, []) for observer in observers: obs = observer() if obs is not None: obs(value) macro_listeners = self.macro_listeners.get(cell_id, []) if not only_last: for macro_object, macro_arg in macro_listeners: try: updated = macro_object.update_cell(macro_arg) except Exception: #TODO: proper logging import traceback traceback.print_exc() aliases = self.cell_aliases.get(cell_id, []) for target_cell_ref in aliases: target_cell = target_cell_ref() if target_cell is not None: if isinstance(target_cell, Signal): #print("cell-cell alias", cell, "=>", target_cell) self._update(target_cell, None, None, worker=worker, only_last=only_last) else: value2 = value if dtype is not None and \ (dtype == "cson" or dtype[0] == "cson") and \ target_cell.dtype is not None and \ (target_cell.dtype == "json" or target_cell.dtype[0] == "json"): if isinstance(value, (str, bytes)): value2 = cson2json(value) target_cell._update(value2, propagate=True) listeners = self.listeners.get(cell_id, []) if only_last: listeners = listeners[-1:] resource_name0 = None if cell.resource is not None: resource_name0 = cell.resource.filepath for input_pin_ref in listeners: input_pin = input_pin_ref() if input_pin is None: continue #TODO: error? if worker is not None and input_pin.worker_ref() is worker: continue value2 = value if dtype is not None and \ (dtype == "cson" or dtype[0] == "cson") and \ input_pin.dtype is not None and \ (input_pin.dtype == "json" or input_pin.dtype[0] == "json"): if isinstance(value, (str, bytes)): value2 = cson2json(value) resource_name = "pin: " + str(input_pin) if resource_name0 is not None: resource_name = resource_name0 + " in " + resource_name input_pin.receive_update(value2, resource_name) def update_from_code(self, cell, only_last=False): import seamless if cell.dtype == "array": value = TransportedArray(cell._data, cell._store) else: value = cell._data #print("manager.update_from_code", cell, head(value)) self._update(cell, cell.dtype, value, only_last=only_last) from .. import run_work from .macro import get_macro_mode if not get_macro_mode(): run_work() def update_from_worker(self, cell_id, value, worker, *, preliminary): import seamless from .cell import Signal cell = self.cells.get(cell_id, None) if cell is None or cell._destroyed: return #cell has died... #print("manager.update_from_worker", cell, head(value), worker) if isinstance(cell, Signal): assert value is None self._update(cell, None, None, worker=worker) else: changed = cell._update(value,propagate=False, preliminary=preliminary) if changed: if cell.dtype == "array": value = TransportedArray(value, cell._store) self._update(cell, cell.dtype, value, worker=worker) def update_registrar_key(self, registrar, key): from .worker import Worker from .macro import MacroObject if registrar not in self.registrar_listeners: return d = self.registrar_listeners[registrar] if key not in d: return for t in list(d[key]): target = t[0]() if target is None: continue if isinstance(target, Worker): namespace_name = t[1] target.receive_registrar_update(registrar.name, key, namespace_name) elif isinstance(target, MacroObject): target.update_cell((registrar.name, key)) else: raise TypeError(target) @classmethod def get_cell_id(cls, cell): return id(cell) def disconnect(self, source, target): from .transformer import Transformer from .cell import Cell, CellLike from .context import Context from .worker import EditPinBase, ExportedEditPin, \ InputPinBase, ExportedInputPin, OutputPinBase, ExportedOutputPin if isinstance(source, EditPinBase): source, target = target, source if isinstance(source, CellLike) and source._like_cell: if isinstance(target, ExportedInputPin): target = target.get_pin() if isinstance(source, Context): assert "_output" in source._pins source = source._pins["_output"] self.remove_listener(source, target) worker = target.worker_ref() if worker is not None: source._on_disconnect(target, worker, incoming = False) elif isinstance(source, OutputPinBase): if isinstance(target, Context): assert "_input" in target._pins target = target._pins["_input"] if isinstance(source, ExportedOutputPin): source = source.get_pin() cell_id = self.get_cell_id(target) ok = False if cell_id in self.cells and \ target in self.cell_to_output_pin: if cell_id not in source._cell_ids: ok = False else: for ref in self.cell_to_output_pin[target]: if ref() is source: self.cell_to_output_pin[target].remove(ref) source._cell_ids.remove(cell_id) ok = True if not ok: raise ValueError("Connection does not exist") if target not in self.cell_to_output_pin: self.cell_to_output_pin[target] = [] self.cell_to_output_pin[target].append(weakref.ref(source)) worker = source.worker_ref() if worker is not None: if isinstance(worker, Transformer): worker._on_disconnect_output() target._on_disconnect(source, worker, incoming = True) else: raise TypeError def connect(self, source, target): from .transformer import Transformer from .cell import Cell, CellLike from .context import Context from .worker import EditPinBase, ExportedEditPin, \ InputPinBase, ExportedInputPin, OutputPinBase, ExportedOutputPin if isinstance(source, EditPinBase): source, target = target, source if isinstance(source, CellLike) and source._like_cell: assert isinstance(target, (InputPinBase, EditPinBase, CellLike)) assert source._get_manager() is self assert target._get_manager() is self if isinstance(target, ExportedInputPin): target = target.get_pin() if isinstance(target, Cell): self.add_cell_alias(source, target) target._on_connect(source, None, incoming = True) if source._status == Cell.StatusFlags.OK: value = source._data if source.dtype is not None and \ (source.dtype == "cson" or source.dtype[0] == "cson") and \ target.dtype is not None and \ (target.dtype == "json" or target.dtype[0] == "json"): if isinstance(value, (str, bytes)): value = cson2json(value) target._update(value,propagate=True) return assert not isinstance(target, Context) #TODO? worker = target.worker_ref() assert worker is not None #weakref may not be dead source._on_connect(target, worker, incoming = False) self.add_listener(source, target) if source._status == Cell.StatusFlags.OK: self.update_from_code(source, only_last=True) else: if isinstance(target, EditPinBase) and target.last_value is not None: self.update_from_worker( self.get_cell_id(source), target.last_value, worker, preliminary=False ) elif isinstance(source, OutputPinBase): assert isinstance(target, CellLike) and target._like_cell if isinstance(target, Context): assert "_input" in target._pins target = target._pins["_input"] if isinstance(source, ExportedOutputPin): source = source.get_pin() worker = source.worker_ref() assert worker is not None #weakref may not be dead target._on_connect(source, worker, incoming = True) cell_id = self.get_cell_id(target) if cell_id not in self.cells: self.cells[cell_id] = target if cell_id not in source._cell_ids: source._cell_ids.append(cell_id) if target not in self.cell_to_output_pin: self.cell_to_output_pin[target] = [] self.cell_to_output_pin[target].append(weakref.ref(source)) if isinstance(worker, Transformer): worker._on_connect_output() elif source.last_value is not None: self.update_from_worker( cell_id, source.last_value, worker, preliminary=False ) else: raise TypeError
class Client(object): # TODO optional HTTP/2 support: this makes multiple queries simultaneously. def __init__(self, api_root_url, schema_path='/schema', fetch_schema=True, **session_kwargs): self._instances = WeakValueDictionary() self._resources = {} self.session = session = requests.Session() for key, value in session_kwargs.items(): setattr(session, key, value) parse_result = urlparse(api_root_url) self._root_url = '{}://{}'.format(parse_result.scheme, parse_result.netloc) self._api_root_url = api_root_url # '{}://{}'.format(parse_result.scheme, parse_result.netloc) self._root_path = parse_result.path self._schema_url = api_root_url + schema_path if fetch_schema: self._fetch_schema() def _fetch_schema(self): schema = self.session \ .get(self._schema_url) \ .json(cls=PotionJSONSchemaDecoder, referrer=self._schema_url, client=self) # NOTE these should perhaps be definitions in Flask-Potion for name, resource_schema in schema['properties'].items(): resource = self.resource_factory(name, resource_schema) setattr(self, upper_camel_case(name), resource) def instance(self, uri, cls=None, default=None, **kwargs): instance = self._instances.get(uri, None) if instance is None: if cls is None: try: cls = self._resources[uri[:uri.rfind('/')]] except KeyError: cls = Reference if isinstance(default, Resource) and default._uri is None: default._status = 200 default._uri = uri instance = default else: instance = cls(uri=uri, **kwargs) self._instances[uri] = instance return instance def fetch(self, uri, cls=PotionJSONDecoder, **kwargs): # TODO handle URL fragments (#properties/id etc.) response = self.session \ .get(urljoin(self._root_url, uri, True)) response.raise_for_status() return response.json(cls=cls, client=self, referrer=uri, **kwargs) def resource_factory(self, name, schema, resource_cls=None): """ Registers a new resource with a given schema. The schema must not have any unresolved references (such as `{"$ref": "#"}` for self-references, or otherwise). A subclass of :class:`Resource` may be provided to add specific functionality to the resulting :class:`Resource`. :param str name: :param dict schema: :param Resource resource_cls: a subclass of :class:`Resource` or None :return: The new :class:`Resource`. """ cls = type(str(upper_camel_case(name)), (resource_cls or Resource, collections.MutableMapping), { '__doc__': schema.get('description', '') }) cls._schema = schema cls._client = self cls._links = links = {} for link_schema in schema['links']: link = Link(self, rel=link_schema['rel'], href=link_schema['href'], method=link_schema['method'], schema=link_schema.get('schema', None), target_schema=link_schema.get('targetSchema', None)) # Set Resource._self, etc. for the special methods as they are managed by the Resource class if link.rel in ('self', 'instances', 'create', 'update', 'destroy'): setattr(cls, '_{}'.format(link.rel), link) links[link.rel] = link if link.rel != 'update': # 'update' is a special case because of MutableMapping.update() setattr(cls, snake_case(link.rel), link) # TODO routes (instance & non-instance) for property_name, property_schema in schema.get('properties', {}).items(): # skip $uri and $id as these are already implemented in Resource and overriding them causes unnecessary # fetches. if property_name.startswith('$'): continue if property_schema.get('readOnly', False): # TODO better error message. Raises AttributeError("can't set attribute") setattr(cls, property_name, property(fget=partial((lambda name, obj: getitem(obj, name)), property_name), doc=property_schema.get('description', None))) else: setattr(cls, property_name, property(fget=partial((lambda name, obj: getitem(obj, name)), property_name), fset=partial((lambda name, obj, value: setitem(obj, name, value)), property_name), fdel=partial((lambda name, obj: delitem(obj, name)), property_name), doc=property_schema.get('description', None))) root = None if 'instances' in links: root = cls._instances.href elif 'self' in links: root = cls._self.href[:cls._self.href.rfind('/')] else: root = self._root_path + '/' + name.replace('_', '-') self._resources[root] = cls return cls
class Container: """ Container used for registering and resolving components. Ideally no component will depend on a reference to the container. """ def __init__(self): self._key_to_registration = dict() self._singleton_instances = dict() self._singleton_instances_lock = threading.Lock() self._weak_references = WeakValueDictionary() self._weak_references_lock = threading.Lock() self._thread_local = threading.local() def register_instance(self, instance: object, register_key=None): key = register_key if register_key else instance.__name__.lower() registration = Registration(instance.__class__) registration.usage = RegistrationUsage.singleton self._key_to_registration[key] = registration with self._singleton_instances_lock: self._singleton_instances[key] = instance def register(self, type_ref: type, register_key=None) -> Registration: key = register_key if register_key else type_ref.__name__.lower() registration = Registration(type_ref) self._key_to_registration[key] = registration return registration def resolve_by_type(self, type_ref: type): reg = Registration(type_ref) return self.__get_instance_for("type::"+type_ref.__name__.lower(), reg) def resolve_by_key(self, register_key): try: found = self._key_to_registration[register_key] except KeyError: raise RegistrationNotFoundError(register_key) return self.__get_instance_for(register_key, found) def __resolve_by_parameter(self, parameter: Parameter): try: if parameter.annotation != Parameter.empty: return self.resolve_by_key(parameter.annotation) return self.resolve_by_key(parameter.name.lower()) except RegistrationNotFoundError: # fixme non registered defaults are expensive... if parameter.default is Parameter.empty: raise return parameter.default def __get_instance_for(self, cache_key, registration: Registration): def create_func(): params = signature(registration.type_ref).parameters args = {name: self.__resolve_by_parameter(param) for name, param in params.items()} return registration.type_ref(**args) if registration.usage is RegistrationUsage.temporal: return create_func() elif registration.usage is RegistrationUsage.singleton: return self.__get_singleton(cache_key, create_func) elif registration.usage is RegistrationUsage.weak_reference: return self.__get_weak(cache_key, create_func) elif registration.usage is RegistrationUsage.thread_local: return self.__get_thread_local(cache_key, create_func) def __get_weak(self, cache_key, create_func): result = self._weak_references.get(cache_key, None) if result is not None: return result with self._weak_references_lock: result = self._weak_references.get(cache_key, None) if result is not None: return result result = create_func() self._weak_references[cache_key] = result return result def __get_singleton(self, cache_key, create_func): if cache_key in self._singleton_instances: return self._singleton_instances[cache_key] with self._singleton_instances_lock: if cache_key in self._singleton_instances: return self._singleton_instances[cache_key] result = create_func() self._singleton_instances[cache_key] = result return result def __get_thread_local(self, cache_key, create_func): local = self._thread_local if 'registration_dictionary' not in local.__dict__: local.registration_dictionary = dict() if cache_key in local.registration_dictionary: return local.registration_dictionary[cache_key] result = create_func() local.registration_dictionary[cache_key] = result return result
class Boss: def __init__(self, os_window_id, opts, args, cached_values): self.window_id_map = WeakValueDictionary() self.cached_values = cached_values self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd) set_boss(self) self.current_font_size = opts.font_size set_font_family(opts) self.opts, self.args = opts, args initialize_renderer() startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None): dpi_changed = False if os_window_id is None: w, h = initial_window_size( self.opts, self.cached_values) if size is None else size cls = wclass or self.args.cls or appname os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls) if startup_id: ctx = init_startup_notification(os_window_id, startup_id) dpi_changed = show_window(os_window_id) if startup_id: end_startup_notification(ctx) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm if dpi_changed: self.on_dpi_change(os_window_id) def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } def match_windows(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) for tm in self.os_window_map.values(): for tab in tm: for window in tab: if window.matches(field, pat): yield window def tab_for_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) tms = tuple(self.os_window_map.values()) found = False if field in ('title', 'id'): for tm in tms: for tab in tm: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def peer_messages_received(self, messages): import json for msg in messages: msg = json.loads(msg.decode('utf-8')) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) session = create_session(opts, args) self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size(opts), startup_id=startup_id) else: safe_print('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = None if self.opts.allow_remote_control: try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = { 'ok': False, 'error': str(err), 'tb': traceback.format_exc() } else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): tm = self.os_window_map.get(os_window_id) if tm is not None: if dpi_changed: if set_dpi_from_os_window(os_window_id): self.on_dpi_change(os_window_id) else: tm.resize() else: tm.resize() def increase_font_size(self): self.change_font_size( min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def _change_font_size(self, new_size=None, on_dpi_change=False): if new_size is not None: self.current_font_size = new_size old_cell_width, old_cell_height = viewport_for_window()[-2:] windows = tuple(filter(None, self.window_id_map.values())) resize_fonts(self.current_font_size, on_dpi_change=on_dpi_change) layout_sprite_map() prerender() for window in windows: window.screen.rescale_images(old_cell_width, old_cell_height) window.screen.refresh_sprite_positions() for tm in self.os_window_map.values(): tm.resize() tm.refresh_sprite_positions() glfw_post_empty_event() def change_font_size(self, new_size): if new_size == self.current_font_size: return self._change_font_size(new_size) def on_dpi_change(self, os_window_id): self._change_font_size() @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, window, data): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'), overlay_for=window.id)) def input_unicode_character(self): w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: overlay_window = tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.unicode_input.main import main; main()' ], overlay_for=w.id)) overlay_window.action_on_close = partial( self.send_unicode_character, w.id) def send_unicode_character(self, target_window_id, source_window): w = self.window_id_map.get(target_window_id) if w is not None: output = str(source_window.screen.linebuf.line(0)) if output.startswith('OK: '): try: text = chr(int(output.partition(' ')[2], 16)) except Exception: import traceback traceback.print_exc() else: w.paste(text) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def open_url_lines(self, lines): self.open_url(''.join(lines)) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() wakeup() self.child_monitor.join() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_sprite_map() destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.buffer_as_ansi() if arg == '@text': return w.buffer_as_text() if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: tab.new_special_window( self.args_to_special_window(args, cwd_from=cwd_from)) else: tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window if w is None: return self.new_window(*args) cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1)
class NodeManager(BaseNodeManager): NODE_CLASS = Node def __init__(self, project): super().__init__() # XXX [!] cycle reference self.project = project self.data = WeakValueDictionary() self.root = None def update_root(self, root_project, root_project_children): self.root = self.new_root_node(root_project, root_project_children) def __setitem__(self, projectid, node): assert self.check_not_exist_node(node) assert projectid == node.projectid self.data[node.projectid] = node assert self.check_exist_node(node) def __delitem__(self, node): assert self.check_exist_node(node) del self.data[node.projectid] assert self.check_not_exist_node(node) def __iter__(self): # how to support lock for iter? just copy dict? return iter(self.data.values()) def __contains__(self, node): if node is None: return False newnode = self.data.get(node.projectid) return node is newnode # ?! def __len__(self): return len(self.data) def __bool__(self): return len(self) != 0 @property def get(self): return self.data.get @property def clear(self): return self.data.clear # TODO: add expend node. (not operation.) def check_exist_node(self, node): original_node = self.get(node.projectid) if original_node is None: raise WFNodeError("{!r} is not exists.".format(node)) elif original_node is not node: raise WFNodeError("{!r} is invalid node.".format(node)) return True def check_not_exist_node(self, node): if node in self: raise WFNodeError("{!r} is already exists.".format(node)) return True def new_void_node(self, uuid=None): return self.NODE_CLASS.from_void(uuid, project=self.project) def new_node_from_json(self, data, parent=None): return self.NODE_CLASS.from_json_with_project(data, parent=parent, project=self.project) def add(self, node, recursion=True): assert recursion is True added_nodes = 0 def register_node(node): nonlocal added_nodes self[node.projectid] = node added_nodes += 1 register_node(node) if recursion: def deep(node): for subnode in node: register_node(subnode) deep(subnode) deep(node) return added_nodes def remove(self, node, recursion=True): assert self.check_exist_node(node) if node.parent is not None: assert self.check_exist_node(node.parent) if node in node.parent: raise WFNodeError("node are still exists in parent node.") removed_nodes = 0 def unregister_node(node): nonlocal removed_nodes del node.parent del self[node] removed_nodes += 1 unregister_node(node) if recursion: def deep(node): if not node: return child_nodes, node.ch = node.ch[:], None for child in child_nodes: unregister_node(node) deep(child) deep(node) return removed_nodes def new_root_node(self, root_project, root_project_children): # XXX [!] project is Project, root_project is root node. ?! if root_project is None: root_project = dict(id=DEFAULT_ROOT_NODE_ID) else: root_project.update(id=DEFAULT_ROOT_NODE_ID) # in shared mode, root will have uuid -(replace)> DEFAULT_ROOT_NODE_ID root_project.update(ch=root_project_children) root = self.new_node_from_json(root_project) self.add(root, recursion=True) return root @property def pretty_print(self): return self.root.pretty_print
class EntityCache(object): """ Cache for entities. Supports add and remove operations as well as lookup by ID and by slug. """ def __init__(self, entities=None, allow_none_id=True): """ :param bool allow_none_id: Flag specifying if calling :meth:`add` with an entity that does not have an ID is allowed. """ # Flag indicating if None IDs are allowed in this cache. self.__allow_none_id = allow_none_id # List of cached entities. This is the only place we are holding a # real reference to the entity. if entities is None: entities = [] self.__entities = entities # Dictionary mapping entity IDs to entities for fast lookup by ID. self.__id_map = WeakValueDictionary() # Dictionary mapping entity slugs to entities for fast lookup by slug. self.__slug_map = {} def get_by_id(self, entity_id): """ Performs a lookup of an entity by its ID. :param int entity_id: entity ID. :return: entity found or ``None``. """ return self.__id_map.get(entity_id) def has_id(self, entity_id): """ Checks if this entity cache holds an entity with the given ID. :return: Boolean result of the check. """ return entity_id in self.__id_map def get_by_slug(self, entity_slug): """ Performs a lookup of an entity by its slug. :param str entity_id: entity slug. :return: entity found or ``None``. """ return self.__slug_map.get(entity_slug) def has_slug(self, entity_slug): return entity_slug in self.__slug_map def add(self, entity): """ Adds the given entity to this cache. :param entity: Entity to add. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises ValueError: If the ID of the entity to add is ``None`` (unless the `allow_none_id` constructor argument was set). """ do_append = self.__check_new(entity) if do_append: self.__entities.append(entity) def remove(self, entity): """ Removes the given entity from this cache. :param entity: Entity to remove. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises KeyError: If the given entity is not in this cache. :raises ValueError: If the ID of the given entity is `None`. """ self.__id_map.pop(entity.id, None) self.__slug_map.pop(entity.slug, None) self.__entities.remove(entity) def update(self, source_data, target_entity): """ Updates the state of the target entity with the given source data. :param target_entity: Entity to update. :type target_entity: Object implementing :class:`everest.interfaces.IEntity`. """ EntityState.set_state_data(target_entity, source_data) def get_all(self): """ Returns the list of all entities in this cache in the order they were added. """ return self.__entities def retrieve(self, filter_expression=None, order_expression=None, slice_key=None): """ Retrieve entities from this cache, possibly after filtering, ordering and slicing. """ ents = iter(self.__entities) if not filter_expression is None: ents = filter_expression(ents) if not order_expression is None: # Ordering always involves a copy and conversion to a list, so # we have to wrap in an iterator. ents = iter(order_expression(ents)) if not slice_key is None: ents = islice(ents, slice_key.start, slice_key.stop) return ents def rebuild(self, entities): """ Rebuilds the ID and slug maps of this cache. This can be necessary when entities obtain their IDs only after they have been flushed to the backend. """ for ent in entities: self.__check_new(ent) def __contains__(self, entity): if not entity.id is None: is_contained = entity.id in self.__id_map else: is_contained = entity in self.__entities return is_contained def __check_new(self, entity): # For certain use cases (e.g., staging), we do not want the entity to # be added to have an ID yet. do_append = True if not entity.id is None: if entity.id in self.__id_map: if not self.__id_map[entity.id] is entity: raise ValueError('Duplicate entity ID "%s". %s' % (entity.id, entity)) else: do_append = False else: self.__id_map[entity.id] = entity elif not self.__allow_none_id: raise ValueError('Entity ID must not be None.') # The slug can be a lazy attribute depending on the # value of other (possibly not yet initialized) attributes which is # why we can not always assume it is available at this point. if do_append and hasattr(entity, 'slug') and not entity.slug is None: ents = self.__slug_map.get(entity.slug) if not ents is None: ents.append(entity) else: self.__slug_map[entity.slug] = WeakList([entity]) return do_append
class Boss: def __init__(self, os_window_id, opts, args, cached_values, new_os_window_trigger): set_layout_options(opts) self.clipboard_buffers = {} self.update_check_process = None self.window_id_map = WeakValueDictionary() self.startup_colors = { k: opts[k] for k in opts if isinstance(opts[k], Color) } self.startup_cursor_text_color = opts.cursor_text_color self.pending_sequences = None self.cached_values = cached_values self.os_window_map = {} self.os_window_death_actions = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.opts, self.args = opts, args startup_sessions = create_sessions( opts, args, default_session=opts.startup_session) self.keymap = self.opts.keymap.copy() if new_os_window_trigger is not None: self.keymap.pop(new_os_window_trigger, None) for startup_session in startup_sessions: self.add_os_window(startup_session, os_window_id=os_window_id) os_window_id = None if args.start_as != 'normal': if args.start_as == 'fullscreen': self.toggle_fullscreen() else: change_os_window_state(args.start_as) if is_macos: from .fast_data_types import cocoa_set_notification_activated_callback cocoa_set_notification_activated_callback( self.notification_activated) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, opts_for_size=None, startup_id=None): if os_window_id is None: opts_for_size = opts_for_size or startup_session.os_window_size or self.opts cls = wclass or self.args.cls or appname with startup_notification_handler( do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback: os_window_id = create_os_window( initial_window_size_func(opts_for_size, self.cached_values), pre_show_callback, appname, wname or self.args.name or cls, cls) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm return os_window_id def list_os_windows(self): with cached_process_data(): active_tab, active_window = self.active_tab, self.active_window active_tab_manager = self.active_tab_manager for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'is_focused': tm is active_tab_manager, 'tabs': list(tm.list_tabs(active_tab, active_window)), } @property def all_tab_managers(self): yield from self.os_window_map.values() @property def all_tabs(self): for tm in self.all_tab_managers: yield from tm @property def all_windows(self): for tab in self.all_tabs: yield from tab def match_windows(self, match): try: field, exp = match.split(':', 1) except ValueError: return if field == 'num': tab = self.active_tab if tab is not None: try: w = tab.get_nth_window(int(exp)) except Exception: return if w is not None: yield w return if field == 'env': kp, vp = exp.partition('=')[::2] if vp: pat = tuple(map(re.compile, (kp, vp))) else: pat = re.compile(kp), None else: pat = re.compile(exp) for window in self.all_windows: if window.matches(field, pat): yield window def tab_for_window(self, window): for tab in self.all_tabs: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): try: field, exp = match.split(':', 1) except ValueError: return pat = re.compile(exp) found = False if field in ('title', 'id'): for tab in self.all_tabs: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for os_window_id, tm in self.os_window_map.items(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return os_window_id def _new_os_window(self, args, cwd_from=None): if isinstance(args, SpecialWindowInstance): sw = args else: sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = next( create_sessions(self.opts, special_window=sw, cwd_from=cwd_from)) return self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) @property def active_window_for_cwd(self): w = self.active_window if w is not None and w.overlay_for is not None and w.overlay_for in self.window_id_map: w = self.window_id_map[w.overlay_for] return w def new_os_window_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._new_os_window(args, cwd_from) def new_os_window_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_os_window(special_window) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control or getattr( window, 'allow_remote_control', False): try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err)} if not getattr(err, 'hide_traceback', False): response['tb'] = traceback.format_exc() else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) if not os.path.isabs(args.directory): args.directory = os.path.join(msg['cwd'], args.directory) for session in create_sessions(opts, args, respect_cwd=True): os_window_id = self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id) if msg.get('notify_on_os_window_death'): self.os_window_death_actions[os_window_id] = partial( self.notify_on_os_window_death, msg['notify_on_os_window_death']) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def toggle_maximized(self): toggle_maximized() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True if self.opts.update_check_interval > 0 and not hasattr( self, 'update_check_started'): from .update_check import run_update_check run_update_check(self.opts.update_check_interval * 60 * 60) self.update_check_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): if dpi_changed: self.on_dpi_change(os_window_id) else: tm = self.os_window_map.get(os_window_id) if tm is not None: tm.resize() def clear_terminal(self, action, only_active): if only_active: windows = [] w = self.active_window if w is not None: windows.append(w) else: windows = self.all_windows reset = action == 'reset' how = 3 if action == 'scrollback' else 2 for w in windows: if action == 'scroll': w.screen.scroll_until_cursor() continue w.screen.cursor.x = w.screen.cursor.y = 0 if reset: w.screen.reset() else: w.screen.erase_in_display(how, False) def increase_font_size(self): # legacy cfs = global_font_size() self.set_font_size(min(self.opts.font_size * 5, cfs + 2.0)) def decrease_font_size(self): # legacy cfs = global_font_size() self.set_font_size(max(MINIMUM_FONT_SIZE, cfs - 2.0)) def restore_font_size(self): # legacy self.set_font_size(self.opts.font_size) def set_font_size(self, new_size): # legacy self.change_font_size(True, None, new_size) def change_font_size(self, all_windows, increment_operation, amt): def calc_new_size(old_size): new_size = old_size if amt == 0: new_size = self.opts.font_size else: if increment_operation: new_size += (1 if increment_operation == '+' else -1) * amt else: new_size = amt new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5)) return new_size if all_windows: current_global_size = global_font_size() new_size = calc_new_size(current_global_size) if new_size != current_global_size: global_font_size(new_size) os_windows = tuple(self.os_window_map.keys()) else: os_windows = [] w = self.active_window if w is not None: os_windows.append(w.os_window_id) if os_windows: final_windows = {} for wid in os_windows: current_size = os_window_font_size(wid) if current_size: new_size = calc_new_size(current_size) if new_size != current_size: final_windows[wid] = new_size if final_windows: self._change_font_size(final_windows) def _change_font_size(self, sz_map): for os_window_id, sz in sz_map.items(): tm = self.os_window_map.get(os_window_id) if tm is not None: os_window_font_size(os_window_id, sz) tm.resize() def on_dpi_change(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: sz = os_window_font_size(os_window_id) if sz: os_window_font_size(os_window_id, sz, True) tm.resize() def _set_os_window_background_opacity(self, os_window_id, opacity): change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0))) def set_background_opacity(self, opacity): window = self.active_window if window is None or not opacity: return if not self.opts.dynamic_background_opacity: return self.show_error( _('Cannot change background opacity'), _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity' )) os_window_id = window.os_window_id if opacity[0] in '+-': old_opacity = background_opacity_of(os_window_id) if old_opacity is None: return opacity = old_opacity + float(opacity) elif opacity == 'default': opacity = self.opts.background_opacity else: opacity = float(opacity) self._set_os_window_background_opacity(os_window_id, opacity) @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.keymap, mods, key, scancode) if key_action is None: sequences = get_shortcut(self.opts.sequence_map, mods, key, scancode) if sequences: self.pending_sequences = sequences set_in_sequence_mode(True) return True else: self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def process_sequence(self, key, scancode, action, mods): if not self.pending_sequences: set_in_sequence_mode(False) remaining = {} matched_action = None for seq, key_action in self.pending_sequences.items(): if shortcut_matches(seq[0], mods, key, scancode): seq = seq[1:] if seq: remaining[seq] = key_action else: matched_action = key_action if remaining: self.pending_sequences = remaining else: self.pending_sequences = None set_in_sequence_mode(False) if matched_action is not None: self.dispatch_action(matched_action) def start_resizing_window(self): w = self.active_window if w is None: return overlay_window = self._run_kitten( 'resize_window', args=[ '--horizontal-increment={}'.format( self.opts.window_resize_step_cells), '--vertical-increment={}'.format( self.opts.window_resize_step_lines) ]) if overlay_window is not None: overlay_window.allow_remote_control = True def resize_layout_window(self, window, increment, is_horizontal, reset=False): tab = window.tabref() if tab is None or not increment: return False if reset: return tab.reset_window_sizes() return tab.resize_window_by(window.id, increment, is_horizontal) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: tm.update_tab_bar_data() tm.mark_tab_bar_dirty() t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) tm.mark_tab_bar_dirty() def update_tab_bar_data(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.update_tab_bar_data() def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) action = self.os_window_death_actions.pop(os_window_id, None) if action is not None: action() def notify_on_os_window_death(self, address): import socket s = socket.socket(family=socket.AF_UNIX) with suppress(Exception): s.connect(address) s.sendall(b'c') with suppress(EnvironmentError): s.shutdown(socket.SHUT_RDWR) s.close() def display_scrollback(self, window, data, cmd): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(cmd, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ kitty_exe(), '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + get_editor() + [confpath] self.new_os_window(*cmd) def get_output(self, source_window, num_lines=1): output = '' s = source_window.screen if num_lines is None: num_lines = s.lines for i in range(min(num_lines, s.lines)): output += str(s.linebuf.line(i)) return output def _run_kitten(self, kitten, args=(), input_data=None, window=None): orig_args, args = list(args), list(args) from kittens.runner import create_kitten_handler end_kitten = create_kitten_handler(kitten, orig_args) if window is None: w = self.active_window tab = self.active_tab else: w = window tab = w.tabref() if end_kitten.no_ui: end_kitten(None, getattr(w, 'id', None), self) return if w is not None and tab is not None and w.overlay_for is None: args[0:0] = [config_dir, kitten] if input_data is None: type_of_input = end_kitten.type_of_input if type_of_input in ('text', 'history', 'ansi', 'ansi-history', 'screen', 'screen-history', 'screen-ansi', 'screen-ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input, add_wrap_markers='screen' in type_of_input).encode('utf-8') elif type_of_input is None: data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) else: data = input_data if isinstance(data, str): data = data.encode('utf-8') copts = { k: self.opts[k] for k in ('select_by_word_characters', 'open_url_with') } overlay_window = tab.new_special_window( SpecialWindow([ kitty_exe(), '+runpy', 'from kittens.runner import main; main()' ] + args, stdin=data, env={ 'KITTY_COMMON_OPTS': json.dumps(copts), 'KITTY_CHILD_PID': w.child.pid, 'PYTHONWARNINGS': 'ignore', 'OVERLAID_WINDOW_LINES': str(w.screen.lines), 'OVERLAID_WINDOW_COLS': str(w.screen.columns), }, cwd=w.cwd_of_child, overlay_for=w.id)) overlay_window.action_on_close = partial(self.on_kitten_finish, w.id, end_kitten) return overlay_window def kitten(self, kitten, *args): import shlex cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] self._run_kitten(kitten, args) def on_kitten_finish(self, target_window_id, end_kitten, source_window): output = self.get_output(source_window, num_lines=None) from kittens.runner import deserialize data = deserialize(output) if data is not None: end_kitten(data, target_window_id, self) def input_unicode_character(self): self._run_kitten('unicode_input') def set_tab_title(self): tab = self.active_tab if tab: args = [ '--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id) ] self._run_kitten('ask', args) def show_error(self, title, msg): self._run_kitten('show_error', args=['--title', title], input_data=msg) def do_set_tab_title(self, title, tab_id): tm = self.active_tab_manager if tm is not None and title: tab_id = int(tab_id) for tab in tm.tabs: if tab.id == tab_id: tab.set_title(title) break def kitty_shell(self, window_type): cmd = ['@', kitty_exe(), '@'] if window_type == 'tab': self._new_tab(cmd) elif window_type == 'os_window': os_window_id = self._new_os_window(cmd) self.os_window_map[os_window_id] elif window_type == 'overlay': w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: tab.new_special_window(SpecialWindow(cmd, overlay_for=w.id)) else: self._new_window(cmd) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) def open_url(self, url, program=None, cwd=None): if url: if isinstance(program, str): program = to_cmdline(program) open_url(url, program or self.opts.open_url_with, cwd=cwd) def open_url_lines(self, lines, program=None): self.open_url(''.join(lines), program) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() self.set_update_check_process() self.update_check_process = None del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection( ) if supports_primary_selection else get_clipboard_string() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: self.copy_to_buffer(self.opts.copy_on_select) def copy_to_buffer(self, buffer_name): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: if buffer_name == 'clipboard': set_clipboard_string(text) elif buffer_name == 'primary': set_primary_selection(text) else: self.clipboard_buffers[buffer_name] = text def paste_from_buffer(self, buffer_name): if buffer_name == 'clipboard': text = get_clipboard_string() elif buffer_name == 'primary': text = get_primary_selection() else: text = self.clipboard_buffers.get(buffer_name) if text: self.paste_to_active_window(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def set_active_tab(self, tab): tm = self.active_tab_manager if tm is not None: return tm.set_active_tab(tab) return False def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) prev_tab = previous_tab def process_stdin_source(self, window=None, stdin=None): w = window or self.active_window env = None if stdin: add_wrap_markers = stdin.endswith('_wrap') if add_wrap_markers: stdin = stdin[:-len('_wrap')] stdin = data_for_at(w, stdin, add_wrap_markers=add_wrap_markers) if stdin is not None: pipe_data = w.pipe_data( stdin, has_wrap_markers=add_wrap_markers) if w else {} if pipe_data: env = { 'KITTY_PIPE_DATA': '{scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}' .format(**pipe_data) } stdin = stdin.encode('utf-8') return env, stdin def special_window_for_cmd(self, cmd, window=None, stdin=None, cwd_from=None, as_overlay=False): w = window or self.active_window env, stdin = self.process_stdin_source(w, stdin) cmdline = [] for arg in cmd: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmdline.append(arg) overlay_for = w.id if as_overlay and w.overlay_for is None else None return SpecialWindow(cmd, stdin, cwd_from=cwd_from, overlay_for=overlay_for, env=env) def pipe(self, source, dest, exe, *args): cmd = [exe] + list(args) window = self.active_window cwd_from = window.child.pid_for_cwd if window else None def create_window(): return self.special_window_for_cmd(cmd, stdin=source, as_overlay=dest == 'overlay', cwd_from=cwd_from) if dest == 'overlay' or dest == 'window': tab = self.active_tab if tab is not None: return tab.new_special_window(create_window()) elif dest == 'tab': tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=create_window(), cwd_from=cwd_from) elif dest == 'os_window': self._new_os_window(create_window(), cwd_from=cwd_from) elif dest in ('clipboard', 'primary'): env, stdin = self.process_stdin_source(stdin=source, window=window) if stdin: func = set_clipboard_string if dest == 'clipboard' else set_primary_selection func(stdin) else: import subprocess env, stdin = self.process_stdin_source(stdin=source, window=window) cwd = None if cwd_from: with suppress(Exception): cwd = cwd_of_process(cwd_from) if stdin: r, w = safe_pipe(False) try: subprocess.Popen(cmd, env=env, stdin=r, cwd=cwd) except Exception: os.close(w) else: thread_write(w, stdin) finally: os.close(r) else: subprocess.Popen(cmd, env=env, cwd=cwd) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window if args[0].startswith('@') and args[0] != '@': stdin = data_for_at(w, args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None, as_neighbor=False): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: return tm.new_tab(special_window=special_window, cwd_from=cwd_from, as_neighbor=as_neighbor) def _create_tab(self, args, cwd_from=None): as_neighbor = False if args and args[0].startswith('!'): as_neighbor = 'neighbor' in args[0][1:].split(',') args = args[1:] self._new_tab(args, as_neighbor=as_neighbor, cwd_from=cwd_from) def new_tab(self, *args): self._create_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._create_tab(args, cwd_from=cwd_from) def new_tab_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_tab(special_window) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: location = None if args and args[0].startswith('!'): location = args[0][1:].lower() args = args[1:] if args: return tab.new_special_window(self.args_to_special_window( args, cwd_from=cwd_from), location=location) else: return tab.new_window(cwd_from=cwd_from, location=location) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window_for_cwd if w is None: return self.new_window(*args) cwd_from = w.child.pid_for_cwd if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def disable_ligatures_in(self, where, strategy): if isinstance(where, str): windows = () if where == 'active': if self.active_window is not None: windows = (self.active_window, ) elif where == 'all': windows = self.all_windows elif where == 'tab': if self.active_tab is not None: windows = tuple(self.active_tab) else: windows = where for window in windows: window.screen.disable_ligatures = strategy window.refresh() def patch_colors(self, spec, cursor_text_color, configured=False): if configured: for k, v in spec.items(): if hasattr(self.opts, k): setattr(self.opts, k, color_from_int(v)) if cursor_text_color is not False: if isinstance(cursor_text_color, int): cursor_text_color = color_from_int(cursor_text_color) self.opts.cursor_text_color = cursor_text_color for tm in self.all_tab_managers: tm.tab_bar.patch_colors(spec) patch_global_colors(spec, configured) def safe_delete_temp_file(self, path): if is_path_in_temp_dir(path): with suppress(FileNotFoundError): os.remove(path) def set_update_check_process(self, process=None): if self.update_check_process is not None: with suppress(Exception): if self.update_check_process.poll() is None: self.update_check_process.kill() self.update_check_process = process def on_monitored_pid_death(self, pid, exit_status): update_check_process = getattr(self, 'update_check_process', None) if update_check_process is not None and pid == update_check_process.pid: self.update_check_process = None from .update_check import process_current_release try: raw = update_check_process.stdout.read().decode('utf-8') except Exception as e: log_error( 'Failed to read data from update check process, with error: {}' .format(e)) else: try: process_current_release(raw) except Exception as e: log_error( 'Failed to process update check data {!r}, with error: {}' .format(raw, e)) def notification_activated(self, identifier): if identifier == 'new-version': from .update_check import notification_activated notification_activated() def dbus_notification_callback(self, activated, *args): from .notify import dbus_notification_created, dbus_notification_activated if activated: dbus_notification_activated(*args) else: dbus_notification_created(*args) def show_bad_config_lines(self, bad_lines): def format_bad_line(bad_line): return '{}:{} in line: {}\n'.format(bad_line.number, bad_line.exception, bad_line.line) msg = '\n'.join(map(format_bad_line, bad_lines)).rstrip() self.show_error(_('Errors in kitty.conf'), msg)
class ConnListener(object): '''Manager for listening socket and connections still in the matchmaking process.''' def __init__(self): # Get a list of potential bind addresses addrs = socket.getaddrinfo(None, PORT, 0, socket.SOCK_STREAM, 0, socket.AI_PASSIVE) # Try to bind to each address socks = [] for family, type, proto, _canonname, addr in addrs: try: sock = socket.socket(family, type, proto) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if family == socket.AF_INET6: # Ensure an IPv6 listener doesn't also bind to IPv4, # since depending on the order of getaddrinfo return # values this could cause the IPv6 bind to fail sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) sock.bind(addr) sock.listen(BACKLOG) sock.setblocking(0) socks.append(sock) except socket.error: pass if not socks: # None of the addresses worked raise ListenError("Couldn't bind listening socket") self._poll = _PendingConnPollSet() for sock in socks: self._poll.register(_ListeningSocket(sock), select.POLLIN) self._nonce_to_pending = WeakValueDictionary() def _accept(self, lsock): '''Accept waiting connections and add them to the pollset.''' try: while True: sock, addr = lsock.accept() sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) host = addr[0] if connection_ok('diamondd', host): pconn = _PendingConn(sock, host) _log.debug('New connection from %s', pconn.peer) self._poll.register(pconn, select.POLLIN) else: sock.close() _log.info('Rejected connection from %s', host) except socket.error: pass def _traffic(self, pconn): '''Handle poll readiness events on the specified pconn.''' try: # Continue trying to read the nonce ret = pconn.read_nonce() if ret is not None: # Have the nonce. if ret == CONTROL: _log.debug('Control connection from %s, nonce %s', pconn.peer, pconn.nonce_str) pconn.send_nonce() self._nonce_to_pending[pconn.nonce] = pconn else: control = self._nonce_to_pending.get(pconn.nonce, None) if control is not None: # We have a match! Clean up pending state and # return the connection handles. _log.debug('Data connection from %s, accepted ' + 'nonce %s', pconn.peer, pconn.nonce_str) pconn.send_nonce() self._poll.unregister(control) self._poll.unregister(pconn) return (control.sock, pconn.sock) else: # No control connection for this data connection. # Close it. _log.warning('Data connection from %s, unknown ' + 'nonce %s', pconn.peer, pconn.nonce_str) self._poll.unregister(pconn) except _ConnectionClosed: # Connection died, clean it up. _nonce_to_pending holds a weak # reference to the pconn, so this should be sufficient to GC # the pconn and close the fd. _log.warning('Connection to %s died during setup', pconn.peer) self._poll.unregister(pconn) return None def accept(self): '''Returns a new (control, data) connection pair.''' while True: for pconn, _flags in self._poll.poll(): if hasattr(pconn, 'accept'): # Listening socket self._accept(pconn) else: # Traffic on a pending connection; attempt to pair it ret = self._traffic(pconn) if ret is not None: return ret # pconn may now be a dead connection; allow it to be GC'd pconn = None def shutdown(self): '''Close listening socket and all pending connections.''' self._poll.close()