class Signal(object): """ A Signal is callable. When called, it calls all the callables in its slots. """ def __init__(self): self._slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self._slots: func, _ = key func(self._slots[key], *args, **kargs) def connect(self, slot): """ Slots must call this to register a callback method. :param slot: callable """ key = (slot.im_func, id(slot.im_self)) self._slots[key] = slot.im_self def disconnect(self, slot): """ They can also unregister their callbacks here. :param slot: callable """ key = (slot.im_func, id(slot.im_self)) if key in self._slots: self._slots.pop(key) def clear(self): """ Clears all slots """ self._slots.clear()
class LRUCache: def __init__(self, max_size): self.LRU = [Node(time(), "none%s"%i) for i in range(max_size)] self.search = WeakValueDictionary() for i in self.LRU: self.search[i.name] = i def __setitem__(self, name, value): q = self.search.get(name, None) if q: q.data = value q.time = time() else: lru = self.LRU[0] self.search.pop(lru.name) lru.data = value lru.time = time() lru.name = name self.search[lru.name] = lru self.LRU.sort() def get(self, name, default=None): pos = None try: pos = self.search.__getitem__(name) pos.time = time() return pos.data except KeyError: if default is not None: return default else: raise
class Signal(object): def __init__(self): self.__slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self.__slots: func, selfid = key if selfid is not None: func(self.__slots[key], *args, **kargs) else: func(*args, **kargs) def __get_key(self, slot): if hasattr(slot, 'im_func'): return (slot.im_func, id(slot.im_self)) else: return (slot, None) def connect(self, slot): key = self.__get_key(slot) if hasattr(slot, 'im_func'): self.__slots[key] = slot.im_self else: self.__slots[key] = slot def disconnect(self, slot): key = self.__get_key(slot) if key in self.__slots: self.__slots.pop(key) def clear(self): self.__slots.clear()
class Signal(object): def __init__(self): self.__slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kargs) def connect(self, slot): key = (slot.__func__, id(slot.__self__)) self.__slots[key] = slot.__self__ def disconnect(self, slot): key = (slot.__func__, id(slot.__self__)) if key in self.__slots: self.__slots.pop(key) def clear(self): self.__slots.clear() ## Sample usage: #class Model(object): # def __init__(self, value): # self.__value = value # self.changed = Signal() # # def set_value(self, value): # self.__value = value # self.changed() # Emit signal # # def get_value(self): # return self.__value # # #class View(object): # def __init__(self, model): # self.model = model # model.changed.connect(self.model_changed) # # def model_changed(self): # print "New value:", self.model.get_value() # # #model = Model(10) #view1 = View(model) #view2 = View(model) #view3 = View(model) # #model.set_value(20) # #del view1 #model.set_value(30) # #model.changed.clear() #model.set_value(40) ### end of http://code.activestate.com/recipes/576477/ }}}
class Signal(object): def __init__(self,sender,max_connections=0, exc_catch=True): self._maxconn=max_connections self._sender=sender self._exc_catch=exc_catch self._slots = WeakValueDictionary() self._lock = threading.Lock() @property def connected(self): return len(self._slots) def connect(self, slot): if self._maxconn>0 and len(self._slots)>=self._maxconn: raise SignalError("Maximum number of connections was exceeded") assert callable(slot), "Signal slots must be callable." # Check for **kwargs try: argspec = inspect.getargspec(slot) except TypeError: try: argspec = inspect.getargspec(slot.__call__) except (TypeError, AttributeError): argspec = None if argspec: assert argspec[2] is not None, \ "Signal receivers must accept keyword arguments (**kwargs)." self._lock.acquire() try: key = (slot.im_func, id(slot.im_self)) self._slots[key] = slot.im_self finally: self._lock.release() def disconnect(self, slot): self._lock.acquire() try: key = (slot.im_func, id(slot.im_self)) if key in self._slots: self._slots.pop(key) finally: self._lock.release() def __call__(self,*args,**kwargs): assert not kwargs.has_key("sender"), \ "'sender' keyword argument is occupied" responses = [] kwargs["sender"]=self._sender for key in self._slots: func, _ = key try: response=func(self._slots[key], *args, **kwargs) responses.append((func,response)) except Exception, err: if self._exc_catch: self.exception("Slot {0} exception: {1}".format(str(func), err)) else: raise Exception(traceback.format_exc()) return responses
class Monitor(QObject): """File monitor This monitor can be used to track single files """ def __init__(self, **kwargs): super(Monitor, self).__init__(**kwargs) self.watched = WeakValueDictionary() self.delMapper = QSignalMapper(self) self.delMapper.mapped[str].connect(self.unmonitorFile) self.watcher = MonitorWithRename(parent=self) self.watcher.fileChanged.connect(self._onFileChanged) def monitorFile(self, path): """Monitor a file and return an object that tracks only `path` :rtype: SingleFileWatcher :return: an object tracking `path`, the same object is returned if the method is called with the same path. """ path = os.path.abspath(path) self.watcher.addPath(path) proxy = self.watched.get(path) if not proxy: proxy = SingleFileWatcher(path) proxy.destroyed.connect(self.delMapper.map) self.delMapper.setMapping(proxy, path) self.watched[path] = proxy return proxy @Slot(str) def unmonitorFile(self, path): """Stop monitoring a file Since there is only one :any:`SingleFileWatcher` object per path, all objects monitoring `path` will not receive notifications anymore. To let only one object stop monitoring the file, simply disconnect its `modified` signal. When the :any:`SingleFileWatcher` object returned by method :any:`monitorFile` is destroyed, the file is automatically un-monitored. """ path = os.path.abspath(path) self.watcher.removePath(path) self.watched.pop(path, None) @Slot(str) def _onFileChanged(self, path): proxy = self.watched.get(path) if proxy: proxy.modified.emit()
class Subscriber(Link, MutableMapping): def __init__(self, connect): super(Subscriber, self).__init__() self._dict = WeakValueDictionary() self._names = WeakKeyDictionary() self.connect = connect self.call() def subscribe(self, obj): self.__setitem__('_', obj) def unsubscribe(self, obj): wr = get_wrapper(obj) self._unsubscribe(wr) keys = wr._unsubscribe(self) keys.append(self._names[wr]) for key in keys: self._dict.pop(key, None) def __setitem__(self, key, obj): wr = get_wrapper(obj) self._names[wr] = key self._subscribe(wr) keys = wr._subscribe(self) keys.append(key) assert not(key != '_' and key in self._dict), 'same name' for key in keys: self._dict[key] = wr.obj return obj def __getitem__(self, key): return self._dict[key] def __delitem__(self, key): self.unsubscribe(self[key]) def __hash__(self): return Link.__hash__(self) def kill(self): for obj in set(self.links): self.unsubscribe(obj) super(Subscriber, self).kill() def call(self): pass def send(self, data): self.connect.send(data) def receive(self, data): receive(self, data)
class DictRegistryBase(RegistryBase, MappingRegistryMixin): """ A registry class with dict-based item storage. Note that the registry just owns a weak reference to the items. """ key_error_class = KeyError def __init__(self): self.mapping = WeakValueDictionary() def perform_cache_clear(self): self.mapping.clear() def register(self, key, value): """ Registration. If the given key has already existed, the function will raise a KeyError. """ if key in self.mapping: raise self.key_error_class(key) self.mapping[key] = value self.perform_register(key, value) def perform_register(self, key, value): pass def unregister(self, key): """ Unregistration. If the given key does not exist, the function will fail silently. """ self.mapping.pop(key, None) self.perform_unregister(key) def perform_unregister(self, key): pass def __getitem__(self, key): """ Shortcuts to access registered items. """ return self.mapping[key]
class Signal(object): def __init__(self): self.__slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kargs) def connect(self, slot): key = (slot.__func__, id(slot.__self__)) self.__slots[key] = slot.__self__ def disconnect(self, slot): key = (slot.__func__, id(slot.__self__)) if key in self.__slots: self.__slots.pop(key) def clear(self): self.__slots.clear()
class WSBroadcaster: def __init__(self): self.wsid_iter = count() self.websockets = WeakValueDictionary() def add_ws(self, ws, username): """Add a websocket to the pool, return its id""" wsid = next(self.wsid_iter) self.websockets[(username, wsid)] = ws self.broadcast_connected_users() return wsid def remove_ws(self, username, wsid): self.websockets.pop((username, wsid)) self.broadcast_connected_users() def list_connected_users(self): return sorted(set(username for username, wsid in self.websockets)) def broadcast_connected_users(self): self.broadcast({ 'type': 'connected_users', 'users': self.list_connected_users(), }) def broadcast(self, message): if message['type'] == 'preview': fn = self.send_bytes data = BSON.encode(message) else: fn = self.send_str data = json.dumps(message) for ws in self.websockets.values(): create_task(fn(ws, data)) async def send_str(self, ws, data): await ws.send_str(data) async def send_bytes(self, ws, data): await ws.send_bytes(data)
class Signal(object): """A lightweight Signal class when Qt is not installed.""" def __init__(self, *types): """Create the Signal object. The type signatures are ignored.""" self.__slots = WeakValueDictionary() def emit(self, *args, **kwargs): """Emit the signal, call all slots that are connected.""" for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kwargs) def connect(self, slot): """Connect this signal to a slot.""" key = (slot.im_func, id(slot.im_self)) self.__slots[key] = slot.im_self def disconnect(self, slot): """Disconnect this signal from a slot.""" key = (slot.im_func, id(slot.im_self)) if key in self.__slots: self.__slots.pop(key)
class Station(object): def __init__(self): self._id_2_protocol = WeakValueDictionary() self._unique_id_factory = UniqueIDFactory() def generate_id(self): return self._unique_id_factory.generate_id() def register(self, protocol): self._id_2_protocol[protocol.id] = protocol def unregister(self, protocol): self._id_2_protocol.pop(protocol.id, None) def outgoing_wcml_message(self, message): raise NotImplementedError def incoming_wcml_message(self, message): raise NotImplementedError def start_service(self): raise NotImplementedError
class Signal(object): """ Signal slot object, used for handling events passed to the objects in the gui. """ def __init__(self): self.__slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kargs) def connect(self, slot): key = (slot.im_func, id(slot.im_self)) self.__slots[key] = slot.im_self def disconnect(self, slot): key = (slot.im_func, id(slot.im_self)) if key in self.__slots: self.__slots.pop(key) def clear(self): self.__slots.clear()
class Signal(object): def __init__(self): self.__slots = WeakValueDictionary() # For keeping references to _FuncHost objects. self.__funchosts = {} def __call__(self, *args, **kargs): for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kargs) def connect(self, slot): if inspect.ismethod(slot): key = (slot.im_func, id(slot.im_self)) self.__slots[key] = slot.im_self else: host = _FuncHost(slot) self.connect(host.meth) # We stick a copy in here just to keep the instance alive. self.__funchosts[slot] = host def disconnect(self, slot): if inspect.ismethod(slot): key = (slot.im_func, id(slot.im_self)) if key in self.__slots: self.__slots.pop(key) else: if slot in self.__funchosts: self.disconnect(self.__funchosts[slot].meth) self.__funchosts.pop(slot) def clear(self): self.__slots.clear() self.__funchosts.clear()
class Layout(object): def __init__(self): self.solver = Solver() self.introns = WeakValueDictionary() def update(self, element): self.solver.autosolve = False for constraint in element.constraints[1]: self.solver.remove_constraint(constraint) for constraint in element.constraints[0]: self.solver.add_constraint(constraint) element.constraints = [], element.constraints[0] self.solver.autosolve = True def add_intron(self, source, intron): self.introns[source] = intron def pop_intron(self, source): return self.introns.pop(source) def get_intron(self, source, otherwise=None): return self.introns.get(source, otherwise)
class Boss: def __init__(self, os_window_id, opts, args): self.window_id_map = WeakValueDictionary() self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd ) set_boss(self) self.current_font_size = opts.font_size set_font_family(opts) self.opts, self.args = opts, args initialize_renderer() startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None): dpi_changed = False if os_window_id is None: w, h = initial_window_size(self.opts) if size is None else size cls = wclass or self.args.cls or appname os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls) if startup_id: ctx = init_startup_notification(os_window_id, startup_id) dpi_changed = show_window(os_window_id) if startup_id: end_startup_notification(ctx) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm if dpi_changed: self.on_dpi_change(os_window_id) def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } def match_windows(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) for tm in self.os_window_map.values(): for tab in tm: for window in tab: if window.matches(field, pat): yield window def tab_for_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) tms = tuple(self.os_window_map.values()) found = False if field in ('title', 'id'): for tm in tms: for tab in tm: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def peer_messages_received(self, messages): import json for msg in messages: msg = json.loads(msg.decode('utf-8')) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) session = create_session(opts, args) self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size(opts), startup_id=startup_id) else: safe_print('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = None if self.opts.allow_remote_control: try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err), 'tb': traceback.format_exc()} else: response = {'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'} if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): tm = self.os_window_map.get(os_window_id) if tm is not None: if dpi_changed: if set_dpi_from_os_window(os_window_id): self.on_dpi_change(os_window_id) else: tm.resize() else: tm.resize() def increase_font_size(self): self.change_font_size( min( self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max( MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def _change_font_size(self, new_size=None): if new_size is not None: self.current_font_size = new_size old_cell_width, old_cell_height = viewport_for_window()[-2:] windows = tuple(filter(None, self.window_id_map.values())) resize_fonts(self.current_font_size) layout_sprite_map() prerender() for window in windows: window.screen.rescale_images(old_cell_width, old_cell_height) window.screen.refresh_sprite_positions() for tm in self.os_window_map.values(): tm.resize() tm.refresh_sprite_positions() glfw_post_empty_event() def change_font_size(self, new_size): if new_size == self.current_font_size: return self._change_font_size(new_size) def on_dpi_change(self, os_window_id): self._change_font_size() @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple(w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, data): if self.opts.scrollback_in_new_tab: self.display_scrollback_in_new_tab(data) else: tab = self.active_tab if tab is not None: tab.new_special_window( SpecialWindow( self.opts.scrollback_pager, data, _('History'))) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def open_url_lines(self, lines): self.open_url(''.join(lines)) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() wakeup() self.child_monitor.join() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_sprite_map() destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.buffer_as_ansi() if arg == '@text': return w.buffer_as_text() if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: tab.new_special_window(self.args_to_special_window(args, cwd_from=cwd_from)) else: tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def display_scrollback_in_new_tab(self, data): tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=SpecialWindow( self.opts.scrollback_pager, data, _('History')))
class Agent(AgentT, Service): """Agent. This is the type of object returned by the ``@app.agent`` decorator. """ supervisor: SupervisorStrategyT = None instances: MutableSequence[ActorRefT] # channel is loaded lazily on .channel property access # to make sure configuration is not accessed when agent created # at module-scope. _channel: Optional[ChannelT] = None _channel_arg: Optional[Union[str, ChannelT]] _channel_kwargs: Dict[str, Any] _channel_iterator: Optional[AsyncIterator] = None _sinks: List[SinkT] _actors: MutableSet[ActorRefT] _actor_by_partition: MutableMapping[TP, ActorRefT] #: This mutable set is used by the first agent we start, #: so that we can update its active_partitions later #: (in on_partitions_assigned, when we know what partitions we get). _pending_active_partitions: Optional[Set[TP]] = None _first_assignment_done: bool = False def __init__(self, fun: AgentFun, *, app: AppT, name: str = None, channel: Union[str, ChannelT] = None, concurrency: int = 1, sink: Iterable[SinkT] = None, on_error: AgentErrorHandler = None, supervisor_strategy: Type[SupervisorStrategyT] = None, help: str = None, key_type: ModelArg = None, value_type: ModelArg = None, isolated_partitions: bool = False, **kwargs: Any) -> None: self.app = app self.fun: AgentFun = fun self.name = name or canonshortname(self.fun) # key-type/value_type arguments only apply when a channel # is not set if key_type is not None: assert channel is None or isinstance(channel, str) self._key_type = key_type if value_type is not None: assert channel is None or isinstance(channel, str) self._value_type = value_type self._channel_arg = channel self._channel_kwargs = kwargs self.concurrency = concurrency or 1 self.isolated_partitions = isolated_partitions self.help = help or '' self._sinks = list(sink) if sink is not None else [] self._on_error: Optional[AgentErrorHandler] = on_error self.supervisor_strategy = supervisor_strategy self._actors = WeakSet() self._actor_by_partition = WeakValueDictionary() if self.isolated_partitions and self.concurrency > 1: raise ImproperlyConfigured( 'Agent concurrency must be 1 when using isolated partitions') Service.__init__(self) async def _start_one(self, *, index: Optional[int] = None, active_partitions: Optional[Set[TP]] = None, stream: StreamT = None, channel: ChannelT = None) -> ActorT: # an index of None means there's only one instance, # and `index is None` is used as a test by functions that # disallows concurrency. index = index if self.concurrency > 1 else None return await self._start_task( index=index, active_partitions=active_partitions, stream=stream, channel=channel, beacon=self.beacon, ) async def _start_one_supervised( self, index: Optional[int] = None, active_partitions: Optional[Set[TP]] = None, stream: StreamT = None) -> ActorT: aref = await self._start_one( index=index, active_partitions=active_partitions, stream=stream, ) self.supervisor.add(aref) await aref.maybe_start() return aref async def _start_for_partitions(self, active_partitions: Set[TP]) -> ActorT: assert active_partitions self.log.info('Starting actor for partitions %s', active_partitions) return await self._start_one_supervised(None, active_partitions) async def on_start(self) -> None: self.supervisor = self._new_supervisor() await self._on_start_supervisor() def _new_supervisor(self) -> SupervisorStrategyT: return self._get_supervisor_strategy()( max_restarts=100.0, over=1.0, replacement=self._replace_actor, loop=self.loop, beacon=self.beacon, ) async def _replace_actor(self, service: ServiceT, index: int) -> ServiceT: aref = cast(ActorRefT, service) return await self._start_one( index=index, active_partitions=aref.active_partitions, stream=aref.stream, channel=cast(ChannelT, aref.stream.channel), ) def _get_supervisor_strategy(self) -> Type[SupervisorStrategyT]: SupervisorStrategy = self.supervisor_strategy if SupervisorStrategy is None: SupervisorStrategy = self.app.conf.agent_supervisor return SupervisorStrategy async def _on_start_supervisor(self) -> None: active_partitions = self._get_active_partitions() channel: ChannelT = cast(ChannelT, None) for i in range(self.concurrency): res = await self._start_one( index=i, active_partitions=active_partitions, channel=channel, ) if channel is None: # First concurrency actor creates channel, # then we reuse it for --concurrency=n. # This way they share the same queue. channel = res.stream.channel self.supervisor.add(res) await self.supervisor.start() def _get_active_partitions(self) -> Optional[Set[TP]]: active_partitions: Optional[Set[TP]] = None if self.isolated_partitions: # when we start our first agent, we create the set of # partitions early, and save it in ._pending_active_partitions. # That way we can update the set once partitions are assigned, # and the actor we started may be assigned one of the partitions. active_partitions = self._pending_active_partitions = set() return active_partitions async def on_stop(self) -> None: # Agents iterate over infinite streams, so we cannot wait for it # to stop. # Instead we cancel it and this forces the stream to ack the # last message processed (but not the message causing the error # to be raised). await self._stop_supervisor() async def _stop_supervisor(self) -> None: if self.supervisor: await self.supervisor.stop() self.supervisor = None def cancel(self) -> None: for actor in self._actors: actor.cancel() async def on_partitions_revoked(self, revoked: Set[TP]) -> None: T = self.app.traced if self.isolated_partitions: # isolated: start/stop actors for each partition await T(self.on_isolated_partitions_revoked)(revoked) else: await T(self.on_shared_partitions_revoked)(revoked) async def on_partitions_assigned(self, assigned: Set[TP]) -> None: T = self.app.traced if self.isolated_partitions: await T(self.on_isolated_partitions_assigned)(assigned) else: await T(self.on_shared_partitions_assigned)(assigned) async def on_isolated_partitions_revoked(self, revoked: Set[TP]) -> None: self.log.dev('Partitions revoked') T = self.app.traced for tp in revoked: aref: Optional[ActorRefT] = self._actor_by_partition.pop(tp, None) if aref is not None: await T(aref.on_isolated_partition_revoked)(tp) async def on_isolated_partitions_assigned(self, assigned: Set[TP]) -> None: T = self.app.traced for tp in sorted(assigned): await T(self._assign_isolated_partition)(tp) async def _assign_isolated_partition(self, tp: TP) -> None: T = self.app.traced if (not self._first_assignment_done and not self._actor_by_partition): self._first_assignment_done = True # if this is the first time we are assigned # we need to reassign the agent we started at boot to # one of the partitions. T(self._on_first_isolated_partition_assigned)(tp) await T(self._maybe_start_isolated)(tp) def _on_first_isolated_partition_assigned(self, tp: TP) -> None: assert self._actors assert len(self._actors) == 1 self._actor_by_partition[tp] = next(iter(self._actors)) if self._pending_active_partitions is not None: assert not self._pending_active_partitions self._pending_active_partitions.add(tp) async def _maybe_start_isolated(self, tp: TP) -> None: try: aref = self._actor_by_partition[tp] except KeyError: aref = await self._start_isolated(tp) self._actor_by_partition[tp] = aref await aref.on_isolated_partition_assigned(tp) async def _start_isolated(self, tp: TP) -> ActorT: return await self._start_for_partitions({tp}) async def on_shared_partitions_revoked(self, revoked: Set[TP]) -> None: ... async def on_shared_partitions_assigned(self, assigned: Set[TP]) -> None: ... def info(self) -> Mapping: return { 'app': self.app, 'fun': self.fun, 'name': self.name, 'channel': self.channel, 'concurrency': self.concurrency, 'help': self.help, 'sinks': self._sinks, 'on_error': self._on_error, 'supervisor_strategy': self.supervisor_strategy, 'isolated_partitions': self.isolated_partitions, } def clone(self, *, cls: Type[AgentT] = None, **kwargs: Any) -> AgentT: return (cls or type(self))(**{**self.info(), **kwargs}) def test_context(self, channel: ChannelT = None, supervisor_strategy: SupervisorStrategyT = None, on_error: AgentErrorHandler = None, **kwargs: Any) -> AgentTestWrapperT: # pragma: no cover # flow control into channel queues are disabled at startup, # so need to resume that. self.app.flow_control.resume() async def on_agent_error(agent: AgentT, exc: BaseException) -> None: if on_error is not None: await on_error(agent, exc) await agent.crash_test_agent(exc) return self.clone( cls=AgentTestWrapper, channel=channel if channel is not None else self.app.channel(), supervisor_strategy=supervisor_strategy or CrashingSupervisor, original_channel=self.channel, on_error=on_agent_error, **kwargs) def _prepare_channel(self, channel: Union[str, ChannelT] = None, internal: bool = True, key_type: ModelArg = None, value_type: ModelArg = None, **kwargs: Any) -> ChannelT: app = self.app channel = f'{app.conf.id}-{self.name}' if channel is None else channel if isinstance(channel, ChannelT): return cast(ChannelT, channel) elif isinstance(channel, str): return app.topic( channel, internal=internal, key_type=key_type, value_type=value_type, **kwargs) raise TypeError( f'Channel must be channel, topic, or str; not {type(channel)}') def __call__(self, *, index: int = None, active_partitions: Set[TP] = None, stream: StreamT = None, channel: ChannelT = None) -> ActorRefT: # The agent function can be reused by other agents/tasks. # For example: # # @app.agent(logs_topic, through='other-topic') # filter_log_errors_(stream): # async for event in stream: # if event.severity == 'error': # yield event # # @app.agent(logs_topic) # def alert_on_log_error(stream): # async for event in filter_log_errors(stream): # alert(f'Error occurred: {event!r}') # # Calling `res = filter_log_errors(it)` will end you up with # an AsyncIterable that you can reuse (but only if the agent # function is an `async def` function that yields) if stream is None: stream = self.stream( channel=channel, concurrency_index=index, active_partitions=active_partitions, ) else: # reusing actor stream after agent restart assert stream.concurrency_index == index assert stream.active_partitions == active_partitions return self.actor_from_stream(stream) def actor_from_stream(self, stream: StreamT) -> ActorRefT: res = self.fun(stream) typ = cast(Type[Actor], (AwaitableActor if isinstance(res, Awaitable) else AsyncIterableActor)) return typ( self, stream, res, index=stream.concurrency_index, active_partitions=stream.active_partitions, loop=self.loop, beacon=self.beacon, ) def add_sink(self, sink: SinkT) -> None: if sink not in self._sinks: self._sinks.append(sink) def stream(self, channel: ChannelT = None, active_partitions: Set[TP] = None, **kwargs: Any) -> StreamT: if channel is None: channel = cast(TopicT, self.channel_iterator).clone( is_iterator=False, active_partitions=active_partitions, ) if active_partitions is not None: assert channel.active_partitions == active_partitions s = self.app.stream( channel, loop=self.loop, active_partitions=active_partitions, **kwargs) s.add_processor(self._maybe_unwrap_reply_request) return s def _maybe_unwrap_reply_request(self, value: V) -> Any: if isinstance(value, ReqRepRequest): return value.value return value async def _start_task(self, *, index: Optional[int], active_partitions: Optional[Set[TP]] = None, stream: StreamT = None, channel: ChannelT = None, beacon: NodeT = None) -> ActorRefT: # If the agent is an async function we simply start it, # if it returns an AsyncIterable/AsyncGenerator we start a task # that will consume it. actor = self( index=index, active_partitions=active_partitions, stream=stream, channel=channel, ) return await self._prepare_actor(actor, beacon) async def _prepare_actor(self, aref: ActorRefT, beacon: NodeT) -> ActorRefT: coro: Any if isinstance(aref, Awaitable): # agent does not yield coro = aref if self._sinks: raise ImproperlyConfigured('Agent must yield to use sinks') else: # agent yields and is an AsyncIterator so we have to consume it. coro = self._slurp(aref, aiter(aref)) task = asyncio.Task(self._execute_task(coro, aref), loop=self.loop) task._beacon = beacon # type: ignore aref.actor_task = task self._actors.add(aref) return aref async def _execute_task(self, coro: Awaitable, aref: ActorRefT) -> None: # This executes the agent task itself, and does exception handling. _current_agent.set(self) try: await coro except asyncio.CancelledError: if self.should_stop: raise except Exception as exc: if self._on_error is not None: await self._on_error(self, exc) # Mark ActorRef as dead, so that supervisor thread # can start a new one. await aref.crash(exc) self.supervisor.wakeup() raise async def _slurp(self, res: ActorRefT, it: AsyncIterator) -> None: # this is used when the agent returns an AsyncIterator, # and simply consumes that async iterator. stream: Optional[StreamT] = None async for value in it: self.log.debug('%r yielded: %r', self.fun, value) if stream is None: stream = res.stream.get_active_stream() event = stream.current_event if event is not None and isinstance(event.value, ReqRepRequest): await self._reply(event.key, value, event.value) await self._delegate_to_sinks(value) async def _delegate_to_sinks(self, value: Any) -> None: for sink in self._sinks: if isinstance(sink, AgentT): await cast(AgentT, sink).send(value=value) elif isinstance(sink, ChannelT): await cast(TopicT, sink).send(value=value) else: await maybe_async(cast(Callable, sink)(value)) async def _reply(self, key: Any, value: Any, req: ReqRepRequest) -> None: assert req.reply_to response = self._response_class(value)( key=key, value=value, correlation_id=req.correlation_id, ) await self.app.send( req.reply_to, key=None, value=response, ) def _response_class(self, value: Any) -> Type[ReqRepResponse]: if isinstance(value, ModelT): return ModelReqRepResponse return ReqRepResponse async def cast(self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None) -> None: await self.send( key=key, value=value, partition=partition, timestamp=timestamp, ) async def ask(self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None, reply_to: ReplyToArg = None, correlation_id: str = None) -> Any: p = await self.ask_nowait( value, key=key, partition=partition, timestamp=timestamp, reply_to=reply_to or self.app.conf.reply_to, correlation_id=correlation_id, force=True, # Send immediately, since we are waiting for result. ) app = cast(App, self.app) await app._reply_consumer.add(p.correlation_id, p) await app.maybe_start_client() return await p async def ask_nowait(self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None, reply_to: ReplyToArg = None, correlation_id: str = None, force: bool = False) -> ReplyPromise: req = self._create_req(key, value, reply_to, correlation_id) await self.channel.send( key=key, value=req, partition=partition, timestamp=timestamp, force=force, ) return ReplyPromise(req.reply_to, req.correlation_id) def _create_req(self, key: K = None, value: V = None, reply_to: ReplyToArg = None, correlation_id: str = None) -> ReqRepRequest: if reply_to is None: raise TypeError('Missing reply_to argument') topic_name = self._get_strtopic(reply_to) correlation_id = correlation_id or str(uuid4()) return self._request_class(value)( value=value, reply_to=topic_name, correlation_id=correlation_id, ) def _request_class(self, value: V) -> Type[ReqRepRequest]: if isinstance(value, ModelT): return ModelReqRepRequest return ReqRepRequest async def send(self, *, key: K = None, value: V = None, partition: int = None, timestamp: float = None, key_serializer: CodecArg = None, value_serializer: CodecArg = None, callback: MessageSentCallback = None, reply_to: ReplyToArg = None, correlation_id: str = None, force: bool = False) -> Awaitable[RecordMetadata]: """Send message to topic used by agent.""" if reply_to: value = self._create_req(key, value, reply_to, correlation_id) return await self.channel.send( key=key, value=value, partition=partition, timestamp=timestamp, key_serializer=key_serializer, value_serializer=value_serializer, force=force, ) def _get_strtopic(self, topic: Union[str, ChannelT, TopicT, AgentT]) -> str: if isinstance(topic, AgentT): return self._get_strtopic(cast(AgentT, topic).channel) if isinstance(topic, TopicT): return cast(TopicT, topic).get_topic_name() if isinstance(topic, ChannelT): raise ValueError('Channels are unnamed topics') return cast(str, topic) async def map(self, values: Union[AsyncIterable, Iterable], key: K = None, reply_to: ReplyToArg = None, ) -> AsyncIterator: # pragma: no cover # Map takes only values, but can provide one key that is used for all. async for value in self.kvmap( ((key, v) async for v in aiter(values)), reply_to): yield value async def kvmap( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg = None, ) -> AsyncIterator[str]: # pragma: no cover # kvmap takes (key, value) pairs. reply_to = self._get_strtopic(reply_to or self.app.conf.reply_to) # BarrierState is the promise that keeps track of pending results. # It contains a list of individual ReplyPromises. barrier = BarrierState(reply_to) async for _ in self._barrier_send(barrier, items, reply_to): # Now that we've sent a message, try to see if we have any # replies. try: _, val = barrier.get_nowait() except asyncio.QueueEmpty: pass else: yield val # All the messages have been sent so finalize the barrier. barrier.finalize() # Then iterate over the results in the group. async for _, value in barrier.iterate(): yield value async def join(self, values: Union[AsyncIterable[V], Iterable[V]], key: K = None, reply_to: ReplyToArg = None, ) -> List[Any]: # pragma: no cover return await self.kvjoin( ((key, value) async for value in aiter(values)), reply_to=reply_to, ) async def kvjoin( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg = None) -> List[Any]: # pragma: no cover reply_to = self._get_strtopic(reply_to or self.app.conf.reply_to) barrier = BarrierState(reply_to) # Map correlation_id -> index posindex: MutableMapping[str, int] = { cid: i async for i, cid in aenumerate( self._barrier_send(barrier, items, reply_to)) } # All the messages have been sent so finalize the barrier. barrier.finalize() # wait until all replies received await barrier # then construct a list in the correct order. values: List = [None] * barrier.total async for correlation_id, value in barrier.iterate(): values[posindex[correlation_id]] = value return values async def _barrier_send( self, barrier: BarrierState, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg) -> AsyncIterator[str]: # pragma: no cover # map: send many tasks to agents # while trying to pop incoming results off. async for key, value in aiter(items): correlation_id = str(uuid4()) p = await self.ask_nowait( key=key, value=value, reply_to=reply_to, correlation_id=correlation_id) # add reply promise to the barrier barrier.add(p) # the ReplyConsumer will call the barrier whenever a new # result comes in. app = cast(App, self.app) await app.maybe_start_client() await app._reply_consumer.add(p.correlation_id, barrier) yield correlation_id def _repr_info(self) -> str: return shorten_fqdn(self.name) def get_topic_names(self) -> Iterable[str]: channel = self.channel if isinstance(channel, TopicT): return channel.topics return [] @property def channel(self) -> ChannelT: if self._channel is None: self._channel = self._prepare_channel( self._channel_arg, key_type=self._key_type, value_type=self._value_type, **self._channel_kwargs, ) return self._channel @channel.setter def channel(self, channel: ChannelT) -> None: self._channel = channel @property def channel_iterator(self) -> AsyncIterator: # The channel is "memoized" here, so subsequent access to # instance.channel_iterator will return the same value. if self._channel_iterator is None: # we do not use aiter(channel) here, because # that will also add it to the topic conductor too early. self._channel_iterator = self.channel.clone(is_iterator=True) return self._channel_iterator @channel_iterator.setter def channel_iterator(self, it: AsyncIterator) -> None: self._channel_iterator = it @property def label(self) -> str: return self._agent_label() def _agent_label(self, name_suffix: str = '') -> str: s = f'{type(self).__name__}{name_suffix}: ' s += f'{shorten_fqdn(qualname(self.fun))}' return s @property def shortlabel(self) -> str: return self._agent_label()
class Boss: def __init__(self, glfw_window, opts, args): self.window_id_map = WeakValueDictionary() startup_session = create_session(opts, args) self.cursor_blinking = True self.window_is_focused = True self.glfw_window_title = None self.shutting_down = False self.child_monitor = ChildMonitor( glfw_window.window_id(), self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None) set_boss(self) self.current_font_size = opts.font_size cell_size.width, cell_size.height = set_font_family(opts) self.opts, self.args = opts, args self.glfw_window = glfw_window glfw_window.framebuffer_size_callback = self.on_window_resize glfw_window.window_focus_callback = self.on_focus load_shader_programs() self.tab_manager = TabManager(opts, args) self.tab_manager.init(startup_session) self.activate_tab_at = self.tab_manager.activate_tab_at layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper) @property def current_tab_bar_height(self): return self.tab_manager.tab_bar_height def __iter__(self): return iter(self.tab_manager) def iterwindows(self): for t in self: yield from t def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def on_child_death(self, window_id): w = self.window_id_map.pop(window_id, None) if w is not None: w.on_child_death() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def on_window_resize(self, window, w, h): viewport_size.width, viewport_size.height = w, h self.tab_manager.resize() def increase_font_size(self): self.change_font_size( min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def change_font_size(self, new_size): if new_size == self.current_font_size: return self.current_font_size = new_size w, h = cell_size.width, cell_size.height windows = tuple(filter(None, self.window_id_map.values())) cell_size.width, cell_size.height = set_font_family( self.opts, override_font_size=self.current_font_size) layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper) for window in windows: window.screen.rescale_images(w, h) self.resize_windows_after_font_size_change() for window in windows: window.screen.refresh_sprite_positions() self.tab_manager.refresh_sprite_positions() def resize_windows_after_font_size_change(self): self.tab_manager.resize() glfw_post_empty_event() def tabbar_visibility_changed(self): self.tab_manager.resize(only_tabs=True) glfw_post_empty_event() @property def active_tab(self): return self.tab_manager.active_tab def is_tab_visible(self, tab): return self.active_tab is tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True key, scancode, action, mods = self.current_key_press_info data = get_sent_data(self.opts.send_text_map, key, scancode, mods, window, action) if data: window.write_to_child(data) return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, window, focused): self.window_is_focused = focused w = self.active_window if w is not None: w.focus_changed(focused) def display_scrollback(self, data): if self.opts.scrollback_in_new_tab: self.display_scrollback_in_new_tab(data) else: tab = self.active_tab if tab is not None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'))) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def send_fake_scroll(self, window_idx, amt, upwards): tab = self.active_tab w = tab.windows[window_idx] k = get_key_map(w.screen)[GLFW_KEY_UP if upwards else GLFW_KEY_DOWN] w.write_to_child(k * amt) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def gui_close_window(self, window): window.destroy() for tab in self.tab_manager: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: self.tab_manager.remove(tab) tab.destroy() if len(self.tab_manager) == 0: if not self.shutting_down: self.glfw_window.set_should_close(True) glfw_post_empty_event() def destroy(self): self.shutting_down = True self.child_monitor.shutdown() wakeup() self.child_monitor.join() for t in self.tab_manager: t.destroy() del self.tab_manager destroy_sprite_map() destroy_global_data() del self.glfw_window def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = self.glfw_window.get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) def next_tab(self): self.tab_manager.next_tab() def previous_tab(self): self.tab_manager.next_tab(-1) def new_tab(self): self.tab_manager.new_tab() def move_tab_forward(self): self.tab_manager.move_tab(1) def move_tab_backward(self): self.tab_manager.move_tab(-1) def display_scrollback_in_new_tab(self, data): self.tab_manager.new_tab(special_window=SpecialWindow( self.opts.scrollback_pager, data, _('History')))
class AsyncTornadoRPCClient(AsyncRabbitConnectionBase): """ Wrap `pika.adapters.tornado_connection.TornadoConnection` to provide a simple RPC client powered by ``tornado.gen.engine`` semantics. """ def __init__(self, *args, **kwargs): self.declare_rpc_queue_lock = Lock() self.connection_open_callbacks = [] self.rpc_queue_callbacks = [] self.rpc_queue = None self.futures = WeakValueDictionary() super(AsyncTornadoRPCClient, self).__init__(*args, **kwargs) def on_closed(self, connection): logger.warning('AsyncRabbitClient.on_close: closed!') self.rpc_queue = None @gen.engine def rpc_queue_declare(self, callback, **kwargs): if not self.declare_rpc_queue_lock.acquire(False): logger.info('RPC Queue is already in the process of ' 'being declared (declare_rpc_queue_lock ' 'could not be acquired).') callback() return try: self.rpc_queue = yield gen.Task(self.queue_declare, exclusive=True, auto_delete=True) yield gen.Task(self.basic_consume, queue=self.rpc_queue) logger.info('Adding callbacks that are waiting for an RPC ' 'queue to the tornado queue.') while self.rpc_queue_callbacks: cb = self.rpc_queue_callbacks.pop() self.io_loop.add_callback(cb) logger.info('Done adding callbacks.') finally: self.declare_rpc_queue_lock.release() callback() @gen.engine def ensure_rpc_queue(self, callback): logger.info('Ensuring that an RPC queue has been declared.') yield Task(self.ensure_connection) if self.rpc_queue: logger.info('The RPC queue is already open.') callback() else: logger.info('Adding callback to list of callbacks ' 'waiting for the RPC queue to be open.') callback = stack_context.wrap(callback) self.rpc_queue_callbacks.append(callback) logger.info('Calling rpc_queue_declare().') rpc_queue = yield gen.Task(self.rpc_queue_declare) logger.info('rpc_queue_declare has been called.') @gen.engine def rpc(self, rpc_request, properties=None, callback=None): ''' Publish an RPC request. Returns a :class:`RPCResponseFuture`. :param rpc_request: An instance of :class:`RPCRequest`. ''' callback = stack_context.wrap(callback) yield Task(self.ensure_connection) yield Task(self.ensure_rpc_queue) if not properties: correlation_id = str(uuid.uuid4()) properties = pika.BasicProperties(reply_to=self.rpc_queue, correlation_id=correlation_id) logger.info('Publishing RPC request with key: %s' % rpc_request.routing_key) self.channel.basic_publish(exchange=rpc_request.exchange, routing_key=rpc_request.routing_key, body=rpc_request.json_params, properties=properties) logger.info('Constructing RPC response future with cid: %s' % correlation_id) future = RPCResponseFuture(correlation_id, timeout=rpc_request.timeout, io_loop=self.io_loop) self.futures[correlation_id] = future callback(future) @gen.engine def basic_publish(self, rpc_request, properties=None, callback=None): yield Task(self.ensure_connection) yield Task(self.ensure_rpc_queue) if not properties: properties = pika.BasicProperties() logger.info('Publishing message request with key: %s' % rpc_request.routing_key) self.channel.basic_publish(exchange=rpc_request.exchange, routing_key=rpc_request.routing_key, body=rpc_request.json_params, properties=properties) logger.info('channel.basic_publish finished.') callback() def consume_message(self, channel, method, header, body): logger.info('RPC response consumed') cid = header.correlation_id try: future = self.futures.pop(cid) response = RPCResponse(channel, method, header, body) cb = partial(future.response_callback, response) self.io_loop.add_callback(cb) except KeyError: logger.warning('AsyncRabbitClient.consume_message received an' ' unrecognized correlation_id: %s. Maybe the' ' RPC took too long and was timed out, or maybe' ' the response was sent more than once.' % cid)
class EntityCache(object): """ Cache for entities. Supports add and remove operations as well as lookup by ID and by slug. """ def __init__(self, entities=None, allow_none_id=True): """ :param bool allow_none_id: Flag specifying if calling :meth:`add` with an entity that does not have an ID is allowed. """ # Flag indicating if None IDs are allowed in this cache. self.__allow_none_id = allow_none_id # List of cached entities. This is the only place we are holding a # real reference to the entity. if entities is None: entities = [] self.__entities = entities # Dictionary mapping entity IDs to entities for fast lookup by ID. self.__id_map = WeakValueDictionary() # Dictionary mapping entity slugs to entities for fast lookup by slug. self.__slug_map = {} def get_by_id(self, entity_id): """ Performs a lookup of an entity by its ID. :param int entity_id: entity ID. :return: entity found or ``None``. """ return self.__id_map.get(entity_id) def has_id(self, entity_id): """ Checks if this entity cache holds an entity with the given ID. :return: Boolean result of the check. """ return entity_id in self.__id_map def get_by_slug(self, entity_slug): """ Performs a lookup of an entity by its slug. :param str entity_id: entity slug. :return: entity found or ``None``. """ return self.__slug_map.get(entity_slug) def has_slug(self, entity_slug): return entity_slug in self.__slug_map def add(self, entity): """ Adds the given entity to this cache. :param entity: Entity to add. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises ValueError: If the ID of the entity to add is ``None`` (unless the `allow_none_id` constructor argument was set). """ do_append = self.__check_new(entity) if do_append: self.__entities.append(entity) def remove(self, entity): """ Removes the given entity from this cache. :param entity: Entity to remove. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises KeyError: If the given entity is not in this cache. :raises ValueError: If the ID of the given entity is `None`. """ self.__id_map.pop(entity.id, None) self.__slug_map.pop(entity.slug, None) self.__entities.remove(entity) def update(self, source_data, target_entity): """ Updates the state of the target entity with the given source data. :param target_entity: Entity to update. :type target_entity: Object implementing :class:`everest.interfaces.IEntity`. """ EntityState.set_state_data(target_entity, source_data) def get_all(self): """ Returns the list of all entities in this cache in the order they were added. """ return self.__entities def retrieve(self, filter_expression=None, order_expression=None, slice_key=None): """ Retrieve entities from this cache, possibly after filtering, ordering and slicing. """ ents = iter(self.__entities) if not filter_expression is None: ents = filter_expression(ents) if not order_expression is None: # Ordering always involves a copy and conversion to a list, so # we have to wrap in an iterator. ents = iter(order_expression(ents)) if not slice_key is None: ents = islice(ents, slice_key.start, slice_key.stop) return ents def rebuild(self, entities): """ Rebuilds the ID and slug maps of this cache. This can be necessary when entities obtain their IDs only after they have been flushed to the backend. """ for ent in entities: self.__check_new(ent) def __contains__(self, entity): if not entity.id is None: is_contained = entity.id in self.__id_map else: is_contained = entity in self.__entities return is_contained def __check_new(self, entity): # For certain use cases (e.g., staging), we do not want the entity to # be added to have an ID yet. do_append = True if not entity.id is None: if entity.id in self.__id_map: if not self.__id_map[entity.id] is entity: raise ValueError('Duplicate entity ID "%s". %s' % (entity.id, entity)) else: do_append = False else: self.__id_map[entity.id] = entity elif not self.__allow_none_id: raise ValueError('Entity ID must not be None.') # The slug can be a lazy attribute depending on the # value of other (possibly not yet initialized) attributes which is # why we can not always assume it is available at this point. if do_append and hasattr(entity, 'slug') and not entity.slug is None: ents = self.__slug_map.get(entity.slug) if not ents is None: ents.append(entity) else: self.__slug_map[entity.slug] = WeakList([entity]) return do_append
class HeapManager(threading.Thread): """ @summary: This class is intended to manage all dataClay objects in runtime's memory. """ """ Logger """ logger = None def __init__(self, theruntime): """ @postcondition: Constructor of the object called from sub-class @param theruntime: Runtime being managed """ """ Memory objects. This dictionary must contain all objects in runtime memory (client or server), as weakrefs. """ self.inmemory_objects = WeakValueDictionary() threading.Thread.__init__(self) self._finished = threading.Event() """ Runtime being monitorized. Java uses abstract functions to get the field in the proper type (EE or client) due to type-check. Not needed here. """ self.runtime = theruntime self.logger = logging.getLogger(__name__) self.daemon = True self.logger.debug("HEAP MANAGER created.") def get_heap(self): return self.inmemory_objects def shutdown(self): """Stop this thread""" self.logger.debug("HEAP MANAGER shutdown request received.") self._finished.set() def run(self): """ @postcondition: Overrides run function """ gc_check_time_interval_seconds = Configuration.MEMMGMT_CHECK_TIME_INTERVAL / 1000.0 while 1: self.logger.trace("HEAP MANAGER THREAD is awake...") if self._finished.isSet(): break self.run_task() # sleep for interval or until shutdown self.logger.trace("HEAP MANAGER THREAD is going to sleep...") self._finished.wait(gc_check_time_interval_seconds) self.logger.debug("HEAP MANAGER THREAD Finished.") def _add_to_inmemory_map(self, dc_object): """ @postcondition: the object is added to inmemory map @param dc_object: object to add """ oid = dc_object.get_object_id() self.inmemory_objects[oid] = dc_object def remove_from_heap(self, object_id): """ @postcondition: Remove reference from Heap. Even if we remove it from the heap, the object won't be Garbage collected till HeapManager flushes the object and releases it. @param object_id: id of object to remove from heap """ self.inmemory_objects.pop(object_id) def get_from_heap(self, object_id): """ @postcondition: Get from heap. @param object_id: id of object to get from heap @return Object with id provided in heap or None if not found. """ try: obj = self.inmemory_objects[object_id] self.logger.debug("Hit in Heap object %s" % str(object_id)) return obj except KeyError: self.logger.debug("Miss in Heap object %s" % str(object_id)) return None def exists_in_heap(self, object_id): """ @postcondition: Exists from heap. @param object_id: id of object to get from heap @return True if exists. False otherwise. """ try: if self.inmemory_objects[object_id] is None: return False else: return True except KeyError: return False def heap_size(self): """ @postcondition: Get heap size. @return Heap size """ return len(self.inmemory_objects) def count_loaded_objs(self): num_loaded_objs = 0 for obj in self.inmemory_objects.values(): if obj.is_loaded(): num_loaded_objs = num_loaded_objs + 1 return num_loaded_objs @abstractmethod def flush_all(self): pass @abstractmethod def run_task(self): pass def cleanReferencesAndLockers(self): """ @postcondition: Clean references and lockers not being used. """ self.runtime.locker_pool.cleanLockers()
class RPC(RPC_): """RPC (without underscore) extends RPC_ with the possibility of sending (and receiving) responses. RPCs differ from normal messages in that they generate an additional "response" message. It adds: * An additional wrapper (built over the one in RPC_) for callbacks that serializes the return (or any exception that is raised) and sends it to a "response" queue. * A new endpoint type "rpc" * A low level publish_rpc method that works similar to Queue.publish except that it does the preparations so that a response can be received. * A high level `rpc` method for calling remote procedures in a friendly way. * The constructor take and additional "loop" argument. The rpc interface is based on Futures. Each calls yields a Future object that can be awaited. To use the RPC client, you must first call the start_client() coroutine-method to create the response queue and register the receiver callback. Implementation details: The Futures for unanswered rpcs are kept in a WeakValueDictionary. If the application looses all references for a Future then it is deleted and can't be recovered. When the response arrives, it will be discarded. """ def __init__(self, loop, *args, **kwargs): """loop must be a asycio.BaseEventLoop-derived object. The rest of the argument are passed on to the superclass.""" super().__init__(*args, **kwargs) self._loop = loop self._correlation_gen = itertools.count() self._open_responses = WeakValueDictionary() self._client_started = False # RPC server: Methods dealing with receiving, processing and # responding to RPCs async def _rpc_receive_wrapper(self, f, channel, body, envelope, properties, ack_mode, **kwargs): try: r = await self._rpc_noret_receive_wrapper(f, channel, body, envelope, properties, ack_mode=ack_mode, **kwargs) except Exception as e: rdata, ctype, cenc = self.format_exception(e) else: rdata, ctype, cenc = self.format_response(r) cid = properties.correlation_id new_props = { 'timestamp': _tstamp(), 'content_type': ctype, 'content_encoding': cenc, 'correlation_id': cid, 'reply_to': properties.reply_to } logger.debug("Sending response: %s", cid) await self.channel.publish(rdata, '', properties.reply_to, properties=new_props) def register_rpc(self, queue_name, routing_key, function, ack_mode=ACK.receive, extended=False, **kwargs): wrapped = functools.partial(self._rpc_receive_wrapper, function, ack_mode=ack_mode, extended=extended) return self.register_queue(queue_name, routing_key, wrapped, **kwargs) @staticmethod def format_response(obj): data = json.dumps({'return': obj}).encode("utf-8") return data, "application/json", "utf-8" @staticmethod def format_exception(exception): """Convert `exception` to a dict and serialize it""" err_info = { 'exc_type': type(exception).__name__, 'exc_message': str(exception), 'traceback': traceback.format_tb(exception.__traceback__) } data = json.dumps(err_info).encode("utf-8") return data, "application/json", "utf-8" # RPC client: Methods dealing with sending RPC requests and # receiving the results. async def start_client(self): """Declare the response queue. You should call this only once. After the first call, further call to start_client() will do nothing. """ if self._client_started: return r = await self.channel.queue_declare(exclusive=True, auto_delete=True) self.response_queue = r['queue'] logger.debug("Return queue is: %s", self.response_queue) await self.channel.basic_consume(self._response_handler, self.response_queue, no_ack=True) self._client_started = True async def _response_handler(self, channel, body, envelope, properties): d = self.decode_response(body) cid = properties.correlation_id try: pending_future = self._open_responses.pop(cid) except KeyError: logger.debug("Discarded response: %s", cid) return if "return" in d: pending_future.set_result(d["return"]) else: pending_future.set_exception( RemoteException(d["exc_message"], d["exc_type"], d["traceback"])) async def publish_rpc(self, data, routing_key, **kwargs): """Publish a message along with the necessary steps to allow for a reply: * Create a future and register it. * Set the reply_to field. * Generate a new correlation id. Returns: future: A new asyncio.Future object associated with the reply to this message. The future is registered in self._open_responses. """ # The correlation_id MUST be a string (why????) cid = str(next(self._correlation_gen)) future = _create_future(self._loop) self._open_responses[cid] = future rpc_properties = { 'reply_to': self.response_queue, 'correlation_id': cid } rpc_properties.update(kwargs.pop("properties", ())) await self.publish(data, routing_key, properties=rpc_properties, **kwargs) return future def rpc(self, proc_name, mandatory=False, immediate=False, **properties): """Return a callable: f(*args, **kwargs) that can be used to do remote procedure calls that send a reply. The callable returns a coroutine that does the remote call and yields an asyncio.Future object. The result (or error) of the Future is set when a reply is received. The RPC object keeps weak references to all Futures created. If the Future is destroyed, the reply to that RPC will be silently ignored. Example: Assuming client is a RPC object ... f = client.rpc("some_function") ... x = await f(1, 2, r = 0) ... result = await x """ return functools.partial(self._call, proc_name, self.publish_rpc, properties, mandatory, immediate) @staticmethod def decode_response(body): d = json.loads(body.decode("utf-8")) return d
class Storage(object): def __init__(self, cache=None): # {(InstanceInfo(instance).model_info.model, instance.pk): InstanceInfo, ...}} self._alive = WeakValueDictionary() # {InstanceInfo(instance): instance, ...}} self._dirty = {} if cache is not None: self._cache = cache else: self._cache = MRUCache(settings.MAX_CACHE) signals.register( ( "instance-deleted", "start-tracking-changes", "stop-tracking-changes", "model-pre-init", "model-post-init", "model-pre-save", "model-post-save", "model-pre-delete", "model-post-delete", "model-pre-update", "model-post-update", # "model-history-reset", # "model-history-redo", # "model-history-undo", "relation-pre-get", "relation-post-get", "relation-pre-set", "relation-post-set", "relation-pre-delete", "relation-post-delete", "relation-pre-add", "relation-post-add", "relation-pre-remove", "relation-post-remove", "model-do-cache", "model-do-not-cache", ), self ) signals.register_with_callback("cache-rollback", self, "rollback") def get(self, query): return QuerySetIterator(query, self) def clear(self): self._dirty.clear() self._alive.clear() self._cache.clear() def set_dirty(self, inst_info): self._dirty[inst_info] = inst_info.get_inst() def cache(self, inf): self._alive.pop((inf.model_info.model, inf._lazypkval), None) self._alive[(inf.model_info.model, inf.get_pk_as_key())] = inf self._cache.add(inf) def uncache(self, inf): if inf in self._cache: self._cache.remove(inf) self._dirty.pop(inf, None) self._alive.pop((inf.model_info.model, inf.get_pk_as_key()), None) self._alive.pop((inf.model_info.model, inf._lazypkval), None) ### signals ### #??? what is this used for? def instance_deleted(self, inst_info): print "instance_deleted", inst_info def model_pre_init(self, instance, **kwargs): signals.fire("stop-tracking-changes", instance=instance) def model_post_init(self, instance, **kwargs): inf = get_inst_info(instance) signals.fire("start-tracking-changes", instance=instance) self._alive[(inf.model_info.model, inf.get_pk_as_key())] = inf # If instance was not initialized with a value for primary key, # then it has not been saved yet and goes into self._dirty. if instance.pk is None: self.set_dirty(inf) def start_tracking_changes(self, instance): get_inst_info(instance)._meta["track-changes"] = True def stop_tracking_changes(self, instance): get_inst_info(instance)._meta["track-changes"] = False def model_pre_save(self, instance): pass def model_post_save(self, instance, created): inf = get_inst_info(instance) if not inf._meta["do-cache"]: return self.cache(inf) self._dirty.pop(inf, None) # On calling Model.delete, tracking of changes is stopped, so start # tracking now. signals.fire("start-tracking-changes", instance=instance) def model_pre_update(self, instance, value, fieldname): # instance._inst_info.record_change(fieldname, value) pass def model_post_update(self, instance, value, fieldname): inf = get_inst_info(instance) if not inf in self._dirty: self.set_dirty(inf) def model_pre_delete(self, instance): signals.fire("stop-tracking-changes", instance=instance) def model_post_delete(self, instance, deleted): if not deleted: return inf = get_inst_info(instance) self.uncache(inf) instance.id = None def model_do_cache(self, instance): inf = get_inst_info(instance) self.cache(inf) self.set_dirty(inf) inf._meta["do-cache"] = True signals.fire("start-tracking-changes", instance=instance) def model_do_not_cache(self, instance): inf = get_inst_info(instance) self.uncache(inf) inf._meta["do-cache"] = False signals.fire("stop-tracking-changes", instance=instance) def model_history_reset(self, instance, **kwargs): raise NotImplementedError def model_history_undo(self, instance, fieldname, **kwargs): raise NotImplementedError def model_history_redo(self, instance, fieldname, **kwargs): raise NotImplementedError def relation_pre_get(self, manager, **kwargs): pass def relation_post_get(self, manager, **kwargs): pass def relation_pre_set(self, manager, values, **kwargs): pass def relation_post_set(self, manager, values, **kwargs): pass def relation_pre_delete(self, manager, **kwargs): pass def relation_post_delete(self, manager, **kwargs): pass def relation_pre_add(self, manager, values, **kwargs): pass def relation_post_add(self, manager, values, **kwargs): pass def relation_pre_remove(self, manager, values, **kwargs): pass def relation_post_remove(self, manager, values, **kwargs): pass
class LRUCache: """A Least-Recently-Used cache. The cache also includes a 'graveyard' of items that have been evicted from the cache. These are stored as weak references throught a `weakref.WeakValueDictionary`. Evicted items can be resurrected if they are retrieved before gc. I purposely chose to *not* implement __getitem__ and __setitem__ to distance this class from builtin dicts and similar objects. Based on: https://docs.python.org/3.9/library/collections.html#ordereddict-examples-and-recipes :param maxsize: The maximum number of items to store in the cache. >>> import gc >>> class A: ... def __init__(self, x): ... self.x = x ... def __repr__(self): ... return 'ALIVE ' + str(self.x) >>> c = LRUCache(maxsize=2) >>> c.set(1, A(1)) >>> c.set(2, A(2)) >>> c.set(3, A(3)) >>> _ = gc.collect() >>> c.get(1, 'DEAD') 'DEAD' >>> c.get(2) ALIVE 2 >>> c.get(3) ALIVE 3 >>> c.set(4, A(4)) >>> _ = gc.collect() >>> c.get(2, 'DEAD') 'DEAD' >>> c.hits 2 >>> c.misses 2 """ def __init__(self, maxsize): self.maxsize = maxsize self.lru = OrderedDict() self.grave = WeakValueDictionary() self.hits = 0 self.misses = 0 self.resurrections = 0 self.rwlock = ReadWriteLock() def get(self, key, default=None): """Retrieve an item from the cache if it exists. :param key: The key to retrieve. :param default: The default value to return if the item doesn't exist. :return: The value associated with key. """ with self.rwlock.read_access: try: value = self.lru[key] self.hits += 1 except KeyError: # try to resurrect value = self.grave.pop(key, None) if value is not None: self.lru[key] = value self.resurrections += 1 else: self.misses += 1 return default self.lru.move_to_end(key) return value def set(self, key, value): """Add a value to the cache. :param key: The key for the item. :param value: The value to store. """ with self.rwlock.write_access: if key in self.grave: del self.grave[key] if key in self.lru: self.lru.move_to_end(key) self.lru[key] = value while len(self.lru) > self.maxsize: # remove old items from the cache # send them to live with the dead (k, v) = self.lru.popitem(last=False) self.grave[k] = v def delete(self, key): """Remove a value from the cache. If there is no item associated with key, do nothing. :param key: The key to delete. """ with self.rwlock.write_access: if key in self.grave: del self.grave[key] if key in self.lru: del self.lru[key]
class ConfCache(QObject): def __init__(self, weak=True): super(ConfCache, self).__init__() if weak: self.cache = WeakValueDictionary() else: self.cache = {} # pylint: disable=redefined-variable-type self.monitor = MonitorWithRename(parent=self) self.monitor.fileChanged.connect(self.onFileChanged) @Slot(str) def onFileChanged(self, path): """Method called when a monitored file changes This method should be reimplemented by subclasses to reload a configuration object. When the cache stores weak-references, config objects may be deleted at anytime, but the associated file will stay monitored. A method reimplementation should check if the config still exists in cache before performing a costly file reload. See :any:`unmonitorCollected`. """ pass def unmonitorCollected(self, path=None): """Stop monitoring file if the config object was garbage-collected. :param path: if None, verifies all monitored files :returns: True if `path` was unmonitored, else False. Meaningless if `path` is None. """ if path is None: for f in self.monitor.files(): self.unmonitorCollected(f) return False else: if path in self.cache: return False self.monitor.removePath(path) return True def addConf(self, path, conf): """Add a config object to cache The config file `path` will be monitored for changes and the `conf` object will be added to cache. :param path: path of the config :param conf: the """ self.cache[path] = conf self.monitor.addPath(path) def delConf(self, path): """Remove a config from cache. The config file at `path` is also unmonitored. """ self.monitor.removePath(path) self.cache.pop(path, None) def get(self, path): """Get the config object for path :returns: the config object in cache, or None if not found or the object was garbage-collected. """ return self.cache.get(path)
class UDPDemux(object): """Explicitly routing UDP demux This class implements a demux that forwards packets from the root socket to sockets belonging to connections. It does this whenever its service method is invoked. Methods: remove_connection -- remove an existing connection service -- distribute datagrams from the root socket to connections forward -- forward a stored datagram to a connection """ _forwarding_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) _forwarding_socket.bind(('127.0.0.1', 0)) def __init__(self, datagram_socket): """Constructor Arguments: datagram_socket -- the root socket; this must be a bound, unconnected datagram socket """ if datagram_socket.type != socket.SOCK_DGRAM: raise InvalidSocketError("datagram_socket is not of " + "type SOCK_DGRAM") try: datagram_socket.getsockname() except: raise InvalidSocketError("datagram_socket is unbound") try: datagram_socket.getpeername() except: pass else: raise InvalidSocketError("datagram_socket is connected") self.datagram_socket = datagram_socket self.payload = "" self.payload_peer_address = None self.connections = WeakValueDictionary() def get_connection(self, address): """Create or retrieve a muxed connection Arguments: address -- a peer endpoint in IPv4/v6 address format; None refers to the connection for unknown peers Return: a bound, connected datagram socket instance """ if self.connections.has_key(address): return self.connections[address] # We need a new datagram socket on a dynamically assigned ephemeral port conn = socket.socket(self._forwarding_socket.family, self._forwarding_socket.type, self._forwarding_socket.proto) conn.bind((self._forwarding_socket.getsockname()[0], 0)) conn.connect(self._forwarding_socket.getsockname()) if not address: conn.setblocking(0) self.connections[address] = conn _logger.debug("Created new connection for address: %s", address) return conn def remove_connection(self, address): """Remove a muxed connection Arguments: address -- an address that was previously returned by the service method and whose connection has not yet been removed Return: the socket object whose connection has been removed """ return self.connections.pop(address) def service(self): """Service the root socket Read from the root socket and forward one datagram to a connection. The call will return without forwarding data if any of the following occurs: * An error is encountered while reading from the root socket * Reading from the root socket times out * The root socket is non-blocking and has no data available * An empty payload is received * A non-empty payload is received from an unknown peer (a peer for which get_connection has not yet been called); in this case, the payload is held by this instance and will be forwarded when the forward method is called Return: if the datagram received was from a new peer, then the peer's address; otherwise None """ self.payload, self.payload_peer_address = \ self.datagram_socket.recvfrom(UDP_MAX_DGRAM_LENGTH) _logger.debug("Received datagram from peer: %s", self.payload_peer_address) if not self.payload: self.payload_peer_address = None return if self.connections.has_key(self.payload_peer_address): self.forward() else: return self.payload_peer_address def forward(self): """Forward a stored datagram When the service method returns the address of a new peer, it holds the datagram from that peer in this instance. In this case, this method will perform the forwarding step. The target connection is the one associated with address None if get_connection has not been called since the service method returned the new peer's address, and the connection associated with the new peer's address if it has. """ assert self.payload assert self.payload_peer_address if self.connections.has_key(self.payload_peer_address): conn = self.connections[self.payload_peer_address] default = False else: conn = self.connections[None] # propagate exception if not created default = True _logger.debug("Forwarding datagram from peer: %s, default: %s", self.payload_peer_address, default) self._forwarding_socket.sendto(self.payload, conn.getsockname()) self.payload = "" self.payload_peer_address = None
class Worker(GearmanProtocolMixin, asyncio.Protocol): def __init__(self, *functions, loop=None, grab_type=Type.GRAB_JOB, timeout=None): super(Worker, self).__init__(loop=loop) self.transport = None self.main_task = None self.functions = OrderedDict() self.running = WeakValueDictionary() self.waiters = [] self.shutting_down = False self.timeout = timeout grab_mapping = { Type.GRAB_JOB: self.grab_job, Type.GRAB_JOB_UNIQ: self.grab_job_uniq, Type.GRAB_JOB_ALL: self.grab_job_all, } try: self.grab = grab_mapping[grab_type] except KeyError: raise RuntimeError( 'Grab type must be one of GRAB_JOB, GRAB_JOB_UNIQ or GRAB_JOB_ALL' ) for func_arg in functions: try: func, name = func_arg self.functions[name] = func except TypeError: name = func_arg.__name__ self.functions[name] = func_arg def connection_made(self, transport): logger.info('Connection is made to %r', transport.get_extra_info('peername')) self.transport = transport if self.timeout is not None: can_do = partial(self.can_do_timeout, timeout=self.timeout) else: can_do = self.can_do for fname in self.functions.keys(): logger.debug('Registering function %s', fname) can_do(fname) self.main_task = self.get_task(self.run()) def connection_lost(self, exc): self.transport = None def get_task(self, coro): return asyncio.ensure_future(coro, loop=self.loop) async def run(self, ): no_job = NoJob() while not self.shutting_down: self.pre_sleep() await self.wait_for(Type.NOOP) response = await self.grab() if response == no_job: continue try: job_info = self._to_job_info(response) func = self.functions.get(job_info.function) if not func: logger.warning('Failed to find function %s in %s', job_info.function, ', '.join(self.functions.keys())) self.work_fail(job_info.handle) continue try: result_or_coro = func(job_info) if asyncio.iscoroutine(result_or_coro): task = self.get_task(result_or_coro) self.running[job_info.handle] = task result = await task else: result = result_or_coro self.work_complete(job_info.handle, result) except Exception as ex: logger.exception('Job (handle %s) resulted with exception', job_info.handle) self.work_exception(job_info.handle, str(ex)) finally: self.running.pop(job_info.handle, None) except AttributeError: logger.error('Unexpected GRAB_JOB response %r', response) async def shutdown(self, graceful=False): logger.debug('Shutting down worker {}gracefully...'.format( '' if graceful else 'un')) self.shutting_down = True sub_tasks = list(self.running.values()) if graceful: if sub_tasks: await asyncio.wait(sub_tasks, loop=self.loop) else: async def cancel_and_wait(tasks): for task in tasks: task.cancel() try: await asyncio.wait(tasks, loop=self.loop) except asyncio.CancelledError: pass if sub_tasks: await cancel_and_wait(sub_tasks) self.main_task.cancel() if self.transport: self.transport.close() @staticmethod def _to_job_info(job_assign): attrs = ['handle', 'function', 'uuid', 'reducer', 'workload'] values = [getattr(job_assign, attr, None) for attr in attrs] return JobInfo(*values) def register_function(self, func, name=''): if not self.transport: raise RuntimeError('Worker must be connected to the daemon') name = name or func.__name__ self.functions[name] = func return self.can_do(name) def grab_job_all(self): self.send(Type.GRAB_JOB_ALL) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_ALL) def grab_job_uniq(self): self.send(Type.GRAB_JOB_UNIQ) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_UNIQ) def grab_job(self): self.send(Type.GRAB_JOB) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN) def pre_sleep(self): self.send(Type.PRE_SLEEP) def can_do(self, function): self.send(Type.CAN_DO, function) def can_do_timeout(self, function, timeout): self.send(Type.CAN_DO_TIMEOUT, function, timeout) def work_fail(self, handle): self.send(Type.WORK_FAIL, handle) def work_exception(self, handle, data): self.send(Type.WORK_EXCEPTION, handle, data) def work_complete(self, handle, result): if result is None: result = '' self.send(Type.WORK_COMPLETE, handle, result) def set_client_id(self, client_id): self.send(Type.SET_CLIENT_ID, client_id)
class RedisStore: def __init__(self, db_host, db_port, db_num, db_pw): self.pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw, decode_responses=True) self.redis = StrictRedis(connection_pool=self.pool) self.redis.ping() self._object_map = WeakValueDictionary() def create_object(self, dbo_class, dbo_dict, update_timestamp=True): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) if not dbo_class: return try: dbo_id = dbo_dict['dbo_id'] except KeyError: dbo_id, dbo_dict = dbo_dict, {} if dbo_id is None or dbo_id == '': log.warn("create_object called with empty dbo_id") return dbo_id = str(dbo_id).lower() if self.object_exists(dbo_class.dbo_key_type, dbo_id): raise ObjectExistsError(dbo_id) dbo = dbo_class() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) dbo.db_created() if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, dbo.dbo_id) self.save_object(dbo, update_timestamp) return dbo def load_object(self, dbo_key, key_type=None, silent=False): if key_type: try: key_type = key_type.dbo_key_type except AttributeError: pass try: dbo_key, dbo_id = ':'.join((key_type, dbo_key)), dbo_key except TypeError: if not silent: log.exception("Invalid dbo_key passed to load_object", stack_info=True) return else: key_type, _, dbo_id = dbo_key.partition(':') cached_dbo = self._object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: if not silent: log.warn("Failed to find {} in database", dbo_key) return return self._json_to_obj(json_str, key_type, dbo_id) def save_object(self, dbo, update_timestamp=False, autosave=False): if update_timestamp: dbo.dbo_ts = int(time.time()) if dbo.dbo_indexes: self._update_indexes(dbo) self._clear_old_refs(dbo) save_root, new_refs = dbo.to_db_value() self.redis.set(dbo.dbo_key, json_encode(save_root)) if new_refs: self._set_new_refs(dbo, new_refs) log.debug("db object {} {}saved", dbo.dbo_key, "auto" if autosave else "") self._object_map[dbo.dbo_key] = dbo return dbo def update_object(self, dbo, dbo_dict): dbo.hydrate(dbo_dict) return self.save_object(dbo, True) def delete_object(self, dbo): key = dbo.dbo_key dbo.db_deleted() self.delete_key(key) self._clear_old_refs(dbo) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, dbo.dbo_id) for children_type in dbo.dbo_children_types: self.delete_object_set( get_dbo_class(children_type), "{}_{}s:{}".format(dbo.dbo_key_type, children_type, dbo.dbo_id)) for ix_name in dbo.dbo_indexes: ix_value = getattr(dbo, ix_name, None) if ix_value is not None and ix_value != '': self.delete_index('ix:{}:{}'.format(dbo.dbo_key_type, ix_name), ix_value) log.debug("object deleted: {}", key) self.evict_object(dbo) def load_cached(self, dbo_key): return self._object_map.get(dbo_key) def object_exists(self, obj_type, obj_id): return self.redis.exists('{}:{}'.format(obj_type, obj_id)) def load_object_set(self, dbo_class, set_key=None): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) key_type = dbo_class.dbo_key_type if not set_key: set_key = dbo_class.dbo_set_key results = set() keys = deque() pipeline = self.redis.pipeline() for key in self.fetch_set_keys(set_key): dbo_key = ':'.join([key_type, key]) try: results.add(self._object_map[dbo_key]) except KeyError: keys.append(key) pipeline.get(dbo_key) for dbo_id, json_str in zip(keys, pipeline.execute()): if json_str: obj = self._json_to_obj(json_str, key_type, dbo_id) if obj: results.add(obj) continue log.warn("Removing missing object from set {}", set_key) self.delete_set_key(set_key, dbo_id) return results def delete_object_set(self, dbo_class, set_key=None): if not set_key: set_key = dbo_class.dbo_set_key for dbo in self.load_object_set(dbo_class, set_key): self.delete_object(dbo) self.delete_key(set_key) def reload_object(self, dbo_key): dbo = self._object_map.get(dbo_key) if dbo: json_str = self.redis.get(dbo_key) if not json_str: log.warn("Failed to find {} in database for reload", dbo_key) return None return self.update_object(dbo, json_decode(json_str)) return self.load_object(dbo_key) def evict_object(self, dbo): self._object_map.pop(dbo.dbo_key, None) def load_value(self, key, default=None): json = self.redis.get(key) if json: return json_decode(json) return default def save_value(self, key, value): self.redis.set(key, json_encode(value)) def fetch_set_keys(self, set_key): return self.redis.smembers(set_key) def add_set_key(self, set_key, *values): self.redis.sadd(set_key, *values) def delete_set_key(self, set_key, value): self.redis.srem(set_key, value) def set_key_exists(self, set_key, value): return self.redis.sismember(set_key, value) def db_counter(self, counter_id, inc=1): return self.redis.incr("counter:{}".format(counter_id), inc) def delete_key(self, key): self.redis.delete(key) def set_index(self, index_name, key, value): return self.redis.hset(index_name, key, value) def get_index(self, index_name, key): return self.redis.hget(index_name, key) def get_full_index(self, index_name): return self.redis.hgetall(index_name) def delete_index(self, index_name, key): return self.redis.hdel(index_name, key) def get_all_hash(self, index_name): return { key: json_decode(value) for key, value in self.redis.hgetall(index_name).items() } def get_hash_keys(self, hash_id): return self.redis.hkeys(hash_id) def set_db_hash(self, hash_id, hash_key, value): return self.redis.hset(hash_id, hash_key, json_encode(value)) def get_db_hash(self, hash_id, hash_key): return json_decode(self.redis.hget(hash_id, hash_key)) def remove_db_hash(self, hash_id, hash_key): self.redis.hdel(hash_id, hash_key) def get_all_db_hash(self, hash_id): return [ json_decode(value) for value in self.redis.hgetall(hash_id).values() ] def get_db_list(self, list_id, start=0, end=-1): return [ json_decode(value) for value in self.redis.lrange(list_id, start, end) ] def add_db_list(self, list_id, value): self.redis.lpush(list_id, json_encode(value)) def trim_db_list(self, list_id, start, end): return self.redis.ltrim(list_id, start, end) def dbo_holders(self, dbo_key, degrees=0): all_keys = set() def find(find_key, degree): holder_keys = self.fetch_set_keys('{}:holders'.format(find_key)) for new_key in holder_keys: if new_key != dbo_key and new_key not in all_keys: all_keys.add(new_key) if degree < degrees: find(new_key, degree + 1) find(dbo_key, 0) return all_keys def _json_to_obj(self, json_str, key_type, dbo_id): dbo_dict = json_decode(json_str) dbo = get_mixed_type(key_type, dbo_dict.get('mixins'))() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) self._object_map[dbo.dbo_key] = dbo return dbo def _update_indexes(self, dbo): try: old_dbo = json_decode(self.redis.get(dbo.dbo_key)) except TypeError: old_dbo = None for ix_name in dbo.dbo_indexes: new_val = getattr(dbo, ix_name, None) old_val = old_dbo.get(ix_name) if old_dbo else None if old_val == new_val: continue ix_key = 'ix:{}:{}'.format(dbo.dbo_key_type, ix_name) if old_val is not None: self.delete_index(ix_key, old_val) if new_val is not None and new_val != '': if self.get_index(ix_key, new_val): raise NonUniqueError(ix_key, new_val) self.set_index(ix_key, new_val, dbo.dbo_id) def _clear_old_refs(self, dbo): dbo_key = dbo.dbo_key ref_key = '{}:refs'.format(dbo_key) for ref_id in self.fetch_set_keys(ref_key): holder_key = '{}:holders'.format(ref_id) self.delete_set_key(holder_key, dbo_key) self.delete_key(ref_key) def _set_new_refs(self, dbo, new_refs): dbo_key = dbo.dbo_key self.add_set_key("{}:refs".format(dbo_key), *new_refs) for ref_id in new_refs: holder_key = '{}:holders'.format(ref_id) self.add_set_key(holder_key, dbo_key)
class BaseConnection(object): def __init__(self, response_queue_size=None, enquire_timeout=None, logger=None): self.proto = Proto(logger) self.sequence_number = 0 self.last_enquire = time.time() self.enquire_timeout = enquire_timeout or 300 self.enquire_response = None self.response_queue = deque([], response_queue_size or 1000) self.response_map = WeakValueDictionary() def next_sequence(self): self.sequence_number += 1 return self.sequence_number def reply(self, cmd): return self.on_send(self.proto.send_bytes(cmd)) def send(self, cmd, callback=None, notify=True): cmd.sequence_number = self.next_sequence() resp = Response(cmd, callback) if notify: self.response_map[cmd.sequence_number] = resp self.response_queue.append(resp) self.reply(cmd) return resp def on_send(self, data): # pragma: no cover pass def on_close(self): # pragma: no cover pass def on_deliver(self, req, resp, reply): # pragma: no cover pass def _make_reply(self, resp): def reply(): reply.called = True return self.reply(resp) reply.called = False return reply def _deliver(self, req, resp): reply = self._make_reply(resp) self.on_deliver(req, resp, reply) if not reply.called: self.reply(resp) def handle(self, cmd): self.last_enquire = time.time() cmd_type = type(cmd) seq = {'sequence_number': cmd.sequence_number} if cmd_type is command.EnquireLink: self.reply(command.EnquireLinkResp(**seq)) elif cmd_type is command.Unbind: self.reply(command.UnbindResp(**seq)) self.on_close() elif cmd_type.is_response: resp = self.response_map.pop(cmd.sequence_number, None) if resp: resp.resolve(cmd) else: resp = cmd_type.response(**seq) self._deliver(cmd, resp) def feed(self, data): for e in self.proto.receive_bytes(data): self.handle(e) def ping(self, response_timeout=10): now = time.time() if self.enquire_response: if self.enquire_response.ready: self.enquire_response = None elif self.enquire_response.expire < now: raise BrokenLink('SMPP link broken: no response from SMSC') elif self.last_enquire + self.enquire_timeout < now: self.enquire_response = self.send(command.EnquireLink()) self.enquire_response.expire = now + response_timeout def unbind(self): return self.send(command.Unbind(), lambda _: self.on_close())
class TaskQueue(AbstractTaskQueue): """Simple in-memory task queue implementation""" @classmethod def factory(cls, url, name=const.DEFAULT, *args, **kw): obj = _REFS.get((url, name)) if obj is None: obj = _REFS[(url, name)] = cls(url, name, *args, **kw) return obj def __init__(self, *args, **kw): super(TaskQueue, self).__init__(*args, **kw) self.queue = Queue() self.results = WeakValueDictionary() self.results_lock = Lock() def _init_result(self, result, status, message): with self.results_lock: if result.id in self.results: return False self.results[result.id] = result result.__status = status result.__value = Queue() result.__task = message result.__args = {} result.__lock = Lock() result.__for = None return True def enqueue_task(self, result, message): if self._init_result(result, const.ENQUEUED, message): self.queue.put(result) return True return False def defer_task(self, result, message, args): if self._init_result(result, const.PENDING, message): results = self.results # keep references to results to prevent GC result.__refs = [results.get(arg) for arg in args] return True return False def undefer_task(self, task_id): result = self.results[task_id] self.queue.put(result) def get(self, timeout=None): try: result = self.queue.get(timeout=timeout) except Empty: return None result.__status = const.PROCESSING return result.id, result.__task def size(self): return len(self.results) def discard_pending(self): with self.results_lock: while True: try: self.queue.get_nowait() except Empty: break self.results.clear() def reserve_argument(self, argument_id, deferred_id): result = self.results.get(argument_id) if result is None: return (False, None) with result.__lock: if result.__for is not None: return (False, None) result.__for = deferred_id try: message = result.__value.get_nowait() except Empty: message = None if message is not None: with self.results_lock: self.results.pop(argument_id, None) return (True, message) def set_argument(self, task_id, argument_id, message): result = self.results[task_id] with self.results_lock: self.results.pop(argument_id, None) with result.__lock: result.__args[argument_id] = message return len(result.__args) == len(result.__refs) def get_arguments(self, task_id): try: return self.results[task_id].__args except KeyError: return {} def set_task_timeout(self, task_id, timeout): pass def get_status(self, task_id): result = self.results.get(task_id) return None if result is None else result.__status def set_result(self, task_id, message, timeout): result = self.results.get(task_id) if result is not None: with result.__lock: result.__value.put(message) return result.__for def pop_result(self, task_id, timeout): result = self.results.get(task_id) if result is None: return const.TASK_EXPIRED # with result.__lock: # if result.__for is not None: # raise NotImplementedError # #return const.RESERVED # result.__for = task_id try: if timeout == 0: value = result.__value.get_nowait() else: value = result.__value.get(timeout=timeout) except Empty: value = None else: self.results.pop(task_id, None) return value def discard_result(self, task_id, task_expired_token): result = self.results.pop(task_id) if result is not None: result.__value.put(task_expired_token)
class UDPDemux(object): """Explicitly routing UDP demux This class implements a demux that forwards packets from the root socket to sockets belonging to connections. It does this whenever its service method is invoked. Methods: remove_connection -- remove an existing connection service -- distribute datagrams from the root socket to connections forward -- forward a stored datagram to a connection """ _forwarding_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) _forwarding_socket.bind(('127.0.0.1', 0)) def __init__(self, datagram_socket): """Constructor Arguments: datagram_socket -- the root socket; this must be a bound, unconnected datagram socket """ if (datagram_socket.type & socket.SOCK_DGRAM) != socket.SOCK_DGRAM: raise InvalidSocketError("datagram_socket is not of " + "type SOCK_DGRAM") try: datagram_socket.getsockname() except: raise InvalidSocketError("datagram_socket is unbound") try: datagram_socket.getpeername() except: pass else: raise InvalidSocketError("datagram_socket is connected") self.datagram_socket = datagram_socket self.payload = "" self.payload_peer_address = None self.connections = WeakValueDictionary() def get_connection(self, address): """Create or retrieve a muxed connection Arguments: address -- a peer endpoint in IPv4/v6 address format; None refers to the connection for unknown peers Return: a bound, connected datagram socket instance """ if address in self.connections: return self.connections[address] # We need a new datagram socket on a dynamically assigned ephemeral port conn = socket.socket(self._forwarding_socket.family, self._forwarding_socket.type, self._forwarding_socket.proto) conn.bind((self._forwarding_socket.getsockname()[0], 0)) conn.connect(self._forwarding_socket.getsockname()) if not address: conn.setblocking(0) self.connections[address] = conn _logger.debug("Created new connection for address: %s", address) return conn def remove_connection(self, address): """Remove a muxed connection Arguments: address -- an address that was previously returned by the service method and whose connection has not yet been removed Return: the socket object whose connection has been removed """ return self.connections.pop(address) def service(self): """Service the root socket Read from the root socket and forward one datagram to a connection. The call will return without forwarding data if any of the following occurs: * An error is encountered while reading from the root socket * Reading from the root socket times out * The root socket is non-blocking and has no data available * An empty payload is received * A non-empty payload is received from an unknown peer (a peer for which get_connection has not yet been called); in this case, the payload is held by this instance and will be forwarded when the forward method is called Return: if the datagram received was from a new peer, then the peer's address; otherwise None """ self.payload, self.payload_peer_address = \ self.datagram_socket.recvfrom(UDP_MAX_DGRAM_LENGTH) _logger.debug("Received datagram from peer: %s", self.payload_peer_address) if not self.payload: self.payload_peer_address = None return if self.payload_peer_address in self.connections: self.forward() else: return self.payload_peer_address def forward(self): """Forward a stored datagram When the service method returns the address of a new peer, it holds the datagram from that peer in this instance. In this case, this method will perform the forwarding step. The target connection is the one associated with address None if get_connection has not been called since the service method returned the new peer's address, and the connection associated with the new peer's address if it has. """ assert self.payload assert self.payload_peer_address if self.payload_peer_address in self.connections: conn = self.connections[self.payload_peer_address] default = False else: conn = self.connections[None] # propagate exception if not created default = True _logger.debug("Forwarding datagram from peer: %s, default: %s", self.payload_peer_address, default) self._forwarding_socket.sendto(self.payload, conn.getsockname()) self.payload = "" self.payload_peer_address = None
class TrollReactor(CBDictInterface): """Base class for Omegle API. """ def __init__(self, transmog=Transmogrifier(), listen=InteractiveViewport(), n=2, refresh=2., debug=0): # Independent setup super(TrollReactor, self).__init__() self.debug = debug self.listeners = WeakValueDictionary() # Argument assignment self.eventQueue = deque() self.connectTransmogrifier(transmog) self.addListeners(listen) self._n = n self.refresh = refresh self._allConnected = False self.reconnectWait = 2. self.idleTime = None self.initializeStrangers() # Now we wait to receive idSet events def connectTransmogrifier(self, transmog): self.transmogrifier = transmog self.transmogrifier.connect(self.eventQueue) def initializeStrangers(self): self._volatile = dict((Stranger(reactor, self, HTTP), None) for _ in xrange(self._n)) self._waiting = len(self._volatile.keys()) self.strangers = {} self.idleTime = time() self._allConnected = False def multicastDisconnect(self, ids): """Announce disconnect for a group of strangers. ids : iterable id strings of strangers from whom to politely disconnect. """ for i in ids: self.strangers[i].announceDisconnect() def restart(self): self.strangers.clear() self.eventQueue.clear() self._allConnected = False sleep(self.reconnectWait) # blocking is OK here. We are trying to *avoid* making connections. self.initializeStrangers() def pumpEvents(self): for id_ in self.strangers: self.strangers[id_].getEventsPage() reactor.callLater(self.refresh, self.pumpEvents) def on_idSet(self, ev): for s in self._volatile: if s.id == ev.id: # we have the stranger that notified self.strangers[s.id] = s # move to {id: stranger} dict self._waiting -= 1 assert self._waiting >= 0, "Too many stranger IDs" if self._waiting == 0: self._allConnected = True self.idleTime = time() self.pumpEvents() def on_error(self, ev): # TODO: handle RECAPCHA pass def addListeners(self, listeners): """Add a listener or group of listeners to the reactor. listeners : CBDictInterface instance or iterable """ listeners = mkIterableSequence(listeners) for listen in listeners: self.listeners[listen] = listen # weak-value dict def removeListener(self, listener): self.listeners.pop(listener) def _processEventQueue(self): while len(self.eventQueue): ev = self.eventQueue.popleft() for listener in self.listeners: listener.notify(ev) self.notify(ev) def deltaIdleTime(self): return time() - self.idleTime def idle(self): """Respond to idle state. This function is run whenever feed encounters a null event, and does nothing by default. Override to define functionality. """ pass def feed(self, events): """Notify the TrollReactor of event(s). """ if not events or events is NULL_EVENT: events = (NULL_EVENT,) self.idle() else: self.idleTime = time() if isEvent(events): # if events is a single event events = (events,) self.transmogrifier(events) self._processEventQueue()
class Signal(object): def __init__(self): self.__slots = WeakValueDictionary() def __call__(self, *args, **kargs): for key in self.__slots: func, _ = key func(self.__slots[key], *args, **kargs) def connect(self, slot): if PYTHON3: key = (slot.__func__, id(slot.__self__)) self.__slots[key] = slot.__self__ else: key = (slot.im_func, id(slot.im_self)) self.__slots[key] = slot.im_self def disconnect(self, slot): if PYTHON3: key = (slot.__func__, id(slot.__self__)) if key in self.__slots: self.__slots.pop(key) else: key = (slot.im_func, id(slot.im_self)) if key in self.__slots: self.__slots.pop(key) def clear(self): self.__slots.clear() ## Sample usage: #class Model(object): # def __init__(self, value): # self.__value = value # self.changed = Signal() # # def set_value(self, value): # self.__value = value # self.changed() # Emit signal # # def get_value(self): # return self.__value # # #class View(object): # def __init__(self, model): # self.model = model # model.changed.connect(self.model_changed) # # def model_changed(self): # print "New value:", self.model.get_value() # # #model = Model(10) #view1 = View(model) #view2 = View(model) #view3 = View(model) # #model.set_value(20) # #del view1 #model.set_value(30) # #model.changed.clear() #model.set_value(40) ### end of http://code.activestate.com/recipes/576477/ }}}
class Boss: def __init__(self, os_window_id, opts, args, cached_values): self.window_id_map = WeakValueDictionary() self.cached_values = cached_values self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.current_font_size = opts.font_size set_font_family(opts) self.opts, self.args = opts, args initialize_renderer() startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None): dpi_changed = False if os_window_id is None: w, h = initial_window_size( self.opts, self.cached_values) if size is None else size cls = wclass or self.args.cls or appname os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls) if startup_id: ctx = init_startup_notification(os_window_id, startup_id) dpi_changed = show_window(os_window_id) if startup_id: end_startup_notification(ctx) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm if dpi_changed: self.on_dpi_change(os_window_id) def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } def match_windows(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) for tm in self.os_window_map.values(): for tab in tm: for window in tab: if window.matches(field, pat): yield window def tab_for_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) tms = tuple(self.os_window_map.values()) found = False if field in ('title', 'id'): for tm in tms: for tab in tm: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control: try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = { 'ok': False, 'error': str(err), 'tb': traceback.format_exc() } else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): import json msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) session = create_session(opts, args) self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size( opts, self.cached_values), startup_id=startup_id) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): tm = self.os_window_map.get(os_window_id) if tm is not None: if dpi_changed: if set_dpi_from_os_window(os_window_id): self.on_dpi_change(os_window_id) else: tm.resize() else: tm.resize() def increase_font_size(self): self.change_font_size( min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def _change_font_size(self, new_size=None, on_dpi_change=False): if new_size is not None: self.current_font_size = new_size old_cell_width, old_cell_height = viewport_for_window()[-2:] windows = tuple(filter(None, self.window_id_map.values())) resize_fonts(self.current_font_size, on_dpi_change=on_dpi_change) layout_sprite_map() prerender() for window in windows: window.screen.rescale_images(old_cell_width, old_cell_height) window.screen.refresh_sprite_positions() for tm in self.os_window_map.values(): tm.resize() tm.refresh_sprite_positions() glfw_post_empty_event() def change_font_size(self, new_size): if new_size == self.current_font_size: return self._change_font_size(new_size) def on_dpi_change(self, os_window_id): self._change_font_size() @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, window, data): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ 'kitty', '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + editor + [confpath] self.new_os_window(*cmd) def input_unicode_character(self): w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: overlay_window = tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.unicode_input.main import main; main()' ], overlay_for=w.id)) overlay_window.action_on_close = partial( self.send_unicode_character, w.id) def send_unicode_character(self, target_window_id, source_window): w = self.window_id_map.get(target_window_id) if w is not None: output = str(source_window.screen.linebuf.line(0)) if output.startswith('OK: '): try: text = chr(int(output.partition(' ')[2], 16)) except Exception: import traceback traceback.print_exc() else: w.paste(text) def run_simple_kitten(self, type_of_input, kitten, *args): import shlex w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] if '--program' not in cmdline: args.extend(('--program', self.opts.open_url_with)) if type_of_input in ('text', 'history', 'ansi', 'ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input).encode('utf-8') elif type_of_input == 'none': data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.{}.main import main; main()'.format(kitten) ] + args, stdin=data, overlay_for=w.id)) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def open_url_lines(self, lines): self.open_url(''.join(lines)) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_sprite_map() destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.as_text(as_ansi=True, add_history=True) if arg == '@text': return w.as_text(add_history=True) if arg == '@screen': return w.as_text() if arg == '@ansi_screen': return w.as_text(as_ansi=True) if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: tab.new_special_window( self.args_to_special_window(args, cwd_from=cwd_from)) else: tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window if w is None: return self.new_window(*args) cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1)
class Entity: def __init__(self, entType, entValue, entField): if isinstance(entField, Field): self.type = entType self.value = entValue self.field = entField self.group = None self.links = WeakValueDictionary() # dict of linked entities self.field.registerEntity(self) # update the entity registry else: raise TypeError("Invalid field argument, field instance expected!") def linkTo(self, eTwo): ''' Linking operation is bi-directional, affects both entities equally.''' # check if entities not already linked if Edge.linkId(self, eTwo) not in self.links.keys(): # update both entities' list of links # create a new edge newlink = Edge(self, eTwo, self.field) self.links[newlink.id] = eTwo eTwo.links[newlink.id] = self # case when the first entity's group is not set if self.group is None: # assuming the second entity has already a group assigned try: eTwo.group.addMember(self) # except the second entity has no group except AttributeError: newGroup = Group(self.field) newGroup.addMember(self) newGroup.addMember(eTwo) # case when the first entity's group is set, but the second entity's is not elif eTwo.group is None: self.group.addMember(eTwo) # case when both entities have groups set and they are different groups elif self.group.name != eTwo.group.name: if self.group.size > eTwo.group.size: # first group wins self.group.annexMembers(eTwo.group) else: # second group wins eTwo.group.annexMembers(self.group) def getLinks(self): ''' Print the list of entities directly linked.''' return self.links.values() def removeLink(self, eTwo): ''' Remove linked entity.''' linkId = Edge.linkId(self, eTwo) self.links.pop(linkId) def __repr__(self): return repr(self.value) def __del__(self): ''' Delete itself from linked entities, and delete links.''' # remove link from linked entity necessary? no because it's a weaklink for linkId in self.links.keys(): self.field.eliminateEdge(linkId) del self
class Boss: def __init__(self, os_window_id, opts, args, cached_values): self.window_id_map = WeakValueDictionary() self.startup_colors = { k: opts[k] for k in opts if isinstance(opts[k], Color) } self.pending_sequences = None self.cached_values = cached_values self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.opts, self.args = opts, args startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, opts_for_size=None, startup_id=None): if os_window_id is None: opts_for_size = opts_for_size or self.opts cls = wclass or self.args.cls or appname with startup_notification_handler( do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback: os_window_id = create_os_window( initial_window_size_func(opts_for_size, self.cached_values), pre_show_callback, appname, wname or self.args.name or cls, cls) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm return os_window_id def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } @property def all_tab_managers(self): yield from self.os_window_map.values() @property def all_tabs(self): for tm in self.all_tab_managers: yield from tm @property def all_windows(self): for tab in self.all_tabs: yield from tab def match_windows(self, match): try: field, exp = match.split(':', 1) except ValueError: return if field == 'num': tab = self.active_tab if tab is not None: try: w = tab.get_nth_window(int(exp)) except Exception: return if w is not None: yield w else: pat = re.compile(exp) for window in self.all_windows: if window.matches(field, pat): yield window def tab_for_window(self, window): for tab in self.all_tabs: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): try: field, exp = match.split(':', 1) except ValueError: return pat = re.compile(exp) found = False if field in ('title', 'id'): for tab in self.all_tabs: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) return self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control or getattr( window, 'allow_remote_control', False): try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err)} if not getattr(err, 'hide_traceback', False): response['tb'] = traceback.format_exc() else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) if not os.path.isabs(args.directory): args.directory = os.path.join(msg['cwd'], args.directory) session = create_session(opts, args, respect_cwd=True) self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): if dpi_changed: self.on_dpi_change(os_window_id) else: tm = self.os_window_map.get(os_window_id) if tm is not None: tm.resize() def increase_font_size(self): # legacy self.set_font_size( min(self.opts.font_size * 5, self.current_font_size + 2.0)) def decrease_font_size(self): # legacy self.set_font_size(self.current_font_size - self.opts.font_size_delta) def restore_font_size(self): # legacy self.set_font_size(self.opts.font_size) def set_font_size(self, new_size): # legacy self.change_font_size(True, None, new_size) def change_font_size(self, all_windows, increment_operation, amt): def calc_new_size(old_size): new_size = old_size if amt == 0: new_size = self.opts.font_size else: if increment_operation: new_size += (1 if increment_operation == '+' else -1) * amt else: new_size = amt new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5)) return new_size if all_windows: current_global_size = global_font_size() new_size = calc_new_size(current_global_size) if new_size != current_global_size: global_font_size(new_size) os_windows = tuple(self.os_window_map.keys()) else: os_windows = [] w = self.active_window if w is not None: os_windows.append(w.os_window_id) if os_windows: final_windows = {} for wid in os_windows: current_size = os_window_font_size(wid) if current_size: new_size = calc_new_size(current_size) if new_size != current_size: final_windows[wid] = new_size if final_windows: self._change_font_size(final_windows) def _change_font_size(self, sz_map): for os_window_id, sz in sz_map.items(): tm = self.os_window_map.get(os_window_id) if tm is not None: os_window_font_size(os_window_id, sz) tm.resize() def on_dpi_change(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: sz = os_window_font_size(os_window_id) if sz: os_window_font_size(os_window_id, sz, True) tm.resize() def _set_os_window_background_opacity(self, os_window_id, opacity): change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0))) def set_background_opacity(self, opacity): window = self.active_window if window is None or not opacity: return if not self.opts.dynamic_background_opacity: return self.show_error( _('Cannot change background opacity'), _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity' )) os_window_id = window.os_window_id if opacity[0] in '+-': opacity = background_opacity_of(os_window_id) if opacity is None: return opacity += float(opacity) elif opacity == 'default': opacity = self.opts.background_opacity else: opacity = float(opacity) self._set_os_window_background_opacity(os_window_id, opacity) @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) if key_action is None: sequences = get_shortcut(self.opts.sequence_map, mods, key, scancode) if sequences: self.pending_sequences = sequences set_in_sequence_mode(True) return True else: self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def process_sequence(self, key, scancode, action, mods): if not self.pending_sequences: set_in_sequence_mode(False) remaining = {} matched_action = None for seq, key_action in self.pending_sequences.items(): if shortcut_matches(seq[0], mods, key, scancode): seq = seq[1:] if seq: remaining[seq] = key_action else: matched_action = key_action if remaining: self.pending_sequences = remaining else: self.pending_sequences = None set_in_sequence_mode(False) if matched_action is not None: self.dispatch_action(matched_action) def start_resizing_window(self): w = self.active_window if w is None: return overlay_window = self._run_kitten( 'resize_window', args=[ '--horizontal-increment={}'.format( self.opts.window_resize_step_cells), '--vertical-increment={}'.format( self.opts.window_resize_step_lines) ]) if overlay_window is not None: overlay_window.allow_remote_control = True def resize_layout_window(self, window, increment, is_horizontal, reset=False): tab = window.tabref() if tab is None or not increment: return False if reset: return tab.reset_window_sizes() return tab.resize_window_by(window.id, increment, is_horizontal) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) tm.mark_tab_bar_dirty() def update_tab_bar_data(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.update_tab_bar_data() def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, window, data): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ 'kitty', '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + editor + [confpath] self.new_os_window(*cmd) def get_output(self, source_window, num_lines=1): output = '' s = source_window.screen if num_lines is None: num_lines = s.lines for i in range(min(num_lines, s.lines)): output += str(s.linebuf.line(i)) return output def _run_kitten(self, kitten, args=(), input_data=None): w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: orig_args, args = list(args), list(args) from kittens.runner import create_kitten_handler end_kitten = create_kitten_handler(kitten, orig_args) args[0:0] = [config_dir, kitten] if input_data is None: type_of_input = end_kitten.type_of_input if type_of_input in ('text', 'history', 'ansi', 'ansi-history', 'screen', 'screen-history', 'screen-ansi', 'screen-ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input, add_wrap_markers='screen' in type_of_input).encode('utf-8') elif type_of_input is None: data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) else: data = input_data if isinstance(data, str): data = data.encode('utf-8') copts = { k: self.opts[k] for k in ('select_by_word_characters', 'open_url_with') } overlay_window = tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.runner import main; main()' ] + args, stdin=data, env={ 'KITTY_COMMON_OPTS': json.dumps(copts), 'PYTHONWARNINGS': 'ignore', 'OVERLAID_WINDOW_LINES': str(w.screen.lines), 'OVERLAID_WINDOW_COLS': str(w.screen.columns), }, overlay_for=w.id)) overlay_window.action_on_close = partial(self.on_kitten_finish, w.id, end_kitten) return overlay_window def kitten(self, kitten, *args): import shlex cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] self._run_kitten(kitten, args) def on_kitten_finish(self, target_window_id, end_kitten, source_window): output = self.get_output(source_window, num_lines=None) from kittens.runner import deserialize data = deserialize(output) if data is not None: end_kitten(data, target_window_id, self) def input_unicode_character(self): self._run_kitten('unicode_input') def set_tab_title(self): tab = self.active_tab if tab: args = [ '--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id) ] self._run_kitten('ask', args) def show_error(self, title, msg): self._run_kitten('show_error', ['--title', title], input_data=msg) def do_set_tab_title(self, title, tab_id): tm = self.active_tab_manager if tm is not None and title: tab_id = int(tab_id) for tab in tm.tabs: if tab.id == tab_id: tab.set_title(title) break def kitty_shell(self, window_type): cmd = ['kitty', '@'] if window_type == 'tab': window = self._new_tab(cmd).active_window elif window_type == 'os_window': os_window_id = self._new_os_window(cmd) window = self.os_window_map[os_window_id].active_window elif window_type == 'overlay': w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: window = tab.new_special_window( SpecialWindow(cmd, overlay_for=w.id)) else: window = None else: window = self._new_window(cmd) if window is not None: window.allow_remote_control = True def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) def open_url(self, url, program=None, cwd=None): if url: if isinstance(program, str): program = to_cmdline(program) open_url(url, program or self.opts.open_url_with, cwd=cwd) def open_url_lines(self, lines, program=None): self.open_url(''.join(lines), program) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection( ) if supports_primary_selection else get_clipboard_string() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def set_active_tab(self, tab): tm = self.active_tab_manager if tm is not None: tm.set_active_tab(tab) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.as_text(as_ansi=True, add_history=True) if arg == '@text': return w.as_text(add_history=True) if arg == '@screen': return w.as_text() if arg == '@ansi_screen': return w.as_text(as_ansi=True) if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: return tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: return tab.new_special_window( self.args_to_special_window(args, cwd_from=cwd_from)) else: return tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window if w is None: return self.new_window(*args) cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def patch_colors(self, spec, configured=False): if configured: for k, v in spec.items(): if hasattr(self.opts, k): setattr(self.opts, k, color_from_int(v)) for tm in self.all_tab_managers: tm.tab_bar.patch_colors(spec)
def pop(self, key, *args): if isinstance(key, bytes): key = key.decode("utf-8") return WeakValueDictionary.pop(self, key, *args)
class _WeakValueDictionary(object): # Maps from OID -> Persistent object, but # only weakly references the Persistent object. This is similar # to ``weakref.WeakValueDictionary``, but is customized depending on the # platform. On PyPy, all objects can cheaply use a WeakRef, so that's # what we actually use. On CPython, though, ``PersistentPy`` cannot be weakly # referenced, so we rely on the fact that the ``id()`` of an object is its # memory location, and we use ``ctypes`` to cast that integer back to # the object. # # To remove stale addresses, we rely on the ``ffi.gc()`` object with the exact # same lifetime as the ``PersistentPy`` object. It calls us, we get the ``id`` # back out of the CData, and clean up. if PYPY: # pragma: no cover def __init__(self): self._data = WeakValueDictionary() def _from_addr(self, addr): return addr def _save_addr(self, oid, obj): return obj cleanup_hook = None else: def __init__(self): # careful not to require ctypes at import time; most likely the # C implementation is in use. import ctypes self._data = {} self._addr_to_oid = {} self._cast = ctypes.cast self._py_object = ctypes.py_object def _save_addr(self, oid, obj): i = id(obj) self._addr_to_oid[i] = oid return i def _from_addr(self, addr): return self._cast(addr, self._py_object).value def cleanup_hook(self, cdata): # This is called during GC, possibly at interpreter shutdown # when the __dict__ of this object may have already been cleared. try: addr_to_oid = self._addr_to_oid except AttributeError: return oid = addr_to_oid.pop(cdata.pobj_id, None) self._data.pop(oid, None) def __contains__(self, oid): return oid in self._data def __len__(self): return len(self._data) def __setitem__(self, key, value): addr = self._save_addr(key, value) self._data[key] = addr def pop(self, oid): return self._from_addr(self._data.pop(oid)) def items(self): from_addr = self._from_addr for oid, addr in self._data.items(): yield oid, from_addr(addr) def get(self, oid, default=None): addr = self._data.get(oid, self) if addr is self: return default return self._from_addr(addr) def __getitem__(self, oid): addr = self._data[oid] return self._from_addr(addr)
class Agent(AgentT, Service): """Agent. This is the type of object returned by the ``@app.agent`` decorator. """ # supervisor is None until the agent is started so we cast to simplify. supervisor: SupervisorStrategyT = cast(SupervisorStrategyT, None) # channel is loaded lazily on .channel property access # to make sure configuration is not accessed when agent created # at module-scope. _channel: Optional[ChannelT] = None _channel_arg: Optional[Union[str, ChannelT]] _channel_kwargs: Dict[str, Any] _channel_iterator: Optional[AsyncIterator] = None _sinks: List[SinkT] _actors: MutableSet[ActorRefT] _actor_by_partition: MutableMapping[TP, ActorRefT] #: This mutable set is used by the first agent we start, #: so that we can update its active_partitions later #: (in on_partitions_assigned, when we know what partitions we get). _pending_active_partitions: Optional[Set[TP]] = None _first_assignment_done: bool = False def __init__( self, fun: AgentFun, *, app: AppT, name: str = None, channel: Union[str, ChannelT] = None, concurrency: int = 1, sink: Iterable[SinkT] = None, on_error: AgentErrorHandler = None, supervisor_strategy: Type[SupervisorStrategyT] = None, help: str = None, schema: SchemaT = None, key_type: ModelArg = None, value_type: ModelArg = None, isolated_partitions: bool = False, use_reply_headers: bool = None, **kwargs: Any, ) -> None: self.app = app self.fun: AgentFun = fun self.name = name or canonshortname(self.fun) # key-type/value_type arguments only apply when a channel # is not set if schema is not None: assert channel is None or isinstance(channel, str) if key_type is not None: assert channel is None or isinstance(channel, str) self._key_type = key_type if value_type is not None: assert channel is None or isinstance(channel, str) self._schema = schema self._value_type = value_type self._channel_arg = channel self._channel_kwargs = kwargs self.concurrency = concurrency or 1 self.isolated_partitions = isolated_partitions self.help = help or "" self._sinks = list(sink) if sink is not None else [] self._on_error: Optional[AgentErrorHandler] = on_error self.supervisor_strategy = supervisor_strategy self._actors = WeakSet() self._actor_by_partition = WeakValueDictionary() if self.isolated_partitions and self.concurrency > 1: raise ImproperlyConfigured( "Agent concurrency must be 1 when using isolated partitions") self.use_reply_headers = use_reply_headers Service.__init__(self) def on_init_dependencies(self) -> Iterable[ServiceT]: """Return list of services dependencies required to start agent.""" # Agent service is now a child of app. self.beacon.reattach(self.app.agents.beacon) return [] def actor_tracebacks(self) -> List[str]: return [actor.traceback() for actor in self._actors] async def _start_one( self, *, index: Optional[int] = None, active_partitions: Optional[Set[TP]] = None, stream: StreamT = None, channel: ChannelT = None, ) -> ActorT: # an index of None means there's only one instance, # and `index is None` is used as a test by functions that # disallows concurrency. index = index if self.concurrency > 1 else None return await self._start_task( index=index, active_partitions=active_partitions, stream=stream, channel=channel, beacon=self.beacon, ) async def _start_one_supervised( self, index: Optional[int] = None, active_partitions: Optional[Set[TP]] = None, stream: StreamT = None, ) -> ActorT: aref = await self._start_one( index=index, active_partitions=active_partitions, stream=stream, ) self.supervisor.add(aref) await aref.maybe_start() return aref async def _start_for_partitions(self, active_partitions: Set[TP]) -> ActorT: assert active_partitions self.log.info("Starting actor for partitions %s", active_partitions) return await self._start_one_supervised(None, active_partitions) async def on_start(self) -> None: """Call when an agent starts.""" self.supervisor = self._new_supervisor() await self._on_start_supervisor() def _new_supervisor(self) -> SupervisorStrategyT: return self._get_supervisor_strategy()( max_restarts=100.0, over=1.0, replacement=self._replace_actor, loop=self.loop, beacon=self.beacon, ) async def _replace_actor(self, service: ServiceT, index: int) -> ServiceT: aref = cast(ActorRefT, service) return await self._start_one( index=index, active_partitions=aref.active_partitions, stream=aref.stream, channel=cast(ChannelT, aref.stream.channel), ) def _get_supervisor_strategy(self) -> Type[SupervisorStrategyT]: SupervisorStrategy = self.supervisor_strategy if SupervisorStrategy is None: return cast(Type[SupervisorStrategyT], self.app.conf.agent_supervisor) else: return SupervisorStrategy async def _on_start_supervisor(self) -> None: active_partitions = self._get_active_partitions() channel: ChannelT = cast(ChannelT, None) for i in range(self.concurrency): res = await self._start_one( index=i, active_partitions=active_partitions, channel=channel, ) if channel is None: # First concurrency actor creates channel, # then we reuse it for --concurrency=n. # This way they share the same queue. channel = res.stream.channel self.supervisor.add(res) await self.supervisor.start() def _get_active_partitions(self) -> Optional[Set[TP]]: active_partitions: Optional[Set[TP]] = None if self.isolated_partitions: # when we start our first agent, we create the set of # partitions early, and save it in ._pending_active_partitions. # That way we can update the set once partitions are assigned, # and the actor we started may be assigned one of the partitions. active_partitions = self._pending_active_partitions = set() return active_partitions async def on_stop(self) -> None: """Call when an agent stops.""" # Agents iterate over infinite streams, so we cannot wait for it # to stop. # Instead we cancel it and this forces the stream to ack the # last message processed (but not the message causing the error # to be raised). await self._stop_supervisor() with suppress(asyncio.CancelledError): await asyncio.gather(*[ aref.actor_task for aref in self._actors if aref.actor_task is not None ]) self._actors.clear() async def _stop_supervisor(self) -> None: if self.supervisor: await self.supervisor.stop() self.supervisor = cast(SupervisorStrategyT, None) def cancel(self) -> None: """Cancel agent and its actor instances running in this process.""" for aref in self._actors: aref.cancel() async def on_partitions_revoked(self, revoked: Set[TP]) -> None: """Call when partitions are revoked.""" T = traced_from_parent_span() if self.isolated_partitions: # isolated: start/stop actors for each partition await T(self.on_isolated_partitions_revoked)(revoked) else: await T(self.on_shared_partitions_revoked)(revoked) async def on_partitions_assigned(self, assigned: Set[TP]) -> None: """Call when partitions are assigned.""" T = traced_from_parent_span() if self.isolated_partitions: await T(self.on_isolated_partitions_assigned)(assigned) else: await T(self.on_shared_partitions_assigned)(assigned) async def on_isolated_partitions_revoked(self, revoked: Set[TP]) -> None: """Call when isolated partitions are revoked.""" self.log.dev("Partitions revoked") T = traced_from_parent_span() for tp in revoked: aref: Optional[ActorRefT] = self._actor_by_partition.pop(tp, None) if aref is not None: await T(aref.on_isolated_partition_revoked)(tp) async def on_isolated_partitions_assigned(self, assigned: Set[TP]) -> None: """Call when isolated partitions are assigned.""" T = traced_from_parent_span() for tp in sorted(assigned): await T(self._assign_isolated_partition)(tp) async def _assign_isolated_partition(self, tp: TP) -> None: T = traced_from_parent_span() if not self._first_assignment_done and not self._actor_by_partition: self._first_assignment_done = True # if this is the first time we are assigned # we need to reassign the agent we started at boot to # one of the partitions. T(self._on_first_isolated_partition_assigned)(tp) await T(self._maybe_start_isolated)(tp) def _on_first_isolated_partition_assigned(self, tp: TP) -> None: assert self._actors assert len(self._actors) == 1 self._actor_by_partition[tp] = next(iter(self._actors)) if self._pending_active_partitions is not None: assert not self._pending_active_partitions self._pending_active_partitions.add(tp) async def _maybe_start_isolated(self, tp: TP) -> None: try: aref = self._actor_by_partition[tp] except KeyError: aref = await self._start_isolated(tp) self._actor_by_partition[tp] = aref await aref.on_isolated_partition_assigned(tp) async def _start_isolated(self, tp: TP) -> ActorT: return await self._start_for_partitions({tp}) async def on_shared_partitions_revoked(self, revoked: Set[TP]) -> None: """Call when non-isolated partitions are revoked.""" ... async def on_shared_partitions_assigned(self, assigned: Set[TP]) -> None: """Call when non-isolated partitions are assigned.""" ... def info(self) -> Mapping: """Return agent attributes as a dictionary.""" return { "app": self.app, "fun": self.fun, "name": self.name, "channel": self.channel, "concurrency": self.concurrency, "help": self.help, "sink": self._sinks, "on_error": self._on_error, "supervisor_strategy": self.supervisor_strategy, "isolated_partitions": self.isolated_partitions, } def clone(self, *, cls: Type[AgentT] = None, **kwargs: Any) -> AgentT: """Create clone of this agent object. Keyword arguments can be passed to override any argument supported by :class:`Agent.__init__ <Agent>`. """ return (cls or type(self))(**{**self.info(), **kwargs}) def test_context( self, channel: ChannelT = None, supervisor_strategy: SupervisorStrategyT = None, on_error: AgentErrorHandler = None, **kwargs: Any, ) -> AgentTestWrapperT: # pragma: no cover """Create new unit-testing wrapper for this agent.""" # flow control into channel queues are disabled at startup, # so need to resume that. self.app.flow_control.resume() async def on_agent_error(agent: AgentT, exc: BaseException) -> None: if on_error is not None: await on_error(agent, exc) await cast(AgentTestWrapper, agent).crash_test_agent(exc) return cast( AgentTestWrapperT, self.clone( cls=AgentTestWrapper, channel=channel if channel is not None else self.app.channel(), supervisor_strategy=supervisor_strategy or CrashingSupervisor, original_channel=self.channel, on_error=on_agent_error, **kwargs, ), ) def _prepare_channel( self, channel: Union[str, ChannelT] = None, internal: bool = True, schema: SchemaT = None, key_type: ModelArg = None, value_type: ModelArg = None, **kwargs: Any, ) -> ChannelT: app = self.app has_prefix = False if channel is None: channel = f"{app.conf.id}-{self.name}" has_prefix = True if isinstance(channel, ChannelT): return channel elif isinstance(channel, str): return app.topic( channel, internal=internal, schema=schema, key_type=key_type, value_type=value_type, has_prefix=has_prefix, **kwargs, ) raise TypeError( f"Channel must be channel, topic, or str; not {type(channel)}") def __call__( self, *, index: int = None, active_partitions: Set[TP] = None, stream: StreamT = None, channel: ChannelT = None, ) -> ActorRefT: """Create new actor instance for this agent.""" # The agent function can be reused by other agents/tasks. # For example: # # @app.agent(logs_topic, through='other-topic') # filter_log_errors_(stream): # async for event in stream: # if event.severity == 'error': # yield event # # @app.agent(logs_topic) # def alert_on_log_error(stream): # async for event in filter_log_errors(stream): # alert(f'Error occurred: {event!r}') # # Calling `res = filter_log_errors(it)` will end you up with # an AsyncIterable that you can reuse (but only if the agent # function is an `async def` function that yields) return self.actor_from_stream(stream, index=index, active_partitions=active_partitions, channel=channel) def actor_from_stream( self, stream: Optional[StreamT], *, index: int = None, active_partitions: Set[TP] = None, channel: ChannelT = None, ) -> ActorRefT: """Create new actor from stream.""" we_created_stream = False actual_stream: StreamT if stream is None: actual_stream = self.stream( channel=channel, concurrency_index=index, active_partitions=active_partitions, ) we_created_stream = True else: # reusing actor stream after agent restart assert stream.concurrency_index == index assert stream.active_partitions == active_partitions actual_stream = stream res = self.fun(actual_stream) if isinstance(res, AsyncIterable): if we_created_stream: actual_stream.add_processor(self._maybe_unwrap_reply_request) return cast( ActorRefT, AsyncIterableActor( self, actual_stream, res, index=actual_stream.concurrency_index, active_partitions=actual_stream.active_partitions, loop=self.loop, beacon=self.beacon, ), ) else: return cast( ActorRefT, AwaitableActor( self, actual_stream, res, index=actual_stream.concurrency_index, active_partitions=actual_stream.active_partitions, loop=self.loop, beacon=self.beacon, ), ) def add_sink(self, sink: SinkT) -> None: """Add new sink to further handle results from this agent.""" if sink not in self._sinks: self._sinks.append(sink) def stream(self, channel: ChannelT = None, active_partitions: Set[TP] = None, **kwargs: Any) -> StreamT: """Create underlying stream used by this agent.""" if channel is None: channel = cast(TopicT, self.channel_iterator).clone( is_iterator=False, active_partitions=active_partitions, ) if active_partitions is not None: assert channel.active_partitions == active_partitions s = self.app.stream( channel, loop=self.loop, active_partitions=active_partitions, prefix=self.name, beacon=self.beacon, **kwargs, ) return s def _maybe_unwrap_reply_request(self, value: V) -> Any: if isinstance(value, ReqRepRequest): return value.value return value async def _start_task( self, *, index: Optional[int], active_partitions: Optional[Set[TP]] = None, stream: StreamT = None, channel: ChannelT = None, beacon: NodeT = None, ) -> ActorRefT: # If the agent is an async function we simply start it, # if it returns an AsyncIterable/AsyncGenerator we start a task # that will consume it. actor = self( index=index, active_partitions=active_partitions, stream=stream, channel=channel, ) return await self._prepare_actor( actor, beacon if beacon is not None else self.beacon) async def _prepare_actor(self, aref: ActorRefT, beacon: NodeT) -> ActorRefT: coro: Any if isinstance(aref, Awaitable): # agent does not yield coro = aref if self._sinks: raise ImproperlyConfigured("Agent must yield to use sinks") else: # agent yields and is an AsyncIterator so we have to consume it. coro = self._slurp(aref, aiter(aref)) task = asyncio.Task(self._execute_actor(coro, aref), loop=self.loop) task._beacon = beacon # type: ignore aref.actor_task = task self._actors.add(aref) return aref async def _execute_actor(self, coro: Awaitable, aref: ActorRefT) -> None: # This executes the agent task itself, and does exception handling. _current_agent.set(self) try: await coro except asyncio.CancelledError: if self.should_stop: raise except Exception as exc: if self._on_error is not None: await self._on_error(self, exc) # Mark ActorRef as dead, so that supervisor thread # can start a new one. await aref.crash(exc) self.supervisor.wakeup() async def _slurp(self, res: ActorRefT, it: AsyncIterator) -> None: # this is used when the agent returns an AsyncIterator, # and simply consumes that async iterator. stream: Optional[StreamT] = None async for value in it: self.log.debug("%r yielded: %r", self.fun, value) if stream is None: stream = res.stream.get_active_stream() event = stream.current_event if event is not None: headers = event.headers reply_to: Optional[str] = None correlation_id: Optional[str] = None if isinstance(event.value, ReqRepRequest): req: ReqRepRequest = event.value reply_to = req.reply_to correlation_id = req.correlation_id elif headers: reply_to_bytes = headers.get("Faust-Ag-ReplyTo") if reply_to_bytes: reply_to = want_str(reply_to_bytes) correlation_id_bytes = headers.get( "Faust-Ag-CorrelationId") if correlation_id_bytes: correlation_id = want_str(correlation_id_bytes) if reply_to is not None: await self._reply(event.key, value, reply_to, cast(str, correlation_id)) await self._delegate_to_sinks(value) async def _delegate_to_sinks(self, value: Any) -> None: for sink in self._sinks: if isinstance(sink, AgentT): await sink.send(value=value) elif isinstance(sink, ChannelT): await cast(TopicT, sink).send(value=value) else: await maybe_async(cast(Callable, sink)(value)) async def _reply(self, key: Any, value: Any, reply_to: str, correlation_id: str) -> None: assert reply_to response = self._response_class(value)( key=key, value=value, correlation_id=correlation_id, ) await self.app.send( reply_to, key=None, value=response, ) def _response_class(self, value: Any) -> Type[ReqRepResponse]: if isinstance(value, ModelT): return ModelReqRepResponse return ReqRepResponse async def cast( self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None, headers: HeadersArg = None, ) -> None: """RPC operation: like :meth:`ask` but do not expect reply. Cast here is like "casting a spell", and will not expect a reply back from the agent. """ await self.send( key=key, value=value, partition=partition, timestamp=timestamp, headers=headers, ) async def ask( self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None, headers: HeadersArg = None, reply_to: ReplyToArg = None, correlation_id: str = None, ) -> Any: """RPC operation: ask agent for result of processing value. This version will wait until the result is available and return the processed value. """ p = await self.ask_nowait( value, key=key, partition=partition, timestamp=timestamp, headers=headers, reply_to=reply_to or self.app.conf.reply_to, correlation_id=correlation_id, force=True, # Send immediately, since we are waiting for result. ) app = cast(_App, self.app) await app._reply_consumer.add(p.correlation_id, p) await app.maybe_start_client() return await p async def ask_nowait( self, value: V = None, *, key: K = None, partition: int = None, timestamp: float = None, headers: HeadersArg = None, reply_to: ReplyToArg = None, correlation_id: str = None, force: bool = False, ) -> ReplyPromise: """RPC operation: ask agent for result of processing value. This version does not wait for the result to arrive, but instead returns a promise of future evaluation. """ if reply_to is None: raise TypeError("Missing reply_to argument") reply_to = self._get_strtopic(reply_to) correlation_id = correlation_id or str(uuid4()) value, headers = self._create_req(key, value, reply_to, correlation_id, headers) await self.channel.send( key=key, value=value, partition=partition, timestamp=timestamp, headers=headers, force=force, ) return ReplyPromise(reply_to, correlation_id) def _create_req( self, key: K = None, value: V = None, reply_to: ReplyToArg = None, correlation_id: str = None, headers: HeadersArg = None, ) -> Tuple[V, Optional[HeadersArg]]: if reply_to is None: raise TypeError("Missing reply_to argument") topic_name = self._get_strtopic(reply_to) correlation_id = correlation_id or str(uuid4()) open_headers = prepare_headers(headers or {}) if self.use_reply_headers: merge_headers( open_headers, { "Faust-Ag-ReplyTo": want_bytes(topic_name), "Faust-Ag-CorrelationId": want_bytes(correlation_id), }, ) return value, open_headers else: # wrap value in envelope req = self._request_class(value)( value=value, reply_to=topic_name, correlation_id=correlation_id, ) return req, open_headers def _request_class(self, value: V) -> Type[ReqRepRequest]: if isinstance(value, ModelT): return ModelReqRepRequest return ReqRepRequest async def send( self, *, key: K = None, value: V = None, partition: int = None, timestamp: float = None, headers: HeadersArg = None, key_serializer: CodecArg = None, value_serializer: CodecArg = None, callback: MessageSentCallback = None, reply_to: ReplyToArg = None, correlation_id: str = None, force: bool = False, ) -> Awaitable[RecordMetadata]: """Send message to topic used by agent.""" if reply_to: value, headers = self._create_req(key, value, reply_to, correlation_id, headers) return await self.channel.send( key=key, value=value, partition=partition, timestamp=timestamp, headers=headers, key_serializer=key_serializer, value_serializer=value_serializer, force=force, ) def _get_strtopic(self, topic: Union[str, ChannelT, TopicT, AgentT]) -> str: if isinstance(topic, AgentT): return self._get_strtopic(topic.channel) if isinstance(topic, TopicT): return topic.get_topic_name() if isinstance(topic, ChannelT): raise ValueError("Channels are unnamed topics") return topic async def map( self, values: Union[AsyncIterable, Iterable], key: K = None, reply_to: ReplyToArg = None, ) -> AsyncIterator: # pragma: no cover """RPC map operation on a list of values. A map operation iterates over results as they arrive. See :meth:`join` and :meth:`kvjoin` if you want them in order. """ # Map takes only values, but can provide one key that is used for all. async for value in self.kvmap(((key, v) async for v in aiter(values)), reply_to): yield value async def kvmap( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg = None, ) -> AsyncIterator[str]: # pragma: no cover """RPC map operation on a list of ``(key, value)`` pairs. A map operation iterates over results as they arrive. See :meth:`join` and :meth:`kvjoin` if you want them in order. """ # kvmap takes (key, value) pairs. reply_to = self._get_strtopic(reply_to or self.app.conf.reply_to) # BarrierState is the promise that keeps track of pending results. # It contains a list of individual ReplyPromises. barrier = BarrierState(reply_to) async for _ in self._barrier_send(barrier, items, reply_to): # Now that we've sent a message, try to see if we have any # replies. try: _, val = barrier.get_nowait() except asyncio.QueueEmpty: pass else: yield val # All the messages have been sent so finalize the barrier. barrier.finalize() # Then iterate over the results in the group. async for _, value in barrier.iterate(): yield value async def join( self, values: Union[AsyncIterable[V], Iterable[V]], key: K = None, reply_to: ReplyToArg = None, ) -> List[Any]: # pragma: no cover """RPC map operation on a list of values. A join returns the results in order, and only returns once all values have been processed. """ return await self.kvjoin( ((key, value) async for value in aiter(values)), reply_to=reply_to, ) async def kvjoin( self, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg = None, ) -> List[Any]: # pragma: no cover """RPC map operation on list of ``(key, value)`` pairs. A join returns the results in order, and only returns once all values have been processed. """ reply_to = self._get_strtopic(reply_to or self.app.conf.reply_to) barrier = BarrierState(reply_to) # Map correlation_id -> index posindex: MutableMapping[str, int] = { cid: i async for i, cid in aenumerate( self._barrier_send(barrier, items, reply_to)) } # All the messages have been sent so finalize the barrier. barrier.finalize() # wait until all replies received await barrier # then construct a list in the correct order. values: List = [None] * barrier.total async for correlation_id, value in barrier.iterate(): values[posindex[correlation_id]] = value return values async def _barrier_send( self, barrier: BarrierState, items: Union[AsyncIterable[Tuple[K, V]], Iterable[Tuple[K, V]]], reply_to: ReplyToArg, ) -> AsyncIterator[str]: # pragma: no cover # map: send many tasks to agents # while trying to pop incoming results off. key: K value: V async for key, value in aiter(items): # type: ignore correlation_id = str(uuid4()) p = await self.ask_nowait(key=key, value=value, reply_to=reply_to, correlation_id=correlation_id) # add reply promise to the barrier barrier.add(p) # the ReplyConsumer will call the barrier whenever a new # result comes in. app = cast(_App, self.app) await app.maybe_start_client() await app._reply_consumer.add(p.correlation_id, barrier) yield correlation_id def _repr_info(self) -> str: return shorten_fqdn(self.name) def get_topic_names(self) -> Iterable[str]: """Return list of topic names this agent subscribes to.""" channel = self.channel if isinstance(channel, TopicT): return channel.topics return [] @property def channel(self) -> ChannelT: """Return channel used by agent.""" if self._channel is None: self._channel = self._prepare_channel( self._channel_arg, schema=self._schema, key_type=self._key_type, value_type=self._value_type, **self._channel_kwargs, ) return self._channel @channel.setter def channel(self, channel: ChannelT) -> None: self._channel = channel @property def channel_iterator(self) -> AsyncIterator: """Return channel agent iterates over.""" # The channel is "memoized" here, so subsequent access to # instance.channel_iterator will return the same value. if self._channel_iterator is None: # we do not use aiter(channel) here, because # that will also add it to the topic conductor too early. self._channel_iterator = self.channel.clone(is_iterator=False) return self._channel_iterator @channel_iterator.setter def channel_iterator(self, it: AsyncIterator) -> None: self._channel_iterator = it @property def label(self) -> str: """Return human-readable description of agent.""" return self._agent_label() def _agent_label(self, name_suffix: str = "") -> str: s = f"{type(self).__name__}{name_suffix}: " s += f"{shorten_fqdn(qualname(self.fun))}" return s @property def shortlabel(self) -> str: """Return short description of agent.""" return self._agent_label()
class TaskManager(object): """ Provides a set of tools to maintain a list of asyncio Tasks that are to be executed during the lifetime of an arbitrary object, usually getting killed with it. """ def __init__(self): self._pending_tasks = WeakValueDictionary() self._task_lock = RLock() self._shutdown = False self._counter = 0 self._logger = logging.getLogger(self.__class__.__name__) self._checker = self.register_task('_check_tasks', self._check_tasks, interval=MAX_TASK_AGE, delay=MAX_TASK_AGE * 1.5) def _check_tasks(self): now = time.time() for name, task in self._pending_tasks.items(): if not task.interval and now - task.start_time > MAX_TASK_AGE: self._logger.warning( 'Non-interval task "%s" has been running for %.2f!', name, now - task.start_time) def replace_task(self, name, *args, **kwargs): """ Replace named task with the new one, cancelling the old one in the process. """ new_task = Future() def cancel_cb(_): try: new_task.set_result(self.register_task(name, *args, **kwargs)) except Exception as e: new_task.set_exception(e) old_task = self.cancel_pending_task(name) old_task.add_done_callback(cancel_cb) return new_task def register_task(self, name, task, *args, delay=None, interval=None, ignore=()): """ Register a Task/(coroutine)function so it can be canceled at shutdown time or by name. """ if not isinstance( task, Task) and not iscoroutinefunction(task) and not callable(task): raise ValueError( 'Register_task takes a Task or a (coroutine)function as a parameter' ) if (interval or delay) and isinstance(task, Task): raise ValueError('Cannot run Task at an interval or with a delay') if not isinstance(ignore, tuple) or not all( (issubclass(e, Exception) for e in ignore)): raise ValueError('Ignore should be a tuple of Exceptions or None') with self._task_lock: if self._shutdown: self._logger.warning("Not adding task %s due to shutdown!", str(task)) if isinstance(task, (Task, Future)): if not task.done(): task.cancel() return task if self.is_pending_task_active(name): raise RuntimeError("Task already exists: '%s'" % name) if iscoroutinefunction(task) or callable(task): task = task if iscoroutinefunction(task) else coroutine(task) if interval: # The default delay for looping calls is the same as the interval delay = interval if delay is None else delay task = ensure_future( interval_runner(delay, interval, task, *args)) elif delay: task = ensure_future(delay_runner(delay, task, *args)) else: task = ensure_future(task(*args)) # Since weak references to list/tuple are not allowed, we're not storing start_time/interval # in _pending_tasks. Instead we add them as attributes to the task. task.start_time = time.time() task.interval = interval assert isinstance(task, Task) def done_cb(future): self._pending_tasks.pop(name, None) try: future.result() except CancelledError: pass except ignore as e: self._logger.error('Task resulted in error: %s', e) self._pending_tasks[name] = task task.add_done_callback(done_cb) return task def register_anonymous_task(self, basename, task, *args, **kwargs): """ Wrapper for register_task to derive a unique name from the basename. """ self._counter += 1 return self.register_task(basename + ' ' + str(self._counter), task, *args, **kwargs) def cancel_pending_task(self, name): """ Cancels the named task """ with self._task_lock: task = self._pending_tasks.get(name, None) if not task: return succeed(None) if not task.done(): task.cancel() self._pending_tasks.pop(name, None) return task def cancel_all_pending_tasks(self): """ Cancels all the registered tasks. This usually should be called when stopping or destroying the object so no tasks are left floating around. """ with self._task_lock: assert all([ isinstance(t, (Task, Future)) for t in self._pending_tasks.values() ]), self._pending_tasks return [ self.cancel_pending_task(name) for name in list(self._pending_tasks.keys()) ] def is_pending_task_active(self, name): """ Return a boolean determining if a task is active. """ with self._task_lock: task = self._pending_tasks.get(name, None) return not task.done() if task else False def get_tasks(self): """ Returns a list of all registered tasks, excluding tasks the are created by the TaskManager itself. """ with self._task_lock: return [ t for t in self._pending_tasks.values() if t != self._checker ] async def wait_for_tasks(self): """ Waits until all registered tasks are done. """ with self._task_lock: tasks = self.get_tasks() if tasks: await gather(*tasks, return_exceptions=True) async def shutdown_task_manager(self): """ Clear the task manager, cancel all pending tasks and disallow new tasks being added. """ with self._task_lock: self._shutdown = True tasks = self.cancel_all_pending_tasks() if tasks: with suppress(CancelledError): await gather(*tasks)
class RedisStore(): def __init__(self, db_host, db_port, db_num, db_pw): self.pool = ConnectionPool(max_connections=2, db=db_num, host=db_host, port=db_port, password=db_pw, decode_responses=True) self.redis = StrictRedis(connection_pool=self.pool) self.redis.ping() self._object_map = WeakValueDictionary() def create_object(self, dbo_class, dbo_dict, update_timestamp=True): dbo_class = get_dbo_class(getattr(dbo_class, 'dbo_key_type', dbo_class)) if not dbo_class: return try: dbo_id = dbo_dict['dbo_id'] except KeyError: dbo_id, dbo_dict = dbo_dict, {} if dbo_id is None or dbo_id == '': warn("create_object called with empty dbo_id") return dbo_id = str(dbo_id).lower() if self.object_exists(dbo_class.dbo_key_type, dbo_id): raise ObjectExistsError(dbo_id) dbo = dbo_class() dbo.dbo_id = dbo_id dbo.hydrate(dbo_dict) dbo.db_created() if dbo.dbo_set_key: self.redis.sadd(dbo.dbo_set_key, dbo.dbo_id) self.save_object(dbo, update_timestamp) return dbo def save_object(self, dbo, update_timestamp=False, autosave=False): if update_timestamp: dbo.dbo_ts = int(time.time()) if dbo.dbo_indexes: self._update_indexes(dbo) self._clear_old_refs(dbo) save_root, new_refs = dbo.to_db_value() self.redis.set(dbo.dbo_key, json_encode(save_root)) if new_refs: self._set_new_refs(dbo, new_refs) debug("db object {} {}saved", dbo.dbo_key, "auto" if autosave else "") self._object_map[dbo.dbo_key] = dbo return dbo def save_raw(self, key, raw): self.redis.set(key, json_encode(raw)) def load_raw(self, key, default=None): json = self.redis.get(key) if json: return json_decode(json) return default def load_cached(self, dbo_key): return self._object_map.get(dbo_key) def load_object(self, dbo_key, key_type=None, silent=False): if key_type: try: key_type = key_type.dbo_key_type except AttributeError: pass try: dbo_key, dbo_id = ':'.join([key_type, dbo_key]), dbo_key except TypeError: if not silent: exception("Invalid dbo_key passed to load_object", stack_info=True) return else: key_type, _, dbo_id = dbo_key.partition(':') cached_dbo = self._object_map.get(dbo_key) if cached_dbo: return cached_dbo json_str = self.redis.get(dbo_key) if not json_str: if not silent: warn("Failed to find {} in database", dbo_key) return return self.load_from_json(json_str, key_type, dbo_id) def load_from_json(self, json_str, key_type, dbo_id): dbo_dict = json_decode(json_str) dbo = get_mixed_type(key_type, dbo_dict.get('mixins'))() dbo.dbo_id = dbo_id self._object_map[dbo.dbo_key] = dbo dbo.hydrate(dbo_dict) return dbo def object_exists(self, obj_type, obj_id): return self.redis.exists('{}:{}'.format(obj_type, obj_id)) def load_object_set(self, dbo_class, set_key=None): key_type = dbo_class.dbo_key_type if not set_key: set_key = dbo_class.dbo_set_key results = set() keys = deque() pipeline = self.redis.pipeline() for key in self.fetch_set_keys(set_key): dbo_key = ':'.join([key_type, key]) try: results.add(self._object_map[dbo_key]) except KeyError: keys.append(key) pipeline.get(dbo_key) for dbo_id, json_str in zip(keys, pipeline.execute()): if json_str: obj = self.load_from_json(json_str, key_type, dbo_id) if obj: results.add(obj) continue warn("Removing missing object from set {}", set_key) self.delete_set_key(set_key, dbo_id) return results def delete_object_set(self, dbo_class, set_key=None): if not set_key: set_key = dbo_class.dbo_set_key for dbo in self.load_object_set(dbo_class, set_key): self.delete_object(dbo) self.delete_key(set_key) def update_object(self, dbo, dbo_dict): dbo.hydrate(dbo_dict) return self.save_object(dbo, True) def delete_object(self, dbo): key = dbo.dbo_key dbo.db_deleted() self.delete_key(key) self._clear_old_refs(dbo) if dbo.dbo_set_key: self.redis.srem(dbo.dbo_set_key, dbo.dbo_id) for children_type in dbo.dbo_children_types: self.delete_object_set(get_dbo_class(children_type), "{}_{}s:{}".format(dbo.dbo_key_type, children_type, dbo.dbo_id)) for ix_name in dbo.dbo_indexes: ix_value = getattr(dbo, ix_name, None) if ix_value is not None and ix_value != '': self.delete_index('ix:{}:{}'.format(dbo.dbo_key_type, ix_name), ix_value) debug("object deleted: {}", key) self.evict_object(dbo) def reload_object(self, dbo_key): dbo = self._object_map.get(dbo_key) if dbo: json_str = self.redis.get(dbo_key) if not json_str: warn("Failed to find {} in database for reload", dbo_key) return None return self.update_object(dbo, json_decode(json_str)) return self.load_object(dbo_key) def evict_object(self, dbo): self._object_map.pop(dbo.dbo_key, None) def fetch_set_keys(self, set_key): return self.redis.smembers(set_key) def add_set_key(self, set_key, *values): self.redis.sadd(set_key, *values) def delete_set_key(self, set_key, value): self.redis.srem(set_key, value) def set_key_exists(self, set_key, value): return self.redis.sismember(set_key, value) def db_counter(self, counter_id, inc=1): return self.redis.incr("counter:{}".format(counter_id), inc) def delete_key(self, key): self.redis.delete(key) def set_index(self, index_name, key, value): return self.redis.hset(index_name, key, value) def get_index(self, index_name, key): return self.redis.hget(index_name, key) def get_full_index(self, index_name): return self.redis.hgetall(index_name) def delete_index(self, index_name, key): return self.redis.hdel(index_name, key) def get_all_hash(self, index_name): return {key: json_decode(value) for key, value in self.redis.hgetall(index_name).items()} def set_db_hash(self, hash_id, hash_key, value): return self.redis.hset(hash_id, hash_key, json_encode(value)) def get_db_hash(self, hash_id, hash_key): return json_decode(self.redis.hget(hash_id, hash_key)) def remove_db_hash(self, hash_id, hash_key): self.redis.hdel(hash_id, hash_key) def get_all_db_hash(self, hash_id): return [json_decode(value) for value in self.redis.hgetall(hash_id).values()] def get_db_list(self, list_id, start=0, end=-1): return [json_decode(value) for value in self.redis.lrange(list_id, start, end)] def add_db_list(self, list_id, value): self.redis.lpush(list_id, json_encode(value)) def trim_db_list(self, list_id, start, end): return self.redis.ltrim(list_id, start, end) def _update_indexes(self, dbo): try: old_dbo = json_decode(self.redis.get(dbo.dbo_key)) except TypeError: old_dbo = None for ix_name in dbo.dbo_indexes: new_val = getattr(dbo, ix_name, None) old_val = old_dbo.get(ix_name) if old_dbo else None if old_val == new_val: continue ix_key = 'ix:{}:{}'.format(dbo.dbo_key_type, ix_name) if old_val is not None: self.delete_index(ix_key, old_val) if new_val is not None and new_val != '': if self.get_index(ix_key, new_val): raise NonUniqueError(ix_key, new_val) self.set_index(ix_key, new_val, dbo.dbo_id) def _clear_old_refs(self, dbo): dbo_key = dbo.dbo_key ref_key = '{}:refs'.format(dbo_key) for ref_id in self.fetch_set_keys(ref_key): holder_key = '{}:holders'.format(ref_id) self.delete_set_key(holder_key, dbo_key) self.delete_key(ref_key) def _set_new_refs(self, dbo, new_refs): dbo_key = dbo.dbo_key self.add_set_key("{}:refs".format(dbo_key), *new_refs) for ref_id in new_refs: holder_key = '{}:holders'.format(ref_id) self.add_set_key(holder_key, dbo_key)
class EntityCache(object): """ Cache for entities. Supports add and remove operations as well as lookup by ID and by slug. """ def __init__(self, allow_none_id=False): """ :param bool allow_none_id: Flag specifying if calling :meth:`add` with an entity that does not have an ID is allowed. """ # self.__allow_none_id = allow_none_id # List of cached entities. This is the only place we are holding a # real reference to the entity. self.__entities = [] # Dictionary mapping entity IDs to entities for fast lookup by ID. self.__id_map = WeakValueDictionary() # Dictionary mapping entity slugs to entities for fast lookup by slug. self.__slug_map = WeakValueDictionary() def get_by_id(self, entity_id): """ Performs a lookup of an entity by its ID. :param int entity_id: entity ID. :return: entity found or ``None``. """ return self.__id_map.get(entity_id) def has_id(self, entity_id): """ Checks if this entity cache holds an entity with the given ID. :return: Boolean result of the check. """ return entity_id in self.__id_map def get_by_slug(self, entity_slug): """ Performs a lookup of an entity by its slug. :param str entity_id: entity slug. :return: entity found or ``None``. """ return self.__slug_map.get(entity_slug) def has_slug(self, entity_slug): return entity_slug in self.__slug_map def add(self, entity): """ Adds the given entity to this cache. :param entity: Entity to add. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises ValueError: If the ID of the entity to add is ``None``. """ # For certain use cases (e.g., staging), we do not want the entity to # be added to have an ID yet. if not entity.id is None: if entity.id in self.__id_map: raise ValueError('Duplicate entity ID "%s".' % entity.id) self.__id_map[entity.id] = entity elif not self.__allow_none_id: raise ValueError('Entity ID must not be None.') # The slug can be a lazy attribute depending on the # value of other (possibly not yet initialized) attributes which is # why we can not always assume it is available at this point. if not entity.slug is None: if entity.slug in self.__slug_map: raise ValueError('Duplicate entity slug "%s".' % entity.slug) self.__slug_map[entity.slug] = entity self.__entities.append(entity) def remove(self, entity): """ Removes the given entity from this cache. :param entity: Entity to remove. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises KeyError: If the given entity is not in this cache. :raises ValueError: If the ID of the given entity is `None`. """ if entity.id is None: raise ValueError('Entity ID must not be None.') del self.__id_map[entity.id] # We may not have the slug in the slug map because it might not have # been available by the time the entity was added. self.__slug_map.pop(entity.slug, None) self.__entities.remove(entity) def replace(self, entity): """ Replaces the current entity that has the same ID as the given new entity with the latter. :param entity: Entity to replace. :type entity: Object implementing :class:`everest.interfaces.IEntity`. :raises KeyError: If the given entity is not in this cache. :raises ValueError: If the ID of the given entity is `None`. """ if entity.id is None: raise ValueError('Entity ID must not be None.') old_entity = self.__id_map[entity.id] self.remove(old_entity) self.add(entity) def iterator(self): """ Returns an iterator over all entities in this cache in the order they were added. """ return iter(self.__entities)
class TrollReactor(CBDictInterface): """Base class for Omegle API. """ def __init__(self, transmog=Transmogrifier(), listen=Viewport(), n=2, refresh=1.5): # Independent setup super(TrollReactor, self).__init__() self.listeners = WeakValueDictionary() # Argument assignment self.eventQueue = deque() self.connectTransmogrifier(transmog) self.addListeners(listen) self._n = n self.refresh = refresh self.initializeStrangers() # Now we wait to receive idSet events def connectTransmogrifier(self, transmog): self.transmogrifier = transmog self.transmogrifier.connect(self.eventQueue) def initializeStrangers(self): self._volatile = {Stranger(reactor, self, HTTP): None for _ in xrange(self._n)} self._waiting = len(self._volatile.keys()) self.strangers = {} def multicastDisconnect(self, ids): """Announce disconnect for a group of strangers. ids : iterable id strings of strangers from whom to politely disconnect. """ for i in ids: self.strangers[i].announceDisconnect() def pumpEvents(self): for id_ in self.strangers: self.strangers[id_].getEventsPage() reactor.callLater(self.refresh, self.pumpEvents) def on_idSet(self, ev): for s in self._volatile: if s.id == ev.id: # we have the stranger that notified self.strangers[s.id] = s # move to {id: stranger} dict self._waiting -= 1 if self._waiting == 0: self.pumpEvents() elif self._waiting < 0: print_stack() stderr.write("ERROR: too many stranger IDs.") reactor.stop() def addListeners(self, listeners): """Add a listener or group of listeners to the reactor. listeners : CBDictInterface instance or iterable """ if not hasattr(listeners, '__iter__'): listeners = (listeners,) for listen in listeners: self.listeners[listen] = listen # weak-value dict def removeListener(self, listener): self.listeners.pop(listener) def _processEventQueue(self): while len(self.eventQueue): ev = self.eventQueue.popleft() for listener in self.listeners: listener.notify(ev) self.notify(ev) def feed(self, events): """Notify the TrollReactor of an event. """ if hasattr(events, '_fields'): events = (events,) # convert to tuple self.transmogrifier(events) self._processEventQueue()
class Boss: def __init__(self, os_window_id, opts, args, cached_values, new_os_window_trigger): set_layout_options(opts) self.clipboard_buffers = {} self.update_check_process = None self.window_id_map = WeakValueDictionary() self.startup_colors = { k: opts[k] for k in opts if isinstance(opts[k], Color) } self.startup_cursor_text_color = opts.cursor_text_color self.pending_sequences = None self.cached_values = cached_values self.os_window_map = {} self.os_window_death_actions = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.opts, self.args = opts, args startup_sessions = create_sessions( opts, args, default_session=opts.startup_session) self.keymap = self.opts.keymap.copy() if new_os_window_trigger is not None: self.keymap.pop(new_os_window_trigger, None) for startup_session in startup_sessions: self.add_os_window(startup_session, os_window_id=os_window_id) os_window_id = None if args.start_as != 'normal': if args.start_as == 'fullscreen': self.toggle_fullscreen() else: change_os_window_state(args.start_as) if is_macos: from .fast_data_types import cocoa_set_notification_activated_callback cocoa_set_notification_activated_callback( self.notification_activated) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, opts_for_size=None, startup_id=None): if os_window_id is None: opts_for_size = opts_for_size or startup_session.os_window_size or self.opts cls = wclass or self.args.cls or appname with startup_notification_handler( do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback: os_window_id = create_os_window( initial_window_size_func(opts_for_size, self.cached_values), pre_show_callback, appname, wname or self.args.name or cls, cls) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm return os_window_id def list_os_windows(self): with cached_process_data(): active_tab, active_window = self.active_tab, self.active_window active_tab_manager = self.active_tab_manager for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'is_focused': tm is active_tab_manager, 'tabs': list(tm.list_tabs(active_tab, active_window)), } @property def all_tab_managers(self): yield from self.os_window_map.values() @property def all_tabs(self): for tm in self.all_tab_managers: yield from tm @property def all_windows(self): for tab in self.all_tabs: yield from tab def match_windows(self, match): try: field, exp = match.split(':', 1) except ValueError: return if field == 'num': tab = self.active_tab if tab is not None: try: w = tab.get_nth_window(int(exp)) except Exception: return if w is not None: yield w return if field == 'env': kp, vp = exp.partition('=')[::2] if vp: pat = tuple(map(re.compile, (kp, vp))) else: pat = re.compile(kp), None else: pat = re.compile(exp) for window in self.all_windows: if window.matches(field, pat): yield window def tab_for_window(self, window): for tab in self.all_tabs: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): try: field, exp = match.split(':', 1) except ValueError: return pat = re.compile(exp) found = False if field in ('title', 'id'): for tab in self.all_tabs: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for os_window_id, tm in self.os_window_map.items(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return os_window_id def _new_os_window(self, args, cwd_from=None): if isinstance(args, SpecialWindowInstance): sw = args else: sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = next( create_sessions(self.opts, special_window=sw, cwd_from=cwd_from)) return self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) @property def active_window_for_cwd(self): w = self.active_window if w is not None and w.overlay_for is not None and w.overlay_for in self.window_id_map: w = self.window_id_map[w.overlay_for] return w def new_os_window_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._new_os_window(args, cwd_from) def new_os_window_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_os_window(special_window) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control or getattr( window, 'allow_remote_control', False): try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err)} if not getattr(err, 'hide_traceback', False): response['tb'] = traceback.format_exc() else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) if not os.path.isabs(args.directory): args.directory = os.path.join(msg['cwd'], args.directory) for session in create_sessions(opts, args, respect_cwd=True): os_window_id = self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id) if msg.get('notify_on_os_window_death'): self.os_window_death_actions[os_window_id] = partial( self.notify_on_os_window_death, msg['notify_on_os_window_death']) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def toggle_maximized(self): toggle_maximized() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True if self.opts.update_check_interval > 0 and not hasattr( self, 'update_check_started'): from .update_check import run_update_check run_update_check(self.opts.update_check_interval * 60 * 60) self.update_check_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): if dpi_changed: self.on_dpi_change(os_window_id) else: tm = self.os_window_map.get(os_window_id) if tm is not None: tm.resize() def clear_terminal(self, action, only_active): if only_active: windows = [] w = self.active_window if w is not None: windows.append(w) else: windows = self.all_windows reset = action == 'reset' how = 3 if action == 'scrollback' else 2 for w in windows: if action == 'scroll': w.screen.scroll_until_cursor() continue w.screen.cursor.x = w.screen.cursor.y = 0 if reset: w.screen.reset() else: w.screen.erase_in_display(how, False) def increase_font_size(self): # legacy cfs = global_font_size() self.set_font_size(min(self.opts.font_size * 5, cfs + 2.0)) def decrease_font_size(self): # legacy cfs = global_font_size() self.set_font_size(max(MINIMUM_FONT_SIZE, cfs - 2.0)) def restore_font_size(self): # legacy self.set_font_size(self.opts.font_size) def set_font_size(self, new_size): # legacy self.change_font_size(True, None, new_size) def change_font_size(self, all_windows, increment_operation, amt): def calc_new_size(old_size): new_size = old_size if amt == 0: new_size = self.opts.font_size else: if increment_operation: new_size += (1 if increment_operation == '+' else -1) * amt else: new_size = amt new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5)) return new_size if all_windows: current_global_size = global_font_size() new_size = calc_new_size(current_global_size) if new_size != current_global_size: global_font_size(new_size) os_windows = tuple(self.os_window_map.keys()) else: os_windows = [] w = self.active_window if w is not None: os_windows.append(w.os_window_id) if os_windows: final_windows = {} for wid in os_windows: current_size = os_window_font_size(wid) if current_size: new_size = calc_new_size(current_size) if new_size != current_size: final_windows[wid] = new_size if final_windows: self._change_font_size(final_windows) def _change_font_size(self, sz_map): for os_window_id, sz in sz_map.items(): tm = self.os_window_map.get(os_window_id) if tm is not None: os_window_font_size(os_window_id, sz) tm.resize() def on_dpi_change(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: sz = os_window_font_size(os_window_id) if sz: os_window_font_size(os_window_id, sz, True) tm.resize() def _set_os_window_background_opacity(self, os_window_id, opacity): change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0))) def set_background_opacity(self, opacity): window = self.active_window if window is None or not opacity: return if not self.opts.dynamic_background_opacity: return self.show_error( _('Cannot change background opacity'), _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity' )) os_window_id = window.os_window_id if opacity[0] in '+-': old_opacity = background_opacity_of(os_window_id) if old_opacity is None: return opacity = old_opacity + float(opacity) elif opacity == 'default': opacity = self.opts.background_opacity else: opacity = float(opacity) self._set_os_window_background_opacity(os_window_id, opacity) @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.keymap, mods, key, scancode) if key_action is None: sequences = get_shortcut(self.opts.sequence_map, mods, key, scancode) if sequences: self.pending_sequences = sequences set_in_sequence_mode(True) return True else: self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def process_sequence(self, key, scancode, action, mods): if not self.pending_sequences: set_in_sequence_mode(False) remaining = {} matched_action = None for seq, key_action in self.pending_sequences.items(): if shortcut_matches(seq[0], mods, key, scancode): seq = seq[1:] if seq: remaining[seq] = key_action else: matched_action = key_action if remaining: self.pending_sequences = remaining else: self.pending_sequences = None set_in_sequence_mode(False) if matched_action is not None: self.dispatch_action(matched_action) def start_resizing_window(self): w = self.active_window if w is None: return overlay_window = self._run_kitten( 'resize_window', args=[ '--horizontal-increment={}'.format( self.opts.window_resize_step_cells), '--vertical-increment={}'.format( self.opts.window_resize_step_lines) ]) if overlay_window is not None: overlay_window.allow_remote_control = True def resize_layout_window(self, window, increment, is_horizontal, reset=False): tab = window.tabref() if tab is None or not increment: return False if reset: return tab.reset_window_sizes() return tab.resize_window_by(window.id, increment, is_horizontal) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: tm.update_tab_bar_data() tm.mark_tab_bar_dirty() t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) tm.mark_tab_bar_dirty() def update_tab_bar_data(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.update_tab_bar_data() def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) action = self.os_window_death_actions.pop(os_window_id, None) if action is not None: action() def notify_on_os_window_death(self, address): import socket s = socket.socket(family=socket.AF_UNIX) with suppress(Exception): s.connect(address) s.sendall(b'c') with suppress(EnvironmentError): s.shutdown(socket.SHUT_RDWR) s.close() def display_scrollback(self, window, data, cmd): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(cmd, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ kitty_exe(), '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + get_editor() + [confpath] self.new_os_window(*cmd) def get_output(self, source_window, num_lines=1): output = '' s = source_window.screen if num_lines is None: num_lines = s.lines for i in range(min(num_lines, s.lines)): output += str(s.linebuf.line(i)) return output def _run_kitten(self, kitten, args=(), input_data=None, window=None): orig_args, args = list(args), list(args) from kittens.runner import create_kitten_handler end_kitten = create_kitten_handler(kitten, orig_args) if window is None: w = self.active_window tab = self.active_tab else: w = window tab = w.tabref() if end_kitten.no_ui: end_kitten(None, getattr(w, 'id', None), self) return if w is not None and tab is not None and w.overlay_for is None: args[0:0] = [config_dir, kitten] if input_data is None: type_of_input = end_kitten.type_of_input if type_of_input in ('text', 'history', 'ansi', 'ansi-history', 'screen', 'screen-history', 'screen-ansi', 'screen-ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input, add_wrap_markers='screen' in type_of_input).encode('utf-8') elif type_of_input is None: data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) else: data = input_data if isinstance(data, str): data = data.encode('utf-8') copts = { k: self.opts[k] for k in ('select_by_word_characters', 'open_url_with') } overlay_window = tab.new_special_window( SpecialWindow([ kitty_exe(), '+runpy', 'from kittens.runner import main; main()' ] + args, stdin=data, env={ 'KITTY_COMMON_OPTS': json.dumps(copts), 'KITTY_CHILD_PID': w.child.pid, 'PYTHONWARNINGS': 'ignore', 'OVERLAID_WINDOW_LINES': str(w.screen.lines), 'OVERLAID_WINDOW_COLS': str(w.screen.columns), }, cwd=w.cwd_of_child, overlay_for=w.id)) overlay_window.action_on_close = partial(self.on_kitten_finish, w.id, end_kitten) return overlay_window def kitten(self, kitten, *args): import shlex cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] self._run_kitten(kitten, args) def on_kitten_finish(self, target_window_id, end_kitten, source_window): output = self.get_output(source_window, num_lines=None) from kittens.runner import deserialize data = deserialize(output) if data is not None: end_kitten(data, target_window_id, self) def input_unicode_character(self): self._run_kitten('unicode_input') def set_tab_title(self): tab = self.active_tab if tab: args = [ '--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id) ] self._run_kitten('ask', args) def show_error(self, title, msg): self._run_kitten('show_error', args=['--title', title], input_data=msg) def do_set_tab_title(self, title, tab_id): tm = self.active_tab_manager if tm is not None and title: tab_id = int(tab_id) for tab in tm.tabs: if tab.id == tab_id: tab.set_title(title) break def kitty_shell(self, window_type): cmd = ['@', kitty_exe(), '@'] if window_type == 'tab': self._new_tab(cmd) elif window_type == 'os_window': os_window_id = self._new_os_window(cmd) self.os_window_map[os_window_id] elif window_type == 'overlay': w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: tab.new_special_window(SpecialWindow(cmd, overlay_for=w.id)) else: self._new_window(cmd) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) def open_url(self, url, program=None, cwd=None): if url: if isinstance(program, str): program = to_cmdline(program) open_url(url, program or self.opts.open_url_with, cwd=cwd) def open_url_lines(self, lines, program=None): self.open_url(''.join(lines), program) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() self.set_update_check_process() self.update_check_process = None del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection( ) if supports_primary_selection else get_clipboard_string() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: self.copy_to_buffer(self.opts.copy_on_select) def copy_to_buffer(self, buffer_name): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: if buffer_name == 'clipboard': set_clipboard_string(text) elif buffer_name == 'primary': set_primary_selection(text) else: self.clipboard_buffers[buffer_name] = text def paste_from_buffer(self, buffer_name): if buffer_name == 'clipboard': text = get_clipboard_string() elif buffer_name == 'primary': text = get_primary_selection() else: text = self.clipboard_buffers.get(buffer_name) if text: self.paste_to_active_window(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def set_active_tab(self, tab): tm = self.active_tab_manager if tm is not None: return tm.set_active_tab(tab) return False def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) prev_tab = previous_tab def process_stdin_source(self, window=None, stdin=None): w = window or self.active_window env = None if stdin: add_wrap_markers = stdin.endswith('_wrap') if add_wrap_markers: stdin = stdin[:-len('_wrap')] stdin = data_for_at(w, stdin, add_wrap_markers=add_wrap_markers) if stdin is not None: pipe_data = w.pipe_data( stdin, has_wrap_markers=add_wrap_markers) if w else {} if pipe_data: env = { 'KITTY_PIPE_DATA': '{scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}' .format(**pipe_data) } stdin = stdin.encode('utf-8') return env, stdin def special_window_for_cmd(self, cmd, window=None, stdin=None, cwd_from=None, as_overlay=False): w = window or self.active_window env, stdin = self.process_stdin_source(w, stdin) cmdline = [] for arg in cmd: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmdline.append(arg) overlay_for = w.id if as_overlay and w.overlay_for is None else None return SpecialWindow(cmd, stdin, cwd_from=cwd_from, overlay_for=overlay_for, env=env) def pipe(self, source, dest, exe, *args): cmd = [exe] + list(args) window = self.active_window cwd_from = window.child.pid_for_cwd if window else None def create_window(): return self.special_window_for_cmd(cmd, stdin=source, as_overlay=dest == 'overlay', cwd_from=cwd_from) if dest == 'overlay' or dest == 'window': tab = self.active_tab if tab is not None: return tab.new_special_window(create_window()) elif dest == 'tab': tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=create_window(), cwd_from=cwd_from) elif dest == 'os_window': self._new_os_window(create_window(), cwd_from=cwd_from) elif dest in ('clipboard', 'primary'): env, stdin = self.process_stdin_source(stdin=source, window=window) if stdin: func = set_clipboard_string if dest == 'clipboard' else set_primary_selection func(stdin) else: import subprocess env, stdin = self.process_stdin_source(stdin=source, window=window) cwd = None if cwd_from: with suppress(Exception): cwd = cwd_of_process(cwd_from) if stdin: r, w = safe_pipe(False) try: subprocess.Popen(cmd, env=env, stdin=r, cwd=cwd) except Exception: os.close(w) else: thread_write(w, stdin) finally: os.close(r) else: subprocess.Popen(cmd, env=env, cwd=cwd) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window if args[0].startswith('@') and args[0] != '@': stdin = data_for_at(w, args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None, as_neighbor=False): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: return tm.new_tab(special_window=special_window, cwd_from=cwd_from, as_neighbor=as_neighbor) def _create_tab(self, args, cwd_from=None): as_neighbor = False if args and args[0].startswith('!'): as_neighbor = 'neighbor' in args[0][1:].split(',') args = args[1:] self._new_tab(args, as_neighbor=as_neighbor, cwd_from=cwd_from) def new_tab(self, *args): self._create_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._create_tab(args, cwd_from=cwd_from) def new_tab_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_tab(special_window) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: location = None if args and args[0].startswith('!'): location = args[0][1:].lower() args = args[1:] if args: return tab.new_special_window(self.args_to_special_window( args, cwd_from=cwd_from), location=location) else: return tab.new_window(cwd_from=cwd_from, location=location) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window_for_cwd if w is None: return self.new_window(*args) cwd_from = w.child.pid_for_cwd if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def disable_ligatures_in(self, where, strategy): if isinstance(where, str): windows = () if where == 'active': if self.active_window is not None: windows = (self.active_window, ) elif where == 'all': windows = self.all_windows elif where == 'tab': if self.active_tab is not None: windows = tuple(self.active_tab) else: windows = where for window in windows: window.screen.disable_ligatures = strategy window.refresh() def patch_colors(self, spec, cursor_text_color, configured=False): if configured: for k, v in spec.items(): if hasattr(self.opts, k): setattr(self.opts, k, color_from_int(v)) if cursor_text_color is not False: if isinstance(cursor_text_color, int): cursor_text_color = color_from_int(cursor_text_color) self.opts.cursor_text_color = cursor_text_color for tm in self.all_tab_managers: tm.tab_bar.patch_colors(spec) patch_global_colors(spec, configured) def safe_delete_temp_file(self, path): if is_path_in_temp_dir(path): with suppress(FileNotFoundError): os.remove(path) def set_update_check_process(self, process=None): if self.update_check_process is not None: with suppress(Exception): if self.update_check_process.poll() is None: self.update_check_process.kill() self.update_check_process = process def on_monitored_pid_death(self, pid, exit_status): update_check_process = getattr(self, 'update_check_process', None) if update_check_process is not None and pid == update_check_process.pid: self.update_check_process = None from .update_check import process_current_release try: raw = update_check_process.stdout.read().decode('utf-8') except Exception as e: log_error( 'Failed to read data from update check process, with error: {}' .format(e)) else: try: process_current_release(raw) except Exception as e: log_error( 'Failed to process update check data {!r}, with error: {}' .format(raw, e)) def notification_activated(self, identifier): if identifier == 'new-version': from .update_check import notification_activated notification_activated() def dbus_notification_callback(self, activated, *args): from .notify import dbus_notification_created, dbus_notification_activated if activated: dbus_notification_activated(*args) else: dbus_notification_created(*args) def show_bad_config_lines(self, bad_lines): def format_bad_line(bad_line): return '{}:{} in line: {}\n'.format(bad_line.number, bad_line.exception, bad_line.line) msg = '\n'.join(map(format_bad_line, bad_lines)).rstrip() self.show_error(_('Errors in kitty.conf'), msg)
class PropAnimation(QtCore.QPropertyAnimation): base_time = 350 disable_animation = True def __init__(self, widget, name, time_factor=1.0, force=False): super(PropAnimation, self).__init__(widget, name) self.stateChanged.connect(self._onStateChanged) self._force = force self.dic = WeakValueDictionary() self.dic['widget'] = widget self.__new = None self.time = time_factor * self.base_time self.setEasingCurve(QtCore.QEasingCurve.InOutQuad) @property def widget(self): return self.dic['widget'] def _onStateChanged(self, state): if state == self.Running: running_animations.append(self) if self.dic.get('__reparent'): self.dic['__old_parent'] = self.widget.parentItem() self.widget.setParentItem(self.dic['__reparent']) elif state == self.Stopped: try: attr = getattr(self.dic['widget'], 'set' + str(self.propertyName()).capitalize()) except RuntimeError: logger.ddebug('runtime error for {}'.format(self)) #FUCKIT return attr(self.__new) self.__new = None if self.dic.get('__reparent'): self.widget.setParentItem(self.dic['__old_parent']) self.dic.pop('__reparent') self.dic.pop('__old_parent') running_animations.remove(self) def setup(self, new, old=None, reparent=None, path_function=None): if self.state() == self.Running and not self._force: return False self.__new = new if old is None: old = getattr(self.dic['widget'], str(self.propertyName())) if hasattr(old, '__call__'): old = old() if reparent: self.dic['__reparent'] = reparent old = self.widget.mapToItem(reparent, old) new = self.widget.mapToItem(reparent, new) if path_function: points = path_function(old, new) self.setKeyValues([]) for i, p in enumerate(points): self.setKeyValueAt(float(i) / len(points), p) self.setDuration(self.time) self.setStartValue(old) self.setEndValue(new) return True def start(self): if self.state() == self.Running and not self._force: return False super(PropAnimation, self).start() return True