class WeakSet(object): def __init__(self): self._data = WeakValueDictionary() def add(self, obj): self._data[id(obj)] = obj def remove(self, obj): try: del self._data[id(obj)] except: raise KeyError(obj) def discard(self, obj): try: self.remove(obj) except: return def __iter__(self): for obj in self._data.values(): yield obj def __len__(self): return len(self._data) def __contains__(self, obj): return id(obj) in self._data def __hash__(self): raise TypeError, "Can't hash a WeakSet."
class Transport(BaseTransport): """Server side of the LocalQueue transport""" def __init__(self,callbacks,cfg): #logger.debug("Server: setting up") self.trace = cfg.get('trace,',0) self.callbacks = callbacks self.p = LocalQueue(cfg) self.p.server = ref(self) # for clients to find me self.clients = WeakValueDictionary() # clients add themselves here @spawned def _process(self,msg): m = self.callbacks.recv(msg.msg) msg.reply(m) def run(self): logger.debug("Server: wait for messages") while self.p.request_q is not None: msg = self.p.request_q.get() #logger.debug("Server: received %r",msg) self._process(msg) def send(self,msg): m = msg msg = RPCmessage(msg, _trace=self.trace) self.last_msgid -= 1 msg.msgid = self.last_msgid if self.trace: logger.debug("Server: send msg %s:\n%s",msg.msgid,format_msg(m)) for c in self.clients.values(): c.reply_q.put(msg)
def _fetch_all(self): set_peers = self._result_cache is None super()._fetch_all() # ModelIterable tests for query sets returning model instances vs values or value lists etc if set_peers and issubclass(self._iterable_class, models.query.ModelIterable): peers = WeakValueDictionary((id(o), o) for o in self._result_cache) for peer in peers.values(): peer._peers = peers
class Master(object): def __init__(self, id, max_slots): self.id = id self.max_slots = max_slots self.lock = Semaphore() self.slots = WeakValueDictionary() def serialize(self): return { 'id': self.id, 'max_slots': self.max_slots, 'slots': len(self.slots), 'workers': sum(len(slot.workers) for slot in self.slots.values()) }
class Supervisor(object): def __init__(self): self.processes = WeakValueDictionary() def register(self, process): self.processes[process.id] = process def process(self, process_id): if process_id in self.processes: return self.processes[process_id] return None def read_all(self): for process in self.processes.values(): process.read()
class WSBroadcaster: def __init__(self): self.wsid_iter = count() self.websockets = WeakValueDictionary() def add_ws(self, ws, username): """Add a websocket to the pool, return its id""" wsid = next(self.wsid_iter) self.websockets[(username, wsid)] = ws self.broadcast_connected_users() return wsid def remove_ws(self, username, wsid): self.websockets.pop((username, wsid)) self.broadcast_connected_users() def list_connected_users(self): return sorted(set(username for username, wsid in self.websockets)) def broadcast_connected_users(self): self.broadcast({ 'type': 'connected_users', 'users': self.list_connected_users(), }) def broadcast(self, message): if message['type'] == 'preview': fn = self.send_bytes data = BSON.encode(message) else: fn = self.send_str data = json.dumps(message) for ws in self.websockets.values(): create_task(fn(ws, data)) async def send_str(self, ws, data): await ws.send_str(data) async def send_bytes(self, ws, data): await ws.send_bytes(data)
class EventManager(object): def __init__(self): self.listeners = Wkd() def add_listener(self, listener, name): self.listeners[name] = listener def remove_listener(self, listener): if listener in self.listeners.keys(): del self.listeners[listener] def post(self, event): en = event.name # if event.name != EV_TICK and event.name != EV_PLAYER_MOVE: # pass #print(type(event)) try: # All listeners if en == EV_TICK or \ en == EV_QUIT or \ en == EV_INIT or \ en == EV_RESIZE: for val in self.listeners.values(): val.notify(event) # Game Engine only elif en == EV_INPUT or en == EV_MOUSE_MOVE or en == EV_MOUSE_CLICK: self.listeners["Game Engine"].notify(event) # Renderer only elif en == EV_PLAYER_MOVE or \ en == EV_PLAYER_STATS or \ en == EV_MODEL_SHARE or \ en == EV_SWORD_SWING: self.listeners["Renderer"].notify(event) except KeyError as ke: print("Error:", ke.message)
class Signal(object): def __init__(self, *args): self.__slots = WeakValueDictionary() for slot in args: self.connect(slot) def __call__(self, slot, *args, **kwargs): """ Emit signal. If slot passed signal will be called only for this slot, for all connected slots otherwise. Calling this method directly lead to immediate signal processing. It may be not thread-safe. Use emit method from this module for delayed calling of signals. """ if slot is not None: slots = (self.__slots[self.key(slot)],) else: slots = self.__slots.values() for func in slots: func(*args, **kwargs) def key(self, slot): """ Get local key name for slot. """ if type(slot) == types.FunctionType: key = (slot.__module__, slot.__name__) elif type(slot) == types.MethodType: key = (slot.__func__, id(slot.__self__)) elif isinstance(slot, basestring): if not slot in registred_slots.keys(): raise ValueError('Slot {0} does not exists.'.format(slot)) key = slot else: raise ValueError('Slot {0} has non-slot type'.format(slot)) return key def connect(self, slot): """ Connect signal to slot. Slot may be function, instance method or name of function perviously registred by `slot` decorator. """ key = self.key(slot) if type(slot) == types.FunctionType: self.__slots[key] = slot elif type(slot) == types.MethodType: self.__slots[key] = partial(slot.__func__, slot.__self__) elif isinstance(slot, basestring): self.__slots[key] = registred_slots[slot] def disconnect(self, slot): """ Remove slot from signal connetions. """ key = self.key(slot) del self.__slots[key] def clear(self): """ Disconnect all slots from signal. """ self.__slots.clear()
class TaskManager(object): """ Provides a set of tools to maintain a list of asyncio Tasks that are to be executed during the lifetime of an arbitrary object, usually getting killed with it. """ def __init__(self): self._pending_tasks = WeakValueDictionary() self._task_lock = RLock() self._shutdown = False self._counter = 0 self._logger = logging.getLogger(self.__class__.__name__) self._checker = self.register_task('_check_tasks', self._check_tasks, interval=MAX_TASK_AGE, delay=MAX_TASK_AGE * 1.5) def _check_tasks(self): now = time.time() for name, task in self._pending_tasks.items(): if not task.interval and now - task.start_time > MAX_TASK_AGE: self._logger.warning( 'Non-interval task "%s" has been running for %.2f!', name, now - task.start_time) def replace_task(self, name, *args, **kwargs): """ Replace named task with the new one, cancelling the old one in the process. """ new_task = Future() def cancel_cb(_): try: new_task.set_result(self.register_task(name, *args, **kwargs)) except Exception as e: new_task.set_exception(e) old_task = self.cancel_pending_task(name) old_task.add_done_callback(cancel_cb) return new_task def register_task(self, name, task, *args, delay=None, interval=None, ignore=()): """ Register a Task/(coroutine)function so it can be canceled at shutdown time or by name. """ if not isinstance( task, Task) and not iscoroutinefunction(task) and not callable(task): raise ValueError( 'Register_task takes a Task or a (coroutine)function as a parameter' ) if (interval or delay) and isinstance(task, Task): raise ValueError('Cannot run Task at an interval or with a delay') if not isinstance(ignore, tuple) or not all( (issubclass(e, Exception) for e in ignore)): raise ValueError('Ignore should be a tuple of Exceptions or None') with self._task_lock: if self._shutdown: self._logger.warning("Not adding task %s due to shutdown!", str(task)) if isinstance(task, (Task, Future)): if not task.done(): task.cancel() return task if self.is_pending_task_active(name): raise RuntimeError("Task already exists: '%s'" % name) if iscoroutinefunction(task) or callable(task): task = task if iscoroutinefunction(task) else coroutine(task) if interval: # The default delay for looping calls is the same as the interval delay = interval if delay is None else delay task = ensure_future( interval_runner(delay, interval, task, *args)) elif delay: task = ensure_future(delay_runner(delay, task, *args)) else: task = ensure_future(task(*args)) # Since weak references to list/tuple are not allowed, we're not storing start_time/interval # in _pending_tasks. Instead we add them as attributes to the task. task.start_time = time.time() task.interval = interval assert isinstance(task, Task) def done_cb(future): self._pending_tasks.pop(name, None) try: future.result() except CancelledError: pass except ignore as e: self._logger.error('Task resulted in error: %s', e) self._pending_tasks[name] = task task.add_done_callback(done_cb) return task def register_anonymous_task(self, basename, task, *args, **kwargs): """ Wrapper for register_task to derive a unique name from the basename. """ self._counter += 1 return self.register_task(basename + ' ' + str(self._counter), task, *args, **kwargs) def cancel_pending_task(self, name): """ Cancels the named task """ with self._task_lock: task = self._pending_tasks.get(name, None) if not task: return succeed(None) if not task.done(): task.cancel() self._pending_tasks.pop(name, None) return task def cancel_all_pending_tasks(self): """ Cancels all the registered tasks. This usually should be called when stopping or destroying the object so no tasks are left floating around. """ with self._task_lock: assert all([ isinstance(t, (Task, Future)) for t in self._pending_tasks.values() ]), self._pending_tasks return [ self.cancel_pending_task(name) for name in list(self._pending_tasks.keys()) ] def is_pending_task_active(self, name): """ Return a boolean determining if a task is active. """ with self._task_lock: task = self._pending_tasks.get(name, None) return not task.done() if task else False def get_tasks(self): """ Returns a list of all registered tasks, excluding tasks the are created by the TaskManager itself. """ with self._task_lock: return [ t for t in self._pending_tasks.values() if t != self._checker ] async def wait_for_tasks(self): """ Waits until all registered tasks are done. """ with self._task_lock: tasks = self.get_tasks() if tasks: await gather(*tasks, return_exceptions=True) async def shutdown_task_manager(self): """ Clear the task manager, cancel all pending tasks and disallow new tasks being added. """ with self._task_lock: self._shutdown = True tasks = self.cancel_all_pending_tasks() if tasks: with suppress(CancelledError): await gather(*tasks)
class Boss: def __init__(self, os_window_id, opts, args, cached_values): self.window_id_map = WeakValueDictionary() self.startup_colors = { k: opts[k] for k in opts if isinstance(opts[k], Color) } self.pending_sequences = None self.cached_values = cached_values self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.opts, self.args = opts, args startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, opts_for_size=None, startup_id=None): if os_window_id is None: opts_for_size = opts_for_size or self.opts cls = wclass or self.args.cls or appname with startup_notification_handler( do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback: os_window_id = create_os_window( initial_window_size_func(opts_for_size, self.cached_values), pre_show_callback, appname, wname or self.args.name or cls, cls) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm return os_window_id def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } @property def all_tab_managers(self): yield from self.os_window_map.values() @property def all_tabs(self): for tm in self.all_tab_managers: yield from tm @property def all_windows(self): for tab in self.all_tabs: yield from tab def match_windows(self, match): try: field, exp = match.split(':', 1) except ValueError: return if field == 'num': tab = self.active_tab if tab is not None: try: w = tab.get_nth_window(int(exp)) except Exception: return if w is not None: yield w else: pat = re.compile(exp) for window in self.all_windows: if window.matches(field, pat): yield window def tab_for_window(self, window): for tab in self.all_tabs: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): try: field, exp = match.split(':', 1) except ValueError: return pat = re.compile(exp) found = False if field in ('title', 'id'): for tab in self.all_tabs: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) return self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control or getattr( window, 'allow_remote_control', False): try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err)} if not getattr(err, 'hide_traceback', False): response['tb'] = traceback.format_exc() else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) if not os.path.isabs(args.directory): args.directory = os.path.join(msg['cwd'], args.directory) session = create_session(opts, args, respect_cwd=True) self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): if dpi_changed: self.on_dpi_change(os_window_id) else: tm = self.os_window_map.get(os_window_id) if tm is not None: tm.resize() def increase_font_size(self): # legacy self.set_font_size( min(self.opts.font_size * 5, self.current_font_size + 2.0)) def decrease_font_size(self): # legacy self.set_font_size(self.current_font_size - self.opts.font_size_delta) def restore_font_size(self): # legacy self.set_font_size(self.opts.font_size) def set_font_size(self, new_size): # legacy self.change_font_size(True, None, new_size) def change_font_size(self, all_windows, increment_operation, amt): def calc_new_size(old_size): new_size = old_size if amt == 0: new_size = self.opts.font_size else: if increment_operation: new_size += (1 if increment_operation == '+' else -1) * amt else: new_size = amt new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5)) return new_size if all_windows: current_global_size = global_font_size() new_size = calc_new_size(current_global_size) if new_size != current_global_size: global_font_size(new_size) os_windows = tuple(self.os_window_map.keys()) else: os_windows = [] w = self.active_window if w is not None: os_windows.append(w.os_window_id) if os_windows: final_windows = {} for wid in os_windows: current_size = os_window_font_size(wid) if current_size: new_size = calc_new_size(current_size) if new_size != current_size: final_windows[wid] = new_size if final_windows: self._change_font_size(final_windows) def _change_font_size(self, sz_map): for os_window_id, sz in sz_map.items(): tm = self.os_window_map.get(os_window_id) if tm is not None: os_window_font_size(os_window_id, sz) tm.resize() def on_dpi_change(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: sz = os_window_font_size(os_window_id) if sz: os_window_font_size(os_window_id, sz, True) tm.resize() def _set_os_window_background_opacity(self, os_window_id, opacity): change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0))) def set_background_opacity(self, opacity): window = self.active_window if window is None or not opacity: return if not self.opts.dynamic_background_opacity: return self.show_error( _('Cannot change background opacity'), _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity' )) os_window_id = window.os_window_id if opacity[0] in '+-': opacity = background_opacity_of(os_window_id) if opacity is None: return opacity += float(opacity) elif opacity == 'default': opacity = self.opts.background_opacity else: opacity = float(opacity) self._set_os_window_background_opacity(os_window_id, opacity) @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) if key_action is None: sequences = get_shortcut(self.opts.sequence_map, mods, key, scancode) if sequences: self.pending_sequences = sequences set_in_sequence_mode(True) return True else: self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def process_sequence(self, key, scancode, action, mods): if not self.pending_sequences: set_in_sequence_mode(False) remaining = {} matched_action = None for seq, key_action in self.pending_sequences.items(): if shortcut_matches(seq[0], mods, key, scancode): seq = seq[1:] if seq: remaining[seq] = key_action else: matched_action = key_action if remaining: self.pending_sequences = remaining else: self.pending_sequences = None set_in_sequence_mode(False) if matched_action is not None: self.dispatch_action(matched_action) def start_resizing_window(self): w = self.active_window if w is None: return overlay_window = self._run_kitten( 'resize_window', args=[ '--horizontal-increment={}'.format( self.opts.window_resize_step_cells), '--vertical-increment={}'.format( self.opts.window_resize_step_lines) ]) if overlay_window is not None: overlay_window.allow_remote_control = True def resize_layout_window(self, window, increment, is_horizontal, reset=False): tab = window.tabref() if tab is None or not increment: return False if reset: return tab.reset_window_sizes() return tab.resize_window_by(window.id, increment, is_horizontal) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) tm.mark_tab_bar_dirty() def update_tab_bar_data(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.update_tab_bar_data() def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, window, data): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ 'kitty', '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + editor + [confpath] self.new_os_window(*cmd) def get_output(self, source_window, num_lines=1): output = '' s = source_window.screen if num_lines is None: num_lines = s.lines for i in range(min(num_lines, s.lines)): output += str(s.linebuf.line(i)) return output def _run_kitten(self, kitten, args=(), input_data=None): w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: orig_args, args = list(args), list(args) from kittens.runner import create_kitten_handler end_kitten = create_kitten_handler(kitten, orig_args) args[0:0] = [config_dir, kitten] if input_data is None: type_of_input = end_kitten.type_of_input if type_of_input in ('text', 'history', 'ansi', 'ansi-history', 'screen', 'screen-history', 'screen-ansi', 'screen-ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input, add_wrap_markers='screen' in type_of_input).encode('utf-8') elif type_of_input is None: data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) else: data = input_data if isinstance(data, str): data = data.encode('utf-8') copts = { k: self.opts[k] for k in ('select_by_word_characters', 'open_url_with') } overlay_window = tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.runner import main; main()' ] + args, stdin=data, env={ 'KITTY_COMMON_OPTS': json.dumps(copts), 'PYTHONWARNINGS': 'ignore', 'OVERLAID_WINDOW_LINES': str(w.screen.lines), 'OVERLAID_WINDOW_COLS': str(w.screen.columns), }, overlay_for=w.id)) overlay_window.action_on_close = partial(self.on_kitten_finish, w.id, end_kitten) return overlay_window def kitten(self, kitten, *args): import shlex cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] self._run_kitten(kitten, args) def on_kitten_finish(self, target_window_id, end_kitten, source_window): output = self.get_output(source_window, num_lines=None) from kittens.runner import deserialize data = deserialize(output) if data is not None: end_kitten(data, target_window_id, self) def input_unicode_character(self): self._run_kitten('unicode_input') def set_tab_title(self): tab = self.active_tab if tab: args = [ '--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id) ] self._run_kitten('ask', args) def show_error(self, title, msg): self._run_kitten('show_error', ['--title', title], input_data=msg) def do_set_tab_title(self, title, tab_id): tm = self.active_tab_manager if tm is not None and title: tab_id = int(tab_id) for tab in tm.tabs: if tab.id == tab_id: tab.set_title(title) break def kitty_shell(self, window_type): cmd = ['kitty', '@'] if window_type == 'tab': window = self._new_tab(cmd).active_window elif window_type == 'os_window': os_window_id = self._new_os_window(cmd) window = self.os_window_map[os_window_id].active_window elif window_type == 'overlay': w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: window = tab.new_special_window( SpecialWindow(cmd, overlay_for=w.id)) else: window = None else: window = self._new_window(cmd) if window is not None: window.allow_remote_control = True def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) def open_url(self, url, program=None, cwd=None): if url: if isinstance(program, str): program = to_cmdline(program) open_url(url, program or self.opts.open_url_with, cwd=cwd) def open_url_lines(self, lines, program=None): self.open_url(''.join(lines), program) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection( ) if supports_primary_selection else get_clipboard_string() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def set_active_tab(self, tab): tm = self.active_tab_manager if tm is not None: tm.set_active_tab(tab) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.as_text(as_ansi=True, add_history=True) if arg == '@text': return w.as_text(add_history=True) if arg == '@screen': return w.as_text() if arg == '@ansi_screen': return w.as_text(as_ansi=True) if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: return tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: return tab.new_special_window( self.args_to_special_window(args, cwd_from=cwd_from)) else: return tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window if w is None: return self.new_window(*args) cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def patch_colors(self, spec, configured=False): if configured: for k, v in spec.items(): if hasattr(self.opts, k): setattr(self.opts, k, color_from_int(v)) for tm in self.all_tab_managers: tm.tab_bar.patch_colors(spec)
class Boss: def __init__(self, os_window_id, opts, args, cached_values): self.window_id_map = WeakValueDictionary() self.cached_values = cached_values self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.current_font_size = opts.font_size set_font_family(opts) self.opts, self.args = opts, args initialize_renderer() startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None): dpi_changed = False if os_window_id is None: w, h = initial_window_size( self.opts, self.cached_values) if size is None else size cls = wclass or self.args.cls or appname os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls) if startup_id: ctx = init_startup_notification(os_window_id, startup_id) dpi_changed = show_window(os_window_id) if startup_id: end_startup_notification(ctx) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm if dpi_changed: self.on_dpi_change(os_window_id) def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } def match_windows(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) for tm in self.os_window_map.values(): for tab in tm: for window in tab: if window.matches(field, pat): yield window def tab_for_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) tms = tuple(self.os_window_map.values()) found = False if field in ('title', 'id'): for tm in tms: for tab in tm: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control: try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = { 'ok': False, 'error': str(err), 'tb': traceback.format_exc() } else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): import json msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) session = create_session(opts, args) self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size( opts, self.cached_values), startup_id=startup_id) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): tm = self.os_window_map.get(os_window_id) if tm is not None: if dpi_changed: if set_dpi_from_os_window(os_window_id): self.on_dpi_change(os_window_id) else: tm.resize() else: tm.resize() def increase_font_size(self): self.change_font_size( min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def _change_font_size(self, new_size=None, on_dpi_change=False): if new_size is not None: self.current_font_size = new_size old_cell_width, old_cell_height = viewport_for_window()[-2:] windows = tuple(filter(None, self.window_id_map.values())) resize_fonts(self.current_font_size, on_dpi_change=on_dpi_change) layout_sprite_map() prerender() for window in windows: window.screen.rescale_images(old_cell_width, old_cell_height) window.screen.refresh_sprite_positions() for tm in self.os_window_map.values(): tm.resize() tm.refresh_sprite_positions() glfw_post_empty_event() def change_font_size(self, new_size): if new_size == self.current_font_size: return self._change_font_size(new_size) def on_dpi_change(self, os_window_id): self._change_font_size() @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, window, data): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ 'kitty', '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + editor + [confpath] self.new_os_window(*cmd) def input_unicode_character(self): w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: overlay_window = tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.unicode_input.main import main; main()' ], overlay_for=w.id)) overlay_window.action_on_close = partial( self.send_unicode_character, w.id) def send_unicode_character(self, target_window_id, source_window): w = self.window_id_map.get(target_window_id) if w is not None: output = str(source_window.screen.linebuf.line(0)) if output.startswith('OK: '): try: text = chr(int(output.partition(' ')[2], 16)) except Exception: import traceback traceback.print_exc() else: w.paste(text) def run_simple_kitten(self, type_of_input, kitten, *args): import shlex w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] if '--program' not in cmdline: args.extend(('--program', self.opts.open_url_with)) if type_of_input in ('text', 'history', 'ansi', 'ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input).encode('utf-8') elif type_of_input == 'none': data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) tab.new_special_window( SpecialWindow([ 'kitty', '+runpy', 'from kittens.{}.main import main; main()'.format(kitten) ] + args, stdin=data, overlay_for=w.id)) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def open_url_lines(self, lines): self.open_url(''.join(lines)) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_sprite_map() destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.as_text(as_ansi=True, add_history=True) if arg == '@text': return w.as_text(add_history=True) if arg == '@screen': return w.as_text() if arg == '@ansi_screen': return w.as_text(as_ansi=True) if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: tab.new_special_window( self.args_to_special_window(args, cwd_from=cwd_from)) else: tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window if w is None: return self.new_window(*args) cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1)
class Server(object): """ A server. """ def __init__(self, db_object): """ Initializes this server object. """ self.user_map = WeakValueDictionary() self.channel_map = {} self.lock = Lock() self.update_from_database_object(db_object) self.needs_reconnect = Flag() def update_from_database_object(self, db_object): """ Instructs this server to update itself from the specified database object, which should be the result of a query against the database. This updates the server's name, the server's protocol class, the server's activated state, and the server's config properties. """ self.name = db_object["name"] self.active = db_object["active"] self.protocol_name = db_object["protocol"] self.config_properties = dict(db_object["config"]) def get_user(self, transient_name): """ Gets a user object representing a user with the specified transient name. """ def init_user_info(self, transient_name, persistent_name=None, display_name=None, group_name=None): """ Called by the protocol constructor to set the initial information for the user. Once the constructor returns, at least the transient name must have been set by calling this method. """ pass def disconnect(self, message=None): """ Disconnects this server. This basically just forwards over to the corresponding protocol. It does not clear any of the server maps; use clear_session_data for that. """ self.protocol.disconnect() def is_connected(self): raise Exception("Not implemented yet") def clear_session_data(self): """ Clears the information for the last session from this server object. This involves clearing the channel map and the user map. This synchronizes on the server lock. This also untracks any users still in the session map. """ with self.lock: for user in self.user_map.values(): user.untrack() # TODO: figure out if the channel map should have a weak map # similar to the user map or not self.channel_map.clear() self.local_user = None def on_action(self, transient_name, message, persistent_name=None, display_name=None, group_name=None): """ Called by a protocol when an action is received. """ pass def on_message(self, transient_name, message, persistent_name=None, display_name=None, group_name=None): """ Called by a protocol when a message is received. """ pass def on_connect(self, transient_name=None, persistent_name=None, display_name=None, group_name=None): """ Called when the server successfully connects. The user information provided to this method is the information for the user that the protocol has connected as. None of it is required. """ pass def on_disconnect(self): """ Called when the protocol disconnects from the server for some reason. """ pass
class UniversalCursors(object): def __init__(self): self.name_cursors = {} self.cursors_orient = WeakKeyDictionary() self.all_canvas = WeakValueDictionary() self.all_axes = WeakValueDictionary() self.backgrounds = {} self.visible = True self.needclear = False def _onmove(self, event): for canvas in self.all_canvas.values(): if not canvas.widgetlock.available(self): return if event.inaxes is None or not self.visible: if self.needclear: self._update(event) for canvas in self.all_canvas.values(): canvas.draw() self.needclear = False return self._update(event) def _update(self, event): # 1/ Reset background for canvas in self.all_canvas.values(): canvas.restore_region(self.backgrounds[id(canvas)]) # 2/ update cursors for cursors in self.cursors_orient.keys(): orient = self.cursors_orient[cursors] if (event.inaxes in [line.get_axes() for line in cursors] and self.visible): visible = True self.needclear = True else: visible = False for line in cursors: if orient == 'vertical': line.set_xdata((event.xdata, event.xdata)) if orient == 'horizontal': line.set_ydata((event.ydata, event.ydata)) line.set_visible(visible) ax = line.get_axes() ax.draw_artist(line) # 3/ update canvas for canvas in self.all_canvas.values(): canvas.blit(canvas.figure.bbox) def _clear(self, event): """clear the cursor""" self.backgrounds = {} for canvas in self.all_canvas.values(): self.backgrounds[id(canvas)] = ( canvas.copy_from_bbox(canvas.figure.bbox)) for cursor in self.cursors_orient.keys(): for line in cursor: line.set_visible(False) def add(self, name, axes=(), orient='vertical', **lineprops): if name in self.name_cursors.keys(): raise NameError class CursorList(list): def __hash__(self): return hash(tuple(self)) self.name_cursors[name] = CursorList() # Required to keep weakref for ax in axes: self.all_axes[id(ax)] = ax ax_canvas = ax.get_figure().canvas if ax_canvas not in self.all_canvas.values(): #if not ax_canvas.supports_blit: # warnings.warn("Must use canvas that support blit") # return self.all_canvas[id(ax_canvas)] = ax_canvas ax_canvas.mpl_connect('motion_notify_event', self._onmove) ax_canvas.mpl_connect('draw_event', self._clear) if orient == 'vertical': line = ax.axvline(ax.get_xbound()[0], visible=False, animated=True, **lineprops) if orient == 'horizontal': line = ax.axhline(ax.get_ybound()[0], visible=False, animated=True, **lineprops) self.name_cursors[name].append(line) self.cursors_orient[self.name_cursors[name]] = orient def remove(self, name): del self.name_cursors[name]
class DispatchTree(object): def __init__(self): # core data self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy()) self.nodes = WeakValueDictionary() self.nodes[0] = self.root self.pools = {} self.renderNodes = {} self.tasks = {} self.rules = [] self.poolShares = {} self.commands = {} # deduced properties self.nodeMaxId = 0 self.poolMaxId = 0 self.renderNodeMaxId = 0 self.taskMaxId = 0 self.commandMaxId = 0 self.poolShareMaxId = 0 self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] # listeners self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange) self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange) # # JSA # self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange) self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange) self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange) self.commandListener = ObjectListener( onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange) self.poolShareListener = ObjectListener(self.onPoolShareCreation) self.modifiedNodes = [] def registerModelListeners(self): BaseNode.changeListeners.append(self.nodeListener) Task.changeListeners.append(self.taskListener) TaskGroup.changeListeners.append(self.taskListener) RenderNode.changeListeners.append(self.renderNodeListener) Pool.changeListeners.append(self.poolListener) Command.changeListeners.append(self.commandListener) PoolShare.changeListeners.append(self.poolShareListener) def destroy(self): BaseNode.changeListeners.remove(self.nodeListener) Task.changeListeners.remove(self.taskListener) RenderNode.changeListeners.remove(self.renderNodeListener) Pool.changeListeners.remove(self.poolListener) Command.changeListeners.remove(self.commandListener) PoolShare.changeListeners.remove(self.poolShareListener) self.root = None self.nodes.clear() self.pools.clear() self.renderNodes.clear() self.tasks.clear() self.rules = None self.commands.clear() self.poolShares = None self.modifiedNodes = None self.toCreateElements = None self.toModifyElements = None self.toArchiveElements = None def findNodeByPath(self, path, default=None): nodenames = splitpath(path) node = self.root for name in nodenames: for child in node.children: if child.name == name: node = child break else: return default return node def updateCompletionAndStatus(self): self.root.updateCompletionAndStatus() def validateDependencies(self): nodes = set() for dependency in self.modifiedNodes: for node in dependency.reverseDependencies: nodes.add(node) del self.modifiedNodes[:] for node in nodes: # logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) ) if not hasattr(node, "task") or node.task is None: continue if isinstance(node, TaskNode): if node.checkDependenciesSatisfaction(): for cmd in node.task.commands: if cmd.status == CMD_BLOCKED: cmd.status = CMD_READY else: for cmd in node.task.commands: if cmd.status == CMD_READY: cmd.status = CMD_BLOCKED # TODO: may be needed to check dependencies on task groups # so far, a hack is done on the client side when submitting: # dependencies of a taksgroup are reported on each task of its heirarchy # # elif isinstance(node, FolderNode): # # if node.checkDependenciesSatisfaction(): # for cmd in node.getAllCommands(): # if cmd.status == CMD_BLOCKED: # cmd.status = CMD_READY # else: # for cmd in node.getAllCommands(): # if cmd.status == CMD_READY: # cmd.status = CMD_BLOCKED def registerNewGraph(self, graph): user = graph['user'] taskDefs = graph['tasks'] poolName = graph['poolName'] if 'maxRN' in graph.items(): maxRN = int(graph['maxRN']) else: maxRN = -1 # # Create objects. # tasks = [None for i in xrange(len(taskDefs))] for (index, taskDef) in enumerate(taskDefs): if taskDef['type'] == 'Task': # logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"]) # logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"]) task = self._createTaskFromJSON(taskDef, user) elif taskDef['type'] == 'TaskGroup': task = self._createTaskGroupFromJSON(taskDef, user) tasks[index] = task root = tasks[graph['root']] # get the pool try: pool = self.pools[poolName] except KeyError: pool = Pool(None, poolName) self.pools[poolName] = pool # # Rebuild full job hierarchy # for (taskDef, task) in zip(taskDefs, tasks): if taskDef['type'] == 'TaskGroup': for taskIndex in taskDef['tasks']: task.addTask(tasks[taskIndex]) tasks[taskIndex].parent = task # # Compute dependencies for each created task or taskgroup object. # dependencies = {} for (taskDef, task) in zip(taskDefs, tasks): taskDependencies = {} if not isinstance(taskDef['dependencies'], list): raise SyntaxError( "Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) if not all(((isinstance(i, int) and isinstance(sl, list) and all( (isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])): raise SyntaxError( "Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) for (taskIndex, statusList) in taskDef['dependencies']: taskDependencies[tasks[taskIndex]] = statusList dependencies[task] = taskDependencies # # Apply rules to generate dispatch tree nodes. # if not self.rules: logger.warning("graph submitted but no rule has been defined") unprocessedTasks = [root] nodes = [] while unprocessedTasks: unprocessedTask = unprocessedTasks.pop(0) for rule in self.rules: try: nodes += rule.apply(unprocessedTask) except RuleError: logger.warning("rule %s failed for graph %s" % (rule, graph)) raise if isinstance(unprocessedTask, TaskGroup): for task in unprocessedTask: unprocessedTasks.append(task) # create the poolshare, if any, and affect it to the node if pool: # FIXME nodes[0] may not be the root node of the graph... ps = PoolShare(None, pool, nodes[0], maxRN) # if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true if maxRN != -1: ps.userDefinedMaxRN = True # # Process dependencies # for rule in self.rules: rule.processDependencies(dependencies) for node in nodes: assert isinstance(node.id, int) self.nodes[node.id] = node # Init number of command in hierarchy self.populateCommandCounts(nodes[0]) return nodes def populateCommandCounts(self, node): """ Updates "commandCount" over a whole hierarchy starting from the given node. """ res = 0 if isinstance(node, FolderNode): for child in node.children: res += self.populateCommandCounts(child) elif isinstance(node, TaskNode): res = len(node.task.commands) node.commandCount = res return res def _createTaskGroupFromJSON(self, taskGroupDefinition, user): # name, parent, arguments, environment, priority, dispatchKey, strategy id = None name = taskGroupDefinition['name'] parent = None arguments = taskGroupDefinition['arguments'] environment = taskGroupDefinition['environment'] requirements = taskGroupDefinition['requirements'] maxRN = taskGroupDefinition['maxRN'] priority = taskGroupDefinition['priority'] dispatchKey = taskGroupDefinition['dispatchKey'] strategy = taskGroupDefinition['strategy'] strategy = loadStrategyClass(strategy.encode()) strategy = strategy() tags = taskGroupDefinition['tags'] timer = None if 'timer' in taskGroupDefinition.keys(): timer = taskGroupDefinition['timer'] return TaskGroup(id, name, parent, user, arguments, environment, requirements, maxRN, priority, dispatchKey, strategy, tags=tags, timer=timer) def _createTaskFromJSON(self, taskDefinition, user): # id, name, parent, user, priority, dispatchKey, runner, arguments, # validationExpression, commands, requirements=[], minNbCores=1, # maxNbCores=0, ramUse=0, environment={} name = taskDefinition['name'] runner = taskDefinition['runner'] arguments = taskDefinition['arguments'] environment = taskDefinition['environment'] requirements = taskDefinition['requirements'] maxRN = taskDefinition['maxRN'] priority = taskDefinition['priority'] dispatchKey = taskDefinition['dispatchKey'] validationExpression = taskDefinition['validationExpression'] minNbCores = taskDefinition['minNbCores'] maxNbCores = taskDefinition['maxNbCores'] ramUse = taskDefinition['ramUse'] lic = taskDefinition['lic'] tags = taskDefinition['tags'] runnerPackages = taskDefinition.get('runnerPackages', '') watcherPackages = taskDefinition.get('watcherPackages', '') timer = None if 'timer' in taskDefinition.keys(): timer = taskDefinition['timer'] maxAttempt = taskDefinition.get('maxAttempt', 1) task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner, arguments, validationExpression, [], requirements, minNbCores, maxNbCores, ramUse, environment, lic=lic, tags=tags, timer=timer, maxAttempt=maxAttempt, runnerPackages=runnerPackages, watcherPackages=watcherPackages) for commandDef in taskDefinition['commands']: description = commandDef['description'] arguments = commandDef['arguments'] cmd = Command(None, description, task, arguments, runnerPackages=runnerPackages, watcherPackages=watcherPackages) task.commands.append(cmd) # import sys # logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd))) return task ## Resets the lists of elements to create or update in the database. # def resetDbElements(self): self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] ## Recalculates the max ids of all elements. Generally called after a reload from db. # def recomputeMaxIds(self): self.nodeMaxId = max([n.id for n in self.nodes.values() ]) if self.nodes else 0 self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) self.poolMaxId = max([p.id for p in self.pools.values() ]) if self.pools else 0 self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId()) self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values() ]) if self.renderNodes else 0 self.renderNodeMaxId = max(self.renderNodeMaxId, StatDB.getRenderNodesMaxId()) self.taskMaxId = max([t.id for t in self.tasks.values() ]) if self.tasks else 0 self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId()) self.commandMaxId = max([c.id for c in self.commands.values() ]) if self.commands else 0 self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId()) self.poolShareMaxId = max([ps.id for ps in self.poolShares.values() ]) if self.poolShares else 0 self.poolShareMaxId = max(self.poolShareMaxId, StatDB.getPoolSharesMaxId()) ## Removes from the dispatchtree the provided element and all its parents and children. # def unregisterElementsFromTree(self, element): # /////////////// Handling of the Task if isinstance(element, Task): del self.tasks[element.id] self.toArchiveElements.append(element) for cmd in element.commands: self.unregisterElementsFromTree(cmd) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskGroup elif isinstance(element, TaskGroup): del self.tasks[element.id] self.toArchiveElements.append(element) for task in element.tasks: self.unregisterElementsFromTree(task) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskNode elif isinstance(element, TaskNode): # remove the element from the children of the parent if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the FolderNode elif isinstance(element, FolderNode): if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the Command elif isinstance(element, Command): del self.commands[element.id] self.toArchiveElements.append(element) ### methods called after interaction with a Task def onTaskCreation(self, task): # logger.info(" -- on task creation: %s" % task) if task.id is None: self.taskMaxId += 1 task.id = self.taskMaxId self.toCreateElements.append(task) else: self.taskMaxId = max(self.taskMaxId, task.id, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId()) self.tasks[task.id] = task def onTaskDestruction(self, task): # logger.info(" -- on task destruction: %s" % task) self.unregisterElementsFromTree(task) def onTaskChange(self, task, field, oldvalue, newvalue): """ Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes However in order to keep track of comments (stored in task's tags[comment] field), we make the following change: - enable task/taskgroups update in DB (cf pulidb.py) - disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs BUT the only field we want to update: "tags" """ if field == "tags": self.toModifyElements.append(task) ### methods called after interaction with a BaseNode def onNodeCreation(self, node): # logger.info(" -- on node creation: %s" % node) if node.id is None: self.nodeMaxId += 1 node.id = self.nodeMaxId self.toCreateElements.append(node) else: self.nodeMaxId = max(self.nodeMaxId, node.id, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) if node.parent is None: node.parent = self.root def onNodeDestruction(self, node): # logger.info(" -- on node destruction: %s" % node) del self.nodes[node.id] def onNodeChange(self, node, field, oldvalue, newvalue): # logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) ) # FIXME: do something when nodes are reparented from or to the root node if node.id is not None: self.toModifyElements.append(node) if field == "status" and node.reverseDependencies: self.modifiedNodes.append(node) ### methods called after interaction with a RenderNode def onRenderNodeCreation(self, renderNode): if renderNode.id is None: self.renderNodeMaxId += 1 renderNode.id = self.renderNodeMaxId self.toCreateElements.append(renderNode) else: self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id, StatDB.getRenderNodesMaxId()) self.renderNodes[renderNode.name] = renderNode def onRenderNodeDestruction(self, rendernode): try: del self.renderNodes[rendernode.name] self.toArchiveElements.append(rendernode) except KeyError: # TOFIX: use of class method vs obj method in changeListener might generate a duplicate call logger.warning("RN %s seems to have been deleted already." % rendernode.name) def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue): if field == "performance": self.toModifyElements.append(rendernode) ### methods called after interaction with a Pool def onPoolCreation(self, pool): if pool.id is None: self.poolMaxId += 1 pool.id = self.poolMaxId self.toCreateElements.append(pool) else: self.poolMaxId = max(self.poolMaxId, pool.id, StatDB.getPoolsMaxId()) self.pools[pool.name] = pool def onPoolDestruction(self, pool): del self.pools[pool.name] self.toArchiveElements.append(pool) def onPoolChange(self, pool, field, oldvalue, newvalue): if pool not in self.toModifyElements: self.toModifyElements.append(pool) ### methods called after interaction with a Command def onCommandCreation(self, command): if command.id is None: self.commandMaxId += 1 command.id = self.commandMaxId self.toCreateElements.append(command) else: self.commandMaxId = max(self.commandMaxId, command.id, StatDB.getCommandsMaxId()) self.commands[command.id] = command def onCommandChange(self, command, field, oldvalue, newvalue): self.toModifyElements.append(command) if command.task is not None: for node in command.task.nodes.values(): node.invalidate() ### methods called after interaction with a Pool def onPoolShareCreation(self, poolShare): if poolShare.id is None: self.poolShareMaxId += 1 poolShare.id = self.poolShareMaxId self.toCreateElements.append(poolShare) else: self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id, StatDB.getPoolSharesMaxId()) self.poolShares[poolShare.id] = poolShare
class DispatchTree(object): def _display_(self): ''' Debug purpose method, returns a basic display of the dispatch tree as html ''' startTimer = time.time() timeout = 2.0 result="<html><head><style>table,th,td { margin: 5px; border-collapse:collapse; border:1px solid black; }</style></head><body font-family='verdana'>" result +="<h3>Pools: %r</h3><table>" % len(self.pools) for i,curr in enumerate(self.pools): result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.pools[curr]) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>Rendernodes: %r</h3><table>" % len(self.renderNodes) for i,curr in enumerate(self.renderNodes): result += "<tr><td>%r</td><td>%r</td></tr>" % (i, self.renderNodes[curr]) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>PoolShares: (attribution de parc pour une tache fille du root, on attribue pas de poolshare aux autres)</h3><table>" for i,curr in enumerate(self.poolShares): result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.poolShares[curr]) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>Main level nodes (proxy info only):</h3><table>" result +="<tr><th>id</th><th>name</th><th>readyCommandCount</th><th>commandCount</th><th>completion</th><th>poolshares</th></tr>" for i,curr in enumerate(self.nodes[1].children): result += "<tr><td>%r</td><td>%s</td><td>%d</td><td>%d</td><td>%.2f</td><td>%s</td></tr>" % (i, curr.name, curr.readyCommandCount, curr.commandCount, curr.completion, curr.poolShares.values()) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>All nodes:</h3><table>" for i,curr in enumerate(self.nodes): result += "<tr><td>%d</td><td>%s</td><td>%r</td></tr>" % (i, curr, self.nodes[curr].name) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>Tasks:</h3><table>" for i,curr in enumerate(self.tasks): result += "<tr><td>%r</td><td>%s</td></tr>" % (i, repr(self.tasks[curr]) ) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>Commands:</h3><table>" for i,curr in enumerate(self.commands): result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.commands[curr] ) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="<h3>Rules:</h3><table>" for i,curr in enumerate(self.rules): result += "<tr><td>%r</td><td>%s</td></tr>" % (i, curr ) if (time.time()-startTimer) > timeout: raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump") result+="</table>" result +="</body></html>" logger.info("DispatchTree printed in %.6f s" % (time.time()-startTimer) ) return result def __init__(self): # core data self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy()) self.nodes = WeakValueDictionary() self.nodes[0] = self.root self.pools = {} self.renderNodes = {} self.tasks = {} self.rules = [] self.poolShares = {} self.commands = {} # deduced properties self.nodeMaxId = 0 self.poolMaxId = 0 self.renderNodeMaxId = 0 self.taskMaxId = 0 self.commandMaxId = 0 self.poolShareMaxId = 0 self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] # listeners self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange) self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange) # # JSA # self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange) self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange) self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange) self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange) self.poolShareListener = ObjectListener(self.onPoolShareCreation) self.modifiedNodes = [] def registerModelListeners(self): BaseNode.changeListeners.append(self.nodeListener) Task.changeListeners.append(self.taskListener) TaskGroup.changeListeners.append(self.taskListener) RenderNode.changeListeners.append(self.renderNodeListener) Pool.changeListeners.append(self.poolListener) Command.changeListeners.append(self.commandListener) PoolShare.changeListeners.append(self.poolShareListener) def destroy(self): BaseNode.changeListeners.remove(self.nodeListener) Task.changeListeners.remove(self.taskListener) RenderNode.changeListeners.remove(self.renderNodeListener) Pool.changeListeners.remove(self.poolListener) Command.changeListeners.remove(self.commandListener) PoolShare.changeListeners.remove(self.poolShareListener) self.root = None self.nodes.clear() self.pools.clear() self.renderNodes.clear() self.tasks.clear() self.rules = None self.commands.clear() self.poolShares = None self.modifiedNodes = None self.toCreateElements = None self.toModifyElements = None self.toArchiveElements = None def findNodeByPath(self, path, default=None): nodenames = splitpath(path) node = self.root for name in nodenames: for child in node.children: if child.name == name: node = child break else: return default return node def updateCompletionAndStatus(self): self.root.updateCompletionAndStatus() def validateDependencies(self): nodes = set() for dependency in self.modifiedNodes: for node in dependency.reverseDependencies: nodes.add(node) del self.modifiedNodes[:] for node in nodes: # logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) ) if not hasattr(node,"task"): continue if isinstance(node, TaskNode): if node.checkDependenciesSatisfaction(): for cmd in node.task.commands: if cmd.status == CMD_BLOCKED: cmd.status = CMD_READY else: for cmd in node.task.commands: if cmd.status == CMD_READY: cmd.status = CMD_BLOCKED # TODO: may be needed to check dependencies on task groups # so far, a hack is done on the client side when submitting: # dependencies of a taksgroup are reported on each task of its heirarchy # # elif isinstance(node, FolderNode): # # if node.checkDependenciesSatisfaction(): # for cmd in node.getAllCommands(): # if cmd.status == CMD_BLOCKED: # cmd.status = CMD_READY # else: # for cmd in node.getAllCommands(): # if cmd.status == CMD_READY: # cmd.status = CMD_BLOCKED def registerNewGraph(self, graph): user = graph['user'] taskDefs = graph['tasks'] poolName = graph['poolName'] if 'maxRN' in graph.items(): maxRN = int(graph['maxRN']) else: maxRN = -1 # # Create objects. # tasks = [None for i in xrange(len(taskDefs))] for (index, taskDef) in enumerate(taskDefs): if taskDef['type'] == 'Task': task = self._createTaskFromJSON(taskDef, user) elif taskDef['type'] == 'TaskGroup': task = self._createTaskGroupFromJSON(taskDef, user) tasks[index] = task root = tasks[graph['root']] # get the pool try: pool = self.pools[poolName] except KeyError: pool = Pool(None, poolName) self.pools[poolName] = pool # # Rebuild full job hierarchy # for (taskDef, task) in zip(taskDefs, tasks): if taskDef['type'] == 'TaskGroup': for taskIndex in taskDef['tasks']: task.addTask(tasks[taskIndex]) tasks[taskIndex].parent = task # # Compute dependencies for each created task or taskgroup object. # dependencies = {} for (taskDef, task) in zip(taskDefs, tasks): taskDependencies = {} if not isinstance(taskDef['dependencies'], list): raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) if not all(((isinstance(i, int) and isinstance(sl, list) and all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])): raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) for (taskIndex, statusList) in taskDef['dependencies']: taskDependencies[tasks[taskIndex]] = statusList dependencies[task] = taskDependencies # # Apply rules to generate dispatch tree nodes. # if not self.rules: logger.warning("graph submitted but no rule has been defined") unprocessedTasks = [root] nodes = [] while unprocessedTasks: unprocessedTask = unprocessedTasks.pop(0) for rule in self.rules: try: nodes += rule.apply(unprocessedTask) except RuleError: logger.warning("rule %s failed for graph %s" % (rule, graph)) raise if isinstance(unprocessedTask, TaskGroup): for task in unprocessedTask: unprocessedTasks.append(task) # create the poolshare, if any, and affect it to the node if pool: # FIXME nodes[0] may not be the root node of the graph... ps = PoolShare(None, pool, nodes[0], maxRN) # if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true if maxRN != -1: ps.userDefinedMaxRN = True # # Process dependencies # for rule in self.rules: rule.processDependencies(dependencies) for node in nodes: assert isinstance(node.id, int) self.nodes[node.id] = node # Init number of command in hierarchy self.populateCommandCounts(nodes[0]) return nodes def populateCommandCounts(self, node): """ Updates "commandCount" over a whole hierarchy starting from the given node. """ res = 0 if isinstance(node, FolderNode): for child in node.children: res += self.populateCommandCounts( child ) elif isinstance(node, TaskNode): res = len(node.task.commands) node.commandCount = res return res def _createTaskGroupFromJSON(self, taskGroupDefinition, user): # name, parent, arguments, environment, priority, dispatchKey, strategy id = None name = taskGroupDefinition['name'] parent = None arguments = taskGroupDefinition['arguments'] environment = taskGroupDefinition['environment'] requirements = taskGroupDefinition['requirements'] maxRN = taskGroupDefinition['maxRN'] priority = taskGroupDefinition['priority'] dispatchKey = taskGroupDefinition['dispatchKey'] strategy = taskGroupDefinition['strategy'] strategy = loadStrategyClass(strategy.encode()) strategy = strategy() tags = taskGroupDefinition['tags'] timer = None if 'timer' in taskGroupDefinition.keys(): timer = taskGroupDefinition['timer'] return TaskGroup(id, name, parent, user, arguments, environment, requirements, maxRN, priority, dispatchKey, strategy, tags=tags, timer=timer) def _createTaskFromJSON(self, taskDefinition, user): # id, name, parent, user, priority, dispatchKey, runner, arguments, # validationExpression, commands, requirements=[], minNbCores=1, # maxNbCores=0, ramUse=0, environment={} name = taskDefinition['name'] runner = taskDefinition['runner'] arguments = taskDefinition['arguments'] environment = taskDefinition['environment'] requirements = taskDefinition['requirements'] maxRN = taskDefinition['maxRN'] priority = taskDefinition['priority'] dispatchKey = taskDefinition['dispatchKey'] validationExpression = taskDefinition['validationExpression'] minNbCores = taskDefinition['minNbCores'] maxNbCores = taskDefinition['maxNbCores'] ramUse = taskDefinition['ramUse'] lic = taskDefinition['lic'] tags = taskDefinition['tags'] timer = None if 'timer' in taskDefinition.keys(): timer = taskDefinition['timer'] task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner, arguments, validationExpression, [], requirements, minNbCores, maxNbCores, ramUse, environment, lic=lic, tags=tags, timer=timer) for commandDef in taskDefinition['commands']: description = commandDef['description'] arguments = commandDef['arguments'] cmd = Command(None, description, task, arguments) task.commands.append(cmd) # import sys # logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd))) return task ## Resets the lists of elements to create or update in the database. # def resetDbElements(self): self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] ## Recalculates the max ids of all elements. Generally called after a reload from db. # def recomputeMaxIds(self): self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0 self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0 self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0 self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0 self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0 self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0 ## Removes from the dispatchtree the provided element and all its parents and children. # def unregisterElementsFromTree(self, element): # /////////////// Handling of the Task if isinstance(element, Task): del self.tasks[element.id] self.toArchiveElements.append(element) for cmd in element.commands: self.unregisterElementsFromTree(cmd) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskGroup elif isinstance(element, TaskGroup): del self.tasks[element.id] self.toArchiveElements.append(element) for task in element.tasks: self.unregisterElementsFromTree(task) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskNode elif isinstance(element, TaskNode): # remove the element from the children of the parent if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the FolderNode elif isinstance(element, FolderNode): if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the Command elif isinstance(element, Command): del self.commands[element.id] self.toArchiveElements.append(element) ### methods called after interaction with a Task def onTaskCreation(self, task): # logger.info(" -- on task creation: %s" % task) if task.id == None: self.taskMaxId += 1 task.id = self.taskMaxId self.toCreateElements.append(task) else: self.taskMaxId = max(self.taskMaxId, task.id) self.tasks[task.id] = task def onTaskDestruction(self, task): # logger.info(" -- on task destruction: %s" % task) self.unregisterElementsFromTree(task) def onTaskChange(self, task, field, oldvalue, newvalue): """ Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes However in order to keep track of comments (stored in task's tags[comment] field), we make the following change: - enable task/taskgroups update in DB (cf pulidb.py) - disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs BUT the only field we want to update: "tags" """ if field == "tags": self.toModifyElements.append(task) ### methods called after interaction with a BaseNode def onNodeCreation(self, node): # logger.info(" -- on node creation: %s" % node) if node.id == None: self.nodeMaxId += 1 node.id = self.nodeMaxId self.toCreateElements.append(node) else: self.nodeMaxId = max(self.nodeMaxId, node.id) if node.parent == None: node.parent = self.root def onNodeDestruction(self, node): # logger.info(" -- on node destruction: %s" % node) del self.nodes[node.id] def onNodeChange(self, node, field, oldvalue, newvalue): # logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) ) # FIXME: do something when nodes are reparented from or to the root node if node.id is not None: self.toModifyElements.append(node) if field == "status" and node.reverseDependencies: self.modifiedNodes.append(node) ### methods called after interaction with a RenderNode def onRenderNodeCreation(self, renderNode): if renderNode.id == None: self.renderNodeMaxId += 1 renderNode.id = self.renderNodeMaxId self.toCreateElements.append(renderNode) else: self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id) self.renderNodes[renderNode.name] = renderNode def onRenderNodeDestruction(self, rendernode): try: del self.renderNodes[rendernode.name] self.toArchiveElements.append(rendernode) except KeyError, e: # TOFIX: use of class method vs obj method in changeListener might generate a duplicate call logger.warning("RN %s seems to have been deleted already." % rendernode.name)
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] Let's test the throttling mechanism. Configure a 2 seconds delay on channel A, end enqueue two jobs. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4:throttle=2') >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') We have only one job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A1>] >>> cm.get_wakeup_time() 102 We have no job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.get_wakeup_time() 102 2 seconds later, we can run the other job (even though the first one is still running, because we have enough capacity). >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob A2>] >>> cm.get_wakeup_time() 104 Let's test throttling in combination with a queue reaching full capacity. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,T:2:throttle=2') >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob T1>] >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob T2>] Channel is now full, so no job to run even though throttling delay is over. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.get_wakeup_time() # no wakeup time, since queue is full 0 >>> pp(list(cm.get_jobs_to_run(now=104))) [] >>> cm.get_wakeup_time() # queue is still full 0 >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob T3>] >>> cm.get_wakeup_time() # queue is full 0 >>> cm.notify(db, 'T', 'T2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 107 Test wakeup time behaviour in presence of eta. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,E:1') >>> cm.notify(db, 'E', 'E1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob E1>] >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'E', 'E1', 1, 0, 10, 105, 'pending') >>> cm.get_wakeup_time() # wakeup at eta 105 >>> pp(list(cm.get_jobs_to_run(now=102))) # but there is capacity [<ChannelJob E2>] >>> pp(list(cm.get_jobs_to_run(now=106))) # no capacity anymore [] >>> cm.get_wakeup_time() # no timed wakeup because no capacity 0 >>> cm.notify(db, 'E', 'E2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=107))) # no capacity anymore [<ChannelJob E1>] >>> cm.get_wakeup_time() 0 Test wakeup time behaviour in a sequential queue. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,S:1:sequential') >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob S1>] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'failed') >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'pending') >>> pp(list(cm.get_jobs_to_run(now=102))) [] No wakeup time because due to eta, because the sequential queue is waiting for a failed job. >>> cm.get_wakeup_time() 0 >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob S1>] >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'done') At this stage, we have S2 with an eta of 105 and since the queue is sequential, we wait for it. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S2>] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S3>] >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @staticmethod def split_strip(s, sep, maxsplit=-1): """Split string and strip each component. >>> ChannelManager.split_strip("foo: bar baz\\n: fred:", ":") ['foo', 'bar baz', 'fred', ''] """ return [x.strip() for x in s.split(sep, maxsplit)] @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...] If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] It ignores whitespace around values, and drops empty entries which would be generated by trailing commas, or commented lines on the Odoo config file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4, ... , ... foo bar:1: k=va lue, ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}] It's also possible to replace commas with line breaks, which is more readable if you're taking the channel configuration from a ConfigParser file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4 ... foo bar:1: k=va lue ... baz ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}, {'capacity': 1, 'name': 'baz'}] """ res = [] config_string = config_string.replace("\n", ",") for channel_config_string in cls.split_strip(config_string, ','): if not channel_config_string: # ignore empty entries (commented lines, trailing commas) continue config = {} config_items = cls.split_strip(channel_config_string, ':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = cls.split_strip(config_item, '=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError('Invalid channel config %s: ' 'incorrect config item %s' % (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' % (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2,seq:1:sequential') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root').sequential False >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('root.autosub.sub').sequential False >>> cm.get_channel_by_name('autosub.sub').capacity 2 >>> cm.get_channel_by_name('seq').capacity 1 >>> cm.get_channel_by_name('seq').sequential True """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning('unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now) def get_wakeup_time(self): return self._root_channel.get_wakeup_time()
class DispatchTree(object): def __init__(self): # core data self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy()) self.nodes = WeakValueDictionary() self.nodes[0] = self.root self.pools = {} self.renderNodes = {} self.tasks = {} self.rules = [] self.poolShares = {} self.commands = {} # deduced properties self.nodeMaxId = 0 self.poolMaxId = 0 self.renderNodeMaxId = 0 self.taskMaxId = 0 self.commandMaxId = 0 self.poolShareMaxId = 0 self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] # listeners self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange) self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange) self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange) self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange) self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange) self.poolShareListener = ObjectListener(self.onPoolShareCreation) self.modifiedNodes = [] def registerModelListeners(self): BaseNode.changeListeners.append(self.nodeListener) Task.changeListeners.append(self.taskListener) TaskGroup.changeListeners.append(self.taskListener) RenderNode.changeListeners.append(self.renderNodeListener) Pool.changeListeners.append(self.poolListener) Command.changeListeners.append(self.commandListener) PoolShare.changeListeners.append(self.poolShareListener) def destroy(self): BaseNode.changeListeners.remove(self.nodeListener) Task.changeListeners.remove(self.taskListener) RenderNode.changeListeners.remove(self.renderNodeListener) Pool.changeListeners.remove(self.poolListener) Command.changeListeners.remove(self.commandListener) PoolShare.changeListeners.remove(self.poolShareListener) self.root = None self.nodes.clear() self.pools.clear() self.renderNodes.clear() self.tasks.clear() self.rules = None self.commands.clear() self.poolShares = None self.modifiedNodes = None self.toCreateElements = None self.toModifyElements = None self.toArchiveElements = None def findNodeByPath(self, path, default=None): nodenames = splitpath(path) node = self.root for name in nodenames: for child in node.children: if child.name == name: node = child break else: return default return node def updateCompletionAndStatus(self): self.root.updateCompletionAndStatus() def validateDependencies(self): nodes = set() for dependency in self.modifiedNodes: for node in dependency.reverseDependencies: nodes.add(node) del self.modifiedNodes[:] for node in nodes: if isinstance(node, TaskNode): if node.checkDependenciesSatisfaction(): for cmd in node.task.commands: if cmd.status == CMD_BLOCKED: cmd.status = CMD_READY else: for cmd in node.task.commands: if cmd.status == CMD_READY: cmd.status = CMD_BLOCKED def registerNewGraph(self, graph): user = graph['user'] taskDefs = graph['tasks'] poolName = graph['poolName'] if 'maxRN' in graph.items(): maxRN = int(graph['maxRN']) else: maxRN = -1 # # Create objects. # tasks = [None for i in xrange(len(taskDefs))] for (index, taskDef) in enumerate(taskDefs): if taskDef['type'] == 'Task': task = self._createTaskFromJSON(taskDef, user) elif taskDef['type'] == 'TaskGroup': task = self._createTaskGroupFromJSON(taskDef, user) tasks[index] = task root = tasks[graph['root']] # get the pool try: pool = self.pools[poolName] except KeyError: pool = Pool(None, poolName) self.pools[poolName] = pool # # Rebuild full job hierarchy # for (taskDef, task) in zip(taskDefs, tasks): if taskDef['type'] == 'TaskGroup': for taskIndex in taskDef['tasks']: task.addTask(tasks[taskIndex]) tasks[taskIndex].parent = task # # Compute dependencies for each created task or taskgroup object. # dependencies = {} for (taskDef, task) in zip(taskDefs, tasks): taskDependencies = {} if not isinstance(taskDef['dependencies'], list): raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) if not all(((isinstance(i, int) and isinstance(sl, list) and all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])): raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies']) for (taskIndex, statusList) in taskDef['dependencies']: taskDependencies[tasks[taskIndex]] = statusList dependencies[task] = taskDependencies # # Apply rules to generate dispatch tree nodes. # if not self.rules: logger.warning("graph submitted but no rule has been defined") unprocessedTasks = [root] nodes = [] while unprocessedTasks: unprocessedTask = unprocessedTasks.pop(0) for rule in self.rules: try: nodes += rule.apply(unprocessedTask) except RuleError: logger.warning("rule %s failed for graph %s" % (rule, graph)) raise if isinstance(unprocessedTask, TaskGroup): for task in unprocessedTask: unprocessedTasks.append(task) # create the poolshare, if any, and affect it to the node if pool: # FIXME nodes[0] may not be the root node of the graph... PoolShare(None, pool, nodes[0], maxRN) # # Process dependencies # for rule in self.rules: rule.processDependencies(dependencies) for node in nodes: assert isinstance(node.id, int) self.nodes[node.id] = node return nodes def _createTaskGroupFromJSON(self, taskGroupDefinition, user): # name, parent, arguments, environment, priority, dispatchKey, strategy id = None name = taskGroupDefinition['name'] parent = None arguments = taskGroupDefinition['arguments'] environment = taskGroupDefinition['environment'] requirements = taskGroupDefinition['requirements'] maxRN = taskGroupDefinition['maxRN'] priority = taskGroupDefinition['priority'] dispatchKey = taskGroupDefinition['dispatchKey'] strategy = taskGroupDefinition['strategy'] strategy = loadStrategyClass(strategy.encode()) strategy = strategy() tags = taskGroupDefinition['tags'] return TaskGroup(id, name, parent, user, arguments, environment, requirements, maxRN, priority, dispatchKey, strategy, tags=tags) def _createTaskFromJSON(self, taskDefinition, user): # id, name, parent, user, priority, dispatchKey, runner, arguments, # validationExpression, commands, requirements=[], minNbCores=1, # maxNbCores=0, ramUse=0, environment={} name = taskDefinition['name'] runner = taskDefinition['runner'] arguments = taskDefinition['arguments'] environment = taskDefinition['environment'] requirements = taskDefinition['requirements'] maxRN = taskDefinition['maxRN'] priority = taskDefinition['priority'] dispatchKey = taskDefinition['dispatchKey'] validationExpression = taskDefinition['validationExpression'] minNbCores = taskDefinition['minNbCores'] maxNbCores = taskDefinition['maxNbCores'] ramUse = taskDefinition['ramUse'] lic = taskDefinition['lic'] tags = taskDefinition['tags'] task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner, arguments, validationExpression, [], requirements, minNbCores, maxNbCores, ramUse, environment, lic=lic, tags=tags) for commandDef in taskDefinition['commands']: description = commandDef['description'] arguments = commandDef['arguments'] task.commands.append(Command(None, description, task, arguments)) return task ## Resets the lists of elements to create or update in the database. # def resetDbElements(self): self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] ## Recalculates the max ids of all elements. Generally called after a reload from db. # def recomputeMaxIds(self): self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0 self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0 self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0 self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0 self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0 self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0 ## Removes from the dispatchtree the provided element and all its parents and children. # def unregisterElementsFromTree(self, element): # /////////////// Handling of the Task if isinstance(element, Task): del self.tasks[element.id] self.toArchiveElements.append(element) for cmd in element.commands: self.unregisterElementsFromTree(cmd) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskGroup elif isinstance(element, TaskGroup): del self.tasks[element.id] self.toArchiveElements.append(element) for task in element.tasks: self.unregisterElementsFromTree(task) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskNode elif isinstance(element, TaskNode): # remove the element from the children of the parent if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the FolderNode elif isinstance(element, FolderNode): if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the Command elif isinstance(element, Command): del self.commands[element.id] self.toArchiveElements.append(element) ### methods called after interaction with a Task def onTaskCreation(self, task): if task.id == None: self.taskMaxId += 1 task.id = self.taskMaxId self.toCreateElements.append(task) else: self.taskMaxId = max(self.taskMaxId, task.id) self.tasks[task.id] = task def onTaskDestruction(self, task): self.unregisterElementsFromTree(task) def onTaskChange(self, task, field, oldvalue, newvalue): self.toModifyElements.append(task) ### methods called after interaction with a BaseNode def onNodeCreation(self, node): if node.id == None: self.nodeMaxId += 1 node.id = self.nodeMaxId self.toCreateElements.append(node) else: self.nodeMaxId = max(self.nodeMaxId, node.id) if node.parent == None: node.parent = self.root def onNodeDestruction(self, node): del self.nodes[node.id] def onNodeChange(self, node, field, oldvalue, newvalue): # FIXME: do something when nodes are reparented from or to the root node if node.id is not None: self.toModifyElements.append(node) if field == "status" and node.reverseDependencies: self.modifiedNodes.append(node) ### methods called after interaction with a RenderNode def onRenderNodeCreation(self, renderNode): if renderNode.id == None: self.renderNodeMaxId += 1 renderNode.id = self.renderNodeMaxId self.toCreateElements.append(renderNode) else: self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id) self.renderNodes[renderNode.name] = renderNode def onRenderNodeDestruction(self, rendernode): del self.renderNodes[rendernode.name] self.toArchiveElements.append(rendernode) def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue): self.toModifyElements.append(rendernode) ### methods called after interaction with a Pool def onPoolCreation(self, pool): if pool.id == None: self.poolMaxId += 1 pool.id = self.poolMaxId self.toCreateElements.append(pool) else: self.poolMaxId = max(self.poolMaxId, pool.id) self.pools[pool.name] = pool def onPoolDestruction(self, pool): del self.pools[pool.name] self.toArchiveElements.append(pool) def onPoolChange(self, pool, field, oldvalue, newvalue): if pool not in self.toModifyElements: self.toModifyElements.append(pool) ### methods called after interaction with a Command def onCommandCreation(self, command): if command.id is None: self.commandMaxId += 1 command.id = self.commandMaxId self.toCreateElements.append(command) else: self.commandMaxId = max(self.commandMaxId, command.id) self.commands[command.id] = command def onCommandChange(self, command, field, oldvalue, newvalue): self.toModifyElements.append(command) for node in command.task.nodes.values(): node.invalidate() ### methods called after interaction with a Pool def onPoolShareCreation(self, poolShare): if poolShare.id is None: self.poolShareMaxId += 1 poolShare.id = self.poolShareMaxId self.toCreateElements.append(poolShare) else: self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id) self.poolShares[poolShare.id] = poolShare
class Silk(SilkObject): _anonymous = None # bool _props = None # list dtype = None # list _positional_args = None # list __slots__ = [ "_parent", "_storage_enum", "_storage_nonjson_children", "_data", "_children", "_is_none", "__weakref__" ] def __init__(self, *args, _mode="any", **kwargs): self._storage_enum = None self._storage_nonjson_children = set() self._children = None if _mode == "parent": self._init( kwargs["parent"], kwargs["storage"], kwargs["data_store"], ) elif _mode == "from_numpy": assert "parent" not in kwargs self._init( None, "numpy", kwargs["data_store"], ) else: assert "parent" not in kwargs assert "storage" not in kwargs assert "data_store" not in kwargs self._init(None, "json", None) if _mode == "any": self.set(*args, **kwargs) elif _mode == "empty": pass elif _mode == "from_json": self.set(*args, prop_setter=_prop_setter_json, **kwargs) else: raise ValueError(_mode) def _init(self, parent, storage, data_store): from .silkarray import SilkArray if parent is not None: if storage == "numpy": self._parent = lambda: parent # hard ref self._parent = weakref.ref(parent) else: self._parent = lambda: None self.storage = storage self._is_none = False self._storage_nonjson_children.clear() if self._children is not None: for child in self._children.values(): child._parent = lambda: None if storage == "json": self._children = {} if data_store is None: data_store = {} elif storage == "numpy": self._children = WeakValueDictionary() assert data_store is not None assert data_store.dtype == np.dtype(self.dtype, align=True) assert data_store.shape == () self._data = data_store return else: raise ValueError(storage) assert storage == "json" for pname, p in self._props.items(): if p["elementary"]: continue t = self._get_typeclass(pname) if pname not in data_store: if issubclass(t, SilkArray): data_store[pname] = [] else: data_store[pname] = {} c_data_store = data_store[pname] self._children[pname] = t( _mode="parent", storage="json", parent=self, data_store=c_data_store, len_data_store=None, ) self._data = data_store def _get_typeclass(self, propname): p = self._props[propname] if "typeclass" in p: t = p["typeclass"] else: typename = p["typename"] t = typenames._silk_types[typename] return t def copy(self, storage="json"): """Returns a copy with the storage in the specified format""" cls = type(self) if storage == "json": json = self.json() ret = cls.from_json(json) for prop in self._props: if not self._props[prop]["elementary"]: child = self._children[prop] is_none = child._is_none ret._children[prop]._is_none = is_none elif storage == "numpy": ret = cls.from_numpy(self.numpy()) else: raise ValueError(storage) return ret @classmethod def from_json(cls, data): data = _filter_json(data) return cls(data, _mode="from_json") @classmethod def from_numpy(cls, data, copy=True,validate=True): """Constructs from a numpy array singleton "data" """ if data.shape != (): raise TypeError("Data must be a singleton") if data.dtype != np.dtype(cls.dtype,align=True): raise TypeError("Data has the wrong dtype") if copy: data = datacopy(data) ret = cls(_mode="from_numpy", data_store=data) if validate: ret.validate() return ret @classmethod def empty(cls): return cls(_mode="empty") def _get_child(self, childname, force=False): from .silkarray import SilkArray if self.storage == "numpy": prop = self._props[childname] is_none = False if prop["optional"]: if not self._data["HAS_" + childname]: is_none = True if is_none and not force: return NoneChild t = self._get_typeclass(childname) len_data_store = None if issubclass(t, SilkArray): if prop.get("var_array", False): len_data_store = self._data["LEN_"+childname] child = t ( _mode = "parent", parent = self, storage = "numpy", data_store = self._data[childname], len_data_store = len_data_store ) self._children[childname] = child return self._children[childname] def set(self, *args, prop_setter=_prop_setter_any, **kwargs): if len(args) == 1 and len(kwargs) == 0: if args[0] is None or isinstance(args[0], SilkObject) and args[0]._is_none: self._is_none = True self._clear_data() return # TODO: make a nice composite exception that stores all exceptions try: self._construct(prop_setter, *args, **kwargs) except Exception: if len(args) == 1 and len(kwargs) == 0: try: a = args[0] try: if isinstance(a, np.void): d = {} for name in a.dtype.fields: if name.startswith("HAS_"): continue name2 = "HAS_" + name if name2 in a.dtype.names and not a[name2]: continue d[name] = a[name] self._construct(prop_setter, **d) else: raise TypeError except Exception: if isinstance(a, dict): self._construct(prop_setter, **a) elif isinstance(a, str): self._parse(a) elif isinstance(a, collections.Iterable) or isinstance(a, np.void): self._construct(prop_setter, *a) elif isinstance(a, SilkObject): d = {prop: getattr(a, prop) for prop in dir(a)} self._construct(prop_setter, **d) elif hasattr(a, "__dict__"): self._construct(prop_setter, **a.__dict__) else: raise TypeError(a) except Exception: raise else: raise self.validate() self._is_none = False def validate(self): pass # overridden during registration def json(self): """Returns a JSON representation of the Silk object """ if self.storage == "json": return _filter_json(self._data) d = {} for attr in self._props: p = self._props[attr] ele = p["elementary"] value = None if ele: if self.storage == "numpy": value = _get_numpy_ele_prop(self, attr) else: value = self._data[attr] if value is not None: t = self._get_typeclass(attr) value = t(value) else: child = self._get_child(attr) if not child._is_none: value = child.json() if value is not None: d[attr] = value return d def numpy(self): """Returns a numpy representation of the Silk object NOTE: for optional members, the entire storage buffer is returned, including (zeroed) elements if the data is not present! the extra field "HAS_xxx" indicates if the data is present. NOTE: for all numpy array members, the entire storage buffer is returned, including (zeroed) elements if the data is not present! the length of each array is stored in the LEN_xxx field TODO: document multidimensional length vector, PTR_LEN_xxx NOTE: for numpy array members of variable shape, an extra field "PTR_xxx" contains a C pointer to the data For this, the dimensionality of the array does not matter, e.g. both for IntegerArray and IntegerArrayArray, the C pointer will be "int *" and both for MyStructArray and MyStructArrayArray, the C pointer will be "MyStruct *" TODO: add and document SHAPE field """ if self.storage == "numpy": return datacopy(self._data) new_obj = self.copy("json") return new_obj.make_numpy() def make_json(self): if self.storage == "json": return self._data elif self.storage == "numpy": json = _filter_json(self.json(), self) parent = self._parent() if parent is not None and parent.storage == "numpy": parent.numpy_shatter() self._init(parent, "json", None) self.set(json, prop_setter=_prop_setter_json) if parent is not None: parent._remove_nonjson_child(self) myname = parent._find_child(id(self)) parent._data[myname] = self._data return self._data elif self.storage == "mixed": for child_id in list(self._storage_nonjson_children): # copy! for child in self._children.values(): if id(child) == child_id: child.make_json() break else: raise Exception("Cannot find child that was marked as 'non-JSON'") # Above will automatically update storage status to "json" return self._data def _restore_array_coupling(self): pass def make_numpy(self,_toplevel=None): """Sets the internal storage to 'numpy' Returns the numpy array that is used as internal storage buffer NOTE: for optional members, the entire storage buffer is returned, including (zeroed) elements if the data is not present! an extra field "HAS_xxx" indicates if the data is present. TODO: update doc NOTE: for numpy array members of variable shape, an extra field "PTR_xxx" contains a C pointer to the data For this, the dimensionality of the array does not matter, e.g. both for IntegerArray and IntegerArrayArray, the C pointer will be "int *" and both for MyStructArray and MyStructArrayArray, the C pointer will be "MyStruct *" """ from .silkarray import SilkArray if self.storage == "numpy": return self._data dtype = np.dtype(self.dtype, align=True) data = np.zeros(dtype=dtype, shape=(1,)) for propname,prop in self._props.items(): if prop["elementary"]: value = getattr(self, propname) _set_numpy_ele_prop(self, propname, value, data) else: child = self._get_child(propname) if not child._is_none: child.make_numpy(_toplevel=False) if isinstance(child, SilkArray): if prop.get("var_array", False): child._restore_array_coupling(data[0], propname) else: data[0][propname] = np.zeros_like(dtype[propname]) slices = [slice(0,v) in child._data.shape] data[0][propname][slices] = child._data else: data[0][propname] = child._data child._data = None self._init(self._parent(), "numpy", data[0]) parent = self._parent() if parent is not None: if parent.storage != "numpy": parent._add_nonjson_child(self) return data[0] def _find_child(self, child_id): for childname, ch in self._children.items(): if child_id == id(ch): return childname raise KeyError def _add_nonjson_child(self, child): childname = self._find_child(id(child)) if self._props[childname].get("var_array", False) and \ self.storage == "numpy": return assert self.storage != "numpy" njc = self._storage_nonjson_children child_id = id(child) if child_id not in njc: njc.add(child_id) if self.storage == "json": self.storage = "mixed" parent = self._parent() if parent is not None: parent._add_nonjson_child(self) def _remove_nonjson_child(self, child): assert self.storage != "numpy" njc = self._storage_nonjson_children child_id = id(child) if child_id in njc: assert self.storage == "mixed", self.storage njc.remove(child_id) if len(njc) == 0: self.storage = "json" parent = self._parent() if parent is not None: parent()._remove_nonjson_child(self) def numpy_shatter(self): """ Breaks up a unified numpy storage into one numpy storage per child """ assert self.storage == "numpy" parent = self._parent() if parent is not None and parent.storage == "numpy": parent.numpy_shatter() data = {} children = {} for prop in self._props: p = self._props[prop] if p["elementary"]: value = getattr(self, prop) if value is not None: if "typeclass" in p: t = p["typeclass"] else: typename = p["typename"] t = typenames._silk_types[typename] value = t(value) data[prop] = value else: child = self._get_child(prop) d = datacopy(child._data) data[prop] = d child._data = d children[prop] = child self._data = data self._children = children self._storage_nonjson_children = set([id(p) for p in children.values()]) self.storage = "mixed" def _construct(self, prop_setter, *args, **kwargs): propdict = {} if len(args) > len(self._positional_args): message = "{0}() takes {1} positional arguments \ but {2} were given".format( self.__class__.__name__, len(self._positional_args), len(args) ) raise TypeError(message) for anr, a in enumerate(args): propdict[self._positional_args[anr]] = a for argname, a in kwargs.items(): if argname in propdict: message = "{0}() got multiple values for argument '{1}'" message = message.format( self.__class__.__name__, argname ) raise TypeError(message) propdict[argname] = a missing = [p for p in self._props if p not in propdict] missing_required = [p for p in missing if not self._props[p]["optional"] and p not in self._props_init] if missing_required: missing_required = ["'{0}'".format(p) for p in missing_required] if len(missing_required) == 1: plural = "" missing_txt = missing_required[0] elif len(missing_required) == 2: plural = "s" missing_txt = missing_required[0] + " and " + \ missing_required[1] else: plural = "s" missing_txt = ", ".join(missing_required[:-1]) + \ ", and " + missing_required[-1] message = "{0}() missing {1} positional argument{2}: {3}".format( self.__class__.__name__, len(missing_required), plural, missing_txt ) raise TypeError(message) for propname in self._props: value = propdict.get(propname, None) if value is None and propname in self._props_init: value = self._props_init[propname] self._set_prop(propname, value, prop_setter) def _parse(self, s): raise NotImplementedError # can be user-defined _storage_names = ("numpy", "json", "mixed") @property def storage(self): return self._storage_names[self._storage_enum] @storage.setter def storage(self, storage): assert storage in self._storage_names, storage self._storage_enum = self._storage_names.index(storage) def __dir__(self): return dir(type(self)) def __setattr__(self, attr, value): if attr.startswith("_") or attr == "storage": object.__setattr__(self, attr, value) else: self._set_prop(attr, value, _prop_setter_any) def _set_prop(self, prop, value, child_prop_setter): try: p = self._props[prop] except KeyError: raise AttributeError(prop) if value is None and not p["optional"]: raise TypeError("'%s' cannot be None" % prop) ele = p["elementary"] if ele: if self.storage == "numpy": _set_numpy_ele_prop(self, prop, value) else: if value is not None: if "typeclass" in p: t = p["typeclass"] else: typename = p["typename"] t = typenames._silk_types[typename] value = t(value) self._data[prop] = value else: child = self._get_child(prop) do_set = True if child is NoneChild: if value is None: do_set = False else: child = self._get_child(prop, force=True) if do_set: if self.storage == "numpy" and p.get("var_array", False): child.set(value) else: child_prop_setter(child, value) if self.storage == "numpy" and p["optional"]: self._data["HAS_"+prop] = (value is not None) def __getattribute__(self, attr): value = object.__getattribute__(self, attr) if attr.startswith("_") or attr in ("storage", "dtype"): return value class_value = getattr(type(self), attr) if value is class_value: raise AttributeError(value) return value def __getattr__(self, attr): try: ele = self._props[attr]["elementary"] except KeyError: raise AttributeError(attr) from None if ele: if self.storage == "numpy": ret = _get_numpy_ele_prop(self, attr) else: ret = self._data.get(attr, None) if ret is None: assert self._props[attr]["optional"] else: ret = self._get_child(attr) if ret._is_none: ret = None return ret def _print(self, spaces): name = "" if not self._anonymous: name = self.__class__.__name__ + " " ret = "{0}(\n".format(name) for propname in self._props: prop = self._props[propname] value = getattr(self, propname) if prop["optional"]: if value is None: continue if self.storage == "numpy" and prop["elementary"]: substr = value if self._data[propname].dtype.kind == 'S': substr = '"' + value + '"' else: substr = str(value) else: substr = value._print(spaces+2) ret += "{0}{1} = {2},\n".format(" " * (spaces+2), propname, substr) ret += "{0})".format(" " * spaces) return ret def __str__(self): return self._print(0) def __repr__(self): return self._print(0) def __eq__(self, other): if not isinstance(other, SilkObject): return False if self.storage == other.storage == "json": return self._data == other._data else: #can't use numpy _data because of PTR and different allocation sizes return self.json() == other.json() def _clear_data(self): d = self._data if self.storage == "numpy": d.fill(np.zeros_like(d)) else: for propname in self._props: prop = self._props[propname] if prop["elementary"]: if propname in d: d.pop(propname) else: child = self._get_child(propname) child._clear_data()
class Worker(GearmanProtocolMixin, asyncio.Protocol): def __init__(self, *functions, loop=None, grab_type=Type.GRAB_JOB, timeout=None): super(Worker, self).__init__(loop=loop) self.transport = None self.main_task = None self.functions = OrderedDict() self.running = WeakValueDictionary() self.waiters = [] self.shutting_down = False self.timeout = timeout grab_mapping = { Type.GRAB_JOB: self.grab_job, Type.GRAB_JOB_UNIQ: self.grab_job_uniq, Type.GRAB_JOB_ALL: self.grab_job_all, } try: self.grab = grab_mapping[grab_type] except KeyError: raise RuntimeError( 'Grab type must be one of GRAB_JOB, GRAB_JOB_UNIQ or GRAB_JOB_ALL' ) for func_arg in functions: try: func, name = func_arg self.functions[name] = func except TypeError: name = func_arg.__name__ self.functions[name] = func_arg def connection_made(self, transport): logger.info('Connection is made to %r', transport.get_extra_info('peername')) self.transport = transport if self.timeout is not None: can_do = partial(self.can_do_timeout, timeout=self.timeout) else: can_do = self.can_do for fname in self.functions.keys(): logger.debug('Registering function %s', fname) can_do(fname) self.main_task = self.get_task(self.run()) def connection_lost(self, exc): self.transport = None def get_task(self, coro): return asyncio.ensure_future(coro, loop=self.loop) async def run(self, ): no_job = NoJob() while not self.shutting_down: self.pre_sleep() await self.wait_for(Type.NOOP) response = await self.grab() if response == no_job: continue try: job_info = self._to_job_info(response) func = self.functions.get(job_info.function) if not func: logger.warning('Failed to find function %s in %s', job_info.function, ', '.join(self.functions.keys())) self.work_fail(job_info.handle) continue try: result_or_coro = func(job_info) if asyncio.iscoroutine(result_or_coro): task = self.get_task(result_or_coro) self.running[job_info.handle] = task result = await task else: result = result_or_coro self.work_complete(job_info.handle, result) except Exception as ex: logger.exception('Job (handle %s) resulted with exception', job_info.handle) self.work_exception(job_info.handle, str(ex)) finally: self.running.pop(job_info.handle, None) except AttributeError: logger.error('Unexpected GRAB_JOB response %r', response) async def shutdown(self, graceful=False): logger.debug('Shutting down worker {}gracefully...'.format( '' if graceful else 'un')) self.shutting_down = True sub_tasks = list(self.running.values()) if graceful: if sub_tasks: await asyncio.wait(sub_tasks, loop=self.loop) else: async def cancel_and_wait(tasks): for task in tasks: task.cancel() try: await asyncio.wait(tasks, loop=self.loop) except asyncio.CancelledError: pass if sub_tasks: await cancel_and_wait(sub_tasks) self.main_task.cancel() if self.transport: self.transport.close() @staticmethod def _to_job_info(job_assign): attrs = ['handle', 'function', 'uuid', 'reducer', 'workload'] values = [getattr(job_assign, attr, None) for attr in attrs] return JobInfo(*values) def register_function(self, func, name=''): if not self.transport: raise RuntimeError('Worker must be connected to the daemon') name = name or func.__name__ self.functions[name] = func return self.can_do(name) def grab_job_all(self): self.send(Type.GRAB_JOB_ALL) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_ALL) def grab_job_uniq(self): self.send(Type.GRAB_JOB_UNIQ) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_UNIQ) def grab_job(self): self.send(Type.GRAB_JOB) return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN) def pre_sleep(self): self.send(Type.PRE_SLEEP) def can_do(self, function): self.send(Type.CAN_DO, function) def can_do_timeout(self, function, timeout): self.send(Type.CAN_DO_TIMEOUT, function, timeout) def work_fail(self, handle): self.send(Type.WORK_FAIL, handle) def work_exception(self, handle, data): self.send(Type.WORK_EXCEPTION, handle, data) def work_complete(self, handle, result): if result is None: result = '' self.send(Type.WORK_COMPLETE, handle, result) def set_client_id(self, client_id): self.send(Type.SET_CLIENT_ID, client_id)
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)?,... If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] """ res = [] for channel_config_string in config_string.split(','): config = {} config_items = channel_config_string.split(':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = config_item.split('=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError('Invalid channel config %s: ', 'incorrect config item %s' (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('autosub.sub').capacity 2 """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning('unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now)
class Boss: def __init__(self, glfw_window, opts, args): self.window_id_map = WeakValueDictionary() startup_session = create_session(opts, args) self.cursor_blinking = True self.window_is_focused = True self.glfw_window_title = None self.shutting_down = False self.child_monitor = ChildMonitor( glfw_window.window_id(), self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None) set_boss(self) self.current_font_size = opts.font_size cell_size.width, cell_size.height = set_font_family(opts) self.opts, self.args = opts, args self.glfw_window = glfw_window glfw_window.framebuffer_size_callback = self.on_window_resize glfw_window.window_focus_callback = self.on_focus load_shader_programs() self.tab_manager = TabManager(opts, args) self.tab_manager.init(startup_session) self.activate_tab_at = self.tab_manager.activate_tab_at layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper) @property def current_tab_bar_height(self): return self.tab_manager.tab_bar_height def __iter__(self): return iter(self.tab_manager) def iterwindows(self): for t in self: yield from t def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def on_child_death(self, window_id): w = self.window_id_map.pop(window_id, None) if w is not None: w.on_child_death() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def on_window_resize(self, window, w, h): viewport_size.width, viewport_size.height = w, h self.tab_manager.resize() def increase_font_size(self): self.change_font_size( min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def change_font_size(self, new_size): if new_size == self.current_font_size: return self.current_font_size = new_size w, h = cell_size.width, cell_size.height windows = tuple(filter(None, self.window_id_map.values())) cell_size.width, cell_size.height = set_font_family( self.opts, override_font_size=self.current_font_size) layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper) for window in windows: window.screen.rescale_images(w, h) self.resize_windows_after_font_size_change() for window in windows: window.screen.refresh_sprite_positions() self.tab_manager.refresh_sprite_positions() def resize_windows_after_font_size_change(self): self.tab_manager.resize() glfw_post_empty_event() def tabbar_visibility_changed(self): self.tab_manager.resize(only_tabs=True) glfw_post_empty_event() @property def active_tab(self): return self.tab_manager.active_tab def is_tab_visible(self, tab): return self.active_tab is tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True key, scancode, action, mods = self.current_key_press_info data = get_sent_data(self.opts.send_text_map, key, scancode, mods, window, action) if data: window.write_to_child(data) return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, window, focused): self.window_is_focused = focused w = self.active_window if w is not None: w.focus_changed(focused) def display_scrollback(self, data): if self.opts.scrollback_in_new_tab: self.display_scrollback_in_new_tab(data) else: tab = self.active_tab if tab is not None: tab.new_special_window( SpecialWindow(self.opts.scrollback_pager, data, _('History'))) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def send_fake_scroll(self, window_idx, amt, upwards): tab = self.active_tab w = tab.windows[window_idx] k = get_key_map(w.screen)[GLFW_KEY_UP if upwards else GLFW_KEY_DOWN] w.write_to_child(k * amt) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def gui_close_window(self, window): window.destroy() for tab in self.tab_manager: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: self.tab_manager.remove(tab) tab.destroy() if len(self.tab_manager) == 0: if not self.shutting_down: self.glfw_window.set_should_close(True) glfw_post_empty_event() def destroy(self): self.shutting_down = True self.child_monitor.shutdown() wakeup() self.child_monitor.join() for t in self.tab_manager: t.destroy() del self.tab_manager destroy_sprite_map() destroy_global_data() del self.glfw_window def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = self.glfw_window.get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) def next_tab(self): self.tab_manager.next_tab() def previous_tab(self): self.tab_manager.next_tab(-1) def new_tab(self): self.tab_manager.new_tab() def move_tab_forward(self): self.tab_manager.move_tab(1) def move_tab_backward(self): self.tab_manager.move_tab(-1) def display_scrollback_in_new_tab(self, data): self.tab_manager.new_tab(special_window=SpecialWindow( self.opts.scrollback_pager, data, _('History')))
class PersistentList(object): """ Sequence object that is persistently stored :param store_uri: URI for storing buckets; see :py:class:`~BaseBucketStore` :type store_uri: :py:class:`str` :param bucket_length: number of items to store per bucket :type bucket_length: :py:class:`int` :param cache_size: number of buckets to LRU-cache in memory :type cache_size: :py:class:`int` """ persistent_defaults = { 'bucket_length': 32, } def __init__(self, store_uri, bucket_length=NOTSET, cache_size=3): self._bucket_store = BaseBucketStore.from_uri(store_uri=store_uri, default_scheme='file') # set empty fields self._bucket_length = None self._bucket_count = 0 self._len = 0 self._bucket_cache = None self._cache_size = None self.bucket_key_fmt = None # load current settings try: for attr, value in self._bucket_store.fetch_head().items(): setattr(self, attr, value) except BucketNotFound: pass # apply new settings self.bucket_length = bucket_length # LRU store for objects fetched from disk self.cache_size = cache_size # weakref store for objects still in use self._active_buckets = WeakValueDictionary() self._active_items = WeakValueDictionary() # calcualate metadata self._length = self._fetch_length() # store new settings self._store_head() # Settings def _store_head(self): """ Store the meta-information of the dict """ self._bucket_store.store_head({ attr: getattr(self, attr) for attr in ('bucket_length', '_bucket_count') }) def _fetch_length(self): """Calculate the length of the list from the persistent store""" if self._bucket_count == 0: return 0 last_bucket = self._fetch_bucket(self.bucket_key_fmt % (self._bucket_count - 1)) return (self._bucket_count - 1) * self._bucket_length + len(last_bucket) def _update_bucket_key_fmt(self): # key: count, salt, index self.bucket_key_fmt = "dlistbkt_%(bucket_length)xs%%x" % { 'bucket_length': self.bucket_length, } @property def _length(self): return self._len @_length.setter def _length(self, value): # detect change of bucket count if self._bucket_count != value // self._bucket_length: self._bucket_count = value // self._bucket_length self._store_head() self._len = value # exposed settings @property def cache_size(self): return self._cache_size @cache_size.setter def cache_size(self, value): self._cache_size = int(value or 1) self._bucket_cache = deque(maxlen=self.cache_size) @property def bucket_length(self): """ Get/Set the ``bucket_length`` of the persistent mapping :note: Setting ``bucket_length`` causes **all** buckets storing data to be recreated. Until the new buckets have been created, changes to the mapping content may be silently dropped. """ return self._bucket_length @bucket_length.setter def bucket_length(self, value): # default if unset if value == NOTSET: if self._bucket_length is not None: return self._bucket_length = self.persistent_defaults['bucket_length'] else: value = int(value) if value < 1: raise ValueError('At least one item per bucket must be used') # no change elif self._bucket_length == value: return # uninitialized, we don't have content yet elif self._bucket_length is None: self._bucket_length = value # TODO: allow resizing backend else: raise NotImplementedError('Changing bucket count not implemented yet') # apply secondary settings self._update_bucket_key_fmt() # bucket management @property def _bucket_keys(self): """List of used bucket keys""" return [self._bucket_key(idx) for idx in range(0, self._length, self._bucket_length)] def _bucket_key(self, index): """ Create the bucket identifier for a given key :param index: key to the content in-memory :return: key to the bucket stored persistently :rtype: str """ if index < 0: index += self._length return self.bucket_key_fmt % (index // self._bucket_length) def _fetch_bucket(self, bucket_key): """ Return a bucket from disk or create a new one :param bucket_key: key for the bucket :return: bucket for ``bucket_key`` :rtype: :py:class:`~DictBucket` """ try: bucket = self._bucket_store.fetch_bucket(bucket_key=bucket_key) except BucketNotFound: self._store_head() bucket = ListBucket() self._active_buckets[bucket_key] = bucket self._bucket_cache.appendleft(bucket) return bucket def _get_bucket(self, bucket_key): """ Return the appropriate bucket May return the cached bucket if available. :param bucket_key: key for the bucket :return: bucket for ``bucket_key`` :rtype: :py:class:`~DictBucket` """ try: return self._active_buckets[bucket_key] except KeyError: return self._fetch_bucket(bucket_key) def _store_bucket(self, bucket_key, bucket=None): """ Store a bucket on disk :param bucket_key: key for the entire bucket """ if bucket is None: try: bucket = self._active_buckets[bucket_key] except KeyError: return if bucket: self._bucket_store.store_bucket(bucket_key=bucket_key, bucket=bucket) # free empty buckets else: self._bucket_store.free_bucket(bucket_key) # cache management # Item cache def _set_cached_item(self, key, item): """Cache reference to existing item""" try: self._active_items[key] = item except TypeError: pass def _get_cached_item(self, key): """Get reference to existing item; raises KeyError if item cannot be fetched""" try: return self._active_items[key] except TypeError: raise KeyError def _del_cached_item(self, key): """Release reference to existing item""" try: del self._active_items[key] except (TypeError, KeyError): pass # paths and files def flush(self): """ Commit all outstanding changes to persistent store """ for bucket_key, bucket in self._active_buckets.values(): self._store_bucket(bucket_key, bucket) # sequence interface def __getitem__(self, pos): if isinstance(pos, slice): return self._get_slice(pos) return self._get_item(pos) def _get_item(self, index): # - use cached reference to existing item # - fetch item from cached reference to existing bucket # - fetch item from fetched bucket try: return self._get_cached_item(index) except KeyError: bucket = self._get_bucket(self._bucket_key(index)) item = bucket[index % self._bucket_length] self._set_cached_item(index, item) return item def _get_slice(self, positions): start_idx, stop_idx, stride = positions.indices(self._length) list_slice = [] # fetch sub-slice from each bucket while start_idx < stop_idx: bucket = self._get_bucket(self._bucket_key(start_idx)) # stop_idx in next bucket if stop_idx // self._bucket_length > start_idx // self._bucket_length: list_slice.extend(bucket[start_idx % self._bucket_length::stride]) slice_length = math.ceil((self._bucket_length - (start_idx % self._bucket_length)) / stride) # stop_idx in this bucket else: list_slice.extend(bucket[start_idx % self._bucket_length:stop_idx % self._bucket_length:stride]) slice_length = math.ceil(((stop_idx - start_idx) % self._bucket_length) / stride) # advance to next bucket start_idx += slice_length * stride return list_slice def __setitem__(self, pos, value): if isinstance(pos, slice): self._set_slice(pos, value) else: self._set_item(pos, value) def _set_slice(self, positions, sequence): start_idx, stop_idx, stride = positions.indices(self._length) sequence = list(sequence) # fetch sub-slice from each bucket while start_idx < stop_idx: bucket_key = self._bucket_key(start_idx) bucket = self._get_bucket(bucket_key) # stop_idx in next bucket if stop_idx // self._bucket_length > start_idx // self._bucket_length: slice_length = math.ceil((self._bucket_length - (start_idx % self._bucket_length)) / stride) bucket[start_idx % self._bucket_length::stride] = sequence[:slice_length] # stop_idx in this bucket else: slice_length = math.ceil(((stop_idx - start_idx) % self._bucket_length) / stride) bucket[start_idx % self._bucket_length:stop_idx % self._bucket_length:stride] = sequence[:slice_length] self._store_bucket(bucket_key, bucket) # advance to next bucket start_idx += slice_length * stride sequence[:] = sequence[slice_length:] def _set_item(self, index, value): bucket_key = self._bucket_key(index) bucket = self._get_bucket(bucket_key) bucket[index % self._bucket_length] = value self._store_bucket(bucket_key, bucket) # update item cache self._set_cached_item(index, value) def __delitem__(self, index): if isinstance(index, slice): self._del_slice(index) else: self._del_item(index) def _del_slice(self, positions): start_idx, stop_idx, stride = positions.indices(self._length) print(start_idx, stop_idx, stride) if start_idx == stop_idx: return # TODO: make this work on sequences for idx in range(start_idx, stop_idx, stride): self._del_item(idx) # consecutive sequence #if stride == 1: # pass def _del_item(self, pos): self._del_cached_item(pos) # delete first element, append first element with next bucket this_bucket_key = self._bucket_key(pos) this_bucket = self._get_bucket(this_bucket_key) # remaining elements move one forward del this_bucket[pos % self._bucket_length] # if this fails, IndexError for entire collection # index of first element of next bucket pos = (pos // self._bucket_length + 1) * self._bucket_length # fill missing last element with first of next bucket while pos < self._length - 1: next_bucket_key = self._bucket_key(pos) next_bucket = self._get_bucket(next_bucket_key) this_bucket.append(next_bucket.pop(0)) self._store_bucket(this_bucket_key, this_bucket) this_bucket_key = next_bucket_key this_bucket = next_bucket pos += self._bucket_length # store last bucket self._store_bucket(this_bucket_key, this_bucket) self._length -= 1 # container protocol def __len__(self): return self._length # list methods def append(self, item): bucket_key = self._bucket_key(self._length) bucket = self._get_bucket(bucket_key) bucket.append(item) self._store_bucket(bucket_key, bucket) # update item cache self._set_cached_item(self._length, item) self._length += 1 def extend(self, sequence): # split sequence into chunks to fill each bucket while sequence: bucket_key = self._bucket_key(self._length) bucket = self._get_bucket(bucket_key) bucket_sequence, sequence = \ sequence[:self._bucket_length - len(bucket)], sequence[self._bucket_length - len(bucket):] bucket.extend(bucket_sequence) self._store_bucket(bucket_key, bucket) # update item cache for idx, item in enumerate(bucket_sequence): self._set_cached_item(self._length + idx, item) self._length += len(bucket_sequence) def clear(self): # clear persistent storage for bucket_key in self._bucket_keys: self._bucket_store.free_bucket(bucket_key=bucket_key) self._length = 0 self._bucket_count = 0 self._store_head() # reset caches self._bucket_cache = deque(maxlen=self.cache_size) self._active_buckets = type(self._active_buckets)() #self._active_items = type(self._active_items)() def insert(self, index, value): raise NotImplementedError def pop(self, index=None): raise NotImplementedError def remove(self, value): raise NotImplementedError def reverse(self, value): raise NotImplementedError def __str__(self): return '[<%s>]' % ('>, <'.join( str(self._get_bucket(bucket_key))[1:-1] for bucket_key in self._bucket_keys ))
class Signal(object): def __init__(self, *args): self.__slots = WeakValueDictionary() for slot in args: self.connect(slot) def __call__(self, slot, *args, **kwargs): """ Emit signal. If slot passed signal will be called only for this slot, for all connected slots otherwise. Calling this method directly lead to immediate signal processing. It may be not thread-safe. Use emit method from this module for delayed calling of signals. """ if slot is not None: slots = (self.__slots[self.key(slot)], ) else: slots = self.__slots.values() for func in slots: func(*args, **kwargs) def key(self, slot): """ Get local key name for slot. """ if type(slot) == types.FunctionType: key = (slot.__module__, slot.__name__) elif type(slot) == types.MethodType: key = (slot.__func__, id(slot.__self__)) elif isinstance(slot, basestring): if not slot in registred_slots.keys(): raise ValueError('Slot {0} does not exists.'.format(slot)) key = slot else: raise ValueError('Slot {0} has non-slot type'.format(slot)) return key def connect(self, slot): """ Connect signal to slot. Slot may be function, instance method or name of function perviously registred by `slot` decorator. """ key = self.key(slot) if type(slot) == types.FunctionType: self.__slots[key] = slot elif type(slot) == types.MethodType: self.__slots[key] = partial(slot.__func__, slot.__self__) elif isinstance(slot, basestring): self.__slots[key] = registred_slots[slot] def disconnect(self, slot): """ Remove slot from signal connetions. """ key = self.key(slot) del self.__slots[key] def clear(self): """ Disconnect all slots from signal. """ self.__slots.clear()
class DispatchTree(object): def __init__(self): # core data self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy()) self.nodes = WeakValueDictionary() self.nodes[0] = self.root self.pools = {} self.renderNodes = {} self.tasks = {} self.rules = [] self.poolShares = {} self.commands = {} # deduced properties self.nodeMaxId = 0 self.poolMaxId = 0 self.renderNodeMaxId = 0 self.taskMaxId = 0 self.commandMaxId = 0 self.poolShareMaxId = 0 self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] # listeners self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange) self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange) # # JSA # self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange) self.renderNodeListener = ObjectListener( self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange ) self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange) self.commandListener = ObjectListener( onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange ) self.poolShareListener = ObjectListener(self.onPoolShareCreation) self.modifiedNodes = [] def registerModelListeners(self): BaseNode.changeListeners.append(self.nodeListener) Task.changeListeners.append(self.taskListener) TaskGroup.changeListeners.append(self.taskListener) RenderNode.changeListeners.append(self.renderNodeListener) Pool.changeListeners.append(self.poolListener) Command.changeListeners.append(self.commandListener) PoolShare.changeListeners.append(self.poolShareListener) def destroy(self): BaseNode.changeListeners.remove(self.nodeListener) Task.changeListeners.remove(self.taskListener) RenderNode.changeListeners.remove(self.renderNodeListener) Pool.changeListeners.remove(self.poolListener) Command.changeListeners.remove(self.commandListener) PoolShare.changeListeners.remove(self.poolShareListener) self.root = None self.nodes.clear() self.pools.clear() self.renderNodes.clear() self.tasks.clear() self.rules = None self.commands.clear() self.poolShares = None self.modifiedNodes = None self.toCreateElements = None self.toModifyElements = None self.toArchiveElements = None def findNodeByPath(self, path, default=None): nodenames = splitpath(path) node = self.root for name in nodenames: for child in node.children: if child.name == name: node = child break else: return default return node def updateCompletionAndStatus(self): self.root.updateCompletionAndStatus() def validateDependencies(self): nodes = set() for dependency in self.modifiedNodes: for node in dependency.reverseDependencies: nodes.add(node) del self.modifiedNodes[:] for node in nodes: # logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) ) if not hasattr(node, "task") or node.task is None: continue if isinstance(node, TaskNode): if node.checkDependenciesSatisfaction(): for cmd in node.task.commands: if cmd.status == CMD_BLOCKED: cmd.status = CMD_READY else: for cmd in node.task.commands: if cmd.status == CMD_READY: cmd.status = CMD_BLOCKED # TODO: may be needed to check dependencies on task groups # so far, a hack is done on the client side when submitting: # dependencies of a taksgroup are reported on each task of its heirarchy # # elif isinstance(node, FolderNode): # # if node.checkDependenciesSatisfaction(): # for cmd in node.getAllCommands(): # if cmd.status == CMD_BLOCKED: # cmd.status = CMD_READY # else: # for cmd in node.getAllCommands(): # if cmd.status == CMD_READY: # cmd.status = CMD_BLOCKED def registerNewGraph(self, graph): user = graph["user"] taskDefs = graph["tasks"] poolName = graph["poolName"] if "maxRN" in graph.items(): maxRN = int(graph["maxRN"]) else: maxRN = -1 # # Create objects. # tasks = [None for i in xrange(len(taskDefs))] for (index, taskDef) in enumerate(taskDefs): if taskDef["type"] == "Task": # logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"]) # logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"]) task = self._createTaskFromJSON(taskDef, user) elif taskDef["type"] == "TaskGroup": task = self._createTaskGroupFromJSON(taskDef, user) tasks[index] = task root = tasks[graph["root"]] # get the pool try: pool = self.pools[poolName] except KeyError: pool = Pool(None, poolName) self.pools[poolName] = pool # # Rebuild full job hierarchy # for (taskDef, task) in zip(taskDefs, tasks): if taskDef["type"] == "TaskGroup": for taskIndex in taskDef["tasks"]: task.addTask(tasks[taskIndex]) tasks[taskIndex].parent = task # # Compute dependencies for each created task or taskgroup object. # dependencies = {} for (taskDef, task) in zip(taskDefs, tasks): taskDependencies = {} if not isinstance(taskDef["dependencies"], list): raise SyntaxError( "Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"] ) if not all( ( (isinstance(i, int) and isinstance(sl, list) and all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef["dependencies"] ) ): raise SyntaxError( "Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"] ) for (taskIndex, statusList) in taskDef["dependencies"]: taskDependencies[tasks[taskIndex]] = statusList dependencies[task] = taskDependencies # # Apply rules to generate dispatch tree nodes. # if not self.rules: logger.warning("graph submitted but no rule has been defined") unprocessedTasks = [root] nodes = [] while unprocessedTasks: unprocessedTask = unprocessedTasks.pop(0) for rule in self.rules: try: nodes += rule.apply(unprocessedTask) except RuleError: logger.warning("rule %s failed for graph %s" % (rule, graph)) raise if isinstance(unprocessedTask, TaskGroup): for task in unprocessedTask: unprocessedTasks.append(task) # create the poolshare, if any, and affect it to the node if pool: # FIXME nodes[0] may not be the root node of the graph... ps = PoolShare(None, pool, nodes[0], maxRN) # if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true if maxRN != -1: ps.userDefinedMaxRN = True # # Process dependencies # for rule in self.rules: rule.processDependencies(dependencies) for node in nodes: assert isinstance(node.id, int) self.nodes[node.id] = node # Init number of command in hierarchy self.populateCommandCounts(nodes[0]) return nodes def populateCommandCounts(self, node): """ Updates "commandCount" over a whole hierarchy starting from the given node. """ res = 0 if isinstance(node, FolderNode): for child in node.children: res += self.populateCommandCounts(child) elif isinstance(node, TaskNode): res = len(node.task.commands) node.commandCount = res return res def _createTaskGroupFromJSON(self, taskGroupDefinition, user): # name, parent, arguments, environment, priority, dispatchKey, strategy id = None name = taskGroupDefinition["name"] parent = None arguments = taskGroupDefinition["arguments"] environment = taskGroupDefinition["environment"] requirements = taskGroupDefinition["requirements"] maxRN = taskGroupDefinition["maxRN"] priority = taskGroupDefinition["priority"] dispatchKey = taskGroupDefinition["dispatchKey"] strategy = taskGroupDefinition["strategy"] strategy = loadStrategyClass(strategy.encode()) strategy = strategy() tags = taskGroupDefinition["tags"] timer = None if "timer" in taskGroupDefinition.keys(): timer = taskGroupDefinition["timer"] return TaskGroup( id, name, parent, user, arguments, environment, requirements, maxRN, priority, dispatchKey, strategy, tags=tags, timer=timer, ) def _createTaskFromJSON(self, taskDefinition, user): # id, name, parent, user, priority, dispatchKey, runner, arguments, # validationExpression, commands, requirements=[], minNbCores=1, # maxNbCores=0, ramUse=0, environment={} name = taskDefinition["name"] runner = taskDefinition["runner"] arguments = taskDefinition["arguments"] environment = taskDefinition["environment"] requirements = taskDefinition["requirements"] maxRN = taskDefinition["maxRN"] priority = taskDefinition["priority"] dispatchKey = taskDefinition["dispatchKey"] validationExpression = taskDefinition["validationExpression"] minNbCores = taskDefinition["minNbCores"] maxNbCores = taskDefinition["maxNbCores"] ramUse = taskDefinition["ramUse"] lic = taskDefinition["lic"] tags = taskDefinition["tags"] runnerPackages = taskDefinition.get("runnerPackages", "") watcherPackages = taskDefinition.get("watcherPackages", "") timer = None if "timer" in taskDefinition.keys(): timer = taskDefinition["timer"] maxAttempt = taskDefinition.get("maxAttempt", 1) task = Task( None, name, None, user, maxRN, priority, dispatchKey, runner, arguments, validationExpression, [], requirements, minNbCores, maxNbCores, ramUse, environment, lic=lic, tags=tags, timer=timer, maxAttempt=maxAttempt, runnerPackages=runnerPackages, watcherPackages=watcherPackages, ) for commandDef in taskDefinition["commands"]: description = commandDef["description"] arguments = commandDef["arguments"] cmd = Command( None, description, task, arguments, runnerPackages=runnerPackages, watcherPackages=watcherPackages ) task.commands.append(cmd) # import sys # logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd))) return task ## Resets the lists of elements to create or update in the database. # def resetDbElements(self): self.toCreateElements = [] self.toModifyElements = [] self.toArchiveElements = [] ## Recalculates the max ids of all elements. Generally called after a reload from db. # def recomputeMaxIds(self): self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0 self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0 self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId()) self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0 self.renderNodeMaxId = max(self.renderNodeMaxId, StatDB.getRenderNodesMaxId()) self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0 self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId()) self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0 self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId()) self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0 self.poolShareMaxId = max(self.poolShareMaxId, StatDB.getPoolSharesMaxId()) ## Removes from the dispatchtree the provided element and all its parents and children. # def unregisterElementsFromTree(self, element): # /////////////// Handling of the Task if isinstance(element, Task): del self.tasks[element.id] self.toArchiveElements.append(element) for cmd in element.commands: self.unregisterElementsFromTree(cmd) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskGroup elif isinstance(element, TaskGroup): del self.tasks[element.id] self.toArchiveElements.append(element) for task in element.tasks: self.unregisterElementsFromTree(task) for node in element.nodes.values(): self.unregisterElementsFromTree(node) # /////////////// Handling of the TaskNode elif isinstance(element, TaskNode): # remove the element from the children of the parent if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the FolderNode elif isinstance(element, FolderNode): if element.parent: element.parent.removeChild(element) if element.poolShares: for poolShare in element.poolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) if element.additionnalPoolShares: for poolShare in element.additionnalPoolShares.values(): del poolShare.pool.poolShares[poolShare.node] del self.poolShares[poolShare.id] self.toArchiveElements.append(poolShare) del self.nodes[element.id] self.toArchiveElements.append(element) for dependency in element.dependencies: self.unregisterElementsFromTree(dependency) # /////////////// Handling of the Command elif isinstance(element, Command): del self.commands[element.id] self.toArchiveElements.append(element) ### methods called after interaction with a Task def onTaskCreation(self, task): # logger.info(" -- on task creation: %s" % task) if task.id is None: self.taskMaxId += 1 task.id = self.taskMaxId self.toCreateElements.append(task) else: self.taskMaxId = max(self.taskMaxId, task.id, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId()) self.tasks[task.id] = task def onTaskDestruction(self, task): # logger.info(" -- on task destruction: %s" % task) self.unregisterElementsFromTree(task) def onTaskChange(self, task, field, oldvalue, newvalue): """ Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes However in order to keep track of comments (stored in task's tags[comment] field), we make the following change: - enable task/taskgroups update in DB (cf pulidb.py) - disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs BUT the only field we want to update: "tags" """ if field == "tags": self.toModifyElements.append(task) ### methods called after interaction with a BaseNode def onNodeCreation(self, node): # logger.info(" -- on node creation: %s" % node) if node.id is None: self.nodeMaxId += 1 node.id = self.nodeMaxId self.toCreateElements.append(node) else: self.nodeMaxId = max(self.nodeMaxId, node.id, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId()) if node.parent is None: node.parent = self.root def onNodeDestruction(self, node): # logger.info(" -- on node destruction: %s" % node) del self.nodes[node.id] def onNodeChange(self, node, field, oldvalue, newvalue): # logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) ) # FIXME: do something when nodes are reparented from or to the root node if node.id is not None: self.toModifyElements.append(node) if field == "status" and node.reverseDependencies: self.modifiedNodes.append(node) ### methods called after interaction with a RenderNode def onRenderNodeCreation(self, renderNode): if renderNode.id is None: self.renderNodeMaxId += 1 renderNode.id = self.renderNodeMaxId self.toCreateElements.append(renderNode) else: self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id, StatDB.getRenderNodesMaxId()) self.renderNodes[renderNode.name] = renderNode def onRenderNodeDestruction(self, rendernode): try: del self.renderNodes[rendernode.name] self.toArchiveElements.append(rendernode) except KeyError: # TOFIX: use of class method vs obj method in changeListener might generate a duplicate call logger.warning("RN %s seems to have been deleted already." % rendernode.name) def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue): if field == "performance": self.toModifyElements.append(rendernode) ### methods called after interaction with a Pool def onPoolCreation(self, pool): if pool.id is None: self.poolMaxId += 1 pool.id = self.poolMaxId self.toCreateElements.append(pool) else: self.poolMaxId = max(self.poolMaxId, pool.id, StatDB.getPoolsMaxId()) self.pools[pool.name] = pool def onPoolDestruction(self, pool): del self.pools[pool.name] self.toArchiveElements.append(pool) def onPoolChange(self, pool, field, oldvalue, newvalue): if pool not in self.toModifyElements: self.toModifyElements.append(pool) ### methods called after interaction with a Command def onCommandCreation(self, command): if command.id is None: self.commandMaxId += 1 command.id = self.commandMaxId self.toCreateElements.append(command) else: self.commandMaxId = max(self.commandMaxId, command.id, StatDB.getCommandsMaxId()) self.commands[command.id] = command def onCommandChange(self, command, field, oldvalue, newvalue): self.toModifyElements.append(command) if command.task is not None: for node in command.task.nodes.values(): node.invalidate() ### methods called after interaction with a Pool def onPoolShareCreation(self, poolShare): if poolShare.id is None: self.poolShareMaxId += 1 poolShare.id = self.poolShareMaxId self.toCreateElements.append(poolShare) else: self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id, StatDB.getPoolSharesMaxId()) self.poolShares[poolShare.id] = poolShare
class NodeManager(BaseNodeManager): NODE_CLASS = Node def __init__(self, project): super().__init__() # XXX [!] cycle reference self.project = project self.data = WeakValueDictionary() self.root = None def update_root(self, root_project, root_project_children): self.root = self.new_root_node(root_project, root_project_children) def __setitem__(self, projectid, node): assert self.check_not_exist_node(node) assert projectid == node.projectid self.data[node.projectid] = node assert self.check_exist_node(node) def __delitem__(self, node): assert self.check_exist_node(node) del self.data[node.projectid] assert self.check_not_exist_node(node) def __iter__(self): # how to support lock for iter? just copy dict? return iter(self.data.values()) def __contains__(self, node): if node is None: return False newnode = self.data.get(node.projectid) return node is newnode # ?! def __len__(self): return len(self.data) def __bool__(self): return len(self) != 0 @property def get(self): return self.data.get @property def clear(self): return self.data.clear # TODO: add expend node. (not operation.) def check_exist_node(self, node): original_node = self.get(node.projectid) if original_node is None: raise WFNodeError("{!r} is not exists.".format(node)) elif original_node is not node: raise WFNodeError("{!r} is invalid node.".format(node)) return True def check_not_exist_node(self, node): if node in self: raise WFNodeError("{!r} is already exists.".format(node)) return True def new_void_node(self, uuid=None): return self.NODE_CLASS.from_void(uuid, project=self.project) def new_node_from_json(self, data, parent=None): return self.NODE_CLASS.from_json_with_project(data, parent=parent, project=self.project) def add(self, node, recursion=True): assert recursion is True added_nodes = 0 def register_node(node): nonlocal added_nodes self[node.projectid] = node added_nodes += 1 register_node(node) if recursion: def deep(node): for subnode in node: register_node(subnode) deep(subnode) deep(node) return added_nodes def remove(self, node, recursion=True): assert self.check_exist_node(node) if node.parent is not None: assert self.check_exist_node(node.parent) if node in node.parent: raise WFNodeError("node are still exists in parent node.") removed_nodes = 0 def unregister_node(node): nonlocal removed_nodes del node.parent del self[node] removed_nodes += 1 unregister_node(node) if recursion: def deep(node): if not node: return child_nodes, node.ch = node.ch[:], None for child in child_nodes: unregister_node(node) deep(child) deep(node) return removed_nodes def new_root_node(self, root_project, root_project_children): # XXX [!] project is Project, root_project is root node. ?! if root_project is None: root_project = dict(id=DEFAULT_ROOT_NODE_ID) else: root_project.update(id=DEFAULT_ROOT_NODE_ID) # in shared mode, root will have uuid -(replace)> DEFAULT_ROOT_NODE_ID root_project.update(ch=root_project_children) root = self.new_node_from_json(root_project) self.add(root, recursion=True) return root @property def pretty_print(self): return self.root.pretty_print
class Container(IContainer): def __init__(self, parent: Optional["Container"] = None): super().__init__() self.__uuid = uuid.uuid4() self.__children = WeakValueDictionary() self.__cache = {} self._parent = parent if parent is not None: parent._register_child(self) self._register_instance_indexer = ContainerRegisterInstanceIndexer( self) self._register_factory_indexer = ContainerRegisterFactoryIndexer(self) self._register_type_indexer = ContainerRegisterTypeIndexer(self) self._resolve_indexer = ContainerResolveIndexer(self.resolve_bean) self._try_resolve_indexer = ContainerResolveIndexer( self.try_resolve_bean) from grundzeug.container.plugins import \ ContainerBeanListResolutionPlugin, \ ContainerSingleValueResolutionPlugin, \ ContainerSpecialResolutionPlugin if parent is None: self._plugins = [ ContainerBeanListResolutionPlugin(), ContainerSingleValueResolutionPlugin(), ContainerSpecialResolutionPlugin() ] else: self._plugins = [] self._plugin_storage = WeakKeyDictionary() def __cache_delete(self, key): if key in self.__cache: del self.__cache[key] def __cache_put(self, key, value): self.__cache[key] = value @property def uuid(self): return self.__uuid @property def children(self) -> typing.List["IContainer"]: return list(self.__children.values()) def _register_child(self, container: IContainer): self.__children[container.uuid] = container def add_plugin(self, plugin: ContainerResolutionPlugin) -> IContainer: if self._parent is not None: self._parent.add_plugin(plugin) else: self._plugins.insert(0, plugin) return self @property def plugins(self): if self._parent is not None: return self._parent.plugins else: return self._plugins @property def parent(self): return self._parent def _register(self, key: RegistrationKey, registration: ContainerRegistration): for plugin in self.plugins: if plugin.register(key=key, registration=registration, container=self): self.__cache_delete(key=key) return def _register_instance(self, instance: BeanT, contract: Optional[ContractT] = None, bean_name: Optional[str] = None) -> "IContainer": if contract is None: contract = type(instance) key = RegistrationKey(contract, bean_name) registration = InstanceContainerRegistration(container=self, key=key, instance=instance) self._register(key, registration) return self @property def register_instance(self) -> IContainerRegisterInstanceIndexer: return self._register_instance_indexer def _register_factory( self, contract: ContractT, factory: Callable[[], Any], bean_name: Optional[str] = None, registration_type: Type[ContainerRegistration] = None ) -> "IContainer": key = RegistrationKey(contract, bean_name) if registration_type is None: registration_type = ContainerFactoryContainerRegistration registration = registration_type(container=self, key=key, factory=factory) self._register(key, registration) return self @property def register_factory(self) -> IContainerRegisterFactoryIndexer: return self._register_factory_indexer def _register_type( self, contract: ContractT, clazz: Optional[type] = None, bean_name: Optional[str] = None, registration_type: Type[ContainerRegistration] = None ) -> "IContainer": if clazz is None: clazz = contract self._register_factory(contract=contract, factory=clazz, bean_name=bean_name, registration_type=registration_type) return self @property def register_type(self) -> IContainerRegisterTypeIndexer: return self._register_type_indexer def try_resolve_bean(self, contract: ContractT, bean_name: Optional[str] = None ) -> Union[BeanT, BEAN_NOT_FOUND_TYPE]: if bean_name is None and contract == Container: return self key = RegistrationKey(contract, bean_name) if key in self.__cache: return self.__cache[key].get() plugins = list(self.plugins) states = [ plugin.resolve_bean_create_initial_state(key, self) for plugin in plugins ] current_container = self while current_container is not None: for i, plugin in enumerate(self.plugins): res = plugin.resolve_bean_reduce(key, states[i], self, current_container) if isinstance(res, ReturnMessage): resolver = res.resolver if resolver.is_cacheable: self.__cache_put(key, resolver) return resolver.get() elif isinstance(res, NotFoundMessage): states[i] = res.state elif isinstance(res, ContinueMessage): states[i] = res.state break else: raise NotImplementedError() current_container = current_container.parent for i, plugin in enumerate(self.plugins): res = plugin.resolve_bean_postprocess(key, states[i], self) if isinstance(res, ReturnMessage): resolver = res.resolver if resolver.is_cacheable: self.__cache_put(key, resolver) return resolver.get() elif isinstance(res, NotFoundMessage): pass else: raise NotImplementedError() return BEAN_NOT_FOUND def resolve_bean(self, contract: ContractT, bean_name: Optional[str] = None) -> BeanT: bean = self.try_resolve_bean(contract=contract, bean_name=bean_name) if bean is BEAN_NOT_FOUND: raise ResolutionFailedError( f"Bean not found: contract={contract}, bean_name={bean_name}") return bean @property def resolve(self) -> IContainerResolveIndexer: return self._resolve_indexer @property def try_resolve(self) -> IContainerResolveIndexer: return self._try_resolve_indexer def inject(self, func: FuncT) -> FuncT: from grundzeug.container.di import inject return inject(self, func) def get_kwargs_to_inject(self, func: FuncT) -> typing.Dict[str, Any]: from grundzeug.container.di import get_kwargs_to_inject return get_kwargs_to_inject(self, func) def get_plugin_storage(self, plugin: ContainerResolutionPlugin): if plugin not in self._plugin_storage: self._plugin_storage[plugin] = {} return self._plugin_storage[plugin]
class HeapManager(threading.Thread): """ @summary: This class is intended to manage all dataClay objects in runtime's memory. """ """ Logger """ logger = None def __init__(self, theruntime): """ @postcondition: Constructor of the object called from sub-class @param theruntime: Runtime being managed """ """ Memory objects. This dictionary must contain all objects in runtime memory (client or server), as weakrefs. """ self.inmemory_objects = WeakValueDictionary() threading.Thread.__init__(self) self._finished = threading.Event() """ Runtime being monitorized. Java uses abstract functions to get the field in the proper type (EE or client) due to type-check. Not needed here. """ self.runtime = theruntime self.logger = logging.getLogger(__name__) self.daemon = True self.logger.debug("HEAP MANAGER created.") def get_heap(self): return self.inmemory_objects def shutdown(self): """Stop this thread""" self.logger.debug("HEAP MANAGER shutdown request received.") self._finished.set() def run(self): """ @postcondition: Overrides run function """ gc_check_time_interval_seconds = Configuration.MEMMGMT_CHECK_TIME_INTERVAL / 1000.0 while 1: self.logger.trace("HEAP MANAGER THREAD is awake...") if self._finished.isSet(): break self.run_task() # sleep for interval or until shutdown self.logger.trace("HEAP MANAGER THREAD is going to sleep...") self._finished.wait(gc_check_time_interval_seconds) self.logger.debug("HEAP MANAGER THREAD Finished.") def _add_to_inmemory_map(self, dc_object): """ @postcondition: the object is added to inmemory map @param dc_object: object to add """ oid = dc_object.get_object_id() self.inmemory_objects[oid] = dc_object def remove_from_heap(self, object_id): """ @postcondition: Remove reference from Heap. Even if we remove it from the heap, the object won't be Garbage collected till HeapManager flushes the object and releases it. @param object_id: id of object to remove from heap """ self.inmemory_objects.pop(object_id) def get_from_heap(self, object_id): """ @postcondition: Get from heap. @param object_id: id of object to get from heap @return Object with id provided in heap or None if not found. """ try: obj = self.inmemory_objects[object_id] self.logger.debug("Hit in Heap object %s" % str(object_id)) return obj except KeyError: self.logger.debug("Miss in Heap object %s" % str(object_id)) return None def exists_in_heap(self, object_id): """ @postcondition: Exists from heap. @param object_id: id of object to get from heap @return True if exists. False otherwise. """ try: if self.inmemory_objects[object_id] is None: return False else: return True except KeyError: return False def heap_size(self): """ @postcondition: Get heap size. @return Heap size """ return len(self.inmemory_objects) def count_loaded_objs(self): num_loaded_objs = 0 for obj in self.inmemory_objects.values(): if obj.is_loaded(): num_loaded_objs = num_loaded_objs + 1 return num_loaded_objs @abstractmethod def flush_all(self): pass @abstractmethod def run_task(self): pass def cleanReferencesAndLockers(self): """ @postcondition: Clean references and lockers not being used. """ self.runtime.locker_pool.cleanLockers()
class WorkerCollection(object): def __init__(self, capabilities, parallel_tasks=10, parallel_tasks_per_worker=10, worker_max_idle=300): self.logger = logging.getLogger('root') self.capabilities = capabilities self.parallel_tasks_per_worker = parallel_tasks_per_worker self.worker_max_idle = worker_max_idle self.workers = WeakValueDictionary() self.task_queue = gevent.queue.JoinableQueue(maxsize=parallel_tasks) def register_response_queue(self, response_queue): self.response_queue = response_queue self.logger.info("Registered worker collection for {caps}".format( caps=", ".join(self.capabilities.keys()))) def get_worker(self, NodeID): if NodeID not in self.workers or self.workers[ NodeID].shutdown_in_progress: self.workers[NodeID] = Worker(self, NodeID, self.response_queue, self.parallel_tasks_per_worker, self.worker_max_idle) return self.workers[NodeID] def remove_worker(self, worker): self.workers = { n: w for n, w in self.workers.items() if w is not worker } def shutdown_workers(self): self.task_queue.join() items = list(self.workers.values()) for i in items: i.shutdown() del items def handle_requests_per_worker(self): self.logger.info("Started forwarding requests") while True: anum, capability, timeout, params, zmq_info = self.task_queue.get() try: worker = self.get_worker(params['NodeID']) capability = self.capabilities[capability] try: worker.add_action( capability.action_class(anum, params['NodeID'], zmq_info, timeout, params, **capability.params)) except Exception as e: self.logger.debug(e) dummy_action = FailedAction(anum, params['NodeID'], zmq_info, timeout, params) dummy_action.statusmsg += "\n" + traceback.format_exc() worker.add_action(dummy_action) except KeyError: self.logger.error( "Unknown capability {cap}".format(cap=capability)) finally: del worker, capability
class Entity: def __init__(self, entType, entValue, entField): if isinstance(entField, Field): self.type = entType self.value = entValue self.field = entField self.group = None self.links = WeakValueDictionary() # dict of linked entities self.field.registerEntity(self) # update the entity registry else: raise TypeError("Invalid field argument, field instance expected!") def linkTo(self, eTwo): ''' Linking operation is bi-directional, affects both entities equally.''' # check if entities not already linked if Edge.linkId(self, eTwo) not in self.links.keys(): # update both entities' list of links # create a new edge newlink = Edge(self, eTwo, self.field) self.links[newlink.id] = eTwo eTwo.links[newlink.id] = self # case when the first entity's group is not set if self.group is None: # assuming the second entity has already a group assigned try: eTwo.group.addMember(self) # except the second entity has no group except AttributeError: newGroup = Group(self.field) newGroup.addMember(self) newGroup.addMember(eTwo) # case when the first entity's group is set, but the second entity's is not elif eTwo.group is None: self.group.addMember(eTwo) # case when both entities have groups set and they are different groups elif self.group.name != eTwo.group.name: if self.group.size > eTwo.group.size: # first group wins self.group.annexMembers(eTwo.group) else: # second group wins eTwo.group.annexMembers(self.group) def getLinks(self): ''' Print the list of entities directly linked.''' return self.links.values() def removeLink(self, eTwo): ''' Remove linked entity.''' linkId = Edge.linkId(self, eTwo) self.links.pop(linkId) def __repr__(self): return repr(self.value) def __del__(self): ''' Delete itself from linked entities, and delete links.''' # remove link from linked entity necessary? no because it's a weaklink for linkId in self.links.keys(): self.field.eliminateEdge(linkId) del self
class Boss: def __init__(self, os_window_id, opts, args): self.window_id_map = WeakValueDictionary() self.os_window_map = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd ) set_boss(self) self.current_font_size = opts.font_size set_font_family(opts) self.opts, self.args = opts, args initialize_renderer() startup_session = create_session(opts, args) self.add_os_window(startup_session, os_window_id=os_window_id) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None): dpi_changed = False if os_window_id is None: w, h = initial_window_size(self.opts) if size is None else size cls = wclass or self.args.cls or appname os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls) if startup_id: ctx = init_startup_notification(os_window_id, startup_id) dpi_changed = show_window(os_window_id) if startup_id: end_startup_notification(ctx) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm if dpi_changed: self.on_dpi_change(os_window_id) def list_os_windows(self): for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'tabs': list(tm.list_tabs()), } def match_windows(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) for tm in self.os_window_map.values(): for tab in tm: for window in tab: if window.matches(field, pat): yield window def tab_for_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): field, exp = match.split(':', 1) pat = re.compile(exp) tms = tuple(self.os_window_map.values()) found = False if field in ('title', 'id'): for tm in tms: for tab in tm: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for tm in self.os_window_map.values(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return def _new_os_window(self, args, cwd_from=None): sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from) self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) def new_os_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_os_window(args, cwd_from) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def peer_messages_received(self, messages): import json for msg in messages: msg = json.loads(msg.decode('utf-8')) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) session = create_session(opts, args) self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size(opts), startup_id=startup_id) else: safe_print('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = None if self.opts.allow_remote_control: try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err), 'tb': traceback.format_exc()} else: response = {'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'} if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) glfw_post_empty_event() def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): tm = self.os_window_map.get(os_window_id) if tm is not None: if dpi_changed: if set_dpi_from_os_window(os_window_id): self.on_dpi_change(os_window_id) else: tm.resize() else: tm.resize() def increase_font_size(self): self.change_font_size( min( self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta)) def decrease_font_size(self): self.change_font_size( max( MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta)) def restore_font_size(self): self.change_font_size(self.opts.font_size) def _change_font_size(self, new_size=None): if new_size is not None: self.current_font_size = new_size old_cell_width, old_cell_height = viewport_for_window()[-2:] windows = tuple(filter(None, self.window_id_map.values())) resize_fonts(self.current_font_size) layout_sprite_map() prerender() for window in windows: window.screen.rescale_images(old_cell_width, old_cell_height) window.screen.refresh_sprite_positions() for tm in self.os_window_map.values(): tm.resize() tm.refresh_sprite_positions() glfw_post_empty_event() def change_font_size(self, new_size): if new_size == self.current_font_size: return self._change_font_size(new_size) def on_dpi_change(self, os_window_id): self._change_font_size() @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.opts.keymap, mods, key, scancode) self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple(w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) def display_scrollback(self, data): if self.opts.scrollback_in_new_tab: self.display_scrollback_in_new_tab(data) else: tab = self.active_tab if tab is not None: tab.new_special_window( SpecialWindow( self.opts.scrollback_pager, data, _('History'))) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) old_focus = tab.active_window if not old_focus.destroyed: old_focus.focus_changed(False) tab.active_window.focus_changed(True) def open_url(self, url): if url: open_url(url, self.opts.open_url_with) def open_url_lines(self, lines): self.open_url(''.join(lines)) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() wakeup() self.child_monitor.join() del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_sprite_map() destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: set_clipboard_string(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window def data_for_at(arg): if arg == '@selection': return w.text_for_selection() if arg == '@ansi': return w.buffer_as_ansi() if arg == '@text': return w.buffer_as_text() if args[0].startswith('@'): stdin = data_for_at(args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=special_window, cwd_from=cwd_from) def new_tab(self, *args): self._new_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_tab(args, cwd_from=cwd_from) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: if args: tab.new_special_window(self.args_to_special_window(args, cwd_from=cwd_from)) else: tab.new_window(cwd_from=cwd_from) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window cwd_from = w.child.pid if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def display_scrollback_in_new_tab(self, data): tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=SpecialWindow( self.opts.scrollback_pager, data, _('History')))
class RpcService(object): """ service for one socket """ UID_LEN = 32 def __init__(self, svr, sock, uid, size=None): if 0: self.svr = RpcServer() self.svr = svr #self._pool = Pool(size=size) self.sock = sock if isinstance(svr, RpcClient): self.sock_addr = svr.addr else: self.sock_addr = self.sock.getpeername() self.uid = str(uid) if len(self.uid) != self.UID_LEN: raise ValueError, 'uid length error: len(uid)=%d <> %d' % ( len(uid), self.UID_LEN) self._slock = Semaphore() self._reconnected = None self.reconnect_timeout = RECONNECT_TIMEOUT #self.iter_id = itertools.cycle(xrange(MAX_INDEX)) self._next_id = 0 self._resps = {} self._proxys = WeakValueDictionary() self.stoped = True self.sock_error = False if HEARTBEAT_TIME > 0: self._heart_time = time.time() self._heart_task = spawn(self.heartbeat) self.shells = {} def next_id(self): self._next_id += 1 if self._next_id >= MAX_INDEX: self._next_id = 1 return self._next_id def start(self): if not self.stoped: return self.stoped = False self._recv_task = spawn(self._recver) self._recver_on_error = False #_services_.append(self.sock_addr) def remote_stop(self): #printf('remote_stop:%s', self.sock_addr) self.sock_error = True self.stop() def close(self): if not self.sock: return try: self.sock._sock.close() except socket_error: pass self.sock.close() self.sock = None def stop(self): if self.stoped: return self.stoped = True self._recv_task.kill(block=0) self._recv_task = None if 1 and not self.sock_error: try: #printf('remote_stop:%s', self.sock_addr) self.call('', 'remote_stop', tuple(), None, no_result=True) sleep(0.01) except: pass self.svr.svc_stop(self) if getattr(self, '_heart_task', None): self._heart_task.kill(block=False) self._heart_task = None try: self._stop_resps() self._stop_proxys() finally: self.close() #_services_.append('-%s' % str(self.sock_addr)) def _stop_resps(self): error = RpcRuntimeError('service stoped') for k, v in self._resps.iteritems(): v.set_exception(error) self._resps.clear() def _stop_proxys(self): if not len(self._proxys): return proxys = self._proxys.values() self._proxys.clear() for p in proxys: p.on_close() ## def _sender(self): ## running = True ## _send = self.sock.sendall ## try: ## for data in self._send_queue: ## _send('%s%s' %(pack('I', len(data)), data)) ## except GreenletExit: ## pass def _recver(self): """ 接收处理数据 """ recv_func = self.sock.recv def _read(c): d = recv_func(c) if d: return d if self.stoped: raise GreenletExit self._recver_on_error = True self._on_socket_error(None) self._recver_on_error = False return None try: sio = StringIO() while not self.stoped: dlen = 4 d = '' while dlen > 0: data = _read(dlen) if data is None: continue d += data dlen -= len(data) dlen = unpack('I', d)[0] #rs = [] sio.seek(0) sio.truncate() while dlen > 0: data = _read(dlen) if data is None: continue #rs.append(data) sio.write(data) dlen -= len(data) #spawn(self._handle, loads(''.join(rs))) sio.seek(0) self._handle(load(sio)) #self._pool.spawn(self._handle, loads(''.join(rs))) except GreenletExit: pass except Exception as err: printf('[RpcService._recver]%s', err) finally: self.stop() def _on_socket_error(self, err): if self.stoped or self.reconnect_timeout <= 0: self.sock_error = True self.stop() return def _reconnect(): #尝试重连或等待重连 while not self.stoped: try: self.svr.reconnect() break except socket_error: pass sleep(0.5) if self._reconnected is None: self._reconnected = AsyncResult() printf('socket error:%s, RpcService try reconnect', err) self.send = self.send_wait if hasattr(self.svr, 'reconnect'): #RpcClient.reconnect spawn(_reconnect) self._wait_reconnect() def _wait_reconnect(self): _reconnected = self._reconnected try: _reconnected.get(timeout=self.reconnect_timeout) except Timeout: pass if not _reconnected.successful(): self.stop() if self.sock_error or _reconnected.exception is None: return self.sock_error = True raise _reconnected.exception def reconnect(self, sock): ## if not self._recver_on_error: ## self._recv_task.kill(exception=socket_error('reconnect')) self.sock = sock self.send = self.send_imme if self._reconnected is not None: self._reconnected.set(True) self._reconnected = None def send_imme(self, *args): data = dumps(args) with self._slock: try: self.sock.sendall('%s%s' % (pack('I', len(data)), data)) except socket_error as err: self._on_socket_error(err) #重新发送 self.sock.sendall('%s%s' % (pack('I', len(data)), data)) ## self._send_queue.put(dumps(args)) def send_wait(self, *args): if self._reconnected is not None: self._wait_reconnect() self.send_imme(*args) send = send_imme def _read_response(self, index, timeout): rs = AsyncResult() self._resps[index] = rs resp = rs.wait(timeout) self._resps.pop(index, None) if not rs.successful(): error = rs.exception if error is None: error = Timeout raise error return resp def _reg_obj(self, obj): if hasattr(obj, 'proxy_pack'): return obj.proxy_pack(), False if isinstance(obj, RpcProxy): return obj._id, False if hasattr(obj, '_rpc_proxy_'): return obj._rpc_proxy_(), True return self.svr.register(obj), False def call(self, obj_id, name, args, kw, no_result=False, timeout=CALL_TIMEORUT, pickle=False, proxy=False): dtype = RT_REQUEST if proxy: objs = args[0] #first arg is proxy(str, RpcProxy or list) if isinstance(objs, (tuple, list)): obj_ids = [] for o in objs: obj, is_pickle = self._reg_obj(o) pickle = pickle or is_pickle obj_ids.append(obj) else: obj, is_pickle = self._reg_obj(objs) pickle = pickle or is_pickle obj_ids = obj args = (obj_ids, ) + args[1:] dtype |= DT_PROXY if pickle: dtype |= DT_PICKLE argkw = pickle_dumps((args, kw), PICKLE_PROTOCOL) else: argkw = dumps((args, kw)) if len(argkw) >= ZIP_LENGTH: dtype |= DT_ZIP argkw = zlib.compress(argkw, ZIP_LEVEL) if no_result: dtype |= ST_NO_RESULT index = self.next_id() #iter_id.next() self.send(dtype, obj_id, index, name, argkw) if no_result: return result = self._read_response(index, timeout) return result def _handle_request(self, parts): dtype, obj_id, index, name, argkw = parts try: obj = self.get_export(obj_id) if obj is None: raise RpcExportNoFound, obj_id func = getattr(obj, name) if not callable(func): raise RpcFuncNoFound, name if dtype & DT_ZIP: argkw = zlib.decompress(argkw) if dtype & DT_PICKLE: args, kw = pickle_loads(argkw) else: args, kw = loads(argkw) if dtype & DT_PROXY: export_ids = args[0] if isinstance(export_ids, (tuple, list)): proxys = [] for e in export_ids: proxys.append(self.get_proxy(e)) else: proxys = self.get_proxy(export_ids) args = (proxys, ) + tuple(args[1:]) if getattr(func, "_block_", True): spawn(self._handle_request_call, func, args, kw, dtype, index, obj_id, name, argkw) else: self._handle_request_call(func, args, kw, dtype, index, obj_id, name, argkw) except Exception as e: log_except('export(%s).%s(%s)', obj_id, name, repr(argkw)) if dtype & ST_NO_RESULT or self.svr.stoped: return self.send(RT_EXCEPTION, index, str(e)) def _handle_request_call(self, func, args, kw, dtype, index, obj_id, name, argkw): try: let = getcurrent() setattr(let, _service_name_, self) if args is None: rs = func() else: rs = func(*args, **kw) if kw is not None else func(*args) if dtype & ST_NO_RESULT: return if not getattr(func, '_rpc_pickle_result_', False): self.send(RT_RESPONSE, index, dumps(rs)) else: self.send(RT_RESPONSE | DT_PICKLE, index, pickle_dumps(rs, PICKLE_PROTOCOL)) except Exception as e: log_except('export(%s).%s(%s)', obj_id, name, repr(argkw)) if dtype & ST_NO_RESULT or self.svr.stoped: return self.send(RT_EXCEPTION, index, str(e)) def _handle_response(self, parts): dtype, index, argkw = parts try: rs = self._resps.pop(index) if dtype & DT_PICKLE: result = pickle_loads(argkw) else: result = loads(argkw) rs.set(result) except KeyError: pass def _handle_exception(self, parts): RT_EXCEPTION, index, error = parts #try: # error = pickle_loads(error) #except: error = RpcCallError(str(error)) try: rs = self._resps.pop(index) rs.set_exception(error) except KeyError: pass def _handle(self, parts): #parts = (parts[0], ) + loads(parts[1]) if len(parts) ==2 else loads(parts[0]) rt = parts[0] & RT_MARK if rt == RT_REQUEST: self._handle_request(parts) elif rt == RT_RESPONSE: self._handle_response(parts) elif rt == RT_EXCEPTION: self._handle_exception(parts) elif rt == RT_HEARTBEAT: self._heart_time = time.time() else: raise ValueError('unknown data:%s' % str(rt)) def heartbeat(self): beat = RT_HEARTBEAT btime = HEARTBEAT_TIME check_times = HEARTBEAT_TIME * max(3, RECONNECT_TIMEOUT) try: while not self.stoped: self.send(beat) sleep(btime) if (self._heart_time + check_times) < time.time(): printf('heartbeat timeout!!!!!!!!') self.sock_error = True break finally: self.stop() @classmethod def handshake_svr(cls, sock): uid = sock.recv(cls.UID_LEN) return uid def handshake_cli(self): self.sock.sendall(self.uid) #######remote call############## def get_export(self, export_id): """ get export obj by export_name """ if not export_id: return self return self.svr.get_export(export_id) def get_proxy(self, export_id, proxy_cls=None): """ remote call: get export obj by id """ if isinstance(export_id, RpcProxy): return export_id if isinstance(export_id, (tuple, list)): export_cls = PROXYS[export_id[0]] return export_cls.proxy_unpack(export_id, svc=self) if proxy_cls in (None, RpcProxy): try: return self._proxys[export_id] except KeyError: proxy_cls = RpcProxy proxy = proxy_cls(export_id, svc=self) self.reg_proxy(export_id, proxy) return proxy else: p = proxy_cls(export_id, svc=self) self.reg_proxy(id(p), p) return p def reg_proxy(self, key, proxy): self._proxys[key] = proxy def stop_shell(self, shell_id): shell = self.shells.pop(shell_id, None) if not shell: return shell.stop() def start_shell(self, console_proxy, pre_prompt): from rpc_shell import RpcShell if self.svr.access and not self.svr.access.access_shell(self): printf('[rpc]shell deny:%s', self.sock_addr) return 0 printf('[rpc]shell start:%s', self.sock_addr) shell = RpcShell(self, console_proxy, pre_prompt=pre_prompt) shell.start() #shell.stop remove shell from self.shells shell_id = id(shell) self.shells[shell_id] = shell return shell_id def stop_console(self, shell_id): self.call('', 'stop_shell', (shell_id, ), None, no_result=True, timeout=20) def start_console(self, pre_prompt='', shell=None): from rpc_shell import RpcLocalConsole, RpcProxyConsole console = RpcLocalConsole(self) if shell is None else RpcProxyConsole( self, shell) shell_id = self.call('', 'start_shell', (console, pre_prompt), None, proxy=True, timeout=20) try: console.wait(shell_id) finally: pass def execute(self, func, args, kw, **others): return self.svr.execute(func, args, kw) def valid_proxy(self, export_id): return self.get_export(export_id) != None
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] Let's test the throttling mechanism. Configure a 2 seconds delay on channel A, end enqueue two jobs. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4:throttle=2') >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') We have only one job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A1>] >>> cm.get_wakeup_time() 102 We have no job to run, because of the throttle. >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.get_wakeup_time() 102 2 seconds later, we can run the other job (even though the first one is still running, because we have enough capacity). >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob A2>] >>> cm.get_wakeup_time() 104 Let's test throttling in combination with a queue reaching full capacity. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,T:2:throttle=2') >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'T', 'T3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob T1>] >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob T2>] Channel is now full, so no job to run even though throttling delay is over. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.get_wakeup_time() # no wakeup time, since queue is full 0 >>> pp(list(cm.get_jobs_to_run(now=104))) [] >>> cm.get_wakeup_time() # queue is still full 0 >>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob T3>] >>> cm.get_wakeup_time() # queue is full 0 >>> cm.notify(db, 'T', 'T2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 107 Test wakeup time behaviour in presence of eta. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,E:1') >>> cm.notify(db, 'E', 'E1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'E', 'E3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob E1>] >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'E', 'E1', 1, 0, 10, 105, 'pending') >>> cm.get_wakeup_time() # wakeup at eta 105 >>> pp(list(cm.get_jobs_to_run(now=102))) # but there is capacity [<ChannelJob E2>] >>> pp(list(cm.get_jobs_to_run(now=106))) # no capacity anymore [] >>> cm.get_wakeup_time() # no timed wakeup because no capacity 0 >>> cm.notify(db, 'E', 'E2', 1, 0, 10, None, 'done') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=107))) # no capacity anymore [<ChannelJob E1>] >>> cm.get_wakeup_time() 0 Test wakeup time behaviour in a sequential queue. >>> cm = ChannelManager() >>> cm.simple_configure('root:4,S:1:sequential') >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob S1>] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'failed') >>> pp(list(cm.get_jobs_to_run(now=101))) [] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'pending') >>> pp(list(cm.get_jobs_to_run(now=102))) [] No wakeup time because due to eta, because the sequential queue is waiting for a failed job. >>> cm.get_wakeup_time() 0 >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending') >>> cm.get_wakeup_time() 105 >>> pp(list(cm.get_jobs_to_run(now=102))) [<ChannelJob S1>] >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'done') At this stage, we have S2 with an eta of 105 and since the queue is sequential, we wait for it. >>> pp(list(cm.get_jobs_to_run(now=103))) [] >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S2>] >>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [<ChannelJob S3>] >>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=105))) [] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...] If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] It ignores whitespace around values, and drops empty entries which would be generated by trailing commas, or commented lines on the Odoo config file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4, ... , ... foo bar:1: k=va lue, ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}] It's also possible to replace commas with line breaks, which is more readable if the channel configuration comes from the odoo config file. >>> pp(ChannelManager.parse_simple_config(''' ... root : 4 ... foo bar:1: k=va lue ... baz ... ''')) [{'capacity': 4, 'name': 'root'}, {'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}, {'capacity': 1, 'name': 'baz'}] """ res = [] config_string = config_string.replace("\n", ",") for channel_config_string in split_strip(config_string, ','): if not channel_config_string: # ignore empty entries (commented lines, trailing commas) continue config = {} config_items = split_strip(channel_config_string, ':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = split_strip(config_item, '=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError('Invalid channel config %s: ' 'incorrect config item %s' % (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' % (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2,seq:1:sequential') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root').sequential False >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('root.autosub.sub').sequential False >>> cm.get_channel_by_name('autosub.sub').capacity 2 >>> cm.get_channel_by_name('seq').capacity 1 >>> cm.get_channel_by_name('seq').sequential True """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) _logger.info("Configured channel: %s", channel) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning( 'unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now) def get_wakeup_time(self): return self._root_channel.get_wakeup_time()
class ChannelManager(object): """ High level interface for channels This class handles: * configuration of channels * high level api to create and remove jobs (notify, remove_job, remove_db) * get jobs to run Here is how the runner will use it. Let's create a channel manager and configure it. >>> from pprint import pprint as pp >>> cm = ChannelManager() >>> cm.simple_configure('root:4,A:4,B:1') >>> db = 'db' Add a few jobs in channel A with priority 10 >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending') >>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending') Add a few jobs in channel B with priority 5 >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending') >>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending') We must now run one job from queue B which has a capacity of 1 and 3 jobs from queue A so the root channel capacity of 4 is filled. >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>] Job A2 is done. Next job to run is A5, even if we have higher priority job in channel B, because channel B has a capacity of 1. >>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A4>] Job B1 is done. Next job to run is B2 because it has higher priority. >>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob B2>] Let's say A1 is done and A6 gets a higher priority. A6 will run next. >>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done') >>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending') >>> pp(list(cm.get_jobs_to_run(now=100))) [<ChannelJob A6>] """ def __init__(self): self._jobs_by_uuid = WeakValueDictionary() self._root_channel = Channel(name='root', parent=None, capacity=1) self._channels_by_name = WeakValueDictionary(root=self._root_channel) @classmethod def parse_simple_config(cls, config_string): """Parse a simple channels configuration string. The general form is as follow: channel(.subchannel)*(:capacity(:key(=value)?)*)?,... If capacity is absent, it defaults to 1. If a key is present without value, it gets True as value. When declaring subchannels, the root channel may be omitted (ie sub:4 is the same as root.sub:4). Returns a list of channel configuration dictionaries. >>> from pprint import pprint as pp >>> pp(ChannelManager.parse_simple_config('root:4')) [{'capacity': 4, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}] >>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:' ... 'sequential:k=v')) [{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}] >>> pp(ChannelManager.parse_simple_config('root')) [{'capacity': 1, 'name': 'root'}] >>> pp(ChannelManager.parse_simple_config('sub:2')) [{'capacity': 2, 'name': 'sub'}] """ res = [] for channel_config_string in config_string.split(','): config = {} config_items = channel_config_string.split(':') name = config_items[0] if not name: raise ValueError('Invalid channel config %s: ' 'missing channel name' % config_string) config['name'] = name if len(config_items) > 1: capacity = config_items[1] try: config['capacity'] = int(capacity) except: raise ValueError('Invalid channel config %s: ' 'invalid capacity %s' % (config_string, capacity)) for config_item in config_items[2:]: kv = config_item.split('=') if len(kv) == 1: k, v = kv[0], True elif len(kv) == 2: k, v = kv else: raise ValueError( 'Invalid channel config %s: ', 'incorrect config item %s' (config_string, config_item)) if k in config: raise ValueError('Invalid channel config %s: ' 'duplicate key %s' (config_string, k)) config[k] = v else: config['capacity'] = 1 res.append(config) return res def simple_configure(self, config_string): """Configure the channel manager from a simple configuration string >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root') >>> c.capacity 1 >>> cm.simple_configure('root:4,autosub.sub:2') >>> cm.get_channel_by_name('root').capacity 4 >>> cm.get_channel_by_name('root.autosub').capacity >>> cm.get_channel_by_name('root.autosub.sub').capacity 2 >>> cm.get_channel_by_name('autosub.sub').capacity 2 """ for config in ChannelManager.parse_simple_config(config_string): self.get_channel_from_config(config) def get_channel_from_config(self, config): """Return a Channel object from a parsed configuration. If the channel does not exist it is created. The configuration is applied on the channel before returning it. If some of the parent channels are missing when creating a subchannel, the parent channels are auto created with an infinite capacity (except for the root channel, which defaults to a capacity of 1 when not configured explicity). """ channel = self.get_channel_by_name(config['name'], autocreate=True) channel.configure(config) return channel def get_channel_by_name(self, channel_name, autocreate=False): """Return a Channel object by its name. If it does not exist and autocreate is True, it is created with a default configuration and inserted in the Channels structure. If autocreate is False and the channel does not exist, an exception is raised. >>> cm = ChannelManager() >>> c = cm.get_channel_by_name('root', autocreate=False) >>> c.name 'root' >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('autosub.sub', autocreate=True) >>> c.name 'sub' >>> c.fullname 'root.autosub.sub' >>> c = cm.get_channel_by_name(None) >>> c.fullname 'root' >>> c = cm.get_channel_by_name('root.sub') >>> c.fullname 'root.sub' >>> c = cm.get_channel_by_name('sub') >>> c.fullname 'root.sub' """ if not channel_name or channel_name == self._root_channel.name: return self._root_channel if not channel_name.startswith(self._root_channel.name + '.'): channel_name = self._root_channel.name + '.' + channel_name if channel_name in self._channels_by_name: return self._channels_by_name[channel_name] if not autocreate: raise ChannelNotFound('Channel %s not found' % channel_name) parent = self._root_channel for subchannel_name in channel_name.split('.')[1:]: subchannel = parent.get_subchannel_by_name(subchannel_name) if not subchannel: subchannel = Channel(subchannel_name, parent, capacity=None) self._channels_by_name[subchannel.fullname] = subchannel parent = subchannel return parent def notify(self, db_name, channel_name, uuid, seq, date_created, priority, eta, state): try: channel = self.get_channel_by_name(channel_name) except ChannelNotFound: _logger.warning( 'unknown channel %s, ' 'using root channel for job %s', channel_name, uuid) channel = self._root_channel job = self._jobs_by_uuid.get(uuid) if job: # db_name is invariant assert job.db_name == db_name # date_created is invariant assert job.date_created == date_created # if one of the job properties that influence # scheduling order has changed, we remove the job # from the queues and create a new job object if (seq != job.seq or priority != job.priority or eta != job.eta or channel != job.channel): _logger.debug("job %s properties changed, rescheduling it", uuid) self.remove_job(uuid) job = None if not job: job = ChannelJob(db_name, channel, uuid, seq, date_created, priority, eta) self._jobs_by_uuid[uuid] = job # state transitions if not state or state == DONE: job.channel.set_done(job) elif state == PENDING: job.channel.set_pending(job) elif state in (ENQUEUED, STARTED): job.channel.set_running(job) elif state == FAILED: job.channel.set_failed(job) else: _logger.error("unexpected state %s for job %s", state, job) def remove_job(self, uuid): job = self._jobs_by_uuid.get(uuid) if job: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def remove_db(self, db_name): for job in self._jobs_by_uuid.values(): if job.db_name == db_name: job.channel.remove(job) del self._jobs_by_uuid[job.uuid] def get_jobs_to_run(self, now): return self._root_channel.get_jobs_to_run(now)
class Boss: def __init__(self, os_window_id, opts, args, cached_values, new_os_window_trigger): set_layout_options(opts) self.clipboard_buffers = {} self.update_check_process = None self.window_id_map = WeakValueDictionary() self.startup_colors = { k: opts[k] for k in opts if isinstance(opts[k], Color) } self.startup_cursor_text_color = opts.cursor_text_color self.pending_sequences = None self.cached_values = cached_values self.os_window_map = {} self.os_window_death_actions = {} self.cursor_blinking = True self.shutting_down = False talk_fd = getattr(single_instance, 'socket', None) talk_fd = -1 if talk_fd is None else talk_fd.fileno() listen_fd = -1 if opts.allow_remote_control and args.listen_on: listen_fd = listen_on(args.listen_on) self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, talk_fd, listen_fd) set_boss(self) self.opts, self.args = opts, args startup_sessions = create_sessions( opts, args, default_session=opts.startup_session) self.keymap = self.opts.keymap.copy() if new_os_window_trigger is not None: self.keymap.pop(new_os_window_trigger, None) for startup_session in startup_sessions: self.add_os_window(startup_session, os_window_id=os_window_id) os_window_id = None if args.start_as != 'normal': if args.start_as == 'fullscreen': self.toggle_fullscreen() else: change_os_window_state(args.start_as) if is_macos: from .fast_data_types import cocoa_set_notification_activated_callback cocoa_set_notification_activated_callback( self.notification_activated) def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, opts_for_size=None, startup_id=None): if os_window_id is None: opts_for_size = opts_for_size or startup_session.os_window_size or self.opts cls = wclass or self.args.cls or appname with startup_notification_handler( do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback: os_window_id = create_os_window( initial_window_size_func(opts_for_size, self.cached_values), pre_show_callback, appname, wname or self.args.name or cls, cls) tm = TabManager(os_window_id, self.opts, self.args, startup_session) self.os_window_map[os_window_id] = tm return os_window_id def list_os_windows(self): with cached_process_data(): active_tab, active_window = self.active_tab, self.active_window active_tab_manager = self.active_tab_manager for os_window_id, tm in self.os_window_map.items(): yield { 'id': os_window_id, 'is_focused': tm is active_tab_manager, 'tabs': list(tm.list_tabs(active_tab, active_window)), } @property def all_tab_managers(self): yield from self.os_window_map.values() @property def all_tabs(self): for tm in self.all_tab_managers: yield from tm @property def all_windows(self): for tab in self.all_tabs: yield from tab def match_windows(self, match): try: field, exp = match.split(':', 1) except ValueError: return if field == 'num': tab = self.active_tab if tab is not None: try: w = tab.get_nth_window(int(exp)) except Exception: return if w is not None: yield w return if field == 'env': kp, vp = exp.partition('=')[::2] if vp: pat = tuple(map(re.compile, (kp, vp))) else: pat = re.compile(kp), None else: pat = re.compile(exp) for window in self.all_windows: if window.matches(field, pat): yield window def tab_for_window(self, window): for tab in self.all_tabs: for w in tab: if w.id == window.id: return tab def match_tabs(self, match): try: field, exp = match.split(':', 1) except ValueError: return pat = re.compile(exp) found = False if field in ('title', 'id'): for tab in self.all_tabs: if tab.matches(field, pat): yield tab found = True if not found: tabs = {self.tab_for_window(w) for w in self.match_windows(match)} for tab in tabs: if tab: yield tab def set_active_window(self, window): for os_window_id, tm in self.os_window_map.items(): for tab in tm: for w in tab: if w.id == window.id: if tab is not self.active_tab: tm.set_active_tab(tab) tab.set_active_window(w) return os_window_id def _new_os_window(self, args, cwd_from=None): if isinstance(args, SpecialWindowInstance): sw = args else: sw = self.args_to_special_window(args, cwd_from) if args else None startup_session = next( create_sessions(self.opts, special_window=sw, cwd_from=cwd_from)) return self.add_os_window(startup_session) def new_os_window(self, *args): self._new_os_window(args) @property def active_window_for_cwd(self): w = self.active_window if w is not None and w.overlay_for is not None and w.overlay_for in self.window_id_map: w = self.window_id_map[w.overlay_for] return w def new_os_window_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._new_os_window(args, cwd_from) def new_os_window_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_os_window(special_window) def add_child(self, window): self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen) self.window_id_map[window.id] = window def _handle_remote_command(self, cmd, window=None): response = None if self.opts.allow_remote_control or getattr( window, 'allow_remote_control', False): try: response = handle_cmd(self, window, cmd) except Exception as err: import traceback response = {'ok': False, 'error': str(err)} if not getattr(err, 'hide_traceback', False): response['tb'] = traceback.format_exc() else: response = { 'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf' } return response def peer_message_received(self, msg): msg = msg.decode('utf-8') cmd_prefix = '\x1bP@kitty-cmd' if msg.startswith(cmd_prefix): cmd = msg[len(cmd_prefix):-2] response = self._handle_remote_command(cmd) if response is not None: response = (cmd_prefix + json.dumps(response) + '\x1b\\').encode('utf-8') return response else: msg = json.loads(msg) if isinstance(msg, dict) and msg.get('cmd') == 'new_instance': startup_id = msg.get('startup_id') args, rest = parse_args(msg['args'][1:]) args.args = rest opts = create_opts(args) if not os.path.isabs(args.directory): args.directory = os.path.join(msg['cwd'], args.directory) for session in create_sessions(opts, args, respect_cwd=True): os_window_id = self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id) if msg.get('notify_on_os_window_death'): self.os_window_death_actions[os_window_id] = partial( self.notify_on_os_window_death, msg['notify_on_os_window_death']) else: log_error('Unknown message received from peer, ignoring') def handle_remote_cmd(self, cmd, window=None): response = self._handle_remote_command(cmd, window) if response is not None: if window is not None: window.send_cmd_response(response) def on_child_death(self, window_id): window = self.window_id_map.pop(window_id, None) if window is None: return if window.action_on_close: try: window.action_on_close(window) except Exception: import traceback traceback.print_exc() os_window_id = window.os_window_id window.destroy() tm = self.os_window_map.get(os_window_id) if tm is None: return for tab in tm: if window in tab: break else: return tab.remove_window(window) if len(tab) == 0: tm.remove(tab) tab.destroy() if len(tm) == 0: if not self.shutting_down: mark_os_window_for_close(os_window_id) def close_window(self, window=None): if window is None: window = self.active_window self.child_monitor.mark_for_close(window.id) def close_tab(self, tab=None): if tab is None: tab = self.active_tab for window in tab: self.close_window(window) def toggle_fullscreen(self): toggle_fullscreen() def toggle_maximized(self): toggle_maximized() def start(self): if not getattr(self, 'io_thread_started', False): self.child_monitor.start() self.io_thread_started = True if self.opts.update_check_interval > 0 and not hasattr( self, 'update_check_started'): from .update_check import run_update_check run_update_check(self.opts.update_check_interval * 60 * 60) self.update_check_started = True def activate_tab_at(self, os_window_id, x): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.activate_tab_at(x) def on_window_resize(self, os_window_id, w, h, dpi_changed): if dpi_changed: self.on_dpi_change(os_window_id) else: tm = self.os_window_map.get(os_window_id) if tm is not None: tm.resize() def clear_terminal(self, action, only_active): if only_active: windows = [] w = self.active_window if w is not None: windows.append(w) else: windows = self.all_windows reset = action == 'reset' how = 3 if action == 'scrollback' else 2 for w in windows: if action == 'scroll': w.screen.scroll_until_cursor() continue w.screen.cursor.x = w.screen.cursor.y = 0 if reset: w.screen.reset() else: w.screen.erase_in_display(how, False) def increase_font_size(self): # legacy cfs = global_font_size() self.set_font_size(min(self.opts.font_size * 5, cfs + 2.0)) def decrease_font_size(self): # legacy cfs = global_font_size() self.set_font_size(max(MINIMUM_FONT_SIZE, cfs - 2.0)) def restore_font_size(self): # legacy self.set_font_size(self.opts.font_size) def set_font_size(self, new_size): # legacy self.change_font_size(True, None, new_size) def change_font_size(self, all_windows, increment_operation, amt): def calc_new_size(old_size): new_size = old_size if amt == 0: new_size = self.opts.font_size else: if increment_operation: new_size += (1 if increment_operation == '+' else -1) * amt else: new_size = amt new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5)) return new_size if all_windows: current_global_size = global_font_size() new_size = calc_new_size(current_global_size) if new_size != current_global_size: global_font_size(new_size) os_windows = tuple(self.os_window_map.keys()) else: os_windows = [] w = self.active_window if w is not None: os_windows.append(w.os_window_id) if os_windows: final_windows = {} for wid in os_windows: current_size = os_window_font_size(wid) if current_size: new_size = calc_new_size(current_size) if new_size != current_size: final_windows[wid] = new_size if final_windows: self._change_font_size(final_windows) def _change_font_size(self, sz_map): for os_window_id, sz in sz_map.items(): tm = self.os_window_map.get(os_window_id) if tm is not None: os_window_font_size(os_window_id, sz) tm.resize() def on_dpi_change(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: sz = os_window_font_size(os_window_id) if sz: os_window_font_size(os_window_id, sz, True) tm.resize() def _set_os_window_background_opacity(self, os_window_id, opacity): change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0))) def set_background_opacity(self, opacity): window = self.active_window if window is None or not opacity: return if not self.opts.dynamic_background_opacity: return self.show_error( _('Cannot change background opacity'), _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity' )) os_window_id = window.os_window_id if opacity[0] in '+-': old_opacity = background_opacity_of(os_window_id) if old_opacity is None: return opacity = old_opacity + float(opacity) elif opacity == 'default': opacity = self.opts.background_opacity else: opacity = float(opacity) self._set_os_window_background_opacity(os_window_id, opacity) @property def active_tab_manager(self): os_window_id = current_os_window() return self.os_window_map.get(os_window_id) @property def active_tab(self): tm = self.active_tab_manager if tm is not None: return tm.active_tab @property def active_window(self): t = self.active_tab if t is not None: return t.active_window def dispatch_special_key(self, key, scancode, action, mods): # Handles shortcuts, return True if the key was consumed key_action = get_shortcut(self.keymap, mods, key, scancode) if key_action is None: sequences = get_shortcut(self.opts.sequence_map, mods, key, scancode) if sequences: self.pending_sequences = sequences set_in_sequence_mode(True) return True else: self.current_key_press_info = key, scancode, action, mods return self.dispatch_action(key_action) def process_sequence(self, key, scancode, action, mods): if not self.pending_sequences: set_in_sequence_mode(False) remaining = {} matched_action = None for seq, key_action in self.pending_sequences.items(): if shortcut_matches(seq[0], mods, key, scancode): seq = seq[1:] if seq: remaining[seq] = key_action else: matched_action = key_action if remaining: self.pending_sequences = remaining else: self.pending_sequences = None set_in_sequence_mode(False) if matched_action is not None: self.dispatch_action(matched_action) def start_resizing_window(self): w = self.active_window if w is None: return overlay_window = self._run_kitten( 'resize_window', args=[ '--horizontal-increment={}'.format( self.opts.window_resize_step_cells), '--vertical-increment={}'.format( self.opts.window_resize_step_lines) ]) if overlay_window is not None: overlay_window.allow_remote_control = True def resize_layout_window(self, window, increment, is_horizontal, reset=False): tab = window.tabref() if tab is None or not increment: return False if reset: return tab.reset_window_sizes() return tab.resize_window_by(window.id, increment, is_horizontal) def default_bg_changed_for(self, window_id): w = self.window_id_map.get(window_id) if w is not None: tm = self.os_window_map.get(w.os_window_id) if tm is not None: tm.update_tab_bar_data() tm.mark_tab_bar_dirty() t = tm.tab_for_id(w.tab_id) if t is not None: t.relayout_borders() def dispatch_action(self, key_action): if key_action is not None: f = getattr(self, key_action.func, None) if f is not None: if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) passthrough = f(*key_action.args) if passthrough is not True: return True tab = self.active_tab if tab is None: return False window = self.active_window if window is None: return False if key_action is not None: f = getattr(tab, key_action.func, getattr(window, key_action.func, None)) if f is not None: passthrough = f(*key_action.args) if self.args.debug_keyboard: print('Keypress matched action:', func_name(f)) if passthrough is not True: return True return False def combine(self, *actions): for key_action in actions: self.dispatch_action(key_action) def on_focus(self, os_window_id, focused): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.focus_changed(focused) tm.mark_tab_bar_dirty() def update_tab_bar_data(self, os_window_id): tm = self.os_window_map.get(os_window_id) if tm is not None: tm.update_tab_bar_data() def on_drop(self, os_window_id, paths): tm = self.os_window_map.get(os_window_id) if tm is not None: w = tm.active_window if w is not None: w.paste('\n'.join(paths)) def on_os_window_closed(self, os_window_id, viewport_width, viewport_height): self.cached_values['window-size'] = viewport_width, viewport_height tm = self.os_window_map.pop(os_window_id, None) if tm is not None: tm.destroy() for window_id in tuple( w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id): self.window_id_map.pop(window_id, None) action = self.os_window_death_actions.pop(os_window_id, None) if action is not None: action() def notify_on_os_window_death(self, address): import socket s = socket.socket(family=socket.AF_UNIX) with suppress(Exception): s.connect(address) s.sendall(b'c') with suppress(EnvironmentError): s.shutdown(socket.SHUT_RDWR) s.close() def display_scrollback(self, window, data, cmd): tab = self.active_tab if tab is not None and window.overlay_for is None: tab.new_special_window( SpecialWindow(cmd, data, _('History'), overlay_for=window.id)) def edit_config_file(self, *a): confpath = prepare_config_file_for_editing() # On macOS vim fails to handle SIGWINCH if it occurs early, so add a # small delay. cmd = [ kitty_exe(), '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])' ] + get_editor() + [confpath] self.new_os_window(*cmd) def get_output(self, source_window, num_lines=1): output = '' s = source_window.screen if num_lines is None: num_lines = s.lines for i in range(min(num_lines, s.lines)): output += str(s.linebuf.line(i)) return output def _run_kitten(self, kitten, args=(), input_data=None, window=None): orig_args, args = list(args), list(args) from kittens.runner import create_kitten_handler end_kitten = create_kitten_handler(kitten, orig_args) if window is None: w = self.active_window tab = self.active_tab else: w = window tab = w.tabref() if end_kitten.no_ui: end_kitten(None, getattr(w, 'id', None), self) return if w is not None and tab is not None and w.overlay_for is None: args[0:0] = [config_dir, kitten] if input_data is None: type_of_input = end_kitten.type_of_input if type_of_input in ('text', 'history', 'ansi', 'ansi-history', 'screen', 'screen-history', 'screen-ansi', 'screen-ansi-history'): data = w.as_text(as_ansi='ansi' in type_of_input, add_history='history' in type_of_input, add_wrap_markers='screen' in type_of_input).encode('utf-8') elif type_of_input is None: data = None else: raise ValueError( 'Unknown type_of_input: {}'.format(type_of_input)) else: data = input_data if isinstance(data, str): data = data.encode('utf-8') copts = { k: self.opts[k] for k in ('select_by_word_characters', 'open_url_with') } overlay_window = tab.new_special_window( SpecialWindow([ kitty_exe(), '+runpy', 'from kittens.runner import main; main()' ] + args, stdin=data, env={ 'KITTY_COMMON_OPTS': json.dumps(copts), 'KITTY_CHILD_PID': w.child.pid, 'PYTHONWARNINGS': 'ignore', 'OVERLAID_WINDOW_LINES': str(w.screen.lines), 'OVERLAID_WINDOW_COLS': str(w.screen.columns), }, cwd=w.cwd_of_child, overlay_for=w.id)) overlay_window.action_on_close = partial(self.on_kitten_finish, w.id, end_kitten) return overlay_window def kitten(self, kitten, *args): import shlex cmdline = args[0] if args else '' args = shlex.split(cmdline) if cmdline else [] self._run_kitten(kitten, args) def on_kitten_finish(self, target_window_id, end_kitten, source_window): output = self.get_output(source_window, num_lines=None) from kittens.runner import deserialize data = deserialize(output) if data is not None: end_kitten(data, target_window_id, self) def input_unicode_character(self): self._run_kitten('unicode_input') def set_tab_title(self): tab = self.active_tab if tab: args = [ '--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id) ] self._run_kitten('ask', args) def show_error(self, title, msg): self._run_kitten('show_error', args=['--title', title], input_data=msg) def do_set_tab_title(self, title, tab_id): tm = self.active_tab_manager if tm is not None and title: tab_id = int(tab_id) for tab in tm.tabs: if tab.id == tab_id: tab.set_title(title) break def kitty_shell(self, window_type): cmd = ['@', kitty_exe(), '@'] if window_type == 'tab': self._new_tab(cmd) elif window_type == 'os_window': os_window_id = self._new_os_window(cmd) self.os_window_map[os_window_id] elif window_type == 'overlay': w = self.active_window tab = self.active_tab if w is not None and tab is not None and w.overlay_for is None: tab.new_special_window(SpecialWindow(cmd, overlay_for=w.id)) else: self._new_window(cmd) def switch_focus_to(self, window_idx): tab = self.active_tab tab.set_active_window_idx(window_idx) def open_url(self, url, program=None, cwd=None): if url: if isinstance(program, str): program = to_cmdline(program) open_url(url, program or self.opts.open_url_with, cwd=cwd) def open_url_lines(self, lines, program=None): self.open_url(''.join(lines), program) def destroy(self): self.shutting_down = True self.child_monitor.shutdown_monitor() self.set_update_check_process() self.update_check_process = None del self.child_monitor for tm in self.os_window_map.values(): tm.destroy() self.os_window_map = {} destroy_global_data() def paste_to_active_window(self, text): if text: w = self.active_window if w is not None: w.paste(text) def paste_from_clipboard(self): text = get_clipboard_string() self.paste_to_active_window(text) def paste_from_selection(self): text = get_primary_selection( ) if supports_primary_selection else get_clipboard_string() self.paste_to_active_window(text) def set_primary_selection(self): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: set_primary_selection(text) if self.opts.copy_on_select: self.copy_to_buffer(self.opts.copy_on_select) def copy_to_buffer(self, buffer_name): w = self.active_window if w is not None and not w.destroyed: text = w.text_for_selection() if text: if buffer_name == 'clipboard': set_clipboard_string(text) elif buffer_name == 'primary': set_primary_selection(text) else: self.clipboard_buffers[buffer_name] = text def paste_from_buffer(self, buffer_name): if buffer_name == 'clipboard': text = get_clipboard_string() elif buffer_name == 'primary': text = get_primary_selection() else: text = self.clipboard_buffers.get(buffer_name) if text: self.paste_to_active_window(text) def goto_tab(self, tab_num): tm = self.active_tab_manager if tm is not None: tm.goto_tab(tab_num - 1) def set_active_tab(self, tab): tm = self.active_tab_manager if tm is not None: return tm.set_active_tab(tab) return False def next_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab() def previous_tab(self): tm = self.active_tab_manager if tm is not None: tm.next_tab(-1) prev_tab = previous_tab def process_stdin_source(self, window=None, stdin=None): w = window or self.active_window env = None if stdin: add_wrap_markers = stdin.endswith('_wrap') if add_wrap_markers: stdin = stdin[:-len('_wrap')] stdin = data_for_at(w, stdin, add_wrap_markers=add_wrap_markers) if stdin is not None: pipe_data = w.pipe_data( stdin, has_wrap_markers=add_wrap_markers) if w else {} if pipe_data: env = { 'KITTY_PIPE_DATA': '{scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}' .format(**pipe_data) } stdin = stdin.encode('utf-8') return env, stdin def special_window_for_cmd(self, cmd, window=None, stdin=None, cwd_from=None, as_overlay=False): w = window or self.active_window env, stdin = self.process_stdin_source(w, stdin) cmdline = [] for arg in cmd: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmdline.append(arg) overlay_for = w.id if as_overlay and w.overlay_for is None else None return SpecialWindow(cmd, stdin, cwd_from=cwd_from, overlay_for=overlay_for, env=env) def pipe(self, source, dest, exe, *args): cmd = [exe] + list(args) window = self.active_window cwd_from = window.child.pid_for_cwd if window else None def create_window(): return self.special_window_for_cmd(cmd, stdin=source, as_overlay=dest == 'overlay', cwd_from=cwd_from) if dest == 'overlay' or dest == 'window': tab = self.active_tab if tab is not None: return tab.new_special_window(create_window()) elif dest == 'tab': tm = self.active_tab_manager if tm is not None: tm.new_tab(special_window=create_window(), cwd_from=cwd_from) elif dest == 'os_window': self._new_os_window(create_window(), cwd_from=cwd_from) elif dest in ('clipboard', 'primary'): env, stdin = self.process_stdin_source(stdin=source, window=window) if stdin: func = set_clipboard_string if dest == 'clipboard' else set_primary_selection func(stdin) else: import subprocess env, stdin = self.process_stdin_source(stdin=source, window=window) cwd = None if cwd_from: with suppress(Exception): cwd = cwd_of_process(cwd_from) if stdin: r, w = safe_pipe(False) try: subprocess.Popen(cmd, env=env, stdin=r, cwd=cwd) except Exception: os.close(w) else: thread_write(w, stdin) finally: os.close(r) else: subprocess.Popen(cmd, env=env, cwd=cwd) def args_to_special_window(self, args, cwd_from=None): args = list(args) stdin = None w = self.active_window if args[0].startswith('@') and args[0] != '@': stdin = data_for_at(w, args[0]) or None if stdin is not None: stdin = stdin.encode('utf-8') del args[0] cmd = [] for arg in args: if arg == '@selection': arg = data_for_at(w, arg) if not arg: continue cmd.append(arg) return SpecialWindow(cmd, stdin, cwd_from=cwd_from) def _new_tab(self, args, cwd_from=None, as_neighbor=False): special_window = None if args: if isinstance(args, SpecialWindowInstance): special_window = args else: special_window = self.args_to_special_window(args, cwd_from=cwd_from) tm = self.active_tab_manager if tm is not None: return tm.new_tab(special_window=special_window, cwd_from=cwd_from, as_neighbor=as_neighbor) def _create_tab(self, args, cwd_from=None): as_neighbor = False if args and args[0].startswith('!'): as_neighbor = 'neighbor' in args[0][1:].split(',') args = args[1:] self._new_tab(args, as_neighbor=as_neighbor, cwd_from=cwd_from) def new_tab(self, *args): self._create_tab(args) def new_tab_with_cwd(self, *args): w = self.active_window_for_cwd cwd_from = w.child.pid_for_cwd if w is not None else None self._create_tab(args, cwd_from=cwd_from) def new_tab_with_wd(self, wd): special_window = SpecialWindow(None, cwd=wd) self._new_tab(special_window) def _new_window(self, args, cwd_from=None): tab = self.active_tab if tab is not None: location = None if args and args[0].startswith('!'): location = args[0][1:].lower() args = args[1:] if args: return tab.new_special_window(self.args_to_special_window( args, cwd_from=cwd_from), location=location) else: return tab.new_window(cwd_from=cwd_from, location=location) def new_window(self, *args): self._new_window(args) def new_window_with_cwd(self, *args): w = self.active_window_for_cwd if w is None: return self.new_window(*args) cwd_from = w.child.pid_for_cwd if w is not None else None self._new_window(args, cwd_from=cwd_from) def move_tab_forward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(1) def move_tab_backward(self): tm = self.active_tab_manager if tm is not None: tm.move_tab(-1) def disable_ligatures_in(self, where, strategy): if isinstance(where, str): windows = () if where == 'active': if self.active_window is not None: windows = (self.active_window, ) elif where == 'all': windows = self.all_windows elif where == 'tab': if self.active_tab is not None: windows = tuple(self.active_tab) else: windows = where for window in windows: window.screen.disable_ligatures = strategy window.refresh() def patch_colors(self, spec, cursor_text_color, configured=False): if configured: for k, v in spec.items(): if hasattr(self.opts, k): setattr(self.opts, k, color_from_int(v)) if cursor_text_color is not False: if isinstance(cursor_text_color, int): cursor_text_color = color_from_int(cursor_text_color) self.opts.cursor_text_color = cursor_text_color for tm in self.all_tab_managers: tm.tab_bar.patch_colors(spec) patch_global_colors(spec, configured) def safe_delete_temp_file(self, path): if is_path_in_temp_dir(path): with suppress(FileNotFoundError): os.remove(path) def set_update_check_process(self, process=None): if self.update_check_process is not None: with suppress(Exception): if self.update_check_process.poll() is None: self.update_check_process.kill() self.update_check_process = process def on_monitored_pid_death(self, pid, exit_status): update_check_process = getattr(self, 'update_check_process', None) if update_check_process is not None and pid == update_check_process.pid: self.update_check_process = None from .update_check import process_current_release try: raw = update_check_process.stdout.read().decode('utf-8') except Exception as e: log_error( 'Failed to read data from update check process, with error: {}' .format(e)) else: try: process_current_release(raw) except Exception as e: log_error( 'Failed to process update check data {!r}, with error: {}' .format(raw, e)) def notification_activated(self, identifier): if identifier == 'new-version': from .update_check import notification_activated notification_activated() def dbus_notification_callback(self, activated, *args): from .notify import dbus_notification_created, dbus_notification_activated if activated: dbus_notification_activated(*args) else: dbus_notification_created(*args) def show_bad_config_lines(self, bad_lines): def format_bad_line(bad_line): return '{}:{} in line: {}\n'.format(bad_line.number, bad_line.exception, bad_line.line) msg = '\n'.join(map(format_bad_line, bad_lines)).rstrip() self.show_error(_('Errors in kitty.conf'), msg)
class SlackHandler(GenericHandler): def __init__(self): super(SlackHandler, self).__init__() session = ClientSession() self.slack_client = SlackAPI(token=None, session=session) self.channels = WeakValueDictionary() async def setup(self, token): self.slack_client._token = token await self.update_team_info() await self.update_channels() asyncio.ensure_future(self.rtm()) print("Logged into Slack") def get_channel(self, serialised) -> Optional[SlackChannel]: channel_id = serialised["id"] try: return self.channels[channel_id] except KeyError: pass rtn = SlackChannel(channel_id, self.slack_client) self.channels[channel_id] = rtn return rtn async def rtm(self): async for event in self.slack_client.rtm(): if isinstance(event, Message): await asyncio.gather(*[channel.on_message_handler(event) for channel in self.channels.values()]) if isinstance(event, Event): if event["type"].startswith("channel_"): await self.update_channels() @property def serialised_channels(self): return self._serialised_channels async def update_channels(self): channels = await self.slack_client.query( slack.methods.CONVERSATIONS_LIST, data={ "exclude_archived": True } ) self._serialised_channels = [ { "type": "slack", "id": channel["id"], "name": channel["name"], "server": self.team_info } for channel in channels["channels"] if channel["is_member"] ] async def update_team_info(self): team_info = await self.slack_client.query("https://slack.com/api/team.info") self.team_info = { "id": team_info["team"]["id"], "name": team_info["team"]["name"], "icon": team_info["team"]["icon"]["image_132"] }