class WeakSet(object):
def __init__(self):
self._data = WeakValueDictionary()
def add(self, obj):
self._data[id(obj)] = obj
def remove(self, obj):
try:
del self._data[id(obj)]
except:
raise KeyError(obj)
def discard(self, obj):
try:
self.remove(obj)
except:
return
def __iter__(self):
for obj in self._data.values():
yield obj
def __len__(self):
return len(self._data)
def __contains__(self, obj):
return id(obj) in self._data
def __hash__(self):
raise TypeError, "Can't hash a WeakSet."
class WeakSet(object):
def __init__(self):
self._data = WeakValueDictionary()
def add(self, obj):
self._data[id(obj)] = obj
def remove(self, obj):
try:
del self._data[id(obj)]
except:
raise KeyError(obj)
def discard(self, obj):
try:
self.remove(obj)
except:
return
def __iter__(self):
for obj in self._data.values():
yield obj
def __len__(self):
return len(self._data)
def __contains__(self, obj):
return id(obj) in self._data
def __hash__(self):
raise TypeError, "Can't hash a WeakSet."
class Transport(BaseTransport):
"""Server side of the LocalQueue transport"""
def __init__(self,callbacks,cfg):
#logger.debug("Server: setting up")
self.trace = cfg.get('trace,',0)
self.callbacks = callbacks
self.p = LocalQueue(cfg)
self.p.server = ref(self) # for clients to find me
self.clients = WeakValueDictionary() # clients add themselves here
@spawned
def _process(self,msg):
m = self.callbacks.recv(msg.msg)
msg.reply(m)
def run(self):
logger.debug("Server: wait for messages")
while self.p.request_q is not None:
msg = self.p.request_q.get()
#logger.debug("Server: received %r",msg)
self._process(msg)
def send(self,msg):
m = msg
msg = RPCmessage(msg, _trace=self.trace)
self.last_msgid -= 1
msg.msgid = self.last_msgid
if self.trace:
logger.debug("Server: send msg %s:\n%s",msg.msgid,format_msg(m))
for c in self.clients.values():
c.reply_q.put(msg)
def _fetch_all(self):
set_peers = self._result_cache is None
super()._fetch_all()
# ModelIterable tests for query sets returning model instances vs values or value lists etc
if set_peers and issubclass(self._iterable_class,
models.query.ModelIterable):
peers = WeakValueDictionary((id(o), o) for o in self._result_cache)
for peer in peers.values():
peer._peers = peers
class Master(object):
def __init__(self, id, max_slots):
self.id = id
self.max_slots = max_slots
self.lock = Semaphore()
self.slots = WeakValueDictionary()
def serialize(self):
return {
'id': self.id,
'max_slots': self.max_slots,
'slots': len(self.slots),
'workers': sum(len(slot.workers) for slot in self.slots.values())
}
class Supervisor(object):
def __init__(self):
self.processes = WeakValueDictionary()
def register(self, process):
self.processes[process.id] = process
def process(self, process_id):
if process_id in self.processes:
return self.processes[process_id]
return None
def read_all(self):
for process in self.processes.values():
process.read()
class WSBroadcaster:
def __init__(self):
self.wsid_iter = count()
self.websockets = WeakValueDictionary()
def add_ws(self, ws, username):
"""Add a websocket to the pool, return its id"""
wsid = next(self.wsid_iter)
self.websockets[(username, wsid)] = ws
self.broadcast_connected_users()
return wsid
def remove_ws(self, username, wsid):
self.websockets.pop((username, wsid))
self.broadcast_connected_users()
def list_connected_users(self):
return sorted(set(username for username, wsid in self.websockets))
def broadcast_connected_users(self):
self.broadcast({
'type': 'connected_users',
'users': self.list_connected_users(),
})
def broadcast(self, message):
if message['type'] == 'preview':
fn = self.send_bytes
data = BSON.encode(message)
else:
fn = self.send_str
data = json.dumps(message)
for ws in self.websockets.values():
create_task(fn(ws, data))
async def send_str(self, ws, data):
await ws.send_str(data)
async def send_bytes(self, ws, data):
await ws.send_bytes(data)
class EventManager(object):
def __init__(self):
self.listeners = Wkd()
def add_listener(self, listener, name):
self.listeners[name] = listener
def remove_listener(self, listener):
if listener in self.listeners.keys():
del self.listeners[listener]
def post(self, event):
en = event.name
# if event.name != EV_TICK and event.name != EV_PLAYER_MOVE:
# pass
#print(type(event))
try:
# All listeners
if en == EV_TICK or \
en == EV_QUIT or \
en == EV_INIT or \
en == EV_RESIZE:
for val in self.listeners.values():
val.notify(event)
# Game Engine only
elif en == EV_INPUT or en == EV_MOUSE_MOVE or en == EV_MOUSE_CLICK:
self.listeners["Game Engine"].notify(event)
# Renderer only
elif en == EV_PLAYER_MOVE or \
en == EV_PLAYER_STATS or \
en == EV_MODEL_SHARE or \
en == EV_SWORD_SWING:
self.listeners["Renderer"].notify(event)
except KeyError as ke:
print("Error:", ke.message)
class Signal(object):
def __init__(self, *args):
self.__slots = WeakValueDictionary()
for slot in args:
self.connect(slot)
def __call__(self, slot, *args, **kwargs):
"""
Emit signal. If slot passed signal will be called only for this
slot, for all connected slots otherwise.
Calling this method directly lead to immediate signal processing.
It may be not thread-safe. Use emit method from this module for
delayed calling of signals.
"""
if slot is not None:
slots = (self.__slots[self.key(slot)],)
else:
slots = self.__slots.values()
for func in slots:
func(*args, **kwargs)
def key(self, slot):
"""
Get local key name for slot.
"""
if type(slot) == types.FunctionType:
key = (slot.__module__, slot.__name__)
elif type(slot) == types.MethodType:
key = (slot.__func__, id(slot.__self__))
elif isinstance(slot, basestring):
if not slot in registred_slots.keys():
raise ValueError('Slot {0} does not exists.'.format(slot))
key = slot
else:
raise ValueError('Slot {0} has non-slot type'.format(slot))
return key
def connect(self, slot):
"""
Connect signal to slot. Slot may be function, instance method
or name of function perviously registred by `slot` decorator.
"""
key = self.key(slot)
if type(slot) == types.FunctionType:
self.__slots[key] = slot
elif type(slot) == types.MethodType:
self.__slots[key] = partial(slot.__func__, slot.__self__)
elif isinstance(slot, basestring):
self.__slots[key] = registred_slots[slot]
def disconnect(self, slot):
"""
Remove slot from signal connetions.
"""
key = self.key(slot)
del self.__slots[key]
def clear(self):
"""
Disconnect all slots from signal.
"""
self.__slots.clear()
class TaskManager(object):
"""
Provides a set of tools to maintain a list of asyncio Tasks that are to be
executed during the lifetime of an arbitrary object, usually getting killed with it.
"""
def __init__(self):
self._pending_tasks = WeakValueDictionary()
self._task_lock = RLock()
self._shutdown = False
self._counter = 0
self._logger = logging.getLogger(self.__class__.__name__)
self._checker = self.register_task('_check_tasks',
self._check_tasks,
interval=MAX_TASK_AGE,
delay=MAX_TASK_AGE * 1.5)
def _check_tasks(self):
now = time.time()
for name, task in self._pending_tasks.items():
if not task.interval and now - task.start_time > MAX_TASK_AGE:
self._logger.warning(
'Non-interval task "%s" has been running for %.2f!', name,
now - task.start_time)
def replace_task(self, name, *args, **kwargs):
"""
Replace named task with the new one, cancelling the old one in the process.
"""
new_task = Future()
def cancel_cb(_):
try:
new_task.set_result(self.register_task(name, *args, **kwargs))
except Exception as e:
new_task.set_exception(e)
old_task = self.cancel_pending_task(name)
old_task.add_done_callback(cancel_cb)
return new_task
def register_task(self,
name,
task,
*args,
delay=None,
interval=None,
ignore=()):
"""
Register a Task/(coroutine)function so it can be canceled at shutdown time or by name.
"""
if not isinstance(
task,
Task) and not iscoroutinefunction(task) and not callable(task):
raise ValueError(
'Register_task takes a Task or a (coroutine)function as a parameter'
)
if (interval or delay) and isinstance(task, Task):
raise ValueError('Cannot run Task at an interval or with a delay')
if not isinstance(ignore, tuple) or not all(
(issubclass(e, Exception) for e in ignore)):
raise ValueError('Ignore should be a tuple of Exceptions or None')
with self._task_lock:
if self._shutdown:
self._logger.warning("Not adding task %s due to shutdown!",
str(task))
if isinstance(task, (Task, Future)):
if not task.done():
task.cancel()
return task
if self.is_pending_task_active(name):
raise RuntimeError("Task already exists: '%s'" % name)
if iscoroutinefunction(task) or callable(task):
task = task if iscoroutinefunction(task) else coroutine(task)
if interval:
# The default delay for looping calls is the same as the interval
delay = interval if delay is None else delay
task = ensure_future(
interval_runner(delay, interval, task, *args))
elif delay:
task = ensure_future(delay_runner(delay, task, *args))
else:
task = ensure_future(task(*args))
# Since weak references to list/tuple are not allowed, we're not storing start_time/interval
# in _pending_tasks. Instead we add them as attributes to the task.
task.start_time = time.time()
task.interval = interval
assert isinstance(task, Task)
def done_cb(future):
self._pending_tasks.pop(name, None)
try:
future.result()
except CancelledError:
pass
except ignore as e:
self._logger.error('Task resulted in error: %s', e)
self._pending_tasks[name] = task
task.add_done_callback(done_cb)
return task
def register_anonymous_task(self, basename, task, *args, **kwargs):
"""
Wrapper for register_task to derive a unique name from the basename.
"""
self._counter += 1
return self.register_task(basename + ' ' + str(self._counter), task,
*args, **kwargs)
def cancel_pending_task(self, name):
"""
Cancels the named task
"""
with self._task_lock:
task = self._pending_tasks.get(name, None)
if not task:
return succeed(None)
if not task.done():
task.cancel()
self._pending_tasks.pop(name, None)
return task
def cancel_all_pending_tasks(self):
"""
Cancels all the registered tasks.
This usually should be called when stopping or destroying the object so no tasks are left floating around.
"""
with self._task_lock:
assert all([
isinstance(t, (Task, Future))
for t in self._pending_tasks.values()
]), self._pending_tasks
return [
self.cancel_pending_task(name)
for name in list(self._pending_tasks.keys())
]
def is_pending_task_active(self, name):
"""
Return a boolean determining if a task is active.
"""
with self._task_lock:
task = self._pending_tasks.get(name, None)
return not task.done() if task else False
def get_tasks(self):
"""
Returns a list of all registered tasks, excluding tasks the are created by the TaskManager itself.
"""
with self._task_lock:
return [
t for t in self._pending_tasks.values() if t != self._checker
]
async def wait_for_tasks(self):
"""
Waits until all registered tasks are done.
"""
with self._task_lock:
tasks = self.get_tasks()
if tasks:
await gather(*tasks, return_exceptions=True)
async def shutdown_task_manager(self):
"""
Clear the task manager, cancel all pending tasks and disallow new tasks being added.
"""
with self._task_lock:
self._shutdown = True
tasks = self.cancel_all_pending_tasks()
if tasks:
with suppress(CancelledError):
await gather(*tasks)
class Boss:
def __init__(self, os_window_id, opts, args, cached_values):
self.window_id_map = WeakValueDictionary()
self.startup_colors = {
k: opts[k]
for k in opts if isinstance(opts[k], Color)
}
self.pending_sequences = None
self.cached_values = cached_values
self.os_window_map = {}
self.cursor_blinking = True
self.shutting_down = False
talk_fd = getattr(single_instance, 'socket', None)
talk_fd = -1 if talk_fd is None else talk_fd.fileno()
listen_fd = -1
if opts.allow_remote_control and args.listen_on:
listen_fd = listen_on(args.listen_on)
self.child_monitor = ChildMonitor(
self.on_child_death,
DumpCommands(args) if args.dump_commands or args.dump_bytes else
None, talk_fd, listen_fd)
set_boss(self)
self.opts, self.args = opts, args
startup_session = create_session(opts, args)
self.add_os_window(startup_session, os_window_id=os_window_id)
def add_os_window(self,
startup_session,
os_window_id=None,
wclass=None,
wname=None,
opts_for_size=None,
startup_id=None):
if os_window_id is None:
opts_for_size = opts_for_size or self.opts
cls = wclass or self.args.cls or appname
with startup_notification_handler(
do_notify=startup_id is not None,
startup_id=startup_id) as pre_show_callback:
os_window_id = create_os_window(
initial_window_size_func(opts_for_size,
self.cached_values),
pre_show_callback, appname, wname or self.args.name or cls,
cls)
tm = TabManager(os_window_id, self.opts, self.args, startup_session)
self.os_window_map[os_window_id] = tm
return os_window_id
def list_os_windows(self):
for os_window_id, tm in self.os_window_map.items():
yield {
'id': os_window_id,
'tabs': list(tm.list_tabs()),
}
@property
def all_tab_managers(self):
yield from self.os_window_map.values()
@property
def all_tabs(self):
for tm in self.all_tab_managers:
yield from tm
@property
def all_windows(self):
for tab in self.all_tabs:
yield from tab
def match_windows(self, match):
try:
field, exp = match.split(':', 1)
except ValueError:
return
if field == 'num':
tab = self.active_tab
if tab is not None:
try:
w = tab.get_nth_window(int(exp))
except Exception:
return
if w is not None:
yield w
else:
pat = re.compile(exp)
for window in self.all_windows:
if window.matches(field, pat):
yield window
def tab_for_window(self, window):
for tab in self.all_tabs:
for w in tab:
if w.id == window.id:
return tab
def match_tabs(self, match):
try:
field, exp = match.split(':', 1)
except ValueError:
return
pat = re.compile(exp)
found = False
if field in ('title', 'id'):
for tab in self.all_tabs:
if tab.matches(field, pat):
yield tab
found = True
if not found:
tabs = {self.tab_for_window(w) for w in self.match_windows(match)}
for tab in tabs:
if tab:
yield tab
def set_active_window(self, window):
for tm in self.os_window_map.values():
for tab in tm:
for w in tab:
if w.id == window.id:
if tab is not self.active_tab:
tm.set_active_tab(tab)
tab.set_active_window(w)
return
def _new_os_window(self, args, cwd_from=None):
sw = self.args_to_special_window(args, cwd_from) if args else None
startup_session = create_session(self.opts,
special_window=sw,
cwd_from=cwd_from)
return self.add_os_window(startup_session)
def new_os_window(self, *args):
self._new_os_window(args)
def new_os_window_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_os_window(args, cwd_from)
def add_child(self, window):
self.child_monitor.add_child(window.id, window.child.pid,
window.child.child_fd, window.screen)
self.window_id_map[window.id] = window
def _handle_remote_command(self, cmd, window=None):
response = None
if self.opts.allow_remote_control or getattr(
window, 'allow_remote_control', False):
try:
response = handle_cmd(self, window, cmd)
except Exception as err:
import traceback
response = {'ok': False, 'error': str(err)}
if not getattr(err, 'hide_traceback', False):
response['tb'] = traceback.format_exc()
else:
response = {
'ok':
False,
'error':
'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'
}
return response
def peer_message_received(self, msg):
msg = msg.decode('utf-8')
cmd_prefix = '\x1bP@kitty-cmd'
if msg.startswith(cmd_prefix):
cmd = msg[len(cmd_prefix):-2]
response = self._handle_remote_command(cmd)
if response is not None:
response = (cmd_prefix + json.dumps(response) +
'\x1b\\').encode('utf-8')
return response
else:
msg = json.loads(msg)
if isinstance(msg, dict) and msg.get('cmd') == 'new_instance':
startup_id = msg.get('startup_id')
args, rest = parse_args(msg['args'][1:])
args.args = rest
opts = create_opts(args)
if not os.path.isabs(args.directory):
args.directory = os.path.join(msg['cwd'], args.directory)
session = create_session(opts, args, respect_cwd=True)
self.add_os_window(session,
wclass=args.cls,
wname=args.name,
opts_for_size=opts,
startup_id=startup_id)
else:
log_error('Unknown message received from peer, ignoring')
def handle_remote_cmd(self, cmd, window=None):
response = self._handle_remote_command(cmd, window)
if response is not None:
if window is not None:
window.send_cmd_response(response)
def on_child_death(self, window_id):
window = self.window_id_map.pop(window_id, None)
if window is None:
return
if window.action_on_close:
try:
window.action_on_close(window)
except Exception:
import traceback
traceback.print_exc()
os_window_id = window.os_window_id
window.destroy()
tm = self.os_window_map.get(os_window_id)
if tm is None:
return
for tab in tm:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
tm.remove(tab)
tab.destroy()
if len(tm) == 0:
if not self.shutting_down:
mark_os_window_for_close(os_window_id)
glfw_post_empty_event()
def close_window(self, window=None):
if window is None:
window = self.active_window
self.child_monitor.mark_for_close(window.id)
def close_tab(self, tab=None):
if tab is None:
tab = self.active_tab
for window in tab:
self.close_window(window)
def toggle_fullscreen(self):
toggle_fullscreen()
def start(self):
if not getattr(self, 'io_thread_started', False):
self.child_monitor.start()
self.io_thread_started = True
def activate_tab_at(self, os_window_id, x):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.activate_tab_at(x)
def on_window_resize(self, os_window_id, w, h, dpi_changed):
if dpi_changed:
self.on_dpi_change(os_window_id)
else:
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.resize()
def increase_font_size(self): # legacy
self.set_font_size(
min(self.opts.font_size * 5, self.current_font_size + 2.0))
def decrease_font_size(self): # legacy
self.set_font_size(self.current_font_size - self.opts.font_size_delta)
def restore_font_size(self): # legacy
self.set_font_size(self.opts.font_size)
def set_font_size(self, new_size): # legacy
self.change_font_size(True, None, new_size)
def change_font_size(self, all_windows, increment_operation, amt):
def calc_new_size(old_size):
new_size = old_size
if amt == 0:
new_size = self.opts.font_size
else:
if increment_operation:
new_size += (1 if increment_operation == '+' else -1) * amt
else:
new_size = amt
new_size = max(MINIMUM_FONT_SIZE,
min(new_size, self.opts.font_size * 5))
return new_size
if all_windows:
current_global_size = global_font_size()
new_size = calc_new_size(current_global_size)
if new_size != current_global_size:
global_font_size(new_size)
os_windows = tuple(self.os_window_map.keys())
else:
os_windows = []
w = self.active_window
if w is not None:
os_windows.append(w.os_window_id)
if os_windows:
final_windows = {}
for wid in os_windows:
current_size = os_window_font_size(wid)
if current_size:
new_size = calc_new_size(current_size)
if new_size != current_size:
final_windows[wid] = new_size
if final_windows:
self._change_font_size(final_windows)
def _change_font_size(self, sz_map):
for os_window_id, sz in sz_map.items():
tm = self.os_window_map.get(os_window_id)
if tm is not None:
os_window_font_size(os_window_id, sz)
tm.resize()
def on_dpi_change(self, os_window_id):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
sz = os_window_font_size(os_window_id)
if sz:
os_window_font_size(os_window_id, sz, True)
tm.resize()
def _set_os_window_background_opacity(self, os_window_id, opacity):
change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0)))
def set_background_opacity(self, opacity):
window = self.active_window
if window is None or not opacity:
return
if not self.opts.dynamic_background_opacity:
return self.show_error(
_('Cannot change background opacity'),
_('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity'
))
os_window_id = window.os_window_id
if opacity[0] in '+-':
opacity = background_opacity_of(os_window_id)
if opacity is None:
return
opacity += float(opacity)
elif opacity == 'default':
opacity = self.opts.background_opacity
else:
opacity = float(opacity)
self._set_os_window_background_opacity(os_window_id, opacity)
@property
def active_tab_manager(self):
os_window_id = current_os_window()
return self.os_window_map.get(os_window_id)
@property
def active_tab(self):
tm = self.active_tab_manager
if tm is not None:
return tm.active_tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
def dispatch_special_key(self, key, scancode, action, mods):
# Handles shortcuts, return True if the key was consumed
key_action = get_shortcut(self.opts.keymap, mods, key, scancode)
if key_action is None:
sequences = get_shortcut(self.opts.sequence_map, mods, key,
scancode)
if sequences:
self.pending_sequences = sequences
set_in_sequence_mode(True)
return True
else:
self.current_key_press_info = key, scancode, action, mods
return self.dispatch_action(key_action)
def process_sequence(self, key, scancode, action, mods):
if not self.pending_sequences:
set_in_sequence_mode(False)
remaining = {}
matched_action = None
for seq, key_action in self.pending_sequences.items():
if shortcut_matches(seq[0], mods, key, scancode):
seq = seq[1:]
if seq:
remaining[seq] = key_action
else:
matched_action = key_action
if remaining:
self.pending_sequences = remaining
else:
self.pending_sequences = None
set_in_sequence_mode(False)
if matched_action is not None:
self.dispatch_action(matched_action)
def start_resizing_window(self):
w = self.active_window
if w is None:
return
overlay_window = self._run_kitten(
'resize_window',
args=[
'--horizontal-increment={}'.format(
self.opts.window_resize_step_cells),
'--vertical-increment={}'.format(
self.opts.window_resize_step_lines)
])
if overlay_window is not None:
overlay_window.allow_remote_control = True
def resize_layout_window(self,
window,
increment,
is_horizontal,
reset=False):
tab = window.tabref()
if tab is None or not increment:
return False
if reset:
return tab.reset_window_sizes()
return tab.resize_window_by(window.id, increment, is_horizontal)
def default_bg_changed_for(self, window_id):
w = self.window_id_map.get(window_id)
if w is not None:
tm = self.os_window_map.get(w.os_window_id)
if tm is not None:
t = tm.tab_for_id(w.tab_id)
if t is not None:
t.relayout_borders()
def dispatch_action(self, key_action):
if key_action is not None:
f = getattr(self, key_action.func, None)
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
tab = self.active_tab
if tab is None:
return False
window = self.active_window
if window is None:
return False
if key_action is not None:
f = getattr(tab, key_action.func,
getattr(window, key_action.func, None))
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
return False
def combine(self, *actions):
for key_action in actions:
self.dispatch_action(key_action)
def on_focus(self, os_window_id, focused):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.focus_changed(focused)
tm.mark_tab_bar_dirty()
def update_tab_bar_data(self, os_window_id):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.update_tab_bar_data()
def on_drop(self, os_window_id, paths):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.paste('\n'.join(paths))
def on_os_window_closed(self, os_window_id, viewport_width,
viewport_height):
self.cached_values['window-size'] = viewport_width, viewport_height
tm = self.os_window_map.pop(os_window_id, None)
if tm is not None:
tm.destroy()
for window_id in tuple(
w.id for w in self.window_id_map.values()
if getattr(w, 'os_window_id', None) == os_window_id):
self.window_id_map.pop(window_id, None)
def display_scrollback(self, window, data):
tab = self.active_tab
if tab is not None and window.overlay_for is None:
tab.new_special_window(
SpecialWindow(self.opts.scrollback_pager,
data,
_('History'),
overlay_for=window.id))
def edit_config_file(self, *a):
confpath = prepare_config_file_for_editing()
# On macOS vim fails to handle SIGWINCH if it occurs early, so add a
# small delay.
cmd = [
'kitty', '+runpy',
'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])'
] + editor + [confpath]
self.new_os_window(*cmd)
def get_output(self, source_window, num_lines=1):
output = ''
s = source_window.screen
if num_lines is None:
num_lines = s.lines
for i in range(min(num_lines, s.lines)):
output += str(s.linebuf.line(i))
return output
def _run_kitten(self, kitten, args=(), input_data=None):
w = self.active_window
tab = self.active_tab
if w is not None and tab is not None and w.overlay_for is None:
orig_args, args = list(args), list(args)
from kittens.runner import create_kitten_handler
end_kitten = create_kitten_handler(kitten, orig_args)
args[0:0] = [config_dir, kitten]
if input_data is None:
type_of_input = end_kitten.type_of_input
if type_of_input in ('text', 'history', 'ansi', 'ansi-history',
'screen', 'screen-history', 'screen-ansi',
'screen-ansi-history'):
data = w.as_text(as_ansi='ansi' in type_of_input,
add_history='history' in type_of_input,
add_wrap_markers='screen'
in type_of_input).encode('utf-8')
elif type_of_input is None:
data = None
else:
raise ValueError(
'Unknown type_of_input: {}'.format(type_of_input))
else:
data = input_data
if isinstance(data, str):
data = data.encode('utf-8')
copts = {
k: self.opts[k]
for k in ('select_by_word_characters', 'open_url_with')
}
overlay_window = tab.new_special_window(
SpecialWindow([
'kitty', '+runpy',
'from kittens.runner import main; main()'
] + args,
stdin=data,
env={
'KITTY_COMMON_OPTS': json.dumps(copts),
'PYTHONWARNINGS': 'ignore',
'OVERLAID_WINDOW_LINES': str(w.screen.lines),
'OVERLAID_WINDOW_COLS':
str(w.screen.columns),
},
overlay_for=w.id))
overlay_window.action_on_close = partial(self.on_kitten_finish,
w.id, end_kitten)
return overlay_window
def kitten(self, kitten, *args):
import shlex
cmdline = args[0] if args else ''
args = shlex.split(cmdline) if cmdline else []
self._run_kitten(kitten, args)
def on_kitten_finish(self, target_window_id, end_kitten, source_window):
output = self.get_output(source_window, num_lines=None)
from kittens.runner import deserialize
data = deserialize(output)
if data is not None:
end_kitten(data, target_window_id, self)
def input_unicode_character(self):
self._run_kitten('unicode_input')
def set_tab_title(self):
tab = self.active_tab
if tab:
args = [
'--name=tab-title', '--message',
_('Enter the new title for this tab below.'),
'do_set_tab_title',
str(tab.id)
]
self._run_kitten('ask', args)
def show_error(self, title, msg):
self._run_kitten('show_error', ['--title', title], input_data=msg)
def do_set_tab_title(self, title, tab_id):
tm = self.active_tab_manager
if tm is not None and title:
tab_id = int(tab_id)
for tab in tm.tabs:
if tab.id == tab_id:
tab.set_title(title)
break
def kitty_shell(self, window_type):
cmd = ['kitty', '@']
if window_type == 'tab':
window = self._new_tab(cmd).active_window
elif window_type == 'os_window':
os_window_id = self._new_os_window(cmd)
window = self.os_window_map[os_window_id].active_window
elif window_type == 'overlay':
w = self.active_window
tab = self.active_tab
if w is not None and tab is not None and w.overlay_for is None:
window = tab.new_special_window(
SpecialWindow(cmd, overlay_for=w.id))
else:
window = None
else:
window = self._new_window(cmd)
if window is not None:
window.allow_remote_control = True
def switch_focus_to(self, window_idx):
tab = self.active_tab
tab.set_active_window_idx(window_idx)
def open_url(self, url, program=None, cwd=None):
if url:
if isinstance(program, str):
program = to_cmdline(program)
open_url(url, program or self.opts.open_url_with, cwd=cwd)
def open_url_lines(self, lines, program=None):
self.open_url(''.join(lines), program)
def destroy(self):
self.shutting_down = True
self.child_monitor.shutdown_monitor()
del self.child_monitor
for tm in self.os_window_map.values():
tm.destroy()
self.os_window_map = {}
destroy_global_data()
def paste_to_active_window(self, text):
if text:
w = self.active_window
if w is not None:
w.paste(text)
def paste_from_clipboard(self):
text = get_clipboard_string()
self.paste_to_active_window(text)
def paste_from_selection(self):
text = get_primary_selection(
) if supports_primary_selection else get_clipboard_string()
self.paste_to_active_window(text)
def set_primary_selection(self):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
set_primary_selection(text)
if self.opts.copy_on_select:
set_clipboard_string(text)
def goto_tab(self, tab_num):
tm = self.active_tab_manager
if tm is not None:
tm.goto_tab(tab_num - 1)
def set_active_tab(self, tab):
tm = self.active_tab_manager
if tm is not None:
tm.set_active_tab(tab)
def next_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab()
def previous_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab(-1)
def args_to_special_window(self, args, cwd_from=None):
args = list(args)
stdin = None
w = self.active_window
def data_for_at(arg):
if arg == '@selection':
return w.text_for_selection()
if arg == '@ansi':
return w.as_text(as_ansi=True, add_history=True)
if arg == '@text':
return w.as_text(add_history=True)
if arg == '@screen':
return w.as_text()
if arg == '@ansi_screen':
return w.as_text(as_ansi=True)
if args[0].startswith('@'):
stdin = data_for_at(args[0]) or None
if stdin is not None:
stdin = stdin.encode('utf-8')
del args[0]
cmd = []
for arg in args:
if arg == '@selection':
arg = data_for_at(arg)
if not arg:
continue
cmd.append(arg)
return SpecialWindow(cmd, stdin, cwd_from=cwd_from)
def _new_tab(self, args, cwd_from=None):
special_window = None
if args:
if isinstance(args, SpecialWindowInstance):
special_window = args
else:
special_window = self.args_to_special_window(args,
cwd_from=cwd_from)
tm = self.active_tab_manager
if tm is not None:
return tm.new_tab(special_window=special_window, cwd_from=cwd_from)
def new_tab(self, *args):
self._new_tab(args)
def new_tab_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_tab(args, cwd_from=cwd_from)
def _new_window(self, args, cwd_from=None):
tab = self.active_tab
if tab is not None:
if args:
return tab.new_special_window(
self.args_to_special_window(args, cwd_from=cwd_from))
else:
return tab.new_window(cwd_from=cwd_from)
def new_window(self, *args):
self._new_window(args)
def new_window_with_cwd(self, *args):
w = self.active_window
if w is None:
return self.new_window(*args)
cwd_from = w.child.pid if w is not None else None
self._new_window(args, cwd_from=cwd_from)
def move_tab_forward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(1)
def move_tab_backward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(-1)
def patch_colors(self, spec, configured=False):
if configured:
for k, v in spec.items():
if hasattr(self.opts, k):
setattr(self.opts, k, color_from_int(v))
for tm in self.all_tab_managers:
tm.tab_bar.patch_colors(spec)
class Boss:
def __init__(self, os_window_id, opts, args, cached_values):
self.window_id_map = WeakValueDictionary()
self.cached_values = cached_values
self.os_window_map = {}
self.cursor_blinking = True
self.shutting_down = False
talk_fd = getattr(single_instance, 'socket', None)
talk_fd = -1 if talk_fd is None else talk_fd.fileno()
listen_fd = -1
if opts.allow_remote_control and args.listen_on:
listen_fd = listen_on(args.listen_on)
self.child_monitor = ChildMonitor(
self.on_child_death,
DumpCommands(args) if args.dump_commands or args.dump_bytes else
None, talk_fd, listen_fd)
set_boss(self)
self.current_font_size = opts.font_size
set_font_family(opts)
self.opts, self.args = opts, args
initialize_renderer()
startup_session = create_session(opts, args)
self.add_os_window(startup_session, os_window_id=os_window_id)
def add_os_window(self,
startup_session,
os_window_id=None,
wclass=None,
wname=None,
size=None,
startup_id=None):
dpi_changed = False
if os_window_id is None:
w, h = initial_window_size(
self.opts, self.cached_values) if size is None else size
cls = wclass or self.args.cls or appname
os_window_id = create_os_window(w, h, appname, wname
or self.args.name or cls, cls)
if startup_id:
ctx = init_startup_notification(os_window_id, startup_id)
dpi_changed = show_window(os_window_id)
if startup_id:
end_startup_notification(ctx)
tm = TabManager(os_window_id, self.opts, self.args, startup_session)
self.os_window_map[os_window_id] = tm
if dpi_changed:
self.on_dpi_change(os_window_id)
def list_os_windows(self):
for os_window_id, tm in self.os_window_map.items():
yield {
'id': os_window_id,
'tabs': list(tm.list_tabs()),
}
def match_windows(self, match):
field, exp = match.split(':', 1)
pat = re.compile(exp)
for tm in self.os_window_map.values():
for tab in tm:
for window in tab:
if window.matches(field, pat):
yield window
def tab_for_window(self, window):
for tm in self.os_window_map.values():
for tab in tm:
for w in tab:
if w.id == window.id:
return tab
def match_tabs(self, match):
field, exp = match.split(':', 1)
pat = re.compile(exp)
tms = tuple(self.os_window_map.values())
found = False
if field in ('title', 'id'):
for tm in tms:
for tab in tm:
if tab.matches(field, pat):
yield tab
found = True
if not found:
tabs = {self.tab_for_window(w) for w in self.match_windows(match)}
for tab in tabs:
if tab:
yield tab
def set_active_window(self, window):
for tm in self.os_window_map.values():
for tab in tm:
for w in tab:
if w.id == window.id:
if tab is not self.active_tab:
tm.set_active_tab(tab)
tab.set_active_window(w)
return
def _new_os_window(self, args, cwd_from=None):
sw = self.args_to_special_window(args, cwd_from) if args else None
startup_session = create_session(self.opts,
special_window=sw,
cwd_from=cwd_from)
self.add_os_window(startup_session)
def new_os_window(self, *args):
self._new_os_window(args)
def new_os_window_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_os_window(args, cwd_from)
def add_child(self, window):
self.child_monitor.add_child(window.id, window.child.pid,
window.child.child_fd, window.screen)
self.window_id_map[window.id] = window
def _handle_remote_command(self, cmd, window=None):
response = None
if self.opts.allow_remote_control:
try:
response = handle_cmd(self, window, cmd)
except Exception as err:
import traceback
response = {
'ok': False,
'error': str(err),
'tb': traceback.format_exc()
}
else:
response = {
'ok':
False,
'error':
'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'
}
return response
def peer_message_received(self, msg):
import json
msg = msg.decode('utf-8')
cmd_prefix = '\x1bP@kitty-cmd'
if msg.startswith(cmd_prefix):
cmd = msg[len(cmd_prefix):-2]
response = self._handle_remote_command(cmd)
if response is not None:
response = (cmd_prefix + json.dumps(response) +
'\x1b\\').encode('utf-8')
return response
else:
msg = json.loads(msg)
if isinstance(msg, dict) and msg.get('cmd') == 'new_instance':
startup_id = msg.get('startup_id')
args, rest = parse_args(msg['args'][1:])
args.args = rest
opts = create_opts(args)
session = create_session(opts, args)
self.add_os_window(session,
wclass=args.cls,
wname=args.name,
size=initial_window_size(
opts, self.cached_values),
startup_id=startup_id)
else:
log_error('Unknown message received from peer, ignoring')
def handle_remote_cmd(self, cmd, window=None):
response = self._handle_remote_command(cmd, window)
if response is not None:
if window is not None:
window.send_cmd_response(response)
def on_child_death(self, window_id):
window = self.window_id_map.pop(window_id, None)
if window is None:
return
if window.action_on_close:
try:
window.action_on_close(window)
except Exception:
import traceback
traceback.print_exc()
os_window_id = window.os_window_id
window.destroy()
tm = self.os_window_map.get(os_window_id)
if tm is None:
return
for tab in tm:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
tm.remove(tab)
tab.destroy()
if len(tm) == 0:
if not self.shutting_down:
mark_os_window_for_close(os_window_id)
glfw_post_empty_event()
def close_window(self, window=None):
if window is None:
window = self.active_window
self.child_monitor.mark_for_close(window.id)
def close_tab(self, tab=None):
if tab is None:
tab = self.active_tab
for window in tab:
self.close_window(window)
def toggle_fullscreen(self):
toggle_fullscreen()
def start(self):
if not getattr(self, 'io_thread_started', False):
self.child_monitor.start()
self.io_thread_started = True
def activate_tab_at(self, os_window_id, x):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.activate_tab_at(x)
def on_window_resize(self, os_window_id, w, h, dpi_changed):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
if dpi_changed:
if set_dpi_from_os_window(os_window_id):
self.on_dpi_change(os_window_id)
else:
tm.resize()
else:
tm.resize()
def increase_font_size(self):
self.change_font_size(
min(self.opts.font_size * 5,
self.current_font_size + self.opts.font_size_delta))
def decrease_font_size(self):
self.change_font_size(
max(MINIMUM_FONT_SIZE,
self.current_font_size - self.opts.font_size_delta))
def restore_font_size(self):
self.change_font_size(self.opts.font_size)
def _change_font_size(self, new_size=None, on_dpi_change=False):
if new_size is not None:
self.current_font_size = new_size
old_cell_width, old_cell_height = viewport_for_window()[-2:]
windows = tuple(filter(None, self.window_id_map.values()))
resize_fonts(self.current_font_size, on_dpi_change=on_dpi_change)
layout_sprite_map()
prerender()
for window in windows:
window.screen.rescale_images(old_cell_width, old_cell_height)
window.screen.refresh_sprite_positions()
for tm in self.os_window_map.values():
tm.resize()
tm.refresh_sprite_positions()
glfw_post_empty_event()
def change_font_size(self, new_size):
if new_size == self.current_font_size:
return
self._change_font_size(new_size)
def on_dpi_change(self, os_window_id):
self._change_font_size()
@property
def active_tab_manager(self):
os_window_id = current_os_window()
return self.os_window_map.get(os_window_id)
@property
def active_tab(self):
tm = self.active_tab_manager
if tm is not None:
return tm.active_tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
def dispatch_special_key(self, key, scancode, action, mods):
# Handles shortcuts, return True if the key was consumed
key_action = get_shortcut(self.opts.keymap, mods, key, scancode)
self.current_key_press_info = key, scancode, action, mods
return self.dispatch_action(key_action)
def default_bg_changed_for(self, window_id):
w = self.window_id_map.get(window_id)
if w is not None:
tm = self.os_window_map.get(w.os_window_id)
if tm is not None:
t = tm.tab_for_id(w.tab_id)
if t is not None:
t.relayout_borders()
def dispatch_action(self, key_action):
if key_action is not None:
f = getattr(self, key_action.func, None)
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
tab = self.active_tab
if tab is None:
return False
window = self.active_window
if window is None:
return False
if key_action is not None:
f = getattr(tab, key_action.func,
getattr(window, key_action.func, None))
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
return False
def combine(self, *actions):
for key_action in actions:
self.dispatch_action(key_action)
def on_focus(self, os_window_id, focused):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.focus_changed(focused)
def on_drop(self, os_window_id, paths):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.paste('\n'.join(paths))
def on_os_window_closed(self, os_window_id, viewport_width,
viewport_height):
self.cached_values['window-size'] = viewport_width, viewport_height
tm = self.os_window_map.pop(os_window_id, None)
if tm is not None:
tm.destroy()
for window_id in tuple(
w.id for w in self.window_id_map.values()
if getattr(w, 'os_window_id', None) == os_window_id):
self.window_id_map.pop(window_id, None)
def display_scrollback(self, window, data):
tab = self.active_tab
if tab is not None and window.overlay_for is None:
tab.new_special_window(
SpecialWindow(self.opts.scrollback_pager,
data,
_('History'),
overlay_for=window.id))
def edit_config_file(self, *a):
confpath = prepare_config_file_for_editing()
# On macOS vim fails to handle SIGWINCH if it occurs early, so add a
# small delay.
cmd = [
'kitty', '+runpy',
'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])'
] + editor + [confpath]
self.new_os_window(*cmd)
def input_unicode_character(self):
w = self.active_window
tab = self.active_tab
if w is not None and tab is not None and w.overlay_for is None:
overlay_window = tab.new_special_window(
SpecialWindow([
'kitty', '+runpy',
'from kittens.unicode_input.main import main; main()'
],
overlay_for=w.id))
overlay_window.action_on_close = partial(
self.send_unicode_character, w.id)
def send_unicode_character(self, target_window_id, source_window):
w = self.window_id_map.get(target_window_id)
if w is not None:
output = str(source_window.screen.linebuf.line(0))
if output.startswith('OK: '):
try:
text = chr(int(output.partition(' ')[2], 16))
except Exception:
import traceback
traceback.print_exc()
else:
w.paste(text)
def run_simple_kitten(self, type_of_input, kitten, *args):
import shlex
w = self.active_window
tab = self.active_tab
if w is not None and tab is not None and w.overlay_for is None:
cmdline = args[0] if args else ''
args = shlex.split(cmdline) if cmdline else []
if '--program' not in cmdline:
args.extend(('--program', self.opts.open_url_with))
if type_of_input in ('text', 'history', 'ansi', 'ansi-history'):
data = w.as_text(as_ansi='ansi' in type_of_input,
add_history='history'
in type_of_input).encode('utf-8')
elif type_of_input == 'none':
data = None
else:
raise ValueError(
'Unknown type_of_input: {}'.format(type_of_input))
tab.new_special_window(
SpecialWindow([
'kitty', '+runpy',
'from kittens.{}.main import main; main()'.format(kitten)
] + args,
stdin=data,
overlay_for=w.id))
def switch_focus_to(self, window_idx):
tab = self.active_tab
tab.set_active_window_idx(window_idx)
old_focus = tab.active_window
if not old_focus.destroyed:
old_focus.focus_changed(False)
tab.active_window.focus_changed(True)
def open_url(self, url):
if url:
open_url(url, self.opts.open_url_with)
def open_url_lines(self, lines):
self.open_url(''.join(lines))
def destroy(self):
self.shutting_down = True
self.child_monitor.shutdown_monitor()
del self.child_monitor
for tm in self.os_window_map.values():
tm.destroy()
self.os_window_map = {}
destroy_sprite_map()
destroy_global_data()
def paste_to_active_window(self, text):
if text:
w = self.active_window
if w is not None:
w.paste(text)
def paste_from_clipboard(self):
text = get_clipboard_string()
self.paste_to_active_window(text)
def paste_from_selection(self):
text = get_primary_selection()
self.paste_to_active_window(text)
def set_primary_selection(self):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
set_primary_selection(text)
if self.opts.copy_on_select:
set_clipboard_string(text)
def goto_tab(self, tab_num):
tm = self.active_tab_manager
if tm is not None:
tm.goto_tab(tab_num - 1)
def next_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab()
def previous_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab(-1)
def args_to_special_window(self, args, cwd_from=None):
args = list(args)
stdin = None
w = self.active_window
def data_for_at(arg):
if arg == '@selection':
return w.text_for_selection()
if arg == '@ansi':
return w.as_text(as_ansi=True, add_history=True)
if arg == '@text':
return w.as_text(add_history=True)
if arg == '@screen':
return w.as_text()
if arg == '@ansi_screen':
return w.as_text(as_ansi=True)
if args[0].startswith('@'):
stdin = data_for_at(args[0]) or None
if stdin is not None:
stdin = stdin.encode('utf-8')
del args[0]
cmd = []
for arg in args:
if arg == '@selection':
arg = data_for_at(arg)
if not arg:
continue
cmd.append(arg)
return SpecialWindow(cmd, stdin, cwd_from=cwd_from)
def _new_tab(self, args, cwd_from=None):
special_window = None
if args:
if isinstance(args, SpecialWindowInstance):
special_window = args
else:
special_window = self.args_to_special_window(args,
cwd_from=cwd_from)
tm = self.active_tab_manager
if tm is not None:
tm.new_tab(special_window=special_window, cwd_from=cwd_from)
def new_tab(self, *args):
self._new_tab(args)
def new_tab_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_tab(args, cwd_from=cwd_from)
def _new_window(self, args, cwd_from=None):
tab = self.active_tab
if tab is not None:
if args:
tab.new_special_window(
self.args_to_special_window(args, cwd_from=cwd_from))
else:
tab.new_window(cwd_from=cwd_from)
def new_window(self, *args):
self._new_window(args)
def new_window_with_cwd(self, *args):
w = self.active_window
if w is None:
return self.new_window(*args)
cwd_from = w.child.pid if w is not None else None
self._new_window(args, cwd_from=cwd_from)
def move_tab_forward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(1)
def move_tab_backward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(-1)
class Server(object):
"""
A server.
"""
def __init__(self, db_object):
"""
Initializes this server object.
"""
self.user_map = WeakValueDictionary()
self.channel_map = {}
self.lock = Lock()
self.update_from_database_object(db_object)
self.needs_reconnect = Flag()
def update_from_database_object(self, db_object):
"""
Instructs this server to update itself from the specified database
object, which should be the result of a query against the database.
This updates the server's name, the server's protocol class, the
server's activated state, and the server's config properties.
"""
self.name = db_object["name"]
self.active = db_object["active"]
self.protocol_name = db_object["protocol"]
self.config_properties = dict(db_object["config"])
def get_user(self, transient_name):
"""
Gets a user object representing a user with the specified transient
name.
"""
def init_user_info(self, transient_name, persistent_name=None,
display_name=None, group_name=None):
"""
Called by the protocol constructor to set the initial information for
the user. Once the constructor returns, at least the transient name
must have been set by calling this method.
"""
pass
def disconnect(self, message=None):
"""
Disconnects this server. This basically just forwards over to the
corresponding protocol. It does not clear any of the server maps; use
clear_session_data for that.
"""
self.protocol.disconnect()
def is_connected(self):
raise Exception("Not implemented yet")
def clear_session_data(self):
"""
Clears the information for the last session from this server object.
This involves clearing the channel map and the user map. This
synchronizes on the server lock. This also untracks any users still in
the session map.
"""
with self.lock:
for user in self.user_map.values():
user.untrack()
# TODO: figure out if the channel map should have a weak map
# similar to the user map or not
self.channel_map.clear()
self.local_user = None
def on_action(self, transient_name, message, persistent_name=None,
display_name=None, group_name=None):
"""
Called by a protocol when an action is received.
"""
pass
def on_message(self, transient_name, message, persistent_name=None,
display_name=None, group_name=None):
"""
Called by a protocol when a message is received.
"""
pass
def on_connect(self, transient_name=None, persistent_name=None,
display_name=None, group_name=None):
"""
Called when the server successfully connects. The user information
provided to this method is the information for the user that the
protocol has connected as. None of it is required.
"""
pass
def on_disconnect(self):
"""
Called when the protocol disconnects from the server for some reason.
"""
pass
class UniversalCursors(object):
def __init__(self):
self.name_cursors = {}
self.cursors_orient = WeakKeyDictionary()
self.all_canvas = WeakValueDictionary()
self.all_axes = WeakValueDictionary()
self.backgrounds = {}
self.visible = True
self.needclear = False
def _onmove(self, event):
for canvas in self.all_canvas.values():
if not canvas.widgetlock.available(self):
return
if event.inaxes is None or not self.visible:
if self.needclear:
self._update(event)
for canvas in self.all_canvas.values():
canvas.draw()
self.needclear = False
return
self._update(event)
def _update(self, event):
# 1/ Reset background
for canvas in self.all_canvas.values():
canvas.restore_region(self.backgrounds[id(canvas)])
# 2/ update cursors
for cursors in self.cursors_orient.keys():
orient = self.cursors_orient[cursors]
if (event.inaxes in [line.get_axes() for line in cursors]
and self.visible):
visible = True
self.needclear = True
else:
visible = False
for line in cursors:
if orient == 'vertical':
line.set_xdata((event.xdata, event.xdata))
if orient == 'horizontal':
line.set_ydata((event.ydata, event.ydata))
line.set_visible(visible)
ax = line.get_axes()
ax.draw_artist(line)
# 3/ update canvas
for canvas in self.all_canvas.values():
canvas.blit(canvas.figure.bbox)
def _clear(self, event):
"""clear the cursor"""
self.backgrounds = {}
for canvas in self.all_canvas.values():
self.backgrounds[id(canvas)] = (
canvas.copy_from_bbox(canvas.figure.bbox))
for cursor in self.cursors_orient.keys():
for line in cursor:
line.set_visible(False)
def add(self, name, axes=(), orient='vertical', **lineprops):
if name in self.name_cursors.keys():
raise NameError
class CursorList(list):
def __hash__(self):
return hash(tuple(self))
self.name_cursors[name] = CursorList() # Required to keep weakref
for ax in axes:
self.all_axes[id(ax)] = ax
ax_canvas = ax.get_figure().canvas
if ax_canvas not in self.all_canvas.values():
#if not ax_canvas.supports_blit:
# warnings.warn("Must use canvas that support blit")
# return
self.all_canvas[id(ax_canvas)] = ax_canvas
ax_canvas.mpl_connect('motion_notify_event', self._onmove)
ax_canvas.mpl_connect('draw_event', self._clear)
if orient == 'vertical':
line = ax.axvline(ax.get_xbound()[0], visible=False,
animated=True, **lineprops)
if orient == 'horizontal':
line = ax.axhline(ax.get_ybound()[0], visible=False,
animated=True, **lineprops)
self.name_cursors[name].append(line)
self.cursors_orient[self.name_cursors[name]] = orient
def remove(self, name):
del self.name_cursors[name]
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0,
FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation,
self.onNodeDestruction,
self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation,
self.onTaskDestruction,
self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation,
self.onRenderNodeDestruction,
self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation,
self.onPoolDestruction,
self.onPoolChange)
self.commandListener = ObjectListener(
onCreationEvent=self.onCommandCreation,
onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node, "task") or node.task is None:
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
# logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"])
# logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"])
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r."
% taskDef['dependencies'])
if not all(((isinstance(i, int) and isinstance(sl, list) and all(
(isinstance(s, int) for s in sl)))
for (i, sl) in taskDef['dependencies'])):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r."
% taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" %
(rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts(child)
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
timer = None
if 'timer' in taskGroupDefinition.keys():
timer = taskGroupDefinition['timer']
return TaskGroup(id,
name,
parent,
user,
arguments,
environment,
requirements,
maxRN,
priority,
dispatchKey,
strategy,
tags=tags,
timer=timer)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
runnerPackages = taskDefinition.get('runnerPackages', '')
watcherPackages = taskDefinition.get('watcherPackages', '')
timer = None
if 'timer' in taskDefinition.keys():
timer = taskDefinition['timer']
maxAttempt = taskDefinition.get('maxAttempt', 1)
task = Task(None,
name,
None,
user,
maxRN,
priority,
dispatchKey,
runner,
arguments,
validationExpression, [],
requirements,
minNbCores,
maxNbCores,
ramUse,
environment,
lic=lic,
tags=tags,
timer=timer,
maxAttempt=maxAttempt,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
cmd = Command(None,
description,
task,
arguments,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()
]) if self.nodes else 0
self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(),
StatDB.getTaskNodesMaxId())
self.poolMaxId = max([p.id for p in self.pools.values()
]) if self.pools else 0
self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId())
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()
]) if self.renderNodes else 0
self.renderNodeMaxId = max(self.renderNodeMaxId,
StatDB.getRenderNodesMaxId())
self.taskMaxId = max([t.id for t in self.tasks.values()
]) if self.tasks else 0
self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(),
StatDB.getTaskGroupsMaxId())
self.commandMaxId = max([c.id for c in self.commands.values()
]) if self.commands else 0
self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId())
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()
]) if self.poolShares else 0
self.poolShareMaxId = max(self.poolShareMaxId,
StatDB.getPoolSharesMaxId())
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id is None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id,
StatDB.getTasksMaxId(),
StatDB.getTaskGroupsMaxId())
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id is None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id,
StatDB.getFolderNodesMaxId(),
StatDB.getTaskNodesMaxId())
if node.parent is None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id is None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id,
StatDB.getRenderNodesMaxId())
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." %
rendernode.name)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
if field == "performance":
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id is None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id,
StatDB.getPoolsMaxId())
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id,
StatDB.getCommandsMaxId())
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
if command.task is not None:
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id,
StatDB.getPoolSharesMaxId())
self.poolShares[poolShare.id] = poolShare
class DispatchTree(object):
def _display_(self):
'''
Debug purpose method, returns a basic display of the dispatch tree as html
'''
startTimer = time.time()
timeout = 2.0
result="<html><head><style>table,th,td { margin: 5px; border-collapse:collapse; border:1px solid black; }</style></head><body font-family='verdana'>"
result +="<h3>Pools: %r</h3><table>" % len(self.pools)
for i,curr in enumerate(self.pools):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.pools[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Rendernodes: %r</h3><table>" % len(self.renderNodes)
for i,curr in enumerate(self.renderNodes):
result += "<tr><td>%r</td><td>%r</td></tr>" % (i, self.renderNodes[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>PoolShares: (attribution de parc pour une tache fille du root, on attribue pas de poolshare aux autres)</h3><table>"
for i,curr in enumerate(self.poolShares):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.poolShares[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Main level nodes (proxy info only):</h3><table>"
result +="<tr><th>id</th><th>name</th><th>readyCommandCount</th><th>commandCount</th><th>completion</th><th>poolshares</th></tr>"
for i,curr in enumerate(self.nodes[1].children):
result += "<tr><td>%r</td><td>%s</td><td>%d</td><td>%d</td><td>%.2f</td><td>%s</td></tr>" % (i, curr.name, curr.readyCommandCount, curr.commandCount, curr.completion, curr.poolShares.values())
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>All nodes:</h3><table>"
for i,curr in enumerate(self.nodes):
result += "<tr><td>%d</td><td>%s</td><td>%r</td></tr>" % (i, curr, self.nodes[curr].name)
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Tasks:</h3><table>"
for i,curr in enumerate(self.tasks):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, repr(self.tasks[curr]) )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Commands:</h3><table>"
for i,curr in enumerate(self.commands):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.commands[curr] )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Rules:</h3><table>"
for i,curr in enumerate(self.rules):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, curr )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="</body></html>"
logger.info("DispatchTree printed in %.6f s" % (time.time()-startTimer) )
return result
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node,"task"):
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
if not all(((isinstance(i, int) and
isinstance(sl, list) and
all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts( child )
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
timer = None
if 'timer' in taskGroupDefinition.keys():
timer = taskGroupDefinition['timer']
return TaskGroup(id, name, parent, user, arguments, environment, requirements,
maxRN, priority, dispatchKey, strategy, tags=tags, timer=timer)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
timer = None
if 'timer' in taskDefinition.keys():
timer = taskDefinition['timer']
task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner,
arguments, validationExpression, [], requirements, minNbCores,
maxNbCores, ramUse, environment, lic=lic, tags=tags, timer=timer)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
cmd = Command(None, description, task, arguments)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id == None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id)
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id == None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id)
if node.parent == None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id == None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id)
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError, e:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." % rendernode.name)
class ChannelManager(object):
""" High level interface for channels
This class handles:
* configuration of channels
* high level api to create and remove jobs (notify, remove_job, remove_db)
* get jobs to run
Here is how the runner will use it.
Let's create a channel manager and configure it.
>>> from pprint import pprint as pp
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4,B:1')
>>> db = 'db'
Add a few jobs in channel A with priority 10
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending')
Add a few jobs in channel B with priority 5
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending')
>>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending')
We must now run one job from queue B which has a capacity of 1
and 3 jobs from queue A so the root channel capacity of 4 is filled.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>]
Job A2 is done. Next job to run is A5, even if we have
higher priority job in channel B, because channel B has a capacity of 1.
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A4>]
Job B1 is done. Next job to run is B2 because it has higher priority.
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B2>]
Let's say A1 is done and A6 gets a higher priority. A6 will run next.
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done')
>>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A6>]
Let's test the throttling mechanism. Configure a 2 seconds delay
on channel A, end enqueue two jobs.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4:throttle=2')
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
We have only one job to run, because of the throttle.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A1>]
>>> cm.get_wakeup_time()
102
We have no job to run, because of the throttle.
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.get_wakeup_time()
102
2 seconds later, we can run the other job (even though the first one
is still running, because we have enough capacity).
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob A2>]
>>> cm.get_wakeup_time()
104
Let's test throttling in combination with a queue reaching full capacity.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,T:2:throttle=2')
>>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'T', 'T2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'T', 'T3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob T1>]
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob T2>]
Channel is now full, so no job to run even though throttling
delay is over.
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> cm.get_wakeup_time() # no wakeup time, since queue is full
0
>>> pp(list(cm.get_jobs_to_run(now=104)))
[]
>>> cm.get_wakeup_time() # queue is still full
0
>>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob T3>]
>>> cm.get_wakeup_time() # queue is full
0
>>> cm.notify(db, 'T', 'T2', 1, 0, 10, None, 'done')
>>> cm.get_wakeup_time()
107
Test wakeup time behaviour in presence of eta.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,E:1')
>>> cm.notify(db, 'E', 'E1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'E', 'E2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'E', 'E3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob E1>]
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.notify(db, 'E', 'E1', 1, 0, 10, 105, 'pending')
>>> cm.get_wakeup_time() # wakeup at eta
105
>>> pp(list(cm.get_jobs_to_run(now=102))) # but there is capacity
[<ChannelJob E2>]
>>> pp(list(cm.get_jobs_to_run(now=106))) # no capacity anymore
[]
>>> cm.get_wakeup_time() # no timed wakeup because no capacity
0
>>> cm.notify(db, 'E', 'E2', 1, 0, 10, None, 'done')
>>> cm.get_wakeup_time()
105
>>> pp(list(cm.get_jobs_to_run(now=107))) # no capacity anymore
[<ChannelJob E1>]
>>> cm.get_wakeup_time()
0
Test wakeup time behaviour in a sequential queue.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,S:1:sequential')
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob S1>]
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'failed')
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=102)))
[]
No wakeup time because due to eta, because the sequential queue
is waiting for a failed job.
>>> cm.get_wakeup_time()
0
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending')
>>> cm.get_wakeup_time()
105
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob S1>]
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'done')
At this stage, we have S2 with an eta of 105 and since the
queue is sequential, we wait for it.
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob S2>]
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob S3>]
>>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[]
"""
def __init__(self):
self._jobs_by_uuid = WeakValueDictionary()
self._root_channel = Channel(name='root', parent=None, capacity=1)
self._channels_by_name = WeakValueDictionary(root=self._root_channel)
@staticmethod
def split_strip(s, sep, maxsplit=-1):
"""Split string and strip each component.
>>> ChannelManager.split_strip("foo: bar baz\\n: fred:", ":")
['foo', 'bar baz', 'fred', '']
"""
return [x.strip() for x in s.split(sep, maxsplit)]
@classmethod
def parse_simple_config(cls, config_string):
"""Parse a simple channels configuration string.
The general form is as follow:
channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...]
If capacity is absent, it defaults to 1.
If a key is present without value, it gets True as value.
When declaring subchannels, the root channel may be omitted
(ie sub:4 is the same as root.sub:4).
Returns a list of channel configuration dictionaries.
>>> from pprint import pprint as pp
>>> pp(ChannelManager.parse_simple_config('root:4'))
[{'capacity': 4, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2'))
[{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:'
... 'sequential:k=v'))
[{'capacity': 4, 'name': 'root'},
{'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}]
>>> pp(ChannelManager.parse_simple_config('root'))
[{'capacity': 1, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('sub:2'))
[{'capacity': 2, 'name': 'sub'}]
It ignores whitespace around values, and drops empty entries which
would be generated by trailing commas, or commented lines on the Odoo
config file.
>>> pp(ChannelManager.parse_simple_config('''
... root : 4,
... ,
... foo bar:1: k=va lue,
... '''))
[{'capacity': 4, 'name': 'root'},
{'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}]
It's also possible to replace commas with line breaks, which is more
readable if you're taking the channel configuration from a ConfigParser
file.
>>> pp(ChannelManager.parse_simple_config('''
... root : 4
... foo bar:1: k=va lue
... baz
... '''))
[{'capacity': 4, 'name': 'root'},
{'capacity': 1, 'k': 'va lue', 'name': 'foo bar'},
{'capacity': 1, 'name': 'baz'}]
"""
res = []
config_string = config_string.replace("\n", ",")
for channel_config_string in cls.split_strip(config_string, ','):
if not channel_config_string:
# ignore empty entries (commented lines, trailing commas)
continue
config = {}
config_items = cls.split_strip(channel_config_string, ':')
name = config_items[0]
if not name:
raise ValueError('Invalid channel config %s: '
'missing channel name' % config_string)
config['name'] = name
if len(config_items) > 1:
capacity = config_items[1]
try:
config['capacity'] = int(capacity)
except:
raise ValueError('Invalid channel config %s: '
'invalid capacity %s' %
(config_string, capacity))
for config_item in config_items[2:]:
kv = cls.split_strip(config_item, '=')
if len(kv) == 1:
k, v = kv[0], True
elif len(kv) == 2:
k, v = kv
else:
raise ValueError('Invalid channel config %s: '
'incorrect config item %s' %
(config_string, config_item))
if k in config:
raise ValueError('Invalid channel config %s: '
'duplicate key %s' %
(config_string, k))
config[k] = v
else:
config['capacity'] = 1
res.append(config)
return res
def simple_configure(self, config_string):
"""Configure the channel manager from a simple configuration string
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root')
>>> c.capacity
1
>>> cm.simple_configure('root:4,autosub.sub:2,seq:1:sequential')
>>> cm.get_channel_by_name('root').capacity
4
>>> cm.get_channel_by_name('root').sequential
False
>>> cm.get_channel_by_name('root.autosub').capacity
>>> cm.get_channel_by_name('root.autosub.sub').capacity
2
>>> cm.get_channel_by_name('root.autosub.sub').sequential
False
>>> cm.get_channel_by_name('autosub.sub').capacity
2
>>> cm.get_channel_by_name('seq').capacity
1
>>> cm.get_channel_by_name('seq').sequential
True
"""
for config in ChannelManager.parse_simple_config(config_string):
self.get_channel_from_config(config)
def get_channel_from_config(self, config):
"""Return a Channel object from a parsed configuration.
If the channel does not exist it is created.
The configuration is applied on the channel before returning it.
If some of the parent channels are missing when creating a subchannel,
the parent channels are auto created with an infinite capacity
(except for the root channel, which defaults to a capacity of 1
when not configured explicity).
"""
channel = self.get_channel_by_name(config['name'], autocreate=True)
channel.configure(config)
return channel
def get_channel_by_name(self, channel_name, autocreate=False):
"""Return a Channel object by its name.
If it does not exist and autocreate is True, it is created
with a default configuration and inserted in the Channels structure.
If autocreate is False and the channel does not exist, an exception
is raised.
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root', autocreate=False)
>>> c.name
'root'
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('autosub.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.autosub.sub'
>>> c = cm.get_channel_by_name(None)
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub')
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub')
>>> c.fullname
'root.sub'
"""
if not channel_name or channel_name == self._root_channel.name:
return self._root_channel
if not channel_name.startswith(self._root_channel.name + '.'):
channel_name = self._root_channel.name + '.' + channel_name
if channel_name in self._channels_by_name:
return self._channels_by_name[channel_name]
if not autocreate:
raise ChannelNotFound('Channel %s not found' % channel_name)
parent = self._root_channel
for subchannel_name in channel_name.split('.')[1:]:
subchannel = parent.get_subchannel_by_name(subchannel_name)
if not subchannel:
subchannel = Channel(subchannel_name, parent, capacity=None)
self._channels_by_name[subchannel.fullname] = subchannel
parent = subchannel
return parent
def notify(self, db_name, channel_name, uuid,
seq, date_created, priority, eta, state):
try:
channel = self.get_channel_by_name(channel_name)
except ChannelNotFound:
_logger.warning('unknown channel %s, '
'using root channel for job %s',
channel_name, uuid)
channel = self._root_channel
job = self._jobs_by_uuid.get(uuid)
if job:
# db_name is invariant
assert job.db_name == db_name
# date_created is invariant
assert job.date_created == date_created
# if one of the job properties that influence
# scheduling order has changed, we remove the job
# from the queues and create a new job object
if (seq != job.seq or
priority != job.priority or
eta != job.eta or
channel != job.channel):
_logger.debug("job %s properties changed, rescheduling it",
uuid)
self.remove_job(uuid)
job = None
if not job:
job = ChannelJob(db_name, channel, uuid,
seq, date_created, priority, eta)
self._jobs_by_uuid[uuid] = job
# state transitions
if not state or state == DONE:
job.channel.set_done(job)
elif state == PENDING:
job.channel.set_pending(job)
elif state in (ENQUEUED, STARTED):
job.channel.set_running(job)
elif state == FAILED:
job.channel.set_failed(job)
else:
_logger.error("unexpected state %s for job %s", state, job)
def remove_job(self, uuid):
job = self._jobs_by_uuid.get(uuid)
if job:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def remove_db(self, db_name):
for job in self._jobs_by_uuid.values():
if job.db_name == db_name:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def get_jobs_to_run(self, now):
return self._root_channel.get_jobs_to_run(now)
def get_wakeup_time(self):
return self._root_channel.get_wakeup_time()
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
if not all(((isinstance(i, int) and
isinstance(sl, list) and
all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
PoolShare(None, pool, nodes[0], maxRN)
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
return nodes
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
return TaskGroup(id, name, parent, user, arguments, environment, requirements,
maxRN, priority, dispatchKey, strategy, tags=tags)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner,
arguments, validationExpression, [], requirements, minNbCores,
maxNbCores, ramUse, environment, lic=lic, tags=tags)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
task.commands.append(Command(None, description, task, arguments))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
if task.id == None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id)
self.tasks[task.id] = task
def onTaskDestruction(self, task):
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
if node.id == None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id)
if node.parent == None:
node.parent = self.root
def onNodeDestruction(self, node):
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id == None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id)
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id == None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id)
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id)
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id)
self.poolShares[poolShare.id] = poolShare
class Silk(SilkObject):
_anonymous = None # bool
_props = None # list
dtype = None # list
_positional_args = None # list
__slots__ = [
"_parent", "_storage_enum", "_storage_nonjson_children",
"_data", "_children", "_is_none", "__weakref__"
]
def __init__(self, *args, _mode="any", **kwargs):
self._storage_enum = None
self._storage_nonjson_children = set()
self._children = None
if _mode == "parent":
self._init(
kwargs["parent"],
kwargs["storage"],
kwargs["data_store"],
)
elif _mode == "from_numpy":
assert "parent" not in kwargs
self._init(
None,
"numpy",
kwargs["data_store"],
)
else:
assert "parent" not in kwargs
assert "storage" not in kwargs
assert "data_store" not in kwargs
self._init(None, "json", None)
if _mode == "any":
self.set(*args, **kwargs)
elif _mode == "empty":
pass
elif _mode == "from_json":
self.set(*args, prop_setter=_prop_setter_json, **kwargs)
else:
raise ValueError(_mode)
def _init(self, parent, storage, data_store):
from .silkarray import SilkArray
if parent is not None:
if storage == "numpy":
self._parent = lambda: parent # hard ref
self._parent = weakref.ref(parent)
else:
self._parent = lambda: None
self.storage = storage
self._is_none = False
self._storage_nonjson_children.clear()
if self._children is not None:
for child in self._children.values():
child._parent = lambda: None
if storage == "json":
self._children = {}
if data_store is None:
data_store = {}
elif storage == "numpy":
self._children = WeakValueDictionary()
assert data_store is not None
assert data_store.dtype == np.dtype(self.dtype, align=True)
assert data_store.shape == ()
self._data = data_store
return
else:
raise ValueError(storage)
assert storage == "json"
for pname, p in self._props.items():
if p["elementary"]:
continue
t = self._get_typeclass(pname)
if pname not in data_store:
if issubclass(t, SilkArray):
data_store[pname] = []
else:
data_store[pname] = {}
c_data_store = data_store[pname]
self._children[pname] = t(
_mode="parent",
storage="json",
parent=self,
data_store=c_data_store,
len_data_store=None,
)
self._data = data_store
def _get_typeclass(self, propname):
p = self._props[propname]
if "typeclass" in p:
t = p["typeclass"]
else:
typename = p["typename"]
t = typenames._silk_types[typename]
return t
def copy(self, storage="json"):
"""Returns a copy with the storage in the specified format"""
cls = type(self)
if storage == "json":
json = self.json()
ret = cls.from_json(json)
for prop in self._props:
if not self._props[prop]["elementary"]:
child = self._children[prop]
is_none = child._is_none
ret._children[prop]._is_none = is_none
elif storage == "numpy":
ret = cls.from_numpy(self.numpy())
else:
raise ValueError(storage)
return ret
@classmethod
def from_json(cls, data):
data = _filter_json(data)
return cls(data, _mode="from_json")
@classmethod
def from_numpy(cls, data, copy=True,validate=True):
"""Constructs from a numpy array singleton "data"
"""
if data.shape != ():
raise TypeError("Data must be a singleton")
if data.dtype != np.dtype(cls.dtype,align=True):
raise TypeError("Data has the wrong dtype")
if copy:
data = datacopy(data)
ret = cls(_mode="from_numpy", data_store=data)
if validate:
ret.validate()
return ret
@classmethod
def empty(cls):
return cls(_mode="empty")
def _get_child(self, childname, force=False):
from .silkarray import SilkArray
if self.storage == "numpy":
prop = self._props[childname]
is_none = False
if prop["optional"]:
if not self._data["HAS_" + childname]:
is_none = True
if is_none and not force:
return NoneChild
t = self._get_typeclass(childname)
len_data_store = None
if issubclass(t, SilkArray):
if prop.get("var_array", False):
len_data_store = self._data["LEN_"+childname]
child = t (
_mode = "parent",
parent = self,
storage = "numpy",
data_store = self._data[childname],
len_data_store = len_data_store
)
self._children[childname] = child
return self._children[childname]
def set(self, *args, prop_setter=_prop_setter_any, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
if args[0] is None or isinstance(args[0], SilkObject) and args[0]._is_none:
self._is_none = True
self._clear_data()
return
# TODO: make a nice composite exception that stores all exceptions
try:
self._construct(prop_setter, *args, **kwargs)
except Exception:
if len(args) == 1 and len(kwargs) == 0:
try:
a = args[0]
try:
if isinstance(a, np.void):
d = {}
for name in a.dtype.fields:
if name.startswith("HAS_"):
continue
name2 = "HAS_" + name
if name2 in a.dtype.names and not a[name2]:
continue
d[name] = a[name]
self._construct(prop_setter, **d)
else:
raise TypeError
except Exception:
if isinstance(a, dict):
self._construct(prop_setter, **a)
elif isinstance(a, str):
self._parse(a)
elif isinstance(a, collections.Iterable) or isinstance(a, np.void):
self._construct(prop_setter, *a)
elif isinstance(a, SilkObject):
d = {prop: getattr(a, prop) for prop in dir(a)}
self._construct(prop_setter, **d)
elif hasattr(a, "__dict__"):
self._construct(prop_setter, **a.__dict__)
else:
raise TypeError(a)
except Exception:
raise
else:
raise
self.validate()
self._is_none = False
def validate(self):
pass # overridden during registration
def json(self):
"""Returns a JSON representation of the Silk object
"""
if self.storage == "json":
return _filter_json(self._data)
d = {}
for attr in self._props:
p = self._props[attr]
ele = p["elementary"]
value = None
if ele:
if self.storage == "numpy":
value = _get_numpy_ele_prop(self, attr)
else:
value = self._data[attr]
if value is not None:
t = self._get_typeclass(attr)
value = t(value)
else:
child = self._get_child(attr)
if not child._is_none:
value = child.json()
if value is not None:
d[attr] = value
return d
def numpy(self):
"""Returns a numpy representation of the Silk object
NOTE: for optional members,
the entire storage buffer is returned,
including (zeroed) elements if the data is not present!
the extra field "HAS_xxx" indicates if the data is present.
NOTE: for all numpy array members,
the entire storage buffer is returned,
including (zeroed) elements if the data is not present!
the length of each array is stored in the LEN_xxx field
TODO: document multidimensional length vector, PTR_LEN_xxx
NOTE: for numpy array members of variable shape,
an extra field "PTR_xxx" contains a C pointer to the data
For this, the dimensionality of the array does not matter,
e.g. both for IntegerArray and IntegerArrayArray,
the C pointer will be "int *"
and both for MyStructArray and MyStructArrayArray,
the C pointer will be "MyStruct *"
TODO: add and document SHAPE field
"""
if self.storage == "numpy":
return datacopy(self._data)
new_obj = self.copy("json")
return new_obj.make_numpy()
def make_json(self):
if self.storage == "json":
return self._data
elif self.storage == "numpy":
json = _filter_json(self.json(), self)
parent = self._parent()
if parent is not None and parent.storage == "numpy":
parent.numpy_shatter()
self._init(parent, "json", None)
self.set(json, prop_setter=_prop_setter_json)
if parent is not None:
parent._remove_nonjson_child(self)
myname = parent._find_child(id(self))
parent._data[myname] = self._data
return self._data
elif self.storage == "mixed":
for child_id in list(self._storage_nonjson_children): # copy!
for child in self._children.values():
if id(child) == child_id:
child.make_json()
break
else:
raise Exception("Cannot find child that was marked as 'non-JSON'")
# Above will automatically update storage status to "json"
return self._data
def _restore_array_coupling(self):
pass
def make_numpy(self,_toplevel=None):
"""Sets the internal storage to 'numpy'
Returns the numpy array that is used as internal storage buffer
NOTE: for optional members,
the entire storage buffer is returned,
including (zeroed) elements if the data is not present!
an extra field "HAS_xxx" indicates if the data is present.
TODO: update doc
NOTE: for numpy array members of variable shape,
an extra field "PTR_xxx" contains a C pointer to the data
For this, the dimensionality of the array does not matter,
e.g. both for IntegerArray and IntegerArrayArray,
the C pointer will be "int *"
and both for MyStructArray and MyStructArrayArray,
the C pointer will be "MyStruct *"
"""
from .silkarray import SilkArray
if self.storage == "numpy":
return self._data
dtype = np.dtype(self.dtype, align=True)
data = np.zeros(dtype=dtype, shape=(1,))
for propname,prop in self._props.items():
if prop["elementary"]:
value = getattr(self, propname)
_set_numpy_ele_prop(self, propname, value, data)
else:
child = self._get_child(propname)
if not child._is_none:
child.make_numpy(_toplevel=False)
if isinstance(child, SilkArray):
if prop.get("var_array", False):
child._restore_array_coupling(data[0], propname)
else:
data[0][propname] = np.zeros_like(dtype[propname])
slices = [slice(0,v) in child._data.shape]
data[0][propname][slices] = child._data
else:
data[0][propname] = child._data
child._data = None
self._init(self._parent(), "numpy", data[0])
parent = self._parent()
if parent is not None:
if parent.storage != "numpy":
parent._add_nonjson_child(self)
return data[0]
def _find_child(self, child_id):
for childname, ch in self._children.items():
if child_id == id(ch):
return childname
raise KeyError
def _add_nonjson_child(self, child):
childname = self._find_child(id(child))
if self._props[childname].get("var_array", False) and \
self.storage == "numpy":
return
assert self.storage != "numpy"
njc = self._storage_nonjson_children
child_id = id(child)
if child_id not in njc:
njc.add(child_id)
if self.storage == "json":
self.storage = "mixed"
parent = self._parent()
if parent is not None:
parent._add_nonjson_child(self)
def _remove_nonjson_child(self, child):
assert self.storage != "numpy"
njc = self._storage_nonjson_children
child_id = id(child)
if child_id in njc:
assert self.storage == "mixed", self.storage
njc.remove(child_id)
if len(njc) == 0:
self.storage = "json"
parent = self._parent()
if parent is not None:
parent()._remove_nonjson_child(self)
def numpy_shatter(self):
"""
Breaks up a unified numpy storage into one numpy storage per child
"""
assert self.storage == "numpy"
parent = self._parent()
if parent is not None and parent.storage == "numpy":
parent.numpy_shatter()
data = {}
children = {}
for prop in self._props:
p = self._props[prop]
if p["elementary"]:
value = getattr(self, prop)
if value is not None:
if "typeclass" in p:
t = p["typeclass"]
else:
typename = p["typename"]
t = typenames._silk_types[typename]
value = t(value)
data[prop] = value
else:
child = self._get_child(prop)
d = datacopy(child._data)
data[prop] = d
child._data = d
children[prop] = child
self._data = data
self._children = children
self._storage_nonjson_children = set([id(p) for p in children.values()])
self.storage = "mixed"
def _construct(self, prop_setter, *args, **kwargs):
propdict = {}
if len(args) > len(self._positional_args):
message = "{0}() takes {1} positional arguments \
but {2} were given".format(
self.__class__.__name__,
len(self._positional_args),
len(args)
)
raise TypeError(message)
for anr, a in enumerate(args):
propdict[self._positional_args[anr]] = a
for argname, a in kwargs.items():
if argname in propdict:
message = "{0}() got multiple values for argument '{1}'"
message = message.format(
self.__class__.__name__,
argname
)
raise TypeError(message)
propdict[argname] = a
missing = [p for p in self._props if p not in propdict]
missing_required = [p for p in missing
if not self._props[p]["optional"]
and p not in self._props_init]
if missing_required:
missing_required = ["'{0}'".format(p) for p in missing_required]
if len(missing_required) == 1:
plural = ""
missing_txt = missing_required[0]
elif len(missing_required) == 2:
plural = "s"
missing_txt = missing_required[0] + " and " + \
missing_required[1]
else:
plural = "s"
missing_txt = ", ".join(missing_required[:-1]) + \
", and " + missing_required[-1]
message = "{0}() missing {1} positional argument{2}: {3}".format(
self.__class__.__name__,
len(missing_required),
plural,
missing_txt
)
raise TypeError(message)
for propname in self._props:
value = propdict.get(propname, None)
if value is None and propname in self._props_init:
value = self._props_init[propname]
self._set_prop(propname, value, prop_setter)
def _parse(self, s):
raise NotImplementedError # can be user-defined
_storage_names = ("numpy", "json", "mixed")
@property
def storage(self):
return self._storage_names[self._storage_enum]
@storage.setter
def storage(self, storage):
assert storage in self._storage_names, storage
self._storage_enum = self._storage_names.index(storage)
def __dir__(self):
return dir(type(self))
def __setattr__(self, attr, value):
if attr.startswith("_") or attr == "storage":
object.__setattr__(self, attr, value)
else:
self._set_prop(attr, value, _prop_setter_any)
def _set_prop(self, prop, value, child_prop_setter):
try:
p = self._props[prop]
except KeyError:
raise AttributeError(prop)
if value is None and not p["optional"]:
raise TypeError("'%s' cannot be None" % prop)
ele = p["elementary"]
if ele:
if self.storage == "numpy":
_set_numpy_ele_prop(self, prop, value)
else:
if value is not None:
if "typeclass" in p:
t = p["typeclass"]
else:
typename = p["typename"]
t = typenames._silk_types[typename]
value = t(value)
self._data[prop] = value
else:
child = self._get_child(prop)
do_set = True
if child is NoneChild:
if value is None:
do_set = False
else:
child = self._get_child(prop, force=True)
if do_set:
if self.storage == "numpy" and p.get("var_array", False):
child.set(value)
else:
child_prop_setter(child, value)
if self.storage == "numpy" and p["optional"]:
self._data["HAS_"+prop] = (value is not None)
def __getattribute__(self, attr):
value = object.__getattribute__(self, attr)
if attr.startswith("_") or attr in ("storage", "dtype"):
return value
class_value = getattr(type(self), attr)
if value is class_value:
raise AttributeError(value)
return value
def __getattr__(self, attr):
try:
ele = self._props[attr]["elementary"]
except KeyError:
raise AttributeError(attr) from None
if ele:
if self.storage == "numpy":
ret = _get_numpy_ele_prop(self, attr)
else:
ret = self._data.get(attr, None)
if ret is None:
assert self._props[attr]["optional"]
else:
ret = self._get_child(attr)
if ret._is_none:
ret = None
return ret
def _print(self, spaces):
name = ""
if not self._anonymous:
name = self.__class__.__name__ + " "
ret = "{0}(\n".format(name)
for propname in self._props:
prop = self._props[propname]
value = getattr(self, propname)
if prop["optional"]:
if value is None:
continue
if self.storage == "numpy" and prop["elementary"]:
substr = value
if self._data[propname].dtype.kind == 'S':
substr = '"' + value + '"'
else:
substr = str(value)
else:
substr = value._print(spaces+2)
ret += "{0}{1} = {2},\n".format(" " * (spaces+2), propname, substr)
ret += "{0})".format(" " * spaces)
return ret
def __str__(self):
return self._print(0)
def __repr__(self):
return self._print(0)
def __eq__(self, other):
if not isinstance(other, SilkObject):
return False
if self.storage == other.storage == "json":
return self._data == other._data
else: #can't use numpy _data because of PTR and different allocation sizes
return self.json() == other.json()
def _clear_data(self):
d = self._data
if self.storage == "numpy":
d.fill(np.zeros_like(d))
else:
for propname in self._props:
prop = self._props[propname]
if prop["elementary"]:
if propname in d:
d.pop(propname)
else:
child = self._get_child(propname)
child._clear_data()
class Worker(GearmanProtocolMixin, asyncio.Protocol):
def __init__(self,
*functions,
loop=None,
grab_type=Type.GRAB_JOB,
timeout=None):
super(Worker, self).__init__(loop=loop)
self.transport = None
self.main_task = None
self.functions = OrderedDict()
self.running = WeakValueDictionary()
self.waiters = []
self.shutting_down = False
self.timeout = timeout
grab_mapping = {
Type.GRAB_JOB: self.grab_job,
Type.GRAB_JOB_UNIQ: self.grab_job_uniq,
Type.GRAB_JOB_ALL: self.grab_job_all,
}
try:
self.grab = grab_mapping[grab_type]
except KeyError:
raise RuntimeError(
'Grab type must be one of GRAB_JOB, GRAB_JOB_UNIQ or GRAB_JOB_ALL'
)
for func_arg in functions:
try:
func, name = func_arg
self.functions[name] = func
except TypeError:
name = func_arg.__name__
self.functions[name] = func_arg
def connection_made(self, transport):
logger.info('Connection is made to %r',
transport.get_extra_info('peername'))
self.transport = transport
if self.timeout is not None:
can_do = partial(self.can_do_timeout, timeout=self.timeout)
else:
can_do = self.can_do
for fname in self.functions.keys():
logger.debug('Registering function %s', fname)
can_do(fname)
self.main_task = self.get_task(self.run())
def connection_lost(self, exc):
self.transport = None
def get_task(self, coro):
return asyncio.ensure_future(coro, loop=self.loop)
async def run(self, ):
no_job = NoJob()
while not self.shutting_down:
self.pre_sleep()
await self.wait_for(Type.NOOP)
response = await self.grab()
if response == no_job:
continue
try:
job_info = self._to_job_info(response)
func = self.functions.get(job_info.function)
if not func:
logger.warning('Failed to find function %s in %s',
job_info.function,
', '.join(self.functions.keys()))
self.work_fail(job_info.handle)
continue
try:
result_or_coro = func(job_info)
if asyncio.iscoroutine(result_or_coro):
task = self.get_task(result_or_coro)
self.running[job_info.handle] = task
result = await task
else:
result = result_or_coro
self.work_complete(job_info.handle, result)
except Exception as ex:
logger.exception('Job (handle %s) resulted with exception',
job_info.handle)
self.work_exception(job_info.handle, str(ex))
finally:
self.running.pop(job_info.handle, None)
except AttributeError:
logger.error('Unexpected GRAB_JOB response %r', response)
async def shutdown(self, graceful=False):
logger.debug('Shutting down worker {}gracefully...'.format(
'' if graceful else 'un'))
self.shutting_down = True
sub_tasks = list(self.running.values())
if graceful:
if sub_tasks:
await asyncio.wait(sub_tasks, loop=self.loop)
else:
async def cancel_and_wait(tasks):
for task in tasks:
task.cancel()
try:
await asyncio.wait(tasks, loop=self.loop)
except asyncio.CancelledError:
pass
if sub_tasks:
await cancel_and_wait(sub_tasks)
self.main_task.cancel()
if self.transport:
self.transport.close()
@staticmethod
def _to_job_info(job_assign):
attrs = ['handle', 'function', 'uuid', 'reducer', 'workload']
values = [getattr(job_assign, attr, None) for attr in attrs]
return JobInfo(*values)
def register_function(self, func, name=''):
if not self.transport:
raise RuntimeError('Worker must be connected to the daemon')
name = name or func.__name__
self.functions[name] = func
return self.can_do(name)
def grab_job_all(self):
self.send(Type.GRAB_JOB_ALL)
return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_ALL)
def grab_job_uniq(self):
self.send(Type.GRAB_JOB_UNIQ)
return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN_UNIQ)
def grab_job(self):
self.send(Type.GRAB_JOB)
return self.wait_for(Type.NO_JOB, Type.JOB_ASSIGN)
def pre_sleep(self):
self.send(Type.PRE_SLEEP)
def can_do(self, function):
self.send(Type.CAN_DO, function)
def can_do_timeout(self, function, timeout):
self.send(Type.CAN_DO_TIMEOUT, function, timeout)
def work_fail(self, handle):
self.send(Type.WORK_FAIL, handle)
def work_exception(self, handle, data):
self.send(Type.WORK_EXCEPTION, handle, data)
def work_complete(self, handle, result):
if result is None:
result = ''
self.send(Type.WORK_COMPLETE, handle, result)
def set_client_id(self, client_id):
self.send(Type.SET_CLIENT_ID, client_id)
class ChannelManager(object):
""" High level interface for channels
This class handles:
* configuration of channels
* high level api to create and remove jobs (notify, remove_job, remove_db)
* get jobs to run
Here is how the runner will use it.
Let's create a channel manager and configure it.
>>> from pprint import pprint as pp
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4,B:1')
>>> db = 'db'
Add a few jobs in channel A with priority 10
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending')
Add a few jobs in channel B with priority 5
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending')
>>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending')
We must now run one job from queue B which has a capacity of 1
and 3 jobs from queue A so the root channel capacity of 4 is filled.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>]
Job A2 is done. Next job to run is A5, even if we have
higher priority job in channel B, because channel B has a capacity of 1.
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A4>]
Job B1 is done. Next job to run is B2 because it has higher priority.
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B2>]
Let's say A1 is done and A6 gets a higher priority. A6 will run next.
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done')
>>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A6>]
"""
def __init__(self):
self._jobs_by_uuid = WeakValueDictionary()
self._root_channel = Channel(name='root', parent=None, capacity=1)
self._channels_by_name = WeakValueDictionary(root=self._root_channel)
@classmethod
def parse_simple_config(cls, config_string):
"""Parse a simple channels configuration string.
The general form is as follow:
channel(.subchannel)*(:capacity(:key(=value)?)*)?,...
If capacity is absent, it defaults to 1.
If a key is present without value, it gets True as value.
When declaring subchannels, the root channel may be omitted
(ie sub:4 is the same as root.sub:4).
Returns a list of channel configuration dictionaries.
>>> from pprint import pprint as pp
>>> pp(ChannelManager.parse_simple_config('root:4'))
[{'capacity': 4, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2'))
[{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:'
... 'sequential:k=v'))
[{'capacity': 4, 'name': 'root'},
{'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}]
>>> pp(ChannelManager.parse_simple_config('root'))
[{'capacity': 1, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('sub:2'))
[{'capacity': 2, 'name': 'sub'}]
"""
res = []
for channel_config_string in config_string.split(','):
config = {}
config_items = channel_config_string.split(':')
name = config_items[0]
if not name:
raise ValueError('Invalid channel config %s: '
'missing channel name' % config_string)
config['name'] = name
if len(config_items) > 1:
capacity = config_items[1]
try:
config['capacity'] = int(capacity)
except:
raise ValueError('Invalid channel config %s: '
'invalid capacity %s' %
(config_string, capacity))
for config_item in config_items[2:]:
kv = config_item.split('=')
if len(kv) == 1:
k, v = kv[0], True
elif len(kv) == 2:
k, v = kv
else:
raise ValueError('Invalid channel config %s: ',
'incorrect config item %s'
(config_string, config_item))
if k in config:
raise ValueError('Invalid channel config %s: '
'duplicate key %s'
(config_string, k))
config[k] = v
else:
config['capacity'] = 1
res.append(config)
return res
def simple_configure(self, config_string):
"""Configure the channel manager from a simple configuration string
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root')
>>> c.capacity
1
>>> cm.simple_configure('root:4,autosub.sub:2')
>>> cm.get_channel_by_name('root').capacity
4
>>> cm.get_channel_by_name('root.autosub').capacity
>>> cm.get_channel_by_name('root.autosub.sub').capacity
2
>>> cm.get_channel_by_name('autosub.sub').capacity
2
"""
for config in ChannelManager.parse_simple_config(config_string):
self.get_channel_from_config(config)
def get_channel_from_config(self, config):
"""Return a Channel object from a parsed configuration.
If the channel does not exist it is created.
The configuration is applied on the channel before returning it.
If some of the parent channels are missing when creating a subchannel,
the parent channels are auto created with an infinite capacity
(except for the root channel, which defaults to a capacity of 1
when not configured explicity).
"""
channel = self.get_channel_by_name(config['name'], autocreate=True)
channel.configure(config)
return channel
def get_channel_by_name(self, channel_name, autocreate=False):
"""Return a Channel object by its name.
If it does not exist and autocreate is True, it is created
with a default configuration and inserted in the Channels structure.
If autocreate is False and the channel does not exist, an exception
is raised.
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root', autocreate=False)
>>> c.name
'root'
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('autosub.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.autosub.sub'
>>> c = cm.get_channel_by_name(None)
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub')
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub')
>>> c.fullname
'root.sub'
"""
if not channel_name or channel_name == self._root_channel.name:
return self._root_channel
if not channel_name.startswith(self._root_channel.name + '.'):
channel_name = self._root_channel.name + '.' + channel_name
if channel_name in self._channels_by_name:
return self._channels_by_name[channel_name]
if not autocreate:
raise ChannelNotFound('Channel %s not found' % channel_name)
parent = self._root_channel
for subchannel_name in channel_name.split('.')[1:]:
subchannel = parent.get_subchannel_by_name(subchannel_name)
if not subchannel:
subchannel = Channel(subchannel_name, parent, capacity=None)
self._channels_by_name[subchannel.fullname] = subchannel
parent = subchannel
return parent
def notify(self, db_name, channel_name, uuid,
seq, date_created, priority, eta, state):
try:
channel = self.get_channel_by_name(channel_name)
except ChannelNotFound:
_logger.warning('unknown channel %s, '
'using root channel for job %s',
channel_name, uuid)
channel = self._root_channel
job = self._jobs_by_uuid.get(uuid)
if job:
# db_name is invariant
assert job.db_name == db_name
# date_created is invariant
assert job.date_created == date_created
# if one of the job properties that influence
# scheduling order has changed, we remove the job
# from the queues and create a new job object
if (seq != job.seq or
priority != job.priority or
eta != job.eta or
channel != job.channel):
_logger.debug("job %s properties changed, rescheduling it",
uuid)
self.remove_job(uuid)
job = None
if not job:
job = ChannelJob(db_name, channel, uuid,
seq, date_created, priority, eta)
self._jobs_by_uuid[uuid] = job
# state transitions
if not state or state == DONE:
job.channel.set_done(job)
elif state == PENDING:
job.channel.set_pending(job)
elif state in (ENQUEUED, STARTED):
job.channel.set_running(job)
elif state == FAILED:
job.channel.set_failed(job)
else:
_logger.error("unexpected state %s for job %s", state, job)
def remove_job(self, uuid):
job = self._jobs_by_uuid.get(uuid)
if job:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def remove_db(self, db_name):
for job in self._jobs_by_uuid.values():
if job.db_name == db_name:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def get_jobs_to_run(self, now):
return self._root_channel.get_jobs_to_run(now)
class Boss:
def __init__(self, glfw_window, opts, args):
self.window_id_map = WeakValueDictionary()
startup_session = create_session(opts, args)
self.cursor_blinking = True
self.window_is_focused = True
self.glfw_window_title = None
self.shutting_down = False
self.child_monitor = ChildMonitor(
glfw_window.window_id(), self.on_child_death,
DumpCommands(args)
if args.dump_commands or args.dump_bytes else None)
set_boss(self)
self.current_font_size = opts.font_size
cell_size.width, cell_size.height = set_font_family(opts)
self.opts, self.args = opts, args
self.glfw_window = glfw_window
glfw_window.framebuffer_size_callback = self.on_window_resize
glfw_window.window_focus_callback = self.on_focus
load_shader_programs()
self.tab_manager = TabManager(opts, args)
self.tab_manager.init(startup_session)
self.activate_tab_at = self.tab_manager.activate_tab_at
layout_sprite_map(cell_size.width, cell_size.height,
render_cell_wrapper)
@property
def current_tab_bar_height(self):
return self.tab_manager.tab_bar_height
def __iter__(self):
return iter(self.tab_manager)
def iterwindows(self):
for t in self:
yield from t
def add_child(self, window):
self.child_monitor.add_child(window.id, window.child.pid,
window.child.child_fd, window.screen)
self.window_id_map[window.id] = window
def on_child_death(self, window_id):
w = self.window_id_map.pop(window_id, None)
if w is not None:
w.on_child_death()
def close_window(self, window=None):
if window is None:
window = self.active_window
self.child_monitor.mark_for_close(window.id)
def close_tab(self, tab=None):
if tab is None:
tab = self.active_tab
for window in tab:
self.close_window(window)
def start(self):
if not getattr(self, 'io_thread_started', False):
self.child_monitor.start()
self.io_thread_started = True
def on_window_resize(self, window, w, h):
viewport_size.width, viewport_size.height = w, h
self.tab_manager.resize()
def increase_font_size(self):
self.change_font_size(
min(self.opts.font_size * 5,
self.current_font_size + self.opts.font_size_delta))
def decrease_font_size(self):
self.change_font_size(
max(MINIMUM_FONT_SIZE,
self.current_font_size - self.opts.font_size_delta))
def restore_font_size(self):
self.change_font_size(self.opts.font_size)
def change_font_size(self, new_size):
if new_size == self.current_font_size:
return
self.current_font_size = new_size
w, h = cell_size.width, cell_size.height
windows = tuple(filter(None, self.window_id_map.values()))
cell_size.width, cell_size.height = set_font_family(
self.opts, override_font_size=self.current_font_size)
layout_sprite_map(cell_size.width, cell_size.height,
render_cell_wrapper)
for window in windows:
window.screen.rescale_images(w, h)
self.resize_windows_after_font_size_change()
for window in windows:
window.screen.refresh_sprite_positions()
self.tab_manager.refresh_sprite_positions()
def resize_windows_after_font_size_change(self):
self.tab_manager.resize()
glfw_post_empty_event()
def tabbar_visibility_changed(self):
self.tab_manager.resize(only_tabs=True)
glfw_post_empty_event()
@property
def active_tab(self):
return self.tab_manager.active_tab
def is_tab_visible(self, tab):
return self.active_tab is tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
def dispatch_special_key(self, key, scancode, action, mods):
# Handles shortcuts, return True if the key was consumed
key_action = get_shortcut(self.opts.keymap, mods, key, scancode)
self.current_key_press_info = key, scancode, action, mods
return self.dispatch_action(key_action)
def dispatch_action(self, key_action):
if key_action is not None:
f = getattr(self, key_action.func, None)
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
tab = self.active_tab
if tab is None:
return False
window = self.active_window
if window is None:
return False
if key_action is not None:
f = getattr(tab, key_action.func,
getattr(window, key_action.func, None))
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
key, scancode, action, mods = self.current_key_press_info
data = get_sent_data(self.opts.send_text_map, key, scancode, mods,
window, action)
if data:
window.write_to_child(data)
return True
return False
def combine(self, *actions):
for key_action in actions:
self.dispatch_action(key_action)
def on_focus(self, window, focused):
self.window_is_focused = focused
w = self.active_window
if w is not None:
w.focus_changed(focused)
def display_scrollback(self, data):
if self.opts.scrollback_in_new_tab:
self.display_scrollback_in_new_tab(data)
else:
tab = self.active_tab
if tab is not None:
tab.new_special_window(
SpecialWindow(self.opts.scrollback_pager, data,
_('History')))
def switch_focus_to(self, window_idx):
tab = self.active_tab
tab.set_active_window_idx(window_idx)
old_focus = tab.active_window
if not old_focus.destroyed:
old_focus.focus_changed(False)
tab.active_window.focus_changed(True)
def send_fake_scroll(self, window_idx, amt, upwards):
tab = self.active_tab
w = tab.windows[window_idx]
k = get_key_map(w.screen)[GLFW_KEY_UP if upwards else GLFW_KEY_DOWN]
w.write_to_child(k * amt)
def open_url(self, url):
if url:
open_url(url, self.opts.open_url_with)
def gui_close_window(self, window):
window.destroy()
for tab in self.tab_manager:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
self.tab_manager.remove(tab)
tab.destroy()
if len(self.tab_manager) == 0:
if not self.shutting_down:
self.glfw_window.set_should_close(True)
glfw_post_empty_event()
def destroy(self):
self.shutting_down = True
self.child_monitor.shutdown()
wakeup()
self.child_monitor.join()
for t in self.tab_manager:
t.destroy()
del self.tab_manager
destroy_sprite_map()
destroy_global_data()
del self.glfw_window
def paste_to_active_window(self, text):
if text:
w = self.active_window
if w is not None:
w.paste(text)
def paste_from_clipboard(self):
text = self.glfw_window.get_clipboard_string()
self.paste_to_active_window(text)
def paste_from_selection(self):
text = get_primary_selection()
self.paste_to_active_window(text)
def set_primary_selection(self):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
set_primary_selection(text)
def next_tab(self):
self.tab_manager.next_tab()
def previous_tab(self):
self.tab_manager.next_tab(-1)
def new_tab(self):
self.tab_manager.new_tab()
def move_tab_forward(self):
self.tab_manager.move_tab(1)
def move_tab_backward(self):
self.tab_manager.move_tab(-1)
def display_scrollback_in_new_tab(self, data):
self.tab_manager.new_tab(special_window=SpecialWindow(
self.opts.scrollback_pager, data, _('History')))
class PersistentList(object):
"""
Sequence object that is persistently stored
:param store_uri: URI for storing buckets; see :py:class:`~BaseBucketStore`
:type store_uri: :py:class:`str`
:param bucket_length: number of items to store per bucket
:type bucket_length: :py:class:`int`
:param cache_size: number of buckets to LRU-cache in memory
:type cache_size: :py:class:`int`
"""
persistent_defaults = {
'bucket_length': 32,
}
def __init__(self, store_uri, bucket_length=NOTSET, cache_size=3):
self._bucket_store = BaseBucketStore.from_uri(store_uri=store_uri, default_scheme='file')
# set empty fields
self._bucket_length = None
self._bucket_count = 0
self._len = 0
self._bucket_cache = None
self._cache_size = None
self.bucket_key_fmt = None
# load current settings
try:
for attr, value in self._bucket_store.fetch_head().items():
setattr(self, attr, value)
except BucketNotFound:
pass
# apply new settings
self.bucket_length = bucket_length
# LRU store for objects fetched from disk
self.cache_size = cache_size
# weakref store for objects still in use
self._active_buckets = WeakValueDictionary()
self._active_items = WeakValueDictionary()
# calcualate metadata
self._length = self._fetch_length()
# store new settings
self._store_head()
# Settings
def _store_head(self):
"""
Store the meta-information of the dict
"""
self._bucket_store.store_head({
attr: getattr(self, attr) for attr in
('bucket_length', '_bucket_count')
})
def _fetch_length(self):
"""Calculate the length of the list from the persistent store"""
if self._bucket_count == 0:
return 0
last_bucket = self._fetch_bucket(self.bucket_key_fmt % (self._bucket_count - 1))
return (self._bucket_count - 1) * self._bucket_length + len(last_bucket)
def _update_bucket_key_fmt(self):
# key: count, salt, index
self.bucket_key_fmt = "dlistbkt_%(bucket_length)xs%%x" % {
'bucket_length': self.bucket_length,
}
@property
def _length(self):
return self._len
@_length.setter
def _length(self, value):
# detect change of bucket count
if self._bucket_count != value // self._bucket_length:
self._bucket_count = value // self._bucket_length
self._store_head()
self._len = value
# exposed settings
@property
def cache_size(self):
return self._cache_size
@cache_size.setter
def cache_size(self, value):
self._cache_size = int(value or 1)
self._bucket_cache = deque(maxlen=self.cache_size)
@property
def bucket_length(self):
"""
Get/Set the ``bucket_length`` of the persistent mapping
:note: Setting ``bucket_length`` causes **all** buckets storing data to be
recreated. Until the new buckets have been created, changes to the
mapping content may be silently dropped.
"""
return self._bucket_length
@bucket_length.setter
def bucket_length(self, value):
# default if unset
if value == NOTSET:
if self._bucket_length is not None:
return
self._bucket_length = self.persistent_defaults['bucket_length']
else:
value = int(value)
if value < 1:
raise ValueError('At least one item per bucket must be used')
# no change
elif self._bucket_length == value:
return
# uninitialized, we don't have content yet
elif self._bucket_length is None:
self._bucket_length = value
# TODO: allow resizing backend
else:
raise NotImplementedError('Changing bucket count not implemented yet')
# apply secondary settings
self._update_bucket_key_fmt()
# bucket management
@property
def _bucket_keys(self):
"""List of used bucket keys"""
return [self._bucket_key(idx) for idx in range(0, self._length, self._bucket_length)]
def _bucket_key(self, index):
"""
Create the bucket identifier for a given key
:param index: key to the content in-memory
:return: key to the bucket stored persistently
:rtype: str
"""
if index < 0:
index += self._length
return self.bucket_key_fmt % (index // self._bucket_length)
def _fetch_bucket(self, bucket_key):
"""
Return a bucket from disk or create a new one
:param bucket_key: key for the bucket
:return: bucket for ``bucket_key``
:rtype: :py:class:`~DictBucket`
"""
try:
bucket = self._bucket_store.fetch_bucket(bucket_key=bucket_key)
except BucketNotFound:
self._store_head()
bucket = ListBucket()
self._active_buckets[bucket_key] = bucket
self._bucket_cache.appendleft(bucket)
return bucket
def _get_bucket(self, bucket_key):
"""
Return the appropriate bucket
May return the cached bucket if available.
:param bucket_key: key for the bucket
:return: bucket for ``bucket_key``
:rtype: :py:class:`~DictBucket`
"""
try:
return self._active_buckets[bucket_key]
except KeyError:
return self._fetch_bucket(bucket_key)
def _store_bucket(self, bucket_key, bucket=None):
"""
Store a bucket on disk
:param bucket_key: key for the entire bucket
"""
if bucket is None:
try:
bucket = self._active_buckets[bucket_key]
except KeyError:
return
if bucket:
self._bucket_store.store_bucket(bucket_key=bucket_key, bucket=bucket)
# free empty buckets
else:
self._bucket_store.free_bucket(bucket_key)
# cache management
# Item cache
def _set_cached_item(self, key, item):
"""Cache reference to existing item"""
try:
self._active_items[key] = item
except TypeError:
pass
def _get_cached_item(self, key):
"""Get reference to existing item; raises KeyError if item cannot be fetched"""
try:
return self._active_items[key]
except TypeError:
raise KeyError
def _del_cached_item(self, key):
"""Release reference to existing item"""
try:
del self._active_items[key]
except (TypeError, KeyError):
pass
# paths and files
def flush(self):
"""
Commit all outstanding changes to persistent store
"""
for bucket_key, bucket in self._active_buckets.values():
self._store_bucket(bucket_key, bucket)
# sequence interface
def __getitem__(self, pos):
if isinstance(pos, slice):
return self._get_slice(pos)
return self._get_item(pos)
def _get_item(self, index):
# - use cached reference to existing item
# - fetch item from cached reference to existing bucket
# - fetch item from fetched bucket
try:
return self._get_cached_item(index)
except KeyError:
bucket = self._get_bucket(self._bucket_key(index))
item = bucket[index % self._bucket_length]
self._set_cached_item(index, item)
return item
def _get_slice(self, positions):
start_idx, stop_idx, stride = positions.indices(self._length)
list_slice = []
# fetch sub-slice from each bucket
while start_idx < stop_idx:
bucket = self._get_bucket(self._bucket_key(start_idx))
# stop_idx in next bucket
if stop_idx // self._bucket_length > start_idx // self._bucket_length:
list_slice.extend(bucket[start_idx % self._bucket_length::stride])
slice_length = math.ceil((self._bucket_length - (start_idx % self._bucket_length)) / stride)
# stop_idx in this bucket
else:
list_slice.extend(bucket[start_idx % self._bucket_length:stop_idx % self._bucket_length:stride])
slice_length = math.ceil(((stop_idx - start_idx) % self._bucket_length) / stride)
# advance to next bucket
start_idx += slice_length * stride
return list_slice
def __setitem__(self, pos, value):
if isinstance(pos, slice):
self._set_slice(pos, value)
else:
self._set_item(pos, value)
def _set_slice(self, positions, sequence):
start_idx, stop_idx, stride = positions.indices(self._length)
sequence = list(sequence)
# fetch sub-slice from each bucket
while start_idx < stop_idx:
bucket_key = self._bucket_key(start_idx)
bucket = self._get_bucket(bucket_key)
# stop_idx in next bucket
if stop_idx // self._bucket_length > start_idx // self._bucket_length:
slice_length = math.ceil((self._bucket_length - (start_idx % self._bucket_length)) / stride)
bucket[start_idx % self._bucket_length::stride] = sequence[:slice_length]
# stop_idx in this bucket
else:
slice_length = math.ceil(((stop_idx - start_idx) % self._bucket_length) / stride)
bucket[start_idx % self._bucket_length:stop_idx % self._bucket_length:stride] = sequence[:slice_length]
self._store_bucket(bucket_key, bucket)
# advance to next bucket
start_idx += slice_length * stride
sequence[:] = sequence[slice_length:]
def _set_item(self, index, value):
bucket_key = self._bucket_key(index)
bucket = self._get_bucket(bucket_key)
bucket[index % self._bucket_length] = value
self._store_bucket(bucket_key, bucket)
# update item cache
self._set_cached_item(index, value)
def __delitem__(self, index):
if isinstance(index, slice):
self._del_slice(index)
else:
self._del_item(index)
def _del_slice(self, positions):
start_idx, stop_idx, stride = positions.indices(self._length)
print(start_idx, stop_idx, stride)
if start_idx == stop_idx:
return
# TODO: make this work on sequences
for idx in range(start_idx, stop_idx, stride):
self._del_item(idx)
# consecutive sequence
#if stride == 1:
# pass
def _del_item(self, pos):
self._del_cached_item(pos)
# delete first element, append first element with next bucket
this_bucket_key = self._bucket_key(pos)
this_bucket = self._get_bucket(this_bucket_key)
# remaining elements move one forward
del this_bucket[pos % self._bucket_length] # if this fails, IndexError for entire collection
# index of first element of next bucket
pos = (pos // self._bucket_length + 1) * self._bucket_length
# fill missing last element with first of next bucket
while pos < self._length - 1:
next_bucket_key = self._bucket_key(pos)
next_bucket = self._get_bucket(next_bucket_key)
this_bucket.append(next_bucket.pop(0))
self._store_bucket(this_bucket_key, this_bucket)
this_bucket_key = next_bucket_key
this_bucket = next_bucket
pos += self._bucket_length
# store last bucket
self._store_bucket(this_bucket_key, this_bucket)
self._length -= 1
# container protocol
def __len__(self):
return self._length
# list methods
def append(self, item):
bucket_key = self._bucket_key(self._length)
bucket = self._get_bucket(bucket_key)
bucket.append(item)
self._store_bucket(bucket_key, bucket)
# update item cache
self._set_cached_item(self._length, item)
self._length += 1
def extend(self, sequence):
# split sequence into chunks to fill each bucket
while sequence:
bucket_key = self._bucket_key(self._length)
bucket = self._get_bucket(bucket_key)
bucket_sequence, sequence = \
sequence[:self._bucket_length - len(bucket)], sequence[self._bucket_length - len(bucket):]
bucket.extend(bucket_sequence)
self._store_bucket(bucket_key, bucket)
# update item cache
for idx, item in enumerate(bucket_sequence):
self._set_cached_item(self._length + idx, item)
self._length += len(bucket_sequence)
def clear(self):
# clear persistent storage
for bucket_key in self._bucket_keys:
self._bucket_store.free_bucket(bucket_key=bucket_key)
self._length = 0
self._bucket_count = 0
self._store_head()
# reset caches
self._bucket_cache = deque(maxlen=self.cache_size)
self._active_buckets = type(self._active_buckets)()
#self._active_items = type(self._active_items)()
def insert(self, index, value):
raise NotImplementedError
def pop(self, index=None):
raise NotImplementedError
def remove(self, value):
raise NotImplementedError
def reverse(self, value):
raise NotImplementedError
def __str__(self):
return '[<%s>]' % ('>, <'.join(
str(self._get_bucket(bucket_key))[1:-1] for bucket_key in self._bucket_keys
))
class Signal(object):
def __init__(self, *args):
self.__slots = WeakValueDictionary()
for slot in args:
self.connect(slot)
def __call__(self, slot, *args, **kwargs):
"""
Emit signal. If slot passed signal will be called only for this
slot, for all connected slots otherwise.
Calling this method directly lead to immediate signal processing.
It may be not thread-safe. Use emit method from this module for
delayed calling of signals.
"""
if slot is not None:
slots = (self.__slots[self.key(slot)], )
else:
slots = self.__slots.values()
for func in slots:
func(*args, **kwargs)
def key(self, slot):
"""
Get local key name for slot.
"""
if type(slot) == types.FunctionType:
key = (slot.__module__, slot.__name__)
elif type(slot) == types.MethodType:
key = (slot.__func__, id(slot.__self__))
elif isinstance(slot, basestring):
if not slot in registred_slots.keys():
raise ValueError('Slot {0} does not exists.'.format(slot))
key = slot
else:
raise ValueError('Slot {0} has non-slot type'.format(slot))
return key
def connect(self, slot):
"""
Connect signal to slot. Slot may be function, instance method
or name of function perviously registred by `slot` decorator.
"""
key = self.key(slot)
if type(slot) == types.FunctionType:
self.__slots[key] = slot
elif type(slot) == types.MethodType:
self.__slots[key] = partial(slot.__func__, slot.__self__)
elif isinstance(slot, basestring):
self.__slots[key] = registred_slots[slot]
def disconnect(self, slot):
"""
Remove slot from signal connetions.
"""
key = self.key(slot)
del self.__slots[key]
def clear(self):
"""
Disconnect all slots from signal.
"""
self.__slots.clear()
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(
self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange
)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(
onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange
)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node, "task") or node.task is None:
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph["user"]
taskDefs = graph["tasks"]
poolName = graph["poolName"]
if "maxRN" in graph.items():
maxRN = int(graph["maxRN"])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef["type"] == "Task":
# logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"])
# logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"])
task = self._createTaskFromJSON(taskDef, user)
elif taskDef["type"] == "TaskGroup":
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph["root"]]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef["type"] == "TaskGroup":
for taskIndex in taskDef["tasks"]:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef["dependencies"], list):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"]
)
if not all(
(
(isinstance(i, int) and isinstance(sl, list) and all((isinstance(s, int) for s in sl)))
for (i, sl) in taskDef["dependencies"]
)
):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"]
)
for (taskIndex, statusList) in taskDef["dependencies"]:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts(child)
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition["name"]
parent = None
arguments = taskGroupDefinition["arguments"]
environment = taskGroupDefinition["environment"]
requirements = taskGroupDefinition["requirements"]
maxRN = taskGroupDefinition["maxRN"]
priority = taskGroupDefinition["priority"]
dispatchKey = taskGroupDefinition["dispatchKey"]
strategy = taskGroupDefinition["strategy"]
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition["tags"]
timer = None
if "timer" in taskGroupDefinition.keys():
timer = taskGroupDefinition["timer"]
return TaskGroup(
id,
name,
parent,
user,
arguments,
environment,
requirements,
maxRN,
priority,
dispatchKey,
strategy,
tags=tags,
timer=timer,
)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition["name"]
runner = taskDefinition["runner"]
arguments = taskDefinition["arguments"]
environment = taskDefinition["environment"]
requirements = taskDefinition["requirements"]
maxRN = taskDefinition["maxRN"]
priority = taskDefinition["priority"]
dispatchKey = taskDefinition["dispatchKey"]
validationExpression = taskDefinition["validationExpression"]
minNbCores = taskDefinition["minNbCores"]
maxNbCores = taskDefinition["maxNbCores"]
ramUse = taskDefinition["ramUse"]
lic = taskDefinition["lic"]
tags = taskDefinition["tags"]
runnerPackages = taskDefinition.get("runnerPackages", "")
watcherPackages = taskDefinition.get("watcherPackages", "")
timer = None
if "timer" in taskDefinition.keys():
timer = taskDefinition["timer"]
maxAttempt = taskDefinition.get("maxAttempt", 1)
task = Task(
None,
name,
None,
user,
maxRN,
priority,
dispatchKey,
runner,
arguments,
validationExpression,
[],
requirements,
minNbCores,
maxNbCores,
ramUse,
environment,
lic=lic,
tags=tags,
timer=timer,
maxAttempt=maxAttempt,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages,
)
for commandDef in taskDefinition["commands"]:
description = commandDef["description"]
arguments = commandDef["arguments"]
cmd = Command(
None, description, task, arguments, runnerPackages=runnerPackages, watcherPackages=watcherPackages
)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId())
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId())
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.renderNodeMaxId = max(self.renderNodeMaxId, StatDB.getRenderNodesMaxId())
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId())
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId())
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
self.poolShareMaxId = max(self.poolShareMaxId, StatDB.getPoolSharesMaxId())
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id is None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId())
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id is None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId())
if node.parent is None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id is None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id, StatDB.getRenderNodesMaxId())
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." % rendernode.name)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
if field == "performance":
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id is None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id, StatDB.getPoolsMaxId())
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id, StatDB.getCommandsMaxId())
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
if command.task is not None:
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id, StatDB.getPoolSharesMaxId())
self.poolShares[poolShare.id] = poolShare
class NodeManager(BaseNodeManager):
NODE_CLASS = Node
def __init__(self, project):
super().__init__()
# XXX [!] cycle reference
self.project = project
self.data = WeakValueDictionary()
self.root = None
def update_root(self, root_project, root_project_children):
self.root = self.new_root_node(root_project, root_project_children)
def __setitem__(self, projectid, node):
assert self.check_not_exist_node(node)
assert projectid == node.projectid
self.data[node.projectid] = node
assert self.check_exist_node(node)
def __delitem__(self, node):
assert self.check_exist_node(node)
del self.data[node.projectid]
assert self.check_not_exist_node(node)
def __iter__(self):
# how to support lock for iter? just copy dict?
return iter(self.data.values())
def __contains__(self, node):
if node is None:
return False
newnode = self.data.get(node.projectid)
return node is newnode # ?!
def __len__(self):
return len(self.data)
def __bool__(self):
return len(self) != 0
@property
def get(self):
return self.data.get
@property
def clear(self):
return self.data.clear
# TODO: add expend node. (not operation.)
def check_exist_node(self, node):
original_node = self.get(node.projectid)
if original_node is None:
raise WFNodeError("{!r} is not exists.".format(node))
elif original_node is not node:
raise WFNodeError("{!r} is invalid node.".format(node))
return True
def check_not_exist_node(self, node):
if node in self:
raise WFNodeError("{!r} is already exists.".format(node))
return True
def new_void_node(self, uuid=None):
return self.NODE_CLASS.from_void(uuid, project=self.project)
def new_node_from_json(self, data, parent=None):
return self.NODE_CLASS.from_json_with_project(data, parent=parent, project=self.project)
def add(self, node, recursion=True):
assert recursion is True
added_nodes = 0
def register_node(node):
nonlocal added_nodes
self[node.projectid] = node
added_nodes += 1
register_node(node)
if recursion:
def deep(node):
for subnode in node:
register_node(subnode)
deep(subnode)
deep(node)
return added_nodes
def remove(self, node, recursion=True):
assert self.check_exist_node(node)
if node.parent is not None:
assert self.check_exist_node(node.parent)
if node in node.parent:
raise WFNodeError("node are still exists in parent node.")
removed_nodes = 0
def unregister_node(node):
nonlocal removed_nodes
del node.parent
del self[node]
removed_nodes += 1
unregister_node(node)
if recursion:
def deep(node):
if not node:
return
child_nodes, node.ch = node.ch[:], None
for child in child_nodes:
unregister_node(node)
deep(child)
deep(node)
return removed_nodes
def new_root_node(self, root_project, root_project_children):
# XXX [!] project is Project, root_project is root node. ?!
if root_project is None:
root_project = dict(id=DEFAULT_ROOT_NODE_ID)
else:
root_project.update(id=DEFAULT_ROOT_NODE_ID)
# in shared mode, root will have uuid -(replace)> DEFAULT_ROOT_NODE_ID
root_project.update(ch=root_project_children)
root = self.new_node_from_json(root_project)
self.add(root, recursion=True)
return root
@property
def pretty_print(self):
return self.root.pretty_print
class Container(IContainer):
def __init__(self, parent: Optional["Container"] = None):
super().__init__()
self.__uuid = uuid.uuid4()
self.__children = WeakValueDictionary()
self.__cache = {}
self._parent = parent
if parent is not None:
parent._register_child(self)
self._register_instance_indexer = ContainerRegisterInstanceIndexer(
self)
self._register_factory_indexer = ContainerRegisterFactoryIndexer(self)
self._register_type_indexer = ContainerRegisterTypeIndexer(self)
self._resolve_indexer = ContainerResolveIndexer(self.resolve_bean)
self._try_resolve_indexer = ContainerResolveIndexer(
self.try_resolve_bean)
from grundzeug.container.plugins import \
ContainerBeanListResolutionPlugin, \
ContainerSingleValueResolutionPlugin, \
ContainerSpecialResolutionPlugin
if parent is None:
self._plugins = [
ContainerBeanListResolutionPlugin(),
ContainerSingleValueResolutionPlugin(),
ContainerSpecialResolutionPlugin()
]
else:
self._plugins = []
self._plugin_storage = WeakKeyDictionary()
def __cache_delete(self, key):
if key in self.__cache:
del self.__cache[key]
def __cache_put(self, key, value):
self.__cache[key] = value
@property
def uuid(self):
return self.__uuid
@property
def children(self) -> typing.List["IContainer"]:
return list(self.__children.values())
def _register_child(self, container: IContainer):
self.__children[container.uuid] = container
def add_plugin(self, plugin: ContainerResolutionPlugin) -> IContainer:
if self._parent is not None:
self._parent.add_plugin(plugin)
else:
self._plugins.insert(0, plugin)
return self
@property
def plugins(self):
if self._parent is not None:
return self._parent.plugins
else:
return self._plugins
@property
def parent(self):
return self._parent
def _register(self, key: RegistrationKey,
registration: ContainerRegistration):
for plugin in self.plugins:
if plugin.register(key=key,
registration=registration,
container=self):
self.__cache_delete(key=key)
return
def _register_instance(self,
instance: BeanT,
contract: Optional[ContractT] = None,
bean_name: Optional[str] = None) -> "IContainer":
if contract is None:
contract = type(instance)
key = RegistrationKey(contract, bean_name)
registration = InstanceContainerRegistration(container=self,
key=key,
instance=instance)
self._register(key, registration)
return self
@property
def register_instance(self) -> IContainerRegisterInstanceIndexer:
return self._register_instance_indexer
def _register_factory(
self,
contract: ContractT,
factory: Callable[[], Any],
bean_name: Optional[str] = None,
registration_type: Type[ContainerRegistration] = None
) -> "IContainer":
key = RegistrationKey(contract, bean_name)
if registration_type is None:
registration_type = ContainerFactoryContainerRegistration
registration = registration_type(container=self,
key=key,
factory=factory)
self._register(key, registration)
return self
@property
def register_factory(self) -> IContainerRegisterFactoryIndexer:
return self._register_factory_indexer
def _register_type(
self,
contract: ContractT,
clazz: Optional[type] = None,
bean_name: Optional[str] = None,
registration_type: Type[ContainerRegistration] = None
) -> "IContainer":
if clazz is None:
clazz = contract
self._register_factory(contract=contract,
factory=clazz,
bean_name=bean_name,
registration_type=registration_type)
return self
@property
def register_type(self) -> IContainerRegisterTypeIndexer:
return self._register_type_indexer
def try_resolve_bean(self,
contract: ContractT,
bean_name: Optional[str] = None
) -> Union[BeanT, BEAN_NOT_FOUND_TYPE]:
if bean_name is None and contract == Container:
return self
key = RegistrationKey(contract, bean_name)
if key in self.__cache:
return self.__cache[key].get()
plugins = list(self.plugins)
states = [
plugin.resolve_bean_create_initial_state(key, self)
for plugin in plugins
]
current_container = self
while current_container is not None:
for i, plugin in enumerate(self.plugins):
res = plugin.resolve_bean_reduce(key, states[i], self,
current_container)
if isinstance(res, ReturnMessage):
resolver = res.resolver
if resolver.is_cacheable:
self.__cache_put(key, resolver)
return resolver.get()
elif isinstance(res, NotFoundMessage):
states[i] = res.state
elif isinstance(res, ContinueMessage):
states[i] = res.state
break
else:
raise NotImplementedError()
current_container = current_container.parent
for i, plugin in enumerate(self.plugins):
res = plugin.resolve_bean_postprocess(key, states[i], self)
if isinstance(res, ReturnMessage):
resolver = res.resolver
if resolver.is_cacheable:
self.__cache_put(key, resolver)
return resolver.get()
elif isinstance(res, NotFoundMessage):
pass
else:
raise NotImplementedError()
return BEAN_NOT_FOUND
def resolve_bean(self,
contract: ContractT,
bean_name: Optional[str] = None) -> BeanT:
bean = self.try_resolve_bean(contract=contract, bean_name=bean_name)
if bean is BEAN_NOT_FOUND:
raise ResolutionFailedError(
f"Bean not found: contract={contract}, bean_name={bean_name}")
return bean
@property
def resolve(self) -> IContainerResolveIndexer:
return self._resolve_indexer
@property
def try_resolve(self) -> IContainerResolveIndexer:
return self._try_resolve_indexer
def inject(self, func: FuncT) -> FuncT:
from grundzeug.container.di import inject
return inject(self, func)
def get_kwargs_to_inject(self, func: FuncT) -> typing.Dict[str, Any]:
from grundzeug.container.di import get_kwargs_to_inject
return get_kwargs_to_inject(self, func)
def get_plugin_storage(self, plugin: ContainerResolutionPlugin):
if plugin not in self._plugin_storage:
self._plugin_storage[plugin] = {}
return self._plugin_storage[plugin]
class HeapManager(threading.Thread):
"""
@summary: This class is intended to manage all dataClay objects in runtime's memory.
"""
""" Logger """
logger = None
def __init__(self, theruntime):
"""
@postcondition: Constructor of the object called from sub-class
@param theruntime: Runtime being managed
"""
""" Memory objects. This dictionary must contain all objects in runtime memory (client or server), as weakrefs. """
self.inmemory_objects = WeakValueDictionary()
threading.Thread.__init__(self)
self._finished = threading.Event()
""" Runtime being monitorized. Java uses abstract functions to get the field in the proper type (EE or client) due to type-check. Not needed here. """
self.runtime = theruntime
self.logger = logging.getLogger(__name__)
self.daemon = True
self.logger.debug("HEAP MANAGER created.")
def get_heap(self):
return self.inmemory_objects
def shutdown(self):
"""Stop this thread"""
self.logger.debug("HEAP MANAGER shutdown request received.")
self._finished.set()
def run(self):
"""
@postcondition: Overrides run function
"""
gc_check_time_interval_seconds = Configuration.MEMMGMT_CHECK_TIME_INTERVAL / 1000.0
while 1:
self.logger.trace("HEAP MANAGER THREAD is awake...")
if self._finished.isSet(): break
self.run_task()
# sleep for interval or until shutdown
self.logger.trace("HEAP MANAGER THREAD is going to sleep...")
self._finished.wait(gc_check_time_interval_seconds)
self.logger.debug("HEAP MANAGER THREAD Finished.")
def _add_to_inmemory_map(self, dc_object):
"""
@postcondition: the object is added to inmemory map
@param dc_object: object to add
"""
oid = dc_object.get_object_id()
self.inmemory_objects[oid] = dc_object
def remove_from_heap(self, object_id):
"""
@postcondition: Remove reference from Heap. Even if we remove it from the heap,
the object won't be Garbage collected till HeapManager flushes the object and releases it.
@param object_id: id of object to remove from heap
"""
self.inmemory_objects.pop(object_id)
def get_from_heap(self, object_id):
"""
@postcondition: Get from heap.
@param object_id: id of object to get from heap
@return Object with id provided in heap or None if not found.
"""
try:
obj = self.inmemory_objects[object_id]
self.logger.debug("Hit in Heap object %s" % str(object_id))
return obj
except KeyError:
self.logger.debug("Miss in Heap object %s" % str(object_id))
return None
def exists_in_heap(self, object_id):
"""
@postcondition: Exists from heap.
@param object_id: id of object to get from heap
@return True if exists. False otherwise.
"""
try:
if self.inmemory_objects[object_id] is None:
return False
else:
return True
except KeyError:
return False
def heap_size(self):
"""
@postcondition: Get heap size.
@return Heap size
"""
return len(self.inmemory_objects)
def count_loaded_objs(self):
num_loaded_objs = 0
for obj in self.inmemory_objects.values():
if obj.is_loaded():
num_loaded_objs = num_loaded_objs + 1
return num_loaded_objs
@abstractmethod
def flush_all(self):
pass
@abstractmethod
def run_task(self):
pass
def cleanReferencesAndLockers(self):
"""
@postcondition: Clean references and lockers not being used.
"""
self.runtime.locker_pool.cleanLockers()
class WorkerCollection(object):
def __init__(self,
capabilities,
parallel_tasks=10,
parallel_tasks_per_worker=10,
worker_max_idle=300):
self.logger = logging.getLogger('root')
self.capabilities = capabilities
self.parallel_tasks_per_worker = parallel_tasks_per_worker
self.worker_max_idle = worker_max_idle
self.workers = WeakValueDictionary()
self.task_queue = gevent.queue.JoinableQueue(maxsize=parallel_tasks)
def register_response_queue(self, response_queue):
self.response_queue = response_queue
self.logger.info("Registered worker collection for {caps}".format(
caps=", ".join(self.capabilities.keys())))
def get_worker(self, NodeID):
if NodeID not in self.workers or self.workers[
NodeID].shutdown_in_progress:
self.workers[NodeID] = Worker(self, NodeID, self.response_queue,
self.parallel_tasks_per_worker,
self.worker_max_idle)
return self.workers[NodeID]
def remove_worker(self, worker):
self.workers = {
n: w
for n, w in self.workers.items() if w is not worker
}
def shutdown_workers(self):
self.task_queue.join()
items = list(self.workers.values())
for i in items:
i.shutdown()
del items
def handle_requests_per_worker(self):
self.logger.info("Started forwarding requests")
while True:
anum, capability, timeout, params, zmq_info = self.task_queue.get()
try:
worker = self.get_worker(params['NodeID'])
capability = self.capabilities[capability]
try:
worker.add_action(
capability.action_class(anum, params['NodeID'],
zmq_info, timeout, params,
**capability.params))
except Exception as e:
self.logger.debug(e)
dummy_action = FailedAction(anum, params['NodeID'],
zmq_info, timeout, params)
dummy_action.statusmsg += "\n" + traceback.format_exc()
worker.add_action(dummy_action)
except KeyError:
self.logger.error(
"Unknown capability {cap}".format(cap=capability))
finally:
del worker, capability
class Entity:
def __init__(self, entType, entValue, entField):
if isinstance(entField, Field):
self.type = entType
self.value = entValue
self.field = entField
self.group = None
self.links = WeakValueDictionary() # dict of linked entities
self.field.registerEntity(self) # update the entity registry
else:
raise TypeError("Invalid field argument, field instance expected!")
def linkTo(self, eTwo):
''' Linking operation is bi-directional, affects both entities equally.'''
# check if entities not already linked
if Edge.linkId(self, eTwo) not in self.links.keys():
# update both entities' list of links
# create a new edge
newlink = Edge(self, eTwo, self.field)
self.links[newlink.id] = eTwo
eTwo.links[newlink.id] = self
# case when the first entity's group is not set
if self.group is None:
# assuming the second entity has already a group assigned
try:
eTwo.group.addMember(self)
# except the second entity has no group
except AttributeError:
newGroup = Group(self.field)
newGroup.addMember(self)
newGroup.addMember(eTwo)
# case when the first entity's group is set, but the second entity's is not
elif eTwo.group is None:
self.group.addMember(eTwo)
# case when both entities have groups set and they are different groups
elif self.group.name != eTwo.group.name:
if self.group.size > eTwo.group.size:
# first group wins
self.group.annexMembers(eTwo.group)
else:
# second group wins
eTwo.group.annexMembers(self.group)
def getLinks(self):
''' Print the list of entities directly linked.'''
return self.links.values()
def removeLink(self, eTwo):
''' Remove linked entity.'''
linkId = Edge.linkId(self, eTwo)
self.links.pop(linkId)
def __repr__(self):
return repr(self.value)
def __del__(self):
''' Delete itself from linked entities, and delete links.'''
# remove link from linked entity necessary? no because it's a weaklink
for linkId in self.links.keys():
self.field.eliminateEdge(linkId)
del self
class Boss:
def __init__(self, os_window_id, opts, args):
self.window_id_map = WeakValueDictionary()
self.os_window_map = {}
self.cursor_blinking = True
self.shutting_down = False
talk_fd = getattr(single_instance, 'socket', None)
talk_fd = -1 if talk_fd is None else talk_fd.fileno()
self.child_monitor = ChildMonitor(
self.on_child_death,
DumpCommands(args) if args.dump_commands or args.dump_bytes else None,
talk_fd
)
set_boss(self)
self.current_font_size = opts.font_size
set_font_family(opts)
self.opts, self.args = opts, args
initialize_renderer()
startup_session = create_session(opts, args)
self.add_os_window(startup_session, os_window_id=os_window_id)
def add_os_window(self, startup_session, os_window_id=None, wclass=None, wname=None, size=None, startup_id=None):
dpi_changed = False
if os_window_id is None:
w, h = initial_window_size(self.opts) if size is None else size
cls = wclass or self.args.cls or appname
os_window_id = create_os_window(w, h, appname, wname or self.args.name or cls, cls)
if startup_id:
ctx = init_startup_notification(os_window_id, startup_id)
dpi_changed = show_window(os_window_id)
if startup_id:
end_startup_notification(ctx)
tm = TabManager(os_window_id, self.opts, self.args, startup_session)
self.os_window_map[os_window_id] = tm
if dpi_changed:
self.on_dpi_change(os_window_id)
def list_os_windows(self):
for os_window_id, tm in self.os_window_map.items():
yield {
'id': os_window_id,
'tabs': list(tm.list_tabs()),
}
def match_windows(self, match):
field, exp = match.split(':', 1)
pat = re.compile(exp)
for tm in self.os_window_map.values():
for tab in tm:
for window in tab:
if window.matches(field, pat):
yield window
def tab_for_window(self, window):
for tm in self.os_window_map.values():
for tab in tm:
for w in tab:
if w.id == window.id:
return tab
def match_tabs(self, match):
field, exp = match.split(':', 1)
pat = re.compile(exp)
tms = tuple(self.os_window_map.values())
found = False
if field in ('title', 'id'):
for tm in tms:
for tab in tm:
if tab.matches(field, pat):
yield tab
found = True
if not found:
tabs = {self.tab_for_window(w) for w in self.match_windows(match)}
for tab in tabs:
if tab:
yield tab
def set_active_window(self, window):
for tm in self.os_window_map.values():
for tab in tm:
for w in tab:
if w.id == window.id:
if tab is not self.active_tab:
tm.set_active_tab(tab)
tab.set_active_window(w)
return
def _new_os_window(self, args, cwd_from=None):
sw = self.args_to_special_window(args, cwd_from) if args else None
startup_session = create_session(self.opts, special_window=sw, cwd_from=cwd_from)
self.add_os_window(startup_session)
def new_os_window(self, *args):
self._new_os_window(args)
def new_os_window_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_os_window(args, cwd_from)
def add_child(self, window):
self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen)
self.window_id_map[window.id] = window
def peer_messages_received(self, messages):
import json
for msg in messages:
msg = json.loads(msg.decode('utf-8'))
if isinstance(msg, dict) and msg.get('cmd') == 'new_instance':
startup_id = msg.get('startup_id')
args, rest = parse_args(msg['args'][1:])
args.args = rest
opts = create_opts(args)
session = create_session(opts, args)
self.add_os_window(session, wclass=args.cls, wname=args.name, size=initial_window_size(opts), startup_id=startup_id)
else:
safe_print('Unknown message received from peer, ignoring')
def handle_remote_cmd(self, cmd, window=None):
response = None
if self.opts.allow_remote_control:
try:
response = handle_cmd(self, window, cmd)
except Exception as err:
import traceback
response = {'ok': False, 'error': str(err), 'tb': traceback.format_exc()}
else:
response = {'ok': False, 'error': 'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'}
if response is not None:
if window is not None:
window.send_cmd_response(response)
def on_child_death(self, window_id):
window = self.window_id_map.pop(window_id, None)
if window is None:
return
os_window_id = window.os_window_id
window.destroy()
tm = self.os_window_map.get(os_window_id)
if tm is None:
return
for tab in tm:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
tm.remove(tab)
tab.destroy()
if len(tm) == 0:
if not self.shutting_down:
mark_os_window_for_close(os_window_id)
glfw_post_empty_event()
def close_window(self, window=None):
if window is None:
window = self.active_window
self.child_monitor.mark_for_close(window.id)
def close_tab(self, tab=None):
if tab is None:
tab = self.active_tab
for window in tab:
self.close_window(window)
def toggle_fullscreen(self):
toggle_fullscreen()
def start(self):
if not getattr(self, 'io_thread_started', False):
self.child_monitor.start()
self.io_thread_started = True
def activate_tab_at(self, os_window_id, x):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.activate_tab_at(x)
def on_window_resize(self, os_window_id, w, h, dpi_changed):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
if dpi_changed:
if set_dpi_from_os_window(os_window_id):
self.on_dpi_change(os_window_id)
else:
tm.resize()
else:
tm.resize()
def increase_font_size(self):
self.change_font_size(
min(
self.opts.font_size * 5, self.current_font_size +
self.opts.font_size_delta))
def decrease_font_size(self):
self.change_font_size(
max(
MINIMUM_FONT_SIZE, self.current_font_size -
self.opts.font_size_delta))
def restore_font_size(self):
self.change_font_size(self.opts.font_size)
def _change_font_size(self, new_size=None):
if new_size is not None:
self.current_font_size = new_size
old_cell_width, old_cell_height = viewport_for_window()[-2:]
windows = tuple(filter(None, self.window_id_map.values()))
resize_fonts(self.current_font_size)
layout_sprite_map()
prerender()
for window in windows:
window.screen.rescale_images(old_cell_width, old_cell_height)
window.screen.refresh_sprite_positions()
for tm in self.os_window_map.values():
tm.resize()
tm.refresh_sprite_positions()
glfw_post_empty_event()
def change_font_size(self, new_size):
if new_size == self.current_font_size:
return
self._change_font_size(new_size)
def on_dpi_change(self, os_window_id):
self._change_font_size()
@property
def active_tab_manager(self):
os_window_id = current_os_window()
return self.os_window_map.get(os_window_id)
@property
def active_tab(self):
tm = self.active_tab_manager
if tm is not None:
return tm.active_tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
def dispatch_special_key(self, key, scancode, action, mods):
# Handles shortcuts, return True if the key was consumed
key_action = get_shortcut(self.opts.keymap, mods, key, scancode)
self.current_key_press_info = key, scancode, action, mods
return self.dispatch_action(key_action)
def dispatch_action(self, key_action):
if key_action is not None:
f = getattr(self, key_action.func, None)
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
tab = self.active_tab
if tab is None:
return False
window = self.active_window
if window is None:
return False
if key_action is not None:
f = getattr(tab, key_action.func, getattr(window, key_action.func, None))
if f is not None:
passthrough = f(*key_action.args)
if passthrough is not True:
return True
return False
def combine(self, *actions):
for key_action in actions:
self.dispatch_action(key_action)
def on_focus(self, os_window_id, focused):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.focus_changed(focused)
def on_drop(self, os_window_id, paths):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.paste('\n'.join(paths))
def on_os_window_closed(self, os_window_id, viewport_width, viewport_height):
cached_values['window-size'] = viewport_width, viewport_height
tm = self.os_window_map.pop(os_window_id, None)
if tm is not None:
tm.destroy()
for window_id in tuple(w.id for w in self.window_id_map.values() if getattr(w, 'os_window_id', None) == os_window_id):
self.window_id_map.pop(window_id, None)
def display_scrollback(self, data):
if self.opts.scrollback_in_new_tab:
self.display_scrollback_in_new_tab(data)
else:
tab = self.active_tab
if tab is not None:
tab.new_special_window(
SpecialWindow(
self.opts.scrollback_pager, data, _('History')))
def switch_focus_to(self, window_idx):
tab = self.active_tab
tab.set_active_window_idx(window_idx)
old_focus = tab.active_window
if not old_focus.destroyed:
old_focus.focus_changed(False)
tab.active_window.focus_changed(True)
def open_url(self, url):
if url:
open_url(url, self.opts.open_url_with)
def open_url_lines(self, lines):
self.open_url(''.join(lines))
def destroy(self):
self.shutting_down = True
self.child_monitor.shutdown_monitor()
wakeup()
self.child_monitor.join()
del self.child_monitor
for tm in self.os_window_map.values():
tm.destroy()
self.os_window_map = {}
destroy_sprite_map()
destroy_global_data()
def paste_to_active_window(self, text):
if text:
w = self.active_window
if w is not None:
w.paste(text)
def paste_from_clipboard(self):
text = get_clipboard_string()
self.paste_to_active_window(text)
def paste_from_selection(self):
text = get_primary_selection()
self.paste_to_active_window(text)
def set_primary_selection(self):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
set_primary_selection(text)
if self.opts.copy_on_select:
set_clipboard_string(text)
def goto_tab(self, tab_num):
tm = self.active_tab_manager
if tm is not None:
tm.goto_tab(tab_num - 1)
def next_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab()
def previous_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab(-1)
def args_to_special_window(self, args, cwd_from=None):
args = list(args)
stdin = None
w = self.active_window
def data_for_at(arg):
if arg == '@selection':
return w.text_for_selection()
if arg == '@ansi':
return w.buffer_as_ansi()
if arg == '@text':
return w.buffer_as_text()
if args[0].startswith('@'):
stdin = data_for_at(args[0]) or None
if stdin is not None:
stdin = stdin.encode('utf-8')
del args[0]
cmd = []
for arg in args:
if arg == '@selection':
arg = data_for_at(arg)
if not arg:
continue
cmd.append(arg)
return SpecialWindow(cmd, stdin, cwd_from=cwd_from)
def _new_tab(self, args, cwd_from=None):
special_window = None
if args:
if isinstance(args, SpecialWindowInstance):
special_window = args
else:
special_window = self.args_to_special_window(args, cwd_from=cwd_from)
tm = self.active_tab_manager
if tm is not None:
tm.new_tab(special_window=special_window, cwd_from=cwd_from)
def new_tab(self, *args):
self._new_tab(args)
def new_tab_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_tab(args, cwd_from=cwd_from)
def _new_window(self, args, cwd_from=None):
tab = self.active_tab
if tab is not None:
if args:
tab.new_special_window(self.args_to_special_window(args, cwd_from=cwd_from))
else:
tab.new_window(cwd_from=cwd_from)
def new_window(self, *args):
self._new_window(args)
def new_window_with_cwd(self, *args):
w = self.active_window
cwd_from = w.child.pid if w is not None else None
self._new_window(args, cwd_from=cwd_from)
def move_tab_forward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(1)
def move_tab_backward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(-1)
def display_scrollback_in_new_tab(self, data):
tm = self.active_tab_manager
if tm is not None:
tm.new_tab(special_window=SpecialWindow(
self.opts.scrollback_pager, data, _('History')))
class RpcService(object):
""" service for one socket """
UID_LEN = 32
def __init__(self, svr, sock, uid, size=None):
if 0:
self.svr = RpcServer()
self.svr = svr
#self._pool = Pool(size=size)
self.sock = sock
if isinstance(svr, RpcClient):
self.sock_addr = svr.addr
else:
self.sock_addr = self.sock.getpeername()
self.uid = str(uid)
if len(self.uid) != self.UID_LEN:
raise ValueError, 'uid length error: len(uid)=%d <> %d' % (
len(uid), self.UID_LEN)
self._slock = Semaphore()
self._reconnected = None
self.reconnect_timeout = RECONNECT_TIMEOUT
#self.iter_id = itertools.cycle(xrange(MAX_INDEX))
self._next_id = 0
self._resps = {}
self._proxys = WeakValueDictionary()
self.stoped = True
self.sock_error = False
if HEARTBEAT_TIME > 0:
self._heart_time = time.time()
self._heart_task = spawn(self.heartbeat)
self.shells = {}
def next_id(self):
self._next_id += 1
if self._next_id >= MAX_INDEX:
self._next_id = 1
return self._next_id
def start(self):
if not self.stoped:
return
self.stoped = False
self._recv_task = spawn(self._recver)
self._recver_on_error = False
#_services_.append(self.sock_addr)
def remote_stop(self):
#printf('remote_stop:%s', self.sock_addr)
self.sock_error = True
self.stop()
def close(self):
if not self.sock:
return
try:
self.sock._sock.close()
except socket_error:
pass
self.sock.close()
self.sock = None
def stop(self):
if self.stoped:
return
self.stoped = True
self._recv_task.kill(block=0)
self._recv_task = None
if 1 and not self.sock_error:
try:
#printf('remote_stop:%s', self.sock_addr)
self.call('', 'remote_stop', tuple(), None, no_result=True)
sleep(0.01)
except:
pass
self.svr.svc_stop(self)
if getattr(self, '_heart_task', None):
self._heart_task.kill(block=False)
self._heart_task = None
try:
self._stop_resps()
self._stop_proxys()
finally:
self.close()
#_services_.append('-%s' % str(self.sock_addr))
def _stop_resps(self):
error = RpcRuntimeError('service stoped')
for k, v in self._resps.iteritems():
v.set_exception(error)
self._resps.clear()
def _stop_proxys(self):
if not len(self._proxys):
return
proxys = self._proxys.values()
self._proxys.clear()
for p in proxys:
p.on_close()
## def _sender(self):
## running = True
## _send = self.sock.sendall
## try:
## for data in self._send_queue:
## _send('%s%s' %(pack('I', len(data)), data))
## except GreenletExit:
## pass
def _recver(self):
""" 接收处理数据 """
recv_func = self.sock.recv
def _read(c):
d = recv_func(c)
if d:
return d
if self.stoped:
raise GreenletExit
self._recver_on_error = True
self._on_socket_error(None)
self._recver_on_error = False
return None
try:
sio = StringIO()
while not self.stoped:
dlen = 4
d = ''
while dlen > 0:
data = _read(dlen)
if data is None:
continue
d += data
dlen -= len(data)
dlen = unpack('I', d)[0]
#rs = []
sio.seek(0)
sio.truncate()
while dlen > 0:
data = _read(dlen)
if data is None:
continue
#rs.append(data)
sio.write(data)
dlen -= len(data)
#spawn(self._handle, loads(''.join(rs)))
sio.seek(0)
self._handle(load(sio))
#self._pool.spawn(self._handle, loads(''.join(rs)))
except GreenletExit:
pass
except Exception as err:
printf('[RpcService._recver]%s', err)
finally:
self.stop()
def _on_socket_error(self, err):
if self.stoped or self.reconnect_timeout <= 0:
self.sock_error = True
self.stop()
return
def _reconnect():
#尝试重连或等待重连
while not self.stoped:
try:
self.svr.reconnect()
break
except socket_error:
pass
sleep(0.5)
if self._reconnected is None:
self._reconnected = AsyncResult()
printf('socket error:%s, RpcService try reconnect', err)
self.send = self.send_wait
if hasattr(self.svr, 'reconnect'): #RpcClient.reconnect
spawn(_reconnect)
self._wait_reconnect()
def _wait_reconnect(self):
_reconnected = self._reconnected
try:
_reconnected.get(timeout=self.reconnect_timeout)
except Timeout:
pass
if not _reconnected.successful():
self.stop()
if self.sock_error or _reconnected.exception is None:
return
self.sock_error = True
raise _reconnected.exception
def reconnect(self, sock):
## if not self._recver_on_error:
## self._recv_task.kill(exception=socket_error('reconnect'))
self.sock = sock
self.send = self.send_imme
if self._reconnected is not None:
self._reconnected.set(True)
self._reconnected = None
def send_imme(self, *args):
data = dumps(args)
with self._slock:
try:
self.sock.sendall('%s%s' % (pack('I', len(data)), data))
except socket_error as err:
self._on_socket_error(err)
#重新发送
self.sock.sendall('%s%s' % (pack('I', len(data)), data))
## self._send_queue.put(dumps(args))
def send_wait(self, *args):
if self._reconnected is not None:
self._wait_reconnect()
self.send_imme(*args)
send = send_imme
def _read_response(self, index, timeout):
rs = AsyncResult()
self._resps[index] = rs
resp = rs.wait(timeout)
self._resps.pop(index, None)
if not rs.successful():
error = rs.exception
if error is None:
error = Timeout
raise error
return resp
def _reg_obj(self, obj):
if hasattr(obj, 'proxy_pack'):
return obj.proxy_pack(), False
if isinstance(obj, RpcProxy):
return obj._id, False
if hasattr(obj, '_rpc_proxy_'):
return obj._rpc_proxy_(), True
return self.svr.register(obj), False
def call(self,
obj_id,
name,
args,
kw,
no_result=False,
timeout=CALL_TIMEORUT,
pickle=False,
proxy=False):
dtype = RT_REQUEST
if proxy:
objs = args[0] #first arg is proxy(str, RpcProxy or list)
if isinstance(objs, (tuple, list)):
obj_ids = []
for o in objs:
obj, is_pickle = self._reg_obj(o)
pickle = pickle or is_pickle
obj_ids.append(obj)
else:
obj, is_pickle = self._reg_obj(objs)
pickle = pickle or is_pickle
obj_ids = obj
args = (obj_ids, ) + args[1:]
dtype |= DT_PROXY
if pickle:
dtype |= DT_PICKLE
argkw = pickle_dumps((args, kw), PICKLE_PROTOCOL)
else:
argkw = dumps((args, kw))
if len(argkw) >= ZIP_LENGTH:
dtype |= DT_ZIP
argkw = zlib.compress(argkw, ZIP_LEVEL)
if no_result:
dtype |= ST_NO_RESULT
index = self.next_id() #iter_id.next()
self.send(dtype, obj_id, index, name, argkw)
if no_result:
return
result = self._read_response(index, timeout)
return result
def _handle_request(self, parts):
dtype, obj_id, index, name, argkw = parts
try:
obj = self.get_export(obj_id)
if obj is None:
raise RpcExportNoFound, obj_id
func = getattr(obj, name)
if not callable(func):
raise RpcFuncNoFound, name
if dtype & DT_ZIP:
argkw = zlib.decompress(argkw)
if dtype & DT_PICKLE:
args, kw = pickle_loads(argkw)
else:
args, kw = loads(argkw)
if dtype & DT_PROXY:
export_ids = args[0]
if isinstance(export_ids, (tuple, list)):
proxys = []
for e in export_ids:
proxys.append(self.get_proxy(e))
else:
proxys = self.get_proxy(export_ids)
args = (proxys, ) + tuple(args[1:])
if getattr(func, "_block_", True):
spawn(self._handle_request_call, func, args, kw, dtype, index,
obj_id, name, argkw)
else:
self._handle_request_call(func, args, kw, dtype, index, obj_id,
name, argkw)
except Exception as e:
log_except('export(%s).%s(%s)', obj_id, name, repr(argkw))
if dtype & ST_NO_RESULT or self.svr.stoped:
return
self.send(RT_EXCEPTION, index, str(e))
def _handle_request_call(self, func, args, kw, dtype, index, obj_id, name,
argkw):
try:
let = getcurrent()
setattr(let, _service_name_, self)
if args is None:
rs = func()
else:
rs = func(*args, **kw) if kw is not None else func(*args)
if dtype & ST_NO_RESULT:
return
if not getattr(func, '_rpc_pickle_result_', False):
self.send(RT_RESPONSE, index, dumps(rs))
else:
self.send(RT_RESPONSE | DT_PICKLE, index,
pickle_dumps(rs, PICKLE_PROTOCOL))
except Exception as e:
log_except('export(%s).%s(%s)', obj_id, name, repr(argkw))
if dtype & ST_NO_RESULT or self.svr.stoped:
return
self.send(RT_EXCEPTION, index, str(e))
def _handle_response(self, parts):
dtype, index, argkw = parts
try:
rs = self._resps.pop(index)
if dtype & DT_PICKLE:
result = pickle_loads(argkw)
else:
result = loads(argkw)
rs.set(result)
except KeyError:
pass
def _handle_exception(self, parts):
RT_EXCEPTION, index, error = parts
#try:
# error = pickle_loads(error)
#except:
error = RpcCallError(str(error))
try:
rs = self._resps.pop(index)
rs.set_exception(error)
except KeyError:
pass
def _handle(self, parts):
#parts = (parts[0], ) + loads(parts[1]) if len(parts) ==2 else loads(parts[0])
rt = parts[0] & RT_MARK
if rt == RT_REQUEST:
self._handle_request(parts)
elif rt == RT_RESPONSE:
self._handle_response(parts)
elif rt == RT_EXCEPTION:
self._handle_exception(parts)
elif rt == RT_HEARTBEAT:
self._heart_time = time.time()
else:
raise ValueError('unknown data:%s' % str(rt))
def heartbeat(self):
beat = RT_HEARTBEAT
btime = HEARTBEAT_TIME
check_times = HEARTBEAT_TIME * max(3, RECONNECT_TIMEOUT)
try:
while not self.stoped:
self.send(beat)
sleep(btime)
if (self._heart_time + check_times) < time.time():
printf('heartbeat timeout!!!!!!!!')
self.sock_error = True
break
finally:
self.stop()
@classmethod
def handshake_svr(cls, sock):
uid = sock.recv(cls.UID_LEN)
return uid
def handshake_cli(self):
self.sock.sendall(self.uid)
#######remote call##############
def get_export(self, export_id):
""" get export obj by export_name """
if not export_id:
return self
return self.svr.get_export(export_id)
def get_proxy(self, export_id, proxy_cls=None):
""" remote call: get export obj by id """
if isinstance(export_id, RpcProxy):
return export_id
if isinstance(export_id, (tuple, list)):
export_cls = PROXYS[export_id[0]]
return export_cls.proxy_unpack(export_id, svc=self)
if proxy_cls in (None, RpcProxy):
try:
return self._proxys[export_id]
except KeyError:
proxy_cls = RpcProxy
proxy = proxy_cls(export_id, svc=self)
self.reg_proxy(export_id, proxy)
return proxy
else:
p = proxy_cls(export_id, svc=self)
self.reg_proxy(id(p), p)
return p
def reg_proxy(self, key, proxy):
self._proxys[key] = proxy
def stop_shell(self, shell_id):
shell = self.shells.pop(shell_id, None)
if not shell:
return
shell.stop()
def start_shell(self, console_proxy, pre_prompt):
from rpc_shell import RpcShell
if self.svr.access and not self.svr.access.access_shell(self):
printf('[rpc]shell deny:%s', self.sock_addr)
return 0
printf('[rpc]shell start:%s', self.sock_addr)
shell = RpcShell(self, console_proxy, pre_prompt=pre_prompt)
shell.start()
#shell.stop remove shell from self.shells
shell_id = id(shell)
self.shells[shell_id] = shell
return shell_id
def stop_console(self, shell_id):
self.call('',
'stop_shell', (shell_id, ),
None,
no_result=True,
timeout=20)
def start_console(self, pre_prompt='', shell=None):
from rpc_shell import RpcLocalConsole, RpcProxyConsole
console = RpcLocalConsole(self) if shell is None else RpcProxyConsole(
self, shell)
shell_id = self.call('',
'start_shell', (console, pre_prompt),
None,
proxy=True,
timeout=20)
try:
console.wait(shell_id)
finally:
pass
def execute(self, func, args, kw, **others):
return self.svr.execute(func, args, kw)
def valid_proxy(self, export_id):
return self.get_export(export_id) != None
class ChannelManager(object):
""" High level interface for channels
This class handles:
* configuration of channels
* high level api to create and remove jobs (notify, remove_job, remove_db)
* get jobs to run
Here is how the runner will use it.
Let's create a channel manager and configure it.
>>> from pprint import pprint as pp
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4,B:1')
>>> db = 'db'
Add a few jobs in channel A with priority 10
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending')
Add a few jobs in channel B with priority 5
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending')
>>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending')
We must now run one job from queue B which has a capacity of 1
and 3 jobs from queue A so the root channel capacity of 4 is filled.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>]
Job A2 is done. Next job to run is A5, even if we have
higher priority job in channel B, because channel B has a capacity of 1.
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A4>]
Job B1 is done. Next job to run is B2 because it has higher priority.
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B2>]
Let's say A1 is done and A6 gets a higher priority. A6 will run next.
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done')
>>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A6>]
Let's test the throttling mechanism. Configure a 2 seconds delay
on channel A, end enqueue two jobs.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4:throttle=2')
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
We have only one job to run, because of the throttle.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A1>]
>>> cm.get_wakeup_time()
102
We have no job to run, because of the throttle.
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.get_wakeup_time()
102
2 seconds later, we can run the other job (even though the first one
is still running, because we have enough capacity).
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob A2>]
>>> cm.get_wakeup_time()
104
Let's test throttling in combination with a queue reaching full capacity.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,T:2:throttle=2')
>>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'T', 'T2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'T', 'T3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob T1>]
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob T2>]
Channel is now full, so no job to run even though throttling
delay is over.
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> cm.get_wakeup_time() # no wakeup time, since queue is full
0
>>> pp(list(cm.get_jobs_to_run(now=104)))
[]
>>> cm.get_wakeup_time() # queue is still full
0
>>> cm.notify(db, 'T', 'T1', 1, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob T3>]
>>> cm.get_wakeup_time() # queue is full
0
>>> cm.notify(db, 'T', 'T2', 1, 0, 10, None, 'done')
>>> cm.get_wakeup_time()
107
Test wakeup time behaviour in presence of eta.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,E:1')
>>> cm.notify(db, 'E', 'E1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'E', 'E2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'E', 'E3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob E1>]
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.notify(db, 'E', 'E1', 1, 0, 10, 105, 'pending')
>>> cm.get_wakeup_time() # wakeup at eta
105
>>> pp(list(cm.get_jobs_to_run(now=102))) # but there is capacity
[<ChannelJob E2>]
>>> pp(list(cm.get_jobs_to_run(now=106))) # no capacity anymore
[]
>>> cm.get_wakeup_time() # no timed wakeup because no capacity
0
>>> cm.notify(db, 'E', 'E2', 1, 0, 10, None, 'done')
>>> cm.get_wakeup_time()
105
>>> pp(list(cm.get_jobs_to_run(now=107))) # no capacity anymore
[<ChannelJob E1>]
>>> cm.get_wakeup_time()
0
Test wakeup time behaviour in a sequential queue.
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,S:1:sequential')
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob S1>]
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'failed')
>>> pp(list(cm.get_jobs_to_run(now=101)))
[]
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=102)))
[]
No wakeup time because due to eta, because the sequential queue
is waiting for a failed job.
>>> cm.get_wakeup_time()
0
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'pending')
>>> cm.get_wakeup_time()
105
>>> pp(list(cm.get_jobs_to_run(now=102)))
[<ChannelJob S1>]
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> cm.notify(db, 'S', 'S1', 1, 0, 10, None, 'done')
At this stage, we have S2 with an eta of 105 and since the
queue is sequential, we wait for it.
>>> pp(list(cm.get_jobs_to_run(now=103)))
[]
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob S2>]
>>> cm.notify(db, 'S', 'S2', 2, 0, 10, 105, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[<ChannelJob S3>]
>>> cm.notify(db, 'S', 'S3', 3, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=105)))
[]
"""
def __init__(self):
self._jobs_by_uuid = WeakValueDictionary()
self._root_channel = Channel(name='root', parent=None, capacity=1)
self._channels_by_name = WeakValueDictionary(root=self._root_channel)
@classmethod
def parse_simple_config(cls, config_string):
"""Parse a simple channels configuration string.
The general form is as follow:
channel(.subchannel)*(:capacity(:key(=value)?)*)? [, ...]
If capacity is absent, it defaults to 1.
If a key is present without value, it gets True as value.
When declaring subchannels, the root channel may be omitted
(ie sub:4 is the same as root.sub:4).
Returns a list of channel configuration dictionaries.
>>> from pprint import pprint as pp
>>> pp(ChannelManager.parse_simple_config('root:4'))
[{'capacity': 4, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2'))
[{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:'
... 'sequential:k=v'))
[{'capacity': 4, 'name': 'root'},
{'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}]
>>> pp(ChannelManager.parse_simple_config('root'))
[{'capacity': 1, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('sub:2'))
[{'capacity': 2, 'name': 'sub'}]
It ignores whitespace around values, and drops empty entries which
would be generated by trailing commas, or commented lines on the Odoo
config file.
>>> pp(ChannelManager.parse_simple_config('''
... root : 4,
... ,
... foo bar:1: k=va lue,
... '''))
[{'capacity': 4, 'name': 'root'},
{'capacity': 1, 'k': 'va lue', 'name': 'foo bar'}]
It's also possible to replace commas with line breaks, which is more
readable if the channel configuration comes from the odoo config file.
>>> pp(ChannelManager.parse_simple_config('''
... root : 4
... foo bar:1: k=va lue
... baz
... '''))
[{'capacity': 4, 'name': 'root'},
{'capacity': 1, 'k': 'va lue', 'name': 'foo bar'},
{'capacity': 1, 'name': 'baz'}]
"""
res = []
config_string = config_string.replace("\n", ",")
for channel_config_string in split_strip(config_string, ','):
if not channel_config_string:
# ignore empty entries (commented lines, trailing commas)
continue
config = {}
config_items = split_strip(channel_config_string, ':')
name = config_items[0]
if not name:
raise ValueError('Invalid channel config %s: '
'missing channel name' % config_string)
config['name'] = name
if len(config_items) > 1:
capacity = config_items[1]
try:
config['capacity'] = int(capacity)
except:
raise ValueError('Invalid channel config %s: '
'invalid capacity %s' %
(config_string, capacity))
for config_item in config_items[2:]:
kv = split_strip(config_item, '=')
if len(kv) == 1:
k, v = kv[0], True
elif len(kv) == 2:
k, v = kv
else:
raise ValueError('Invalid channel config %s: '
'incorrect config item %s' %
(config_string, config_item))
if k in config:
raise ValueError('Invalid channel config %s: '
'duplicate key %s' %
(config_string, k))
config[k] = v
else:
config['capacity'] = 1
res.append(config)
return res
def simple_configure(self, config_string):
"""Configure the channel manager from a simple configuration string
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root')
>>> c.capacity
1
>>> cm.simple_configure('root:4,autosub.sub:2,seq:1:sequential')
>>> cm.get_channel_by_name('root').capacity
4
>>> cm.get_channel_by_name('root').sequential
False
>>> cm.get_channel_by_name('root.autosub').capacity
>>> cm.get_channel_by_name('root.autosub.sub').capacity
2
>>> cm.get_channel_by_name('root.autosub.sub').sequential
False
>>> cm.get_channel_by_name('autosub.sub').capacity
2
>>> cm.get_channel_by_name('seq').capacity
1
>>> cm.get_channel_by_name('seq').sequential
True
"""
for config in ChannelManager.parse_simple_config(config_string):
self.get_channel_from_config(config)
def get_channel_from_config(self, config):
"""Return a Channel object from a parsed configuration.
If the channel does not exist it is created.
The configuration is applied on the channel before returning it.
If some of the parent channels are missing when creating a subchannel,
the parent channels are auto created with an infinite capacity
(except for the root channel, which defaults to a capacity of 1
when not configured explicity).
"""
channel = self.get_channel_by_name(config['name'], autocreate=True)
channel.configure(config)
_logger.info("Configured channel: %s", channel)
return channel
def get_channel_by_name(self, channel_name, autocreate=False):
"""Return a Channel object by its name.
If it does not exist and autocreate is True, it is created
with a default configuration and inserted in the Channels structure.
If autocreate is False and the channel does not exist, an exception
is raised.
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root', autocreate=False)
>>> c.name
'root'
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('autosub.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.autosub.sub'
>>> c = cm.get_channel_by_name(None)
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub')
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub')
>>> c.fullname
'root.sub'
"""
if not channel_name or channel_name == self._root_channel.name:
return self._root_channel
if not channel_name.startswith(self._root_channel.name + '.'):
channel_name = self._root_channel.name + '.' + channel_name
if channel_name in self._channels_by_name:
return self._channels_by_name[channel_name]
if not autocreate:
raise ChannelNotFound('Channel %s not found' % channel_name)
parent = self._root_channel
for subchannel_name in channel_name.split('.')[1:]:
subchannel = parent.get_subchannel_by_name(subchannel_name)
if not subchannel:
subchannel = Channel(subchannel_name, parent, capacity=None)
self._channels_by_name[subchannel.fullname] = subchannel
parent = subchannel
return parent
def notify(self, db_name, channel_name, uuid, seq, date_created, priority,
eta, state):
try:
channel = self.get_channel_by_name(channel_name)
except ChannelNotFound:
_logger.warning(
'unknown channel %s, '
'using root channel for job %s', channel_name, uuid)
channel = self._root_channel
job = self._jobs_by_uuid.get(uuid)
if job:
# db_name is invariant
assert job.db_name == db_name
# date_created is invariant
assert job.date_created == date_created
# if one of the job properties that influence
# scheduling order has changed, we remove the job
# from the queues and create a new job object
if (seq != job.seq or priority != job.priority or eta != job.eta
or channel != job.channel):
_logger.debug("job %s properties changed, rescheduling it",
uuid)
self.remove_job(uuid)
job = None
if not job:
job = ChannelJob(db_name, channel, uuid, seq, date_created,
priority, eta)
self._jobs_by_uuid[uuid] = job
# state transitions
if not state or state == DONE:
job.channel.set_done(job)
elif state == PENDING:
job.channel.set_pending(job)
elif state in (ENQUEUED, STARTED):
job.channel.set_running(job)
elif state == FAILED:
job.channel.set_failed(job)
else:
_logger.error("unexpected state %s for job %s", state, job)
def remove_job(self, uuid):
job = self._jobs_by_uuid.get(uuid)
if job:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def remove_db(self, db_name):
for job in self._jobs_by_uuid.values():
if job.db_name == db_name:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def get_jobs_to_run(self, now):
return self._root_channel.get_jobs_to_run(now)
def get_wakeup_time(self):
return self._root_channel.get_wakeup_time()
class ChannelManager(object):
""" High level interface for channels
This class handles:
* configuration of channels
* high level api to create and remove jobs (notify, remove_job, remove_db)
* get jobs to run
Here is how the runner will use it.
Let's create a channel manager and configure it.
>>> from pprint import pprint as pp
>>> cm = ChannelManager()
>>> cm.simple_configure('root:4,A:4,B:1')
>>> db = 'db'
Add a few jobs in channel A with priority 10
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A3', 3, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A4', 4, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A5', 5, 0, 10, None, 'pending')
>>> cm.notify(db, 'A', 'A6', 6, 0, 10, None, 'pending')
Add a few jobs in channel B with priority 5
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'pending')
>>> cm.notify(db, 'B', 'B2', 2, 0, 5, None, 'pending')
We must now run one job from queue B which has a capacity of 1
and 3 jobs from queue A so the root channel capacity of 4 is filled.
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B1>, <ChannelJob A1>, <ChannelJob A2>, <ChannelJob A3>]
Job A2 is done. Next job to run is A5, even if we have
higher priority job in channel B, because channel B has a capacity of 1.
>>> cm.notify(db, 'A', 'A2', 2, 0, 10, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A4>]
Job B1 is done. Next job to run is B2 because it has higher priority.
>>> cm.notify(db, 'B', 'B1', 1, 0, 5, None, 'done')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob B2>]
Let's say A1 is done and A6 gets a higher priority. A6 will run next.
>>> cm.notify(db, 'A', 'A1', 1, 0, 10, None, 'done')
>>> cm.notify(db, 'A', 'A6', 6, 0, 5, None, 'pending')
>>> pp(list(cm.get_jobs_to_run(now=100)))
[<ChannelJob A6>]
"""
def __init__(self):
self._jobs_by_uuid = WeakValueDictionary()
self._root_channel = Channel(name='root', parent=None, capacity=1)
self._channels_by_name = WeakValueDictionary(root=self._root_channel)
@classmethod
def parse_simple_config(cls, config_string):
"""Parse a simple channels configuration string.
The general form is as follow:
channel(.subchannel)*(:capacity(:key(=value)?)*)?,...
If capacity is absent, it defaults to 1.
If a key is present without value, it gets True as value.
When declaring subchannels, the root channel may be omitted
(ie sub:4 is the same as root.sub:4).
Returns a list of channel configuration dictionaries.
>>> from pprint import pprint as pp
>>> pp(ChannelManager.parse_simple_config('root:4'))
[{'capacity': 4, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2'))
[{'capacity': 4, 'name': 'root'}, {'capacity': 2, 'name': 'root.sub'}]
>>> pp(ChannelManager.parse_simple_config('root:4,root.sub:2:'
... 'sequential:k=v'))
[{'capacity': 4, 'name': 'root'},
{'capacity': 2, 'k': 'v', 'name': 'root.sub', 'sequential': True}]
>>> pp(ChannelManager.parse_simple_config('root'))
[{'capacity': 1, 'name': 'root'}]
>>> pp(ChannelManager.parse_simple_config('sub:2'))
[{'capacity': 2, 'name': 'sub'}]
"""
res = []
for channel_config_string in config_string.split(','):
config = {}
config_items = channel_config_string.split(':')
name = config_items[0]
if not name:
raise ValueError('Invalid channel config %s: '
'missing channel name' % config_string)
config['name'] = name
if len(config_items) > 1:
capacity = config_items[1]
try:
config['capacity'] = int(capacity)
except:
raise ValueError('Invalid channel config %s: '
'invalid capacity %s' %
(config_string, capacity))
for config_item in config_items[2:]:
kv = config_item.split('=')
if len(kv) == 1:
k, v = kv[0], True
elif len(kv) == 2:
k, v = kv
else:
raise ValueError(
'Invalid channel config %s: ',
'incorrect config item %s' (config_string,
config_item))
if k in config:
raise ValueError('Invalid channel config %s: '
'duplicate key %s' (config_string, k))
config[k] = v
else:
config['capacity'] = 1
res.append(config)
return res
def simple_configure(self, config_string):
"""Configure the channel manager from a simple configuration string
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root')
>>> c.capacity
1
>>> cm.simple_configure('root:4,autosub.sub:2')
>>> cm.get_channel_by_name('root').capacity
4
>>> cm.get_channel_by_name('root.autosub').capacity
>>> cm.get_channel_by_name('root.autosub.sub').capacity
2
>>> cm.get_channel_by_name('autosub.sub').capacity
2
"""
for config in ChannelManager.parse_simple_config(config_string):
self.get_channel_from_config(config)
def get_channel_from_config(self, config):
"""Return a Channel object from a parsed configuration.
If the channel does not exist it is created.
The configuration is applied on the channel before returning it.
If some of the parent channels are missing when creating a subchannel,
the parent channels are auto created with an infinite capacity
(except for the root channel, which defaults to a capacity of 1
when not configured explicity).
"""
channel = self.get_channel_by_name(config['name'], autocreate=True)
channel.configure(config)
return channel
def get_channel_by_name(self, channel_name, autocreate=False):
"""Return a Channel object by its name.
If it does not exist and autocreate is True, it is created
with a default configuration and inserted in the Channels structure.
If autocreate is False and the channel does not exist, an exception
is raised.
>>> cm = ChannelManager()
>>> c = cm.get_channel_by_name('root', autocreate=False)
>>> c.name
'root'
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('autosub.sub', autocreate=True)
>>> c.name
'sub'
>>> c.fullname
'root.autosub.sub'
>>> c = cm.get_channel_by_name(None)
>>> c.fullname
'root'
>>> c = cm.get_channel_by_name('root.sub')
>>> c.fullname
'root.sub'
>>> c = cm.get_channel_by_name('sub')
>>> c.fullname
'root.sub'
"""
if not channel_name or channel_name == self._root_channel.name:
return self._root_channel
if not channel_name.startswith(self._root_channel.name + '.'):
channel_name = self._root_channel.name + '.' + channel_name
if channel_name in self._channels_by_name:
return self._channels_by_name[channel_name]
if not autocreate:
raise ChannelNotFound('Channel %s not found' % channel_name)
parent = self._root_channel
for subchannel_name in channel_name.split('.')[1:]:
subchannel = parent.get_subchannel_by_name(subchannel_name)
if not subchannel:
subchannel = Channel(subchannel_name, parent, capacity=None)
self._channels_by_name[subchannel.fullname] = subchannel
parent = subchannel
return parent
def notify(self, db_name, channel_name, uuid, seq, date_created, priority,
eta, state):
try:
channel = self.get_channel_by_name(channel_name)
except ChannelNotFound:
_logger.warning(
'unknown channel %s, '
'using root channel for job %s', channel_name, uuid)
channel = self._root_channel
job = self._jobs_by_uuid.get(uuid)
if job:
# db_name is invariant
assert job.db_name == db_name
# date_created is invariant
assert job.date_created == date_created
# if one of the job properties that influence
# scheduling order has changed, we remove the job
# from the queues and create a new job object
if (seq != job.seq or priority != job.priority or eta != job.eta
or channel != job.channel):
_logger.debug("job %s properties changed, rescheduling it",
uuid)
self.remove_job(uuid)
job = None
if not job:
job = ChannelJob(db_name, channel, uuid, seq, date_created,
priority, eta)
self._jobs_by_uuid[uuid] = job
# state transitions
if not state or state == DONE:
job.channel.set_done(job)
elif state == PENDING:
job.channel.set_pending(job)
elif state in (ENQUEUED, STARTED):
job.channel.set_running(job)
elif state == FAILED:
job.channel.set_failed(job)
else:
_logger.error("unexpected state %s for job %s", state, job)
def remove_job(self, uuid):
job = self._jobs_by_uuid.get(uuid)
if job:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def remove_db(self, db_name):
for job in self._jobs_by_uuid.values():
if job.db_name == db_name:
job.channel.remove(job)
del self._jobs_by_uuid[job.uuid]
def get_jobs_to_run(self, now):
return self._root_channel.get_jobs_to_run(now)
class Boss:
def __init__(self, os_window_id, opts, args, cached_values,
new_os_window_trigger):
set_layout_options(opts)
self.clipboard_buffers = {}
self.update_check_process = None
self.window_id_map = WeakValueDictionary()
self.startup_colors = {
k: opts[k]
for k in opts if isinstance(opts[k], Color)
}
self.startup_cursor_text_color = opts.cursor_text_color
self.pending_sequences = None
self.cached_values = cached_values
self.os_window_map = {}
self.os_window_death_actions = {}
self.cursor_blinking = True
self.shutting_down = False
talk_fd = getattr(single_instance, 'socket', None)
talk_fd = -1 if talk_fd is None else talk_fd.fileno()
listen_fd = -1
if opts.allow_remote_control and args.listen_on:
listen_fd = listen_on(args.listen_on)
self.child_monitor = ChildMonitor(
self.on_child_death,
DumpCommands(args) if args.dump_commands or args.dump_bytes else
None, talk_fd, listen_fd)
set_boss(self)
self.opts, self.args = opts, args
startup_sessions = create_sessions(
opts, args, default_session=opts.startup_session)
self.keymap = self.opts.keymap.copy()
if new_os_window_trigger is not None:
self.keymap.pop(new_os_window_trigger, None)
for startup_session in startup_sessions:
self.add_os_window(startup_session, os_window_id=os_window_id)
os_window_id = None
if args.start_as != 'normal':
if args.start_as == 'fullscreen':
self.toggle_fullscreen()
else:
change_os_window_state(args.start_as)
if is_macos:
from .fast_data_types import cocoa_set_notification_activated_callback
cocoa_set_notification_activated_callback(
self.notification_activated)
def add_os_window(self,
startup_session,
os_window_id=None,
wclass=None,
wname=None,
opts_for_size=None,
startup_id=None):
if os_window_id is None:
opts_for_size = opts_for_size or startup_session.os_window_size or self.opts
cls = wclass or self.args.cls or appname
with startup_notification_handler(
do_notify=startup_id is not None,
startup_id=startup_id) as pre_show_callback:
os_window_id = create_os_window(
initial_window_size_func(opts_for_size,
self.cached_values),
pre_show_callback, appname, wname or self.args.name or cls,
cls)
tm = TabManager(os_window_id, self.opts, self.args, startup_session)
self.os_window_map[os_window_id] = tm
return os_window_id
def list_os_windows(self):
with cached_process_data():
active_tab, active_window = self.active_tab, self.active_window
active_tab_manager = self.active_tab_manager
for os_window_id, tm in self.os_window_map.items():
yield {
'id': os_window_id,
'is_focused': tm is active_tab_manager,
'tabs': list(tm.list_tabs(active_tab, active_window)),
}
@property
def all_tab_managers(self):
yield from self.os_window_map.values()
@property
def all_tabs(self):
for tm in self.all_tab_managers:
yield from tm
@property
def all_windows(self):
for tab in self.all_tabs:
yield from tab
def match_windows(self, match):
try:
field, exp = match.split(':', 1)
except ValueError:
return
if field == 'num':
tab = self.active_tab
if tab is not None:
try:
w = tab.get_nth_window(int(exp))
except Exception:
return
if w is not None:
yield w
return
if field == 'env':
kp, vp = exp.partition('=')[::2]
if vp:
pat = tuple(map(re.compile, (kp, vp)))
else:
pat = re.compile(kp), None
else:
pat = re.compile(exp)
for window in self.all_windows:
if window.matches(field, pat):
yield window
def tab_for_window(self, window):
for tab in self.all_tabs:
for w in tab:
if w.id == window.id:
return tab
def match_tabs(self, match):
try:
field, exp = match.split(':', 1)
except ValueError:
return
pat = re.compile(exp)
found = False
if field in ('title', 'id'):
for tab in self.all_tabs:
if tab.matches(field, pat):
yield tab
found = True
if not found:
tabs = {self.tab_for_window(w) for w in self.match_windows(match)}
for tab in tabs:
if tab:
yield tab
def set_active_window(self, window):
for os_window_id, tm in self.os_window_map.items():
for tab in tm:
for w in tab:
if w.id == window.id:
if tab is not self.active_tab:
tm.set_active_tab(tab)
tab.set_active_window(w)
return os_window_id
def _new_os_window(self, args, cwd_from=None):
if isinstance(args, SpecialWindowInstance):
sw = args
else:
sw = self.args_to_special_window(args, cwd_from) if args else None
startup_session = next(
create_sessions(self.opts, special_window=sw, cwd_from=cwd_from))
return self.add_os_window(startup_session)
def new_os_window(self, *args):
self._new_os_window(args)
@property
def active_window_for_cwd(self):
w = self.active_window
if w is not None and w.overlay_for is not None and w.overlay_for in self.window_id_map:
w = self.window_id_map[w.overlay_for]
return w
def new_os_window_with_cwd(self, *args):
w = self.active_window_for_cwd
cwd_from = w.child.pid_for_cwd if w is not None else None
self._new_os_window(args, cwd_from)
def new_os_window_with_wd(self, wd):
special_window = SpecialWindow(None, cwd=wd)
self._new_os_window(special_window)
def add_child(self, window):
self.child_monitor.add_child(window.id, window.child.pid,
window.child.child_fd, window.screen)
self.window_id_map[window.id] = window
def _handle_remote_command(self, cmd, window=None):
response = None
if self.opts.allow_remote_control or getattr(
window, 'allow_remote_control', False):
try:
response = handle_cmd(self, window, cmd)
except Exception as err:
import traceback
response = {'ok': False, 'error': str(err)}
if not getattr(err, 'hide_traceback', False):
response['tb'] = traceback.format_exc()
else:
response = {
'ok':
False,
'error':
'Remote control is disabled. Add allow_remote_control yes to your kitty.conf'
}
return response
def peer_message_received(self, msg):
msg = msg.decode('utf-8')
cmd_prefix = '\x1bP@kitty-cmd'
if msg.startswith(cmd_prefix):
cmd = msg[len(cmd_prefix):-2]
response = self._handle_remote_command(cmd)
if response is not None:
response = (cmd_prefix + json.dumps(response) +
'\x1b\\').encode('utf-8')
return response
else:
msg = json.loads(msg)
if isinstance(msg, dict) and msg.get('cmd') == 'new_instance':
startup_id = msg.get('startup_id')
args, rest = parse_args(msg['args'][1:])
args.args = rest
opts = create_opts(args)
if not os.path.isabs(args.directory):
args.directory = os.path.join(msg['cwd'], args.directory)
for session in create_sessions(opts, args, respect_cwd=True):
os_window_id = self.add_os_window(session,
wclass=args.cls,
wname=args.name,
opts_for_size=opts,
startup_id=startup_id)
if msg.get('notify_on_os_window_death'):
self.os_window_death_actions[os_window_id] = partial(
self.notify_on_os_window_death,
msg['notify_on_os_window_death'])
else:
log_error('Unknown message received from peer, ignoring')
def handle_remote_cmd(self, cmd, window=None):
response = self._handle_remote_command(cmd, window)
if response is not None:
if window is not None:
window.send_cmd_response(response)
def on_child_death(self, window_id):
window = self.window_id_map.pop(window_id, None)
if window is None:
return
if window.action_on_close:
try:
window.action_on_close(window)
except Exception:
import traceback
traceback.print_exc()
os_window_id = window.os_window_id
window.destroy()
tm = self.os_window_map.get(os_window_id)
if tm is None:
return
for tab in tm:
if window in tab:
break
else:
return
tab.remove_window(window)
if len(tab) == 0:
tm.remove(tab)
tab.destroy()
if len(tm) == 0:
if not self.shutting_down:
mark_os_window_for_close(os_window_id)
def close_window(self, window=None):
if window is None:
window = self.active_window
self.child_monitor.mark_for_close(window.id)
def close_tab(self, tab=None):
if tab is None:
tab = self.active_tab
for window in tab:
self.close_window(window)
def toggle_fullscreen(self):
toggle_fullscreen()
def toggle_maximized(self):
toggle_maximized()
def start(self):
if not getattr(self, 'io_thread_started', False):
self.child_monitor.start()
self.io_thread_started = True
if self.opts.update_check_interval > 0 and not hasattr(
self, 'update_check_started'):
from .update_check import run_update_check
run_update_check(self.opts.update_check_interval * 60 * 60)
self.update_check_started = True
def activate_tab_at(self, os_window_id, x):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.activate_tab_at(x)
def on_window_resize(self, os_window_id, w, h, dpi_changed):
if dpi_changed:
self.on_dpi_change(os_window_id)
else:
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.resize()
def clear_terminal(self, action, only_active):
if only_active:
windows = []
w = self.active_window
if w is not None:
windows.append(w)
else:
windows = self.all_windows
reset = action == 'reset'
how = 3 if action == 'scrollback' else 2
for w in windows:
if action == 'scroll':
w.screen.scroll_until_cursor()
continue
w.screen.cursor.x = w.screen.cursor.y = 0
if reset:
w.screen.reset()
else:
w.screen.erase_in_display(how, False)
def increase_font_size(self): # legacy
cfs = global_font_size()
self.set_font_size(min(self.opts.font_size * 5, cfs + 2.0))
def decrease_font_size(self): # legacy
cfs = global_font_size()
self.set_font_size(max(MINIMUM_FONT_SIZE, cfs - 2.0))
def restore_font_size(self): # legacy
self.set_font_size(self.opts.font_size)
def set_font_size(self, new_size): # legacy
self.change_font_size(True, None, new_size)
def change_font_size(self, all_windows, increment_operation, amt):
def calc_new_size(old_size):
new_size = old_size
if amt == 0:
new_size = self.opts.font_size
else:
if increment_operation:
new_size += (1 if increment_operation == '+' else -1) * amt
else:
new_size = amt
new_size = max(MINIMUM_FONT_SIZE,
min(new_size, self.opts.font_size * 5))
return new_size
if all_windows:
current_global_size = global_font_size()
new_size = calc_new_size(current_global_size)
if new_size != current_global_size:
global_font_size(new_size)
os_windows = tuple(self.os_window_map.keys())
else:
os_windows = []
w = self.active_window
if w is not None:
os_windows.append(w.os_window_id)
if os_windows:
final_windows = {}
for wid in os_windows:
current_size = os_window_font_size(wid)
if current_size:
new_size = calc_new_size(current_size)
if new_size != current_size:
final_windows[wid] = new_size
if final_windows:
self._change_font_size(final_windows)
def _change_font_size(self, sz_map):
for os_window_id, sz in sz_map.items():
tm = self.os_window_map.get(os_window_id)
if tm is not None:
os_window_font_size(os_window_id, sz)
tm.resize()
def on_dpi_change(self, os_window_id):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
sz = os_window_font_size(os_window_id)
if sz:
os_window_font_size(os_window_id, sz, True)
tm.resize()
def _set_os_window_background_opacity(self, os_window_id, opacity):
change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0)))
def set_background_opacity(self, opacity):
window = self.active_window
if window is None or not opacity:
return
if not self.opts.dynamic_background_opacity:
return self.show_error(
_('Cannot change background opacity'),
_('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity'
))
os_window_id = window.os_window_id
if opacity[0] in '+-':
old_opacity = background_opacity_of(os_window_id)
if old_opacity is None:
return
opacity = old_opacity + float(opacity)
elif opacity == 'default':
opacity = self.opts.background_opacity
else:
opacity = float(opacity)
self._set_os_window_background_opacity(os_window_id, opacity)
@property
def active_tab_manager(self):
os_window_id = current_os_window()
return self.os_window_map.get(os_window_id)
@property
def active_tab(self):
tm = self.active_tab_manager
if tm is not None:
return tm.active_tab
@property
def active_window(self):
t = self.active_tab
if t is not None:
return t.active_window
def dispatch_special_key(self, key, scancode, action, mods):
# Handles shortcuts, return True if the key was consumed
key_action = get_shortcut(self.keymap, mods, key, scancode)
if key_action is None:
sequences = get_shortcut(self.opts.sequence_map, mods, key,
scancode)
if sequences:
self.pending_sequences = sequences
set_in_sequence_mode(True)
return True
else:
self.current_key_press_info = key, scancode, action, mods
return self.dispatch_action(key_action)
def process_sequence(self, key, scancode, action, mods):
if not self.pending_sequences:
set_in_sequence_mode(False)
remaining = {}
matched_action = None
for seq, key_action in self.pending_sequences.items():
if shortcut_matches(seq[0], mods, key, scancode):
seq = seq[1:]
if seq:
remaining[seq] = key_action
else:
matched_action = key_action
if remaining:
self.pending_sequences = remaining
else:
self.pending_sequences = None
set_in_sequence_mode(False)
if matched_action is not None:
self.dispatch_action(matched_action)
def start_resizing_window(self):
w = self.active_window
if w is None:
return
overlay_window = self._run_kitten(
'resize_window',
args=[
'--horizontal-increment={}'.format(
self.opts.window_resize_step_cells),
'--vertical-increment={}'.format(
self.opts.window_resize_step_lines)
])
if overlay_window is not None:
overlay_window.allow_remote_control = True
def resize_layout_window(self,
window,
increment,
is_horizontal,
reset=False):
tab = window.tabref()
if tab is None or not increment:
return False
if reset:
return tab.reset_window_sizes()
return tab.resize_window_by(window.id, increment, is_horizontal)
def default_bg_changed_for(self, window_id):
w = self.window_id_map.get(window_id)
if w is not None:
tm = self.os_window_map.get(w.os_window_id)
if tm is not None:
tm.update_tab_bar_data()
tm.mark_tab_bar_dirty()
t = tm.tab_for_id(w.tab_id)
if t is not None:
t.relayout_borders()
def dispatch_action(self, key_action):
if key_action is not None:
f = getattr(self, key_action.func, None)
if f is not None:
if self.args.debug_keyboard:
print('Keypress matched action:', func_name(f))
passthrough = f(*key_action.args)
if passthrough is not True:
return True
tab = self.active_tab
if tab is None:
return False
window = self.active_window
if window is None:
return False
if key_action is not None:
f = getattr(tab, key_action.func,
getattr(window, key_action.func, None))
if f is not None:
passthrough = f(*key_action.args)
if self.args.debug_keyboard:
print('Keypress matched action:', func_name(f))
if passthrough is not True:
return True
return False
def combine(self, *actions):
for key_action in actions:
self.dispatch_action(key_action)
def on_focus(self, os_window_id, focused):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.focus_changed(focused)
tm.mark_tab_bar_dirty()
def update_tab_bar_data(self, os_window_id):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
tm.update_tab_bar_data()
def on_drop(self, os_window_id, paths):
tm = self.os_window_map.get(os_window_id)
if tm is not None:
w = tm.active_window
if w is not None:
w.paste('\n'.join(paths))
def on_os_window_closed(self, os_window_id, viewport_width,
viewport_height):
self.cached_values['window-size'] = viewport_width, viewport_height
tm = self.os_window_map.pop(os_window_id, None)
if tm is not None:
tm.destroy()
for window_id in tuple(
w.id for w in self.window_id_map.values()
if getattr(w, 'os_window_id', None) == os_window_id):
self.window_id_map.pop(window_id, None)
action = self.os_window_death_actions.pop(os_window_id, None)
if action is not None:
action()
def notify_on_os_window_death(self, address):
import socket
s = socket.socket(family=socket.AF_UNIX)
with suppress(Exception):
s.connect(address)
s.sendall(b'c')
with suppress(EnvironmentError):
s.shutdown(socket.SHUT_RDWR)
s.close()
def display_scrollback(self, window, data, cmd):
tab = self.active_tab
if tab is not None and window.overlay_for is None:
tab.new_special_window(
SpecialWindow(cmd, data, _('History'), overlay_for=window.id))
def edit_config_file(self, *a):
confpath = prepare_config_file_for_editing()
# On macOS vim fails to handle SIGWINCH if it occurs early, so add a
# small delay.
cmd = [
kitty_exe(), '+runpy',
'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])'
] + get_editor() + [confpath]
self.new_os_window(*cmd)
def get_output(self, source_window, num_lines=1):
output = ''
s = source_window.screen
if num_lines is None:
num_lines = s.lines
for i in range(min(num_lines, s.lines)):
output += str(s.linebuf.line(i))
return output
def _run_kitten(self, kitten, args=(), input_data=None, window=None):
orig_args, args = list(args), list(args)
from kittens.runner import create_kitten_handler
end_kitten = create_kitten_handler(kitten, orig_args)
if window is None:
w = self.active_window
tab = self.active_tab
else:
w = window
tab = w.tabref()
if end_kitten.no_ui:
end_kitten(None, getattr(w, 'id', None), self)
return
if w is not None and tab is not None and w.overlay_for is None:
args[0:0] = [config_dir, kitten]
if input_data is None:
type_of_input = end_kitten.type_of_input
if type_of_input in ('text', 'history', 'ansi', 'ansi-history',
'screen', 'screen-history', 'screen-ansi',
'screen-ansi-history'):
data = w.as_text(as_ansi='ansi' in type_of_input,
add_history='history' in type_of_input,
add_wrap_markers='screen'
in type_of_input).encode('utf-8')
elif type_of_input is None:
data = None
else:
raise ValueError(
'Unknown type_of_input: {}'.format(type_of_input))
else:
data = input_data
if isinstance(data, str):
data = data.encode('utf-8')
copts = {
k: self.opts[k]
for k in ('select_by_word_characters', 'open_url_with')
}
overlay_window = tab.new_special_window(
SpecialWindow([
kitty_exe(), '+runpy',
'from kittens.runner import main; main()'
] + args,
stdin=data,
env={
'KITTY_COMMON_OPTS': json.dumps(copts),
'KITTY_CHILD_PID': w.child.pid,
'PYTHONWARNINGS': 'ignore',
'OVERLAID_WINDOW_LINES': str(w.screen.lines),
'OVERLAID_WINDOW_COLS':
str(w.screen.columns),
},
cwd=w.cwd_of_child,
overlay_for=w.id))
overlay_window.action_on_close = partial(self.on_kitten_finish,
w.id, end_kitten)
return overlay_window
def kitten(self, kitten, *args):
import shlex
cmdline = args[0] if args else ''
args = shlex.split(cmdline) if cmdline else []
self._run_kitten(kitten, args)
def on_kitten_finish(self, target_window_id, end_kitten, source_window):
output = self.get_output(source_window, num_lines=None)
from kittens.runner import deserialize
data = deserialize(output)
if data is not None:
end_kitten(data, target_window_id, self)
def input_unicode_character(self):
self._run_kitten('unicode_input')
def set_tab_title(self):
tab = self.active_tab
if tab:
args = [
'--name=tab-title', '--message',
_('Enter the new title for this tab below.'),
'do_set_tab_title',
str(tab.id)
]
self._run_kitten('ask', args)
def show_error(self, title, msg):
self._run_kitten('show_error', args=['--title', title], input_data=msg)
def do_set_tab_title(self, title, tab_id):
tm = self.active_tab_manager
if tm is not None and title:
tab_id = int(tab_id)
for tab in tm.tabs:
if tab.id == tab_id:
tab.set_title(title)
break
def kitty_shell(self, window_type):
cmd = ['@', kitty_exe(), '@']
if window_type == 'tab':
self._new_tab(cmd)
elif window_type == 'os_window':
os_window_id = self._new_os_window(cmd)
self.os_window_map[os_window_id]
elif window_type == 'overlay':
w = self.active_window
tab = self.active_tab
if w is not None and tab is not None and w.overlay_for is None:
tab.new_special_window(SpecialWindow(cmd, overlay_for=w.id))
else:
self._new_window(cmd)
def switch_focus_to(self, window_idx):
tab = self.active_tab
tab.set_active_window_idx(window_idx)
def open_url(self, url, program=None, cwd=None):
if url:
if isinstance(program, str):
program = to_cmdline(program)
open_url(url, program or self.opts.open_url_with, cwd=cwd)
def open_url_lines(self, lines, program=None):
self.open_url(''.join(lines), program)
def destroy(self):
self.shutting_down = True
self.child_monitor.shutdown_monitor()
self.set_update_check_process()
self.update_check_process = None
del self.child_monitor
for tm in self.os_window_map.values():
tm.destroy()
self.os_window_map = {}
destroy_global_data()
def paste_to_active_window(self, text):
if text:
w = self.active_window
if w is not None:
w.paste(text)
def paste_from_clipboard(self):
text = get_clipboard_string()
self.paste_to_active_window(text)
def paste_from_selection(self):
text = get_primary_selection(
) if supports_primary_selection else get_clipboard_string()
self.paste_to_active_window(text)
def set_primary_selection(self):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
set_primary_selection(text)
if self.opts.copy_on_select:
self.copy_to_buffer(self.opts.copy_on_select)
def copy_to_buffer(self, buffer_name):
w = self.active_window
if w is not None and not w.destroyed:
text = w.text_for_selection()
if text:
if buffer_name == 'clipboard':
set_clipboard_string(text)
elif buffer_name == 'primary':
set_primary_selection(text)
else:
self.clipboard_buffers[buffer_name] = text
def paste_from_buffer(self, buffer_name):
if buffer_name == 'clipboard':
text = get_clipboard_string()
elif buffer_name == 'primary':
text = get_primary_selection()
else:
text = self.clipboard_buffers.get(buffer_name)
if text:
self.paste_to_active_window(text)
def goto_tab(self, tab_num):
tm = self.active_tab_manager
if tm is not None:
tm.goto_tab(tab_num - 1)
def set_active_tab(self, tab):
tm = self.active_tab_manager
if tm is not None:
return tm.set_active_tab(tab)
return False
def next_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab()
def previous_tab(self):
tm = self.active_tab_manager
if tm is not None:
tm.next_tab(-1)
prev_tab = previous_tab
def process_stdin_source(self, window=None, stdin=None):
w = window or self.active_window
env = None
if stdin:
add_wrap_markers = stdin.endswith('_wrap')
if add_wrap_markers:
stdin = stdin[:-len('_wrap')]
stdin = data_for_at(w, stdin, add_wrap_markers=add_wrap_markers)
if stdin is not None:
pipe_data = w.pipe_data(
stdin, has_wrap_markers=add_wrap_markers) if w else {}
if pipe_data:
env = {
'KITTY_PIPE_DATA':
'{scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}'
.format(**pipe_data)
}
stdin = stdin.encode('utf-8')
return env, stdin
def special_window_for_cmd(self,
cmd,
window=None,
stdin=None,
cwd_from=None,
as_overlay=False):
w = window or self.active_window
env, stdin = self.process_stdin_source(w, stdin)
cmdline = []
for arg in cmd:
if arg == '@selection':
arg = data_for_at(w, arg)
if not arg:
continue
cmdline.append(arg)
overlay_for = w.id if as_overlay and w.overlay_for is None else None
return SpecialWindow(cmd,
stdin,
cwd_from=cwd_from,
overlay_for=overlay_for,
env=env)
def pipe(self, source, dest, exe, *args):
cmd = [exe] + list(args)
window = self.active_window
cwd_from = window.child.pid_for_cwd if window else None
def create_window():
return self.special_window_for_cmd(cmd,
stdin=source,
as_overlay=dest == 'overlay',
cwd_from=cwd_from)
if dest == 'overlay' or dest == 'window':
tab = self.active_tab
if tab is not None:
return tab.new_special_window(create_window())
elif dest == 'tab':
tm = self.active_tab_manager
if tm is not None:
tm.new_tab(special_window=create_window(), cwd_from=cwd_from)
elif dest == 'os_window':
self._new_os_window(create_window(), cwd_from=cwd_from)
elif dest in ('clipboard', 'primary'):
env, stdin = self.process_stdin_source(stdin=source, window=window)
if stdin:
func = set_clipboard_string if dest == 'clipboard' else set_primary_selection
func(stdin)
else:
import subprocess
env, stdin = self.process_stdin_source(stdin=source, window=window)
cwd = None
if cwd_from:
with suppress(Exception):
cwd = cwd_of_process(cwd_from)
if stdin:
r, w = safe_pipe(False)
try:
subprocess.Popen(cmd, env=env, stdin=r, cwd=cwd)
except Exception:
os.close(w)
else:
thread_write(w, stdin)
finally:
os.close(r)
else:
subprocess.Popen(cmd, env=env, cwd=cwd)
def args_to_special_window(self, args, cwd_from=None):
args = list(args)
stdin = None
w = self.active_window
if args[0].startswith('@') and args[0] != '@':
stdin = data_for_at(w, args[0]) or None
if stdin is not None:
stdin = stdin.encode('utf-8')
del args[0]
cmd = []
for arg in args:
if arg == '@selection':
arg = data_for_at(w, arg)
if not arg:
continue
cmd.append(arg)
return SpecialWindow(cmd, stdin, cwd_from=cwd_from)
def _new_tab(self, args, cwd_from=None, as_neighbor=False):
special_window = None
if args:
if isinstance(args, SpecialWindowInstance):
special_window = args
else:
special_window = self.args_to_special_window(args,
cwd_from=cwd_from)
tm = self.active_tab_manager
if tm is not None:
return tm.new_tab(special_window=special_window,
cwd_from=cwd_from,
as_neighbor=as_neighbor)
def _create_tab(self, args, cwd_from=None):
as_neighbor = False
if args and args[0].startswith('!'):
as_neighbor = 'neighbor' in args[0][1:].split(',')
args = args[1:]
self._new_tab(args, as_neighbor=as_neighbor, cwd_from=cwd_from)
def new_tab(self, *args):
self._create_tab(args)
def new_tab_with_cwd(self, *args):
w = self.active_window_for_cwd
cwd_from = w.child.pid_for_cwd if w is not None else None
self._create_tab(args, cwd_from=cwd_from)
def new_tab_with_wd(self, wd):
special_window = SpecialWindow(None, cwd=wd)
self._new_tab(special_window)
def _new_window(self, args, cwd_from=None):
tab = self.active_tab
if tab is not None:
location = None
if args and args[0].startswith('!'):
location = args[0][1:].lower()
args = args[1:]
if args:
return tab.new_special_window(self.args_to_special_window(
args, cwd_from=cwd_from),
location=location)
else:
return tab.new_window(cwd_from=cwd_from, location=location)
def new_window(self, *args):
self._new_window(args)
def new_window_with_cwd(self, *args):
w = self.active_window_for_cwd
if w is None:
return self.new_window(*args)
cwd_from = w.child.pid_for_cwd if w is not None else None
self._new_window(args, cwd_from=cwd_from)
def move_tab_forward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(1)
def move_tab_backward(self):
tm = self.active_tab_manager
if tm is not None:
tm.move_tab(-1)
def disable_ligatures_in(self, where, strategy):
if isinstance(where, str):
windows = ()
if where == 'active':
if self.active_window is not None:
windows = (self.active_window, )
elif where == 'all':
windows = self.all_windows
elif where == 'tab':
if self.active_tab is not None:
windows = tuple(self.active_tab)
else:
windows = where
for window in windows:
window.screen.disable_ligatures = strategy
window.refresh()
def patch_colors(self, spec, cursor_text_color, configured=False):
if configured:
for k, v in spec.items():
if hasattr(self.opts, k):
setattr(self.opts, k, color_from_int(v))
if cursor_text_color is not False:
if isinstance(cursor_text_color, int):
cursor_text_color = color_from_int(cursor_text_color)
self.opts.cursor_text_color = cursor_text_color
for tm in self.all_tab_managers:
tm.tab_bar.patch_colors(spec)
patch_global_colors(spec, configured)
def safe_delete_temp_file(self, path):
if is_path_in_temp_dir(path):
with suppress(FileNotFoundError):
os.remove(path)
def set_update_check_process(self, process=None):
if self.update_check_process is not None:
with suppress(Exception):
if self.update_check_process.poll() is None:
self.update_check_process.kill()
self.update_check_process = process
def on_monitored_pid_death(self, pid, exit_status):
update_check_process = getattr(self, 'update_check_process', None)
if update_check_process is not None and pid == update_check_process.pid:
self.update_check_process = None
from .update_check import process_current_release
try:
raw = update_check_process.stdout.read().decode('utf-8')
except Exception as e:
log_error(
'Failed to read data from update check process, with error: {}'
.format(e))
else:
try:
process_current_release(raw)
except Exception as e:
log_error(
'Failed to process update check data {!r}, with error: {}'
.format(raw, e))
def notification_activated(self, identifier):
if identifier == 'new-version':
from .update_check import notification_activated
notification_activated()
def dbus_notification_callback(self, activated, *args):
from .notify import dbus_notification_created, dbus_notification_activated
if activated:
dbus_notification_activated(*args)
else:
dbus_notification_created(*args)
def show_bad_config_lines(self, bad_lines):
def format_bad_line(bad_line):
return '{}:{} in line: {}\n'.format(bad_line.number,
bad_line.exception,
bad_line.line)
msg = '\n'.join(map(format_bad_line, bad_lines)).rstrip()
self.show_error(_('Errors in kitty.conf'), msg)
class SlackHandler(GenericHandler):
def __init__(self):
super(SlackHandler, self).__init__()
session = ClientSession()
self.slack_client = SlackAPI(token=None, session=session)
self.channels = WeakValueDictionary()
async def setup(self, token):
self.slack_client._token = token
await self.update_team_info()
await self.update_channels()
asyncio.ensure_future(self.rtm())
print("Logged into Slack")
def get_channel(self, serialised) -> Optional[SlackChannel]:
channel_id = serialised["id"]
try:
return self.channels[channel_id]
except KeyError:
pass
rtn = SlackChannel(channel_id, self.slack_client)
self.channels[channel_id] = rtn
return rtn
async def rtm(self):
async for event in self.slack_client.rtm():
if isinstance(event, Message):
await asyncio.gather(*[channel.on_message_handler(event) for channel in self.channels.values()])
if isinstance(event, Event):
if event["type"].startswith("channel_"):
await self.update_channels()
@property
def serialised_channels(self):
return self._serialised_channels
async def update_channels(self):
channels = await self.slack_client.query(
slack.methods.CONVERSATIONS_LIST,
data={
"exclude_archived": True
}
)
self._serialised_channels = [
{
"type": "slack",
"id": channel["id"],
"name": channel["name"],
"server": self.team_info
}
for channel in channels["channels"]
if channel["is_member"]
]
async def update_team_info(self):
team_info = await self.slack_client.query("https://slack.com/api/team.info")
self.team_info = {
"id": team_info["team"]["id"],
"name": team_info["team"]["name"],
"icon": team_info["team"]["icon"]["image_132"]
}
