def __init__(self, logger=None): if logger is None: logger = logging.getLogger(object.__repr__(self)) super(Manager, self).__init__({}, logger) self.running = False self.running_event = threading.Event() self.modules = dict() self.to_stop = list() # objects to stop self.daemons = list() # and to join self.valueTypes = {'str': str, 'float': float, 'bool': bool, 'int': int} self.insts = dict() # module -> concrete modules implementing module self.modules_implementing = dict() self.insts_implementing = dict() self.add_module_definition('module', ModuleDefinition()) self.add_module_definition('manager', ModuleDefinition()) self.add_module_definition('threadPool', ModuleDefinition( vsettings={'minFree': VSettingDefinition('int', 4), 'maxFree': VSettingDefinition('int', 8), 'min': VSettingDefinition('int', 8)}, implementedBy='mirte.threadPool.ThreadPool')) self.register_instance('manager', 'manager', self, {}, {}) self.create_instance('threadPool', 'threadPool', {}) self.sleep_event = KeyboardInterruptableEvent() # set of paths of the mirteFiles that already have been loaded self.loaded_mirteFiles = set([])
class Manager(Module): def __init__(self, logger=None): if logger is None: logger = logging.getLogger(object.__repr__(self)) super(Manager, self).__init__({}, logger) self.running = False self.running_event = threading.Event() self.modules = dict() self.to_stop = list() # objects to stop self.daemons = list() # and to join self.valueTypes = {'str': str, 'float': float, 'bool': bool, 'int': int} self.insts = dict() # module -> concrete modules implementing module self.modules_implementing = dict() self.insts_implementing = dict() self.add_module_definition('module', ModuleDefinition()) self.add_module_definition('manager', ModuleDefinition()) self.add_module_definition('threadPool', ModuleDefinition( vsettings={'minFree': VSettingDefinition('int', 4), 'maxFree': VSettingDefinition('int', 8), 'min': VSettingDefinition('int', 8)}, implementedBy='mirte.threadPool.ThreadPool')) self.register_instance('manager', 'manager', self, {}, {}) self.create_instance('threadPool', 'threadPool', {}) self.sleep_event = KeyboardInterruptableEvent() # set of paths of the mirteFiles that already have been loaded self.loaded_mirteFiles = set([]) def _get_all(self, _type): """ Gets all instances implementing type <_type> """ if _type not in self.modules: raise ValueError("No such module, %s" % _type) if not self.insts_implementing.get(_type, None): raise ValueError("No instance implementing %s" % _type) return self.insts_implementing[_type] def get_a(self, _type): """ Gets an instance implementing type <_type> """ return self.insts[self._get_a(_type)].object def _get_a(self, _type): """ Gets an instance implementing type <_type> """ tmp = self._get_all(_type) ret = pick(tmp) if len(tmp) != 1: self.l.warn(("get_a: %s all implement %s; " + "picking %s") % (tmp, _type, ret)) return ret def got_a(self, _type): """ Returns whether there is an instance implementing <_type> """ if _type not in self.modules: raise ValueError("No such module, %s" % _type) return (_type in self.insts_implementing and self.insts_implementing[_type]) class GoCa_Plan(object): """ A partial plan for a get_or_create_a call """ def __init__(self, man, targets, insts=None, insts_implementing=None): self.man = man self.targets = targets self.insts = dict() if insts is None else insts self.insts_implementing = (dict() if insts_implementing is None else insts_implementing) def free_instance_name_like(self, name): if (name not in self.insts and name not in self.man.insts): return name suffix = 2 while True: shot = "%s-%s" % (name, suffix) if (shot not in self.insts and name not in self.man.insts): return shot suffix += 1 def got_a(self, _type): if self.man.got_a(_type): return True return (_type in self.insts_implementing and self.insts_implementing[_type]) def get_all(self, _type): ret = list() if self.man.got_a(_type): ret.extend(self.man._get_all(_type)) if _type in self.insts_implementing: ret.extend(self.insts_implementing[_type]) return ret def get_a(self, _type): return pick(self.get_all(_type)) @property def finished(self): return not self.targets def plan_a(self, mod): name = self.free_instance_name_like(mod) self.insts[name] = (name, mod, {}) md = self.man.modules[mod] for mod2 in chain(md.inherits, (mod,)): if mod2 not in self.insts_implementing: self.insts_implementing[mod2] = list() self.insts_implementing[mod2].append(name) for depName, dep in six.iteritems(md.deps): if dep.type in self.targets: self.targets[dep.type].append( (name, depName)) else: self.targets[dep.type] = [ (name, depName)] return name def branches(self): choices = dict() for target in self.targets: if self.got_a(target): choices[target] = [(True, name) for name in self.get_all(target)] continue choices[target] = [(False, name) for name in self.man.modules_implementing[ target]] choices_t = list(six.iteritems(choices)) for choice in product(*[list(range(len(v))) for k, v in choices_t]): plan2 = Manager.GoCa_Plan(self.man, dict(), dict(self.insts), dict(self.insts_implementing)) tmp = [(choices_t[n][0], choices_t[n][1][m]) for n, m in enumerate(choice)] for target, inst_or_mod in tmp: if inst_or_mod[0]: name = inst_or_mod[1] else: name = plan2.plan_a(inst_or_mod[1]) for _in, depName in self.targets[target]: plan2.insts[_in][2][depName] = name yield plan2 def execute(self): insts = frozenset(six.iterkeys(self.insts)) inst_list = tuple(sort_by_successors( insts, lambda inst: [self.insts[inst][2][k] for k in self.man.modules[self.insts[inst][1]].deps if self.insts[inst][2][k] in insts] )) for name in reversed(inst_list): self.man.create_instance(*self.insts[name]) def get_or_create_a(self, _type): """ Gets or creates an instance of type <_type> """ return self.insts[self._get_or_create_a(_type)].object def _get_or_create_a(self, _type): """ Gets or creates an instance of type <_type> """ self.l.debug("get_or_create_a: %s" % _type) stack = [Manager.GoCa_Plan(self, {_type: ()})] while stack: p = stack.pop() if p.finished: p.execute() return p.get_a(_type) for c in p.branches(): stack.append(c) raise NotImplementedError def free_instance_name_like(self, name): if name not in self.insts: return name suffix = 2 while True: shot = "%s-%s" % (name, suffix) if shot not in self.insts: return shot suffix += 1 def add_module_definition(self, name, definition): if name in self.modules: raise ValueError("Duplicate module name") self.modules[name] = definition if definition.implementedBy is not None: for t in chain(definition.inherits, (name,)): if t not in self.modules_implementing: self.insts_implementing[t] = set() self.modules_implementing[t] = set() self.modules_implementing[t].add(name) def update_instance(self, name, settings): """ Updates settings of instance <name> with the dictionary <settings>. """ if name not in self.insts: raise ValueError("There's no instance named %s" % name) if 'module' in settings: raise ValueError(("Can't change module of existing instan" + "ce %s") % name) self.l.info('update instance %-15s' % (name)) for k, v in six.iteritems(settings): self.change_setting(name, k, v) def create_instance(self, name, moduleName, settings): """ Creates an instance of <moduleName> at <name> with <settings>. """ if name in self.insts: raise ValueError("There's already an instance named %s" % name) if moduleName not in self.modules: raise ValueError("There's no module %s" % moduleName) md = self.modules[moduleName] deps = dict() for k, v in six.iteritems(md.deps): if k not in settings: settings[k] = self._get_or_create_a(v.type) if settings[k] is None: if not v.allow_null: raise ValueError("`null' not allowed for %s" % k) elif settings[k] not in self.insts: raise ValueError("No such instance %s" % settings[k]) else: settings[k] = self.insts[settings[k]].object deps[k] = settings[k] for k, v in six.iteritems(md.vsettings): if k not in settings: settings[k] = v.default if v.default is None: self.l.warn('%s:%s not set' % (name, k)) self.l.info('create_instance %-15s %s' % (name, md.implementedBy)) cl = get_by_path(md.implementedBy) il = logging.getLogger(name) obj = cl(settings, il) self.register_instance(name, moduleName, obj, settings, deps) return obj def register_instance(self, name, moduleName, obj, settings, deps): md = self.modules[moduleName] self.insts[name] = InstanceInfo(name, moduleName, obj, settings, deps) for mn in chain(md.inherits, (moduleName,)): if mn not in self.insts_implementing: self.insts_implementing[mn] = set() self.insts_implementing[mn].add(name) if md.run: self.to_stop.append(name) self.daemons.append(name) if self.running: self._run_instance(name) elif hasattr(obj, 'stop'): self.to_stop.append(name) def _run_instance(self, name): ii = self.insts[name] self.insts['threadPool'].object.execute_named( self._daemon_entry, "mirte run %s" % name, ii) def _daemon_entry(self, ii): try: ii.object.run() except Exception: self.l.exception(("Module %s exited " + "abnormally") % ii.name) return self.l.info("Module %s exited normally" % ii.name) def run(self): assert not self.running self.running = True self.running_event.set() tp = self.insts['threadPool'].object tp.start() # Note that self.daemons is already dependency ordered for us for name in self.daemons: self._run_instance(name) while self.running: try: self.sleep_event.wait() except KeyboardInterrupt: self.l.warn("Keyboard interrupt") self.stop() self.l.info("Woke up") self.l.info("Stopping modules") for name in reversed(self.to_stop): ii = self.insts[name] self.l.info(" %s" % ii.name) ii.object.stop() self.l.info("Joining threadPool") tp.join() def change_setting(self, instance_name, key, raw_value): """ Change the settings <key> to <raw_value> of an instance named <instance_name>. <raw_value> should be a string and is properly converted. """ ii = self.insts[instance_name] mo = self.modules[ii.module] if key in mo.deps: if raw_value not in self.insts: raise ValueError("No such instance %s" % raw_value) vii = self.insts[raw_value] vmo = self.modules[vii.module] if not (mo.deps[key].type in vmo.inherits or mo.deps[key].type == vii.module): raise ValueError("%s isn't a %s" % ( raw_value, mo.deps[key].type)) value = vii.object elif key in mo.vsettings: value = self.valueTypes[mo.vsettings[key].type]( raw_value) else: raise ValueError("No such settings %s" % key) self.l.info("Changing %s.%s to %s" % (instance_name, key, raw_value)) ii.settings[key] = value ii.object.change_setting(key, value) def stop(self): if not self.running: return self.running_event.clear() self.running = False self.sleep_event.set()