def testUpdateBundlerBasket(self): basket = self.vishnu.getBundlerBasket() self.assertEqual(basket, self.vishnu.getBundlerBasket()) # Force registry rebuild from flumotion.common import registry registry.getRegistry().verify(force=True) self.assertNotEqual(basket, self.vishnu.getBundlerBasket())
def getBundlerBasket(self): """ Return a bundler basket to unbundle from. If the registry files were updated since the last time, the bundlerbasket will be rebuilt. @since: 0.2.2 @rtype: L{flumotion.common.bundle.BundlerBasket} """ if registry.getRegistry().rebuildNeeded(): self.info("Registry changed, rebuilding") registry.getRegistry().verify(force=True) self.bundlerBasket = registry.getRegistry().makeBundlerBasket() return self.bundlerBasket
def testInit(self): r = registry.getRegistry() components = [c.getType() for c in r.getComponents()] for type in components: # skip test components - see test_config.py if type.startswith('test-'): continue log.debug('test', 'testing component type %s' % type) defs = r.getComponent(type) try: entry = defs.getEntryByType('component') except KeyError, e: self.fail( 'KeyError while trying to get component entry for %s' % type) moduleName = defs.getSource() methodName = entry.getFunction() # call __init__ without the config arg; this will load # modules, get the entry point, then fail with too-few # arguments. would be nice to __init__ with the right # config, but that is component-specific... self.assertRaises(TypeError, reflectcall.reflectCall, moduleName, methodName)
def perspective_getPlugEntry(self, plugType, entryType): """ Get the entry point for a piece of bundled code in a plug by type. @param plugType: the plug @type plugType: a string @param entryType: location of the entry point @type entryType: a string Returns: a (filename, methodName) tuple, or raises:: - NoBundleError if the entry location does not exist """ assert plugType is not None self.debug('getting entry of type %s for plug type %s', entryType, plugType) try: plugRegistryEntry = registry.getRegistry().getPlug(plugType) entry = plugRegistryEntry.getEntryByType(entryType) except KeyError: self.warning("Could not find bundle for %s(%s)" % ( plugType, entryType)) raise errors.NoBundleError("entry type %s in plug type %s" % (entryType, plugType)) self.debug('entry point is in file path %s and function %s' % ( entry.location, entry.function)) return (entry.location, entry.function)
def createComponent(self, avatarId, type, nice, conf): """ Create a component of the given type with the given nice level. @param avatarId: avatarId the component should use to log in @type avatarId: str @param type: type of the component to create @type type: str @param nice: the nice level to create the component at @type nice: int @param conf: the component's config dict @type conf: dict @returns: a deferred that will give the avatarId the component will use to log in to the manager """ self.debug('creating %s (%s) on worker %s with nice level %d', avatarId, type, self.avatarId, nice) defs = registry.getRegistry().getComponent(type) try: entry = defs.getEntryByType('component') # FIXME: use entry.getModuleName() (doesn't work atm?) moduleName = defs.getSource() methodName = entry.getFunction() except KeyError: self.warning('no "component" entry in registry of type %s, %s', type, 'falling back to createComponent') moduleName = defs.getSource() methodName = "createComponent" self.debug('call remote create') return self.mindCallRemote('create', avatarId, type, moduleName, methodName, nice, conf)
def _loadManagerBouncer(self, conf): if not (conf.bouncer): self.warning('no bouncer defined, nothing can access the ' 'manager') return defer.succeed(None) self.debug('going to start manager bouncer %s of type %s', conf.bouncer.name, conf.bouncer.type) defs = registry.getRegistry().getComponent(conf.bouncer.type) entry = defs.getEntryByType('component') # FIXME: use entry.getModuleName() (doesn't work atm?) moduleName = defs.getSource() methodName = entry.getFunction() bouncer = reflectcall.createComponent(moduleName, methodName, conf.bouncer.getConfigDict()) d = bouncer.waitForHappy() def setupCallback(result): bouncer.debug('started') self.setBouncer(bouncer) def setupErrback(failure): self.warning('Error starting manager bouncer') d.addCallbacks(setupCallback, setupErrback) return d
def perspective_getEntryByType(self, componentType, entryType): """ Get the entry point for a piece of bundled code in a component by type. @param componentType: the component @type componentType: a string @param entryType: location of the entry point @type entryType: a string Returns: a (filename, methodName) tuple, or raises:: - NoBundleError if the entry location does not exist """ assert componentType is not None self.debug('getting entry of type %s for component type %s', entryType, componentType) try: componentRegistryEntry = registry.getRegistry().getComponent( componentType) # FIXME: add logic here for default entry points and functions entry = componentRegistryEntry.getEntryByType(entryType) except KeyError: self.warning("Could not find bundle for %s(%s)" % ( componentType, entryType)) raise errors.NoBundleError("entry type %s in component type %s" % (entryType, componentType)) filename = os.path.join(componentRegistryEntry.base, entry.location) self.debug('entry point is in file path %s and function %s' % ( filename, entry.function)) return (filename, entry.function)
def __init__(self, name, parent, type, label, propertyList, plugList, worker, eatersList, isClockMaster, project, version, virtualFeeds=None): self.name = name self.parent = parent self.type = type self.label = label self.worker = worker self.defs = registry.getRegistry().getComponent(self.type) try: self.config = self._buildConfig(propertyList, plugList, eatersList, isClockMaster, project, version, virtualFeeds) except errors.ConfigError, e: # reuse the original exception? e.args = ("While parsing component %s: %s" % (name, log.getExceptionMessage(e)), ) raise
def perspective_getScenarioByType(self, scenarioType, entryType): """ Remote method that gets the scenario of a given type. @param scenarioType: the component @type scenarioType: a string Returns: a (filename, methodName) tuple, or raises:: - NoBundleError if the entry location does not exist """ assert scenarioType is not None self.debug('getting entry of type %s for scenario type %s', entryType, scenarioType) try: scenarioRegistryEntry = registry.getRegistry().getScenarioByType( scenarioType) # FIXME: add logic here for default entry points and functions entry = scenarioRegistryEntry.getEntryByType(entryType) except KeyError: self.warning("Could not find bundle for %s(%s)" % ( scenarioType, entryType)) raise errors.NoBundleError("entry type %s in component type %s" % (entryType, scenarioType)) filename = os.path.join(scenarioRegistryEntry.getBase(), entry.getLocation()) self.debug('entry point is in file path %s and function %s' % ( filename, entry.function)) return (filename, entry.getFunction())
def perspective_getScenarios(self): """ Get all the scenarios defined on the registry. @rtype : List of L{IScenarioAssistantPlugin} """ r = registry.getRegistry() return r.getScenarios()
def setUp(self): # load and verify registry from flumotion.common import registry reg = registry.getRegistry() self.vishnu = manager.Vishnu('test', unsafeTracebacks=1) self._workers = {} # id -> avatar self._components = {} # id -> avatar
def __init__(self, type_, class_, props=None, name=None, plugs=None, cfg=None): self.comp = None self.comp_class = class_ if cfg is None: cfg = {} self.cfg = cfg self.auto_link = True self.debug_msgs = [] self.sync = None self.sync_master = None if ComponentWrapper._registry is None: ComponentWrapper._registry = registry.getRegistry() cfg['type'] = type_ reg = ComponentWrapper._registry.getComponent(type_) if not 'source' in cfg: cfg['source'] = [] if not 'eater' in cfg: cfg['eater'] = dict([(e.getName(), []) for e in reg.getEaters() if e.getRequired()]) if not 'feed' in cfg: cfg['feed'] = reg.getFeeders()[:] if plugs is not None: cfg['plugs'] = plugs if not 'plugs' in cfg: cfg['plugs'] = dict([(s, []) for s in reg.getSockets()]) if name: cfg['name'] = name if not 'name' in cfg: cfg['name'] = ComponentWrapper.get_unique_name() self.name = cfg['name'] if not 'parent' in cfg: cfg['parent'] = 'default' if not 'avatarId' in cfg: cfg['avatarId'] = common.componentId(cfg['parent'], self.name) if props is not None: cfg['properties'] = props if not 'properties' in cfg: cfg['properties'] = {} if not 'clock-master' in cfg: cfg['clock-master'] = None self.sync_master = cfg['clock-master'] if reg.getNeedsSynchronization(): self.sync = reg.getClockPriority()
def perspective_loadConfiguration(self, xml, saveAs=None): """ Load the given XML configuration into the manager. If the optional saveAs parameter is passed, the XML snippet will be saved to disk in the manager's flows directory. @param xml: the XML configuration snippet. @type xml: str @param saveAs: The name of a file to save the XML as. @type saveAs: str """ if saveAs: output = self._saveFlowFile(saveAs) # Update the registry if needed, so that new/changed component types # can be parsed. registry.getRegistry().verify() f = StringIO(xml) res = self.vishnu.loadComponentConfigurationXML(f, self.remoteIdentity) f.close() if saveAs: def success(res): self.debug('loadConfiguration succeeded, writing flow to %r', output) output.truncate(0) output.write(xml) output.close() return res def failure(res): self.debug('loadConfiguration failed, leaving %r as it was', output) output.close() return res res.addCallbacks(success, failure) return res
def _getProcedure(self, type): r = registry.getRegistry() c = r.getComponent(type) try: entry = c.getEntryByType("component") except KeyError: err("Component %s has no component entry" % self.name) importname = entry.getModuleName(c.getBase()) try: module = reflect.namedAny(importname) except Exception, e: err("Could not load module %s for component %s: %s" % (importname, self.name, e))
def perspective_getComponentEntry(self, componentType): """Fetches a ComponentRegistryEntry given a componentType @param componentType: component type @type componentType: string @returns: the component @rtype: L{ComponentRegistryEntry} """ try: componentRegistryEntry = registry.getRegistry().getComponent( componentType) except KeyError: return None return componentRegistryEntry
def _getProcedure(self, type): r = registry.getRegistry() c = r.getComponent(type) try: entry = c.getEntryByType('component') except KeyError: err('Component %s has no component entry' % self.name) importname = entry.getModuleName(c.getBase()) try: module = reflect.namedAny(importname) except Exception, e: err('Could not load module %s for component %s: %s' % (importname, self.name, e))
def __init__(self, plugType, propertyList): try: defs = registry.getRegistry().getPlug(plugType) except KeyError: raise ConfigError("unknown plug type: %s" % plugType) self.type = plugType self.socket = defs.getSocket() self.properties = buildPropertyDict(propertyList, defs.getProperties()) self.config = {'type': self.type, 'socket': self.socket, 'entries': self._parseEntries(defs), 'properties': self.properties}
def __init__(self, plugType, propertyList): try: defs = registry.getRegistry().getPlug(plugType) except KeyError: raise ConfigError("unknown plug type: %s" % plugType) self.type = plugType self.socket = defs.getSocket() self.properties = buildPropertyDict(propertyList, defs.getProperties()) self.config = { 'type': self.type, 'socket': self.socket, 'entries': self._parseEntries(defs), 'properties': self.properties }
def _loadManagerPlugs(self, conf): # Load plugs for socket, plugs in conf.plugs.items(): if not socket in self.plugs: self.plugs[socket] = [] for args in plugs: self.debug('loading plug type %s for socket %s' % (args['type'], socket)) defs = registry.getRegistry().getPlug(args['type']) e = defs.getEntry() call = reflectcall.reflectCallCatching plug = call(errors.ConfigError, e.getModuleName(), e.getFunction(), args) self.plugs[socket].append(plug)
def __init__(self, type, name): self.type = type self.name = name self.properties = [] self.plugs = [] self.source = [] self.config_entry = None r = registry.getRegistry() if not r.hasComponent(self.type): err('Unknown component type: %s' % self.type) self._reg = r.getComponent(self.type) if self._reg.getNeedsSynchronization(): self.clock_priority = self._reg.getClockPriority() else: self.clock_priority = None
def resolve_links(self, component_types): for link in self.get_links(): assert link[0] and link[2] if not link[0] in component_types: err('Invalid grammar: no feeder component %s to link from' % ( link[0], )) if not link[2] in component_types: err('Invalid grammar: no eater component %s to link to' % ( link[2], )) reg = registry.getRegistry() for link in self.get_links(): compname = link[0] comptype = component_types[compname] compreg = reg.getComponent(comptype) if link[1]: if not link[1] in compreg.getFeeders(): err('Component %s has no feeder named %s' % ( compname, link[1])) # leave link[1] unchanged else: if not compreg.getFeeders(): err('Component %s has no feeders' % compname) link[1] = compreg.getFeeders()[0] for link in self.get_links(): compname = link[2] comptype = component_types[compname] compreg = reg.getComponent(comptype) eaters = compreg.getEaters() if link[3]: if not link[3] in [x.getName() for x in eaters]: err('Component %s has no eater named %s' % ( compname, link[3])) # leave link[1] unchanged else: if not eaters: err('Component %s has no eaters' % compname) link[3] = eaters[0].getName() feeders = dict([(name, []) for name in component_types]) for link in self.get_links(): feeders[link[2]].append('%s:%s' % (link[0], link[1])) return feeders
def __init__(self, name, unsafeTracebacks=0, configDir=None): # create a Dispatcher which will hand out avatars to clients # connecting to me self.dispatcher = Dispatcher(self.computeIdentity) self.workerHeaven = self._createHeaven(interfaces.IWorkerMedium, worker.WorkerHeaven) self.componentHeaven = self._createHeaven(interfaces.IComponentMedium, component.ComponentHeaven) self.adminHeaven = self._createHeaven(interfaces.IAdminMedium, admin.AdminHeaven) self.running = True def setStopped(): self.running = False reactor.addSystemEventTrigger('before', 'shutdown', setStopped) if configDir is not None: self.configDir = configDir else: self.configDir = os.path.join(configure.configdir, "managers", name) self.bouncer = None # used by manager to authenticate worker/component self.bundlerBasket = registry.getRegistry().makeBundlerBasket() self._componentMappers = {} # any object -> ComponentMapper self.state = planet.ManagerPlanetState() self.state.set('name', name) self.state.set('version', configure.version) self.plugs = {} # socket -> list of plugs # create a portal so that I can be connected to, through our dispatcher # implementing the IRealm and a bouncer self.portal = fportal.BouncerPortal(self.dispatcher, None) #unsafeTracebacks = 1 # for debugging tracebacks to clients self.factory = pb.PBServerFactory(self.portal, unsafeTracebacks=unsafeTracebacks) self.connectionInfo = {} self.setConnectionInfo(None, None, None)
def perspective_getWizardEntries(self, types=None, provides=None, accepts=None): """ Fetches the wizard entries which matches the parameters sent in @param types: list of component types to fetch, is usually something like ['video-producer'] or ['audio-encoder'] @type types: list of strings @param provides: formats provided, eg ['jpeg', 'speex'] @type provides: list of strings @param accepts: formats accepted, eg ['theora'] @type accepts: list of strings @returns: L{componentui.WizardEntryState} """ def extract(wizards): for wizard in wizards: if types is not None: if wizard.type not in types: continue if provides is not None: for formatProvided in wizard.provides: if formatProvided.media_type in provides: break else: continue if accepts is not None: for formatAccepted in wizard.accepts: if formatAccepted.media_type in accepts: break else: continue yield wizard retval = [] r = registry.getRegistry() for component in r.getComponents(): retval += extract(component.wizards) for plug in r.getPlugs(): retval += extract(plug.wizards) del r return retval
def upgradeEaters(conf): def parseFeedId(feedId): if feedId.find(':') == -1: return "%s:default" % feedId else: return feedId eaterConfig = conf.get('eater', {}) sourceConfig = conf.get('source', []) if eaterConfig == {} and sourceConfig != []: eaters = registry.getRegistry().getComponent( conf.get('type')).getEaters() eatersDict = {} eatersTuple = [(None, parseFeedId(s)) for s in sourceConfig] eatersDict = buildEatersDict(eatersTuple, eaters) conf['eater'] = eatersDict if sourceConfig: sources = [] for s in sourceConfig: sources.append(parseFeedId(s)) conf['source'] = sources
def _validateManagerPlugs(conf): for socket, plugs in conf.plugs.items(): for args in plugs: defs = registry.getRegistry().getPlug(args['type']) e = defs.getEntry() e.getModuleName()
def main(args): from flumotion.common import setup setup.setupPackagePath() usage_str = ('Usage: %prog [options] [COMPONENT-OR-PLUG' ' [FULL-PROPERTY-NAME]]') fpname_str = ("FULL-PROPERTY-NAME: represents a fully qualified" " property name, including the names of the containing" " properties: " "...[property-name:]property-name") parser = OptionParser(usage=usage_str, description=fpname_str, domain="flumotion-inspect") log.debug('inspect', 'Parsing arguments (%r)' % ', '.join(args)) options, args = parser.parse_args(args) r = registry.getRegistry() if len(args) == 1: # print all components components = [(c.getType(), c) for c in r.getComponents()] components.sort() print '\nAvailable components:\n' for name, c in components: print ' %s' % name plugs = [(p.getType(), p) for p in r.getPlugs()] plugs.sort() print '\nAvailable plugs:\n' for name, p in plugs: print ' %s' % name print elif len(args) == 2: cname = args[1] handled = False if r.hasComponent(cname): handled = True c = r.getComponent(cname) print '\nComponent:' print ' %s' % cname desc = c.getDescription() if desc: print ' %s' % desc print '\nSource:' print ' %s' % c.getSource() print ' in %s' % c.getBase() print '\nEaters:' if c.getEaters(): for e in c.getEaters(): print(' %s (%s%s)' % (e.getName(), e.getRequired() and 'required' or 'optional', (e.getMultiple() and ', multiple ok' or ''))) else: print ' (None)' print '\nFeeders:' if c.getFeeders(): for e in c.getFeeders(): print ' %s' % e else: print ' (None)' print '\nFeatures:' features = [(p.getType(), p) for p in c.getEntries()] features.sort() if features: for k, v in features: print ' %s: %s:%s' % (k, v.getLocation(), v.getFunction()) else: print ' (None)' print '\nProperties:' printProperties(c.getProperties(), 0) sockets = c.getSockets() print '\nClocking:' print ' Needs synchronisation: %r' % c.getNeedsSynchronization() if (c.getClockPriority() is not None and c.getNeedsSynchronization()): print ' Clock priority: %d' % c.getClockPriority() print '\nSockets:' for socket in sockets: print ' %s' % socket print if r.hasPlug(cname): handled = True p = r.getPlug(cname) print '\nPlug type:' print ' %s' % cname desc = p.getDescription() if desc: print ' %s' % desc print '\nEntry:' e = p.getEntry() print ' %s() in %s' % (e.getFunction(), e.getModuleName()) print '\nProperties:' printProperties(p.getProperties(), 0) print if not handled: parser.exit(status=1, msg=('Unknown component or plug `%s\'\n' % cname)) elif len(args) == 3: cname = args[1] pname = args[2] ppath = pname.split(':') handled = False if r.hasComponent(cname): handled = True c = r.getComponent(cname) try: prop = getNestedProperty(c, ppath) except _NestedPropertyError, npe: parser.exit(status=1, msg='%s\n' % npe.message) print '\nComponent:' print ' %s' % cname desc = c.getDescription() if desc: print ' %s' % desc print '\nProperty:' printProperty(prop, len(prop.getName())) print if r.hasPlug(cname): handled = True p = r.getPlug(cname) try: prop = getNestedProperty(p, ppath) except _NestedPropertyError, npe: parser.exit(status=1, msg='%s\n' % npe.message) print '\nPlug:' print ' %s' % cname print '\nType:' print ' %s' % p.getType() print '\nProperty:' printProperty(prop, len(prop.getName())) print
<compound-property name="cp1" multiple="true" required="true" _description="A property"> <property name="one" type="string" required="true" _description="A property"/> </compound-property> <compound-property name="cp2" multiple="false" required="false" _description="A property"> <property name="two" type="int" required="false" _description="A property"/> </compound-property> </properties> </plug> </plugs> </registry>""" reg = registry.getRegistry() reg.addFromString(regchunk) def ConfigXML(string, parser=config.PlanetConfigParser): f = StringIO(string) conf = parser(f) f.close() return conf def ManagerConfigXML(string): return ConfigXML(string, config.ManagerConfigParser) class TestFunctions(testsuite.TestCase):
def main(args): from flumotion.common import setup setup.setupPackagePath() usage_str = ('Usage: %prog [options] [COMPONENT-OR-PLUG' ' [FULL-PROPERTY-NAME]]') fpname_str = ("FULL-PROPERTY-NAME: represents a fully qualified" " property name, including the names of the containing" " properties: " "...[property-name:]property-name") parser = OptionParser(usage=usage_str, description=fpname_str, domain="flumotion-inspect") log.debug('inspect', 'Parsing arguments (%r)' % ', '.join(args)) options, args = parser.parse_args(args) r = registry.getRegistry() if len(args) == 1: # print all components components = [(c.getType(), c) for c in r.getComponents()] components.sort() print '\nAvailable components:\n' for name, c in components: print ' %s' % name plugs = [(p.getType(), p) for p in r.getPlugs()] plugs.sort() print '\nAvailable plugs:\n' for name, p in plugs: print ' %s' % name print elif len(args) == 2: cname = args[1] handled = False if r.hasComponent(cname): handled = True c = r.getComponent(cname) print '\nComponent:' print ' %s' % cname desc = c.getDescription() if desc: print ' %s' % desc print '\nSource:' print ' %s' % c.getSource() print ' in %s' % c.getBase() print '\nEaters:' if c.getEaters(): for e in c.getEaters(): print (' %s (%s%s)' % (e.getName(), e.getRequired() and 'required' or 'optional', (e.getMultiple() and ', multiple ok' or ''))) else: print ' (None)' print '\nFeeders:' if c.getFeeders(): for e in c.getFeeders(): print ' %s' % e else: print ' (None)' print '\nFeatures:' features = [(p.getType(), p) for p in c.getEntries()] features.sort() if features: for k, v in features: print ' %s: %s:%s' % (k, v.getLocation(), v.getFunction()) else: print ' (None)' print '\nProperties:' printProperties(c.getProperties(), 0) sockets = c.getSockets() print '\nClocking:' print ' Needs synchronisation: %r' % c.getNeedsSynchronization() if (c.getClockPriority() is not None and c.getNeedsSynchronization()): print ' Clock priority: %d' % c.getClockPriority() print '\nSockets:' for socket in sockets: print ' %s' % socket print if r.hasPlug(cname): handled = True p = r.getPlug(cname) print '\nPlug type:' print ' %s' % cname desc = p.getDescription() if desc: print ' %s' % desc print '\nEntry:' e = p.getEntry() print ' %s() in %s' % (e.getFunction(), e.getModuleName()) print '\nProperties:' printProperties(p.getProperties(), 0) print if not handled: parser.exit(status=1, msg=('Unknown component or plug `%s\'\n' % cname)) elif len(args) == 3: cname = args[1] pname = args[2] ppath = pname.split(':') handled = False if r.hasComponent(cname): handled = True c = r.getComponent(cname) try: prop = getNestedProperty(c, ppath) except _NestedPropertyError, npe: parser.exit(status=1, msg='%s\n' % npe.message) print '\nComponent:' print ' %s' % cname desc = c.getDescription() if desc: print ' %s' % desc print '\nProperty:' printProperty(prop, len(prop.getName())) print if r.hasPlug(cname): handled = True p = r.getPlug(cname) try: prop = getNestedProperty(p, ppath) except _NestedPropertyError, npe: parser.exit(status=1, msg='%s\n' % npe.message) print '\nPlug:' print ' %s' % cname print '\nType:' print ' %s' % p.getType() print '\nProperty:' printProperty(prop, len(prop.getName())) print
def testDefault(self): self.failUnless(hasattr(registry, 'getRegistry')) reg = registry.getRegistry() self.failUnless(isinstance(reg, registry.ComponentRegistry))