示例#1
0
    def __init__(self, pad, name, setActive, setInactive, reconnectEater,
                 *args):
        PadMonitor.__init__(self, pad, name, setActive, setInactive)

        self._reconnectPoller = Poller(lambda: reconnectEater(*args),
                                       self.PAD_MONITOR_CHECK_INTERVAL,
                                       start=False)
示例#2
0
    def init(self):
        """
        A subclass should do as little as possible in its init method.
        In particular, it should not try to access resources.

        Failures during init are marshalled back to the manager through
        the worker's remote_create method, since there is no component state
        proxied to the manager yet at the time of init.
        """
        self.state = planet.WorkerJobState()

        self.name = self.config['name']

        self.state.set('pid', os.getpid())
        self.setMood(moods.waking)

        self.medium = None # the medium connecting us to the manager's avatar

        self.uiState = componentui.WorkerComponentUIState()
        self.uiState.addKey('cpu-percent')
        self.uiState.addKey('start-time')
        self.uiState.addKey('current-time')
        self.uiState.addKey('virtual-size')

        self.plugs = {}

        self._happyWaits = []

        # Start the cpu-usage updating.
        self._lastTime = time.time()
        self._lastClock = time.clock()
        self._cpuPoller = Poller(self._pollCPU, 5)
        self._memoryPoller = Poller(self._pollMemory, 60)

        self._shutdownHook = None
示例#3
0
    def __init__(self, pad, name, setActive, setInactive):
        """
        @type  pad:         L{gst.Pad}
        @type  name:        str
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._last_data_time = -1  # system time in epoch secs of last reception
        self._pad = pad
        self.name = name
        self._active = False
        self._first = True
        self._running = True

        self._doSetActive = []
        self._doSetInactive = []
        self.addWatch(setActive, setInactive)

        # This dict sillyness is because python's dict operations are atomic
        # w.r.t. the GIL.
        self._probe_id = {}

        self.check_poller = Poller(self._probe_timeout,
                                   self.PAD_MONITOR_PROBE_INTERVAL,
                                   immediately=True)

        self.watch_poller = Poller(self._check_timeout,
                                   self.PAD_MONITOR_CHECK_INTERVAL)
示例#4
0
    def __init__(self, pad, name, setActive, setInactive):
        """
        @type  pad:         L{gst.Pad}
        @type  name:        str
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._last_data_time = -1 # system time in epoch secs of last reception
        self._pad = pad
        self.name = name
        self._active = False
        self._first = True
        self._running = True

        self._doSetActive = []
        self._doSetInactive = []
        self.addWatch(setActive, setInactive)

        # This dict sillyness is because python's dict operations are atomic
        # w.r.t. the GIL.
        self._probe_id = {}

        self.check_poller = Poller(self._probe_timeout,
                                   self.PAD_MONITOR_PROBE_INTERVAL,
                                   immediately=True)

        self.watch_poller = Poller(self._check_timeout,
                                   self.PAD_MONITOR_CHECK_INTERVAL)
示例#5
0
文件: rrd.py 项目: flyapen/UgFlu
class ComponentRRDPlug(base.ComponentPlug):
    """Class to create or update a RRD file with statistics"""

    ### ComponentPlug methods

    def start(self, component):
        self._rrdpoller = None

        self._component = component

        if not self._hasImport():
            return

        properties = self.args['properties']
        self._clientsPath = properties['clients-connected-file']
        self._bytesPath = properties['bytes-transferred-file']
        self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
        self._RRDPaths = self._getRRDPaths()
        # call to update_rrd with a poll interval
        timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
        self._rrdpoller = Poller(self._updateRRD, timeout)

    def stop(self, component):
        if self._rrdpoller:
            self._rrdpoller.stop()

    def _hasImport(self):
        """Check rrdtool availability"""
        if not rrdtool:
            m = messages.Warning(T_(N_(
                "Cannot import module '%s'.\n"), 'rrdtool'),
                                    mid='rrdtool-import-error')
            m.add(T_(N_(
                "The RRD plug for this component is disabled.")))
            self._component.addMessage(m)
            return False

        return True

    def _updateRRD(self):
        """Update data in RRD file"""
        for path in self._RRDPaths:
            value = None
            if path == self._clientsPath:
                value = self._component.getClients()
            elif path == self._bytesPath:
                value = self._component.getBytesSent()

            try:
                rrdtool.update(path, 'N:%i' % value)
                self.debug('RRD file [%s] updated with value: %r',
                    path, value)
            except rrdtool.error, e:
                # We could get an error from rrdtool on converting the
                # value to a double or from not finding the file
                self.warning('RRD error: %r',
                             log.getExceptionMessage(e))
示例#6
0
文件: rrd.py 项目: sharky93/flumotion
class ComponentRRDPlug(base.ComponentPlug):
    """Class to create or update a RRD file with statistics"""

    ### ComponentPlug methods

    def start(self, component):
        self._rrdpoller = None

        self._component = component

        if not self._hasImport():
            return

        properties = self.args['properties']
        self._clientsPath = properties['clients-connected-file']
        self._bytesPath = properties['bytes-transferred-file']
        self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
        self._RRDPaths = self._getRRDPaths()
        # call to update_rrd with a poll interval
        timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
        self._rrdpoller = Poller(self._updateRRD, timeout)

    def stop(self, component):
        if self._rrdpoller:
            self._rrdpoller.stop()

    def _hasImport(self):
        """Check rrdtool availability"""
        if not rrdtool:
            m = messages.Warning(T_(N_(
                "Cannot import module '%s'.\n"), 'rrdtool'),
                                    mid='rrdtool-import-error')
            m.add(T_(N_(
                "The RRD plug for this component is disabled.")))
            self._component.addMessage(m)
            return False

        return True

    def _updateRRD(self):
        """Update data in RRD file"""
        for path in self._RRDPaths:
            value = None
            if path == self._clientsPath:
                value = self._component.getClients()
            elif path == self._bytesPath:
                value = self._component.getBytesSent()

            try:
                rrdtool.update(path, 'N:%i' % value)
                self.debug('RRD file [%s] updated with value: %r',
                    path, value)
            except rrdtool.error, e:
                # We could get an error from rrdtool on converting the
                # value to a double or from not finding the file
                self.warning('RRD error: %r',
                             log.getExceptionMessage(e))
示例#7
0
    def start(self, component):
        self._rrdpoller = None

        self._component = component

        if not self._hasImport():
            return

        properties = self.args['properties']
        self._clientsPath = properties['clients-connected-file']
        self._bytesPath = properties['bytes-transferred-file']
        self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
        self._RRDPaths = self._getRRDPaths()
        # call to update_rrd with a poll interval
        timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
        self._rrdpoller = Poller(self._updateRRD, timeout)
示例#8
0
    def __init__(self, pad, name, setActive, setInactive,
                 reconnectEater, *args):
        PadMonitor.__init__(self, pad, name, setActive, setInactive)

        self._reconnectPoller = Poller(lambda: reconnectEater(*args),
                                       self.PAD_MONITOR_CHECK_INTERVAL,
                                       start=False)
示例#9
0
文件: plug.py 项目: zaheerm/Flumotion
    def init(self):
        self.medium = None
        self.enabled = True
        self._idCounter = 0
        self._idFormat = time.strftime("%Y%m%d%H%M%S-%%d")
        self._keycards = {}  # keycard id -> Keycard

        self._expirer = Poller(self._expire, self.KEYCARD_EXPIRE_INTERVAL, start=False)
示例#10
0
    def init(self):
        """
        A subclass should do as little as possible in its init method.
        In particular, it should not try to access resources.

        Failures during init are marshalled back to the manager through
        the worker's remote_create method, since there is no component state
        proxied to the manager yet at the time of init.
        """
        self.state = planet.WorkerJobState()

        self.name = self.config['name']

        self.state.set('pid', os.getpid())
        self.setMood(moods.waking)

        self.medium = None  # the medium connecting us to the manager's avatar

        self.uiState = componentui.WorkerComponentUIState()
        self.uiState.addKey('cpu-percent')
        self.uiState.addKey('start-time')
        self.uiState.addKey('current-time')
        self.uiState.addKey('virtual-size')
        self.uiState.addKey('total-memory')
        self.uiState.addKey('num-cpus')
        self.uiState.addKey('flu-debug')
        self.uiState.addKey('properties')

        self.uiState.addHook(self)

        self.plugs = {}

        self._happyWaits = []

        # Start the cpu-usage updating.
        self._lastTime = time.time()
        self._lastClock = time.clock()
        self._cpuPoller = Poller(self._pollCPU, 5, start=False)
        self._memoryPoller = Poller(self._pollMemory, 60, start=False)
        self._cpuPollerDC = None
        self._memoryPollerDC = None
        self._shutdownHook = None
示例#11
0
    def __init__(self, pad, name, setActive, setInactive):
        self._last_data_time = -1
        self._pad = pad
        self.name = name
        self._active = False
        self._first = True
        self._running = True

        self._doSetActive = []
        self._doSetInactive = []
        self.addWatch(setActive, setInactive)

        # This dict sillyness is because python's dict operations are atomic
        # w.r.t. the GIL.
        self._probe_id = {}

        self.check_poller = Poller(self._check_timeout,
                                   self.PAD_MONITOR_PROBE_FREQUENCY,
                                   immediately=True)

        self.watch_poller = Poller(self._watch_timeout,
                                   self.PAD_MONITOR_TIMEOUT)
示例#12
0
文件: rrd.py 项目: flyapen/UgFlu
    def start(self, component):
        self._rrdpoller = None

        self._component = component

        if not self._hasImport():
            return

        properties = self.args['properties']
        self._clientsPath = properties['clients-connected-file']
        self._bytesPath = properties['bytes-transferred-file']
        self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
        self._RRDPaths = self._getRRDPaths()
        # call to update_rrd with a poll interval
        timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
        self._rrdpoller = Poller(self._updateRRD, timeout)
示例#13
0
class EaterPadMonitor(PadMonitor):

    def __init__(self, pad, name, setActive, setInactive,
                 reconnectEater, *args):
        PadMonitor.__init__(self, pad, name, setActive, setInactive)

        self._reconnectPoller = Poller(lambda: reconnectEater(*args),
                                       self.PAD_MONITOR_CHECK_INTERVAL,
                                       start=False)

    def setInactive(self):
        PadMonitor.setInactive(self)

        # It might be that we got detached while calling
        # PadMonitor.setInactive() For example someone might have
        # stopped the component as it went hungry, which would happen
        # inside the PadMonitor.setInactive() call. The component
        # would then detach us and the reconnect poller would get
        # stopped. If that happened don't bother restarting it, as it
        # will result in the reactor ending up in an unclean state.
        #
        # A prominent example of such situation is
        # flumotion.test.test_component_disker, where the component
        # gets stopped right after it goes hungry
        if self._running:
            # If an eater received a buffer before being marked as
            # disconnected, and still within the buffer check
            # interval, the next eaterCheck call could accidentally
            # think the eater was reconnected properly.  Setting this
            # to 0 here avoids that happening in eaterCheck.
            self._last_data_time = 0

            self.debug('starting the reconnect poller')
            self._reconnectPoller.start(immediately=True)

    def setActive(self):
        PadMonitor.setActive(self)
        self.debug('stopping the reconnect poller')
        self._reconnectPoller.stop()

    def detach(self):
        PadMonitor.detach(self)
        self.debug('stopping the reconnect poller')
        self._reconnectPoller.stop()
示例#14
0
class EaterPadMonitor(PadMonitor):
    def __init__(self, pad, name, setActive, setInactive, reconnectEater,
                 *args):
        PadMonitor.__init__(self, pad, name, setActive, setInactive)

        self._reconnectPoller = Poller(lambda: reconnectEater(*args),
                                       self.PAD_MONITOR_CHECK_INTERVAL,
                                       start=False)

    def setInactive(self):
        PadMonitor.setInactive(self)

        # It might be that we got detached while calling
        # PadMonitor.setInactive() For example someone might have
        # stopped the component as it went hungry, which would happen
        # inside the PadMonitor.setInactive() call. The component
        # would then detach us and the reconnect poller would get
        # stopped. If that happened don't bother restarting it, as it
        # will result in the reactor ending up in an unclean state.
        #
        # A prominent example of such situation is
        # flumotion.test.test_component_disker, where the component
        # gets stopped right after it goes hungry
        if self._running:
            # If an eater received a buffer before being marked as
            # disconnected, and still within the buffer check
            # interval, the next eaterCheck call could accidentally
            # think the eater was reconnected properly.  Setting this
            # to 0 here avoids that happening in eaterCheck.
            self._last_data_time = 0

            self.debug('starting the reconnect poller')
            self._reconnectPoller.start(immediately=True)

    def setActive(self):
        PadMonitor.setActive(self)
        self.debug('stopping the reconnect poller')
        self._reconnectPoller.stop()

    def detach(self):
        PadMonitor.detach(self)
        self.debug('stopping the reconnect poller')
        self._reconnectPoller.stop()
示例#15
0
class BaseComponent(common.InitMixin, log.Loggable):
    """
    I am the base class for all Flumotion components.

    @ivar name:    the name of the component
    @type name:    string
    @ivar medium:  the component's medium
    @type medium:  L{BaseComponentMedium}
    @ivar uiState: state of the component to be shown in a UI.
                   Contains at least the following keys.
                    - cpu-percent:  percentage of CPU use in last interval
                    - start-time:   time when component was started, in epoch
                                    seconds
                    - current-time: current time in epoch seconds, as seen on
                                    component's machine, which might be out of
                                    sync
                    - virtual-size: virtual memory size in bytes
                   Subclasses can add additional keys for their respective UI.
    @type uiState: L{componentui.WorkerComponentUIState}

    @cvar componentMediumClass: the medium class to use for this component
    @type componentMediumClass: child class of L{BaseComponentMedium}
    """

    logCategory = 'basecomp'
    componentMediumClass = BaseComponentMedium

    implements(IStateCacheableListener)

    def __init__(self, config, haveError=None):
        """
        Subclasses should not override __init__ at all.

        Instead, they should implement init(), which will be called
        by this implementation automatically.

        L{flumotion.common.common.InitMixin} for more details.
        """
        self.debug("initializing %r with config %r", type(self), config)
        self.config = config
        self._haveError = haveError

        # this will call self.init() for all implementors of init()
        common.InitMixin.__init__(self)

        self.setup()

    # BaseComponent interface for subclasses related to component protocol

    def init(self):
        """
        A subclass should do as little as possible in its init method.
        In particular, it should not try to access resources.

        Failures during init are marshalled back to the manager through
        the worker's remote_create method, since there is no component state
        proxied to the manager yet at the time of init.
        """
        self.state = planet.WorkerJobState()

        self.name = self.config['name']

        self.state.set('pid', os.getpid())
        self.setMood(moods.waking)

        self.medium = None  # the medium connecting us to the manager's avatar

        self.uiState = componentui.WorkerComponentUIState()
        self.uiState.addKey('cpu-percent')
        self.uiState.addKey('start-time')
        self.uiState.addKey('current-time')
        self.uiState.addKey('virtual-size')
        self.uiState.addKey('total-memory')
        self.uiState.addKey('num-cpus')
        self.uiState.addKey('flu-debug')
        self.uiState.addKey('properties')

        self.uiState.addHook(self)

        self.plugs = {}

        self._happyWaits = []

        # Start the cpu-usage updating.
        self._lastTime = time.time()
        self._lastClock = time.clock()
        self._cpuPoller = Poller(self._pollCPU, 5, start=False)
        self._memoryPoller = Poller(self._pollMemory, 60, start=False)
        self._cpuPollerDC = None
        self._memoryPollerDC = None
        self._shutdownHook = None

    ### IStateCacheable Interface

    def observerAppend(self, observer, num):
        """
        Triggered when a uiState observer was added.

        Default implementation is to start the memory and cpu pollers.

        Note:
        Subclasses can override me but should chain me up to start these
        pollers
        """
        self.debug("observer has started watching us, starting pollers")
        if not self._cpuPoller.running and not self._cpuPollerDC:
            self._cpuPollerDC = reactor.callLater(0,
                                                  self._cpuPoller.start,
                                                  immediately=True)
        if not self._memoryPoller.running and not self._memoryPollerDC:
            self._memoryPollerDC = reactor.callLater(0,
                                                     self._memoryPoller.start,
                                                     immediately=True)

    def observerRemove(self, observer, num):
        """
        Triggered when a uiState observer has left.

        Default implementation is to stop the memory and cpu pollers
        when the total number of observers denoted by the 'num'
        argument becomes zero.

        Note:
        Subclasses can override me but should chain me up to stop these
        pollers
        """
        if num == 0:
            self.debug("no more observers left, shutting down pollers")
            # Cancel any pending callLaters
            if self._cpuPollerDC:
                self._cpuPollerDC.cancel()
                self._cpuPollerDC = None
            if self._memoryPollerDC:
                self._memoryPollerDC.cancel()
                self._memoryPollerDC = None

            if self._cpuPoller:
                self._cpuPoller.stop()
            if self._memoryPoller:
                self._memoryPoller.stop()

    def do_check(self):
        """
        Subclasses can implement me to run any checks before the component
        performs setup.

        Messages can be added to the component state's 'messages' list key.
        Any error messages added will trigger the component going to sad,
        with L{flumotion.common.errors.ComponentSetupError} being raised
        before getting to setup stage; do_setup() will not be called.

        In the event of a fatal problem that can't be expressed through an
        error message, this method should raise an exception or return a
        failure.

        It is not necessary to chain up in this function. The return
        value may be a deferred.
        """
        return defer.maybeDeferred(self.check_properties,
                                   self.config['properties'], self.addMessage)

    def check_properties(self, properties, addMessage):
        """
        BaseComponent convenience vmethod for running checks.

        A component implementation can override this method to run any
        checks that it needs to. Typically, a check_properties
        implementation will call the provided addMessage() callback to
        note warnings or errors. For errors, addMessage() will set
        component's mood to sad, which will abort the init process
        before getting to do_setup().

        @param properties: The component's properties
        @type properties: dict of string => object
        @param addMessage: Thunk to add a message to the component
                           state. Will raise an exception if the
                           message is of level ERROR.
        @type addMessage: L{flumotion.common.messages.Message} -> None
        """
        pass

    def do_setup(self):
        """
        Subclasses can implement me to set up the component before it is
        started.  It should set up the component, possibly opening files
        and resources.
        Non-programming errors should not be raised, but returned as a
        failing deferred.

        The return value may be a deferred.
        """
        plug_starts = []
        for socket, plugs in self.config['plugs'].items():
            self.plugs[socket] = []
            for plug in plugs:
                entry = plug['entries']['default']
                instance = reflectcall.reflectCall(entry['module-name'],
                                                   entry['function-name'],
                                                   plug)
                self.plugs[socket].append(instance)
                self.debug('Starting plug %r on socket %s', instance, socket)
                plug_starts.append(instance.start)

        # Call check methods, starting from the base class and working down to
        # subclasses.
        checks = common.get_all_methods(self, 'do_check', False)

        def checkErrorCallback(result):
            # if the mood is now sad, it means an error was encountered
            # during check, and we should return a failure here.
            # since the checks are responsible for adding a message,
            # this is a handled error.
            current = self.state.get('mood')
            if current == moods.sad.value:
                self.warning('Running checks made the component sad.')
                raise errors.ComponentSetupHandledError()

        checks.append(checkErrorCallback)

        return _maybeDeferredChain(plug_starts + checks, self)

    def do_stop(self):
        """
        BaseComponent vmethod for stopping.
        The component should do any cleanup it needs, but must not set the
        component's mood to sleeping.

        @Returns: L{twisted.internet.defer.Deferred}
        """
        plug_stops = []
        for socket, plugs in self.plugs.items():
            for plug in plugs:
                self.debug('Stopping plug %r on socket %s', plug, socket)
                plug_stops.append(plug.stop)

        for message in self.state.get('messages'):
            # FIXME: not necessary
            self.state.remove('messages', message)

        # Cancel any pending callLaters
        if self._cpuPollerDC:
            self._cpuPollerDC.cancel()
            self._cpuPollerDC = None
        if self._memoryPollerDC:
            self._memoryPollerDC.cancel()
            self._memoryPollerDC = None

        if self._cpuPoller:
            self._cpuPoller.stop()
            self._cpuPoller = None
        if self._memoryPoller:
            self._memoryPoller.stop()
            self._memoryPoller = None

        if self._shutdownHook:
            self.debug('_stoppedCallback: firing shutdown hook')
            self._shutdownHook()

        return _maybeDeferredChain(plug_stops, self)

    ### BaseComponent implementation related to compoment protocol

    def setup(self):
        """
        Sets up the component.  Called during __init__, so be sure not
        to raise exceptions, instead adding messages to the component
        state.
        """
        def run_setups():
            setups = common.get_all_methods(self, 'do_setup', False)
            return _maybeDeferredChain(setups, self)

        def setup_complete(_):
            self.debug('setup completed')
            self.setup_completed()

        def got_error(failure):
            txt = log.getFailureMessage(failure)
            self.debug('got_error: %s', txt)
            if not failure.check(errors.ComponentSetupHandledError):
                self.warning('Setup failed: %s', txt)
                m = messages.Error(T_(N_("Could not setup component.")),
                                   debug=txt,
                                   mid="component-setup-%s" % self.name)
                # will call setMood(moods.sad)
                self.addMessage(m)

            # swallow
            return None

        self.setMood(moods.waking)
        self.uiState.set('start-time', time.time())

        self.uiState.set('total-memory', self._getTotalMemory())
        self.uiState.set('num-cpus', self._getNumberOfCPUs())
        self.uiState.set('flu-debug', log.getDebug())

        d = run_setups()
        d.addCallbacks(setup_complete, got_error)
        # all status info via messages and the mood

    def setup_completed(self):
        self.debug('turning happy')
        self.setMood(moods.happy)

    def setShutdownHook(self, shutdownHook):
        """
        Set the shutdown hook for this component (replacing any previous hook).
        When a component is stopped, then this hook will be fired.
        """
        self._shutdownHook = shutdownHook

    def stop(self):
        """
        Tell the component to stop.
        The connection to the manager will be closed.
        The job process will also finish.
        """
        self.debug('BaseComponent.stop')

        # Set ourselves to waking while we're shutting down.
        self.setMood(moods.waking)

        # Run stop methods, starting from the subclass, up to this base class.
        stops = common.get_all_methods(self, 'do_stop', True)
        return _maybeDeferredChain(stops, self)

    ### BaseComponent public methods

    def getName(self):
        return self.name

    def setWorkerName(self, workerName):
        self.state.set('workerName', workerName)

    def getWorkerName(self):
        return self.state.get('workerName')

    def setMedium(self, medium):
        assert isinstance(medium, BaseComponentMedium)
        self.medium = medium
        self.medium.logName = self.getName()
        for plugs in self.plugs.values():
            for plug in plugs:
                self._export_plug_interface(plug, medium)

    def setMood(self, mood):
        """
        Set the given mood on the component if it's different from the current
        one.
        """
        current = self.state.get('mood')

        if current == mood.value:
            self.log('already in mood %r' % mood)
            return
        elif current == moods.sad.value:
            self.info('tried to set mood to %r, but already sad :-(' % mood)
            return

        self.doLog(log.DEBUG, -2, 'MOOD changed to %r by caller', mood)
        self.state.set('mood', mood.value)

        if mood == moods.happy:
            while self._happyWaits:
                self._happyWaits.pop(0).callback(None)
        elif mood == moods.sad:
            while self._happyWaits:
                self._happyWaits.pop(0).errback(errors.ComponentStartError())

    def getMood(self):
        """
        Gets the mood on the component.

        @rtype: int
        """
        return self.state.get('mood')

    def waitForHappy(self):
        mood = self.getMood()
        if mood == moods.happy.value:
            return defer.succeed(None)
        elif mood == moods.sad.value:
            return defer.fail(errors.ComponentStartError())
        else:
            d = defer.Deferred()
            self._happyWaits.append(d)
            return d

    def addMessage(self, message):
        """
        Add a message to the component.
        If any of the messages is an error, the component will turn sad.

        @type  message: L{flumotion.common.messages.Message}
        """
        self.removeMessage(message.id)
        self.state.append('messages', message)
        if message.level == messages.ERROR:
            self.debug('error message, turning sad')
            self.setMood(moods.sad)
            if self._haveError:
                self._haveError(message)

    def removeMessage(self, mid):
        """
        Remove a message with a given id from the component.
        @type  mid: str
        """
        for msg in self.state.get('messages', []):
            if msg.id == mid:
                self.state.remove('messages', msg)

    def warnDeprecatedProperties(self, list):
        """
        Add a warning messages for deprecated properties.

        @param list:       list of property names.
        @type  list:       list of str
        """
        msg = ("Your configuration uses deprecated properties.  "
               "Please update your configuration and correct them.\n")
        m = messages.Warning(T_(N_(msg)), mid="deprecated")
        for prop in list:
            m.add(T_(N_("Please remove '%s' property.\n"), prop))
        self.addMessage(m)
        self.warning(msg)

    def fixRenamedProperties(self, properties, list):
        """
        Fix properties that have been renamed from a previous version,
        and add a warning for them.

        @param properties: properties; will be modified as a result.
        @type  properties: dict
        @param list:       list of (old, new) tuples of property names.
        @type  list:       list of tuple of (str, str)
        """
        found = []
        for old, new in list:
            if old in properties:
                found.append((old, new))

        if found:
            m = messages.Warning(T_(
                N_("Your configuration uses deprecated properties.  "
                   "Please update your configuration and correct them.\n")),
                                 mid="deprecated")
            for old, new in found:
                m.add(T_(N_("Please rename '%s' to '%s'.\n"), old, new))
                self.debug("Setting new property '%s' to %r", new,
                           properties[old])
                properties[new] = properties[old]
                del properties[old]
            self.addMessage(m)

    def adminCallRemote(self, methodName, *args, **kwargs):
        """
        Call a remote method on all admin client views on this component.

        This gets serialized through the manager and multiplexed to all
        admin clients, and from there on to all views connected to each
        admin client model.

        Because there can be any number of admin clients that this call
        will go out do, it does not make sense to have one return value.
        This function will return None always.
        """
        if self.medium:
            self.medium.callRemote("adminCallRemote", methodName, *args,
                                   **kwargs)
        else:
            self.debug('asked to adminCallRemote(%s, *%r, **%r), but '
                       'no manager.' % (methodName, args, kwargs))

    def modifyProperty(self, property_name, value):
        """
        Modifies a property of the compoment.

        Components with modifiable properties (properties that can be changed
        on the fly) should implement modify_property_(propertyName) to receive
        the call

        @param property_name: Name of the property to change
        @type  property_name: str
        @param value: Value to set
        """
        # Transform property name to camel case:
        # max-reconnections-delay -> MaxReconnectionsDelay
        p = ''.join([t.title() for t in property_name.split('-')])
        method_name = "modify_property_%s" % p
        if not hasattr(self, method_name):
            raise errors.PropertyNotModifiableError("%s" % (property_name))
        method = getattr(self, method_name)
        if not method(value):
            return False
        self.config['properties'][property_name] = value
        self.uiState.set('properties', self.config['properties'])
        return True

    def checkPropertyType(self, property_name, value, allowed_type):
        """
        Check that the value to be set in a property is of the correct type

        @returns: True if the value is of the correct type
        """
        if type(value) != allowed_type:
            self.warning(
                "Could not set the property %s in %s. "
                "'value' must be of %s", property_name, self, allowed_type)
            return False
        return True

    def _export_plug_interface(self, plug, medium):
        try:
            namespace = plug.get_namespace()
        except AttributeError:
            self.debug(
                "Plug %r does not provide namespace, "
                "its interface will not be exposed", plug)
            return

        self.debug("Exposing plug's %r interface in namespace %r", plug,
                   namespace)
        for method in filter(
                callable,
            [getattr(plug, m) for m in dir(plug) if m.startswith('remote_')]):
            if namespace:
                name = "".join(("remote_", namespace, "_",
                                method.__name__[len("remote_"):]))
            else:
                name = method.__name__
            self.debug("Exposing method %r as %r in %r", method, name, medium)
            setattr(medium, name, method)

    def _pollCPU(self):
        self._cpuPollerDC = None
        # update CPU time stats
        nowTime = time.time()
        nowClock = time.clock()
        deltaTime = nowTime - self._lastTime
        deltaClock = nowClock - self._lastClock
        self._lastTime = nowTime
        self._lastClock = nowClock
        # deltaClock can be < 0 if time.clock() wrapped around
        if deltaClock >= 0:
            CPU = deltaClock / deltaTime
            self.log('latest CPU use: %r', CPU)
            self.uiState.set('cpu-percent', CPU)

        self.uiState.set('current-time', nowTime)

    def _pollMemory(self):
        self._memoryPollerDC = None
        # Figure out our virtual memory size and report that.
        # I don't know a nicer way to find vsize than groping /proc/
        handle = open('/proc/%d/stat' % os.getpid())
        line = handle.read()
        handle.close()
        fields = line.split()
        # field 1 (comm) could potentially contain spaces and thus split over
        # multiple list items, but our processes do not contain spaces
        vsize = int(fields[22])
        self.log('vsize is %d', vsize)
        self.uiState.set('virtual-size', vsize)

    def _getTotalMemory(self):
        f = open("/proc/meminfo")
        memtotal = f.readline()
        f.close()
        return int(memtotal[memtotal.index(":") + 1:-3]) * 1024

    def _getNumberOfCPUs(self):
        try:
            return open('/proc/cpuinfo').read().count('processor\t:')
        except IOError:
            self.debug('Can not determine number of CPUs on this system')
            return 1
示例#16
0
class PadMonitor(log.Loggable):
    """
    I monitor data flow on a GStreamer pad.
    I regularly schedule a buffer probe call at PAD_MONITOR_PROBE_INTERVAL.
    I regularly schedule a check call at PAD_MONITOR_CHECK_INTERVAL
    that makes sure a buffer probe was triggered since the last check call.
    """

    PAD_MONITOR_PROBE_INTERVAL = 5.0
    PAD_MONITOR_CHECK_INTERVAL = PAD_MONITOR_PROBE_INTERVAL * 2.5

    def __init__(self, pad, name, setActive, setInactive):
        """
        @type  pad:         L{gst.Pad}
        @type  name:        str
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._last_data_time = -1  # system time in epoch secs of last reception
        self._pad = pad
        self.name = name
        self._active = False
        self._first = True
        self._running = True

        self._doSetActive = []
        self._doSetInactive = []
        self.addWatch(setActive, setInactive)

        # This dict sillyness is because python's dict operations are atomic
        # w.r.t. the GIL.
        self._probe_id = {}

        self.check_poller = Poller(self._probe_timeout,
                                   self.PAD_MONITOR_PROBE_INTERVAL,
                                   immediately=True)

        self.watch_poller = Poller(self._check_timeout,
                                   self.PAD_MONITOR_CHECK_INTERVAL)

    def logMessage(self, message, *args):
        if self._first:
            self.debug(message, *args)
        else:
            self.log(message, *args)

    def isActive(self):
        return self._active

    def detach(self):
        self.check_poller.stop()
        self.watch_poller.stop()
        self._running = False

        # implementation closely tied to _probe_timeout wrt to GIL
        # tricks, threadsafety, and getting the probe deferred to
        # actually return
        d, probe_id = self._probe_id.pop("id", (None, None))
        if probe_id:
            self._pad.remove_buffer_probe(probe_id)
            d.callback(None)

    def _probe_timeout(self):
        # called every so often to install a probe callback

        def probe_cb(pad, buffer):
            """
            Periodically scheduled buffer probe, that ensures that we're
            currently actually having dataflow through our eater
            elements.

            Called from GStreamer threads.

            @param pad:       The gst.Pad srcpad for one eater in this
                              component.
            @param buffer:    A gst.Buffer that has arrived on this pad
            """
            self._last_data_time = time.time()

            self.logMessage('buffer probe on %s has timestamp %s', self.name,
                            gst.TIME_ARGS(buffer.timestamp))

            deferred, probe_id = self._probe_id.pop("id", (None, None))
            if probe_id:
                # This will be None only if detach() has been called.
                self._pad.remove_buffer_probe(probe_id)

                reactor.callFromThread(deferred.callback, None)
                # Data received! Return to happy ASAP:
                reactor.callFromThread(self.watch_poller.run)

            self._first = False

            # let the buffer through
            return True

        d = defer.Deferred()
        # FIXME: this is racy: evaluate RHS, drop GIL, buffer probe
        # fires before __setitem__ in LHS; need a mutex
        self._probe_id['id'] = (d, self._pad.add_buffer_probe(probe_cb))
        return d

    def _check_timeout(self):
        # called every so often to check that a probe callback was triggered
        self.log('last buffer for %s at %r', self.name, self._last_data_time)

        now = time.time()

        if self._last_data_time < 0:
            # We never received any data in the first timeout period...
            self._last_data_time = 0
            self.setInactive()
        elif self._last_data_time == 0:
            # still no data...
            pass
        else:
            # We received data at some time in the past.
            delta = now - self._last_data_time

            if self._active and delta > self.PAD_MONITOR_CHECK_INTERVAL:
                self.info(
                    "No data received on pad %s for > %r seconds, "
                    "setting to hungry", self.name,
                    self.PAD_MONITOR_CHECK_INTERVAL)
                self.setInactive()
            elif not self._active and delta < self.PAD_MONITOR_CHECK_INTERVAL:
                self.info("Receiving data again on pad %s, flow active",
                          self.name)
                self.setActive()

    def addWatch(self, setActive, setInactive):
        """
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._doSetActive.append(setActive)
        self._doSetInactive.append(setInactive)

    def setInactive(self):
        self._active = False
        for setInactive in self._doSetInactive:
            setInactive(self.name)

    def setActive(self):
        self._active = True
        for setActive in self._doSetActive:
            setActive(self.name)
示例#17
0
文件: plug.py 项目: zaheerm/Flumotion
class BouncerPlug(pbase.ComponentPlug, common.InitMixin):
    """
    I am the base class for all bouncer plugs.

    FIXME: expireKeycardIds has been added to the component bouncer.
           Because the plug version is not yet used, and will be
           refactored/redesigned in a near future, the modifications
           have not been duplicated here.

    @cvar keycardClasses: tuple of all classes of keycards this bouncer can
                          authenticate, in order of preference
    @type keycardClasses: tuple of L{flumotion.common.keycards.Keycard}
                          class objects
    """

    keycardClasses = ()
    logCategory = "bouncer"

    KEYCARD_EXPIRE_INTERVAL = 2 * 60

    def __init__(self, *args, **kwargs):
        pbase.ComponentPlug.__init__(self, *args, **kwargs)
        common.InitMixin.__init__(self)

    def init(self):
        self.medium = None
        self.enabled = True
        self._idCounter = 0
        self._idFormat = time.strftime("%Y%m%d%H%M%S-%%d")
        self._keycards = {}  # keycard id -> Keycard

        self._expirer = Poller(self._expire, self.KEYCARD_EXPIRE_INTERVAL, start=False)

    def typeAllowed(self, keycard):
        """
        Verify if the keycard is an instance of a Keycard class specified
        in the bouncer's keycardClasses variable.
        """
        return isinstance(keycard, self.keycardClasses)

    def setEnabled(self, enabled):
        if not enabled and self.enabled:
            # If we were enabled and are being set to disabled, eject the warp
            # core^w^w^w^wexpire all existing keycards
            self.expireAllKeycards()
            self._expirer.stop()

        self.enabled = enabled

    def setMedium(self, medium):
        self.medium = medium

    def stop(self, component):
        self.setEnabled(False)

    def _expire(self):
        for k in self._keycards.values():
            if hasattr(k, "ttl"):
                k.ttl -= self._expirer.timeout
                if k.ttl <= 0:
                    self.expireKeycardId(k.id)

    def authenticate(self, keycard):
        if not self.typeAllowed(keycard):
            self.warning("keycard %r is not an allowed keycard class", keycard)
            return None

        if self.enabled:
            if not self._expirer.running and hasattr(keycard, "ttl"):
                self.debug("installing keycard timeout poller")
                self._expirer.start()
            return defer.maybeDeferred(self.do_authenticate, keycard)
        else:
            self.debug("Bouncer disabled, refusing authentication")
            return None

    def do_authenticate(self, keycard):
        """
        Must be overridden by subclasses.

        Authenticate the given keycard.
        Return the keycard with state AUTHENTICATED to authenticate,
        with state REQUESTING to continue the authentication process,
        or None to deny the keycard, or a deferred which should have the same
        eventual value.
        """
        raise NotImplementedError("authenticate not overridden")

    def hasKeycard(self, keycard):
        return keycard in self._keycards.values()

    def generateKeycardId(self):
        keycardId = self._idFormat % self._idCounter
        self._idCounter += 1
        return keycardId

    def addKeycard(self, keycard):
        """
        Adds a keycard to the bouncer.
        Can be called with the same keycard more than one time.
        If the keycard has already been added successfully,
        adding it again will succeed and return True.

        @param keycard: the keycard to add.
        @return: if the bouncer accepts the keycard.
        """
        # give keycard an id and store it in our hash
        if keycard.id in self._keycards:
            # already in there
            return True

        keycard.id = self.generateKeycardId()

        if hasattr(keycard, "ttl") and keycard.ttl <= 0:
            self.log("immediately expiring keycard %r", keycard)
            return False

        self._keycards[keycard.id] = keycard

        self.debug("added keycard with id %s" % keycard.id)
        return True

    def removeKeycard(self, keycard):
        if not keycard.id in self._keycards:
            raise KeyError

        del self._keycards[keycard.id]

        self.debug("removed keycard with id %s" % keycard.id)

    def removeKeycardId(self, keycardId):
        self.debug("removing keycard with id %s" % keycardId)
        if not keycardId in self._keycards:
            raise KeyError

        keycard = self._keycards[keycardId]
        self.removeKeycard(keycard)

    def keepAlive(self, issuerName, ttl):
        for k in self._keycards.itervalues():
            if hasattr(k, "issuerName") and k.issuerName == issuerName:
                k.ttl = ttl

    def expireAllKeycards(self):
        return defer.DeferredList([self.expireKeycardId(keycardId) for keycardId in self._keycards.keys()])

    def expireKeycardId(self, keycardId):
        self.log("expiring keycard with id %r", keycardId)
        if not keycardId in self._keycards:
            raise KeyError

        keycard = self._keycards.pop(keycardId)

        return self.medium.callRemote("expireKeycard", keycard.requesterId, keycard.id)
示例#18
0
class BaseComponent(common.InitMixin, log.Loggable):
    """
    I am the base class for all Flumotion components.

    @ivar name:    the name of the component
    @type name:    string
    @ivar medium:  the component's medium
    @type medium:  L{BaseComponentMedium}
    @ivar uiState: state of the component to be shown in a UI.
                   Contains at least the following keys.
                    - cpu-percent:  percentage of CPU use in last interval
                    - start-time:   time when component was started, in epoch
                                    seconds
                    - current-time: current time in epoch seconds, as seen on
                                    component's machine, which might be out of
                                    sync
                    - virtual-size: virtual memory size in bytes
                   Subclasses can add additional keys for their respective UI.
    @type uiState: L{componentui.WorkerComponentUIState}

    @cvar componentMediumClass: the medium class to use for this component
    @type componentMediumClass: child class of L{BaseComponentMedium}
    """

    logCategory = 'basecomp'
    componentMediumClass = BaseComponentMedium

    implements(IStateCacheableListener)

    def __init__(self, config, haveError=None):
        """
        Subclasses should not override __init__ at all.

        Instead, they should implement init(), which will be called
        by this implementation automatically.

        L{flumotion.common.common.InitMixin} for more details.
        """
        self.debug("initializing %r with config %r", type(self), config)
        self.config = config
        self._haveError = haveError

        # this will call self.init() for all implementors of init()
        common.InitMixin.__init__(self)

        self.setup()

    # BaseComponent interface for subclasses related to component protocol

    def init(self):
        """
        A subclass should do as little as possible in its init method.
        In particular, it should not try to access resources.

        Failures during init are marshalled back to the manager through
        the worker's remote_create method, since there is no component state
        proxied to the manager yet at the time of init.
        """
        self.state = planet.WorkerJobState()

        self.name = self.config['name']

        self.state.set('pid', os.getpid())
        self.setMood(moods.waking)

        self.medium = None # the medium connecting us to the manager's avatar

        self.uiState = componentui.WorkerComponentUIState()
        self.uiState.addKey('cpu-percent')
        self.uiState.addKey('start-time')
        self.uiState.addKey('current-time')
        self.uiState.addKey('virtual-size')
        self.uiState.addKey('total-memory')

        self.uiState.addHook(self)

        self.plugs = {}

        self._happyWaits = []

        # Start the cpu-usage updating.
        self._lastTime = time.time()
        self._lastClock = time.clock()
        self._cpuPoller = Poller(self._pollCPU, 5, start=False)
        self._memoryPoller = Poller(self._pollMemory, 60, start=False)
        self._cpuPollerDC = None
        self._memoryPollerDC = None
        self._shutdownHook = None

    ### IStateCacheable Interface

    def observerAppend(self, observer, num):
        """
        Triggered when a uiState observer was added.

        Default implementation is to start the memory and cpu pollers.

        Note:
        Subclasses can override me but should chain me up to start these
        pollers
        """
        self.debug("observer has started watching us, starting pollers")
        if not self._cpuPoller.running and not self._cpuPollerDC:
            self._cpuPollerDC = reactor.callLater(0,
                                                  self._cpuPoller.start,
                                                  immediately=True)
        if not self._memoryPoller.running and not self._memoryPollerDC:
            self._memoryPollerDC = reactor.callLater(0,
                                                     self._memoryPoller.start,
                                                     immediately=True)

    def observerRemove(self, observer, num):
        """
        Triggered when a uiState observer has left.

        Default implementation is to stop the memory and cpu pollers
        when the total number of observers denoted by the 'num'
        argument becomes zero.

        Note:
        Subclasses can override me but should chain me up to stop these
        pollers
        """
        if num == 0:
            self.debug("no more observers left, shutting down pollers")
            # Cancel any pending callLaters
            if self._cpuPollerDC:
                self._cpuPollerDC.cancel()
                self._cpuPollerDC = None
            if self._memoryPollerDC:
                self._memoryPollerDC.cancel()
                self._memoryPollerDC = None

            if self._cpuPoller:
                self._cpuPoller.stop()
            if self._memoryPoller:
                self._memoryPoller.stop()

    def do_check(self):
        """
        Subclasses can implement me to run any checks before the component
        performs setup.

        Messages can be added to the component state's 'messages' list key.
        Any error messages added will trigger the component going to sad,
        with L{flumotion.common.errors.ComponentSetupError} being raised
        before getting to setup stage; do_setup() will not be called.

        In the event of a fatal problem that can't be expressed through an
        error message, this method should raise an exception or return a
        failure.

        It is not necessary to chain up in this function. The return
        value may be a deferred.
        """
        return defer.maybeDeferred(self.check_properties,
                                   self.config['properties'],
                                   self.addMessage)

    def check_properties(self, properties, addMessage):
        """
        BaseComponent convenience vmethod for running checks.

        A component implementation can override this method to run any
        checks that it needs to. Typically, a check_properties
        implementation will call the provided addMessage() callback to
        note warnings or errors. For errors, addMessage() will set
        component's mood to sad, which will abort the init process
        before getting to do_setup().

        @param properties: The component's properties
        @type properties: dict of string => object
        @param addMessage: Thunk to add a message to the component
                           state. Will raise an exception if the
                           message is of level ERROR.
        @type addMessage: L{flumotion.common.messages.Message} -> None
        """
        pass

    def do_setup(self):
        """
        Subclasses can implement me to set up the component before it is
        started.  It should set up the component, possibly opening files
        and resources.
        Non-programming errors should not be raised, but returned as a
        failing deferred.

        The return value may be a deferred.
        """
        for socket, plugs in self.config['plugs'].items():
            self.plugs[socket] = []
            for plug in plugs:
                entry = plug['entries']['default']
                instance = reflectcall.reflectCall(entry['module-name'],
                                                   entry['function-name'],
                                                   plug)
                self.plugs[socket].append(instance)
                self.debug('Starting plug %r on socket %s',
                           instance, socket)
                instance.start(self)

        # Call check methods, starting from the base class and working down to
        # subclasses.
        checks = common.get_all_methods(self, 'do_check', False)

        def checkErrorCallback(result):
            # if the mood is now sad, it means an error was encountered
            # during check, and we should return a failure here.
            # since the checks are responsible for adding a message,
            # this is a handled error.
            current = self.state.get('mood')
            if current == moods.sad.value:
                self.warning('Running checks made the component sad.')
                raise errors.ComponentSetupHandledError()

        checks.append(checkErrorCallback)
        return _maybeDeferredChain(checks, self)

    def do_stop(self):
        """
        BaseComponent vmethod for stopping.
        The component should do any cleanup it needs, but must not set the
        component's mood to sleeping.

        @Returns: L{twisted.internet.defer.Deferred}
        """
        for socket, plugs in self.plugs.items():
            for plug in plugs:
                self.debug('Stopping plug %r on socket %s', plug, socket)
                plug.stop(self)

        for message in self.state.get('messages'):
            # FIXME: not necessary
            self.state.remove('messages', message)

        # Cancel any pending callLaters
        if self._cpuPollerDC:
            self._cpuPollerDC.cancel()
            self._cpuPollerDC = None
        if self._memoryPollerDC:
            self._memoryPollerDC.cancel()
            self._memoryPollerDC = None

        if self._cpuPoller:
            self._cpuPoller.stop()
            self._cpuPoller = None
        if self._memoryPoller:
            self._memoryPoller.stop()
            self._memoryPoller = None

        if self._shutdownHook:
            self.debug('_stoppedCallback: firing shutdown hook')
            self._shutdownHook()

    ### BaseComponent implementation related to compoment protocol

    def setup(self):
        """
        Sets up the component.  Called during __init__, so be sure not
        to raise exceptions, instead adding messages to the component
        state.
        """

        def run_setups():
            setups = common.get_all_methods(self, 'do_setup', False)
            return _maybeDeferredChain(setups, self)

        def setup_complete(_):
            self.debug('setup completed')
            self.setup_completed()

        def got_error(failure):
            txt = log.getFailureMessage(failure)
            self.debug('got_error: %s', txt)
            if not failure.check(errors.ComponentSetupHandledError):
                self.warning('Setup failed: %s', txt)
                m = messages.Error(T_(N_("Could not setup component.")),
                                   debug=txt,
                                   mid="component-setup-%s" % self.name)
                # will call setMood(moods.sad)
                self.addMessage(m)

            # swallow
            return None

        self.setMood(moods.waking)
        self.uiState.set('start-time', time.time())

        self.uiState.set('total-memory', self._getTotalMemory())
        d = run_setups()
        d.addCallbacks(setup_complete, got_error)
        # all status info via messages and the mood

    def setup_completed(self):
        self.debug('turning happy')
        self.setMood(moods.happy)

    def setShutdownHook(self, shutdownHook):
        """
        Set the shutdown hook for this component (replacing any previous hook).
        When a component is stopped, then this hook will be fired.
        """
        self._shutdownHook = shutdownHook

    def stop(self):
        """
        Tell the component to stop.
        The connection to the manager will be closed.
        The job process will also finish.
        """
        self.debug('BaseComponent.stop')

        # Set ourselves to waking while we're shutting down.
        self.setMood(moods.waking)

        # Run stop methods, starting from the subclass, up to this base class.
        stops = common.get_all_methods(self, 'do_stop', True)
        return _maybeDeferredChain(stops, self)

    ### BaseComponent public methods

    def getName(self):
        return self.name

    def setWorkerName(self, workerName):
        self.state.set('workerName', workerName)

    def getWorkerName(self):
        return self.state.get('workerName')

    def setMedium(self, medium):
        assert isinstance(medium, BaseComponentMedium)
        self.medium = medium
        self.medium.logName = self.getName()

    def setMood(self, mood):
        """
        Set the given mood on the component if it's different from the current
        one.
        """
        current = self.state.get('mood')

        if current == mood.value:
            self.log('already in mood %r' % mood)
            return
        elif current == moods.sad.value:
            self.info('tried to set mood to %r, but already sad :-(' % mood)
            return

        self.doLog(log.DEBUG, -2, 'MOOD changed to %r by caller', mood)
        self.state.set('mood', mood.value)

        if mood == moods.happy:
            while self._happyWaits:
                self._happyWaits.pop(0).callback(None)
        elif mood == moods.sad:
            while self._happyWaits:
                self._happyWaits.pop(0).errback(errors.ComponentStartError())

    def getMood(self):
        """
        Gets the mood on the component.

        @rtype: int
        """
        return self.state.get('mood')

    def waitForHappy(self):
        mood = self.getMood()
        if mood == moods.happy.value:
            return defer.succeed(None)
        elif mood == moods.sad.value:
            return defer.fail(errors.ComponentStartError())
        else:
            d = defer.Deferred()
            self._happyWaits.append(d)
            return d

    def addMessage(self, message):
        """
        Add a message to the component.
        If any of the messages is an error, the component will turn sad.

        @type  message: L{flumotion.common.messages.Message}
        """
        self.state.append('messages', message)
        if message.level == messages.ERROR:
            self.debug('error message, turning sad')
            self.setMood(moods.sad)
            if self._haveError:
                self._haveError(message)

    def fixRenamedProperties(self, properties, list):
        """
        Fix properties that have been renamed from a previous version,
        and add a warning for them.

        @param properties: properties; will be modified as a result.
        @type  properties: dict
        @param list:       list of (old, new) tuples of property names.
        @type  list:       list of tuple of (str, str)
        """
        found = []
        for old, new in list:
            if old in properties:
                found.append((old, new))

        if found:
            m = messages.Warning(T_(N_(
                "Your configuration uses deprecated properties.  "
                "Please update your configuration and correct them.\n")),
                mid="deprecated")
            for old, new in found:
                m.add(T_(N_(
                "Please rename '%s' to '%s'.\n"),
                        old, new))
                self.debug("Setting new property '%s' to %r", new,
                    properties[old])
                properties[new] = properties[old]
                del properties[old]
            self.addMessage(m)

    def adminCallRemote(self, methodName, *args, **kwargs):
        """
        Call a remote method on all admin client views on this component.

        This gets serialized through the manager and multiplexed to all
        admin clients, and from there on to all views connected to each
        admin client model.

        Because there can be any number of admin clients that this call
        will go out do, it does not make sense to have one return value.
        This function will return None always.
        """
        if self.medium:
            self.medium.callRemote("adminCallRemote", methodName,
                                   *args, **kwargs)
        else:
            self.debug('asked to adminCallRemote(%s, *%r, **%r), but '
                       'no manager.'
                       % (methodName, args, kwargs))

    def _pollCPU(self):
        self._cpuPollerDC = None
        # update CPU time stats
        nowTime = time.time()
        nowClock = time.clock()
        deltaTime = nowTime - self._lastTime
        deltaClock = nowClock - self._lastClock
        self._lastTime = nowTime
        self._lastClock = nowClock
        # deltaClock can be < 0 if time.clock() wrapped around
        if deltaClock >= 0:
            CPU = deltaClock/deltaTime
            self.log('latest CPU use: %r', CPU)
            self.uiState.set('cpu-percent', CPU)

        self.uiState.set('current-time', nowTime)

    def _pollMemory(self):
        self._memoryPollerDC = None
        # Figure out our virtual memory size and report that.
        # I don't know a nicer way to find vsize than groping /proc/
        handle = open('/proc/%d/stat' % os.getpid())
        line = handle.read()
        handle.close()
        fields = line.split()
        # field 1 (comm) could potentially contain spaces and thus split over
        # multiple list items, but our processes do not contain spaces
        vsize = int(fields[22])
        self.log('vsize is %d', vsize)
        self.uiState.set('virtual-size', vsize)

    def _getTotalMemory(self):
        f = open("/proc/meminfo")
        memtotal = f.readline()
        f.close()
        return int(memtotal[memtotal.index(":") + 1: -3]) * 1024
示例#19
0
文件: bouncer.py 项目: flyapen/UgFlu
class Bouncer(component.BaseComponent):
    """
    I am the base class for all bouncers.

    @cvar keycardClasses: tuple of all classes of keycards this bouncer can
                          authenticate, in order of preference
    @type keycardClasses: tuple of L{flumotion.common.keycards.Keycard}
                          class objects
    """
    keycardClasses = ()
    componentMediumClass = BouncerMedium
    logCategory = 'bouncer'

    KEYCARD_EXPIRE_INTERVAL = 2 * 60 # expire every 2 minutes

    def init(self):
        self._idCounter = 0
        self._idFormat = time.strftime('%Y%m%d%H%M%S-%%d')
        self._keycards = {} # keycard id -> Keycard

        self._expirer = Poller(self._expire,
                               self.KEYCARD_EXPIRE_INTERVAL,
                               start=False)
        self.enabled = True

    def setDomain(self, name):
        self.domain = name

    def getDomain(self):
        return self.domain

    def typeAllowed(self, keycard):
        """
        Verify if the keycard is an instance of a Keycard class specified
        in the bouncer's keycardClasses variable.
        """
        return isinstance(keycard, self.keycardClasses)

    def setEnabled(self, enabled):

        def callAndPassthru(result, method, *args):
            method(*args)
            return result

        if not enabled and self.enabled:
            # If we were enabled and are being set to disabled, eject the warp
            # core^w^w^w^wexpire all existing keycards
            self.enabled = False
            self._expirer.stop()
            d = self.expireAllKeycards()
            d.addCallback(callAndPassthru, self.on_disabled)
            return d
        self.enabled = enabled
        d = defer.succeed(0)
        d.addCallback(callAndPassthru, self.on_enabled)
        return d

    def getEnabled(self):
        return self.enabled

    def do_stop(self):
        return self.setEnabled(False)

    def authenticate(self, keycard):
        if not self.typeAllowed(keycard):
            self.warning('keycard %r is not an allowed keycard class', keycard)
            return None

        if not self.do_validate(keycard):
            self.warning('keycard %r is not a valid keycard instance', keycard)
            return None

        if self.enabled:
            if not self._expirer.running and hasattr(keycard, 'ttl'):
                self.debug('installing keycard timeout poller')
                self._expirer.start()
            return defer.maybeDeferred(self.do_authenticate, keycard)
        else:
            self.debug("Bouncer disabled, refusing authentication")
            return None

    def do_expireKeycards(self, elapsed):
        """
        Override to expire keycards managed by sub-classes.

        @param elapsed: time in second since the last expiration call.
        @type elapsed: int
        @returns: if there is more keycard to expire. If False is returned,
                  the expirer poller MAY be stopped.
        @rtype: bool
        """
        for k in self._keycards.values():
            if hasattr(k, 'ttl'):
                k.ttl -= elapsed
                if k.ttl <= 0:
                    self.expireKeycardId(k.id)
        return len(self._keycards) > 0

    def do_validate(self, keycard):
        """
        Override to check keycards before authentication steps.
        Should return True if the keycard is valid, False otherwise.
        #FIXME: This belong to the base bouncer class

        @param keycard: the keycard that should be validated
                        before authentication
        @type keycard: flumotion.common.keycards.Keycard
        @returns: True if the keycard is accepted, False otherwise
        @rtype: bool
        """
        return True

    def do_authenticate(self, keycard):
        """
        Must be overridden by subclasses.

        Authenticate the given keycard.
        Return the keycard with state AUTHENTICATED to authenticate,
        with state REQUESTING to continue the authentication process,
        or REFUSED to deny the keycard or a deferred which should
        have the same eventual value.

        FIXME: Currently, a return value of 'None' is treated
        as rejecting the keycard. This is unintuitive.

        FIXME: in fact, for authentication sessions like challenge/response,
        returning a keycard with state REFUSED instead of None
        will not work properly and may enter in an asynchronous infinit loop.
        """
        raise NotImplementedError("authenticate not overridden")

    def on_keycardAdded(self, keycard):
        """
        Override to update sub-class specific data related to keycards.
        Called when the base bouncer accepts and references a new keycard.
        """

    def on_keycardRemoved(self, keycard):
        """
        Override to cleanup sub-class specific data related to keycards.
        Called when the base bouncer has cleanup his references to a keycard.
        """

    def on_enabled(self):
        """
        Override to initialize sub-class specific data
        when the bouncer is enabled.
        """

    def on_disabled(self):
        """
        Override to cleanup sub-class specific data
        when the bouncer is disabled.
        """

    def hasKeycard(self, keycard):
        return keycard in self._keycards.values()

    def generateKeycardId(self):
        # FIXME: what if it already had one ?
        # FIXME: deal with wraparound ?
        keycardId = self._idFormat % self._idCounter
        self._idCounter += 1
        return keycardId

    def addKeycard(self, keycard):
        """
        Adds a keycard to the bouncer.
        Can be called with the same keycard more than one time.
        If the keycard has already been added successfully,
        adding it again will succeed and return True.

        @param keycard: the keycard to add.
        @return: if the bouncer accepts the keycard.
        """
        # give keycard an id and store it in our hash
        if keycard.id in self._keycards:
            # already in there
            return True

        keycardId = self.generateKeycardId()
        keycard.id = keycardId

        if hasattr(keycard, 'ttl') and keycard.ttl <= 0:
            self.log('immediately expiring keycard %r', keycard)
            return False

        self._addKeycard(keycard)
        return True

    def removeKeycard(self, keycard):
        if not keycard.id in self._keycards:
            raise KeyError

        del self._keycards[keycard.id]
        self.on_keycardRemoved(keycard)

        self.info("removed keycard with id %s" % keycard.id)

    def removeKeycardId(self, keycardId):
        self.debug("removing keycard with id %s" % keycardId)
        if not keycardId in self._keycards:
            raise KeyError

        keycard = self._keycards[keycardId]
        self.removeKeycard(keycard)

    def keepAlive(self, issuerName, ttl):
        for k in self._keycards.itervalues():
            if hasattr(k, 'issuerName') and k.issuerName == issuerName:
                k.ttl = ttl

    def expireAllKeycards(self):
        return self.expireKeycardIds(self._keycards.keys())

    def expireKeycardId(self, keycardId):
        self.log("expiring keycard with id %r", keycardId)
        if not keycardId in self._keycards:
            raise KeyError

        keycard = self._keycards[keycardId]
        self.removeKeycardId(keycardId)

        if self.medium:
            return self.medium.callRemote('expireKeycard',
                                          keycard.requesterId, keycard.id)
        else:
            return defer.succeed(None)

    def expireKeycardIds(self, keycardIds):
        self.log("expiring keycards with id %r", keycardIds)
        d = defer.Deferred()
        self._expireNextKeycardBlock(0, keycardIds, d)
        return d

    def _expireNextKeycardBlock(self, total, keycardIds, finished):
        keycardBlock = keycardIds[:EXPIRE_BLOCK_SIZE]
        keycardIds = keycardIds[EXPIRE_BLOCK_SIZE:]
        idByReq = {}

        for keycardId in keycardBlock:
            if keycardId in self._keycards:
                keycard = self._keycards[keycardId]
                requesterId = keycard.requesterId
                idByReq.setdefault(requesterId, []).append(keycardId)
                self.removeKeycardId(keycardId)

        if not (idByReq and self.medium):
            finished.callback(total)
            return

        defs = [self.medium.callRemote('expireKeycards', rid, ids)
                for rid, ids in idByReq.items()]
        dl = defer.DeferredList(defs, consumeErrors=True)

        def countExpirations(results, total):
            return sum([v for s, v in results if s and v]) + total

        dl.addCallback(countExpirations, total)
        dl.addCallback(self._expireNextKeycardBlock, keycardIds, finished)

    def _addKeycard(self, keycard):
        """
        Adds a keycard without checking.
        Used by sub-class knowing what they do.
        """
        self._keycards[keycard.id] = keycard
        self.on_keycardAdded(keycard)

        self.debug("added keycard with id %s, ttl %r", keycard.id,
                   getattr(keycard, 'ttl', None))

    def _expire(self):
        if not self.do_expireKeycards(self._expirer.timeout):
            if self._expirer.running:
                self.debug('no more keycards, removing timeout poller')
                self._expirer.stop()
示例#20
0
class PadMonitor(log.Loggable):
    """
    I monitor data flow on a GStreamer pad.
    I regularly schedule a buffer probe call at PAD_MONITOR_PROBE_INTERVAL.
    I regularly schedule a check call at PAD_MONITOR_CHECK_INTERVAL
    that makes sure a buffer probe was triggered since the last check call.
    """

    PAD_MONITOR_PROBE_INTERVAL = 5.0
    PAD_MONITOR_CHECK_INTERVAL = PAD_MONITOR_PROBE_INTERVAL * 2.5

    def __init__(self, pad, name, setActive, setInactive):
        """
        @type  pad:         L{gst.Pad}
        @type  name:        str
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._last_data_time = -1 # system time in epoch secs of last reception
        self._pad = pad
        self.name = name
        self._active = False
        self._first = True
        self._running = True

        self._doSetActive = []
        self._doSetInactive = []
        self.addWatch(setActive, setInactive)

        # This dict sillyness is because python's dict operations are atomic
        # w.r.t. the GIL.
        self._probe_id = {}

        self.check_poller = Poller(self._probe_timeout,
                                   self.PAD_MONITOR_PROBE_INTERVAL,
                                   immediately=True)

        self.watch_poller = Poller(self._check_timeout,
                                   self.PAD_MONITOR_CHECK_INTERVAL)

    def logMessage(self, message, *args):
        if self._first:
            self.debug(message, *args)
        else:
            self.log(message, *args)

    def isActive(self):
        return self._active

    def detach(self):
        self.check_poller.stop()
        self.watch_poller.stop()
        self._running = False

        # implementation closely tied to _probe_timeout wrt to GIL
        # tricks, threadsafety, and getting the probe deferred to
        # actually return
        d, probe_id = self._probe_id.pop("id", (None, None))
        if probe_id:
            self._pad.remove_buffer_probe(probe_id)
            d.callback(None)

    def _probe_timeout(self):
        # called every so often to install a probe callback

        def probe_cb(pad, buffer):
            """
            Periodically scheduled buffer probe, that ensures that we're
            currently actually having dataflow through our eater
            elements.

            Called from GStreamer threads.

            @param pad:       The gst.Pad srcpad for one eater in this
                              component.
            @param buffer:    A gst.Buffer that has arrived on this pad
            """
            self._last_data_time = time.time()

            self.logMessage('buffer probe on %s has timestamp %s', self.name,
                            gst.TIME_ARGS(buffer.timestamp))

            deferred, probe_id = self._probe_id.pop("id", (None, None))
            if probe_id:
                # This will be None only if detach() has been called.
                self._pad.remove_buffer_probe(probe_id)

                reactor.callFromThread(deferred.callback, None)
                # Data received! Return to happy ASAP:
                reactor.callFromThread(self.watch_poller.run)

            self._first = False

            # let the buffer through
            return True

        d = defer.Deferred()
        # FIXME: this is racy: evaluate RHS, drop GIL, buffer probe
        # fires before __setitem__ in LHS; need a mutex
        self._probe_id['id'] = (d, self._pad.add_buffer_probe(probe_cb))
        return d

    def _check_timeout(self):
        # called every so often to check that a probe callback was triggered
        self.log('last buffer for %s at %r', self.name, self._last_data_time)

        now = time.time()

        if self._last_data_time < 0:
            # We never received any data in the first timeout period...
            self._last_data_time = 0
            self.setInactive()
        elif self._last_data_time == 0:
            # still no data...
            pass
        else:
            # We received data at some time in the past.
            delta = now - self._last_data_time

            if self._active and delta > self.PAD_MONITOR_CHECK_INTERVAL:
                self.info("No data received on pad %s for > %r seconds, "
                          "setting to hungry",
                          self.name, self.PAD_MONITOR_CHECK_INTERVAL)
                self.setInactive()
            elif not self._active and delta < self.PAD_MONITOR_CHECK_INTERVAL:
                self.info("Receiving data again on pad %s, flow active",
                    self.name)
                self.setActive()

    def addWatch(self, setActive, setInactive):
        """
        @param setActive:   a callable that will be called when the pad is
                            considered active, taking the name of the monitor.
        @type  setActive:   callable
        @param setInactive: a callable that will be called when the pad is
                            considered inactive, taking the name of the
                            monitor.
        @type  setInactive: callable
        """
        self._doSetActive.append(setActive)
        self._doSetInactive.append(setInactive)

    def setInactive(self):
        self._active = False
        for setInactive in self._doSetInactive:
            setInactive(self.name)

    def setActive(self):
        self._active = True
        for setActive in self._doSetActive:
            setActive(self.name)