class ComponentRRDPlug(base.ComponentPlug):
"""Class to create or update a RRD file with statistics"""
### ComponentPlug methods
def start(self, component):
self._rrdpoller = None
self._component = component
if not self._hasImport():
return
properties = self.args['properties']
self._clientsPath = properties['clients-connected-file']
self._bytesPath = properties['bytes-transferred-file']
self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
self._RRDPaths = self._getRRDPaths()
# call to update_rrd with a poll interval
timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
self._rrdpoller = Poller(self._updateRRD, timeout)
def stop(self, component):
if self._rrdpoller:
self._rrdpoller.stop()
def _hasImport(self):
"""Check rrdtool availability"""
if not rrdtool:
m = messages.Warning(T_(N_(
"Cannot import module '%s'.\n"), 'rrdtool'),
mid='rrdtool-import-error')
m.add(T_(N_(
"The RRD plug for this component is disabled.")))
self._component.addMessage(m)
return False
return True
def _updateRRD(self):
"""Update data in RRD file"""
for path in self._RRDPaths:
value = None
if path == self._clientsPath:
value = self._component.getClients()
elif path == self._bytesPath:
value = self._component.getBytesSent()
try:
rrdtool.update(path, 'N:%i' % value)
self.debug('RRD file [%s] updated with value: %r',
path, value)
except rrdtool.error, e:
# We could get an error from rrdtool on converting the
# value to a double or from not finding the file
self.warning('RRD error: %r',
log.getExceptionMessage(e))
class ComponentRRDPlug(base.ComponentPlug):
"""Class to create or update a RRD file with statistics"""
### ComponentPlug methods
def start(self, component):
self._rrdpoller = None
self._component = component
if not self._hasImport():
return
properties = self.args['properties']
self._clientsPath = properties['clients-connected-file']
self._bytesPath = properties['bytes-transferred-file']
self._stepSize = properties.get('step-size', _DEFAULT_STEP_SIZE)
self._RRDPaths = self._getRRDPaths()
# call to update_rrd with a poll interval
timeout = properties.get('poll-interval', _DEFAULT_POLL_INTERVAL)
self._rrdpoller = Poller(self._updateRRD, timeout)
def stop(self, component):
if self._rrdpoller:
self._rrdpoller.stop()
def _hasImport(self):
"""Check rrdtool availability"""
if not rrdtool:
m = messages.Warning(T_(N_(
"Cannot import module '%s'.\n"), 'rrdtool'),
mid='rrdtool-import-error')
m.add(T_(N_(
"The RRD plug for this component is disabled.")))
self._component.addMessage(m)
return False
return True
def _updateRRD(self):
"""Update data in RRD file"""
for path in self._RRDPaths:
value = None
if path == self._clientsPath:
value = self._component.getClients()
elif path == self._bytesPath:
value = self._component.getBytesSent()
try:
rrdtool.update(path, 'N:%i' % value)
self.debug('RRD file [%s] updated with value: %r',
path, value)
except rrdtool.error, e:
# We could get an error from rrdtool on converting the
# value to a double or from not finding the file
self.warning('RRD error: %r',
log.getExceptionMessage(e))
class EaterPadMonitor(PadMonitor):
def __init__(self, pad, name, setActive, setInactive,
reconnectEater, *args):
PadMonitor.__init__(self, pad, name, setActive, setInactive)
self._reconnectPoller = Poller(lambda: reconnectEater(*args),
self.PAD_MONITOR_CHECK_INTERVAL,
start=False)
def setInactive(self):
PadMonitor.setInactive(self)
# It might be that we got detached while calling
# PadMonitor.setInactive() For example someone might have
# stopped the component as it went hungry, which would happen
# inside the PadMonitor.setInactive() call. The component
# would then detach us and the reconnect poller would get
# stopped. If that happened don't bother restarting it, as it
# will result in the reactor ending up in an unclean state.
#
# A prominent example of such situation is
# flumotion.test.test_component_disker, where the component
# gets stopped right after it goes hungry
if self._running:
# If an eater received a buffer before being marked as
# disconnected, and still within the buffer check
# interval, the next eaterCheck call could accidentally
# think the eater was reconnected properly. Setting this
# to 0 here avoids that happening in eaterCheck.
self._last_data_time = 0
self.debug('starting the reconnect poller')
self._reconnectPoller.start(immediately=True)
def setActive(self):
PadMonitor.setActive(self)
self.debug('stopping the reconnect poller')
self._reconnectPoller.stop()
def detach(self):
PadMonitor.detach(self)
self.debug('stopping the reconnect poller')
self._reconnectPoller.stop()
class EaterPadMonitor(PadMonitor):
def __init__(self, pad, name, setActive, setInactive, reconnectEater,
*args):
PadMonitor.__init__(self, pad, name, setActive, setInactive)
self._reconnectPoller = Poller(lambda: reconnectEater(*args),
self.PAD_MONITOR_CHECK_INTERVAL,
start=False)
def setInactive(self):
PadMonitor.setInactive(self)
# It might be that we got detached while calling
# PadMonitor.setInactive() For example someone might have
# stopped the component as it went hungry, which would happen
# inside the PadMonitor.setInactive() call. The component
# would then detach us and the reconnect poller would get
# stopped. If that happened don't bother restarting it, as it
# will result in the reactor ending up in an unclean state.
#
# A prominent example of such situation is
# flumotion.test.test_component_disker, where the component
# gets stopped right after it goes hungry
if self._running:
# If an eater received a buffer before being marked as
# disconnected, and still within the buffer check
# interval, the next eaterCheck call could accidentally
# think the eater was reconnected properly. Setting this
# to 0 here avoids that happening in eaterCheck.
self._last_data_time = 0
self.debug('starting the reconnect poller')
self._reconnectPoller.start(immediately=True)
def setActive(self):
PadMonitor.setActive(self)
self.debug('stopping the reconnect poller')
self._reconnectPoller.stop()
def detach(self):
PadMonitor.detach(self)
self.debug('stopping the reconnect poller')
self._reconnectPoller.stop()
class BaseComponent(common.InitMixin, log.Loggable):
"""
I am the base class for all Flumotion components.
@ivar name: the name of the component
@type name: string
@ivar medium: the component's medium
@type medium: L{BaseComponentMedium}
@ivar uiState: state of the component to be shown in a UI.
Contains at least the following keys.
- cpu-percent: percentage of CPU use in last interval
- start-time: time when component was started, in epoch
seconds
- current-time: current time in epoch seconds, as seen on
component's machine, which might be out of
sync
- virtual-size: virtual memory size in bytes
Subclasses can add additional keys for their respective UI.
@type uiState: L{componentui.WorkerComponentUIState}
@cvar componentMediumClass: the medium class to use for this component
@type componentMediumClass: child class of L{BaseComponentMedium}
"""
logCategory = 'basecomp'
componentMediumClass = BaseComponentMedium
implements(IStateCacheableListener)
def __init__(self, config, haveError=None):
"""
Subclasses should not override __init__ at all.
Instead, they should implement init(), which will be called
by this implementation automatically.
L{flumotion.common.common.InitMixin} for more details.
"""
self.debug("initializing %r with config %r", type(self), config)
self.config = config
self._haveError = haveError
# this will call self.init() for all implementors of init()
common.InitMixin.__init__(self)
self.setup()
# BaseComponent interface for subclasses related to component protocol
def init(self):
"""
A subclass should do as little as possible in its init method.
In particular, it should not try to access resources.
Failures during init are marshalled back to the manager through
the worker's remote_create method, since there is no component state
proxied to the manager yet at the time of init.
"""
self.state = planet.WorkerJobState()
self.name = self.config['name']
self.state.set('pid', os.getpid())
self.setMood(moods.waking)
self.medium = None # the medium connecting us to the manager's avatar
self.uiState = componentui.WorkerComponentUIState()
self.uiState.addKey('cpu-percent')
self.uiState.addKey('start-time')
self.uiState.addKey('current-time')
self.uiState.addKey('virtual-size')
self.uiState.addKey('total-memory')
self.uiState.addKey('num-cpus')
self.uiState.addKey('flu-debug')
self.uiState.addKey('properties')
self.uiState.addHook(self)
self.plugs = {}
self._happyWaits = []
# Start the cpu-usage updating.
self._lastTime = time.time()
self._lastClock = time.clock()
self._cpuPoller = Poller(self._pollCPU, 5, start=False)
self._memoryPoller = Poller(self._pollMemory, 60, start=False)
self._cpuPollerDC = None
self._memoryPollerDC = None
self._shutdownHook = None
### IStateCacheable Interface
def observerAppend(self, observer, num):
"""
Triggered when a uiState observer was added.
Default implementation is to start the memory and cpu pollers.
Note:
Subclasses can override me but should chain me up to start these
pollers
"""
self.debug("observer has started watching us, starting pollers")
if not self._cpuPoller.running and not self._cpuPollerDC:
self._cpuPollerDC = reactor.callLater(0,
self._cpuPoller.start,
immediately=True)
if not self._memoryPoller.running and not self._memoryPollerDC:
self._memoryPollerDC = reactor.callLater(0,
self._memoryPoller.start,
immediately=True)
def observerRemove(self, observer, num):
"""
Triggered when a uiState observer has left.
Default implementation is to stop the memory and cpu pollers
when the total number of observers denoted by the 'num'
argument becomes zero.
Note:
Subclasses can override me but should chain me up to stop these
pollers
"""
if num == 0:
self.debug("no more observers left, shutting down pollers")
# Cancel any pending callLaters
if self._cpuPollerDC:
self._cpuPollerDC.cancel()
self._cpuPollerDC = None
if self._memoryPollerDC:
self._memoryPollerDC.cancel()
self._memoryPollerDC = None
if self._cpuPoller:
self._cpuPoller.stop()
if self._memoryPoller:
self._memoryPoller.stop()
def do_check(self):
"""
Subclasses can implement me to run any checks before the component
performs setup.
Messages can be added to the component state's 'messages' list key.
Any error messages added will trigger the component going to sad,
with L{flumotion.common.errors.ComponentSetupError} being raised
before getting to setup stage; do_setup() will not be called.
In the event of a fatal problem that can't be expressed through an
error message, this method should raise an exception or return a
failure.
It is not necessary to chain up in this function. The return
value may be a deferred.
"""
return defer.maybeDeferred(self.check_properties,
self.config['properties'], self.addMessage)
def check_properties(self, properties, addMessage):
"""
BaseComponent convenience vmethod for running checks.
A component implementation can override this method to run any
checks that it needs to. Typically, a check_properties
implementation will call the provided addMessage() callback to
note warnings or errors. For errors, addMessage() will set
component's mood to sad, which will abort the init process
before getting to do_setup().
@param properties: The component's properties
@type properties: dict of string => object
@param addMessage: Thunk to add a message to the component
state. Will raise an exception if the
message is of level ERROR.
@type addMessage: L{flumotion.common.messages.Message} -> None
"""
pass
def do_setup(self):
"""
Subclasses can implement me to set up the component before it is
started. It should set up the component, possibly opening files
and resources.
Non-programming errors should not be raised, but returned as a
failing deferred.
The return value may be a deferred.
"""
plug_starts = []
for socket, plugs in self.config['plugs'].items():
self.plugs[socket] = []
for plug in plugs:
entry = plug['entries']['default']
instance = reflectcall.reflectCall(entry['module-name'],
entry['function-name'],
plug)
self.plugs[socket].append(instance)
self.debug('Starting plug %r on socket %s', instance, socket)
plug_starts.append(instance.start)
# Call check methods, starting from the base class and working down to
# subclasses.
checks = common.get_all_methods(self, 'do_check', False)
def checkErrorCallback(result):
# if the mood is now sad, it means an error was encountered
# during check, and we should return a failure here.
# since the checks are responsible for adding a message,
# this is a handled error.
current = self.state.get('mood')
if current == moods.sad.value:
self.warning('Running checks made the component sad.')
raise errors.ComponentSetupHandledError()
checks.append(checkErrorCallback)
return _maybeDeferredChain(plug_starts + checks, self)
def do_stop(self):
"""
BaseComponent vmethod for stopping.
The component should do any cleanup it needs, but must not set the
component's mood to sleeping.
@Returns: L{twisted.internet.defer.Deferred}
"""
plug_stops = []
for socket, plugs in self.plugs.items():
for plug in plugs:
self.debug('Stopping plug %r on socket %s', plug, socket)
plug_stops.append(plug.stop)
for message in self.state.get('messages'):
# FIXME: not necessary
self.state.remove('messages', message)
# Cancel any pending callLaters
if self._cpuPollerDC:
self._cpuPollerDC.cancel()
self._cpuPollerDC = None
if self._memoryPollerDC:
self._memoryPollerDC.cancel()
self._memoryPollerDC = None
if self._cpuPoller:
self._cpuPoller.stop()
self._cpuPoller = None
if self._memoryPoller:
self._memoryPoller.stop()
self._memoryPoller = None
if self._shutdownHook:
self.debug('_stoppedCallback: firing shutdown hook')
self._shutdownHook()
return _maybeDeferredChain(plug_stops, self)
### BaseComponent implementation related to compoment protocol
def setup(self):
"""
Sets up the component. Called during __init__, so be sure not
to raise exceptions, instead adding messages to the component
state.
"""
def run_setups():
setups = common.get_all_methods(self, 'do_setup', False)
return _maybeDeferredChain(setups, self)
def setup_complete(_):
self.debug('setup completed')
self.setup_completed()
def got_error(failure):
txt = log.getFailureMessage(failure)
self.debug('got_error: %s', txt)
if not failure.check(errors.ComponentSetupHandledError):
self.warning('Setup failed: %s', txt)
m = messages.Error(T_(N_("Could not setup component.")),
debug=txt,
mid="component-setup-%s" % self.name)
# will call setMood(moods.sad)
self.addMessage(m)
# swallow
return None
self.setMood(moods.waking)
self.uiState.set('start-time', time.time())
self.uiState.set('total-memory', self._getTotalMemory())
self.uiState.set('num-cpus', self._getNumberOfCPUs())
self.uiState.set('flu-debug', log.getDebug())
d = run_setups()
d.addCallbacks(setup_complete, got_error)
# all status info via messages and the mood
def setup_completed(self):
self.debug('turning happy')
self.setMood(moods.happy)
def setShutdownHook(self, shutdownHook):
"""
Set the shutdown hook for this component (replacing any previous hook).
When a component is stopped, then this hook will be fired.
"""
self._shutdownHook = shutdownHook
def stop(self):
"""
Tell the component to stop.
The connection to the manager will be closed.
The job process will also finish.
"""
self.debug('BaseComponent.stop')
# Set ourselves to waking while we're shutting down.
self.setMood(moods.waking)
# Run stop methods, starting from the subclass, up to this base class.
stops = common.get_all_methods(self, 'do_stop', True)
return _maybeDeferredChain(stops, self)
### BaseComponent public methods
def getName(self):
return self.name
def setWorkerName(self, workerName):
self.state.set('workerName', workerName)
def getWorkerName(self):
return self.state.get('workerName')
def setMedium(self, medium):
assert isinstance(medium, BaseComponentMedium)
self.medium = medium
self.medium.logName = self.getName()
for plugs in self.plugs.values():
for plug in plugs:
self._export_plug_interface(plug, medium)
def setMood(self, mood):
"""
Set the given mood on the component if it's different from the current
one.
"""
current = self.state.get('mood')
if current == mood.value:
self.log('already in mood %r' % mood)
return
elif current == moods.sad.value:
self.info('tried to set mood to %r, but already sad :-(' % mood)
return
self.doLog(log.DEBUG, -2, 'MOOD changed to %r by caller', mood)
self.state.set('mood', mood.value)
if mood == moods.happy:
while self._happyWaits:
self._happyWaits.pop(0).callback(None)
elif mood == moods.sad:
while self._happyWaits:
self._happyWaits.pop(0).errback(errors.ComponentStartError())
def getMood(self):
"""
Gets the mood on the component.
@rtype: int
"""
return self.state.get('mood')
def waitForHappy(self):
mood = self.getMood()
if mood == moods.happy.value:
return defer.succeed(None)
elif mood == moods.sad.value:
return defer.fail(errors.ComponentStartError())
else:
d = defer.Deferred()
self._happyWaits.append(d)
return d
def addMessage(self, message):
"""
Add a message to the component.
If any of the messages is an error, the component will turn sad.
@type message: L{flumotion.common.messages.Message}
"""
self.removeMessage(message.id)
self.state.append('messages', message)
if message.level == messages.ERROR:
self.debug('error message, turning sad')
self.setMood(moods.sad)
if self._haveError:
self._haveError(message)
def removeMessage(self, mid):
"""
Remove a message with a given id from the component.
@type mid: str
"""
for msg in self.state.get('messages', []):
if msg.id == mid:
self.state.remove('messages', msg)
def warnDeprecatedProperties(self, list):
"""
Add a warning messages for deprecated properties.
@param list: list of property names.
@type list: list of str
"""
msg = ("Your configuration uses deprecated properties. "
"Please update your configuration and correct them.\n")
m = messages.Warning(T_(N_(msg)), mid="deprecated")
for prop in list:
m.add(T_(N_("Please remove '%s' property.\n"), prop))
self.addMessage(m)
self.warning(msg)
def fixRenamedProperties(self, properties, list):
"""
Fix properties that have been renamed from a previous version,
and add a warning for them.
@param properties: properties; will be modified as a result.
@type properties: dict
@param list: list of (old, new) tuples of property names.
@type list: list of tuple of (str, str)
"""
found = []
for old, new in list:
if old in properties:
found.append((old, new))
if found:
m = messages.Warning(T_(
N_("Your configuration uses deprecated properties. "
"Please update your configuration and correct them.\n")),
mid="deprecated")
for old, new in found:
m.add(T_(N_("Please rename '%s' to '%s'.\n"), old, new))
self.debug("Setting new property '%s' to %r", new,
properties[old])
properties[new] = properties[old]
del properties[old]
self.addMessage(m)
def adminCallRemote(self, methodName, *args, **kwargs):
"""
Call a remote method on all admin client views on this component.
This gets serialized through the manager and multiplexed to all
admin clients, and from there on to all views connected to each
admin client model.
Because there can be any number of admin clients that this call
will go out do, it does not make sense to have one return value.
This function will return None always.
"""
if self.medium:
self.medium.callRemote("adminCallRemote", methodName, *args,
**kwargs)
else:
self.debug('asked to adminCallRemote(%s, *%r, **%r), but '
'no manager.' % (methodName, args, kwargs))
def modifyProperty(self, property_name, value):
"""
Modifies a property of the compoment.
Components with modifiable properties (properties that can be changed
on the fly) should implement modify_property_(propertyName) to receive
the call
@param property_name: Name of the property to change
@type property_name: str
@param value: Value to set
"""
# Transform property name to camel case:
# max-reconnections-delay -> MaxReconnectionsDelay
p = ''.join([t.title() for t in property_name.split('-')])
method_name = "modify_property_%s" % p
if not hasattr(self, method_name):
raise errors.PropertyNotModifiableError("%s" % (property_name))
method = getattr(self, method_name)
if not method(value):
return False
self.config['properties'][property_name] = value
self.uiState.set('properties', self.config['properties'])
return True
def checkPropertyType(self, property_name, value, allowed_type):
"""
Check that the value to be set in a property is of the correct type
@returns: True if the value is of the correct type
"""
if type(value) != allowed_type:
self.warning(
"Could not set the property %s in %s. "
"'value' must be of %s", property_name, self, allowed_type)
return False
return True
def _export_plug_interface(self, plug, medium):
try:
namespace = plug.get_namespace()
except AttributeError:
self.debug(
"Plug %r does not provide namespace, "
"its interface will not be exposed", plug)
return
self.debug("Exposing plug's %r interface in namespace %r", plug,
namespace)
for method in filter(
callable,
[getattr(plug, m) for m in dir(plug) if m.startswith('remote_')]):
if namespace:
name = "".join(("remote_", namespace, "_",
method.__name__[len("remote_"):]))
else:
name = method.__name__
self.debug("Exposing method %r as %r in %r", method, name, medium)
setattr(medium, name, method)
def _pollCPU(self):
self._cpuPollerDC = None
# update CPU time stats
nowTime = time.time()
nowClock = time.clock()
deltaTime = nowTime - self._lastTime
deltaClock = nowClock - self._lastClock
self._lastTime = nowTime
self._lastClock = nowClock
# deltaClock can be < 0 if time.clock() wrapped around
if deltaClock >= 0:
CPU = deltaClock / deltaTime
self.log('latest CPU use: %r', CPU)
self.uiState.set('cpu-percent', CPU)
self.uiState.set('current-time', nowTime)
def _pollMemory(self):
self._memoryPollerDC = None
# Figure out our virtual memory size and report that.
# I don't know a nicer way to find vsize than groping /proc/
handle = open('/proc/%d/stat' % os.getpid())
line = handle.read()
handle.close()
fields = line.split()
# field 1 (comm) could potentially contain spaces and thus split over
# multiple list items, but our processes do not contain spaces
vsize = int(fields[22])
self.log('vsize is %d', vsize)
self.uiState.set('virtual-size', vsize)
def _getTotalMemory(self):
f = open("/proc/meminfo")
memtotal = f.readline()
f.close()
return int(memtotal[memtotal.index(":") + 1:-3]) * 1024
def _getNumberOfCPUs(self):
try:
return open('/proc/cpuinfo').read().count('processor\t:')
except IOError:
self.debug('Can not determine number of CPUs on this system')
return 1
class PadMonitor(log.Loggable):
"""
I monitor data flow on a GStreamer pad.
I regularly schedule a buffer probe call at PAD_MONITOR_PROBE_INTERVAL.
I regularly schedule a check call at PAD_MONITOR_CHECK_INTERVAL
that makes sure a buffer probe was triggered since the last check call.
"""
PAD_MONITOR_PROBE_INTERVAL = 5.0
PAD_MONITOR_CHECK_INTERVAL = PAD_MONITOR_PROBE_INTERVAL * 2.5
def __init__(self, pad, name, setActive, setInactive):
"""
@type pad: L{gst.Pad}
@type name: str
@param setActive: a callable that will be called when the pad is
considered active, taking the name of the monitor.
@type setActive: callable
@param setInactive: a callable that will be called when the pad is
considered inactive, taking the name of the
monitor.
@type setInactive: callable
"""
self._last_data_time = -1 # system time in epoch secs of last reception
self._pad = pad
self.name = name
self._active = False
self._first = True
self._running = True
self._doSetActive = []
self._doSetInactive = []
self.addWatch(setActive, setInactive)
# This dict sillyness is because python's dict operations are atomic
# w.r.t. the GIL.
self._probe_id = {}
self.check_poller = Poller(self._probe_timeout,
self.PAD_MONITOR_PROBE_INTERVAL,
immediately=True)
self.watch_poller = Poller(self._check_timeout,
self.PAD_MONITOR_CHECK_INTERVAL)
def logMessage(self, message, *args):
if self._first:
self.debug(message, *args)
else:
self.log(message, *args)
def isActive(self):
return self._active
def detach(self):
self.check_poller.stop()
self.watch_poller.stop()
self._running = False
# implementation closely tied to _probe_timeout wrt to GIL
# tricks, threadsafety, and getting the probe deferred to
# actually return
d, probe_id = self._probe_id.pop("id", (None, None))
if probe_id:
self._pad.remove_buffer_probe(probe_id)
d.callback(None)
def _probe_timeout(self):
# called every so often to install a probe callback
def probe_cb(pad, buffer):
"""
Periodically scheduled buffer probe, that ensures that we're
currently actually having dataflow through our eater
elements.
Called from GStreamer threads.
@param pad: The gst.Pad srcpad for one eater in this
component.
@param buffer: A gst.Buffer that has arrived on this pad
"""
self._last_data_time = time.time()
self.logMessage('buffer probe on %s has timestamp %s', self.name,
gst.TIME_ARGS(buffer.timestamp))
deferred, probe_id = self._probe_id.pop("id", (None, None))
if probe_id:
# This will be None only if detach() has been called.
self._pad.remove_buffer_probe(probe_id)
reactor.callFromThread(deferred.callback, None)
# Data received! Return to happy ASAP:
reactor.callFromThread(self.watch_poller.run)
self._first = False
# let the buffer through
return True
d = defer.Deferred()
# FIXME: this is racy: evaluate RHS, drop GIL, buffer probe
# fires before __setitem__ in LHS; need a mutex
self._probe_id['id'] = (d, self._pad.add_buffer_probe(probe_cb))
return d
def _check_timeout(self):
# called every so often to check that a probe callback was triggered
self.log('last buffer for %s at %r', self.name, self._last_data_time)
now = time.time()
if self._last_data_time < 0:
# We never received any data in the first timeout period...
self._last_data_time = 0
self.setInactive()
elif self._last_data_time == 0:
# still no data...
pass
else:
# We received data at some time in the past.
delta = now - self._last_data_time
if self._active and delta > self.PAD_MONITOR_CHECK_INTERVAL:
self.info(
"No data received on pad %s for > %r seconds, "
"setting to hungry", self.name,
self.PAD_MONITOR_CHECK_INTERVAL)
self.setInactive()
elif not self._active and delta < self.PAD_MONITOR_CHECK_INTERVAL:
self.info("Receiving data again on pad %s, flow active",
self.name)
self.setActive()
def addWatch(self, setActive, setInactive):
"""
@param setActive: a callable that will be called when the pad is
considered active, taking the name of the monitor.
@type setActive: callable
@param setInactive: a callable that will be called when the pad is
considered inactive, taking the name of the
monitor.
@type setInactive: callable
"""
self._doSetActive.append(setActive)
self._doSetInactive.append(setInactive)
def setInactive(self):
self._active = False
for setInactive in self._doSetInactive:
setInactive(self.name)
def setActive(self):
self._active = True
for setActive in self._doSetActive:
setActive(self.name)
class BouncerPlug(pbase.ComponentPlug, common.InitMixin):
"""
I am the base class for all bouncer plugs.
FIXME: expireKeycardIds has been added to the component bouncer.
Because the plug version is not yet used, and will be
refactored/redesigned in a near future, the modifications
have not been duplicated here.
@cvar keycardClasses: tuple of all classes of keycards this bouncer can
authenticate, in order of preference
@type keycardClasses: tuple of L{flumotion.common.keycards.Keycard}
class objects
"""
keycardClasses = ()
logCategory = "bouncer"
KEYCARD_EXPIRE_INTERVAL = 2 * 60
def __init__(self, *args, **kwargs):
pbase.ComponentPlug.__init__(self, *args, **kwargs)
common.InitMixin.__init__(self)
def init(self):
self.medium = None
self.enabled = True
self._idCounter = 0
self._idFormat = time.strftime("%Y%m%d%H%M%S-%%d")
self._keycards = {} # keycard id -> Keycard
self._expirer = Poller(self._expire, self.KEYCARD_EXPIRE_INTERVAL, start=False)
def typeAllowed(self, keycard):
"""
Verify if the keycard is an instance of a Keycard class specified
in the bouncer's keycardClasses variable.
"""
return isinstance(keycard, self.keycardClasses)
def setEnabled(self, enabled):
if not enabled and self.enabled:
# If we were enabled and are being set to disabled, eject the warp
# core^w^w^w^wexpire all existing keycards
self.expireAllKeycards()
self._expirer.stop()
self.enabled = enabled
def setMedium(self, medium):
self.medium = medium
def stop(self, component):
self.setEnabled(False)
def _expire(self):
for k in self._keycards.values():
if hasattr(k, "ttl"):
k.ttl -= self._expirer.timeout
if k.ttl <= 0:
self.expireKeycardId(k.id)
def authenticate(self, keycard):
if not self.typeAllowed(keycard):
self.warning("keycard %r is not an allowed keycard class", keycard)
return None
if self.enabled:
if not self._expirer.running and hasattr(keycard, "ttl"):
self.debug("installing keycard timeout poller")
self._expirer.start()
return defer.maybeDeferred(self.do_authenticate, keycard)
else:
self.debug("Bouncer disabled, refusing authentication")
return None
def do_authenticate(self, keycard):
"""
Must be overridden by subclasses.
Authenticate the given keycard.
Return the keycard with state AUTHENTICATED to authenticate,
with state REQUESTING to continue the authentication process,
or None to deny the keycard, or a deferred which should have the same
eventual value.
"""
raise NotImplementedError("authenticate not overridden")
def hasKeycard(self, keycard):
return keycard in self._keycards.values()
def generateKeycardId(self):
keycardId = self._idFormat % self._idCounter
self._idCounter += 1
return keycardId
def addKeycard(self, keycard):
"""
Adds a keycard to the bouncer.
Can be called with the same keycard more than one time.
If the keycard has already been added successfully,
adding it again will succeed and return True.
@param keycard: the keycard to add.
@return: if the bouncer accepts the keycard.
"""
# give keycard an id and store it in our hash
if keycard.id in self._keycards:
# already in there
return True
keycard.id = self.generateKeycardId()
if hasattr(keycard, "ttl") and keycard.ttl <= 0:
self.log("immediately expiring keycard %r", keycard)
return False
self._keycards[keycard.id] = keycard
self.debug("added keycard with id %s" % keycard.id)
return True
def removeKeycard(self, keycard):
if not keycard.id in self._keycards:
raise KeyError
del self._keycards[keycard.id]
self.debug("removed keycard with id %s" % keycard.id)
def removeKeycardId(self, keycardId):
self.debug("removing keycard with id %s" % keycardId)
if not keycardId in self._keycards:
raise KeyError
keycard = self._keycards[keycardId]
self.removeKeycard(keycard)
def keepAlive(self, issuerName, ttl):
for k in self._keycards.itervalues():
if hasattr(k, "issuerName") and k.issuerName == issuerName:
k.ttl = ttl
def expireAllKeycards(self):
return defer.DeferredList([self.expireKeycardId(keycardId) for keycardId in self._keycards.keys()])
def expireKeycardId(self, keycardId):
self.log("expiring keycard with id %r", keycardId)
if not keycardId in self._keycards:
raise KeyError
keycard = self._keycards.pop(keycardId)
return self.medium.callRemote("expireKeycard", keycard.requesterId, keycard.id)
class BaseComponent(common.InitMixin, log.Loggable):
"""
I am the base class for all Flumotion components.
@ivar name: the name of the component
@type name: string
@ivar medium: the component's medium
@type medium: L{BaseComponentMedium}
@ivar uiState: state of the component to be shown in a UI.
Contains at least the following keys.
- cpu-percent: percentage of CPU use in last interval
- start-time: time when component was started, in epoch
seconds
- current-time: current time in epoch seconds, as seen on
component's machine, which might be out of
sync
- virtual-size: virtual memory size in bytes
Subclasses can add additional keys for their respective UI.
@type uiState: L{componentui.WorkerComponentUIState}
@cvar componentMediumClass: the medium class to use for this component
@type componentMediumClass: child class of L{BaseComponentMedium}
"""
logCategory = 'basecomp'
componentMediumClass = BaseComponentMedium
implements(IStateCacheableListener)
def __init__(self, config, haveError=None):
"""
Subclasses should not override __init__ at all.
Instead, they should implement init(), which will be called
by this implementation automatically.
L{flumotion.common.common.InitMixin} for more details.
"""
self.debug("initializing %r with config %r", type(self), config)
self.config = config
self._haveError = haveError
# this will call self.init() for all implementors of init()
common.InitMixin.__init__(self)
self.setup()
# BaseComponent interface for subclasses related to component protocol
def init(self):
"""
A subclass should do as little as possible in its init method.
In particular, it should not try to access resources.
Failures during init are marshalled back to the manager through
the worker's remote_create method, since there is no component state
proxied to the manager yet at the time of init.
"""
self.state = planet.WorkerJobState()
self.name = self.config['name']
self.state.set('pid', os.getpid())
self.setMood(moods.waking)
self.medium = None # the medium connecting us to the manager's avatar
self.uiState = componentui.WorkerComponentUIState()
self.uiState.addKey('cpu-percent')
self.uiState.addKey('start-time')
self.uiState.addKey('current-time')
self.uiState.addKey('virtual-size')
self.uiState.addKey('total-memory')
self.uiState.addHook(self)
self.plugs = {}
self._happyWaits = []
# Start the cpu-usage updating.
self._lastTime = time.time()
self._lastClock = time.clock()
self._cpuPoller = Poller(self._pollCPU, 5, start=False)
self._memoryPoller = Poller(self._pollMemory, 60, start=False)
self._cpuPollerDC = None
self._memoryPollerDC = None
self._shutdownHook = None
### IStateCacheable Interface
def observerAppend(self, observer, num):
"""
Triggered when a uiState observer was added.
Default implementation is to start the memory and cpu pollers.
Note:
Subclasses can override me but should chain me up to start these
pollers
"""
self.debug("observer has started watching us, starting pollers")
if not self._cpuPoller.running and not self._cpuPollerDC:
self._cpuPollerDC = reactor.callLater(0,
self._cpuPoller.start,
immediately=True)
if not self._memoryPoller.running and not self._memoryPollerDC:
self._memoryPollerDC = reactor.callLater(0,
self._memoryPoller.start,
immediately=True)
def observerRemove(self, observer, num):
"""
Triggered when a uiState observer has left.
Default implementation is to stop the memory and cpu pollers
when the total number of observers denoted by the 'num'
argument becomes zero.
Note:
Subclasses can override me but should chain me up to stop these
pollers
"""
if num == 0:
self.debug("no more observers left, shutting down pollers")
# Cancel any pending callLaters
if self._cpuPollerDC:
self._cpuPollerDC.cancel()
self._cpuPollerDC = None
if self._memoryPollerDC:
self._memoryPollerDC.cancel()
self._memoryPollerDC = None
if self._cpuPoller:
self._cpuPoller.stop()
if self._memoryPoller:
self._memoryPoller.stop()
def do_check(self):
"""
Subclasses can implement me to run any checks before the component
performs setup.
Messages can be added to the component state's 'messages' list key.
Any error messages added will trigger the component going to sad,
with L{flumotion.common.errors.ComponentSetupError} being raised
before getting to setup stage; do_setup() will not be called.
In the event of a fatal problem that can't be expressed through an
error message, this method should raise an exception or return a
failure.
It is not necessary to chain up in this function. The return
value may be a deferred.
"""
return defer.maybeDeferred(self.check_properties,
self.config['properties'],
self.addMessage)
def check_properties(self, properties, addMessage):
"""
BaseComponent convenience vmethod for running checks.
A component implementation can override this method to run any
checks that it needs to. Typically, a check_properties
implementation will call the provided addMessage() callback to
note warnings or errors. For errors, addMessage() will set
component's mood to sad, which will abort the init process
before getting to do_setup().
@param properties: The component's properties
@type properties: dict of string => object
@param addMessage: Thunk to add a message to the component
state. Will raise an exception if the
message is of level ERROR.
@type addMessage: L{flumotion.common.messages.Message} -> None
"""
pass
def do_setup(self):
"""
Subclasses can implement me to set up the component before it is
started. It should set up the component, possibly opening files
and resources.
Non-programming errors should not be raised, but returned as a
failing deferred.
The return value may be a deferred.
"""
for socket, plugs in self.config['plugs'].items():
self.plugs[socket] = []
for plug in plugs:
entry = plug['entries']['default']
instance = reflectcall.reflectCall(entry['module-name'],
entry['function-name'],
plug)
self.plugs[socket].append(instance)
self.debug('Starting plug %r on socket %s',
instance, socket)
instance.start(self)
# Call check methods, starting from the base class and working down to
# subclasses.
checks = common.get_all_methods(self, 'do_check', False)
def checkErrorCallback(result):
# if the mood is now sad, it means an error was encountered
# during check, and we should return a failure here.
# since the checks are responsible for adding a message,
# this is a handled error.
current = self.state.get('mood')
if current == moods.sad.value:
self.warning('Running checks made the component sad.')
raise errors.ComponentSetupHandledError()
checks.append(checkErrorCallback)
return _maybeDeferredChain(checks, self)
def do_stop(self):
"""
BaseComponent vmethod for stopping.
The component should do any cleanup it needs, but must not set the
component's mood to sleeping.
@Returns: L{twisted.internet.defer.Deferred}
"""
for socket, plugs in self.plugs.items():
for plug in plugs:
self.debug('Stopping plug %r on socket %s', plug, socket)
plug.stop(self)
for message in self.state.get('messages'):
# FIXME: not necessary
self.state.remove('messages', message)
# Cancel any pending callLaters
if self._cpuPollerDC:
self._cpuPollerDC.cancel()
self._cpuPollerDC = None
if self._memoryPollerDC:
self._memoryPollerDC.cancel()
self._memoryPollerDC = None
if self._cpuPoller:
self._cpuPoller.stop()
self._cpuPoller = None
if self._memoryPoller:
self._memoryPoller.stop()
self._memoryPoller = None
if self._shutdownHook:
self.debug('_stoppedCallback: firing shutdown hook')
self._shutdownHook()
### BaseComponent implementation related to compoment protocol
def setup(self):
"""
Sets up the component. Called during __init__, so be sure not
to raise exceptions, instead adding messages to the component
state.
"""
def run_setups():
setups = common.get_all_methods(self, 'do_setup', False)
return _maybeDeferredChain(setups, self)
def setup_complete(_):
self.debug('setup completed')
self.setup_completed()
def got_error(failure):
txt = log.getFailureMessage(failure)
self.debug('got_error: %s', txt)
if not failure.check(errors.ComponentSetupHandledError):
self.warning('Setup failed: %s', txt)
m = messages.Error(T_(N_("Could not setup component.")),
debug=txt,
mid="component-setup-%s" % self.name)
# will call setMood(moods.sad)
self.addMessage(m)
# swallow
return None
self.setMood(moods.waking)
self.uiState.set('start-time', time.time())
self.uiState.set('total-memory', self._getTotalMemory())
d = run_setups()
d.addCallbacks(setup_complete, got_error)
# all status info via messages and the mood
def setup_completed(self):
self.debug('turning happy')
self.setMood(moods.happy)
def setShutdownHook(self, shutdownHook):
"""
Set the shutdown hook for this component (replacing any previous hook).
When a component is stopped, then this hook will be fired.
"""
self._shutdownHook = shutdownHook
def stop(self):
"""
Tell the component to stop.
The connection to the manager will be closed.
The job process will also finish.
"""
self.debug('BaseComponent.stop')
# Set ourselves to waking while we're shutting down.
self.setMood(moods.waking)
# Run stop methods, starting from the subclass, up to this base class.
stops = common.get_all_methods(self, 'do_stop', True)
return _maybeDeferredChain(stops, self)
### BaseComponent public methods
def getName(self):
return self.name
def setWorkerName(self, workerName):
self.state.set('workerName', workerName)
def getWorkerName(self):
return self.state.get('workerName')
def setMedium(self, medium):
assert isinstance(medium, BaseComponentMedium)
self.medium = medium
self.medium.logName = self.getName()
def setMood(self, mood):
"""
Set the given mood on the component if it's different from the current
one.
"""
current = self.state.get('mood')
if current == mood.value:
self.log('already in mood %r' % mood)
return
elif current == moods.sad.value:
self.info('tried to set mood to %r, but already sad :-(' % mood)
return
self.doLog(log.DEBUG, -2, 'MOOD changed to %r by caller', mood)
self.state.set('mood', mood.value)
if mood == moods.happy:
while self._happyWaits:
self._happyWaits.pop(0).callback(None)
elif mood == moods.sad:
while self._happyWaits:
self._happyWaits.pop(0).errback(errors.ComponentStartError())
def getMood(self):
"""
Gets the mood on the component.
@rtype: int
"""
return self.state.get('mood')
def waitForHappy(self):
mood = self.getMood()
if mood == moods.happy.value:
return defer.succeed(None)
elif mood == moods.sad.value:
return defer.fail(errors.ComponentStartError())
else:
d = defer.Deferred()
self._happyWaits.append(d)
return d
def addMessage(self, message):
"""
Add a message to the component.
If any of the messages is an error, the component will turn sad.
@type message: L{flumotion.common.messages.Message}
"""
self.state.append('messages', message)
if message.level == messages.ERROR:
self.debug('error message, turning sad')
self.setMood(moods.sad)
if self._haveError:
self._haveError(message)
def fixRenamedProperties(self, properties, list):
"""
Fix properties that have been renamed from a previous version,
and add a warning for them.
@param properties: properties; will be modified as a result.
@type properties: dict
@param list: list of (old, new) tuples of property names.
@type list: list of tuple of (str, str)
"""
found = []
for old, new in list:
if old in properties:
found.append((old, new))
if found:
m = messages.Warning(T_(N_(
"Your configuration uses deprecated properties. "
"Please update your configuration and correct them.\n")),
mid="deprecated")
for old, new in found:
m.add(T_(N_(
"Please rename '%s' to '%s'.\n"),
old, new))
self.debug("Setting new property '%s' to %r", new,
properties[old])
properties[new] = properties[old]
del properties[old]
self.addMessage(m)
def adminCallRemote(self, methodName, *args, **kwargs):
"""
Call a remote method on all admin client views on this component.
This gets serialized through the manager and multiplexed to all
admin clients, and from there on to all views connected to each
admin client model.
Because there can be any number of admin clients that this call
will go out do, it does not make sense to have one return value.
This function will return None always.
"""
if self.medium:
self.medium.callRemote("adminCallRemote", methodName,
*args, **kwargs)
else:
self.debug('asked to adminCallRemote(%s, *%r, **%r), but '
'no manager.'
% (methodName, args, kwargs))
def _pollCPU(self):
self._cpuPollerDC = None
# update CPU time stats
nowTime = time.time()
nowClock = time.clock()
deltaTime = nowTime - self._lastTime
deltaClock = nowClock - self._lastClock
self._lastTime = nowTime
self._lastClock = nowClock
# deltaClock can be < 0 if time.clock() wrapped around
if deltaClock >= 0:
CPU = deltaClock/deltaTime
self.log('latest CPU use: %r', CPU)
self.uiState.set('cpu-percent', CPU)
self.uiState.set('current-time', nowTime)
def _pollMemory(self):
self._memoryPollerDC = None
# Figure out our virtual memory size and report that.
# I don't know a nicer way to find vsize than groping /proc/
handle = open('/proc/%d/stat' % os.getpid())
line = handle.read()
handle.close()
fields = line.split()
# field 1 (comm) could potentially contain spaces and thus split over
# multiple list items, but our processes do not contain spaces
vsize = int(fields[22])
self.log('vsize is %d', vsize)
self.uiState.set('virtual-size', vsize)
def _getTotalMemory(self):
f = open("/proc/meminfo")
memtotal = f.readline()
f.close()
return int(memtotal[memtotal.index(":") + 1: -3]) * 1024
class PadMonitor(log.Loggable):
"""
I monitor data flow on a GStreamer pad.
I regularly schedule a buffer probe call at PAD_MONITOR_PROBE_INTERVAL.
I regularly schedule a check call at PAD_MONITOR_CHECK_INTERVAL
that makes sure a buffer probe was triggered since the last check call.
"""
PAD_MONITOR_PROBE_INTERVAL = 5.0
PAD_MONITOR_CHECK_INTERVAL = PAD_MONITOR_PROBE_INTERVAL * 2.5
def __init__(self, pad, name, setActive, setInactive):
"""
@type pad: L{gst.Pad}
@type name: str
@param setActive: a callable that will be called when the pad is
considered active, taking the name of the monitor.
@type setActive: callable
@param setInactive: a callable that will be called when the pad is
considered inactive, taking the name of the
monitor.
@type setInactive: callable
"""
self._last_data_time = -1 # system time in epoch secs of last reception
self._pad = pad
self.name = name
self._active = False
self._first = True
self._running = True
self._doSetActive = []
self._doSetInactive = []
self.addWatch(setActive, setInactive)
# This dict sillyness is because python's dict operations are atomic
# w.r.t. the GIL.
self._probe_id = {}
self.check_poller = Poller(self._probe_timeout,
self.PAD_MONITOR_PROBE_INTERVAL,
immediately=True)
self.watch_poller = Poller(self._check_timeout,
self.PAD_MONITOR_CHECK_INTERVAL)
def logMessage(self, message, *args):
if self._first:
self.debug(message, *args)
else:
self.log(message, *args)
def isActive(self):
return self._active
def detach(self):
self.check_poller.stop()
self.watch_poller.stop()
self._running = False
# implementation closely tied to _probe_timeout wrt to GIL
# tricks, threadsafety, and getting the probe deferred to
# actually return
d, probe_id = self._probe_id.pop("id", (None, None))
if probe_id:
self._pad.remove_buffer_probe(probe_id)
d.callback(None)
def _probe_timeout(self):
# called every so often to install a probe callback
def probe_cb(pad, buffer):
"""
Periodically scheduled buffer probe, that ensures that we're
currently actually having dataflow through our eater
elements.
Called from GStreamer threads.
@param pad: The gst.Pad srcpad for one eater in this
component.
@param buffer: A gst.Buffer that has arrived on this pad
"""
self._last_data_time = time.time()
self.logMessage('buffer probe on %s has timestamp %s', self.name,
gst.TIME_ARGS(buffer.timestamp))
deferred, probe_id = self._probe_id.pop("id", (None, None))
if probe_id:
# This will be None only if detach() has been called.
self._pad.remove_buffer_probe(probe_id)
reactor.callFromThread(deferred.callback, None)
# Data received! Return to happy ASAP:
reactor.callFromThread(self.watch_poller.run)
self._first = False
# let the buffer through
return True
d = defer.Deferred()
# FIXME: this is racy: evaluate RHS, drop GIL, buffer probe
# fires before __setitem__ in LHS; need a mutex
self._probe_id['id'] = (d, self._pad.add_buffer_probe(probe_cb))
return d
def _check_timeout(self):
# called every so often to check that a probe callback was triggered
self.log('last buffer for %s at %r', self.name, self._last_data_time)
now = time.time()
if self._last_data_time < 0:
# We never received any data in the first timeout period...
self._last_data_time = 0
self.setInactive()
elif self._last_data_time == 0:
# still no data...
pass
else:
# We received data at some time in the past.
delta = now - self._last_data_time
if self._active and delta > self.PAD_MONITOR_CHECK_INTERVAL:
self.info("No data received on pad %s for > %r seconds, "
"setting to hungry",
self.name, self.PAD_MONITOR_CHECK_INTERVAL)
self.setInactive()
elif not self._active and delta < self.PAD_MONITOR_CHECK_INTERVAL:
self.info("Receiving data again on pad %s, flow active",
self.name)
self.setActive()
def addWatch(self, setActive, setInactive):
"""
@param setActive: a callable that will be called when the pad is
considered active, taking the name of the monitor.
@type setActive: callable
@param setInactive: a callable that will be called when the pad is
considered inactive, taking the name of the
monitor.
@type setInactive: callable
"""
self._doSetActive.append(setActive)
self._doSetInactive.append(setInactive)
def setInactive(self):
self._active = False
for setInactive in self._doSetInactive:
setInactive(self.name)
def setActive(self):
self._active = True
for setActive in self._doSetActive:
setActive(self.name)
class Bouncer(component.BaseComponent):
"""
I am the base class for all bouncers.
@cvar keycardClasses: tuple of all classes of keycards this bouncer can
authenticate, in order of preference
@type keycardClasses: tuple of L{flumotion.common.keycards.Keycard}
class objects
"""
keycardClasses = ()
componentMediumClass = BouncerMedium
logCategory = 'bouncer'
KEYCARD_EXPIRE_INTERVAL = 2 * 60 # expire every 2 minutes
def init(self):
self._idCounter = 0
self._idFormat = time.strftime('%Y%m%d%H%M%S-%%d')
self._keycards = {} # keycard id -> Keycard
self._expirer = Poller(self._expire,
self.KEYCARD_EXPIRE_INTERVAL,
start=False)
self.enabled = True
def setDomain(self, name):
self.domain = name
def getDomain(self):
return self.domain
def typeAllowed(self, keycard):
"""
Verify if the keycard is an instance of a Keycard class specified
in the bouncer's keycardClasses variable.
"""
return isinstance(keycard, self.keycardClasses)
def setEnabled(self, enabled):
def callAndPassthru(result, method, *args):
method(*args)
return result
if not enabled and self.enabled:
# If we were enabled and are being set to disabled, eject the warp
# core^w^w^w^wexpire all existing keycards
self.enabled = False
self._expirer.stop()
d = self.expireAllKeycards()
d.addCallback(callAndPassthru, self.on_disabled)
return d
self.enabled = enabled
d = defer.succeed(0)
d.addCallback(callAndPassthru, self.on_enabled)
return d
def getEnabled(self):
return self.enabled
def do_stop(self):
return self.setEnabled(False)
def authenticate(self, keycard):
if not self.typeAllowed(keycard):
self.warning('keycard %r is not an allowed keycard class', keycard)
return None
if not self.do_validate(keycard):
self.warning('keycard %r is not a valid keycard instance', keycard)
return None
if self.enabled:
if not self._expirer.running and hasattr(keycard, 'ttl'):
self.debug('installing keycard timeout poller')
self._expirer.start()
return defer.maybeDeferred(self.do_authenticate, keycard)
else:
self.debug("Bouncer disabled, refusing authentication")
return None
def do_expireKeycards(self, elapsed):
"""
Override to expire keycards managed by sub-classes.
@param elapsed: time in second since the last expiration call.
@type elapsed: int
@returns: if there is more keycard to expire. If False is returned,
the expirer poller MAY be stopped.
@rtype: bool
"""
for k in self._keycards.values():
if hasattr(k, 'ttl'):
k.ttl -= elapsed
if k.ttl <= 0:
self.expireKeycardId(k.id)
return len(self._keycards) > 0
def do_validate(self, keycard):
"""
Override to check keycards before authentication steps.
Should return True if the keycard is valid, False otherwise.
#FIXME: This belong to the base bouncer class
@param keycard: the keycard that should be validated
before authentication
@type keycard: flumotion.common.keycards.Keycard
@returns: True if the keycard is accepted, False otherwise
@rtype: bool
"""
return True
def do_authenticate(self, keycard):
"""
Must be overridden by subclasses.
Authenticate the given keycard.
Return the keycard with state AUTHENTICATED to authenticate,
with state REQUESTING to continue the authentication process,
or REFUSED to deny the keycard or a deferred which should
have the same eventual value.
FIXME: Currently, a return value of 'None' is treated
as rejecting the keycard. This is unintuitive.
FIXME: in fact, for authentication sessions like challenge/response,
returning a keycard with state REFUSED instead of None
will not work properly and may enter in an asynchronous infinit loop.
"""
raise NotImplementedError("authenticate not overridden")
def on_keycardAdded(self, keycard):
"""
Override to update sub-class specific data related to keycards.
Called when the base bouncer accepts and references a new keycard.
"""
def on_keycardRemoved(self, keycard):
"""
Override to cleanup sub-class specific data related to keycards.
Called when the base bouncer has cleanup his references to a keycard.
"""
def on_enabled(self):
"""
Override to initialize sub-class specific data
when the bouncer is enabled.
"""
def on_disabled(self):
"""
Override to cleanup sub-class specific data
when the bouncer is disabled.
"""
def hasKeycard(self, keycard):
return keycard in self._keycards.values()
def generateKeycardId(self):
# FIXME: what if it already had one ?
# FIXME: deal with wraparound ?
keycardId = self._idFormat % self._idCounter
self._idCounter += 1
return keycardId
def addKeycard(self, keycard):
"""
Adds a keycard to the bouncer.
Can be called with the same keycard more than one time.
If the keycard has already been added successfully,
adding it again will succeed and return True.
@param keycard: the keycard to add.
@return: if the bouncer accepts the keycard.
"""
# give keycard an id and store it in our hash
if keycard.id in self._keycards:
# already in there
return True
keycardId = self.generateKeycardId()
keycard.id = keycardId
if hasattr(keycard, 'ttl') and keycard.ttl <= 0:
self.log('immediately expiring keycard %r', keycard)
return False
self._addKeycard(keycard)
return True
def removeKeycard(self, keycard):
if not keycard.id in self._keycards:
raise KeyError
del self._keycards[keycard.id]
self.on_keycardRemoved(keycard)
self.info("removed keycard with id %s" % keycard.id)
def removeKeycardId(self, keycardId):
self.debug("removing keycard with id %s" % keycardId)
if not keycardId in self._keycards:
raise KeyError
keycard = self._keycards[keycardId]
self.removeKeycard(keycard)
def keepAlive(self, issuerName, ttl):
for k in self._keycards.itervalues():
if hasattr(k, 'issuerName') and k.issuerName == issuerName:
k.ttl = ttl
def expireAllKeycards(self):
return self.expireKeycardIds(self._keycards.keys())
def expireKeycardId(self, keycardId):
self.log("expiring keycard with id %r", keycardId)
if not keycardId in self._keycards:
raise KeyError
keycard = self._keycards[keycardId]
self.removeKeycardId(keycardId)
if self.medium:
return self.medium.callRemote('expireKeycard',
keycard.requesterId, keycard.id)
else:
return defer.succeed(None)
def expireKeycardIds(self, keycardIds):
self.log("expiring keycards with id %r", keycardIds)
d = defer.Deferred()
self._expireNextKeycardBlock(0, keycardIds, d)
return d
def _expireNextKeycardBlock(self, total, keycardIds, finished):
keycardBlock = keycardIds[:EXPIRE_BLOCK_SIZE]
keycardIds = keycardIds[EXPIRE_BLOCK_SIZE:]
idByReq = {}
for keycardId in keycardBlock:
if keycardId in self._keycards:
keycard = self._keycards[keycardId]
requesterId = keycard.requesterId
idByReq.setdefault(requesterId, []).append(keycardId)
self.removeKeycardId(keycardId)
if not (idByReq and self.medium):
finished.callback(total)
return
defs = [self.medium.callRemote('expireKeycards', rid, ids)
for rid, ids in idByReq.items()]
dl = defer.DeferredList(defs, consumeErrors=True)
def countExpirations(results, total):
return sum([v for s, v in results if s and v]) + total
dl.addCallback(countExpirations, total)
dl.addCallback(self._expireNextKeycardBlock, keycardIds, finished)
def _addKeycard(self, keycard):
"""
Adds a keycard without checking.
Used by sub-class knowing what they do.
"""
self._keycards[keycard.id] = keycard
self.on_keycardAdded(keycard)
self.debug("added keycard with id %s, ttl %r", keycard.id,
getattr(keycard, 'ttl', None))
def _expire(self):
if not self.do_expireKeycards(self._expirer.timeout):
if self._expirer.running:
self.debug('no more keycards, removing timeout poller')
self._expirer.stop()