def _event(self, result, event_type):
if Event.exists(event_type):
Event(event_type).fire(alert=self,
severity=result[0], event=result[1],
otherInfo=result[2]
)
return self #transform the callback so the caller get's us
def __init__(self, service, name, callback, **kwargs):
"""
"""
#get ready for python3
if not hasattr(callback, '__call__'):
raise AssertionError("%s is not callable" % (callback.__name__, ))
event_name = str(service) + '-' + str(name)
self.event = Event(event_name)
self.name = name
self.loop = None
self.service = service
#borderline laziness on my part
self.log = AppManager(service).log
self.condition = kwargs.get('condition', None)
self.recurring = float(kwargs.get('recurring', 0))
self.silent = kwargs.get('silent', False)
assert not (self.recurring and self.condition), \
"recurring and condition args are mutually exclusive"
self.func = callback.__name__
if not self.silent:
self.event.subscribe(self.announce)
#send data only if we have it, keeps compatibility with legacy api
self.event.subscribe(lambda x: (hasattr(x, 'params') and x.params) \
and callback(x) or callback())
#our event loop is controlled by the Service
if self.recurring > 0.0:
self.loop = LoopingCall(self.event.fire)
self.occurred() #automatically start re-occuring events
def _setenabled(self, enabled):
enabled = bool(enabled)
status = self._getenabled()
self.info['enabled'] = enabled
if (enabled != status) and enabled:
Event('instance-enabled').fire(instance=self)
elif (enabled != status) and not enabled:
Event('instance-disabled').fire(instance=self)
def stopService(self):
Event('instance-started').unsubscribe(self.write)
Event('signal').unsubscribe(self.write)
Event('journal-error').unsubscribe(self._journal_failure)
if self._task.running: self._task.stop()
if not self.writing.called:
self.writing.addBoth(lambda x: Service.stopService(self) and x or x)
return self.writing #the main service will deal with it
Service.stopService(self)
def notification(conversation, event, string):
#hopefully you know how to parse this string
if Event.exists(event):
context = {
'conversation': conversation,
'message': string,
'event': event,
}
Event(event).fire(**context)
def startService(self):
self._task = task.LoopingCall(self.write)
#delay first journaling
config.reactor.callLater(
config.JOURNAL_FREQUENCY,
self._start_hook,
config.JOURNAL_FREQUENCY
)
#minimize the chances of losing started instances
Event('journal-error').subscribe(self._journal_failure)
Event('instance-started').subscribe(self.write)
Event('signal').subscribe(self.write)
Service.startService(self)
def blocking_journal_write(self):
if not self._task.running: return #account for lazy task start
now = int( time.time() )
snapshot = '%d.pickle' % now
path = os.path.join(config.JOURNAL_DIR, snapshot)
outfile = open(path, 'wb')
c = 0
for obj in gc.get_objects():
if isinstance(obj, Entity) and obj.serializable and \
obj.__class__.isValid(obj):
try:
data = obj.serialize()
outfile.write(data)
c += 1
except:
#don't ever reference the ``obj`` in here, it is probably
#an invalid Entity and will raise an Exception on access.
failure = err('Exception Serializing an Object')
Event('journal-error').fire(failure=failure, journal=path)
outfile.close()
plural = 's'
if c == 1: plural = ''
log('stored %d object%s' % (c,plural))
old = sorted( map(int, list_snapshots()) )[:-config.JOURNAL_RETENTION]
for timestamp in old:
path = os.path.join(config.JOURNAL_DIR, str(timestamp) + SUFFIX)
os.unlink(path)
def remove_hooks(occurrence):
"""remove hook from droned's default server resource"""
if occurrence.service.SERVICENAME not in (dependant_service, SERVICENAME):
return
global service
service = False
try:
resource = get_resource()
if SERVICENAME in resource.children:
resource.children.pop(SERVICENAME)
except: pass
Event('service-stopped').unsubscribe(remove_hooks)
if occurrence.service.SERVICENAME == 'drone':
Event('service-stopped').fire(service=__module__)
else:
Event('service-started').unsubscribe(add_hooks)
def __init__(self, service, name, callback, **kwargs):
"""
"""
#get ready for python3
if not hasattr(callback, '__call__'):
raise AssertionError("%s is not callable" % (callback.__name__,))
event_name = str(service) + '-' + str(name)
self.event = Event(event_name)
self.name = name
self.loop = None
self.service = service
#borderline laziness on my part
self.log = AppManager(service).log
self.condition = kwargs.get('condition', None)
self.recurring = float(kwargs.get('recurring', 0))
self.silent = kwargs.get('silent', False)
assert not (self.recurring and self.condition), \
"recurring and condition args are mutually exclusive"
self.func = callback.__name__
if not self.silent:
self.event.subscribe(self.announce)
#send data only if we have it, keeps compatibility with legacy api
self.event.subscribe(lambda x: (hasattr(x, 'params') and x.params) \
and callback(x) or callback())
#our event loop is controlled by the Service
if self.recurring > 0.0:
self.loop = LoopingCall(self.event.fire)
self.occurred() #automatically start re-occuring events
def connectionLost(self, xmlstream):
log('connection lost')
self.authenticated = False
if self.broadcastTask.running:
self.broadcastTask.stop()
if self.connected:
Event('jabber-offline').fire()
self.xmlstream = None
def construct(state):
conversation = Conversation(state['buddy'])
conversation.context['subscriptions'] = set(state['subscriptions'])
if state.get('authorized'):
conversation.grantAuthorization(notify=False)
for name in state['subscriptions']:
Event(name).subscribe(conversation.notify)
return conversation
def notification(conversation, event, string):
#hopefully you know how to parse this string
if Event.exists(event):
context = {
'conversation': conversation,
'message': string,
'event': event,
}
Event(event).fire(**context)
def _pluginSetup(self, plugin):
"""get romeo configuration bound in order to update the instance
This gets called by the factory below.
Note: ROMEO configuration always overrides the ApplicationPlugin's
default constructor!!!!
"""
plugin.log('applying configuration from romeo')
tmp = getattr(plugin, 'INSTANCES',
1) #allow constructor to set this up
plugin.INSTANCES = plugin.configuration.get('INSTANCES', tmp)
#get the real settings from configuration
plugin.STARTUP_INFO.update(plugin.configuration.get(
'STARTUP_INFO', {}))
plugin.STARTUP_INFO['START_ARGS'] = tuple(
plugin.STARTUP_INFO['START_ARGS'])
plugin.SHUTDOWN_INFO.update(
plugin.configuration.get('SHUTDOWN_INFO', {}))
plugin.SHUTDOWN_INFO['STOP_ARGS'] = tuple(
plugin.SHUTDOWN_INFO['STOP_ARGS'])
#how long to wait before searching for a newly started process
tmp = getattr(plugin, 'SEARCH_DELAY',
5.0) #allow constructor to set this up
plugin.SEARCH_DELAY = plugin.configuration.get('SEARCH_DELAY', tmp)
#how long to wait on a command before timing out!!! Timeout implies failure!
tmp = getattr(plugin, 'DEFAULT_TIMEOUT',
120) #allow constructor to set this up
plugin.DEFAULT_TIMEOUT = plugin.configuration.get(
'DEFAULT_TIMEOUT', tmp) #seconds
#how to assimilate an application
tmp = getattr(plugin, 'ASSIMILATION_PATTERN',
None) #allow constructor to set this up
plugin.ASSIMILATION_PATTERN = plugin.configuration.get(
'ASSIMILATION_PATTERN', tmp)
#prepare the process regular expression
plugin.PROCESS_REGEX = None
if plugin.ASSIMILATION_PATTERN:
plugin.PROCESS_REGEX = re.compile(plugin.ASSIMILATION_PATTERN,
re.I)
#if you don't like the default behavior of addInstance override it
for i in range(plugin.INSTANCES):
try:
plugin.getInstance(i)
except AssertionError:
plugin.addInstance(i)
tmp = getattr(plugin, 'AUTO_RECOVER',
False) #allow constructor to set this up
#configure automatic restart after crash
if plugin.configuration.get('AUTO_RECOVER', tmp):
Event('instance-crashed').subscribe(plugin.recoverInstance)
#allow romeo to hint that droned manages the daemonization of this app.
if plugin.configuration.get('MANAGED', False):
plugin.startProtoKwargs.update({'daemonize': True})
plugin.log('plugin is configured and ready to be used')
def subscribe(conversation, event):
subscriptions = set(conversation.context.get('subscriptions', set()))
if event == 'all':
events = Event.objects
else:
events = [Event(event)]
for event in events:
conversation.say("Subscribed to %s events" % event.name)
subscriptions.add(event.name)
event.subscribe(conversation.notify)
conversation.context['subscriptions'] = subscriptions
def assimilate(selfs, appname, appversion, applabel):
"""attempt to manage an unmanaged process"""
if self.running:
ai = AppInstance(self.server, AppVersion(App(appname), appversion),
applabel)
if ai.running: return False #this instance is already running
ai.updateInfo({'pid': self.process.pid})
if ai.running and ai.pid == self.process.pid:
Event('scab-lost').fire(scab=self)
return True
return False
def unsubscribe(conversation, event):
subscriptions = set(conversation.context.get('subscriptions', []))
if event == 'all':
eventList = Event.objects
else:
eventList = [Event(event)]
for event in eventList:
conversation.say("Unsubscribed from %s events" % event.name)
event.unsubscribe(conversation.notify)
subscriptions.discard(event.name)
conversation.context['subscriptions'] = sorted(subscriptions)
def newfunc():
try:
if obj.running():
log('Stopping Service')
func() #we don't really care about the return
if obj.running():
raise AssertionError("%s is still running" % (name, ))
Event('service-stopped').fire(service=obj)
log('Stopped Service')
except:
return Failure()
return True
def signal_emitter(self, signum, frame):
"""trap as many os signals as we can and the send the signal as an event
!!only call this right before reactor.run
"""
if signum == signal.SIGTERM:
#suppress further signals
signal.signal(signal.SIGTERM, signal.SIG_IGN)
self.log('Received SIGTERM Shutting Down in 5 seconds')
self.reactor.callLater(5.0, self.reactor.stop)
#imported below in this file
Event('signal').fire(signum=signum,
signame=self.SIGNALS[signum],
frame=frame)
def _setversion(self, version):
"""sets the self.appversion and self.version"""
checkVersion = hasattr(self, '_version') #could be reconstructing
if checkVersion:
checkVersion = self._getappversion()
if IDroneModelAppVersion.providedBy(version):
self._version = IDroneModelAppVersion(version)
else:
self._version = IDroneModelAppVersion(
AppVersion.makeAppVersion(self.app.name, version))
if checkVersion:
data = {
'instance': self,
'version': self._version,
'previous': checkVersion
}
if checkVersion < self._version:
if checkVersion.major < self._version.major:
Event('new-major-release').fire(**data)
else:
Event('new-release-version').fire(**data)
return #done
Event('release-change').fire(**data)
def connectionAuthenticated(self, xmlstream):
log('connection authenticated')
self.authenticated = True
if not self.broadcastTask.running:
self.broadcastTask.start(
self.SERVICECONFIG.JABBER_BROADCAST_INTERVAL)
xmlstream.addObserver('/message', self.receivedMessage)
xmlstream.addObserver('/presence', self.receivedPresence)
xmlstream.addObserver('/iq', self.receivedIQ)
xmlstream.addObserver('/error', self.receivedError)
Event('jabber-online').fire()
while self.sendQueue:
self.xmlstream.send(self.sendQueue.pop(0))
def newfunc():
try:
if not self.SERVICE_STATE[name]:
raise AssertionError('%s is disabled' % (name, ))
if not obj.running():
log('Starting Service')
func() #don't really care about the return
if not obj.running():
raise AssertionError("%s not running" % (name, ))
Event('service-started').fire(service=obj)
log('Started Service')
except:
return Failure()
return True
def stopService(self):
"""Stop All AppManager Services"""
for x in ['allapps', 'applist']:
if x in drone.builtins:
del drone.builtins[x]
Event('instance-started').unsubscribe(self.reset_tracking)
for manager in AppManager.objects:
if manager.running:
mesg = 'Stopping Application Manager'
logWithContext(type=manager.name, route=SERVICENAME)(mesg)
manager.stop()
#plugins are stateless by design
pluginFactory.delete_plugin(manager.model)
if self._task.running: self._task.stop()
Service.stopService(self)
def startService(self):
"""Start All AppManager Services"""
if self.scanning.called: #need to pre-populate values
self.scanning = defer.maybeDeferred(self._first_scan)
self.first_run = True
self._task = task.LoopingCall(self.scan_app_instances)
#plugins will be created and loaded when needed
for shortname in config.APPLICATIONS.keys():
manager = None
try:
applog = logWithContext(type=shortname, route=SERVICENAME)
applog('Loading Application Plugin')
applog('Creating Application Manager')
manager = AppManager(shortname)
manager.parentService = self
#check and see if the model is bound
if not AppManager(shortname).running:
applog('Starting Application Manager')
manager.start()
except:
failure = Failure()
#bad plugin, not adaptable
failures = (InvalidPlugin, TypeError)
if failure.check(*failures) and manager:
log('plugin for %s is invalid' % (manager.name, ))
manager.action.__class__.delete(manager.action)
try:
pluginFactory.delete_plugin(manager.model)
except:
pass #silence
AppManager.delete(manager)
if not config.EXCESSIVE_LOGGING:
continue #avoid extra logging
try:
failure.raiseException()
except:
crashReport('ApplicationLoader', self)
Service.startService(self)
Event('instance-started').subscribe(self.reset_tracking)
#wire allapps action into the server
drone.builtins.update({
'allapps': self.allapps_action,
'applist': self.applist_action,
})
#delay scanning by some interval
config.reactor.callLater(SERVICECONFIG.initial_delay,
self._start_all_tasks)
event.unsubscribe(conversation.notify)
def joinEnvironmentalChatRoom(event):
"""determine if we should join a chatroom"""
chat = ChatRoom(config.ROMEO_ENV_NAME)
#make sure the drone can be managed by the room
username = config.ROMEO_ENV_NAME
jbserver = jconfig.JABBER_CHAT_SERVICE
jid = "%(username)s@%(jbserver)s" % locals()
Team('support').addMember(jid)
#get the conversation context set to some sane defaults
conversation = Conversation(jid)
#grant the room access to the server
if jconfig.JABBER_TRUST_ROOM:
conversation.grantAuthorization(notify=False)
#be vain assume all conversations revolve around ourself
context = {
'server': Server(config.HOSTNAME),
'subject': Server(config.HOSTNAME),
}
conversation.context.update(context)
#finally join the room
chat.join()
if jconfig and jconfig.JABBER_JOIN_CHATROOM:
Event('jabber-online').subscribe(joinEnvironmentalChatRoom)
Event('jabber-online').subscribe(notify_online)
Event('jabber-offline').subscribe(remove_conversation_subscriptions)
def _scan(self):
for pid in listProcesses():
try:
AppProcess(Server(config.HOSTNAME), pid)
except InvalidProcess:
pass
except IOError:
pass #happens on linux when a pid dies
except:
err('process table scanning error')
#scan for crashed instances
for app in App.objects:
if not app.__class__.isValid(app): continue
for ai in app.localappinstances:
if not ai.__class__.isValid(ai): continue
if ai in self.tracking: continue
if not self.first_run: #avoid process table races
self.tracking.add(ai)
config.reactor.callLater(SERVICECONFIG.recovery_period,
self.tracking.discard, ai)
if not ai.enabled: continue #skip disabled
result = None
if ai.running and not ai.shouldBeRunning:
ai.shouldBeRunning = True
if ai.running: continue
manager = AppManager(ai.app.name)
if not manager.running:
continue #skip managers that are not running
if not manager.discover:
continue #app manager claims to be ok
#look for processes that we can assimilate
d = manager.model.findProcesses()
wfd = defer.waitForDeferred(d)
yield wfd
for (pid, result) in wfd.getResult():
d = manager.model.assimilateProcess(result)
wfd2 = defer.waitForDeferred(d)
yield wfd2
ai2 = wfd2.getResult()
if ai2 and isinstance(ai2, AppInstance) and ai2 is ai:
Event('instance-found').fire(instance=ai)
manager.log('Sucessfully assimilated PID %d' % ai2.pid)
if ai.running: continue #may have assimilated the app
if not ai.crashed: continue
if not ai.enabled:
continue #disabled instances are not actionable
if self.first_run: continue #avoid process table races
Event('instance-crashed').fire(instance=ai)
#cool off on eventing for a little while
#keep the process objects up to date
for process in AppProcess.objects:
try:
if not process.localInstall: continue
if not process.running:
AppProcess.delete(process)
except:
err('process book keeping error')
d = defer.Deferred()
config.reactor.callLater(0.01, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
self.first_run = False
yield 'done'
def stopInstance(self, label):
"""Stops an application instance by label
@param label: (string)
@fires Event('instance-stopped')
return defer.Deferred()
"""
result = {}
template = '[%(application)s,%(label)s] %(description)s'
context = {'code': 254}
thisInst = None
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = False
if not thisInst.running:
context.update(self.model.statusInstance(label))
raise DroneCommandFailed(context)
pid = thisInst.process.pid
self.log("Trying to shutdown %d gracefully" % (pid, ))
def failed(result):
"""attempting to be consistant"""
self.log("Failed to shutdown process gracefully")
return result
def success(result):
"""attempting to be consistant"""
self.log("process %d gracefully shutdown" % (pid, ))
return result
d = self._start_stop_common(label, 'stopInstance')
d.addCallback(success)
d.addErrback(failed)
d.addErrback(self._killInstance, thisInst)
wfd = defer.waitForDeferred(d)
yield wfd
#refresh the instance as it can change
thisInst = self.model.getInstance(label)
result = wfd.getResult()
if isinstance(result, dict):
context.update(result)
elif isinstance(result, DroneCommandFailed):
context.update(result.resultContext)
if not thisInst.running:
context['code'] = 0
Event('instance-stopped').fire(instance=thisInst)
raise AssertionError('ignore me')
raise DroneCommandFailed(context)
except AssertionError:
#update the instance model
wfd = defer.waitForDeferred(self.statusInstance(label))
yield wfd
result = wfd.getResult()
result['code'] = context['code']
except:
failure = Failure()
if failure.check(DroneCommandFailed):
context = failure.value.resultContext
template = '%(description)s'
else:
temp = "%s: %s" % (getException(failure),
failure.getErrorMessage())
context = {'error': failure, 'code': 253, 'description': temp}
result = self.resultContext(template, thisInst, **context)
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = False
except:
pass
yield result
def __del__(self):
if self.loop.running: self.loop.stop()
try: Event.delete(self.event)
except: pass
class ApplicationEvent(Entity):
"""The Extents Eventing For AppManagers"""
implements(IDroneModelApplicationEvent)
serializable = False
reactor = property(lambda s: reactor)
def __del__(self):
if self.loop.running: self.loop.stop()
try: Event.delete(self.event)
except: pass
#TODO document
def __init__(self, service, name, callback, **kwargs):
"""
"""
#get ready for python3
if not hasattr(callback, '__call__'):
raise AssertionError("%s is not callable" % (callback.__name__,))
event_name = str(service) + '-' + str(name)
self.event = Event(event_name)
self.name = name
self.loop = None
self.service = service
#borderline laziness on my part
self.log = AppManager(service).log
self.condition = kwargs.get('condition', None)
self.recurring = float(kwargs.get('recurring', 0))
self.silent = kwargs.get('silent', False)
assert not (self.recurring and self.condition), \
"recurring and condition args are mutually exclusive"
self.func = callback.__name__
if not self.silent:
self.event.subscribe(self.announce)
#send data only if we have it, keeps compatibility with legacy api
self.event.subscribe(lambda x: (hasattr(x, 'params') and x.params) \
and callback(x) or callback())
#our event loop is controlled by the Service
if self.recurring > 0.0:
self.loop = LoopingCall(self.event.fire)
self.occurred() #automatically start re-occuring events
def announce(self, *args, **kargs):
"""Anounce the Occurrence of an Event"""
self.log("%s event occurred calling %s" % (self.name, self.func))
return True
def occurred(self):
"""Check for Occurrence or Start Event Loop"""
if not self.condition and self.loop and not self.loop.running:
self.loop.start(self.recurring)
elif self.condition and self.condition():
self.event.fire()
return True
return False
#TODO document
def trigger(self, data=None, delay=0.0):
"""Trigger an event, the return object can be cancelled"""
kwargs = {}
if data:
kwargs['data'] = data
return self.reactor.callLater(delay, self.event.fire, **kwargs)
def start():
global service
service = True
Event('service-started').subscribe(add_hooks)
Event('service-stopped').subscribe(remove_hooks)
class ApplicationEvent(Entity):
"""The Extents Eventing For AppManagers"""
implements(IDroneModelApplicationEvent)
serializable = False
reactor = property(lambda s: config.reactor)
def __del__(self):
if self.loop.running: self.loop.stop()
try:
Event.delete(self.event)
except:
pass
#TODO document
def __init__(self, service, name, callback, **kwargs):
"""
"""
#get ready for python3
if not hasattr(callback, '__call__'):
raise AssertionError("%s is not callable" % (callback.__name__, ))
event_name = str(service) + '-' + str(name)
self.event = Event(event_name)
self.name = name
self.loop = None
self.service = service
#borderline laziness on my part
self.log = AppManager(service).log
self.condition = kwargs.get('condition', None)
self.recurring = float(kwargs.get('recurring', 0))
self.silent = kwargs.get('silent', False)
assert not (self.recurring and self.condition), \
"recurring and condition args are mutually exclusive"
self.func = callback.__name__
if not self.silent:
self.event.subscribe(self.announce)
#send data only if we have it, keeps compatibility with legacy api
self.event.subscribe(lambda x: (hasattr(x, 'params') and x.params) \
and callback(x) or callback())
#our event loop is controlled by the Service
if self.recurring > 0.0:
self.loop = LoopingCall(self.event.fire)
self.occurred() #automatically start re-occuring events
def announce(self, *args, **kargs):
"""Anounce the Occurrence of an Event"""
self.log("%s event occurred calling %s" % (self.name, self.func))
return True
def occurred(self):
"""Check for Occurrence or Start Event Loop"""
if not self.condition and self.loop and not self.loop.running:
self.loop.start(self.recurring)
elif self.condition and self.condition():
self.event.fire()
return True
return False
#TODO document
def trigger(self, data=None, delay=0.0):
"""Trigger an event, the return object can be cancelled"""
kwargs = {}
if data:
kwargs['data'] = data
return self.reactor.callLater(delay, self.event.fire, **kwargs)
def __init__(self, *args, **kwargs):
Event('instance-found').subscribe(self._check_version)
def startInstance(self, label):
"""Starts an application instance by label
@param label: (string)
@fires Event('instance-started')
return defer.Deferred()
"""
template = '[%(application)s,%(label)s] %(description)s'
context = {'description': 'Failed to Start', 'code': 254}
result = {}
thisInst = None
try:
if self.model.getInstance(label).running:
context.update(self.model.statusInstance(label))
raise DroneCommandFailed(context)
d = self._start_stop_common(label, 'startInstance')
wfd = defer.waitForDeferred(d)
yield wfd
result = wfd.getResult()
d = self.statusInstance(label)
wfd = defer.waitForDeferred(d)
yield wfd
result.update(wfd.getResult())
#refresh the instance as it can change
thisInst = self.model.getInstance(label)
if isinstance(result, dict):
context.update(result)
elif isinstance(result, DroneCommandFailed):
context.update(result.resultContext)
if thisInst.running:
Event('instance-started').fire(instance=thisInst)
context['code'] = 0
raise AssertionError('ignore')
raise DroneCommandFailed(context)
except AssertionError:
#update the instance model
wfd = defer.waitForDeferred(self.statusInstance(label))
yield wfd
result = wfd.getResult()
except:
thisInst = self.model.getInstance(label)
failure = Failure()
if failure.check(DroneCommandFailed):
template = '%(description)s'
context = failure.value.resultContext
else:
#log the error, allowing for debugging
self.debugReport()
#be nice and return something to the end user
temp = "%s: %s" % (getException(failure),
failure.getErrorMessage())
context = {'error': failure, 'code': 253, 'description': temp}
result = self.resultContext(template, thisInst, **context)
try:
thisInst = self.model.getInstance(label)
thisInst.shouldBeRunning = True
except:
pass
yield result
def doesNotRunOn(self, server):
if server in self.shouldRunOn:
self.shouldRunOn.remove(server)
Event('app-servers-change').fire(app=self,
server=server,
change='removed')
def __del__(self):
if self.loop.running: self.loop.stop()
try:
Event.delete(self.event)
except:
pass