def _borg(self):
"""assimilation routine"""
server = Server(config.HOSTNAME)
#wait for scanning to finish
wfd = defer.waitForDeferred(self.scanning)
yield wfd
try: wfd.getResult()
except: err('scanning had an error')
for manager in AppManager.objects:
if not manager.running:
continue #skip managers that are not running
if not manager.discover:
continue #appmanager is happy, no need to look
d = manager.model.findProcesses()
wfd = defer.waitForDeferred(d)
yield wfd
for (pid, result) in wfd.getResult():
d = manager.model.assimilateProcess(result)
wfd2 = defer.waitForDeferred(d)
yield wfd2
ai = wfd2.getResult()
if ai and isinstance(ai, AppInstance):
ai.pid = int(result['pid']) #just make sure!!!
Event('instance-found').fire(instance=ai)
manager.log('Sucessfully assimilated PID %(pid)d' % result)
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
yield 'done'
def _scan(self):
for pid in listProcesses():
try: AppProcess(Server(config.HOSTNAME), pid)
except InvalidProcess: pass
except IOError: pass #happens on linux when a pid dies
except: err('process table scanning error')
#scan for crashed instances
for app in App.objects:
if not app.__class__.isValid(app): continue
for ai in app.localappinstances:
if not ai.__class__.isValid(ai): continue
if ai in self.tracking: continue
if not self.first_run: #avoid process table races
self.tracking.add(ai)
config.reactor.callLater(SERVICECONFIG.recovery_period,
self.tracking.discard, ai
)
if not ai.enabled: continue #skip disabled
result = None
if ai.running and not ai.shouldBeRunning:
ai.shouldBeRunning = True
if ai.running: continue
manager = AppManager(ai.app.name)
if not manager.running:
continue #skip managers that are not running
if not manager.discover:
continue #app manager claims to be ok
#look for processes that we can assimilate
d = manager.model.findProcesses()
wfd = defer.waitForDeferred(d)
yield wfd
for (pid, result) in wfd.getResult():
d = manager.model.assimilateProcess(result)
wfd2 = defer.waitForDeferred(d)
yield wfd2
ai2 = wfd2.getResult()
if ai2 and isinstance(ai2, AppInstance) and ai2 is ai:
Event('instance-found').fire(instance=ai)
manager.log('Sucessfully assimilated PID %d' % ai2.pid)
if ai.running: continue #may have assimilated the app
if not ai.crashed: continue
if not ai.enabled: continue #disabled instances are not actionable
if self.first_run: continue #avoid process table races
Event('instance-crashed').fire(instance=ai)
#cool off on eventing for a little while
#keep the process objects up to date
for process in AppProcess.objects:
try:
if not process.localInstall: continue
if not process.running:
AppProcess.delete(process)
except:
err('process book keeping error')
d = defer.Deferred()
config.reactor.callLater(0.01, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
self.first_run = False
yield 'done'
def _updateVersion(self, result, label):
thisInst = self.getInstance(label)
try:
ts = rpm.TransactionSet()
mi = ts.dbMatch('name', self.name)
for h in mi:
if not h: break
version = AppVersion.makeVersion(self.name, h['version'])
if version > thisInst.appversion:
thisInst.version = version
except:
err('exception while setting version')
return result
def _updateVersion(self, result, label):
thisInst = self.getInstance(label)
try:
ts = rpm.TransactionSet()
mi = ts.dbMatch('name', self.name)
for h in mi:
if not h: break
version = AppVersion.makeVersion(self.name, h['version'])
if version > thisInst.appversion:
thisInst.version = version
except:
err('exception while setting version')
return result
def findProcesses(self):
"""Attempt to find a process by an ASSIMILATION pattern.
This is a relatively naive attempt to find an application
that works in most cases.
NOTE:
If your ASSIMILATION pattern includes group matches the
dictionary will be updated with the output of
``groupdict() from re.search``. If droned is able to
read the environment settings from the application that
will be inlcuded in the result dictionary as well.
@callback (list) - sorted([(int('PID'), dict), ...])
@errback (twisted.python.failure.Failure())
@return defer.Deferred()
"""
candidates = {}
def safe_process(pid):
try: #because processes can be invalid
return AppProcess(Server(config.HOSTNAME), pid)
except:
err('problem')
return None
if self.PROCESS_REGEX:
#rescan the whole system for processes
for process in (safe_process(pid) for pid in listProcesses()):
try:
if not process: continue
if not process.__class__.isValid(process): continue
if process.pid in candidates: continue
if process.ppid != 1: continue #droned wants your daemons
if not process.running: continue
if not process.localInstall: continue
if process.managed: continue #already managed
cmd = ' '.join(process.cmdline)
if not cmd: continue
match = self.PROCESS_REGEX.search(cmd)
if not match: continue
#remember we tried to set some VARS on startInstance
_result = dict(**process.environ)
#allows us to set interesting parameters in the regex
_result.update(match.groupdict())
_result.update({'pid': process.pid})
candidates[_result['pid']] = _result
except:
self.log('error searching for process check console log')
err('error searching for process')
return sorted([(pid, d) for (pid, d) in candidates.items() if pid > 1])
def findProcesses(self):
"""Attempt to find a process by an ASSIMILATION pattern.
This is a relatively naive attempt to find an application
that works in most cases.
NOTE:
If your ASSIMILATION pattern includes group matches the
dictionary will be updated with the output of
``groupdict() from re.search``. If droned is able to
read the environment settings from the application that
will be inlcuded in the result dictionary as well.
@callback (list) - sorted([(int('PID'), dict), ...])
@errback (twisted.python.failure.Failure())
@return defer.Deferred()
"""
candidates = {}
def safe_process(pid):
try: #because processes can be invalid
return AppProcess(Server(config.HOSTNAME), pid)
except:
err('problem')
return None
if self.PROCESS_REGEX:
#rescan the whole system for processes
for process in ( safe_process(pid) for pid in listProcesses() ):
try:
if not process: continue
if not process.__class__.isValid(process): continue
if process.pid in candidates: continue
if process.ppid != 1: continue #droned wants your daemons
if not process.running: continue
if not process.localInstall: continue
if process.managed: continue #already managed
cmd = ' '.join(process.cmdline)
if not cmd: continue
match = self.PROCESS_REGEX.search(cmd)
if not match: continue
#remember we tried to set some VARS on startInstance
_result = dict(**process.environ)
#allows us to set interesting parameters in the regex
_result.update(match.groupdict())
_result.update({'pid': process.pid})
candidates[_result['pid']] = _result
except:
self.log('error searching for process check console log')
err('error searching for process')
return sorted([(pid, d) for (pid, d) in candidates.items() if pid > 1])
def startService(self):
self.handlers = []
stat_handlers = StatBlockLoader.load()
for STAT in SERVICECONFIG.STATS:
STAT_BLOCK = STAT['STAT']
try:
for sh in stat_handlers:
if sh.TYPE == STAT_BLOCK['TYPE']:
inst = sh(STAT_BLOCK)
self.handlers.append((STAT_BLOCK,inst))
except:
f = Failure()
err("Error while initializing systemstats service.")
self.setup_actions()
Service.startService(self)
def blocking_journal_write(self):
if not self._task.running: return #account for lazy task start
now = int( time.time() )
snapshot = '%d.pickle' % now
path = os.path.join(config.JOURNAL_DIR, snapshot)
outfile = open(path, 'wb')
c = 0
for obj in gc.get_objects():
if isinstance(obj, Entity) and obj.serializable and \
obj.__class__.isValid(obj):
try:
data = obj.serialize()
outfile.write(data)
c += 1
except:
#don't ever reference the ``obj`` in here, it is probably
#an invalid Entity and will raise an Exception on access.
failure = err('Exception Serializing an Object')
Event('journal-error').fire(failure=failure, journal=path)
outfile.close()
plural = 's'
if c == 1: plural = ''
log('stored %d object%s' % (c,plural))
old = sorted( map(int, list_snapshots()) )[:-config.JOURNAL_RETENTION]
for timestamp in old:
path = os.path.join(config.JOURNAL_DIR, str(timestamp) + SUFFIX)
os.unlink(path)
def fire(self, **params):
if not self.enabled: return
occurrence = Occurrence(self, **params)
if config.DEBUG_EVENTS:
params = ', '.join("%s=%s" % i for i in params.items())
log('%s.fire(%s)' % (self, params))
for obj in list(self.subscribers):
try:
if isinstance(obj, Deferred):
if not obj.called:
obj.callback(occurrence)
self.subscribers.remove(obj)
else:
obj(occurrence)
except:
log('%s.fire() subscriber %s raised an exception' % (self, obj), error=True, failure=Failure())
err()
def load():
global temporary_storage
snapshots = sorted( map(int, list_snapshots()) )
if not snapshots: return
path = os.path.join(config.JOURNAL_DIR, str(snapshots[-1]) + SUFFIX)
timestamp = time.ctime(snapshots[-1])
log('loading %s (%s)' % (path,timestamp))
journal = open(path, 'rb')
c = 0
while True:
try:
temporary_storage.append(Entity.deserialize(journal))
except EOFError:
journal.close()
break
except: err('problem unspooling')
c += 1
log('loaded %d objects' % c)
def fire(self, **params):
if not self.enabled:
return
occurrence = Occurrence(self, **params)
if config.DEBUG_EVENTS:
params = ", ".join("%s=%s" % i for i in params.items())
log("%s.fire(%s)" % (self, params))
for obj in list(self.subscribers):
try:
if isinstance(obj, Deferred):
if not obj.called:
obj.callback(occurrence)
self.subscribers.remove(obj)
else:
obj(occurrence)
except:
log("%s.fire() subscriber %s raised an exception" % (self, obj), error=True, failure=Failure())
err()
def load():
snapshots = sorted(map(int, list_snapshots()))
if not snapshots:
return
path = os.path.join(config.JOURNAL_DIR, str(snapshots[-1]) + SUFFIX)
timestamp = time.ctime(snapshots[-1])
log("loading %s (%s)" % (path, timestamp))
journal = open(path, "rb")
c = 0
while True:
try:
obj = Entity.deserialize(journal)
except EOFError:
journal.close()
break
except:
err("problem unspooling")
c += 1
log("loaded %d objects" % c)
def loadAppPlugins(self):
"""load all of the application plugins"""
#find all application plugins
my_dir = os.path.dirname(__file__)
for filename in os.listdir(my_dir):
if not filename.endswith('.py'): continue
if filename == '__init__.py': continue
modname = filename[:-3]
try:
mod = __import__(__name__ + '.' + modname, {}, {}, [modname])
except:
err('Application Plugin Loader Caught Exception')
continue #horribly broken module ... skipping
for name,obj in vars(mod).items():
if name in self._classMap: continue
try:
#need the interfaces to be what we expect
if IDroneDApplication.implementedBy(obj):
self._classMap[name] = copy.deepcopy(obj)
except TypeError: pass
except:
err('Application Plugin Scanner Caught Exception')
def loadAppPlugins(self):
"""load all of the application plugins"""
#find all application plugins
my_dir = os.path.dirname(__file__)
for filename in os.listdir(my_dir):
if not filename.endswith('.py'): continue
if filename == '__init__.py': continue
modname = filename[:-3]
try:
mod = __import__(__name__ + '.' + modname, {}, {}, [modname])
except:
err('Application Plugin Loader Caught Exception')
continue #horribly broken module ... skipping
for name, obj in vars(mod).items():
if name in self._classMap: continue
try:
#need the interfaces to be what we expect
if IDroneDApplication.implementedBy(obj):
self._classMap[name] = copy.deepcopy(obj)
except TypeError:
pass
except:
err('Application Plugin Scanner Caught Exception')
def __init__(self, server, pid):
self._pid = pid
self.server = IDroneModelServer(server)
self._created = time.time()
#don't set self._process
try:
try: #re-constructing, can cause problems with this
if IKittNullProcess.providedBy(self.process.process):
raise InvalidProcess("Invalid PID (%s)" % pid)
except AttributeError:
if isinstance(self.process, NullProcess):
raise InvalidProcess("Invalid PID (%s)" % pid)
raise #re-raise do avoid ending up in a pickle, literally
except InvalidProcess:
AppProcess.delete(self) #make sure we are invalid
raise InvalidProcess("Invalid PID (%s)" % pid)
except IOError: #linux and solaris kitt.proc.LiveProcess use files
AppProcess.delete(self) #make sure we are invalid
raise InvalidProcess("Invalid PID (%s)" % pid)
except:
err('wtf happened here .. seriously i do not know!!!')
AppProcess.delete(self) #make sure we are invalid
raise
def loadAll():
"""loads all of the responders"""
responders.clear()
my_dir = os.path.dirname(__file__)
for filename in os.listdir(my_dir):
if not filename.endswith('.py'): continue
if filename == '__init__.py': continue
modname = filename[:-3]
log('Loading responder module %s' % modname)
try:
#python2.4 __import__ implementation doesn't accept **kwargs
mod = __import__(__name__ + '.' + modname, {}, {}, [modname])
except:
err('skipping responder module %s due to errors' % modname)
continue
for name,obj in vars(mod).items():
if getattr(obj, 'is_responder', False) and hasattr(obj, 'pattern'):
try:
responders[ re.compile(obj.pattern) ] = obj
obj.module = modname
except:
err("Failed to compile pattern \"%s\" for %s" % (obj.pattern, obj))
raise
def receivedPresence(self, e):
log('received presence: %s' % e.toXml())
if e.getAttribute('type') == 'subscribe':
log('received authorization request from %s' % e['from'])
response = Element( ('','presence') )
response['to'] = e['from']
response['type'] = 'subscribed'
log('sending auth response: %s' % response.toXml())
self.xmlstream.send(response)
buddy = str(e['from'])
if not Conversation.exists(buddy):
self.requestAuthorization(buddy)
elif e.getAttribute('type') == 'unavailable':
#fix for openfire jabber server randomly kicking clients out and prevent kicks
CHAT = '@%s/%s' % (self.SERVICECONFIG.JABBER_CHAT_SERVICE,self.SERVICECONFIG.JABBER_CHAT_NICK)
if e['to'] == self.jid.full() and e['from'].endswith(CHAT) and \
"status code='307'" in e.toXml():
try:
log('%s has kicked me' % (e['from'],))
self.joinChatRoom(e['from'].split(CHAT)[0])
log('successfully rejoined room')
except:
err('Failed to recover from /kick')
def __init__(self, server, pid):
self._pid = pid
self.server = IDroneModelServer(server)
self._created = time.time()
#don't set self._process
try:
try: #re-constructing, can cause problems with this
if IKittNullProcess.providedBy(self.process.process):
raise InvalidProcess("Invalid PID (%s)" % pid)
except AttributeError:
if isinstance(self.process, NullProcess):
raise InvalidProcess("Invalid PID (%s)" % pid)
raise #re-raise do avoid ending up in a pickle, literally
except InvalidProcess:
if config.HOSTNAME == self.server.hostname:
AppProcess.delete(self) #make sure we are invalid
raise InvalidProcess("Invalid PID (%s)" % pid)
except IOError: #linux and solaris kitt.proc.LiveProcess use files
AppProcess.delete(self) #make sure we are invalid
raise InvalidProcess("Invalid PID (%s)" % pid)
except:
err('wtf happened here .. seriously i do not know!!!')
AppProcess.delete(self) #make sure we are invalid
raise
def loadAll():
"""loads all of the responders"""
responders.clear()
my_dir = os.path.dirname(__file__)
for filename in os.listdir(my_dir):
if not filename.endswith('.py'): continue
if filename == '__init__.py': continue
modname = filename[:-3]
log('Loading responder module %s' % modname)
try:
#python2.4 __import__ implementation doesn't accept **kwargs
mod = __import__(__name__ + '.' + modname, {}, {}, [modname])
except:
err('skipping responder module %s due to errors' % modname)
continue
for name, obj in vars(mod).items():
if getattr(obj, 'is_responder', False) and hasattr(obj, 'pattern'):
try:
responders[re.compile(obj.pattern)] = obj
obj.module = modname
except:
err("Failed to compile pattern \"%s\" for %s" %
(obj.pattern, obj))
raise
def receivedPresence(self, e):
log('received presence: %s' % e.toXml())
if e.getAttribute('type') == 'subscribe':
log('received authorization request from %s' % e['from'])
response = Element(('', 'presence'))
response['to'] = e['from']
response['type'] = 'subscribed'
log('sending auth response: %s' % response.toXml())
self.xmlstream.send(response)
buddy = str(e['from'])
if not Conversation.exists(buddy):
self.requestAuthorization(buddy)
elif e.getAttribute('type') == 'unavailable':
#fix for openfire jabber server randomly kicking clients out and prevent kicks
CHAT = '@%s/%s' % (self.SERVICECONFIG.JABBER_CHAT_SERVICE,
self.SERVICECONFIG.JABBER_CHAT_NICK)
if e['to'] == self.jid.full() and e['from'].endswith(CHAT) and \
"status code='307'" in e.toXml():
try:
log('%s has kicked me' % (e['from'], ))
self.joinChatRoom(e['from'].split(CHAT)[0])
log('successfully rejoined room')
except:
err('Failed to recover from /kick')
def _scan(self):
#wait for assimilation to finish
wfd = defer.waitForDeferred(self.assimilating)
yield wfd
try: wfd.getResult()
except: err('assimilation had an error')
#scan for all processes, hopefully we have seen them before
for pid in listProcesses():
try: AppProcess(Server(config.HOSTNAME), pid)
except InvalidProcess: pass
except IOError: pass #happens on linux when a pid dies
except: err('process table scanning error')
#scan for crashed instances
for app in App.objects:
if not app.__class__.isValid(app): continue
for ai in app.localappinstances:
if not ai.__class__.isValid(ai): continue
if ai.running and not ai.shouldBeRunning:
ai.shouldBeRunning = True
continue #work around first assimilated/never started
if not ai.crashed: continue
if not ai.enabled: continue #disabled instances are not actionable
if ai in self.tracking: continue
Event('instance-crashed').fire(instance=ai)
#cool off on eventing for a little while
self.tracking.add(ai)
reactor.callLater(SERVICECONFIG.recovery_period,
self.tracking.discard, ai
)
#keep the process objects up to date
for process in AppProcess.objects:
try:
if not process.localInstall: continue
if not process.running:
AppProcess.delete(process)
except:
err('process book keeping error')
d = defer.Deferred()
reactor.callLater(0.01, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
yield 'done'
def safe_process(pid):
try: #because processes can be invalid
return AppProcess(Server(config.HOSTNAME), pid)
except:
err('problem')
return None
def _scan(self):
for pid in listProcesses():
try:
AppProcess(Server(config.HOSTNAME), pid)
except InvalidProcess:
pass
except IOError:
pass #happens on linux when a pid dies
except:
err('process table scanning error')
#scan for crashed instances
for app in App.objects:
if not app.__class__.isValid(app): continue
for ai in app.localappinstances:
if not ai.__class__.isValid(ai): continue
if ai in self.tracking: continue
if not self.first_run: #avoid process table races
self.tracking.add(ai)
config.reactor.callLater(SERVICECONFIG.recovery_period,
self.tracking.discard, ai)
if not ai.enabled: continue #skip disabled
result = None
if ai.running and not ai.shouldBeRunning:
ai.shouldBeRunning = True
if ai.running: continue
manager = AppManager(ai.app.name)
if not manager.running:
continue #skip managers that are not running
if not manager.discover:
continue #app manager claims to be ok
#look for processes that we can assimilate
d = manager.model.findProcesses()
wfd = defer.waitForDeferred(d)
yield wfd
for (pid, result) in wfd.getResult():
d = manager.model.assimilateProcess(result)
wfd2 = defer.waitForDeferred(d)
yield wfd2
ai2 = wfd2.getResult()
if ai2 and isinstance(ai2, AppInstance) and ai2 is ai:
Event('instance-found').fire(instance=ai)
manager.log('Sucessfully assimilated PID %d' % ai2.pid)
if ai.running: continue #may have assimilated the app
if not ai.crashed: continue
if not ai.enabled:
continue #disabled instances are not actionable
if self.first_run: continue #avoid process table races
Event('instance-crashed').fire(instance=ai)
#cool off on eventing for a little while
#keep the process objects up to date
for process in AppProcess.objects:
try:
if not process.localInstall: continue
if not process.running:
AppProcess.delete(process)
except:
err('process book keeping error')
d = defer.Deferred()
config.reactor.callLater(0.01, d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
self.first_run = False
yield 'done'
def safe_process(pid):
try: #because processes can be invalid
return AppProcess(Server(config.HOSTNAME), pid)
except:
err('problem')
return None
