def log(self, message, label=None): """route logging messages to the application log and allow for custom labeling to be applied @param message: (string) @param label: (string) or (None) @return None """ info = self.name if label: info += ',%(label)s' % locals() logWithContext(type=info, route='application')(message)
def stopService(self): """Stop All AppManager Services""" for x in ['allapps', 'applist']: if x in drone.builtins: del drone.builtins[x] Event('instance-started').unsubscribe(self.reset_tracking) for manager in AppManager.objects: if manager.running: mesg = 'Stopping Application Manager' logWithContext(type=manager.name,route=SERVICENAME)(mesg) manager.stop() #plugins are stateless by design pluginFactory.delete_plugin(manager.model) if self._task.running: self._task.stop() Service.stopService(self)
def stopService(self): """Stop All AppManager Services""" for x in ['allapps', 'applist']: if x in drone.builtins: del drone.builtins[x] Event('instance-started').unsubscribe(self.reset_tracking) for manager in AppManager.objects: if manager.running: mesg = 'Stopping Application Manager' logWithContext(type=manager.name, route=SERVICENAME)(mesg) manager.stop() #plugins are stateless by design pluginFactory.delete_plugin(manager.model) if self._task.running: self._task.stop() Service.stopService(self)
def startService(self): """Start All AppManager Services""" if self.scanning.called: #need to pre-populate values self.scanning = defer.maybeDeferred(self._first_scan) self.first_run = True self._task = task.LoopingCall(self.scan_app_instances) #plugins will be created and loaded when needed for shortname in config.APPLICATIONS.keys(): manager = None try: applog = logWithContext(type=shortname, route=SERVICENAME) applog('Loading Application Plugin') applog('Creating Application Manager') manager = AppManager(shortname) manager.parentService = self #check and see if the model is bound if not AppManager(shortname).running: applog('Starting Application Manager') manager.start() except: failure = Failure() #bad plugin, not adaptable failures = (InvalidPlugin, TypeError) if failure.check(*failures) and manager: log('plugin for %s is invalid' % (manager.name, )) manager.action.__class__.delete(manager.action) try: pluginFactory.delete_plugin(manager.model) except: pass #silence AppManager.delete(manager) if not config.EXCESSIVE_LOGGING: continue #avoid extra logging try: failure.raiseException() except: crashReport('ApplicationLoader', self) Service.startService(self) Event('instance-started').subscribe(self.reset_tracking) #wire allapps action into the server drone.builtins.update({ 'allapps': self.allapps_action, 'applist': self.applist_action, }) #delay scanning by some interval config.reactor.callLater(SERVICECONFIG.initial_delay, self._start_all_tasks)
def startService(self): """Start All AppManager Services""" if self.scanning.called: #need to pre-populate values self.scanning = defer.maybeDeferred(self._first_scan) self.first_run = True self._task = task.LoopingCall(self.scan_app_instances) #plugins will be created and loaded when needed for shortname in config.APPLICATIONS.keys(): manager = None try: applog = logWithContext(type=shortname,route=SERVICENAME) applog('Loading Application Plugin') applog('Creating Application Manager') manager = AppManager(shortname) manager.parentService = self #check and see if the model is bound if not AppManager(shortname).running: applog('Starting Application Manager') manager.start() except: failure = Failure() #bad plugin, not adaptable failures = (InvalidPlugin, TypeError) if failure.check(*failures) and manager: log('plugin for %s is invalid' % (manager.name,)) manager.action.__class__.delete(manager.action) try: pluginFactory.delete_plugin(manager.model) except: pass #silence AppManager.delete(manager) if not config.EXCESSIVE_LOGGING: continue #avoid extra logging try: failure.raiseException() except: crashReport('ApplicationLoader', self) Service.startService(self) Event('instance-started').subscribe(self.reset_tracking) #wire allapps action into the server drone.builtins.update({ 'allapps': self.allapps_action, 'applist': self.applist_action, }) #delay scanning by some interval config.reactor.callLater(SERVICECONFIG.initial_delay, self._start_all_tasks)
def __init__(self, *args, **kwargs): self.deferredResult = args[-1] #deferred result is always the last arg self.debug = False #capture application output as it would appear on a terminal self.OUTPUT = '' #capture STDERR self.STDERR = '' #capture STDOUT self.STDOUT = '' self.PID = 0 #for debugging the protocol if 'debug' in kwargs and kwargs['debug'] == True: self.debug = True #for logging if 'logger' in kwargs: self.log = kwargs['logger'] else: self.log = logWithContext(type='console')
def __init__(self, *args, **kwargs): self.deferredResult = args[-1] #deferred result is always the last arg self.debug = False self._pid = 0 #capture application output as it would appear on a terminal self.OUTPUT = '' #capture STDERR self.STDERR = '' #capture STDOUT self.STDOUT = '' self.PID = 0 #for debugging the protocol if 'debug' in kwargs and kwargs['debug'] == True: self.debug = True #for logging if 'logger' in kwargs: self.log = kwargs['logger'] else: self.log = logWithContext(type='console')
#TODO detect buddies that go offline # if we have a Conversation then unsubscribe .notify from all events def receivedIQ(self, e): log('received iq: %s' % e.toXml()) def receivedError(self, f): log('received error: %s' % str(f)) def initFailed(self, failure): log('Failed to initialize jabber connection:\n%s' % failure.getTraceback()) self.stop() if failure.check(SASLAuthError): log('Will attempt to reconnect in 15 seconds...') reactor.callLater(15, self.start) #setup logging after class definition log = logWithContext(type=JabberClient.SERVICENAME) def validateXml(xml): stream = elementStream() stream.DocumentStartEvent = lambda e: None stream.ElementEvent = lambda e: None stream.DocumentEndEvent = lambda: None stream.parse(xml) #will throw a ParserError if xml is not well-formed # Avoid import circularity from droned.models.conversation import Conversation, ChatRoom from droned.models.event import Event from droned.models.team import Team
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### from twisted.internet.task import LoopingCall from twisted.internet.defer import Deferred from droned.logging import logWithContext import config log = logWithContext(type='action') class ServerManager(object): runningCommands = 0 def __init__(self, server): self.server = server self.queuedCommands = [] self.queueRunner = LoopingCall(self._runQueued) def run(self, command, **kwargs): if config.DO_NOTHING_MODE: command = 'ping' deferredResult = Deferred()
parentService = None service = None SERVICENAME = 'manhole' SERVICECONFIG = dictwrapper() def install(self, _parentService): self.parentService = _parentService realm = TerminalRealm() realm.chainedProtocolFactory.protocolFactory = lambda _: Manhole(globals()) login = {self.SERVICECONFIG.USERNAME: self.SERVICECONFIG.PASSWORD} credChecker = checkers.InMemoryUsernamePasswordDatabaseDontUse(**login) sshportal = portal.Portal(realm) sshportal.registerChecker(credChecker) self.sessionFactory = ConchFactory(sshportal) def start(self): if self.running(): return self.service = internet.TCPServer(self.SERVICECONFIG.PORT, self.sessionFactory) self.service.setServiceParent(self.parentService) def stop(self): if self.service: self.service.disownServiceParent() self.service.stopService() self.service = None def running(self): return bool(self.service) and self.service.running log = logWithContext(type=ManholeService.SERVICENAME)
self.ftpserver.portal = ftpportal self.ftpserver.protocol = ftp.FTP self.service = internet.TCPServer(int(self.SERVICECONFIG.PORT), self.ftpserver) self.service.setServiceParent(self.parentService) def stop(self): if self.service: self.service.disownServiceParent() self.service.stopService() self.service = None def running(self): return bool(self.service) and self.service.running log = logWithContext(type=Ftp.SERVICENAME) class GenericFTPRealm: implements(portal.IRealm) def __init__(self, anonymousRoot, site_root): self.anonymousRoot = filepath.FilePath(anonymousRoot) self.site_root = filepath.FilePath(site_root) def requestAvatar(self, avatarId, mind, *interfaces): for iface in interfaces: if iface is IFTPShell: if avatarId is checkers.ANONYMOUS: avatar = FTPAnonymousShell(self.anonymousRoot) else: avatar = FTPShell(self.site_root)
This service when properly configured will keep the filesystem cleaned up when running. keep the most recent 10 copies of files that match the pattern # files that don't match the pattern are ignored. Janitizer.garbage = { '/tmp/example1/log/directory' : [ ( 'foo_[a-z].+\.log.*', 10) ] } """ #logging context log = logWithContext(type=SERVICENAME) def ageCompare(f1, f2): t1 = os.path.getmtime(f1) t2 = os.path.getmtime(f2) if t1 > t2: return 1 if t2 == t2: return 0 if t2 < t2: return -1 class Janitizer(Service): minute = property(lambda foo: 60) hour = property(lambda foo: 3600) day = property(lambda foo: 86400) week = property(lambda f: 604800)
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### from twisted.internet.task import LoopingCall from twisted.internet.defer import Deferred from droned.logging import logWithContext import config log = logWithContext(type='action') class ServerManager(object): runningCommands = 0 def __init__(self, server): self.server = server self.queuedCommands = [] self.queueRunner = LoopingCall(self._runQueued) def run(self, command, **kwargs): if config.DO_NOTHING_MODE: command = 'ping' deferredResult = Deferred() self.queuedCommands.append((command, kwargs, deferredResult))
#raise NotImplementedError(WORK_IN_PROGRESS_MESSAGE) import time from zope.interface import Interface, Attribute from twisted.python.failure import Failure from twisted.internet import defer, task from twisted.application.service import Service from droned.logging import logWithContext, err from droned.models.systemstats import StatBlockLoader from droned.models.action import AdminAction #becoming a service provider module from kitt.interfaces import moduleProvides, IDroneDService moduleProvides(IDroneDService) #requirement log = logWithContext(type=SERVICENAME) class SystemStats(Service): writing = defer.succeed(None) def stop_collect(self,*args): log("disabling all metric collection") for h in self.handlers: h[1].disable_collect() def start_collect(self,*args): log("enabling metic collection") for h in self.handlers: h[1].enable_collect() def stop_output(self,*args):
def log(self, message): """where to send logging information""" logWithContext(type=self.action, route='console')(str(message))
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### import traceback from twisted.internet.defer import Deferred from twisted.python.failure import Failure from droned.entity import Entity from droned.logging import logWithContext, err import config log = logWithContext(type='events') class Event(Entity): enabled = True def __init__(self, name): self.name = name self.subscribers = set() def fire(self, **params): if not self.enabled: return occurrence = Occurrence(self, **params) if config.DEBUG_EVENTS: params = ', '.join("%s=%s" % i for i in params.items()) log('%s.fire(%s)' % (self, params))
# Everybody Loves Static Files root.putChild("static", static.File(self.SERVICECONFIG.STATIC_ROOT)) # Install in the Reactor main_site = server.Site(root) self.service = internet.TCPServer(self.SERVICECONFIG.PORT, main_site) self.service.setServiceParent(self.parentService) def stop(self): if self.service: self.service.disownServiceParent() self.service.stopService() self.service = None def running(self): return bool(self.service) and self.service.running class Root(resource.Resource): def __init__(self, wsgi_resource): resource.Resource.__init__(self) self.wsgi_resource = wsgi_resource def getChild(self, path, request): path0 = request.prepath.pop(0) request.postpath.insert(0, path0) return self.wsgi_resource log = logWithContext(type=Django.SERVICENAME)
def build_plugin(self, name): """build or lookup plugins @param name C{str} name of the application @return L{IDroneDApplication} provider """ if name in self._instanceMap: return self._instanceMap[name] #create a logging router logging = logWithContext(type="%s,plugin" % (name, ), route='application') logging('dynamically building new plugin class') pluginConfig = self._config.get(name, {}) #look up the application class by name className = pluginConfig.get('CLASS') if isinstance(className, bool) and not className: return None #this is explicitly disabled in config #get the positional constructor args from configuration classArgs = tuple(pluginConfig.get('CLASSARGS', ())) #get the constructor keyword args from configuration classKwargs = pluginConfig.get('CLASSKWARGS', {}) #try to get your application plugin, defaults to builtin bases = (self._classMap.get(className, self._default), ) MRO = set() for x in bases: MRO.add(x) for i in x.__bases__: MRO.add(i) #check if mro is ok MRO = any([b for b in MRO if ApplicationPlugin is b or \ issubclass(b, ApplicationPlugin)]) if not MRO: i = 'adding %s to %s for MRO safety' % \ (ApplicationPlugin, bases[0]) logging(i) #log what we are doing to take the mystory out of it. bases += (ApplicationPlugin, ) else: logging('Method Resolution Order looks good') logging('Adding Storage API to the Plugin.') bases = (StorageMixin, ) + bases ####################################################################### # new class defaults!!!! ####################################################################### #setup the plugin by dynamically building a new class based #off of the provided reference design. newClassName = "Plugin(app=%s, class=%s)" % (name, bases[1].__name__) logging('creating plugin %s' % (newClassName, )) plugin = self.makeAdaptable( type( newClassName, bases, { 'name': property(lambda s: name), 'service': property(lambda s: IDroneModelAppManager(s)), 'configuration': property(lambda s: pluginConfig), 'reactor': property(lambda s: config.reactor), 'STARTUP_INFO': { 'START_USEPTY': 0, 'START_CHILDFD': { 0: 'w', 1: 'r', 2: 'r' }, 'START_ENV': {}, 'START_PATH': None, 'START_CMD': '/bin/true', 'START_ARGS': (), }, 'SHUTDOWN_INFO': { 'STOP_USEPTY': 0, 'STOP_CHILDFD': { 0: 'w', 1: 'r', 2: 'r' }, 'STOP_ENV': {}, 'STOP_PATH': None, 'STOP_CMD': '/bin/true', 'STOP_ARGS': (), }, 'startProtocol': ApplicationProtocol, 'startProtoArgs': tuple(), 'startProtoKwargs': dict(), 'stopProtocol': ApplicationProtocol, 'stopProtoArgs': tuple(), 'stopProtoKwargs': dict(), })) #this guarentees we have a known API to work against #instantiate the plugin object with it's parameters self._instanceMap[name] = plugin(*classArgs, **classKwargs) self._instanceMap[name].log('application plugin interface initialized') self._pluginSetup(self._instanceMap[name]) return self._instanceMap[name]
from twisted.web import server, static, resource from twisted.internet import defer, task import config from kitt.util import unpackify from kitt.decorators import deferredAsThread from kitt import blaster from droned.logging import logWithContext from droned.entity import Entity from droned.clients import cancelTask from droned.clients.blaster import blast import time import gc #setup some logging contexts http_log = logWithContext(type='http', route=SERVICENAME) server_log = logWithContext(type=SERVICENAME) gremlin_log = logWithContext(type='gremlin', route=SERVICENAME) DIGEST_INIT = None try: #newer versions of python import hashlib DIGEST_INIT = hashlib.sha1 except ImportError: #python2.4? import sha DIGEST_INIT = sha.new try: from cStringIO import StringIO except ImportError: from StringIO import StringIO
log('received iq: %s' % e.toXml()) def receivedError(self, f): log('received error: %s' % str(f)) def initFailed(self, failure): log('Failed to initialize jabber connection:\n%s' % failure.getTraceback()) self.stop() if failure.check(SASLAuthError): log('Will attempt to reconnect in 15 seconds...') config.reactor.callLater(15, self.start) #setup logging after class definition log = logWithContext(type=JabberClient.SERVICENAME) def validateXml(xml): stream = elementStream() stream.DocumentStartEvent = lambda e: None stream.ElementEvent = lambda e: None stream.DocumentEndEvent = lambda: None stream.parse(xml) #will throw a ParserError if xml is not well-formed # Avoid import circularity from droned.models.conversation import Conversation, ChatRoom from droned.models.event import Event from droned.models.team import Team
def build_plugin(self, name): """build or lookup plugins @param name C{str} name of the application @return L{IDroneDApplication} provider """ if name in self._instanceMap: return self._instanceMap[name] #create a logging router logging = logWithContext( type="%s,plugin" % (name,), route='application' ) logging('dynamically building new plugin class') pluginConfig = self._config.get(name, {}) #look up the application class by name className = pluginConfig.get('CLASS') if isinstance(className, bool) and not className: return None #this is explicitly disabled in config #get the positional constructor args from configuration classArgs = tuple(pluginConfig.get('CLASSARGS', ())) #get the constructor keyword args from configuration classKwargs = pluginConfig.get('CLASSKWARGS', {}) #try to get your application plugin, defaults to builtin bases = (self._classMap.get(className, self._default),) MRO = set() for x in bases: MRO.add(x) for i in x.__bases__: MRO.add(i) #check if mro is ok MRO = any([b for b in MRO if ApplicationPlugin is b or \ issubclass(b, ApplicationPlugin)]) if not MRO: i = 'adding %s to %s for MRO safety' % \ (ApplicationPlugin, bases[0]) logging(i) #log what we are doing to take the mystory out of it. bases += (ApplicationPlugin,) else: logging('Method Resolution Order looks good') logging('Adding Storage API to the Plugin.') bases = (StorageMixin,) + bases ####################################################################### # new class defaults!!!! ####################################################################### #setup the plugin by dynamically building a new class based #off of the provided reference design. newClassName = "Plugin(app=%s, class=%s)" % (name, bases[1].__name__) logging('creating plugin %s' % (newClassName,)) plugin = self.makeAdaptable(type(newClassName, bases, { 'name': property(lambda s: name), 'service': property(lambda s: IDroneModelAppManager(s)), 'configuration': property(lambda s: pluginConfig), 'reactor': property(lambda s: reactor), 'STARTUP_INFO': { 'START_USEPTY' : 0, 'START_CHILDFD' : {0:'w',1:'r',2:'r'}, 'START_ENV' : {}, 'START_PATH' : None, 'START_CMD' : '/bin/true', 'START_ARGS' : (), }, 'SHUTDOWN_INFO': { 'STOP_USEPTY' : 0, 'STOP_CHILDFD' : {0:'w',1:'r',2:'r'}, 'STOP_ENV' : {}, 'STOP_PATH' : None, 'STOP_CMD' : '/bin/true', 'STOP_ARGS' : (), }, 'startProtocol': ApplicationProtocol, 'startProtoArgs': tuple(), 'startProtoKwargs': dict(), 'stopProtocol': ApplicationProtocol, 'stopProtoArgs': tuple(), 'stopProtoKwargs': dict(), })) #this guarentees we have a known API to work against #instantiate the plugin object with it's parameters self._instanceMap[name] = plugin(*classArgs, **classKwargs) self._instanceMap[name].log('application plugin interface initialized') self._pluginSetup(self._instanceMap[name]) return self._instanceMap[name]
from twisted.application import internet from twisted.web import server, static, resource from twisted.internet import reactor, defer import config from kitt.util import unpackify from kitt.decorators import deferredAsThread from kitt import blaster from droned.logging import logWithContext from droned.entity import Entity from droned.clients import cancelTask import time import gc #setup some logging contexts http_log = logWithContext(type='http', route=SERVICENAME) server_log = logWithContext(type=SERVICENAME) gremlin_log = logWithContext(type='gremlin', route=SERVICENAME) DIGEST_INIT = None try: #newer versions of python import hashlib DIGEST_INIT = hashlib.sha1 except ImportError: #python2.4? import sha DIGEST_INIT = sha.new try: from cStringIO import StringIO except ImportError: from StringIO import StringIO
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### import traceback from twisted.internet.defer import Deferred from twisted.python.failure import Failure from droned.entity import Entity from droned.logging import logWithContext, err import config log = logWithContext(type="events") class Event(Entity): enabled = True def __init__(self, name): self.name = name self.subscribers = set() def fire(self, **params): if not self.enabled: return occurrence = Occurrence(self, **params) if config.DEBUG_EVENTS: params = ", ".join("%s=%s" % i for i in params.items())