def log(self, message, label=None):
"""route logging messages to the application log and allow for custom
labeling to be applied
@param message: (string)
@param label: (string) or (None)
@return None
"""
info = self.name
if label: info += ',%(label)s' % locals()
logWithContext(type=info, route='application')(message)
def log(self, message, label=None):
"""route logging messages to the application log and allow for custom
labeling to be applied
@param message: (string)
@param label: (string) or (None)
@return None
"""
info = self.name
if label: info += ',%(label)s' % locals()
logWithContext(type=info, route='application')(message)
def stopService(self):
"""Stop All AppManager Services"""
for x in ['allapps', 'applist']:
if x in drone.builtins:
del drone.builtins[x]
Event('instance-started').unsubscribe(self.reset_tracking)
for manager in AppManager.objects:
if manager.running:
mesg = 'Stopping Application Manager'
logWithContext(type=manager.name,route=SERVICENAME)(mesg)
manager.stop()
#plugins are stateless by design
pluginFactory.delete_plugin(manager.model)
if self._task.running: self._task.stop()
Service.stopService(self)
def stopService(self):
"""Stop All AppManager Services"""
for x in ['allapps', 'applist']:
if x in drone.builtins:
del drone.builtins[x]
Event('instance-started').unsubscribe(self.reset_tracking)
for manager in AppManager.objects:
if manager.running:
mesg = 'Stopping Application Manager'
logWithContext(type=manager.name, route=SERVICENAME)(mesg)
manager.stop()
#plugins are stateless by design
pluginFactory.delete_plugin(manager.model)
if self._task.running: self._task.stop()
Service.stopService(self)
def startService(self):
"""Start All AppManager Services"""
if self.scanning.called: #need to pre-populate values
self.scanning = defer.maybeDeferred(self._first_scan)
self.first_run = True
self._task = task.LoopingCall(self.scan_app_instances)
#plugins will be created and loaded when needed
for shortname in config.APPLICATIONS.keys():
manager = None
try:
applog = logWithContext(type=shortname, route=SERVICENAME)
applog('Loading Application Plugin')
applog('Creating Application Manager')
manager = AppManager(shortname)
manager.parentService = self
#check and see if the model is bound
if not AppManager(shortname).running:
applog('Starting Application Manager')
manager.start()
except:
failure = Failure()
#bad plugin, not adaptable
failures = (InvalidPlugin, TypeError)
if failure.check(*failures) and manager:
log('plugin for %s is invalid' % (manager.name, ))
manager.action.__class__.delete(manager.action)
try:
pluginFactory.delete_plugin(manager.model)
except:
pass #silence
AppManager.delete(manager)
if not config.EXCESSIVE_LOGGING:
continue #avoid extra logging
try:
failure.raiseException()
except:
crashReport('ApplicationLoader', self)
Service.startService(self)
Event('instance-started').subscribe(self.reset_tracking)
#wire allapps action into the server
drone.builtins.update({
'allapps': self.allapps_action,
'applist': self.applist_action,
})
#delay scanning by some interval
config.reactor.callLater(SERVICECONFIG.initial_delay,
self._start_all_tasks)
def startService(self):
"""Start All AppManager Services"""
if self.scanning.called: #need to pre-populate values
self.scanning = defer.maybeDeferred(self._first_scan)
self.first_run = True
self._task = task.LoopingCall(self.scan_app_instances)
#plugins will be created and loaded when needed
for shortname in config.APPLICATIONS.keys():
manager = None
try:
applog = logWithContext(type=shortname,route=SERVICENAME)
applog('Loading Application Plugin')
applog('Creating Application Manager')
manager = AppManager(shortname)
manager.parentService = self
#check and see if the model is bound
if not AppManager(shortname).running:
applog('Starting Application Manager')
manager.start()
except:
failure = Failure()
#bad plugin, not adaptable
failures = (InvalidPlugin, TypeError)
if failure.check(*failures) and manager:
log('plugin for %s is invalid' % (manager.name,))
manager.action.__class__.delete(manager.action)
try: pluginFactory.delete_plugin(manager.model)
except: pass #silence
AppManager.delete(manager)
if not config.EXCESSIVE_LOGGING: continue #avoid extra logging
try: failure.raiseException()
except: crashReport('ApplicationLoader', self)
Service.startService(self)
Event('instance-started').subscribe(self.reset_tracking)
#wire allapps action into the server
drone.builtins.update({
'allapps': self.allapps_action,
'applist': self.applist_action,
})
#delay scanning by some interval
config.reactor.callLater(SERVICECONFIG.initial_delay, self._start_all_tasks)
def __init__(self, *args, **kwargs):
self.deferredResult = args[-1] #deferred result is always the last arg
self.debug = False
#capture application output as it would appear on a terminal
self.OUTPUT = ''
#capture STDERR
self.STDERR = ''
#capture STDOUT
self.STDOUT = ''
self.PID = 0
#for debugging the protocol
if 'debug' in kwargs and kwargs['debug'] == True:
self.debug = True
#for logging
if 'logger' in kwargs:
self.log = kwargs['logger']
else:
self.log = logWithContext(type='console')
def __init__(self, *args, **kwargs):
self.deferredResult = args[-1] #deferred result is always the last arg
self.debug = False
self._pid = 0
#capture application output as it would appear on a terminal
self.OUTPUT = ''
#capture STDERR
self.STDERR = ''
#capture STDOUT
self.STDOUT = ''
self.PID = 0
#for debugging the protocol
if 'debug' in kwargs and kwargs['debug'] == True:
self.debug = True
#for logging
if 'logger' in kwargs:
self.log = kwargs['logger']
else:
self.log = logWithContext(type='console')
#TODO detect buddies that go offline
# if we have a Conversation then unsubscribe .notify from all events
def receivedIQ(self, e):
log('received iq: %s' % e.toXml())
def receivedError(self, f):
log('received error: %s' % str(f))
def initFailed(self, failure):
log('Failed to initialize jabber connection:\n%s' % failure.getTraceback())
self.stop()
if failure.check(SASLAuthError):
log('Will attempt to reconnect in 15 seconds...')
reactor.callLater(15, self.start)
#setup logging after class definition
log = logWithContext(type=JabberClient.SERVICENAME)
def validateXml(xml):
stream = elementStream()
stream.DocumentStartEvent = lambda e: None
stream.ElementEvent = lambda e: None
stream.DocumentEndEvent = lambda: None
stream.parse(xml) #will throw a ParserError if xml is not well-formed
# Avoid import circularity
from droned.models.conversation import Conversation, ChatRoom
from droned.models.event import Event
from droned.models.team import Team
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
from twisted.internet.task import LoopingCall
from twisted.internet.defer import Deferred
from droned.logging import logWithContext
import config
log = logWithContext(type='action')
class ServerManager(object):
runningCommands = 0
def __init__(self, server):
self.server = server
self.queuedCommands = []
self.queueRunner = LoopingCall(self._runQueued)
def run(self, command, **kwargs):
if config.DO_NOTHING_MODE:
command = 'ping'
deferredResult = Deferred()
parentService = None
service = None
SERVICENAME = 'manhole'
SERVICECONFIG = dictwrapper()
def install(self, _parentService):
self.parentService = _parentService
realm = TerminalRealm()
realm.chainedProtocolFactory.protocolFactory = lambda _: Manhole(globals())
login = {self.SERVICECONFIG.USERNAME: self.SERVICECONFIG.PASSWORD}
credChecker = checkers.InMemoryUsernamePasswordDatabaseDontUse(**login)
sshportal = portal.Portal(realm)
sshportal.registerChecker(credChecker)
self.sessionFactory = ConchFactory(sshportal)
def start(self):
if self.running(): return
self.service = internet.TCPServer(self.SERVICECONFIG.PORT, self.sessionFactory)
self.service.setServiceParent(self.parentService)
def stop(self):
if self.service:
self.service.disownServiceParent()
self.service.stopService()
self.service = None
def running(self):
return bool(self.service) and self.service.running
log = logWithContext(type=ManholeService.SERVICENAME)
self.ftpserver.portal = ftpportal
self.ftpserver.protocol = ftp.FTP
self.service = internet.TCPServer(int(self.SERVICECONFIG.PORT), self.ftpserver)
self.service.setServiceParent(self.parentService)
def stop(self):
if self.service:
self.service.disownServiceParent()
self.service.stopService()
self.service = None
def running(self):
return bool(self.service) and self.service.running
log = logWithContext(type=Ftp.SERVICENAME)
class GenericFTPRealm:
implements(portal.IRealm)
def __init__(self, anonymousRoot, site_root):
self.anonymousRoot = filepath.FilePath(anonymousRoot)
self.site_root = filepath.FilePath(site_root)
def requestAvatar(self, avatarId, mind, *interfaces):
for iface in interfaces:
if iface is IFTPShell:
if avatarId is checkers.ANONYMOUS:
avatar = FTPAnonymousShell(self.anonymousRoot)
else:
avatar = FTPShell(self.site_root)
This service when properly configured will keep the filesystem
cleaned up when running.
keep the most recent 10 copies of files that match the pattern
# files that don't match the pattern are ignored.
Janitizer.garbage = {
'/tmp/example1/log/directory' : [
( 'foo_[a-z].+\.log.*', 10)
]
}
"""
#logging context
log = logWithContext(type=SERVICENAME)
def ageCompare(f1, f2):
t1 = os.path.getmtime(f1)
t2 = os.path.getmtime(f2)
if t1 > t2: return 1
if t2 == t2: return 0
if t2 < t2: return -1
class Janitizer(Service):
minute = property(lambda foo: 60)
hour = property(lambda foo: 3600)
day = property(lambda foo: 86400)
week = property(lambda f: 604800)
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
from twisted.internet.task import LoopingCall
from twisted.internet.defer import Deferred
from droned.logging import logWithContext
import config
log = logWithContext(type='action')
class ServerManager(object):
runningCommands = 0
def __init__(self, server):
self.server = server
self.queuedCommands = []
self.queueRunner = LoopingCall(self._runQueued)
def run(self, command, **kwargs):
if config.DO_NOTHING_MODE:
command = 'ping'
deferredResult = Deferred()
self.queuedCommands.append((command, kwargs, deferredResult))
#raise NotImplementedError(WORK_IN_PROGRESS_MESSAGE)
import time
from zope.interface import Interface, Attribute
from twisted.python.failure import Failure
from twisted.internet import defer, task
from twisted.application.service import Service
from droned.logging import logWithContext, err
from droned.models.systemstats import StatBlockLoader
from droned.models.action import AdminAction
#becoming a service provider module
from kitt.interfaces import moduleProvides, IDroneDService
moduleProvides(IDroneDService) #requirement
log = logWithContext(type=SERVICENAME)
class SystemStats(Service):
writing = defer.succeed(None)
def stop_collect(self,*args):
log("disabling all metric collection")
for h in self.handlers:
h[1].disable_collect()
def start_collect(self,*args):
log("enabling metic collection")
for h in self.handlers:
h[1].enable_collect()
def stop_output(self,*args):
def log(self, message):
"""where to send logging information"""
logWithContext(type=self.action, route='console')(str(message))
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
import traceback
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from droned.entity import Entity
from droned.logging import logWithContext, err
import config
log = logWithContext(type='events')
class Event(Entity):
enabled = True
def __init__(self, name):
self.name = name
self.subscribers = set()
def fire(self, **params):
if not self.enabled: return
occurrence = Occurrence(self, **params)
if config.DEBUG_EVENTS:
params = ', '.join("%s=%s" % i for i in params.items())
log('%s.fire(%s)' % (self, params))
# Everybody Loves Static Files
root.putChild("static", static.File(self.SERVICECONFIG.STATIC_ROOT))
# Install in the Reactor
main_site = server.Site(root)
self.service = internet.TCPServer(self.SERVICECONFIG.PORT, main_site)
self.service.setServiceParent(self.parentService)
def stop(self):
if self.service:
self.service.disownServiceParent()
self.service.stopService()
self.service = None
def running(self):
return bool(self.service) and self.service.running
class Root(resource.Resource):
def __init__(self, wsgi_resource):
resource.Resource.__init__(self)
self.wsgi_resource = wsgi_resource
def getChild(self, path, request):
path0 = request.prepath.pop(0)
request.postpath.insert(0, path0)
return self.wsgi_resource
log = logWithContext(type=Django.SERVICENAME)
def build_plugin(self, name):
"""build or lookup plugins
@param name C{str} name of the application
@return L{IDroneDApplication} provider
"""
if name in self._instanceMap:
return self._instanceMap[name]
#create a logging router
logging = logWithContext(type="%s,plugin" % (name, ),
route='application')
logging('dynamically building new plugin class')
pluginConfig = self._config.get(name, {})
#look up the application class by name
className = pluginConfig.get('CLASS')
if isinstance(className, bool) and not className:
return None #this is explicitly disabled in config
#get the positional constructor args from configuration
classArgs = tuple(pluginConfig.get('CLASSARGS', ()))
#get the constructor keyword args from configuration
classKwargs = pluginConfig.get('CLASSKWARGS', {})
#try to get your application plugin, defaults to builtin
bases = (self._classMap.get(className, self._default), )
MRO = set()
for x in bases:
MRO.add(x)
for i in x.__bases__:
MRO.add(i)
#check if mro is ok
MRO = any([b for b in MRO if ApplicationPlugin is b or \
issubclass(b, ApplicationPlugin)])
if not MRO:
i = 'adding %s to %s for MRO safety' % \
(ApplicationPlugin, bases[0])
logging(i) #log what we are doing to take the mystory out of it.
bases += (ApplicationPlugin, )
else:
logging('Method Resolution Order looks good')
logging('Adding Storage API to the Plugin.')
bases = (StorageMixin, ) + bases
#######################################################################
# new class defaults!!!!
#######################################################################
#setup the plugin by dynamically building a new class based
#off of the provided reference design.
newClassName = "Plugin(app=%s, class=%s)" % (name, bases[1].__name__)
logging('creating plugin %s' % (newClassName, ))
plugin = self.makeAdaptable(
type(
newClassName, bases, {
'name': property(lambda s: name),
'service': property(lambda s: IDroneModelAppManager(s)),
'configuration': property(lambda s: pluginConfig),
'reactor': property(lambda s: config.reactor),
'STARTUP_INFO': {
'START_USEPTY': 0,
'START_CHILDFD': {
0: 'w',
1: 'r',
2: 'r'
},
'START_ENV': {},
'START_PATH': None,
'START_CMD': '/bin/true',
'START_ARGS': (),
},
'SHUTDOWN_INFO': {
'STOP_USEPTY': 0,
'STOP_CHILDFD': {
0: 'w',
1: 'r',
2: 'r'
},
'STOP_ENV': {},
'STOP_PATH': None,
'STOP_CMD': '/bin/true',
'STOP_ARGS': (),
},
'startProtocol': ApplicationProtocol,
'startProtoArgs': tuple(),
'startProtoKwargs': dict(),
'stopProtocol': ApplicationProtocol,
'stopProtoArgs': tuple(),
'stopProtoKwargs': dict(),
})) #this guarentees we have a known API to work against
#instantiate the plugin object with it's parameters
self._instanceMap[name] = plugin(*classArgs, **classKwargs)
self._instanceMap[name].log('application plugin interface initialized')
self._pluginSetup(self._instanceMap[name])
return self._instanceMap[name]
from twisted.web import server, static, resource
from twisted.internet import defer, task
import config
from kitt.util import unpackify
from kitt.decorators import deferredAsThread
from kitt import blaster
from droned.logging import logWithContext
from droned.entity import Entity
from droned.clients import cancelTask
from droned.clients.blaster import blast
import time
import gc
#setup some logging contexts
http_log = logWithContext(type='http', route=SERVICENAME)
server_log = logWithContext(type=SERVICENAME)
gremlin_log = logWithContext(type='gremlin', route=SERVICENAME)
DIGEST_INIT = None
try: #newer versions of python
import hashlib
DIGEST_INIT = hashlib.sha1
except ImportError: #python2.4?
import sha
DIGEST_INIT = sha.new
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
log('received iq: %s' % e.toXml())
def receivedError(self, f):
log('received error: %s' % str(f))
def initFailed(self, failure):
log('Failed to initialize jabber connection:\n%s' %
failure.getTraceback())
self.stop()
if failure.check(SASLAuthError):
log('Will attempt to reconnect in 15 seconds...')
config.reactor.callLater(15, self.start)
#setup logging after class definition
log = logWithContext(type=JabberClient.SERVICENAME)
def validateXml(xml):
stream = elementStream()
stream.DocumentStartEvent = lambda e: None
stream.ElementEvent = lambda e: None
stream.DocumentEndEvent = lambda: None
stream.parse(xml) #will throw a ParserError if xml is not well-formed
# Avoid import circularity
from droned.models.conversation import Conversation, ChatRoom
from droned.models.event import Event
from droned.models.team import Team
def build_plugin(self, name):
"""build or lookup plugins
@param name C{str} name of the application
@return L{IDroneDApplication} provider
"""
if name in self._instanceMap:
return self._instanceMap[name]
#create a logging router
logging = logWithContext(
type="%s,plugin" % (name,),
route='application'
)
logging('dynamically building new plugin class')
pluginConfig = self._config.get(name, {})
#look up the application class by name
className = pluginConfig.get('CLASS')
if isinstance(className, bool) and not className:
return None #this is explicitly disabled in config
#get the positional constructor args from configuration
classArgs = tuple(pluginConfig.get('CLASSARGS', ()))
#get the constructor keyword args from configuration
classKwargs = pluginConfig.get('CLASSKWARGS', {})
#try to get your application plugin, defaults to builtin
bases = (self._classMap.get(className, self._default),)
MRO = set()
for x in bases:
MRO.add(x)
for i in x.__bases__: MRO.add(i)
#check if mro is ok
MRO = any([b for b in MRO if ApplicationPlugin is b or \
issubclass(b, ApplicationPlugin)])
if not MRO:
i = 'adding %s to %s for MRO safety' % \
(ApplicationPlugin, bases[0])
logging(i) #log what we are doing to take the mystory out of it.
bases += (ApplicationPlugin,)
else: logging('Method Resolution Order looks good')
logging('Adding Storage API to the Plugin.')
bases = (StorageMixin,) + bases
#######################################################################
# new class defaults!!!!
#######################################################################
#setup the plugin by dynamically building a new class based
#off of the provided reference design.
newClassName = "Plugin(app=%s, class=%s)" % (name, bases[1].__name__)
logging('creating plugin %s' % (newClassName,))
plugin = self.makeAdaptable(type(newClassName, bases, {
'name': property(lambda s: name),
'service': property(lambda s: IDroneModelAppManager(s)),
'configuration': property(lambda s: pluginConfig),
'reactor': property(lambda s: reactor),
'STARTUP_INFO': {
'START_USEPTY' : 0,
'START_CHILDFD' : {0:'w',1:'r',2:'r'},
'START_ENV' : {},
'START_PATH' : None,
'START_CMD' : '/bin/true',
'START_ARGS' : (),
},
'SHUTDOWN_INFO': {
'STOP_USEPTY' : 0,
'STOP_CHILDFD' : {0:'w',1:'r',2:'r'},
'STOP_ENV' : {},
'STOP_PATH' : None,
'STOP_CMD' : '/bin/true',
'STOP_ARGS' : (),
},
'startProtocol': ApplicationProtocol,
'startProtoArgs': tuple(),
'startProtoKwargs': dict(),
'stopProtocol': ApplicationProtocol,
'stopProtoArgs': tuple(),
'stopProtoKwargs': dict(),
})) #this guarentees we have a known API to work against
#instantiate the plugin object with it's parameters
self._instanceMap[name] = plugin(*classArgs, **classKwargs)
self._instanceMap[name].log('application plugin interface initialized')
self._pluginSetup(self._instanceMap[name])
return self._instanceMap[name]
from twisted.application import internet
from twisted.web import server, static, resource
from twisted.internet import reactor, defer
import config
from kitt.util import unpackify
from kitt.decorators import deferredAsThread
from kitt import blaster
from droned.logging import logWithContext
from droned.entity import Entity
from droned.clients import cancelTask
import time
import gc
#setup some logging contexts
http_log = logWithContext(type='http', route=SERVICENAME)
server_log = logWithContext(type=SERVICENAME)
gremlin_log = logWithContext(type='gremlin', route=SERVICENAME)
DIGEST_INIT = None
try: #newer versions of python
import hashlib
DIGEST_INIT = hashlib.sha1
except ImportError: #python2.4?
import sha
DIGEST_INIT = sha.new
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
import traceback
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from droned.entity import Entity
from droned.logging import logWithContext, err
import config
log = logWithContext(type="events")
class Event(Entity):
enabled = True
def __init__(self, name):
self.name = name
self.subscribers = set()
def fire(self, **params):
if not self.enabled:
return
occurrence = Occurrence(self, **params)
if config.DEBUG_EVENTS:
params = ", ".join("%s=%s" % i for i in params.items())
def log(self, message):
"""where to send logging information"""
logWithContext(type=self.action, route='console')(str(message))