Esempio n. 1
0
 def configure(self, ns):
     # load configuration
     section = ns.section("syslog")
     self.nprocs = section.getInt("num processes", None)
     self.port = section.getInt("listen port", 10514)
     # configure server logging
     logconfigfile = section.getString('log config file', "%s.logconfig" % ns.appname)
     getLogger('tornado')
     if section.getBoolean("debug", False):
         startLogging(StdoutHandler(), DEBUG, logconfigfile)
     else:
         startLogging(None)
     self.handler = TCPHandler(max_buffer_size=65535)
Esempio n. 2
0
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, datetime, dateutil.tz, xmlrpclib
from getpass import getpass
from logging import StreamHandler, DEBUG, Formatter
from pprint import pformat
from twisted.web.xmlrpc import Proxy
from twisted.web.error import Error as TwistedWebError
from twisted.internet import reactor
from terane.bier.evid import EVID
from terane.loggers import getLogger, startLogging, StdoutHandler, DEBUG

logger = getLogger('terane.commands.search.searcher')

class Searcher(object):
    def configure(self, settings):
        # load configuration
        section = settings.section("search")
        self.host = section.getString("host", 'localhost:45565')
        self.username = section.getString("username", None)
        self.password = section.getString("password", None)
        if section.getBoolean("prompt password", False):
            self.password = getpass("Password: "******"limit", 100)
        self.reverse = section.getBoolean("display reverse", False)
        self.longfmt = section.getBoolean("long format", False)
        self.indices = section.getList(str, "use indices", None)
        self.tz = section.getString("timezone", None)
Esempio n. 3
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import sys
from terane.sources.file import StdinSource
from terane.sinks.syslog import SyslogSink
from terane.plugin import PluginManager
from terane.pipeline import Pipeline, parsenodespec, makepipeline
from terane.loggers import getLogger, startLogging, StdoutHandler, DEBUG

logger = getLogger('terane.toolbox.etl.etl')

class ETL(object):
    """
    ETL contains all of the logic necessary to perform an extract-transform-load.
    """
    def configure(self, ns):
        # load configuration
        section = ns.section("etl")
        # publish to the specified sink
        self.store = section.getString("sink", "main")
        # configure pipeline
        source = StdinSource()
        source.configure(section)
        sink = SyslogSink()
        sink.configure(section)
Esempio n. 4
0
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, datetime
from dateutil.tz import tzutc
from zope.interface import implements
from terane.manager import IManager, Manager
from terane.settings import ConfigureError
from terane.bier.interfaces import *
from terane.bier.event import Event
from terane.loggers import getLogger

logger = getLogger('terane.bier')


class EventManager(Manager):
    """
    """

    implements(IManager, IEventFactory, IFieldStore)

    def __init__(self, pluginstore):
        Manager.__init__(self)
        self.setName("events")
        self._pluginstore = pluginstore
        self._fields = dict()
        self._idstore = None
        self._idcache = []
Esempio n. 5
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import re, time, dateutil.parser
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.filters import Filter, IFilter, FilterError
from terane.bier.event import Contract
from terane.loggers import getLogger

logger = getLogger("terane.filters.mysql")

class MysqlServerFilter(Filter):
    """
    Parse the event.message in MySQL server log format.
    """

    implements(IFilter)

    def configure(self, section):
        self._regex = re.compile(r'(?P<date>\d{6})\w+(?P<time>\d\d:\d\d\:\d\d)\w+(?P<msg>.*)')
        self._contract = Contract().sign()

    def getContract(self):
        return self._contract
Esempio n. 6
0
# Terane is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from terane.loggers import getLogger

logger = getLogger('terane.auth.acl')


class Permission(object):
    """
    """

    ALLOW = 1
    DENY = 2
    PASS = 3

    def __init__(self, perm, action, **refs):
        # parse the permission name
        _parts = []
        for part in perm.split('::'):
            part = part.upper()
Esempio n. 7
0
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from dateutil.parser import parse
from xmlrpclib import Fault
from twisted.web.xmlrpc import Proxy
from terane.bier.evid import EVID
from terane.commands.console.switcher import Window
from terane.commands.console.results import ResultsListbox
from terane.commands.console.console import console
from terane.commands.console.ui import useMainThread
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.tail')


class Tailer(Window):
    def __init__(self, args):
        title = "Tail '%s'" % args
        self._query = args
        self._lastId = None
        self._results = ResultsListbox()
        self.interval = 2
        self._url = "http://%s/XMLRPC" % console.host
        self._user = console.username
        self._pass = console.password
        self._delayed = None
        self._deferred = None
        logger.debug("using proxy url %s" % self._url)
Esempio n. 8
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, datetime, dateutil.tz, xmlrpclib
from getpass import getpass
from logging import StreamHandler, DEBUG, Formatter
from twisted.web.xmlrpc import Proxy
from twisted.internet import reactor
from terane.bier.evid import EVID
from terane.loggers import getLogger, startLogging, StdoutHandler, DEBUG

logger = getLogger('terane.commands.tail.tailer')


class Tailer(object):
    def configure(self, settings):
        # load configuration
        section = settings.section("tail")
        self.host = section.getString("host", 'localhost:45565')
        self.username = section.getString("username", None)
        self.password = section.getString("password", None)
        if section.getBoolean("prompt password", False):
            self.password = getpass("Password: "******"limit", 100)
        self.longfmt = section.getBoolean("long format", False)
        self.indices = section.getList(str, "use indices", None)
        self.refresh = section.getInt("refresh", 3)
Esempio n. 9
0
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, getopt
from ConfigParser import RawConfigParser
from terane.loggers import getLogger
from terane import versionstring

logger = getLogger('terane.settings')


class ConfigureError(Exception):
    """
    Configuration parsing failed.
    """
    pass


class Option(object):
    def __init__(self,
                 shortname,
                 longname,
                 section,
                 override,
Esempio n. 10
0
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os
from zope.interface import implements
from zope.component import getUtility
from terane.plugins import Plugin, IPlugin
from terane.sched import IScheduler
from terane.bier.event import Contract
from terane.bier.writing import WriterWorker
from terane.outputs import Output, IOutput, ISearchable
from terane.outputs.store.env import Env
from terane.outputs.store.index import Index
from terane.outputs.store.logfd import LogFD
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store')

class StoreOutput(Output):

    implements(IOutput, ISearchable)

    def __init__(self, plugin, name, fieldstore):
        self._plugin = plugin
        self.setName(name)
        self._fieldstore = fieldstore
        self._index = None
        self._contract = Contract().sign()

    def configure(self, section):
        self._indexName = section.getString("index name", self.name)
        if self._indexName in self._plugin._outputs:
Esempio n. 11
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import datetime, dateutil.tz
import pyparsing as pp
from terane.loggers import getLogger

logger = getLogger('terane.bier.ql.date')


def _makeUTC(dt):
    "Convert the supplied datetime object to UTC, if necessary."
    # if no timezone is specified, then assume local tz
    if dt.tzinfo == None:
        dt = dt.replace(tzinfo=dateutil.tz.tzlocal())
    # convert to UTC, if necessary
    if not dt.tzinfo == dateutil.tz.tzutc():
        dt = dt.astimezone(dateutil.tz.tzutc())
    # return a timezone-naive datetime object for Whoosh
    return dt.replace(tzinfo=None) - dt.utcoffset()


# --------------------
Esempio n. 12
0
import time, datetime, pickle
from threading import Lock
from uuid import UUID, uuid4, uuid5
from zope.interface import implements
from twisted.internet.defer import succeed
from terane.bier import IIndex
from terane.bier.evid import EVID, EVID_MIN
from terane.bier.fields import SchemaError
from terane.outputs.store import backend
from terane.outputs.store.segment import Segment
from terane.outputs.store.searching import IndexSearcher
from terane.outputs.store.writing import IndexWriter
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.index')


class Index(backend.Index):
    """
    Stores events, which are a collection of fields.  Internally, an Index is
    made up of multiple Segments.  Instantiation opens the Index, creating it
    if necessary.  The index will be created in the specified environment, and
    thus protected transactionally.

    :param env: The DB environment.
    :type env: :class:`terane.db.backend.Env`
    :param name: The name of the index.
    :type name: str
    """
Esempio n. 13
0
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from zope.interface import implements
from twisted.application.service import Service
from twisted.internet.protocol import ServerFactory
from terane.manager import IManager, Manager
from terane.plugins import IPluginStore
from terane.auth import IAuthManager
from terane.queries import IQueryManager
from terane.protocols import IProtocol
from terane.settings import ConfigureError
from terane.loggers import getLogger

logger = getLogger('terane.listeners')


class Listener(Service):
    def __init__(self, name, protocol):
        self.setName(name)
        self._protocol = protocol

    def configure(self, section):
        self.listenAddress = section.getString("listen address", "0.0.0.0")
        self.listenPort = section.getInt("listen port",
                                         self._protocol.getDefaultPort())
        self.listenBacklog = section.getInt("listen backlog", 50)

    def startService(self):
        protoFactory = self._protocol.makeFactory()
Esempio n. 14
0
from zope.component import provideUtility
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.internet.defer import maybeDeferred
from terane.sched import Scheduler, IScheduler
from terane.plugins import PluginManager
from terane.bier import EventManager
from terane.auth import AuthManager
from terane.routes import RouteManager
from terane.queries import QueryManager
from terane.listeners import ListenerManager
from terane.stats import stats
from terane.loggers import getLogger, startLogging, StdoutHandler, FileHandler
from terane.loggers import ERROR, WARNING, INFO, DEBUG

logger = getLogger('terane.commands.server.server')


class ServerError(Exception):
    pass


class Server(MultiService):
    def configure(self, settings):
        self.settings = settings
        section = settings.section('server')
        self.user = section.getString('runtime user', None)
        self.group = section.getString('runtime group', None)
        self.pidfile = section.getPath('pid file',
                                       '/var/lib/terane/terane-server.pid')
        self.debug = section.getBoolean('debug', False)
Esempio n. 15
0
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from csv import DictReader
from twisted.internet import reactor
from terane.commands.console.switcher import Window
from terane.commands.console.results import ResultsListbox
from terane.commands.console.console import console
from terane.commands.console.ui import useMainThread
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.outfile')

class Outfile(Window):
    def __init__(self, args):
        self._path = os.path.expanduser(args)
        title = "Results from file %s" % self._path
        self._results = ResultsListbox()
        Window.__init__(self, title, self._results)

    def startService(self):
        logger.debug("startService")
        try:
            # load data from path
            with file(self._path, 'r') as f:
                logger.debug("opened outfile %s" % self._path)
                reader = DictReader(f)
Esempio n. 16
0
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys
from twisted.internet.defer import Deferred
from twisted.spread.pb import PBClientFactory, DeadReferenceError
from twisted.cred.credentials import Anonymous
from twisted.internet import reactor
from twisted.python.failure import Failure
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.outputs import Output, IOutput
from terane.loggers import getLogger
from terane.stats import getStat

logger = getLogger('terane.outputs.forward')


class ForwardOutput(Output):

    implements(IOutput)

    def configure(self, section):
        self.forwardserver = section.getString('forwarding address', None)
        self.forwardport = section.getInt('forwarding port', None)
        self.retryinterval = section.getInt('retry interval', 10)
        self.forwardedevents = getStat(
            "terane.output.%s.forwardedevents" % self.name, 0)
        self.stalerefs = getStat("terane.output.%s.stalerefs" % self.name, 0)

    def startService(self):
Esempio n. 17
0
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from dateutil.parser import parse
from xmlrpclib import Fault
from twisted.web.xmlrpc import Proxy
from terane.bier.evid import EVID
from terane.commands.console.switcher import Window
from terane.commands.console.results import ResultsListbox
from terane.commands.console.console import console
from terane.commands.console.ui import useMainThread
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.search')

class Searcher(Window):
    def __init__(self, args):
        # configure the searcher
        title = "Search results for '%s'" % args
        self._query = args
        self._results = ResultsListbox()
        self._url = "http://%s/XMLRPC" % console.host
        self._user = console.username
        self._pass = console.password
        self._deferred = None
        Window.__init__(self, title, self._results)

    def startService(self):
        Window.startService(self)
Esempio n. 18
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import re, time, dateutil.parser
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.filters import Filter, IFilter, FilterError
from terane.bier.event import Contract
from terane.loggers import getLogger

logger = getLogger("terane.filters.mysql")


class MysqlServerFilter(Filter):
    """
    Parse the event.message in MySQL server log format.
    """

    implements(IFilter)

    def configure(self, section):
        self._regex = re.compile(
            r'(?P<date>\d{6})\w+(?P<time>\d\d:\d\d\:\d\d)\w+(?P<msg>.*)')
        self._contract = Contract().sign()

    def getContract(self):
Esempio n. 19
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import re, time, email.utils, datetime
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.filters import Filter, IFilter, FilterError, StopFiltering
from terane.bier.event import Contract, Assertion
from terane.loggers import getLogger

logger = getLogger("terane.filters.datetime")

class RFC2822DatetimeFilter(Filter):
    """
    Given an event field with a RFC2822-compatible datetime string, convert the
    string to a datetime.datetime and store it as the event ts.
    """

    implements(IFilter)

    def configure(self, section):
        self._fieldname = section.getString('source field', 'rfc2822_date')
        self._expected = section.getBoolean('expects source', False)
        self._guaranteed = section.getBoolean('guarantees source', False)
        self._contract = Contract()
        self._contract.addAssertion( unicode(self._fieldname), u'text',
Esempio n. 20
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, fcntl
from terane.outputs.store import backend
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.backend.env')


class Env(backend.Env):
    """
    """
    def __init__(self, dbdir, options=dict()):
        # create berkeleydb-specific directories under the dbdir root
        self.dbdir = dbdir
        if not os.path.exists(self.dbdir):
            os.mkdir(dbdir)
        datadir = os.path.join(self.dbdir, "data")
        if not os.path.exists(datadir):
            os.mkdir(datadir)
        envdir = os.path.join(self.dbdir, "env")
        if not os.path.exists(envdir):
Esempio n. 21
0
File: ui.py Progetto: msfrank/terane
#
# Terane is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import functools
from twisted.internet import reactor
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.ui')


def useMainThread(fn):
    """
    A decorator for methods which should be run in a separate thread.
    """
    @functools.wraps(fn)
    def _threadWrapper(*args, **kwds):
        return reactor.callFromThread(fn, *args, **kwds)

    return _threadWrapper
Esempio n. 22
0
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import pyparsing as pp
from terane.bier.matching import Every, AND, OR, NOT
from terane.bier.ql.term import subjectTerm
from terane.bier.ql.date import subjectDate
from terane.loggers import getLogger

logger = getLogger('terane.bier.ql.queries')


def _parseNotGroup(tokens):
    return NOT(tokens[0][0])


def _parseAndGroup(tokens):
    tokens = tokens[0]
    q = tokens[0]
    if len(tokens) > 1:
        i = 1
        while i < len(tokens):
            q = AND([q, tokens[i]])
            i += 1
    return q
Esempio n. 23
0
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, socket
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.inputs import Input, IInput
from terane.signals import Signal
from terane.bier.event import Contract, Assertion
from terane.loggers import getLogger

logger = getLogger('terane.inputs.file')

class FileInput(Input):

    implements(IInput)

    def __init__(self, plugin, name, evfactory):
        self._plugin = plugin
        self.setName(name)
        self._evfactory = evfactory
        self._dispatcher = Signal()
        self._delayed = None
        self._deferred = None
        self._file = None
        self._prevstats = None
        self._position = None
Esempio n. 24
0
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from twisted.python.failure import Failure
from terane.bier.interfaces import IIndex, IWriter
from terane.bier.evid import EVID
from terane.bier.event import Event
from terane.loggers import getLogger

logger = getLogger('terane.bier.writing')


class WriterError(Exception):
    pass


class WriterWorker(object):
    """
    A worker which writes an event to the specified index.  Instances of this
    class must be submitted to a :class:`terane.sched.Task` to be scheduled.
    """
    def __init__(self, event, index):
        """
        :param event: The event to write.
        :type event: :class:`terane.bier.event.Event`
Esempio n. 25
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import hashlib
from zope.interface import implements
from twisted.internet.defer import succeed, fail
from twisted.cred.checkers import ICredentialsChecker, ANONYMOUS
from twisted.cred.credentials import IUsernamePassword, IAnonymous, Anonymous
from twisted.cred.error import UnauthorizedLogin
from terane.loggers import getLogger

logger = getLogger('terane.auth.credentials')


class User(object):
    def __init__(self, name, passwd, roles):
        self.name = name
        self.passwd = passwd
        self.roles = roles


anonUser = User('_ANONYMOUS', '', ['_ANONYMOUS'])


class AnonymousOnly(object):
    """
    """
Esempio n. 26
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from twisted.application.service import Service, MultiService
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.switcher')


class Window(Service, urwid.WidgetWrap):
    def __init__(self, title, body):
        self.body = body
        self.title = title
        self.wid = None
        self.prev = None
        self.next = None
        urwid.WidgetWrap.__init__(self, body)

    def redraw(self):
        pass

Esempio n. 27
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, re, urwid, datetime, dateutil.tz
from itertools import cycle, islice
from csv import DictWriter
from terane.commands.console import filters
from terane.commands.console.console import console
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.results')


class Result(urwid.WidgetWrap):
    def __init__(self, evid, defaultfield, defaultvalue, fields):
        self.evid = evid
        self.ts = datetime.datetime.fromtimestamp(evid.ts, dateutil.tz.tzutc())
        self.offset = evid.offset
        self.defaultfield = defaultfield
        self.defaultvalue = defaultvalue
        self.fields = dict(fields)
        self.visible = True
        self.highlighted = False
        self._text = urwid.Text('', wrap='any')
        urwid.WidgetWrap.__init__(self, self._text)
Esempio n. 28
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from zope.interface import Interface, implements
from twisted.cred.portal import Portal
from terane.manager import IManager, Manager
from terane.settings import ConfigureError
from terane.auth.credentials import PasswdFile, AnonymousOnly
from terane.auth.acl import ACL, Permission
from terane.loggers import getLogger

logger = getLogger('terane.auth')

class IAuthManager(Interface):
    def getPortal(realm):
        """
        Return a Portal integrating the specified realm with the credentials loaded
        into the authorization manager.

        :param realm: The realm.
        :type realm: An object providing :class:`twisted.cred.portal.IRealm`
        :returns: The portal.
        :rtype: :class:`twisted.cred.portal.Portal`
        """
    def canAccess(userName, objectType, objectName, *perms):
        """
        Returns a boolean indicating whether or not the specified userName has the
Esempio n. 29
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, signal, traceback
from tornado.tcpserver import TCPServer
from tornado.iostream import StreamClosedError
from tornado.ioloop import IOLoop
from tornado.process import task_id
from loggerglue.rfc5424 import SyslogEntry, syslog_msg
from terane.loggers import getLogger, startLogging, StdoutHandler, DEBUG

logger = getLogger('terane.toolbox.relay.server')

class Server(object):
    """
    Receives TCP syslog messages and processes them using the
    specified pipeline.
    """
    def configure(self, ns):
        # load configuration
        section = ns.section("syslog")
        self.nprocs = section.getInt("num processes", None)
        self.port = section.getInt("listen port", 10514)
        # configure server logging
        logconfigfile = section.getString('log config file', "%s.logconfig" % ns.appname)
        getLogger('tornado')
        if section.getBoolean("debug", False):
Esempio n. 30
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import datetime, calendar, dateutil.tz, re
from zope.interface import implements
from terane.plugins import ILoadable, IPlugin, Plugin
from terane.bier.interfaces import IField
from terane.bier.matching import Term, Phrase, RangeGreaterThan, RangeLessThan
from terane.loggers import getLogger

logger = getLogger('terane.bier.schema')

class SchemaError(Exception):
    pass

class QualifiedField(object):
    """
    """
    def __init__(self, fieldname, fieldtype, field):
        self.fieldname = unicode(fieldname)
        self.fieldtype = unicode(fieldtype)
        self.field = field

    def parseValue(self, value):
        return self.field.parseValue(value)
Esempio n. 31
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import math
from zope.interface import implements
from twisted.internet.threads import deferToThread
from twisted.internet.defer import succeed, inlineCallbacks, returnValue
from terane.bier import ISearcher, IPostingList, IEventStore
from terane.bier.evid import EVID
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.searching')


class IndexSearcher(object):
    """
    IndexSearcher searches an entire index by searching each Segment individually
    and merging the results.
    """

    implements(ISearcher)

    def __init__(self, ix):
        """
        :param ix: The index to search.
        :type ix: :class:`terane.outputs.store.index.Index`
        """
Esempio n. 32
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import bisect
from zope.interface import implements
from twisted.internet.defer import inlineCallbacks, returnValue, succeed, fail
from terane.bier.interfaces import IMatcher, IPostingList
from terane.bier.event import Contract
from terane.bier.evid import EVID
from terane.loggers import getLogger

logger = getLogger('terane.bier.matching')

class QueryTerm(object):

    implements(IMatcher)

    def __init__(self, fieldname, fieldtype, fieldfunc, value):
        """
        :param fieldname: The name of the field to search.
        :type fieldname: str
        :param value: The term to search for in the field.
        :type value: unicode
        """
        self.fieldname = fieldname
        self.fieldtype = fieldtype
        self.fieldfunc = fieldfunc
Esempio n. 33
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from twisted.internet import reactor
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.input')


class Input(urwid.FlowWidget):
    """
    The Input widget allows user input in a style similar to Vi(m).  There are
    two modes that the widget can be in: view and command (find and rfind modes
    are submodes of the command mode).  View mode is the default mode.  The user
    enters command mode, find mode, or rfind mode by typing ':', '/', or '?',
    respectively.  In command mode, user input is displayed on the screen.  The
    user exits command mode by either pressing enter/return to submit the
    command, or pressing escape to discard the command.  While in command mode,
    the user can also press up or down to scroll through command history.
    """

    VIEW_MODE = 0
Esempio n. 34
0
# 
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import pyparsing as pp
from terane.bier.matching import Every, AND, OR, NOT
from terane.bier.ql.term import subjectTerm
from terane.bier.ql.date import subjectDate
from terane.loggers import getLogger

logger = getLogger('terane.bier.ql.queries')

def _parseNotGroup(tokens):
    return NOT(tokens[0][0])

def _parseAndGroup(tokens):
    tokens = tokens[0]
    q = tokens[0]
    if len(tokens) > 1:
        i = 1
        while i < len(tokens):
            q = AND([q, tokens[i]])
            i += 1
    return q

def _parseOrGroup(tokens):
Esempio n. 35
0
import sys, traceback
from terane.toolbox.admin.action import ActionMap,Action
from terane.toolbox.admin.route.append import AppendRouteCommand
from terane.toolbox.admin.source.create import CreateSyslogUdpSourceCommand, CreateSyslogTcpSourceCommand
from terane.toolbox.admin.source.delete import DeleteSourceCommand
from terane.toolbox.admin.source.list import ListSourcesCommand
from terane.toolbox.admin.source.show import ShowSourceCommand
from terane.toolbox.admin.sink.create import CreateCassandraSinkCommand
from terane.toolbox.admin.sink.delete import DeleteSinkCommand
from terane.toolbox.admin.sink.list import ListSinksCommand
from terane.toolbox.admin.sink.show import ShowSinkCommand
from terane.settings import Option, LongOption, Switch, ConfigureError
from terane.loggers import getLogger

logger = getLogger('terane.toolbox.admin')

actions = ActionMap(usage="[OPTIONS...] COMMAND", description="Terane cluster administrative commands", section="admin", options=[
    Option("H", "host", override="host", help="Connect to terane server HOST", metavar="HOST"),
    Option("u", "username", override="username", help="Authenticate with username USER", metavar="USER"),
    Option("p", "password", override="password", help="Authenticate with password PASS", metavar="PASS"),
    Switch("P", "prompt-password", override="prompt password", help="Prompt for a password"),
    LongOption("log-config", override="log config file", help="use logging configuration file FILE", metavar="FILE"),
    Switch("d", "debug", override="debug", help="Print debugging information")], actions=[
    Action("route", usage="COMMAND", description="Manipulate routes in a Terane cluster", actions=[
        Action("append", callback=None,
               usage="ROUTE",
               description="Append ROUTE to the routing table"),
        Action("insert", callback=None,
               usage="POSITION ROUTE",
               description="Insert ROUTE at the specified POSITION in the routing table"),
Esempio n. 36
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import time, datetime, calendar, copy
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import cooperate
from terane.bier.interfaces import IIndex, ISearcher, IPostingList, IEventStore
from terane.bier.evid import EVID
from terane.loggers import getLogger

logger = getLogger('terane.bier.searching')


class SearcherError(Exception):
    pass


class Period(object):
    """
    A time range within which to constain a query.
    """
    def __init__(self, start, end, startexcl, endexcl):
        """
        :param start: The start of the time range.
        :type start: :class:`datetime.datetime`
        :param end: The end of the time range.
Esempio n. 37
0
import time, datetime, pickle
from threading import Lock
from uuid import UUID, uuid4, uuid5
from zope.interface import implements
from twisted.internet.defer import succeed
from terane.bier import IIndex
from terane.bier.evid import EVID, EVID_MIN
from terane.bier.fields import SchemaError
from terane.outputs.store import backend
from terane.outputs.store.segment import Segment
from terane.outputs.store.searching import IndexSearcher
from terane.outputs.store.writing import IndexWriter
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.index')

class Index(backend.Index):
    """
    Stores events, which are a collection of fields.  Internally, an Index is
    made up of multiple Segments.  Instantiation opens the Index, creating it
    if necessary.  The index will be created in the specified environment, and
    thus protected transactionally.

    :param env: The DB environment.
    :type env: :class:`terane.db.backend.Env`
    :param name: The name of the index.
    :type name: str
    """

    implements(IIndex)
Esempio n. 38
0
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys
from twisted.internet.defer import Deferred
from twisted.spread.pb import PBClientFactory, DeadReferenceError
from twisted.cred.credentials import Anonymous
from twisted.internet import reactor
from twisted.python.failure import Failure
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.outputs import Output, IOutput
from terane.loggers import getLogger
from terane.stats import getStat

logger = getLogger('terane.outputs.forward')

class ForwardOutput(Output):

    implements(IOutput)

    def configure(self, section):
        self.forwardserver = section.getString('forwarding address', None)
        self.forwardport = section.getInt('forwarding port', None)
        self.retryinterval = section.getInt('retry interval', 10)
        self.forwardedevents = getStat("terane.output.%s.forwardedevents" % self.name, 0)
        self.stalerefs = getStat("terane.output.%s.stalerefs" % self.name, 0)
        
    def startService(self):
        Output.startService(self)
        self._client = None
Esempio n. 39
0
# 
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from twisted.internet import reactor
from terane.toolbox.admin.command import AdminCommand
from terane.api.context import ApiContext
from terane.api.sink import DescribeSinkRequest
from terane.loggers import getLogger

logger = getLogger('terane.toolbox.admin.sink.show')

class ShowSinkCommand(AdminCommand):
    """
    """
    def configure(self, ns):
        (name,) = ns.getArgs(str, names=["NAME"], maximum=1)
        self.name = name
        AdminCommand.configure(self, ns)

    def printResult(self, sink):
        name = sink['name']
        del sink['name']
        print "  name: " + name
        sinkType = sink['sinkType']
        del sink['sinkType']
Esempio n. 40
0
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, urwid
from dateutil.parser import parse
from xmlrpclib import Fault
from twisted.web.xmlrpc import Proxy
from terane.bier.evid import EVID
from terane.commands.console.switcher import Window
from terane.commands.console.results import ResultsListbox
from terane.commands.console.console import console
from terane.commands.console.ui import useMainThread
from terane.loggers import getLogger

logger = getLogger('terane.commands.console.tail')

class Tailer(Window):
    def __init__(self, args):
        title = "Tail '%s'" % args
        self._query = args
        self._lastId = None
        self._results = ResultsListbox()
        self.interval = 2
        self._url = "http://%s/XMLRPC" % console.host
        self._user = console.username
        self._pass = console.password
        self._delayed = None
        self._deferred = None
        logger.debug("using proxy url %s" % self._url)
        Window.__init__(self, title, self._results)
Esempio n. 41
0
# 
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from twisted.internet import reactor
from terane.toolbox.admin.command import AdminCommand
from terane.api.context import ApiContext
from terane.api.sink import DeleteSinkRequest
from terane.loggers import getLogger

logger = getLogger('terane.toolbox.admin.sink.delete')

class DeleteSinkCommand(AdminCommand):
    """
    """
    def configure(self, settings):
        section = settings.section("admin:sink:delete")
        (name,) = settings.getArgs(str, names=["NAME"], maximum=1)
        self.name = name
        AdminCommand.configure(self, ns)

    def printResult(self, result):
        reactor.stop()

    def printError(self, failure):
        try:
Esempio n. 42
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import pickle
from zope.interface import implements
from terane.registry import getRegistry
from terane.bier import IField, ISchema
from terane.bier.fields import QualifiedField
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.schema')

class Schema(object):

    implements(ISchema)

    def __init__(self, index, fieldstore):
        self._index = index
        self._fields = {}
        self._fieldstore = fieldstore
        # load schema data from the db
        with self._index.new_txn() as txn:
            for fieldname,fieldspec in self._index.iter_fields(txn):
                self._fields[fieldname] = pickle.loads(str(fieldspec))
                # verify that the field type is consistent
                for fieldtype,stored in self._fields[fieldname].items():
Esempio n. 43
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import re
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.filters import Filter, IFilter, FilterError, StopFiltering
from terane.bier.event import Contract

from terane.loggers import getLogger

logger = getLogger("terane.filters.regex")


class RegexFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._infield = section.getString('regex field', '_raw')
        regex = section.getString('regex pattern', None)
        if regex == None:
            raise ConfigureError(
                "[filter:%s] missing required parameter 'regex pattern'" %
                self.name)
        try:
            self._regex = re.compile(regex)
Esempio n. 44
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import re, time, dateutil.parser
from zope.interface import implements
from terane.plugins import Plugin, IPlugin
from terane.filters import Filter, IFilter, FilterError
from terane.bier.event import Contract
from terane.loggers import getLogger

logger = getLogger("terane.filters.syslog")

class SyslogFilter(Filter):

    implements(IFilter)

    def configure(self, section):
        self._linematcher = re.compile(r'(?P<ts>[A-Za-z]{3} [ \d]\d \d\d:\d\d\:\d\d) (?P<hostname>\S*) (?P<msg>.*)')
        self._tagmatcher = re.compile(r'^(\S+)\[(\d+)\]:$|^(\S+):$')
        self._contract = Contract()
        self._contract.addAssertion(u'syslog_pid', u'int', guarantees=False)
        self._contract.addAssertion(u'syslog_tag', u'text', guarantees=False)
        self._contract.sign()

    def getContract(self):
        return self._contract
Esempio n. 45
0
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from pkg_resources import Environment, working_set
from terane.loggers import getLogger

logger = getLogger("terane.plugin")

class IPlugin(object):

    def __init__(self, *args, **kwargs):
        pass

    def configure(self, section):
        pass

    def init(self):
        pass

    def fini(self):
        pass
Esempio n. 46
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from terane.plugin import IPlugin
from terane.event import FieldIdentifier
from terane.loggers import getLogger

logger = getLogger('terane.sinks.debug')

class DebugSink(IPlugin):
    """
    Dump received events to stdout.
    """

    def __str__(self):
        return "DebugSink()"

    def consume(self, event):
        if event.id == None:
            print "-"
        else:
            print event.id
        for (fieldname,fieldtype),value in event.items():
Esempio n. 47
0
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import time, datetime, calendar, copy
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import cooperate
from terane.bier.interfaces import IIndex, ISearcher, IPostingList, IEventStore
from terane.bier.evid import EVID
from terane.loggers import getLogger

logger = getLogger('terane.bier.searching')

class SearcherError(Exception):
    pass

class Period(object):
    """
    A time range within which to constain a query.
    """
    def __init__(self, start, end, startexcl, endexcl):
        """
        :param start: The start of the time range.
        :type start: :class:`datetime.datetime`
        :param end: The end of the time range.
        :type end: :class:`datetime.datetime`
        :param startexcl: If True, then the start of the range is exclusive.
Esempio n. 48
0
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import time
from zope.interface import Interface, implements
from twisted.internet.defer import succeed
from twisted.python.failure import Failure
from terane.manager import IManager, Manager
from terane.registry import getRegistry
from terane.routes import IIndexStore
from terane.bier.evid import EVID
from terane.bier.ql import parseIterQuery, parseTailQuery
from terane.bier.searching import searchIndices, Period, SearcherError
from terane.loggers import getLogger

logger = getLogger('terane.queries')

class QueryExecutionError(Exception):
    """
    There was an error while executing the plan.
    """
    pass

class QueryResult(object):
    """
    The result from executing a query method.  A QueryResult consists of the data
    itself, which is a list of results; and metadata, which is a mapping of keys to
    values.
    """

    def __init__(self, meta={}, data=[]):
Esempio n. 49
0
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, sys, getopt
from ConfigParser import RawConfigParser
from terane.loggers import getLogger
from terane import versionstring

logger = getLogger("terane.settings")


class ConfigureError(Exception):
    """
    Configuration parsing failed.
    """

    pass


class Option(object):
    def __init__(self, shortname, longname, section, override, help=None, metavar=None):
        self.shortname = shortname
        self.shortident = "%s:" % shortname
        self.longname = longname
Esempio n. 50
0
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import time
import cPickle as pickle
from zope.interface import implements
from twisted.internet.defer import succeed
from twisted.internet.threads import deferToThread
from terane.bier import IWriter
from terane.bier.fields import QualifiedField
from terane.bier.writing import WriterError
from terane.loggers import getLogger

logger = getLogger('terane.outputs.store.writing')

class WriterExpired(WriterError):
    pass

class IndexWriter(object):

    implements(IWriter)

    def __init__(self, ix):
        self._ix = ix
        logger.trace("[writer %s] waiting for segmentLock" % self)
        with ix._segmentLock:
            logger.trace("[writer %s] acquired segmentLock" % self)
            self._segment = ix._current
        logger.trace("[writer %s] released segmentLock" % self)
Esempio n. 51
0
# 
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, errno
from twisted.application.service import Service
from twisted.internet import task
from terane.settings import ConfigureError
from terane.loggers import getLogger

logger = getLogger('terane.stats')

class Stat(object):

    def __init__(self, name, ivalue, itype, volatile):
        self.name = name
        self._volatile = volatile
        self._ivalue = ivalue
        self._itype = itype
        self._value = itype(ivalue)

    def _getvalue(self):
        return self._value

    def _setvalue(self, v):
        self._value = self._itype(v)
Esempio n. 52
0
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import os, signal
from tornado.tcpserver import TCPServer
from tornado.iostream import StreamClosedError
from tornado.ioloop import IOLoop
from tornado.process import task_id
from loggerglue.rfc5424 import SyslogEntry, syslog_msg
from terane.plugin import IPlugin
from terane.settings import ConfigureError
from terane.loggers import getLogger, startLogging, StdoutHandler, DEBUG

logger = getLogger('terane.toolbox.syslog.sink')

class SyslogSink(IPlugin):
    """
    """
    def __init__(self, ioloop=None):
        self.ioloop = ioloop
        self.host = 'localhost'
        self.port = 514

    def configure(self, section):
        pass

    def init(self):
        self.queue = list()
        self.stream = None
Esempio n. 53
0
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

from zope.interface import Interface, implements
from twisted.application.service import Service
from twisted.internet.task import cooperate
from terane.manager import IManager, Manager
from terane.plugins import IPluginStore
from terane.bier import IEventFactory, IFieldStore
from terane.inputs import IInput
from terane.outputs import IOutput, ISearchable
from terane.filters import IFilter, StopFiltering
from terane.signals import SignalCancelled
from terane.settings import ConfigureError
from terane.loggers import getLogger

logger = getLogger('terane.routes')

class EventProcessor(object):
    def __init__(self, event, filters, fieldstore):
        self.event = event
        self._filters = filters
        self._fieldstore = fieldstore

    def next(self):
        if len(self._filters) == 0:
            raise StopIteration()
        filter = self._filters.pop(0)
        contract = filter.getContract()
        contract.validateEventBefore(self.event, self._fieldstore)
        self.event = filter.filter(self.event)
        contract.validateEventAfter(self.event, self._fieldstore)
Esempio n. 54
0
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Terane is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Terane.  If not, see <http://www.gnu.org/licenses/>.

import pyparsing as pp
from terane.bier.matching import QueryTerm
from terane.loggers import getLogger

logger = getLogger('terane.bier.ql')

# --------------------
# keyValueTerm grammar
# --------------------
# <alphanums>             ::= alphabetic characters 'a' through 'z', 'A' through 'Z', digits 0 through 9
# <unreservedSymbols>     ::= '_' | '+' | '-' | '*' | '/' | '\' | ',' | '.' | '&' | '|' | '^' | '~' | '@' | '#' | '$' | '%' | ':' | ';'
# <unreservedChars>       ::= <alphas> | <digits> | <unreservedSymbols>
# <subjectWord>           ::= <quotedString> | <unreservedChars>+
# <subjectFieldName>      ::= <unreservedChars>+
# <subjectFieldType>      ::= <unreservedChars>+
# <keyValueTerm>          ::= <subjectWord> | <subjectFieldName> '=' <subjectWord>

fieldName = pp.Word(pp.alphanums + '_')
unquotedValue = pp.Word(pp.alphanums + '_+*/\\,.&|^~@#$%:;')
value = pp.QuotedString('\'') | pp.QuotedString('"') | unquotedValue