# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import os import socket import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.dealer.FileOutput') class FileOutput(object): def __init__(self, *args, **kw): self.file_output = conf.get("dealer", "file_path") def name(self): return 'file_output' def register(self, *args, **kw): return self.run def run(self, notification): log.debug("Notification received: %r" % (notification), level=9) if self.file_output == None: return notification
# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import json import outputs import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.dealer') class BonnieDealer(object): def __init__(self, *args, **kw): self.output_modules = {} for _class in outputs.list_classes(): __class = _class() self.output_modules[__class] = __class.register(callback=self.register_output) def register_output(self, interests): pass def accept_notification(self, notification): parsed = json.loads(notification) event = parsed['event']
# # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import logging import os import pdb import sys import traceback import bonnie from bonnie.translate import _ log = bonnie.getLogger('bonnie.plugins') conf = bonnie.getConf() class KolabPlugins(object): """ Detects, loads and interfaces with plugins for different Kolab components. """ def __init__(self): """ Searches the plugin directory for plugins, and loads them into a list. """ self.plugins = {} for plugin_path in [
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import json import pykolab from pykolab.imap import IMAP import bonnie from bonnie.utils import parse_imap_uri from bonnie.utils import imap_folder_path conf = bonnie.getConf() log = bonnie.getLogger('bonnie.collector.IMAPDataHandler') class IMAPDataHandler(object): """ Collector handler to provide metadata from IMAP """ def __init__(self, *args, **kw): # load pykolab conf conf = pykolab.getConf() if not hasattr(conf, 'defaults'): conf.finalize_conf(fatal=False) self.imap = IMAP() def register(self, callback):
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import datetime import time from bonnie.broker.state import init_db, Job, Worker import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.broker.ZMQBroker') def add(identity, state = b'READY'): db = init_db('workers') db.add(Worker(identity, state)) db.commit() def count(): db = init_db('workers') result = db.query(Worker).count() return result def count_by_state(state): db = init_db('workers') result = db.query(Worker).filter_by(state=state).count() return result
# # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import datetime from dateutil.parser import parse from dateutil.tz import tzutc import riak import json import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker.RiakOutput') class RiakOutput(object): def __init__(self, *args, **kw): riak_output_address = conf.get( 'worker', 'riak_output_address' ) if riak_output_address == None: riak_output_address = 'localhost' self.riak = riak.Riak( host=riak_output_address )
""" This is a worker node pulling jobs from a ZMQ broker. """ import os import random import socket import time import zmq import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker.ZMQInput') class ZMQInput(object): running = False def __init__(self, *args, **kw): self.state = b"READY" self.job_uuid = None self.lastping = 0 self.report_timestamp = 0 self.busy_start = 0 def name(self): return 'zmq_input' def register(self, *args, **kw):
import re import json import urllib import hashlib import random import time import datetime import elasticsearch from dateutil.tz import tzutc from bonnie.utils import parse_imap_uri from bonnie.worker.storage import CachedDict import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker.ElasticSearchStorage') class ElasticSearchStorage(object): """ Storage node writing object data into Elasticsearch """ default_index = 'objects' default_doctype = 'object' folders_index = 'objects' folders_doctype = 'folder' users_index = 'objects' users_doctype = 'user' def __init__(self, *args, **kw): elasticsearch_output_address = conf.get( 'worker',
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import os import socket import zmq from zmq.eventloop import ioloop, zmqstream import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.dealer.ZMQOutput') class ZMQOutput(object): def __init__(self, *args, **kw): self.context = zmq.Context() ioloop.install() zmq_broker_address = conf.get('dealer', 'zmq_broker_address') if zmq_broker_address == None: zmq_broker_address = "tcp://localhost:5570" self.dealer = self.context.socket(zmq.DEALER) self.dealer.identity = (u"Dealer-%s-%s" % (socket.getfqdn(), os.getpid())).encode('ascii') self.dealer.connect(zmq_broker_address)
import os import urllib import urlparse import datetime import subprocess from dateutil.parser import parse as parse_date from dateutil.tz import tzutc from email import message_from_string from email.header import decode_header from email.utils import getaddresses import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.utils') def expand_uidset(uidset): """ Expand the given UID set string into a complete set of values Examples: 1,2,5 => [1,2,5] or 1:4 => [1,2,3,4] """ _uids = [] for _uid in uidset.split(','): if len(_uid.split(':')) > 1: for __uid in range((int)(_uid.split(':')[0]), (int)(_uid.split(':')[1])+1): _uids.append("%d" % (__uid)) else: _uids.append(str(_uid))
# # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # """ Changelog handler for groupware objects """ import re import time import bonnie from dateutil.tz import tzutc from bonnie.worker.handlers import HandlerBase log = bonnie.getLogger("bonnie.worker.changelog") # timestamp (* 10) at the year 2014 REVBASE = 13885344000 class ChangelogHandler(HandlerBase): events = ["MessageAppend", "vnd.cmu.MessageMove"] def __init__(self, *args, **kw): HandlerBase.__init__(self, *args, **kw) def register(self, callback): kw = {"callback": self.run} interests = dict((event, kw) for event in self.events) self.worker = callback(interests)
# import os import time import json from email import message_from_string import bonnie from bonnie import utils from bonnie.utils import expand_uidset from bonnie.utils import parse_imap_uri from bonnie.utils import decode_message_headers conf = bonnie.getConf() log = bonnie.getLogger('bonnie.collector.MessageDataHandler') class MessageDataHandler(object): def __init__(self, *args, **kw): pass def register(self, callback): interests = { 'FETCH': { 'callback': self.retrieve_contents_for_messages }, 'HEADER': { 'callback': self.retrieve_headers_for_messages } }
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import os import sys import grp import pwd import signal import traceback import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie') class BonnieDaemon(object): """ A standard daemon process abstraction layer for Bonnie. This class provides the following capabilities for Bonnie daemons (through inheritance): * standard command-line options * :func:`dropping privileges <drop_privileges>` * :func:`signal handling <signal_handlers>` """ pidfile = "/var/run/bonnie/bonnie.pid" def __init__(self, *args, **kw):
from sqlalchemy.orm import relationship from sqlalchemy.schema import Index try: from sqlalchemy.orm import relationship except: from sqlalchemy.orm import relation as relationship try: from sqlalchemy.orm import sessionmaker except: from sqlalchemy.orm import create_session import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.broker.state') DeclarativeBase = declarative_base() db = None collector_interest_table = Table( 'collector_interest', DeclarativeBase.metadata, Column('collector_id', Integer, ForeignKey('collector.id')), Column('interest_id', Integer, ForeignKey('interest.id')) ) ## ## Classes ##
# along with this program. If not, see <http://www.gnu.org/licenses/>. # import os import sys import inputs import handlers import multiprocessing from distutils import version from bonnie.utils import parse_imap_uri from bonnie.daemon import BonnieDaemon import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.collector') class BonnieCollector(BonnieDaemon): pidfile = "/var/run/bonnie/collector.pid" def __init__(self, *args, **kw): super(BonnieCollector, self).__init__(*args, **kw) self.input_interests = {} self.input_modules = {} self.handler_interests = {} self.num_threads = int(conf.get('collector', 'num_threads', 5)) self.num_threads_busy = 0
import datetime import hashlib import json import re import urllib import random import riak import time from dateutil.tz import tzutc from bonnie.utils import parse_imap_uri from bonnie.worker.storage import CachedDict import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker.RiakStorage') class RiakStorage(object): """ Storage node writing object data into Riak """ default_index = 'objects' default_doctype = 'object' folders_index = 'objects' folders_doctype = 'folder' users_index = 'objects' users_doctype = 'user' def __init__(self, *args, **kw): riak_output_address = conf.get( 'worker',
This is a collector node pulling jobs from a ZMQ broker. """ import datetime import json import os import socket import time import zmq from zmq.eventloop import ioloop, zmqstream import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.collector.ZMQInput') class ZMQInput(object): report_timestamp = 0 state = b"READY" def __init__(self, *args, **kw): self.interests = [] self.context = zmq.Context() zmq_broker_address = conf.get('collector', 'zmq_broker_address') if zmq_broker_address == None: zmq_broker_address = "tcp://localhost:5571" self.identity = u"Collector-%s" % (socket.getfqdn())
import json import time import handlers import inputs import outputs import storage import signal from bonnie.translate import _ from bonnie.daemon import BonnieDaemon from multiprocessing import Process import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker') class BonnieWorker(BonnieDaemon): """ Bonnie Worker specific version of a :class:`Bonnie Daemon <bonnie.daemon.BonnieDaemon>` """ #: The process ID file to use. pidfile = "/var/run/bonnie/worker.pid" def __init__(self, *args, **kw): worker_group = conf.add_cli_parser_option_group("Worker Options") worker_group.add_option( "-n", "--num-childs",
# This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import logging import os from optparse import OptionParser from ConfigParser import SafeConfigParser import bonnie log = bonnie.getLogger('bonnie.conf') from bonnie.translate import _ class Conf(object): cfg_parser = None def __init__(self): self.create_options() self.config = SafeConfigParser() if os.path.exists('/etc/bonnie/bonnie.conf'): self.config.read('/etc/bonnie/bonnie.conf') elif os.path.exists(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf')): self.config.read(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf')) self.defaults = Defaults()
def __init__(self, *args, **kw): super(MessageHandlerBase, self).__init__(*args, **kw) self.log = bonnie.getLogger('bonnie.worker.' + self.event)
# # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import datetime from dateutil.parser import parse from dateutil.tz import tzutc import elasticsearch import json import bonnie conf = bonnie.getConf() log = bonnie.getLogger('bonnie.worker.ElasticSearchOutput') class ElasticSearchOutput(object): def __init__(self, *args, **kw): elasticsearch_output_address = conf.get( 'worker', 'elasticsearch_output_address' ) if elasticsearch_output_address == None: elasticsearch_output_address = 'localhost' self.es = elasticsearch.Elasticsearch( host=elasticsearch_output_address )