#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import socket
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.dealer.FileOutput')
class FileOutput(object):
def __init__(self, *args, **kw):
self.file_output = conf.get("dealer", "file_path")
def name(self):
return 'file_output'
def register(self, *args, **kw):
return self.run
def run(self, notification):
log.debug("Notification received: %r" % (notification), level=9)
if self.file_output == None:
return notification
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
import outputs
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.dealer')
class BonnieDealer(object):
def __init__(self, *args, **kw):
self.output_modules = {}
for _class in outputs.list_classes():
__class = _class()
self.output_modules[__class] = __class.register(callback=self.register_output)
def register_output(self, interests):
pass
def accept_notification(self, notification):
parsed = json.loads(notification)
event = parsed['event']
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
import pdb
import sys
import traceback
import bonnie
from bonnie.translate import _
log = bonnie.getLogger('bonnie.plugins')
conf = bonnie.getConf()
class KolabPlugins(object):
"""
Detects, loads and interfaces with plugins for different
Kolab components.
"""
def __init__(self):
"""
Searches the plugin directory for plugins, and loads
them into a list.
"""
self.plugins = {}
for plugin_path in [
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
import pykolab
from pykolab.imap import IMAP
import bonnie
from bonnie.utils import parse_imap_uri
from bonnie.utils import imap_folder_path
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.collector.IMAPDataHandler')
class IMAPDataHandler(object):
"""
Collector handler to provide metadata from IMAP
"""
def __init__(self, *args, **kw):
# load pykolab conf
conf = pykolab.getConf()
if not hasattr(conf, 'defaults'):
conf.finalize_conf(fatal=False)
self.imap = IMAP()
def register(self, callback):
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
import time
from bonnie.broker.state import init_db, Job, Worker
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.broker.ZMQBroker')
def add(identity, state = b'READY'):
db = init_db('workers')
db.add(Worker(identity, state))
db.commit()
def count():
db = init_db('workers')
result = db.query(Worker).count()
return result
def count_by_state(state):
db = init_db('workers')
result = db.query(Worker).filter_by(state=state).count()
return result
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc
import riak
import json
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker.RiakOutput')
class RiakOutput(object):
def __init__(self, *args, **kw):
riak_output_address = conf.get(
'worker',
'riak_output_address'
)
if riak_output_address == None:
riak_output_address = 'localhost'
self.riak = riak.Riak(
host=riak_output_address
)
"""
This is a worker node pulling jobs from a ZMQ broker.
"""
import os
import random
import socket
import time
import zmq
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker.ZMQInput')
class ZMQInput(object):
running = False
def __init__(self, *args, **kw):
self.state = b"READY"
self.job_uuid = None
self.lastping = 0
self.report_timestamp = 0
self.busy_start = 0
def name(self):
return 'zmq_input'
def register(self, *args, **kw):
import re
import json
import urllib
import hashlib
import random
import time
import datetime
import elasticsearch
from dateutil.tz import tzutc
from bonnie.utils import parse_imap_uri
from bonnie.worker.storage import CachedDict
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker.ElasticSearchStorage')
class ElasticSearchStorage(object):
"""
Storage node writing object data into Elasticsearch
"""
default_index = 'objects'
default_doctype = 'object'
folders_index = 'objects'
folders_doctype = 'folder'
users_index = 'objects'
users_doctype = 'user'
def __init__(self, *args, **kw):
elasticsearch_output_address = conf.get(
'worker',
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import socket
import zmq
from zmq.eventloop import ioloop, zmqstream
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.dealer.ZMQOutput')
class ZMQOutput(object):
def __init__(self, *args, **kw):
self.context = zmq.Context()
ioloop.install()
zmq_broker_address = conf.get('dealer', 'zmq_broker_address')
if zmq_broker_address == None:
zmq_broker_address = "tcp://localhost:5570"
self.dealer = self.context.socket(zmq.DEALER)
self.dealer.identity = (u"Dealer-%s-%s" % (socket.getfqdn(), os.getpid())).encode('ascii')
self.dealer.connect(zmq_broker_address)
import os
import urllib
import urlparse
import datetime
import subprocess
from dateutil.parser import parse as parse_date
from dateutil.tz import tzutc
from email import message_from_string
from email.header import decode_header
from email.utils import getaddresses
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.utils')
def expand_uidset(uidset):
"""
Expand the given UID set string into a complete set of values
Examples: 1,2,5 => [1,2,5] or 1:4 => [1,2,3,4]
"""
_uids = []
for _uid in uidset.split(','):
if len(_uid.split(':')) > 1:
for __uid in range((int)(_uid.split(':')[0]), (int)(_uid.split(':')[1])+1):
_uids.append("%d" % (__uid))
else:
_uids.append(str(_uid))
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Changelog handler for groupware objects
"""
import re
import time
import bonnie
from dateutil.tz import tzutc
from bonnie.worker.handlers import HandlerBase
log = bonnie.getLogger("bonnie.worker.changelog")
# timestamp (* 10) at the year 2014
REVBASE = 13885344000
class ChangelogHandler(HandlerBase):
events = ["MessageAppend", "vnd.cmu.MessageMove"]
def __init__(self, *args, **kw):
HandlerBase.__init__(self, *args, **kw)
def register(self, callback):
kw = {"callback": self.run}
interests = dict((event, kw) for event in self.events)
self.worker = callback(interests)
#
import os
import time
import json
from email import message_from_string
import bonnie
from bonnie import utils
from bonnie.utils import expand_uidset
from bonnie.utils import parse_imap_uri
from bonnie.utils import decode_message_headers
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.collector.MessageDataHandler')
class MessageDataHandler(object):
def __init__(self, *args, **kw):
pass
def register(self, callback):
interests = {
'FETCH': {
'callback': self.retrieve_contents_for_messages
},
'HEADER': {
'callback': self.retrieve_headers_for_messages
}
}
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import grp
import pwd
import signal
import traceback
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie')
class BonnieDaemon(object):
"""
A standard daemon process abstraction layer for Bonnie.
This class provides the following capabilities for Bonnie
daemons (through inheritance):
* standard command-line options
* :func:`dropping privileges <drop_privileges>`
* :func:`signal handling <signal_handlers>`
"""
pidfile = "/var/run/bonnie/bonnie.pid"
def __init__(self, *args, **kw):
from sqlalchemy.orm import relationship
from sqlalchemy.schema import Index
try:
from sqlalchemy.orm import relationship
except:
from sqlalchemy.orm import relation as relationship
try:
from sqlalchemy.orm import sessionmaker
except:
from sqlalchemy.orm import create_session
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.broker.state')
DeclarativeBase = declarative_base()
db = None
collector_interest_table = Table(
'collector_interest',
DeclarativeBase.metadata,
Column('collector_id', Integer, ForeignKey('collector.id')),
Column('interest_id', Integer, ForeignKey('interest.id'))
)
##
## Classes
##
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import sys
import inputs
import handlers
import multiprocessing
from distutils import version
from bonnie.utils import parse_imap_uri
from bonnie.daemon import BonnieDaemon
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.collector')
class BonnieCollector(BonnieDaemon):
pidfile = "/var/run/bonnie/collector.pid"
def __init__(self, *args, **kw):
super(BonnieCollector, self).__init__(*args, **kw)
self.input_interests = {}
self.input_modules = {}
self.handler_interests = {}
self.num_threads = int(conf.get('collector', 'num_threads', 5))
self.num_threads_busy = 0
import datetime
import hashlib
import json
import re
import urllib
import random
import riak
import time
from dateutil.tz import tzutc
from bonnie.utils import parse_imap_uri
from bonnie.worker.storage import CachedDict
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker.RiakStorage')
class RiakStorage(object):
"""
Storage node writing object data into Riak
"""
default_index = 'objects'
default_doctype = 'object'
folders_index = 'objects'
folders_doctype = 'folder'
users_index = 'objects'
users_doctype = 'user'
def __init__(self, *args, **kw):
riak_output_address = conf.get(
'worker',
This is a collector node pulling jobs from a ZMQ broker.
"""
import datetime
import json
import os
import socket
import time
import zmq
from zmq.eventloop import ioloop, zmqstream
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.collector.ZMQInput')
class ZMQInput(object):
report_timestamp = 0
state = b"READY"
def __init__(self, *args, **kw):
self.interests = []
self.context = zmq.Context()
zmq_broker_address = conf.get('collector', 'zmq_broker_address')
if zmq_broker_address == None:
zmq_broker_address = "tcp://localhost:5571"
self.identity = u"Collector-%s" % (socket.getfqdn())
import json
import time
import handlers
import inputs
import outputs
import storage
import signal
from bonnie.translate import _
from bonnie.daemon import BonnieDaemon
from multiprocessing import Process
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker')
class BonnieWorker(BonnieDaemon):
"""
Bonnie Worker specific version of a :class:`Bonnie Daemon <bonnie.daemon.BonnieDaemon>`
"""
#: The process ID file to use.
pidfile = "/var/run/bonnie/worker.pid"
def __init__(self, *args, **kw):
worker_group = conf.add_cli_parser_option_group("Worker Options")
worker_group.add_option(
"-n",
"--num-childs",
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import os
from optparse import OptionParser
from ConfigParser import SafeConfigParser
import bonnie
log = bonnie.getLogger('bonnie.conf')
from bonnie.translate import _
class Conf(object):
cfg_parser = None
def __init__(self):
self.create_options()
self.config = SafeConfigParser()
if os.path.exists('/etc/bonnie/bonnie.conf'):
self.config.read('/etc/bonnie/bonnie.conf')
elif os.path.exists(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf')):
self.config.read(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf'))
self.defaults = Defaults()
def __init__(self, *args, **kw):
super(MessageHandlerBase, self).__init__(*args, **kw)
self.log = bonnie.getLogger('bonnie.worker.' + self.event)
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
from dateutil.parser import parse
from dateutil.tz import tzutc
import elasticsearch
import json
import bonnie
conf = bonnie.getConf()
log = bonnie.getLogger('bonnie.worker.ElasticSearchOutput')
class ElasticSearchOutput(object):
def __init__(self, *args, **kw):
elasticsearch_output_address = conf.get(
'worker',
'elasticsearch_output_address'
)
if elasticsearch_output_address == None:
elasticsearch_output_address = 'localhost'
self.es = elasticsearch.Elasticsearch(
host=elasticsearch_output_address
)
