def __init__(self, name, config, uri):
self.queues = {}
self.config = config
self.uri = uri
self.log = get_logger(name)
for qname in config.persist_queues:
num_workers = self.config.persist_queues[qname][1]
if num_workers > 1:
self.queues[qname] = MultiWorkerQueue(qname,
MemcachedPersistQueue, uri, num_workers)
else:
self.queues[qname] = MemcachedPersistQueue(qname, uri)
def __init__(self, qprefix, qtype, uri, num_workers):
self.qprefix = qprefix
self.qtype = qtype
self.num_workers = num_workers
self.queues = {}
self.worker_map = {}
self.log = get_logger('MultiWorkerQueue')
self.worker_load = []
for i in range(1, num_workers + 1):
name = "%s_%d" % (qprefix, i)
self.queues[name] = qtype(name, uri)
self.worker_load.append([i, 0])
def __init__(self, config, qname, persistq):
self.log = get_logger("espersistd.%s" % qname)
self.config = config
self.qname = qname
self.running = False
self.sleeping = False
if persistq:
self.persistq = persistq
else:
self.persistq = MemcachedPersistQueue(qname, config.espersistd_uri)
self.data_count = 0
self.last_stats = time.time()
def __init__(self, name, config):
self.config = config
self.sinks = []
self.log = get_logger("espersist.client")
if not self.config.espoll_persist_uri:
self.log.warning(
"espoll_persist_uri not defined: all data will be discarded")
return
for uri in config.espoll_persist_uri:
(kind, kind_uri) = uri.split(':', 1)
sink = eval('%s(name, config, "%s")' % (kind, kind_uri))
self.sinks.append(sink)
def __init__(self, qname, memcached_uri):
super(MemcachedPersistQueue, self).__init__(qname)
self.log = get_logger("MemcachedPersistQueue_%s" % self.qname)
# self.mc = memcache.Client([memcached_uri])
self.mc = memcache.Client([memcached_uri],
pickler=JsonSerializer, unpickler=JsonSerializer)
self.last_added = '%s_%s_last_added' % (self.PREFIX, self.qname)
la = self.mc.get(self.last_added)
if not la:
self.mc.set(self.last_added, 0)
self.last_read = '%s_%s_last_read' % (self.PREFIX, self.qname)
lr = self.mc.get(self.last_read)
if not lr:
self.mc.set(self.last_read, 0)
def __init__(self, name, opts, args, config):
self.name = name
self.opts = opts
self.args = args
self.config = config
self.hostname = socket.gethostname()
self.log = get_logger(self.name)
self.running = False
self.last_reload = time.time()
self.last_penalty_empty = time.time()
self.reload_interval = 30
self.penalty_interval = 300
self.devices = Device.objects.active_as_dict()
self.persistq = Queue.Queue()
self.snmp_poller = AsyncSNMPPoller(config=self.config,
name="espolld.snmp_poller")
self.pollers = {}
def __init__(self, name, opts, args, config):
self.name = name
self.opts = opts
self.args = args
self.config = config
self.hostname = socket.gethostname()
self.log = get_logger(self.name)
self.running = False
self.last_reload = time.time()
self.last_penalty_empty = time.time()
self.reload_interval = 30
self.penalty_interval = 300
self.devices = Device.objects.active_as_dict()
self.persistq = Queue.Queue()
self.snmp_poller = AsyncSNMPPoller(config=self.config,
name="espolld.snmp_poller")
self.pollers = {}
class PersistManager(object):
def __init__(self, name, config, opts):
self.name = name
self.config = config
self.opts = opts
self.runing = False
self.processes = {}
if tsdb:
if config.tsdb_root and not os.path.isdir(config.tsdb_root):
try:
tsdb.TSDB.create(config.tsdb_root)
except Exception, e:
print >>sys.stderr, "unable to create TSDB root: %s: %s" % (config.tsdb_root, str(e))
init_logging(name, config.syslog_facility, level=config.syslog_priority,
debug=opts.debug)
self.log = get_logger(name)
# save the location of the calling script for later use
# (os.path.abspath uses current directory and daemonize does a cd /)
self.caller_path = os.path.abspath(__main__.__file__)
if not self.opts.debug:
exc_handler = setup_exc_handler(name, config)
exc_handler.install()
daemonize(name, config.pid_dir,
log_stdout_stderr=config.syslog_facility)
os.umask(0022)
setproctitle(name)
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def __init__(self, config=None, name="AsyncSNMPPoller", maxrepetitions=25):
self.maxrepetitions = maxrepetitions
self.name = name
self.config = config
self.reqmap = {}
self.sessions = SNMPManager(local_dir="/usr/local/share/snmp",
threaded_processor=True)
self.log = get_logger(self.name)
if self.config:
for mib_dir in self.config.mib_dirs:
self.log.info("add mib dir %s" % mib_dir)
self.sessions.add_mib_dir(mib_dir)
self.sessions.refresh_mibs()
for mib in self.config.mibs:
self.log.info("add mib %s" % mib)
self.sessions.read_module(mib)
self.sessions.bind('response', '1', None, self._callback)
self.sessions.bind('timeout', '1', None, self._errback)
def __init__(self, config, device, oidset, poller, persistq):
self.config = config
self.device = device
self.oidset = oidset
self.poller = poller
self.persistq = persistq
self.name = "espolld." + self.device.name + "." + self.oidset.name
self.next_poll = int(time.time() - 1)
self.oids = self.oidset.oids
# in some pollers we poll oids beyond the ones which are used
# for that poller, so we make a copy in poll_oids
self.poll_oids = [o.name for o in self.oids.all()]
self.running = True
self.log = get_logger(self.name)
self.poller_args = {}
if self.oidset.poller_args is not None:
for arg in self.oidset.poller_args.split():
(var, val) = arg.split('=')
self.poller_args[var] = val
self.polling_round = 0
def __init__(self, config=None, name="AsyncSNMPPoller", maxrepetitions=25):
self.maxrepetitions = maxrepetitions
self.name = name
self.config = config
self.reqmap = {}
self.sessions = SNMPManager(local_dir="/usr/local/share/snmp",
threaded_processor=True)
self.log = get_logger(self.name)
if self.config:
for mib_dir in self.config.mib_dirs:
self.log.info("add mib dir %s" % mib_dir)
self.sessions.add_mib_dir(mib_dir)
self.sessions.refresh_mibs()
for mib in self.config.mibs:
self.log.info("add mib %s" % mib)
self.sessions.read_module(mib)
self.sessions.bind('response', '1', None, self._callback)
self.sessions.bind('timeout', '1', None, self._errback)
def __init__(self, config, device, oidset, poller, persistq):
self.config = config
self.device = device
self.oidset = oidset
self.poller = poller
self.persistq = persistq
self.name = "espolld." + self.device.name + "." + self.oidset.name
self.next_poll = int(time.time() - 1)
self.oids = self.oidset.oids
# in some pollers we poll oids beyond the ones which are used
# for that poller, so we make a copy in poll_oids
self.poll_oids = [o.name for o in self.oids.all()]
self.running = True
self.log = get_logger(self.name)
self.poller_args = {}
if self.oidset.poller_args is not None:
for arg in self.oidset.poller_args.split():
(var, val) = arg.split('=')
self.poller_args[var] = val
self.polling_round = 0
from esmond.api.api_v2 import (DataObject, _get_ersatz_esmond_api_queryset,
DjangoModelPerm)
from esmond.api.perfsonar.types import *
from esmond.cassandra import KEY_DELIMITER, CASSANDRA_DB, AGG_TYPES, ConnectionException, RawRateData, BaseRateBin, RawData, AggregationBin
from esmond.config import get_config_path, get_config
from esmond.util import get_logger
#
# Logger
#
log = get_logger(__name__)
#
# Cassandra db connection
#
try:
db = CASSANDRA_DB(get_config(get_config_path()))
EVENT_TYPE_CF_MAP = {
'histogram': db.raw_cf,
'integer': db.rate_cf,
'json': db.raw_cf,
'percentage': db.agg_cf,
'subinterval': db.raw_cf,
'float': db.agg_cf
}
except ConnectionException, e:
(opts, args) = oparse.parse_args(args=argv)
opts.config_file = os.path.abspath(opts.config_file)
try:
config = get_config(opts.config_file, opts)
except ConfigError, e:
print >>sys.stderr, e
sys.exit(1)
name = "espersistd.%s" % opts.role
if opts.qname:
name += ".%s" % opts.qname
log = get_logger(name)
if opts.role == 'manager':
try:
PersistManager(name, config, opts).run()
except Exception, e:
log.error("Problem with manager module: %s" % e, exc_info=True)
raise
sys.exit(1)
elif opts.role == 'worker':
try:
worker(name, config, opts)
except Exception, e:
log.error("Problem with worker module: %s" % e, exc_info=True)
raise
sys.exit(1)