def __init__(self, name, config, uri): self.queues = {} self.config = config self.uri = uri self.log = get_logger(name) for qname in config.persist_queues: num_workers = self.config.persist_queues[qname][1] if num_workers > 1: self.queues[qname] = MultiWorkerQueue(qname, MemcachedPersistQueue, uri, num_workers) else: self.queues[qname] = MemcachedPersistQueue(qname, uri)
def __init__(self, qprefix, qtype, uri, num_workers): self.qprefix = qprefix self.qtype = qtype self.num_workers = num_workers self.queues = {} self.worker_map = {} self.log = get_logger('MultiWorkerQueue') self.worker_load = [] for i in range(1, num_workers + 1): name = "%s_%d" % (qprefix, i) self.queues[name] = qtype(name, uri) self.worker_load.append([i, 0])
def __init__(self, config, qname, persistq): self.log = get_logger("espersistd.%s" % qname) self.config = config self.qname = qname self.running = False self.sleeping = False if persistq: self.persistq = persistq else: self.persistq = MemcachedPersistQueue(qname, config.espersistd_uri) self.data_count = 0 self.last_stats = time.time()
def __init__(self, name, config): self.config = config self.sinks = [] self.log = get_logger("espersist.client") if not self.config.espoll_persist_uri: self.log.warning( "espoll_persist_uri not defined: all data will be discarded") return for uri in config.espoll_persist_uri: (kind, kind_uri) = uri.split(':', 1) sink = eval('%s(name, config, "%s")' % (kind, kind_uri)) self.sinks.append(sink)
def __init__(self, qname, memcached_uri): super(MemcachedPersistQueue, self).__init__(qname) self.log = get_logger("MemcachedPersistQueue_%s" % self.qname) # self.mc = memcache.Client([memcached_uri]) self.mc = memcache.Client([memcached_uri], pickler=JsonSerializer, unpickler=JsonSerializer) self.last_added = '%s_%s_last_added' % (self.PREFIX, self.qname) la = self.mc.get(self.last_added) if not la: self.mc.set(self.last_added, 0) self.last_read = '%s_%s_last_read' % (self.PREFIX, self.qname) lr = self.mc.get(self.last_read) if not lr: self.mc.set(self.last_read, 0)
def __init__(self, name, opts, args, config): self.name = name self.opts = opts self.args = args self.config = config self.hostname = socket.gethostname() self.log = get_logger(self.name) self.running = False self.last_reload = time.time() self.last_penalty_empty = time.time() self.reload_interval = 30 self.penalty_interval = 300 self.devices = Device.objects.active_as_dict() self.persistq = Queue.Queue() self.snmp_poller = AsyncSNMPPoller(config=self.config, name="espolld.snmp_poller") self.pollers = {}
class PersistManager(object): def __init__(self, name, config, opts): self.name = name self.config = config self.opts = opts self.runing = False self.processes = {} if tsdb: if config.tsdb_root and not os.path.isdir(config.tsdb_root): try: tsdb.TSDB.create(config.tsdb_root) except Exception, e: print >>sys.stderr, "unable to create TSDB root: %s: %s" % (config.tsdb_root, str(e)) init_logging(name, config.syslog_facility, level=config.syslog_priority, debug=opts.debug) self.log = get_logger(name) # save the location of the calling script for later use # (os.path.abspath uses current directory and daemonize does a cd /) self.caller_path = os.path.abspath(__main__.__file__) if not self.opts.debug: exc_handler = setup_exc_handler(name, config) exc_handler.install() daemonize(name, config.pid_dir, log_stdout_stderr=config.syslog_facility) os.umask(0022) setproctitle(name) signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop)
def __init__(self, config=None, name="AsyncSNMPPoller", maxrepetitions=25): self.maxrepetitions = maxrepetitions self.name = name self.config = config self.reqmap = {} self.sessions = SNMPManager(local_dir="/usr/local/share/snmp", threaded_processor=True) self.log = get_logger(self.name) if self.config: for mib_dir in self.config.mib_dirs: self.log.info("add mib dir %s" % mib_dir) self.sessions.add_mib_dir(mib_dir) self.sessions.refresh_mibs() for mib in self.config.mibs: self.log.info("add mib %s" % mib) self.sessions.read_module(mib) self.sessions.bind('response', '1', None, self._callback) self.sessions.bind('timeout', '1', None, self._errback)
def __init__(self, config, device, oidset, poller, persistq): self.config = config self.device = device self.oidset = oidset self.poller = poller self.persistq = persistq self.name = "espolld." + self.device.name + "." + self.oidset.name self.next_poll = int(time.time() - 1) self.oids = self.oidset.oids # in some pollers we poll oids beyond the ones which are used # for that poller, so we make a copy in poll_oids self.poll_oids = [o.name for o in self.oids.all()] self.running = True self.log = get_logger(self.name) self.poller_args = {} if self.oidset.poller_args is not None: for arg in self.oidset.poller_args.split(): (var, val) = arg.split('=') self.poller_args[var] = val self.polling_round = 0
from esmond.api.api_v2 import (DataObject, _get_ersatz_esmond_api_queryset, DjangoModelPerm) from esmond.api.perfsonar.types import * from esmond.cassandra import KEY_DELIMITER, CASSANDRA_DB, AGG_TYPES, ConnectionException, RawRateData, BaseRateBin, RawData, AggregationBin from esmond.config import get_config_path, get_config from esmond.util import get_logger # # Logger # log = get_logger(__name__) # # Cassandra db connection # try: db = CASSANDRA_DB(get_config(get_config_path())) EVENT_TYPE_CF_MAP = { 'histogram': db.raw_cf, 'integer': db.rate_cf, 'json': db.raw_cf, 'percentage': db.agg_cf, 'subinterval': db.raw_cf, 'float': db.agg_cf } except ConnectionException, e:
(opts, args) = oparse.parse_args(args=argv) opts.config_file = os.path.abspath(opts.config_file) try: config = get_config(opts.config_file, opts) except ConfigError, e: print >>sys.stderr, e sys.exit(1) name = "espersistd.%s" % opts.role if opts.qname: name += ".%s" % opts.qname log = get_logger(name) if opts.role == 'manager': try: PersistManager(name, config, opts).run() except Exception, e: log.error("Problem with manager module: %s" % e, exc_info=True) raise sys.exit(1) elif opts.role == 'worker': try: worker(name, config, opts) except Exception, e: log.error("Problem with worker module: %s" % e, exc_info=True) raise sys.exit(1)