Python RuleManager Examples

Example #1

File: service.py Project: zhpn/carbon

def setupReceivers(root_service, settings):
  from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver

  for protocol, interface, port in [
      (MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT),
      (MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT)
    ]:
    if port:
      factory = ServerFactory()
      factory.protocol = protocol
      service = TCPServer(port, factory, interface=interface)
      service.setServiceParent(root_service)

  if settings.ENABLE_UDP_LISTENER:
      service = UDPServer(int(settings.UDP_RECEIVER_PORT),
                          MetricDatagramReceiver(),
                          interface=settings.UDP_RECEIVER_INTERFACE)
      service.setServiceParent(root_service)

  if settings.ENABLE_AMQP:
    from carbon import amqp_listener
    amqp_host = settings.AMQP_HOST
    amqp_port = settings.AMQP_PORT
    amqp_user = settings.AMQP_USER
    amqp_password = settings.AMQP_PASSWORD
    amqp_verbose = settings.AMQP_VERBOSE
    amqp_vhost = settings.AMQP_VHOST
    amqp_spec = settings.AMQP_SPEC
    amqp_exchange_name = settings.AMQP_EXCHANGE

    factory = amqp_listener.createAMQPListener(
      amqp_user,
      amqp_password,
      vhost=amqp_vhost,
      spec=amqp_spec,
      exchange_name=amqp_exchange_name,
      verbose=amqp_verbose)
    service = TCPClient(amqp_host, amqp_port, factory)
    service.setServiceParent(root_service)

  if settings.ENABLE_MANHOLE:
    from carbon import manhole

    # Configure application components
    if settings.RELAY_METHOD == 'rules':
      router = RelayRulesRouter(settings["relay-rules"])
    elif settings.RELAY_METHOD == 'consistent-hashing':
      router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
    elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
      from carbon.aggregator.rules import RuleManager
      RuleManager.read_from(settings["aggregation-rules"])
      router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
    elif settings.RELAY_METHOD == 'remove-node-consistent-hashing':
      router = RemoveNodeConsistentHashingRouter(settings.REPLICATION_FACTOR, settings.REMOVE_NODE_INDEX)
    factory = manhole.createManholeListener()
    service = TCPServer(
      settings.MANHOLE_PORT,
      factory,
      interface=settings.MANHOLE_INTERFACE)
    service.setServiceParent(root_service)

Example #2

File: service.py Project: laiwei/carbon

def createAggregatorService(config):
    from carbon.aggregator import receiver
    from carbon.aggregator.rules import RuleManager
    from carbon.routers import ConsistentHashingRouter
    from carbon.client import CarbonClientManager
    from carbon.rewrite import RewriteRuleManager
    from carbon.conf import settings
    from carbon import events

    root_service = createBaseService(config)

    # Configure application components
    router = ConsistentHashingRouter()
    client_manager = CarbonClientManager(router)
    client_manager.setServiceParent(root_service)

    events.metricReceived.addHandler(receiver.process)
    events.metricGenerated.addHandler(client_manager.sendDatapoint)

    RuleManager.read_from(settings["aggregation-rules"])
    if exists(settings["rewrite-rules"]):
        RewriteRuleManager.read_from(settings["rewrite-rules"])

    if not settings.DESTINATIONS:
      raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")

    for destination in util.parseDestinations(settings.DESTINATIONS):
      client_manager.startClient(destination)

    return root_service

Example #3

File: service.py Project: jacklesplat/ql_emc_graphite

def createAggregatorService(config):
    from carbon.aggregator import receiver
    from carbon.aggregator.rules import RuleManager
    from carbon.routers import ConsistentHashingRouter
    from carbon.client import CarbonClientManager
    from carbon.rewrite import RewriteRuleManager
    from carbon.conf import settings
    from carbon import events

    root_service = createBaseService(config)

    # Configure application components
    router = ConsistentHashingRouter(
        settings.REPLICATION_FACTOR,
        diverse_replicas=settings.DIVERSE_REPLICAS)
    client_manager = CarbonClientManager(router)
    client_manager.setServiceParent(root_service)

    events.metricReceived.addHandler(receiver.process)
    events.metricGenerated.addHandler(client_manager.sendDatapoint)

    RuleManager.read_from(settings["aggregation-rules"])
    if exists(settings["rewrite-rules"]):
        RewriteRuleManager.read_from(settings["rewrite-rules"])

    if not settings.DESTINATIONS:
        raise CarbonConfigException(
            "Required setting DESTINATIONS is missing from carbon.conf")

    for destination in util.parseDestinations(settings.DESTINATIONS):
        client_manager.startClient(destination)

    return root_service

Example #4

File: service.py Project: jacklesplat/ql_emc_graphite

def createRelayService(config):
    from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
    from carbon.client import CarbonClientManager
    from carbon.conf import settings
    from carbon import events

    root_service = createBaseService(config)

    # Configure application components
    if settings.RELAY_METHOD == 'rules':
        router = RelayRulesRouter(settings["relay-rules"])
    elif settings.RELAY_METHOD == 'consistent-hashing':
        router = ConsistentHashingRouter(
            settings.REPLICATION_FACTOR,
            diverse_replicas=settings.DIVERSE_REPLICAS)
    elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
        from carbon.aggregator.rules import RuleManager
        RuleManager.read_from(settings["aggregation-rules"])
        router = AggregatedConsistentHashingRouter(RuleManager,
                                                   settings.REPLICATION_FACTOR)

    client_manager = CarbonClientManager(router)
    client_manager.setServiceParent(root_service)

    events.metricReceived.addHandler(client_manager.sendDatapoint)
    events.metricGenerated.addHandler(client_manager.sendDatapoint)

    if not settings.DESTINATIONS:
        raise CarbonConfigException(
            "Required setting DESTINATIONS is missing from carbon.conf")

    for destination in util.parseDestinations(settings.DESTINATIONS):
        client_manager.startClient(destination)

    return root_service

Example #5

def setupRelayProcessor(root_service, settings):
    from carbon.routers import AggregatedConsistentHashingRouter, \
        ConsistentHashingRouter, RelayRulesRouter
    from carbon.client import CarbonClientManager

    if settings.RELAY_METHOD == 'consistent-hashing':
        router = ConsistentHashingRouter(
            settings.REPLICATION_FACTOR,
            diverse_replicas=settings.DIVERSE_REPLICAS)
    elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
        from carbon.aggregator.rules import RuleManager
        aggregation_rules_path = settings["aggregation-rules"]
        RuleManager.read_from(aggregation_rules_path)
        router = AggregatedConsistentHashingRouter(
            RuleManager,
            settings.REPLICATION_FACTOR,
            diverse_replicas=settings.DIVERSE_REPLICAS)
    elif settings.RELAY_METHOD == 'rules':
        router = RelayRulesRouter(settings["relay-rules"])

    state.client_manager = CarbonClientManager(router)
    state.client_manager.setServiceParent(root_service)

    for destination in util.parseDestinations(settings.DESTINATIONS):
        state.client_manager.startClient(destination)

Example #6

File: service.py Project: laiwei/carbon

def createRelayService(config):
    from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
    from carbon.client import CarbonClientManager
    from carbon.conf import settings
    from carbon import events

    root_service = createBaseService(config)

    # Configure application components
    if settings.RELAY_METHOD == 'rules':
      router = RelayRulesRouter(settings["relay-rules"])
    elif settings.RELAY_METHOD == 'consistent-hashing':
      router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
    elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
      from carbon.aggregator.rules import RuleManager
      RuleManager.read_from(settings["aggregation-rules"])
      router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)

    client_manager = CarbonClientManager(router)
    client_manager.setServiceParent(root_service)

    events.metricReceived.addHandler(client_manager.sendDatapoint)
    events.metricGenerated.addHandler(client_manager.sendDatapoint)
    events.specialMetricReceived.addHandler(client_manager.sendHighPriorityDatapoint)
    events.specialMetricGenerated.addHandler(client_manager.sendHighPriorityDatapoint)

    if not settings.DESTINATIONS:
      raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")

    for destination in util.parseDestinations(settings.DESTINATIONS):
      client_manager.startClient(destination)

    return root_service

Example #7

File: routers.py Project: xbernpa/carbon

  def __init__(self, settings):
    from carbon.aggregator.rules import RuleManager
    aggregation_rules_path = settings["aggregation-rules"]
    if aggregation_rules_path:
      RuleManager.read_from(aggregation_rules_path)

    self.hash_router = ConsistentHashingRouter(settings)
    self.agg_rules_manager = RuleManager

Example #8

File: service.py Project: connectthefuture/wbs-graphite

def setupAggregatorProcessor(root_service, settings):
  from carbon.aggregator.processor import AggregationProcessor  # Register the plugin class
  from carbon.aggregator.rules import RuleManager

  aggregation_rules_path = settings["aggregation-rules"]
  if not exists(aggregation_rules_path):
    raise CarbonConfigException("aggregation processor: file does not exist {0}".format(aggregation_rules_path))
  RuleManager.read_from(aggregation_rules_path)

Example #9

File: service.py Project: zhpn/carbon

def setupAggregatorProcessor(root_service, settings):
  from carbon.aggregator.processor import AggregationProcessor  # Register the plugin class
  from carbon.aggregator.rules import RuleManager

  aggregation_rules_path = settings["aggregation-rules"]
  if not exists(aggregation_rules_path):
    raise CarbonConfigException("aggregation processor: file does not exist {0}".format(aggregation_rules_path))
  RuleManager.read_from(aggregation_rules_path)

Example #10

File: cluster.py Project: klynch/carbonate

    def __init__(self, config, cluster='main', aggregation_rules=None):
        relay_method = config.relay_method(cluster=cluster)
        if relay_method == "consistent-hashing":
            self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
        elif relay_method == "aggregated-consistent-hashing":
            if aggregation_rules:
                RuleManager.read_from(aggregation_rules)
            self.ring = AggregatedConsistentHashingRouter(RuleManager, config.replication_factor(cluster))

        try:
            dest_list = config.destinations(cluster)
            self.destinations = util.parseDestinations(dest_list)
        except ValueError as e:
            raise SystemExit("Unable to parse destinations!" + str(e))

        for d in self.destinations:
            self.ring.addDestination(d)

Example #11

File: service.py Project: zhpn/carbon

def setupRelayProcessor(root_service, settings):
  from carbon.routers import AggregatedConsistentHashingRouter, \
      ConsistentHashingRouter, RelayRulesRouter
  from carbon.client import CarbonClientManager

  if settings.RELAY_METHOD == 'consistent-hashing':
    router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
  elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
    from carbon.aggregator.rules import RuleManager
    aggregation_rules_path = settings["aggregation-rules"]
    RuleManager.read_from(aggregation_rules_path)
    router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
  elif settings.RELAY_METHOD == 'rules':
    router = RelayRulesRouter(settings["relay-rules"])

  state.client_manager = CarbonClientManager(router)
  state.client_manager.setServiceParent(root_service)

  for destination in util.parseDestinations(settings.DESTINATIONS):
    state.client_manager.startClient(destination)

Example #12

File: service.py Project: mtodd/graphite

def createAggregatorService(config):
    from carbon.events import metricReceived
    from carbon.aggregator import receiver
    from carbon.aggregator.rules import RuleManager
    from carbon.aggregator import client
    from carbon.rewrite import RewriteRuleManager
    from carbon.conf import settings

    root_service = createBaseService(config)

    # Configure application components
    metricReceived.installHandler(receiver.process)
    RuleManager.read_from(settings["aggregation-rules"])
    if exists(settings["rewrite-rules"]):
        RewriteRuleManager.read_from(settings["rewrite-rules"])

    client.connect(settings["DESTINATION_HOST"],
                   int(settings["DESTINATION_PORT"]))

    return root_service

Example #13

    def __init__(self, config, cluster='main', aggregation_rules=None):
        relay_method = config.relay_method(cluster=cluster)
        if relay_method == "consistent-hashing":
            # Support multiple versions of carbon, the API changed in 0.10.
            args = inspect.getargspec(ConsistentHashingRouter.__init__).args
            if 'replication_factor' in args:
                r = ConsistentHashingRouter(config.replication_factor(cluster))
            else:

                class Settings(object):
                    REPLICATION_FACTOR = config.replication_factor(cluster)
                    DIVERSE_REPLICAS = False

                r = ConsistentHashingRouter(Settings())

            # 'hash_type' was added only in carbon 1.0.2 or master
            args = inspect.getargspec(ConsistentHashRing.__init__).args
            if 'hash_type' in args:
                r.ring = ConsistentHashRing(
                    hash_type=config.hashing_type(cluster))
        elif relay_method == "aggregated-consistent-hashing":
            if aggregation_rules:
                RuleManager.read_from(aggregation_rules)
            r = AggregatedConsistentHashingRouter(
                RuleManager, config.replication_factor(cluster))

        self.ring = r

        try:
            dest_list = config.destinations(cluster)
            self.destinations = util.parseDestinations(dest_list)
        except ValueError as e:
            raise SystemExit("Unable to parse destinations!" + str(e))

        for d in self.destinations:
            self.ring.addDestination(d)

Example #14

File: test_aggregator_processor.py Project: zasca/carbon

 def tearDown(self):
     instrumentation.stats.clear()
     BufferManager.clear()
     RuleManager.clear()

Example #15

File: test_aggregator_processor.py Project: graphite-project/carbon

 def tearDown(self):
   instrumentation.stats.clear()
   BufferManager.clear()
   RuleManager.clear()

Example #16

File: carbon-aggregator.py Project: katzj/graphite

from carbon.conf import settings
settings.readFrom(options.config, 'aggregator')


# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize

RuleManager.read_from(options.rules)

rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
  RewriteRuleManager.read_from(rewrite_rules_conf)

# --debug
if options.debug:
  logToStdout()

else:
  if not isdir(options.logdir):
    os.makedirs(options.logdir)

  daemonize()

Example #17

File: carbon-aggregator.py Project: ImanIsmayana/graphite

# Read config (we want failures to occur before daemonizing)
from carbon.conf import settings
settings.readFrom(options.config, 'aggregator')

# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize

RuleManager.read_from(options.rules)

rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
    RewriteRuleManager.read_from(rewrite_rules_conf)

# --debug
if options.debug:
    logToStdout()

else:
    if not isdir(options.logdir):
        os.makedirs(options.logdir)

    daemonize()