Esempio n. 1
0
 def test_constructor(self):
     settings = {
         'CACHE_WRITE_STRATEGY': 'max',
     }
     settings_patch = patch.dict('carbon.conf.settings', settings)
     settings_patch.start()
     cache = MetricCache()
     self.assertNotEqual(cache, None)
     self.assertTrue(isinstance(cache.strategy, MaxStrategy))
Esempio n. 2
0
    def stringReceived(self, rawRequest):
        request = self.unpickler.loads(rawRequest)
        cache = MetricCache()
        if request['type'] == 'cache-query':
            metric = request['metric']
            datapoints = list(cache.get(metric, {}).items())
            result = dict(datapoints=datapoints)
            if settings.LOG_CACHE_HITS:
                log.query('[%s] cache query for \"%s\" returned %d values' %
                          (self.peerAddr, metric, len(datapoints)))
            instrumentation.increment('cacheQueries')

        elif request['type'] == 'cache-query-bulk':
            datapointsByMetric = {}
            metrics = request['metrics']
            for metric in metrics:
                datapointsByMetric[metric] = list(
                    cache.get(metric, {}).items())

            result = dict(datapointsByMetric=datapointsByMetric)

            if settings.LOG_CACHE_HITS:
                log.query(
                    '[%s] cache query bulk for \"%d\" metrics returned %d values'
                    % (self.peerAddr, len(metrics),
                       sum([
                           len(datapoints)
                           for datapoints in datapointsByMetric.values()
                       ])))
            instrumentation.increment('cacheBulkQueries')
            instrumentation.append('cacheBulkQuerySize', len(metrics))

        elif request['type'] == 'get-metadata':
            result = management.getMetadata(request['metric'], request['key'])

        elif request['type'] == 'set-metadata':
            result = management.setMetadata(request['metric'], request['key'],
                                            request['value'])

        else:
            result = dict(error="Invalid request type \"%s\"" %
                          request['type'])

        response = pickle.dumps(result, protocol=2)
        self.sendString(response)
Esempio n. 3
0
def optimalWriteOrder():
    """Generates metrics with the most cached values first and applies a soft
  rate limit on new metrics"""
    cache = MetricCache()
    while cache:
        (metric, datapoints) = cache.drain_metric()
        dbFileExists = state.database.exists(metric)

        if not dbFileExists and CREATE_BUCKET:
            # If our tokenbucket has enough tokens available to create a new metric
            # file then yield the metric data to complete that operation. Otherwise
            # we'll just drop the metric on the ground and move on to the next
            # metric.
            # XXX This behavior should probably be configurable to no tdrop metrics
            # when rate limitng unless our cache is too big or some other legit
            # reason.
            if CREATE_BUCKET.drain(1):
                yield (metric, datapoints, dbFileExists)
            continue

        yield (metric, datapoints, dbFileExists)
Esempio n. 4
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"

  cache = MetricCache()
  while cache:
    (metric, datapoints) = cache.drain_metric()
    if metric is None:
      # end the loop
      break

    dbFileExists = state.database.exists(metric)

    if not dbFileExists:
      if CREATE_BUCKET and not CREATE_BUCKET.drain(1):
        # If our tokenbucket doesn't have enough tokens available to create a new metric
        # file then we'll just drop the metric on the ground and move on to the next
        # metric.
        # XXX This behavior should probably be configurable to no tdrop metrics
        # when rate limitng unless our cache is too big or some other legit
        # reason.
        instrumentation.increment('droppedCreates')
        continue

      archiveConfig = None
      xFilesFactor, aggregationMethod = None, None

      for schema in SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
          archiveConfig = [archive.getTuple() for archive in schema.archives]
          break

      for schema in AGGREGATION_SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched aggregation schema %s'
                        % (metric, schema.name))
          xFilesFactor, aggregationMethod = schema.archives
          break

      if not archiveConfig:
        raise Exception(("No storage schema matched the metric '%s',"
                         " check your storage-schemas.conf file.") % metric)

      if settings.LOG_CREATES:
        log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
                    (metric, archiveConfig, xFilesFactor, aggregationMethod))
      try:
        state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
        if settings.ENABLE_TAGS:
          tagQueue.add(metric)
        instrumentation.increment('creates')
      except Exception as e:
        log.err()
        log.msg("Error creating %s: %s" % (metric, e))
        instrumentation.increment('errors')
        continue

    # If we've got a rate limit configured lets makes sure we enforce it
    waitTime = 0
    if UPDATE_BUCKET:
      t1 = time.time()
      yield UPDATE_BUCKET.drain(1, blocking=True)
      waitTime = time.time() - t1

    try:
      t1 = time.time()
      # If we have duplicated points, always pick the last. update_many()
      # has no guaranted behavior for that, and in fact the current implementation
      # will keep the first point in the list.
      datapoints = dict(datapoints).items()
      state.database.write(metric, datapoints)
      if settings.ENABLE_TAGS:
        tagQueue.update(metric)
      updateTime = time.time() - t1
    except Exception as e:
      log.err()
      log.msg("Error writing to %s: %s" % (metric, e))
      instrumentation.increment('errors')
    else:
      pointCount = len(datapoints)
      instrumentation.increment('committedPoints', pointCount)
      instrumentation.append('updateTimes', updateTime)
      if settings.LOG_UPDATES:
        if waitTime > 0.001:
          log.updates("wrote %d datapoints for %s in %.5f seconds after waiting %.5f seconds" % (
            pointCount, metric, updateTime, waitTime))
        else:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (
            pointCount, metric, updateTime))
Esempio n. 5
0
def writeCachedDataPoints():
    "Write datapoints until the MetricCache is completely empty"

    cache = MetricCache()
    while cache:
        dataWritten = False

        for (metric, datapoints, dbFileExists) in optimalWriteOrder():
            dataWritten = True

            if not dbFileExists:
                archiveConfig = None
                xFilesFactor, aggregationMethod = None, None

                for schema in SCHEMAS:
                    if schema.matches(metric):
                        if settings.LOG_CREATES:
                            log.creates('new metric %s matched schema %s' %
                                        (metric, schema.name))
                        archiveConfig = [
                            archive.getTuple() for archive in schema.archives
                        ]
                        break

                for schema in AGGREGATION_SCHEMAS:
                    if schema.matches(metric):
                        if settings.LOG_CREATES:
                            log.creates(
                                'new metric %s matched aggregation schema %s' %
                                (metric, schema.name))
                        xFilesFactor, aggregationMethod = schema.archives
                        break

                if not archiveConfig:
                    raise Exception(
                        "No storage schema matched the metric '%s', check your storage-schemas.conf file."
                        % metric)

                if settings.LOG_CREATES:
                    log.creates(
                        "creating database metric %s (archive=%s xff=%s agg=%s)"
                        % (metric, archiveConfig, xFilesFactor,
                           aggregationMethod))
                try:
                    state.database.create(metric, archiveConfig, xFilesFactor,
                                          aggregationMethod)
                    instrumentation.increment('creates')
                except Exception, e:
                    log.err()
                    log.msg("Error creating %s: %s" % (metric, e))
                    instrumentation.increment('errors')
                    continue
            # If we've got a rate limit configured lets makes sure we enforce it
            if UPDATE_BUCKET:
                UPDATE_BUCKET.drain(1, blocking=True)
            try:
                t1 = time.time()
                # If we have duplicated points, always pick the last. update_many()
                # has no guaranted behavior for that, and in fact the current implementation
                # will keep the first point in the list.
                datapoints = dict(datapoints).items()
                state.database.write(metric, datapoints)
                updateTime = time.time() - t1
            except Exception, e:
                log.err()
                log.msg("Error writing to %s: %s" % (metric, e))
                instrumentation.increment('errors')
            else:
                pointCount = len(datapoints)
                instrumentation.increment('committedPoints', pointCount)
                instrumentation.append('updateTimes', updateTime)
                if settings.LOG_UPDATES:
                    log.updates("wrote %d datapoints for %s in %.5f seconds" %
                                (pointCount, metric, updateTime))