Esempio n. 1
0
    def _createOneMetric(self):
        try:
            metric, metric_name = self._metricsToCreate.get(True, 1)
        except queue.Empty:
            return

        CREATES_DEQUEUED.inc()

        existing_metric = self.accessor.get_metric(metric.name)

        if metric == existing_metric:
            # Update current cache value with the one coming from DB.
            # It contains the last updated_on value set in DB that
            # will be used when touching the metadata
            self.cache.cache_set(metric.name, existing_metric)
            return

        if existing_metric:
            # The retention policy is different, update it.
            log.creates(
                "updating database metric %s (%s)"
                % (metric.name, metric.metadata.as_string_dict())
            )
        else:
            # The metric doesn't exists, create it.
            log.creates("creating database metric %s" % metric.name)

        # we will create it right now, no need to touch it
        metric.updated_on = datetime.datetime.now()

        self.cache.create_metric(metric)
        self.tag(metric_name)
        CREATES.inc()
Esempio n. 2
0
    def _createOneMetric(self):
        try:
            metric, metric_name = self._metricsToCreate.get(True, 1)
        except queue.Empty:
            return

        CREATES_DEQUEUED.inc()

        existing_metric = self.accessor.get_metric(metric.name)

        if metric == existing_metric:
            return

        if existing_metric:
            # The retention policy is different, update it.
            log.creates(
                "updating database metric %s (%s)"
                % (metric.name, metric.metadata.as_string_dict())
            )
        else:
            # The metric doesn't exists, create it.
            log.creates("creating database metric %s" % metric.name)

        self.cache.create_metric(metric)
        self.tag(metric_name)
        CREATES.inc()
Esempio n. 3
0
    def test_write_to_logfile(self):
        with TemporaryDirectory() as tmpdir:
            o = log.CarbonLogObserver()
            o.log_to_dir(tmpdir)
            addObserver(o)
            log.creates('😀😀😀😀 test !!!!')
            removeObserver(o)

            with open(path.join(tmpdir, 'creates.log')) as logfile:
                read_line = logfile.readline()
                self.assertRegexpMatches(read_line, '.*😀😀😀😀 test !!!!')
Esempio n. 4
0
    def _createOneMetric(self):
        try:
            metric = self._metricsToCreate.get(True, 1)
        except Queue.Empty:
            return

        existing_metric = self.accessor.get_metric(metric.name)

        if metric == existing_metric:
            return

        if existing_metric:
            # The retention policy is different, update it.
            log.creates("updating database metric %s (%s)" % (
                metric.name, metric.metadata.as_string_dict()))
        else:
            # The metric doesn't exists, create it.
            log.creates("creating database metric %s" % metric.name)

        self.cache.create_metric(metric)
        CREATES.inc()
Esempio n. 5
0
def createWhisperFile(metric, dbFilePath, dbFileExists):
  if not dbFileExists:
    archiveConfig = None
    xFilesFactor, aggregationMethod = None, None

    for schema in schemas:
      if schema.matches(metric):
        log.creates('new metric %s matched schema %s' % (metric, schema.name))
        archiveConfig = [archive.getTuple() for archive in schema.archives]
        break

    for schema in agg_schemas:
      if schema.matches(metric):
        log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
        xFilesFactor, aggregationMethod = schema.archives
        break

    if not archiveConfig:
          raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)

    dbDir = dirname(dbFilePath)
    try:
      os.makedirs(dbDir)
    except OSError as e:
      if e.errno != errno.EEXIST:
        log.err("%s" % e)
    log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
                (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
    try:
      whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
      instrumentation.increment('creates')
    except Exception, e:
      log.err("Error creating %s: %s" % (dbFilePath, e))
      return False
Esempio n. 6
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"

  while MetricCache:
    dataWritten = False

    for (metric, datapoints, dbFileExists) in optimalWriteOrder():
      dataWritten = True

      if not dbFileExists:
        archiveConfig = None
        xFilesFactor, aggregationMethod = None, None

        for schema in SCHEMAS:
          if schema.matches(metric):
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
            archiveConfig = [archive.getTuple() for archive in schema.archives]
            break

        for schema in AGGREGATION_SCHEMAS:
          if schema.matches(metric):
            log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
            xFilesFactor, aggregationMethod = schema.archives
            break

        if not archiveConfig:
          raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)

        log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
                    (metric, archiveConfig, xFilesFactor, aggregationMethod))
        try:
            state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
            instrumentation.increment('creates')
        except Exception, e:
            log.err()
            log.msg("Error creating %s: %s" % (metric, e))
            instrumentation.increment('errors')
            continue
      # If we've got a rate limit configured lets makes sure we enforce it
      if UPDATE_BUCKET:
        UPDATE_BUCKET.drain(1, blocking=True)
      try:
        t1 = time.time()
        # If we have duplicated points, always pick the last. update_many()
        # has no guaranted behavior for that, and in fact the current implementation
        # will keep the first point in the list.
        datapoints = dict(datapoints).items()
        state.database.write(metric, datapoints)
        updateTime = time.time() - t1
      except Exception, e:
        log.err()
        log.msg("Error writing to %s: %s" % (metric, e))
        instrumentation.increment('errors')
      else:
        pointCount = len(datapoints)
        instrumentation.increment('committedPoints', pointCount)
        instrumentation.append('updateTimes', updateTime)
        if settings.LOG_UPDATES:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
Esempio n. 7
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"

  while MetricCache:
    dataWritten = False

    for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
      dataWritten = True

      if not dbFileExists:
        archiveConfig = None
        xFilesFactor, aggregationMethod = None, None

        for schema in SCHEMAS:
          if schema.matches(metric):
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
            archiveConfig = [archive.getTuple() for archive in schema.archives]
            break

        for schema in AGGREGATION_SCHEMAS:
          if schema.matches(metric):
            log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
            xFilesFactor, aggregationMethod = schema.archives
            break

        if not archiveConfig:
          raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)

        log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
                    (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
        try:
            state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
            instrumentation.increment('creates')
        except Exception:
            log.err("Error creating %s" % (dbFilePath))
            continue
      # If we've got a rate limit configured lets makes sure we enforce it
      if UPDATE_BUCKET:
        UPDATE_BUCKET.drain(1, blocking=True)
      try:
        t1 = time.time()
        state.database.write(metric, datapoints)
        updateTime = time.time() - t1
      except Exception:
        log.msg("Error writing to %s" % (dbFilePath))
        log.err()
        instrumentation.increment('errors')
      else:
        pointCount = len(datapoints)
        instrumentation.increment('committedPoints', pointCount)
        instrumentation.append('updateTimes', updateTime)
        if settings.LOG_UPDATES:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))

    # Avoid churning CPU when only new metrics are in the cache
    if not dataWritten:
      time.sleep(0.1)
Esempio n. 8
0
def writeCachedDataPoints():
    "Write datapoints until the MetricCache is completely empty"
    updates = 0
    lastSecond = 0

    while MetricCache:
        dataWritten = False

        for (metric, datapoints, dbFilePath,
             dbFileExists) in optimalWriteOrder():
            dataWritten = True

            if not dbFileExists:
                archiveConfig = None
                xFilesFactor, aggregationMethod = None, None

                for schema in schemas:
                    if schema.matches(metric):
                        log.creates('new metric %s matched schema %s' %
                                    (metric, schema.name))
                        archiveConfig = [
                            archive.getTuple() for archive in schema.archives
                        ]
                        break

                for schema in agg_schemas:
                    if schema.matches(metric):
                        log.creates(
                            'new metric %s matched aggregation schema %s' %
                            (metric, schema.name))
                        xFilesFactor, aggregationMethod = schema.archives
                        break

                if not archiveConfig:
                    raise Exception(
                        "No storage schema matched the metric '%s', check your storage-schemas.conf file."
                        % metric)

                dbDir = dirname(dbFilePath)
                try:
                    os.makedirs(dbDir, 0755)
                except OSError as e:
                    log.err("%s" % e)
                log.creates(
                    "creating database file %s (archive=%s xff=%s agg=%s)" %
                    (dbFilePath, archiveConfig, xFilesFactor,
                     aggregationMethod))
                whisper.create(dbFilePath, archiveConfig, xFilesFactor,
                               aggregationMethod,
                               settings.WHISPER_SPARSE_CREATE,
                               settings.WHISPER_FALLOCATE_CREATE)
                instrumentation.increment('creates')

            try:
                t1 = time.time()
                whisper.update_many(dbFilePath, datapoints)
                t2 = time.time()
                updateTime = t2 - t1
            except:
                log.msg("Error writing to %s" % (dbFilePath))
                log.err()
                instrumentation.increment('errors')
            else:
                pointCount = len(datapoints)
                instrumentation.increment('committedPoints', pointCount)
                instrumentation.append('updateTimes', updateTime)

                if settings.LOG_UPDATES:
                    log.updates("wrote %d datapoints for %s in %.5f seconds" %
                                (pointCount, metric, updateTime))

                # Rate limit update operations
                thisSecond = int(t2)

                if thisSecond != lastSecond:
                    lastSecond = thisSecond
                    updates = 0
                else:
                    updates += 1
                    if updates >= settings.MAX_UPDATES_PER_SECOND:
                        time.sleep(int(t2 + 1) - t2)

        # Avoid churning CPU when only new metrics are in the cache
        if not dataWritten:
            time.sleep(0.1)
Esempio n. 9
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"
  updates = 0
  lastSecond = 0

  while MetricCache:
    dataWritten = False

    for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
      dataWritten = True

      if not dbFileExists:
        archiveConfig = None
        xFilesFactor, aggregationMethod = None, None

        for schema in schemas:
          if schema.matches(metric):
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
            archiveConfig = [archive.getTuple() for archive in schema.archives]
            break

        for schema in agg_schemas:
          if schema.matches(metric):
            log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
            xFilesFactor, aggregationMethod = schema.archives
            break

        if not archiveConfig:
          raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)

        dbDir = dirname(dbFilePath)
        try:
            os.makedirs(dbDir, 0755)
        except OSError as e:
            log.err("%s" % e)
        log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
                    (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
        whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
        instrumentation.increment('creates')

      try:
        t1 = time.time()
        whisper.update_many(dbFilePath, datapoints)
        t2 = time.time()
        updateTime = t2 - t1
      except:
        log.msg("Error writing to %s" % (dbFilePath))
        log.err()
        instrumentation.increment('errors')
      else:
        pointCount = len(datapoints)
        instrumentation.increment('committedPoints', pointCount)
        instrumentation.append('updateTimes', updateTime)

        if settings.LOG_UPDATES:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))

        # Rate limit update operations
        thisSecond = int(t2)

        if thisSecond != lastSecond:
          lastSecond = thisSecond
          updates = 0
        else:
          updates += 1
          if updates >= settings.MAX_UPDATES_PER_SECOND:
            time.sleep(int(t2 + 1) - t2)

    # Avoid churning CPU when only new metrics are in the cache
    if not dataWritten:
      time.sleep(0.1)
Esempio n. 10
0
def writeCachedDataPoints():
    "Write datapoints until the MetricCache is completely empty"

    cache = MetricCache()
    while cache:
        dataWritten = False

        for (metric, datapoints, dbFileExists) in optimalWriteOrder():
            dataWritten = True

            if not dbFileExists:
                archiveConfig = None
                xFilesFactor, aggregationMethod = None, None

                for schema in SCHEMAS:
                    if schema.matches(metric):
                        if settings.LOG_CREATES:
                            log.creates('new metric %s matched schema %s' %
                                        (metric, schema.name))
                        archiveConfig = [
                            archive.getTuple() for archive in schema.archives
                        ]
                        break

                for schema in AGGREGATION_SCHEMAS:
                    if schema.matches(metric):
                        if settings.LOG_CREATES:
                            log.creates(
                                'new metric %s matched aggregation schema %s' %
                                (metric, schema.name))
                        xFilesFactor, aggregationMethod = schema.archives
                        break

                if not archiveConfig:
                    raise Exception(
                        "No storage schema matched the metric '%s', check your storage-schemas.conf file."
                        % metric)

                if settings.LOG_CREATES:
                    log.creates(
                        "creating database metric %s (archive=%s xff=%s agg=%s)"
                        % (metric, archiveConfig, xFilesFactor,
                           aggregationMethod))
                try:
                    state.database.create(metric, archiveConfig, xFilesFactor,
                                          aggregationMethod)
                    instrumentation.increment('creates')
                except Exception, e:
                    log.err()
                    log.msg("Error creating %s: %s" % (metric, e))
                    instrumentation.increment('errors')
                    continue
            # If we've got a rate limit configured lets makes sure we enforce it
            if UPDATE_BUCKET:
                UPDATE_BUCKET.drain(1, blocking=True)
            try:
                t1 = time.time()
                # If we have duplicated points, always pick the last. update_many()
                # has no guaranted behavior for that, and in fact the current implementation
                # will keep the first point in the list.
                datapoints = dict(datapoints).items()
                state.database.write(metric, datapoints)
                updateTime = time.time() - t1
            except Exception, e:
                log.err()
                log.msg("Error writing to %s: %s" % (metric, e))
                instrumentation.increment('errors')
            else:
                pointCount = len(datapoints)
                instrumentation.increment('committedPoints', pointCount)
                instrumentation.append('updateTimes', updateTime)
                if settings.LOG_UPDATES:
                    log.updates("wrote %d datapoints for %s in %.5f seconds" %
                                (pointCount, metric, updateTime))
Esempio n. 11
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"
  updates = 0
  lastSecond = 0

  while MetricCache:
    dataWritten = False

    for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
      dataWritten = True

      if not dbFileExists:
        archiveConfig = None

        for schema in schemas:
          if schema.matches(metric):
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
            archiveConfig = [archive.getTuple() for archive in schema.archives]
            break

        if not archiveConfig:
          raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)

        dbDir = dirname(dbFilePath)
        os.system("mkdir -p -m 755 '%s'" % dbDir)

        log.creates("creating database file %s" % dbFilePath)
        whisper.create(dbFilePath, archiveConfig)
        os.chmod(dbFilePath, 0755)
        instrumentation.increment('creates')

        # Create metadata file
        dbFileName = basename(dbFilePath)
        metaFilePath = join(dbDir, dbFileName[ :-len('.wsp') ] + '.context.pickle')
        createMetaFile(metric, schema, metaFilePath)

      try:
        t1 = time.time()
        whisper.update_many(dbFilePath, datapoints)
        t2 = time.time()
        updateTime = t2 - t1
      except:
        log.err()
        instrumentation.increment('errors')
      else:
        pointCount = len(datapoints)
        instrumentation.increment('committedPoints', pointCount)
        instrumentation.append('updateTimes', updateTime)

        if settings.LOG_UPDATES:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))

        # Rate limit update operations
        thisSecond = int(t2)

        if thisSecond != lastSecond:
          lastSecond = thisSecond
          updates = 0
        else:
          updates += 1
          if updates >= settings.MAX_UPDATES_PER_SECOND:
            time.sleep( int(t2 + 1) - t2 )

    # Avoid churning CPU when only new metrics are in the cache
    if not dataWritten:
      time.sleep(0.1)
Esempio n. 12
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"

  cache = MetricCache()
  while cache:
    (metric, datapoints) = cache.drain_metric()
    if metric is None:
      # end the loop
      break

    dbFileExists = state.database.exists(metric)

    if not dbFileExists:
      if CREATE_BUCKET and not CREATE_BUCKET.drain(1):
        # If our tokenbucket doesn't have enough tokens available to create a new metric
        # file then we'll just drop the metric on the ground and move on to the next
        # metric.
        # XXX This behavior should probably be configurable to no tdrop metrics
        # when rate limitng unless our cache is too big or some other legit
        # reason.
        instrumentation.increment('droppedCreates')
        continue

      archiveConfig = None
      xFilesFactor, aggregationMethod = None, None

      for schema in SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
          archiveConfig = [archive.getTuple() for archive in schema.archives]
          break

      for schema in AGGREGATION_SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched aggregation schema %s'
                        % (metric, schema.name))
          xFilesFactor, aggregationMethod = schema.archives
          break

      if not archiveConfig:
        raise Exception(("No storage schema matched the metric '%s',"
                         " check your storage-schemas.conf file.") % metric)

      if settings.LOG_CREATES:
        log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
                    (metric, archiveConfig, xFilesFactor, aggregationMethod))
      try:
        state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
        if settings.ENABLE_TAGS:
          tagQueue.add(metric)
        instrumentation.increment('creates')
      except Exception as e:
        log.err()
        log.msg("Error creating %s: %s" % (metric, e))
        instrumentation.increment('errors')
        continue

    # If we've got a rate limit configured lets makes sure we enforce it
    waitTime = 0
    if UPDATE_BUCKET:
      t1 = time.time()
      UPDATE_BUCKET.drain(1, blocking=True)
      waitTime = time.time() - t1

    try:
      t1 = time.time()
      # If we have duplicated points, always pick the last. update_many()
      # has no guaranted behavior for that, and in fact the current implementation
      # will keep the first point in the list.
      datapoints = dict(datapoints).items()
      state.database.write(metric, datapoints)
      if settings.ENABLE_TAGS:
        tagQueue.update(metric)
      updateTime = time.time() - t1
    except Exception as e:
      log.err()
      log.msg("Error writing to %s: %s" % (metric, e))
      instrumentation.increment('errors')
    else:
      pointCount = len(datapoints)
      instrumentation.increment('committedPoints', pointCount)
      instrumentation.append('updateTimes', updateTime)
      if settings.LOG_UPDATES:
        if waitTime > 0.001:
          log.updates("wrote %d datapoints for %s in %.5f seconds after waiting %.5f seconds" % (
            pointCount, metric, updateTime, waitTime))
        else:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (
            pointCount, metric, updateTime))
Esempio n. 13
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"
  
  global  updates 
  global  lastSecond 

#  while MetricCache:
#  dataWritten = False

  for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
    dataWritten = True

    if not dbFileExists:
      archiveConfig = None
      xFilesFactor, aggregationMethod = None, None

      for schema in schemas:
        if schema.matches(metric):
          log.creates('new metric %s matched schema %s' % (metric, schema.name))
          archiveConfig = [archive.getTuple() for archive in schema.archives]
          break

      for schema in agg_schemas:
        if schema.matches(metric):
          log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
          xFilesFactor, aggregationMethod = schema.archives
          break

      if not archiveConfig:
        raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
      try: 
        dbDir = dirname(dbFilePath)
        os.system("umask u=rwx,go=rx ; mkdir -p -m 755 '%s'" % dbDir)

        log.creates("creating database file %s (archive=%s xff=%s agg=%s)" % 
                  (dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
        whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod)
        os.chmod(dbFilePath, 0755)
        instrumentation.increment('creates')
      except IOError as e:
        log.msg("IOError: {0}".format(e))

    try:
      t1 = time.time()
      whisper.update_many(dbFilePath, datapoints)
      t2 = time.time()
      updateTime = t2 - t1
    except:
      log.err()
      instrumentation.increment('errors')
    else:
      pointCount = len(datapoints)
      instrumentation.increment('committedPoints', pointCount)
      instrumentation.append('updateTimes', updateTime)

      if settings.LOG_UPDATES:
        log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))

      # Rate limit update operations
      thisSecond = int(t2)

      if thisSecond != lastSecond:
        lastSecond = thisSecond
        updates = 0
      else:
        updates += 1
        if updates >= settings.MAX_UPDATES_PER_SECOND:
          time.sleep( int(t2 + 1) - t2 )
  time.sleep(5)
Esempio n. 14
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"
  updates = 0
  lastSecond = 0

  while MetricCache:
    dataWritten = False

    for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
      dataWritten = True

      if not dbFileExists:
        for schema in schemas:
          if schema.matches(metric):
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
            archiveConfig = [archive.getTuple() for archive in schema.archives]
            break

        dbDir = dirname(dbFilePath)
        os.system("mkdir -p '%s'" % dbDir)

        log.creates("creating database file %s" % dbFilePath)
        whisper.create(dbFilePath, archiveConfig)
        increment('creates')

        # Create metadata file
        dbFileName = basename(dbFilePath)
        metaFilePath = join(dbDir, dbFileName[ :-len('.wsp') ] + '.context.pickle')
        createMetaFile(metric, schema, metaFilePath)

      data = {}
      try:
        t1 = time.time()
         
        counts = {}

        for dp in datapoints:
          key = long(dp[0] / 60)
	  if not key in data:         
            data[key] = 0
            counts[key] = 0

          data[key] += dp[1]
          counts[key] += 1

        if "/rt" in dbFilePath or "latency_ms" in dbFilePath:
          print "AVERAGING", dbFilePath
          for key, value in data.iteritems():
             data[key] = value / counts[key]
           
        for dptime, datavalue in data.iteritems():
          whisper.update(dbFilePath, datavalue, dptime * 60)

        t2 = time.time()
        updateTime = t2 - t1
      except:
        log.err()
        increment('errors')
      else:
        pointCount = len(data)
        log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
        increment('committedPoints', pointCount)
        append('updateTimes', updateTime)

        # Rate limit update operations
        thisSecond = int(t2)

        if thisSecond != lastSecond:
          lastSecond = thisSecond
          updates = 0
        else:
          updates += 1
          if updates >= settings.MAX_UPDATES_PER_SECOND:
            time.sleep( int(t2 + 1) - t2 )

    # Avoid churning CPU when only new metrics are in the cache
    if not dataWritten:
      time.sleep(0.1)
Esempio n. 15
0
def writeCachedDataPoints():
  "Write datapoints until the MetricCache is completely empty"

  cache = MetricCache()
  while cache:
    (metric, datapoints) = cache.drain_metric()
    if metric is None:
      # end the loop
      break

    dbFileExists = state.database.exists(metric)

    if not dbFileExists:
      if CREATE_BUCKET and not CREATE_BUCKET.drain(1):
        # If our tokenbucket doesn't have enough tokens available to create a new metric
        # file then we'll just drop the metric on the ground and move on to the next
        # metric.
        # XXX This behavior should probably be configurable to no tdrop metrics
        # when rate limitng unless our cache is too big or some other legit
        # reason.
        instrumentation.increment('droppedCreates')
        continue

      archiveConfig = None
      xFilesFactor, aggregationMethod = None, None

      for schema in SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched schema %s' % (metric, schema.name))
          archiveConfig = [archive.getTuple() for archive in schema.archives]
          break

      for schema in AGGREGATION_SCHEMAS:
        if schema.matches(metric):
          if settings.LOG_CREATES:
            log.creates('new metric %s matched aggregation schema %s'
                        % (metric, schema.name))
          xFilesFactor, aggregationMethod = schema.archives
          break

      if not archiveConfig:
        raise Exception(("No storage schema matched the metric '%s',"
                         " check your storage-schemas.conf file.") % metric)

      if settings.LOG_CREATES:
        log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
                    (metric, archiveConfig, xFilesFactor, aggregationMethod))
      try:
        state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
        if settings.ENABLE_TAGS:
          tagQueue.add(metric)
        instrumentation.increment('creates')
      except Exception as e:
        log.err()
        log.msg("Error creating %s: %s" % (metric, e))
        instrumentation.increment('errors')
        continue

    # If we've got a rate limit configured lets makes sure we enforce it
    waitTime = 0
    if UPDATE_BUCKET:
      t1 = time.time()
      yield UPDATE_BUCKET.drain(1, blocking=True)
      waitTime = time.time() - t1

    try:
      t1 = time.time()
      # If we have duplicated points, always pick the last. update_many()
      # has no guaranted behavior for that, and in fact the current implementation
      # will keep the first point in the list.
      datapoints = dict(datapoints).items()
      state.database.write(metric, datapoints)
      if settings.ENABLE_TAGS:
        tagQueue.update(metric)
      updateTime = time.time() - t1
    except Exception as e:
      log.err()
      log.msg("Error writing to %s: %s" % (metric, e))
      instrumentation.increment('errors')
    else:
      pointCount = len(datapoints)
      instrumentation.increment('committedPoints', pointCount)
      instrumentation.append('updateTimes', updateTime)
      if settings.LOG_UPDATES:
        if waitTime > 0.001:
          log.updates("wrote %d datapoints for %s in %.5f seconds after waiting %.5f seconds" % (
            pointCount, metric, updateTime, waitTime))
        else:
          log.updates("wrote %d datapoints for %s in %.5f seconds" % (
            pointCount, metric, updateTime))
Esempio n. 16
0
def writeCachedDataPoints():
    "Write datapoints until the MetricCache is completely empty"

    while MetricCache:
        dataWritten = False

        for (metric, datapoints, dbFilePath,
             dbFileExists) in optimalWriteOrder():
            dataWritten = True

            if not dbFileExists:
                archiveConfig = None
                xFilesFactor, aggregationMethod = None, None

                for schema in SCHEMAS:
                    if schema.matches(metric):
                        log.creates('new metric %s matched schema %s' %
                                    (metric, schema.name))
                        archiveConfig = [
                            archive.getTuple() for archive in schema.archives
                        ]
                        break

                for schema in AGGREGATION_SCHEMAS:
                    if schema.matches(metric):
                        log.creates(
                            'new metric %s matched aggregation schema %s' %
                            (metric, schema.name))
                        xFilesFactor, aggregationMethod = schema.archives
                        break

                if not archiveConfig:
                    raise Exception(
                        "No storage schema matched the metric '%s', check your storage-schemas.conf file."
                        % metric)

                dbDir = dirname(dbFilePath)
                try:
                    if not exists(dbDir):
                        os.makedirs(dbDir)
                except OSError, e:
                    log.err("%s" % e)
                log.creates(
                    "creating database file %s (archive=%s xff=%s agg=%s)" %
                    (dbFilePath, archiveConfig, xFilesFactor,
                     aggregationMethod))
                try:
                    whisper.create(dbFilePath, archiveConfig, xFilesFactor,
                                   aggregationMethod,
                                   settings.WHISPER_SPARSE_CREATE,
                                   settings.WHISPER_FALLOCATE_CREATE)
                    instrumentation.increment('creates')
                except:
                    log.err("Error creating %s" % (dbFilePath))
                    continue
            # If we've got a rate limit configured lets makes sure we enforce it
            if UPDATE_BUCKET:
                UPDATE_BUCKET.drain(1, blocking=True)
            try:
                t1 = time.time()
                whisper.update_many(dbFilePath, datapoints)
                updateTime = time.time() - t1
            except Exception:
                log.msg("Error writing to %s" % (dbFilePath))
                log.err()
                instrumentation.increment('errors')
            else:
                pointCount = len(datapoints)
                instrumentation.increment('committedPoints', pointCount)
                instrumentation.append('updateTimes', updateTime)
                if settings.LOG_UPDATES:
                    log.updates("wrote %d datapoints for %s in %.5f seconds" %
                                (pointCount, metric, updateTime))

        # Avoid churning CPU when only new metrics are in the cache
        if not dataWritten:
            time.sleep(0.1)
Esempio n. 17
0
def writeCachedDataPoints():
    "Write datapoints until the MetricCache is completely empty"
    updates = 0
    lastSecond = 0

    while MetricCache:
        dataWritten = False

        for (metric, datapoints, dbFilePath,
             dbFileExists) in optimalWriteOrder():
            dataWritten = True

            if not dbFileExists:
                archiveConfig = None

                for schema in schemas:
                    if schema.matches(metric):
                        log.creates('new metric %s matched schema %s' %
                                    (metric, schema.name))
                        archiveConfig = [
                            archive.getTuple() for archive in schema.archives
                        ]
                        break

                if not archiveConfig:
                    raise Exception(
                        "No storage schema matched the metric '%s', check your storage-schemas.conf file."
                        % metric)

                dbDir = dirname(dbFilePath)
                os.system("mkdir -p -m 755 '%s'" % dbDir)

                log.creates("creating database file %s" % dbFilePath)
                whisper.create(dbFilePath, archiveConfig)
                os.chmod(dbFilePath, 0755)
                increment('creates')

                # Create metadata file
                dbFileName = basename(dbFilePath)
                metaFilePath = join(
                    dbDir, dbFileName[:-len('.wsp')] + '.context.pickle')
                createMetaFile(metric, schema, metaFilePath)

            try:
                t1 = time.time()
                whisper.update_many(dbFilePath, datapoints)
                t2 = time.time()
                updateTime = t2 - t1
            except:
                log.err()
                increment('errors')
            else:
                pointCount = len(datapoints)
                increment('committedPoints', pointCount)
                append('updateTimes', updateTime)

                if settings.LOG_UPDATES:
                    log.updates("wrote %d datapoints for %s in %.5f seconds" %
                                (pointCount, metric, updateTime))

                # Rate limit update operations
                thisSecond = int(t2)

                if thisSecond != lastSecond:
                    lastSecond = thisSecond
                    updates = 0
                else:
                    updates += 1
                    if updates >= settings.MAX_UPDATES_PER_SECOND:
                        time.sleep(int(t2 + 1) - t2)

        # Avoid churning CPU when only new metrics are in the cache
        if not dataWritten:
            time.sleep(0.1)