def optimalWriteOrder(): """Generates metrics with the most cached values first and applies a soft rate limit on new metrics""" global lastCreateInterval global createCount metrics = MetricCache.counts() time_ = time.time() metrics.sort(key=lambda item: item[1], reverse=True) # by queue size, descending log.debug("Sorted %d cache queues in %.6f seconds" % (len(metrics), time.time() - time_)) for metric, queueSize in metrics: if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK: events.cacheSpaceAvailable() dbFileExists = APP_DB.exists(metric) if not dbFileExists: createCount += 1 now = time.time() if now - lastCreateInterval >= 60: lastCreateInterval = now createCount = 1 elif createCount >= settings.MAX_CREATES_PER_MINUTE: # dropping queued up datapoints for new metrics prevents filling up the entire cache # when a bunch of new metrics are received. try: MetricCache.pop(metric) except KeyError: pass continue try: # metrics can momentarily disappear from the MetricCache due to the implementation of MetricCache.store() datapoints = MetricCache.pop(metric) except KeyError: log.msg("MetricCache contention, skipping %s update for now" % metric) continue # we simply move on to the next metric when this race condition occurs yield (metric, datapoints, dbFileExists)
def writeCachedDataPoints(): "Write datapoints until the MetricCache is completely empty" updates = 0 last_second = 0 while MetricCache: dataWritten = False for (metric, datapoints, dbFileExists) in optimalWriteOrder(): dataWritten = True if not dbFileExists: archiveConfig = None xFilesFactor, aggregationMethod = None, None for schema in schemas: if schema.matches(metric): log.creates('new metric %s matched schema %s' % (metric, schema.name)) archiveConfig = [archive.getTuple() for archive in schema.archives] break for schema in agg_schemas: if schema.matches(metric): log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name)) xFilesFactor, aggregationMethod = schema.archives break if not archiveConfig: raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric) log.creates("creating database metric %s (metric=%s xff=%s agg=%s)" % (metric, archiveConfig, xFilesFactor, aggregationMethod)) try: APP_DB.create(metric, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE) except OSError as e: log.err("%s" % e) instrumentation.increment('creates') try: time1 = time.time() APP_DB.update_many(metric, datapoints) time2 = time.time() update_time = time2 - time1 except: log.msg("Error writing to %s" % (metric)) log.err() instrumentation.increment('errors') else: point_count = len(datapoints) instrumentation.increment('committedPoints', point_count) instrumentation.append('updateTimes', update_time) if settings.LOG_UPDATES: log.updates("wrote %d datapoints for %s in %.5f seconds" % (point_count, metric, update_time)) # Rate limit update operations this_second = int(time2) if this_second != last_second: last_second = this_second updates = 0 else: updates += 1 if updates >= settings.MAX_UPDATES_PER_SECOND: time.sleep(int(time2 + 1) - time2) # Avoid churning CPU when only new metrics are in the cache if not dataWritten: time.sleep(0.1)
def writeCachedDataPoints(): "Write datapoints until the MetricCache is completely empty" updates = 0 last_second = 0 while MetricCache: dataWritten = False for (metric, datapoints, dbFileExists) in optimalWriteOrder(): dataWritten = True if not dbFileExists: archiveConfig = None xFilesFactor, aggregationMethod = None, None for schema in schemas: if schema.matches(metric): log.creates('new metric %s matched schema %s' % (metric, schema.name)) archiveConfig = [ archive.getTuple() for archive in schema.archives ] break for schema in agg_schemas: if schema.matches(metric): log.creates( 'new metric %s matched aggregation schema %s' % (metric, schema.name)) xFilesFactor, aggregationMethod = schema.archives break if not archiveConfig: raise Exception( "No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric) log.creates( "creating database metric %s (metric=%s xff=%s agg=%s)" % (metric, archiveConfig, xFilesFactor, aggregationMethod)) try: APP_DB.create(metric, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE) except OSError as e: log.err("%s" % e) instrumentation.increment('creates') try: time1 = time.time() APP_DB.update_many(metric, datapoints) time2 = time.time() update_time = time2 - time1 except: log.msg("Error writing to %s" % (metric)) log.err() instrumentation.increment('errors') else: point_count = len(datapoints) instrumentation.increment('committedPoints', point_count) instrumentation.append('updateTimes', update_time) if settings.LOG_UPDATES: log.updates("wrote %d datapoints for %s in %.5f seconds" % (point_count, metric, update_time)) # Rate limit update operations this_second = int(time2) if this_second != last_second: last_second = this_second updates = 0 else: updates += 1 if updates >= settings.MAX_UPDATES_PER_SECOND: time.sleep(int(time2 + 1) - time2) # Avoid churning CPU when only new metrics are in the cache if not dataWritten: time.sleep(0.1)