def optimalWriteOrder():
"""Generates metrics with the most cached values first and applies a soft
rate limit on new metrics"""
global lastCreateInterval
global createCount
metrics = MetricCache.counts()
time_ = time.time()
metrics.sort(key=lambda item: item[1],
reverse=True) # by queue size, descending
log.debug("Sorted %d cache queues in %.6f seconds" %
(len(metrics), time.time() - time_))
for metric, queueSize in metrics:
if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK:
events.cacheSpaceAvailable()
dbFileExists = APP_DB.exists(metric)
if not dbFileExists:
createCount += 1
now = time.time()
if now - lastCreateInterval >= 60:
lastCreateInterval = now
createCount = 1
elif createCount >= settings.MAX_CREATES_PER_MINUTE:
# dropping queued up datapoints for new metrics prevents filling up the entire cache
# when a bunch of new metrics are received.
try:
MetricCache.pop(metric)
except KeyError:
pass
continue
try: # metrics can momentarily disappear from the MetricCache due to the implementation of MetricCache.store()
datapoints = MetricCache.pop(metric)
except KeyError:
log.msg("MetricCache contention, skipping %s update for now" %
metric)
continue # we simply move on to the next metric when this race condition occurs
yield (metric, datapoints, dbFileExists)
def optimalWriteOrder():
"""Generates metrics with the most cached values first and applies a soft
rate limit on new metrics"""
global lastCreateInterval
global createCount
metrics = MetricCache.counts()
time_ = time.time()
metrics.sort(key=lambda item: item[1], reverse=True) # by queue size, descending
log.debug("Sorted %d cache queues in %.6f seconds" % (len(metrics),
time.time() - time_))
for metric, queueSize in metrics:
if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK:
events.cacheSpaceAvailable()
dbFileExists = APP_DB.exists(metric)
if not dbFileExists:
createCount += 1
now = time.time()
if now - lastCreateInterval >= 60:
lastCreateInterval = now
createCount = 1
elif createCount >= settings.MAX_CREATES_PER_MINUTE:
# dropping queued up datapoints for new metrics prevents filling up the entire cache
# when a bunch of new metrics are received.
try:
MetricCache.pop(metric)
except KeyError:
pass
continue
try: # metrics can momentarily disappear from the MetricCache due to the implementation of MetricCache.store()
datapoints = MetricCache.pop(metric)
except KeyError:
log.msg("MetricCache contention, skipping %s update for now" % metric)
continue # we simply move on to the next metric when this race condition occurs
yield (metric, datapoints, dbFileExists)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
last_second = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
log.creates("creating database metric %s (metric=%s xff=%s agg=%s)" %
(metric, archiveConfig, xFilesFactor, aggregationMethod))
try:
APP_DB.create(metric, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
except OSError as e:
log.err("%s" % e)
instrumentation.increment('creates')
try:
time1 = time.time()
APP_DB.update_many(metric, datapoints)
time2 = time.time()
update_time = time2 - time1
except:
log.msg("Error writing to %s" % (metric))
log.err()
instrumentation.increment('errors')
else:
point_count = len(datapoints)
instrumentation.increment('committedPoints', point_count)
instrumentation.append('updateTimes', update_time)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (point_count, metric, update_time))
# Rate limit update operations
this_second = int(time2)
if this_second != last_second:
last_second = this_second
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep(int(time2 + 1) - time2)
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
last_second = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' %
(metric, schema.name))
archiveConfig = [
archive.getTuple() for archive in schema.archives
]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates(
'new metric %s matched aggregation schema %s' %
(metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(
"No storage schema matched the metric '%s', check your storage-schemas.conf file."
% metric)
log.creates(
"creating database metric %s (metric=%s xff=%s agg=%s)" %
(metric, archiveConfig, xFilesFactor, aggregationMethod))
try:
APP_DB.create(metric, archiveConfig, xFilesFactor,
aggregationMethod,
settings.WHISPER_SPARSE_CREATE,
settings.WHISPER_FALLOCATE_CREATE)
except OSError as e:
log.err("%s" % e)
instrumentation.increment('creates')
try:
time1 = time.time()
APP_DB.update_many(metric, datapoints)
time2 = time.time()
update_time = time2 - time1
except:
log.msg("Error writing to %s" % (metric))
log.err()
instrumentation.increment('errors')
else:
point_count = len(datapoints)
instrumentation.increment('committedPoints', point_count)
instrumentation.append('updateTimes', update_time)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" %
(point_count, metric, update_time))
# Rate limit update operations
this_second = int(time2)
if this_second != last_second:
last_second = this_second
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep(int(time2 + 1) - time2)
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
