def _createOneMetric(self):
try:
metric, metric_name = self._metricsToCreate.get(True, 1)
except queue.Empty:
return
CREATES_DEQUEUED.inc()
existing_metric = self.accessor.get_metric(metric.name)
if metric == existing_metric:
# Update current cache value with the one coming from DB.
# It contains the last updated_on value set in DB that
# will be used when touching the metadata
self.cache.cache_set(metric.name, existing_metric)
return
if existing_metric:
# The retention policy is different, update it.
log.creates(
"updating database metric %s (%s)"
% (metric.name, metric.metadata.as_string_dict())
)
else:
# The metric doesn't exists, create it.
log.creates("creating database metric %s" % metric.name)
# we will create it right now, no need to touch it
metric.updated_on = datetime.datetime.now()
self.cache.create_metric(metric)
self.tag(metric_name)
CREATES.inc()
def _createOneMetric(self):
try:
metric, metric_name = self._metricsToCreate.get(True, 1)
except queue.Empty:
return
CREATES_DEQUEUED.inc()
existing_metric = self.accessor.get_metric(metric.name)
if metric == existing_metric:
return
if existing_metric:
# The retention policy is different, update it.
log.creates(
"updating database metric %s (%s)"
% (metric.name, metric.metadata.as_string_dict())
)
else:
# The metric doesn't exists, create it.
log.creates("creating database metric %s" % metric.name)
self.cache.create_metric(metric)
self.tag(metric_name)
CREATES.inc()
def test_write_to_logfile(self):
with TemporaryDirectory() as tmpdir:
o = log.CarbonLogObserver()
o.log_to_dir(tmpdir)
addObserver(o)
log.creates('😀😀😀😀 test !!!!')
removeObserver(o)
with open(path.join(tmpdir, 'creates.log')) as logfile:
read_line = logfile.readline()
self.assertRegexpMatches(read_line, '.*😀😀😀😀 test !!!!')
def _createOneMetric(self):
try:
metric = self._metricsToCreate.get(True, 1)
except Queue.Empty:
return
existing_metric = self.accessor.get_metric(metric.name)
if metric == existing_metric:
return
if existing_metric:
# The retention policy is different, update it.
log.creates("updating database metric %s (%s)" % (
metric.name, metric.metadata.as_string_dict()))
else:
# The metric doesn't exists, create it.
log.creates("creating database metric %s" % metric.name)
self.cache.create_metric(metric)
CREATES.inc()
def createWhisperFile(metric, dbFilePath, dbFileExists):
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
dbDir = dirname(dbFilePath)
try:
os.makedirs(dbDir)
except OSError as e:
if e.errno != errno.EEXIST:
log.err("%s" % e)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
try:
whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
instrumentation.increment('creates')
except Exception, e:
log.err("Error creating %s: %s" % (dbFilePath, e))
return False
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
(metric, archiveConfig, xFilesFactor, aggregationMethod))
try:
state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
instrumentation.increment('creates')
except Exception, e:
log.err()
log.msg("Error creating %s: %s" % (metric, e))
instrumentation.increment('errors')
continue
# If we've got a rate limit configured lets makes sure we enforce it
if UPDATE_BUCKET:
UPDATE_BUCKET.drain(1, blocking=True)
try:
t1 = time.time()
# If we have duplicated points, always pick the last. update_many()
# has no guaranted behavior for that, and in fact the current implementation
# will keep the first point in the list.
datapoints = dict(datapoints).items()
state.database.write(metric, datapoints)
updateTime = time.time() - t1
except Exception, e:
log.err()
log.msg("Error writing to %s: %s" % (metric, e))
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
try:
state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
instrumentation.increment('creates')
except Exception:
log.err("Error creating %s" % (dbFilePath))
continue
# If we've got a rate limit configured lets makes sure we enforce it
if UPDATE_BUCKET:
UPDATE_BUCKET.drain(1, blocking=True)
try:
t1 = time.time()
state.database.write(metric, datapoints)
updateTime = time.time() - t1
except Exception:
log.msg("Error writing to %s" % (dbFilePath))
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
lastSecond = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath,
dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' %
(metric, schema.name))
archiveConfig = [
archive.getTuple() for archive in schema.archives
]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates(
'new metric %s matched aggregation schema %s' %
(metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(
"No storage schema matched the metric '%s', check your storage-schemas.conf file."
% metric)
dbDir = dirname(dbFilePath)
try:
os.makedirs(dbDir, 0755)
except OSError as e:
log.err("%s" % e)
log.creates(
"creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor,
aggregationMethod))
whisper.create(dbFilePath, archiveConfig, xFilesFactor,
aggregationMethod,
settings.WHISPER_SPARSE_CREATE,
settings.WHISPER_FALLOCATE_CREATE)
instrumentation.increment('creates')
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
t2 = time.time()
updateTime = t2 - t1
except:
log.msg("Error writing to %s" % (dbFilePath))
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" %
(pointCount, metric, updateTime))
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep(int(t2 + 1) - t2)
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
lastSecond = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
dbDir = dirname(dbFilePath)
try:
os.makedirs(dbDir, 0755)
except OSError as e:
log.err("%s" % e)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
instrumentation.increment('creates')
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
t2 = time.time()
updateTime = t2 - t1
except:
log.msg("Error writing to %s" % (dbFilePath))
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep(int(t2 + 1) - t2)
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
cache = MetricCache()
while cache:
dataWritten = False
for (metric, datapoints, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates('new metric %s matched schema %s' %
(metric, schema.name))
archiveConfig = [
archive.getTuple() for archive in schema.archives
]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates(
'new metric %s matched aggregation schema %s' %
(metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(
"No storage schema matched the metric '%s', check your storage-schemas.conf file."
% metric)
if settings.LOG_CREATES:
log.creates(
"creating database metric %s (archive=%s xff=%s agg=%s)"
% (metric, archiveConfig, xFilesFactor,
aggregationMethod))
try:
state.database.create(metric, archiveConfig, xFilesFactor,
aggregationMethod)
instrumentation.increment('creates')
except Exception, e:
log.err()
log.msg("Error creating %s: %s" % (metric, e))
instrumentation.increment('errors')
continue
# If we've got a rate limit configured lets makes sure we enforce it
if UPDATE_BUCKET:
UPDATE_BUCKET.drain(1, blocking=True)
try:
t1 = time.time()
# If we have duplicated points, always pick the last. update_many()
# has no guaranted behavior for that, and in fact the current implementation
# will keep the first point in the list.
datapoints = dict(datapoints).items()
state.database.write(metric, datapoints)
updateTime = time.time() - t1
except Exception, e:
log.err()
log.msg("Error writing to %s: %s" % (metric, e))
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" %
(pointCount, metric, updateTime))
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
lastSecond = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
dbDir = dirname(dbFilePath)
os.system("mkdir -p -m 755 '%s'" % dbDir)
log.creates("creating database file %s" % dbFilePath)
whisper.create(dbFilePath, archiveConfig)
os.chmod(dbFilePath, 0755)
instrumentation.increment('creates')
# Create metadata file
dbFileName = basename(dbFilePath)
metaFilePath = join(dbDir, dbFileName[ :-len('.wsp') ] + '.context.pickle')
createMetaFile(metric, schema, metaFilePath)
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
t2 = time.time()
updateTime = t2 - t1
except:
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep( int(t2 + 1) - t2 )
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
cache = MetricCache()
while cache:
(metric, datapoints) = cache.drain_metric()
if metric is None:
# end the loop
break
dbFileExists = state.database.exists(metric)
if not dbFileExists:
if CREATE_BUCKET and not CREATE_BUCKET.drain(1):
# If our tokenbucket doesn't have enough tokens available to create a new metric
# file then we'll just drop the metric on the ground and move on to the next
# metric.
# XXX This behavior should probably be configurable to no tdrop metrics
# when rate limitng unless our cache is too big or some other legit
# reason.
instrumentation.increment('droppedCreates')
continue
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates('new metric %s matched aggregation schema %s'
% (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(("No storage schema matched the metric '%s',"
" check your storage-schemas.conf file.") % metric)
if settings.LOG_CREATES:
log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
(metric, archiveConfig, xFilesFactor, aggregationMethod))
try:
state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
if settings.ENABLE_TAGS:
tagQueue.add(metric)
instrumentation.increment('creates')
except Exception as e:
log.err()
log.msg("Error creating %s: %s" % (metric, e))
instrumentation.increment('errors')
continue
# If we've got a rate limit configured lets makes sure we enforce it
waitTime = 0
if UPDATE_BUCKET:
t1 = time.time()
UPDATE_BUCKET.drain(1, blocking=True)
waitTime = time.time() - t1
try:
t1 = time.time()
# If we have duplicated points, always pick the last. update_many()
# has no guaranted behavior for that, and in fact the current implementation
# will keep the first point in the list.
datapoints = dict(datapoints).items()
state.database.write(metric, datapoints)
if settings.ENABLE_TAGS:
tagQueue.update(metric)
updateTime = time.time() - t1
except Exception as e:
log.err()
log.msg("Error writing to %s: %s" % (metric, e))
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
if waitTime > 0.001:
log.updates("wrote %d datapoints for %s in %.5f seconds after waiting %.5f seconds" % (
pointCount, metric, updateTime, waitTime))
else:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (
pointCount, metric, updateTime))
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
global updates
global lastSecond
# while MetricCache:
# dataWritten = False
for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
try:
dbDir = dirname(dbFilePath)
os.system("umask u=rwx,go=rx ; mkdir -p -m 755 '%s'" % dbDir)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod)
os.chmod(dbFilePath, 0755)
instrumentation.increment('creates')
except IOError as e:
log.msg("IOError: {0}".format(e))
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
t2 = time.time()
updateTime = t2 - t1
except:
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep( int(t2 + 1) - t2 )
time.sleep(5)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
lastSecond = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath, dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
dbDir = dirname(dbFilePath)
os.system("mkdir -p '%s'" % dbDir)
log.creates("creating database file %s" % dbFilePath)
whisper.create(dbFilePath, archiveConfig)
increment('creates')
# Create metadata file
dbFileName = basename(dbFilePath)
metaFilePath = join(dbDir, dbFileName[ :-len('.wsp') ] + '.context.pickle')
createMetaFile(metric, schema, metaFilePath)
data = {}
try:
t1 = time.time()
counts = {}
for dp in datapoints:
key = long(dp[0] / 60)
if not key in data:
data[key] = 0
counts[key] = 0
data[key] += dp[1]
counts[key] += 1
if "/rt" in dbFilePath or "latency_ms" in dbFilePath:
print "AVERAGING", dbFilePath
for key, value in data.iteritems():
data[key] = value / counts[key]
for dptime, datavalue in data.iteritems():
whisper.update(dbFilePath, datavalue, dptime * 60)
t2 = time.time()
updateTime = t2 - t1
except:
log.err()
increment('errors')
else:
pointCount = len(data)
log.updates("wrote %d datapoints for %s in %.5f seconds" % (pointCount, metric, updateTime))
increment('committedPoints', pointCount)
append('updateTimes', updateTime)
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep( int(t2 + 1) - t2 )
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
cache = MetricCache()
while cache:
(metric, datapoints) = cache.drain_metric()
if metric is None:
# end the loop
break
dbFileExists = state.database.exists(metric)
if not dbFileExists:
if CREATE_BUCKET and not CREATE_BUCKET.drain(1):
# If our tokenbucket doesn't have enough tokens available to create a new metric
# file then we'll just drop the metric on the ground and move on to the next
# metric.
# XXX This behavior should probably be configurable to no tdrop metrics
# when rate limitng unless our cache is too big or some other legit
# reason.
instrumentation.increment('droppedCreates')
continue
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
if settings.LOG_CREATES:
log.creates('new metric %s matched aggregation schema %s'
% (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(("No storage schema matched the metric '%s',"
" check your storage-schemas.conf file.") % metric)
if settings.LOG_CREATES:
log.creates("creating database metric %s (archive=%s xff=%s agg=%s)" %
(metric, archiveConfig, xFilesFactor, aggregationMethod))
try:
state.database.create(metric, archiveConfig, xFilesFactor, aggregationMethod)
if settings.ENABLE_TAGS:
tagQueue.add(metric)
instrumentation.increment('creates')
except Exception as e:
log.err()
log.msg("Error creating %s: %s" % (metric, e))
instrumentation.increment('errors')
continue
# If we've got a rate limit configured lets makes sure we enforce it
waitTime = 0
if UPDATE_BUCKET:
t1 = time.time()
yield UPDATE_BUCKET.drain(1, blocking=True)
waitTime = time.time() - t1
try:
t1 = time.time()
# If we have duplicated points, always pick the last. update_many()
# has no guaranted behavior for that, and in fact the current implementation
# will keep the first point in the list.
datapoints = dict(datapoints).items()
state.database.write(metric, datapoints)
if settings.ENABLE_TAGS:
tagQueue.update(metric)
updateTime = time.time() - t1
except Exception as e:
log.err()
log.msg("Error writing to %s: %s" % (metric, e))
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
if waitTime > 0.001:
log.updates("wrote %d datapoints for %s in %.5f seconds after waiting %.5f seconds" % (
pointCount, metric, updateTime, waitTime))
else:
log.updates("wrote %d datapoints for %s in %.5f seconds" % (
pointCount, metric, updateTime))
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath,
dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in SCHEMAS:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' %
(metric, schema.name))
archiveConfig = [
archive.getTuple() for archive in schema.archives
]
break
for schema in AGGREGATION_SCHEMAS:
if schema.matches(metric):
log.creates(
'new metric %s matched aggregation schema %s' %
(metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception(
"No storage schema matched the metric '%s', check your storage-schemas.conf file."
% metric)
dbDir = dirname(dbFilePath)
try:
if not exists(dbDir):
os.makedirs(dbDir)
except OSError, e:
log.err("%s" % e)
log.creates(
"creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor,
aggregationMethod))
try:
whisper.create(dbFilePath, archiveConfig, xFilesFactor,
aggregationMethod,
settings.WHISPER_SPARSE_CREATE,
settings.WHISPER_FALLOCATE_CREATE)
instrumentation.increment('creates')
except:
log.err("Error creating %s" % (dbFilePath))
continue
# If we've got a rate limit configured lets makes sure we enforce it
if UPDATE_BUCKET:
UPDATE_BUCKET.drain(1, blocking=True)
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
updateTime = time.time() - t1
except Exception:
log.msg("Error writing to %s" % (dbFilePath))
log.err()
instrumentation.increment('errors')
else:
pointCount = len(datapoints)
instrumentation.increment('committedPoints', pointCount)
instrumentation.append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" %
(pointCount, metric, updateTime))
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)
def writeCachedDataPoints():
"Write datapoints until the MetricCache is completely empty"
updates = 0
lastSecond = 0
while MetricCache:
dataWritten = False
for (metric, datapoints, dbFilePath,
dbFileExists) in optimalWriteOrder():
dataWritten = True
if not dbFileExists:
archiveConfig = None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' %
(metric, schema.name))
archiveConfig = [
archive.getTuple() for archive in schema.archives
]
break
if not archiveConfig:
raise Exception(
"No storage schema matched the metric '%s', check your storage-schemas.conf file."
% metric)
dbDir = dirname(dbFilePath)
os.system("mkdir -p -m 755 '%s'" % dbDir)
log.creates("creating database file %s" % dbFilePath)
whisper.create(dbFilePath, archiveConfig)
os.chmod(dbFilePath, 0755)
increment('creates')
# Create metadata file
dbFileName = basename(dbFilePath)
metaFilePath = join(
dbDir, dbFileName[:-len('.wsp')] + '.context.pickle')
createMetaFile(metric, schema, metaFilePath)
try:
t1 = time.time()
whisper.update_many(dbFilePath, datapoints)
t2 = time.time()
updateTime = t2 - t1
except:
log.err()
increment('errors')
else:
pointCount = len(datapoints)
increment('committedPoints', pointCount)
append('updateTimes', updateTime)
if settings.LOG_UPDATES:
log.updates("wrote %d datapoints for %s in %.5f seconds" %
(pointCount, metric, updateTime))
# Rate limit update operations
thisSecond = int(t2)
if thisSecond != lastSecond:
lastSecond = thisSecond
updates = 0
else:
updates += 1
if updates >= settings.MAX_UPDATES_PER_SECOND:
time.sleep(int(t2 + 1) - t2)
# Avoid churning CPU when only new metrics are in the cache
if not dataWritten:
time.sleep(0.1)