def waterlevel(db_name):
"""Reduce alert frequency after initial alert, reset on all-clear"""
(times, fail_buffer) = whisper.fetch(db_name, 315550800)
if fail_buffer.count(1) > 2:
new_whisper_db_name = db_name + '.wsp2'
whisper.create(new_whisper_db_name, FOLLOWUP, aggregationMethod='last')
whisper.update(new_whisper_db_name, 1)
os.rename(new_whisper_db_name, db_name)
for admin in sys.argv[2:]:
os.system('mail -s "' + sys.argv[1] + '" ' + admin + '</dev/null')
if fail_buffer.count(1) == 0:
if whisper.info(
db_name)['archives'][0]['secondsPerPoint'] == FOLLOWUP[0][0]:
new_whisper_db_name = db_name + '.wsp2'
whisper.create(new_whisper_db_name,
RETAINER,
aggregationMethod='last')
whisper.update(new_whisper_db_name, 0)
os.rename(new_whisper_db_name, db_name)
for admin in sys.argv[2:]:
os.system('mail -s "' + sys.argv[1] + '" ' + admin +
'</dev/null')
return (0)
def benchmark_create_update_fetch():
path, archive_list, tear_down = set_up_create()
# start timer
start_time = time.clock()
for i in range(100):
whisper.create(path, archive_list)
seconds_ago = 3500
current_value = 0.5
increment = 0.2
now = time.time()
# file_update closes the file so we have to reopen every time
for i in range(seconds_ago):
whisper.update(path, current_value, now - seconds_ago + i)
current_value += increment
from_time = now - seconds_ago
until_time = from_time + 1000
whisper.fetch(path, from_time, until_time)
tear_down()
# end timer
end_time = time.clock()
elapsed_time = end_time - start_time
print "Executed 100 iterations in %ss (%i ns/op)" % (
elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100)
def test_single_metric(self):
xfilesfactor = 0.5
aggregation_method = "last"
# This retentions are such that every other point is present in both
# archives. Test validates that duplicate points gets inserted only once.
retentions = [(1, 10), (2, 10)]
high_precision_duration = retentions[0][0] * retentions[0][1]
low_precision_duration = retentions[1][0] * retentions[1][1]
now = int(time.time())
time_from, time_to = now - low_precision_duration, now
points = [(float(t), float(now-t)) for t in xrange(time_from, time_to)]
metric = "test_metric"
metric_path = os_path.join(self.tempdir, metric + ".wsp")
whisper.create(metric_path, retentions, xfilesfactor, aggregation_method)
whisper.update_many(metric_path, points)
self._call_main()
metric = self.accessor.get_metric(metric)
self.assertTrue(metric)
self.assertEqual(metric.name, metric.name)
self.assertEqual(metric.aggregator.carbon_name, aggregation_method)
self.assertEqual(metric.carbon_xfilesfactor, xfilesfactor)
self.assertEqual(metric.retention.as_string, "10*1s:10*2s")
points_again = list(self.accessor.fetch_points(
metric, time_from, time_to, metric.retention[0]))
self.assertEqual(points[-high_precision_duration:], points_again)
def update(self, updates):
"""
whisperをまとめて更新する
"""
# updatesをbox_id別に管理する
sorted_updates = defaultdict(lambda: defaultdict(int))
# whisperのtimestampは秒単位なので、秒以下はまとめる
for box_id, timestamp in updates:
sorted_updates[box_id][int(timestamp)] += 1
for box_id, timestamps in sorted_updates.items():
# whisperの中でSORTかけてた
# timestamps = sorted(timestamps.items())
timestamps = timestamps.items()
db_path = os.path.join(self.dir_prefix,
TimedDB.make_db_name(box_id))
if not os.path.exists(db_path):
logger.info('create whsiper db for box %s at path %s', box_id,
db_path)
whisper.create(db_path,
WHISPER_ARCHIVES,
xFilesFactor=0.5,
aggregationMethod='sum',
sparse=False,
useFallocate=True)
with get_lock(db_path):
whisper.update_many(db_path, timestamps)
def test_fetch(self):
"""fetch info from database """
# check a db that doesnt exist
with self.assertRaises(Exception):
whisper.fetch("this_db_does_not_exist", 0)
# SECOND MINUTE HOUR DAY
retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
whisper.create(self.db, retention)
# check a db with an invalid time range
with self.assertRaises(whisper.InvalidTimeInterval):
whisper.fetch(self.db, time.time(), time.time() - 6000)
fetch = whisper.fetch(self.db, 0)
# check time range
self.assertEqual(fetch[0][1] - fetch[0][0],
retention[-1][0] * retention[-1][1])
# check number of points
self.assertEqual(len(fetch[1]), retention[-1][1])
# check step size
self.assertEqual(fetch[0][2], retention[-1][0])
self._removedb()
def test_file_fetch_edge_cases(self):
"""
Test some of the edge cases in file_fetch() that should return
None or raise an exception
"""
whisper.create(self.filename, [(1, 60)])
with open(self.filename, 'rb') as fh:
msg = "Invalid time interval: from time '{0}' is after until time '{1}'"
until_time = 0
from_time = int(time.time()) + 100
with AssertRaisesException(
whisper.InvalidTimeInterval(msg.format(from_time, until_time))):
whisper.file_fetch(fh, fromTime=from_time, untilTime=until_time)
# fromTime > now aka metrics from the future
self.assertIsNone(
whisper.file_fetch(fh, fromTime=int(time.time()) + 100,
untilTime=int(time.time()) + 200),
)
# untilTime > oldest time stored in the archive
headers = whisper.info(self.filename)
the_past = int(time.time()) - headers['maxRetention'] - 200
self.assertIsNone(
whisper.file_fetch(fh, fromTime=the_past - 1, untilTime=the_past),
)
# untilTime > now, change untilTime to now
now = int(time.time())
self.assertEqual(
whisper.file_fetch(fh, fromTime=now, untilTime=now + 200, now=now),
((now + 1, now + 2, 1), [None]),
)
def test_render_view(self):
url = reverse('graphite.render.views.renderView')
response = self.client.get(url, {'target': 'test', 'format': 'json'})
self.assertEqual(json.loads(response.content), [])
self.assertTrue(response.has_header('Expires'))
self.assertTrue(response.has_header('Last-Modified'))
self.assertTrue(response.has_header('Cache-Control'))
response = self.client.get(url, {'target': 'test'})
self.assertEqual(response['Content-Type'], 'image/png')
self.assertTrue(response.has_header('Expires'))
self.assertTrue(response.has_header('Last-Modified'))
self.assertTrue(response.has_header('Cache-Control'))
self.addCleanup(self.wipe_whisper)
whisper.create(self.db, [(1, 60)])
ts = int(time.time())
whisper.update(self.db, 0.5, ts - 2)
whisper.update(self.db, 0.4, ts - 1)
whisper.update(self.db, 0.6, ts)
response = self.client.get(url, {'target': 'test', 'format': 'json'})
data = json.loads(response.content)
end = data[0]['datapoints'][-4:]
self.assertEqual(
end, [[None, ts - 3], [0.5, ts - 2], [0.4, ts - 1], [0.6, ts]])
def test_fetch(self):
"""
fetch info from database
"""
# Don't use AssertRaisesException due to a super obscure bug in
# python2.6 which returns an IOError in the 2nd argument of __exit__
# in a context manager as a tuple. See this for a minimal reproducer:
# http://git.io/cKz30g
with self.assertRaises(IOError):
# check a db that doesnt exist
whisper.fetch("this_db_does_not_exist", 0)
# SECOND MINUTE HOUR DAY
retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
whisper.create(self.filename, retention)
# check a db with an invalid time range
now = int(time.time())
past = now - 6000
msg = "Invalid time interval: from time '{0}' is after until time '{1}'"
with AssertRaisesException(whisper.InvalidTimeInterval(msg.format(now, past))):
whisper.fetch(self.filename, now, past)
fetch = whisper.fetch(self.filename, 0)
# check time range
self.assertEqual(fetch[0][1] - fetch[0][0],
retention[-1][0] * retention[-1][1])
# check number of points
self.assertEqual(len(fetch[1]), retention[-1][1])
# check step size
self.assertEqual(fetch[0][2], retention[-1][0])
def test_fetch(self):
"""fetch info from database """
# check a db that doesnt exist
with self.assertRaises(Exception):
whisper.fetch("this_db_does_not_exist", 0)
# SECOND MINUTE HOUR DAY
retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
whisper.create(self.db, retention)
# check a db with an invalid time range
with self.assertRaises(whisper.InvalidTimeInterval):
whisper.fetch(self.db, time.time(), time.time()-6000)
fetch = whisper.fetch(self.db, 0)
# check time range
self.assertEqual(fetch[0][1] - fetch[0][0],
retention[-1][0] * retention[-1][1])
# check number of points
self.assertEqual(len(fetch[1]), retention[-1][1])
# check step size
self.assertEqual(fetch[0][2], retention[-1][0])
self._removedb()
def createWhisperFile(metric, dbFilePath, dbFileExists):
if not dbFileExists:
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
dbDir = dirname(dbFilePath)
try:
os.makedirs(dbDir)
except OSError as e:
if e.errno != errno.EEXIST:
log.err("%s" % e)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
try:
whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE, settings.WHISPER_FALLOCATE_CREATE)
instrumentation.increment('creates')
except Exception, e:
log.err("Error creating %s: %s" % (dbFilePath, e))
return False
def test_single_metric(self):
xfilesfactor = 0.5
aggregation_method = "last"
# This retentions are such that every other point is present in both
# archives. Test validates that duplicate points gets inserted only once.
retentions = [(1, 10), (2, 10)]
high_precision_duration = retentions[0][0] * retentions[0][1]
low_precision_duration = retentions[1][0] * retentions[1][1]
now = int(time.time())
time_from, time_to = now - low_precision_duration, now
points = [(float(t), float(now - t))
for t in xrange(time_from, time_to)]
metric = "test_metric"
metric_path = os_path.join(self.tempdir, metric + ".wsp")
whisper.create(metric_path, retentions, xfilesfactor,
aggregation_method)
whisper.update_many(metric_path, points)
self._call_main()
metric = self.accessor.get_metric(metric)
self.assertTrue(metric)
self.assertEqual(metric.name, metric.name)
self.assertEqual(metric.aggregator.carbon_name, aggregation_method)
self.assertEqual(metric.carbon_xfilesfactor, xfilesfactor)
self.assertEqual(metric.retention.as_string, "10*1s:10*2s")
points_again = list(
self.accessor.fetch_points(metric, time_from, time_to,
metric.retention[0]))
self.assertEqual(points[-high_precision_duration:], points_again)
def test_render_view(self):
url = reverse('graphite.render.views.renderView')
response = self.client.get(url, {'target': 'test', 'format': 'json'})
self.assertEqual(json.loads(response.content), [])
self.assertTrue(response.has_header('Expires'))
self.assertTrue(response.has_header('Last-Modified'))
self.assertTrue(response.has_header('Cache-Control'))
response = self.client.get(url, {'target': 'test'})
self.assertEqual(response['Content-Type'], 'image/png')
self.assertTrue(response.has_header('Expires'))
self.assertTrue(response.has_header('Last-Modified'))
self.assertTrue(response.has_header('Cache-Control'))
self.addCleanup(self.wipe_whisper)
whisper.create(self.db, [(1, 60)])
ts = int(time.time())
whisper.update(self.db, 0.5, ts - 2)
whisper.update(self.db, 0.4, ts - 1)
whisper.update(self.db, 0.6, ts)
response = self.client.get(url, {'target': 'test', 'format': 'json'})
data = json.loads(response.content)
end = data[0]['datapoints'][-4:]
self.assertEqual(
end, [[None, ts - 3], [0.5, ts - 2], [0.4, ts - 1], [0.6, ts]])
def create_db(self, metric):
archiveConfig = None
xFilesFactor, aggregationMethod = None, None
for schema in schemas:
if schema.matches(metric):
log.creates('new metric %s matched schema %s' % (metric, schema.name))
archiveConfig = [archive.getTuple() for archive in schema.archives]
break
for schema in agg_schemas:
if schema.matches(metric):
log.creates('new metric %s matched aggregation schema %s' % (metric, schema.name))
xFilesFactor, aggregationMethod = schema.archives
break
if not archiveConfig:
raise Exception("No storage schema matched the metric '%s', check your storage-schemas.conf file." % metric)
dbDir = dirname(dbFilePath)
os.system("mkdir -p -m 755 '%s'" % dbDir)
log.creates("creating database file %s (archive=%s xff=%s agg=%s)" %
(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod))
whisper.create(dbFilePath, archiveConfig, xFilesFactor, aggregationMethod, settings.WHISPER_SPARSE_CREATE)
os.chmod(dbFilePath, 0755)
def _populate_data(self):
self.db = os.path.join(settings.WHISPER_DIR, 'test.wsp')
whisper.create(self.db, [(1, 60)])
ts = int(time.time())
for i, value in enumerate(reversed(self._test_data)):
whisper.update(self.db, value, ts - i)
self.ts = ts
def test_update_many_excess(self):
# given an empty db
wsp = "test_update_many_excess.wsp"
self.addCleanup(self._remove, wsp)
archive_len = 3
archive_step = 1
whisper.create(wsp, [(archive_step, archive_len)])
# given too many points than the db can hold
excess_len = 1
num_input_points = archive_len + excess_len
test_now = int(time.time())
input_start = test_now - num_input_points + archive_step
input_points = [(input_start + i, random.random() * 10)
for i in range(num_input_points)]
# when the db is updated with too many points
whisper.update_many(wsp, input_points, now=test_now)
# then only the most recent input points (those at the end) were written
actual_time_info = whisper.fetch(wsp, 0, now=test_now)[0]
self.assertEqual(actual_time_info,
(input_points[-archive_len][0],
input_points[-1][0] + archive_step, # untilInterval = newest + step
archive_step))
def test_normal(self):
whisper.create(self.filename, [(1, 60), (60, 60)])
whisper.CACHE_HEADERS = True
whisper.info(self.filename)
whisper.info(self.filename)
whisper.CACHE_HEADERS = False
def test_setAggregation(self):
"""Create a db, change aggregation, xFilesFactor, then use info() to validate"""
retention = [(1, 60), (60, 60)]
# create a new db with a valid configuration
whisper.create(self.db, retention)
#set setting every AggregationMethod available
for ag in whisper.aggregationMethods:
for xff in 0.0,0.2,0.4,0.7,0.75,1.0:
#original xFilesFactor
info0 = whisper.info(self.db)
#optional xFilesFactor not passed
whisper.setAggregationMethod(self.db, ag)
#original value should not change
info1 = whisper.info(self.db)
self.assertEqual(info0['xFilesFactor'],info1['xFilesFactor'])
#the selected aggregation method should have applied
self.assertEqual(ag,info1['aggregationMethod'])
#optional xFilesFactor used
whisper.setAggregationMethod(self.db, ag, xff)
#new info should match what we just set it to
info2 = whisper.info(self.db)
#packing and unpacking because
#AssertionError: 0.20000000298023224 != 0.2
target_xff = struct.unpack("!f", struct.pack("!f",xff))[0]
self.assertEqual(info2['xFilesFactor'], target_xff)
#same aggregationMethod asssertion again, but double-checking since
#we are playing with packed values and seek()
self.assertEqual(ag,info2['aggregationMethod'])
self._removedb()
def test_setAggregation(self):
"""Create a db, change aggregation, xFilesFactor, then use info() to validate"""
retention = [(1, 60), (60, 60)]
# create a new db with a valid configuration
whisper.create(self.db, retention)
#set setting every AggregationMethod available
for ag in whisper.aggregationMethods:
for xff in 0.0, 0.2, 0.4, 0.7, 0.75, 1.0:
#original xFilesFactor
info0 = whisper.info(self.db)
#optional xFilesFactor not passed
whisper.setAggregationMethod(self.db, ag)
#original value should not change
info1 = whisper.info(self.db)
self.assertEqual(info0['xFilesFactor'], info1['xFilesFactor'])
#the selected aggregation method should have applied
self.assertEqual(ag, info1['aggregationMethod'])
#optional xFilesFactor used
whisper.setAggregationMethod(self.db, ag, xff)
#new info should match what we just set it to
info2 = whisper.info(self.db)
#packing and unpacking because
#AssertionError: 0.20000000298023224 != 0.2
target_xff = struct.unpack("!f", struct.pack("!f", xff))[0]
self.assertEqual(info2['xFilesFactor'], target_xff)
#same aggregationMethod asssertion again, but double-checking since
#we are playing with packed values and seek()
self.assertEqual(ag, info2['aggregationMethod'])
self._removedb()
def benchmark_create_update_fetch(): path, archive_list, tear_down = set_up_create() # start timer start_time = time.clock() for i in range(100): whisper.create(path, archive_list) seconds_ago = 3500 current_value = 0.5 increment = 0.2 now = time.time() # file_update closes the file so we have to reopen every time for i in range(seconds_ago): whisper.update(path, current_value, now - seconds_ago + i) current_value += increment from_time = now - seconds_ago until_time = from_time + 1000 whisper.fetch(path, from_time, until_time) tear_down() # end timer end_time = time.clock() elapsed_time = end_time - start_time print "Executed 100 iterations in %ss (%i ns/op)" % (elapsed_time, (elapsed_time * 1000 * 1000 * 1000) / 100)
def populate_data(self):
self.db = os.path.join(settings.WHISPER_DIR, 'test.wsp')
whisper.create(self.db, [(1, 60)])
ts = int(time.time())
for i, value in enumerate(reversed(self._test_data)):
whisper.update(self.db, value, ts - i)
self.ts = ts
def test_file_fetch_edge_cases(self):
"""
Test some of the edge cases in file_fetch() that should return
None or raise an exception
"""
whisper.create(self.filename, [(1, 60)])
with open(self.filename, 'rb') as fh:
msg = "Invalid time interval: from time '{0}' is after until time '{1}'"
until_time = 0
from_time = int(time.time()) + 100
with AssertRaisesException(
whisper.InvalidTimeInterval(msg.format(from_time, until_time))):
whisper.file_fetch(fh, fromTime=from_time, untilTime=until_time)
# fromTime > now aka metrics from the future
self.assertIsNone(
whisper.file_fetch(fh, fromTime=int(time.time()) + 100,
untilTime=int(time.time()) + 200),
)
# untilTime > oldest time stored in the archive
headers = whisper.info(self.filename)
the_past = int(time.time()) - headers['maxRetention'] - 200
self.assertIsNone(
whisper.file_fetch(fh, fromTime=the_past - 1, untilTime=the_past),
)
# untilTime > now, change untilTime to now
now = int(time.time())
self.assertEqual(
whisper.file_fetch(fh, fromTime=now, untilTime=now + 200, now=now),
((now + 1, now + 2, 1), [None]),
)
def test_fetch(self):
"""
fetch info from database
"""
# Don't use AssertRaisesException due to a super obscure bug in
# python2.6 which returns an IOError in the 2nd argument of __exit__
# in a context manager as a tuple. See this for a minimal reproducer:
# http://git.io/cKz30g
with self.assertRaises(IOError):
# check a db that doesnt exist
whisper.fetch("this_db_does_not_exist", 0)
# SECOND MINUTE HOUR DAY
retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
whisper.create(self.filename, retention)
# check a db with an invalid time range
now = int(time.time())
past = now - 6000
msg = "Invalid time interval: from time '{0}' is after until time '{1}'"
with AssertRaisesException(whisper.InvalidTimeInterval(msg.format(now, past))):
whisper.fetch(self.filename, now, past)
fetch = whisper.fetch(self.filename, 0)
# check time range
self.assertEqual(fetch[0][1] - fetch[0][0],
retention[-1][0] * retention[-1][1])
# check number of points
self.assertEqual(len(fetch[1]), retention[-1][1])
# check step size
self.assertEqual(fetch[0][2], retention[-1][0])
def test_normal(self):
whisper.create(self.filename, [(1, 60), (60, 60)])
whisper.CACHE_HEADERS = True
whisper.info(self.filename)
whisper.info(self.filename)
whisper.CACHE_HEADERS = False
def test_update_many_excess(self):
# given an empty db
wsp = "test_update_many_excess.wsp"
self.addCleanup(self._remove, wsp)
archive_len = 3
archive_step = 1
whisper.create(wsp, [(archive_step, archive_len)])
# given too many points than the db can hold
excess_len = 1
num_input_points = archive_len + excess_len
test_now = int(time.time())
input_start = test_now - num_input_points + archive_step
input_points = [(input_start + i, random.random() * 10)
for i in range(num_input_points)]
# when the db is updated with too many points
whisper.update_many(wsp, input_points, now=test_now)
# then only the most recent input points (those at the end) were written
actual_time_info = whisper.fetch(wsp, 0, now=test_now)[0]
self.assertEqual(actual_time_info,
(input_points[-archive_len][0],
input_points[-1][0] + archive_step, # untilInterval = newest + step
archive_step))
def _create_dbs(self):
for db in (
('test', 'foo.wsp'),
('test', 'bar', 'baz.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
def test_00_create_empty_whisper(self):
"""
Create a whisper file with one archive of 5 points where each point
covers 60 second, and default xFilesFactor and aggregationMethod.
"""
self.assertTrue(not os.path.exists(FILENAME))
whisper.create(FILENAME, [(SECONDS_PER_POINT, NUMBER_OF_POINTS)])
self.assertTrue(os.path.exists(FILENAME))
def create_rrd_file(self, postfix, overwrite=False):
path = self.get_rrd_file(postfix)
if os.path.exists(path) and not overwrite:
return
whisper.create(path, self.archiveList, self.xFilesFactor,
self.get_aggregationMethod(postfix))
def create(self, filename):
whisper.create(
filename,
self.archives,
xFilesFactor=self.x_files_factor,
aggregationMethod=self.aggregation_method,
sparse=self.sparse,
)
def record_metering_data(self, data):
record = copy.deepcopy(data)
timestamp = record["timestamp"].replace(second=0, microsecond=0)
timestamp = int((timestamp - datetime.datetime(1970, 1, 1)).total_seconds())
value = float(record["counter_volume"])
record_path = (
env_variables["whisper_path"] + data["resource_id"] + "_" + data["counter_name"].replace(".", "_") + ".wsp"
)
# if not os.path.isdir(os.path.dirname(record_path)):
# os.makedirs(os.path.dirname(record_path))
if not os.path.isfile(record_path):
whisper.create(record_path, archieve_list)
whisper.update(record_path, value, timestamp)
# add resource & meter to sqlite db
conn = sqlite3.connect(env_variables["sql_db_path"])
c = conn.cursor()
c.execute("select count(*) from resources where resource_id=?", (data["resource_id"],))
r = c.fetchone()[0]
if r == 0:
c.execute(
"insert into resources (resource_id, user_id, project_id, source_id, resource_metadata)"
+ "values (?,?,?,?,?)",
(
data["resource_id"],
data["user_id"],
data["project_id"],
data["source"],
json.dumps(data["resource_metadata"]),
),
)
c.execute(
"select count(*) from meters where name=? and resource_id=?", (data["counter_name"], data["resource_id"])
)
r = c.fetchone()[0]
if r == 0:
c.execute(
"insert into meters (name, type, unit, resource_id, project_id, user_id, source)"
+ "values (?,?,?,?,?,?,?)",
(
data["counter_name"],
data["counter_type"],
data["counter_unit"],
data["resource_id"],
data["project_id"],
data["user_id"],
data["source"],
),
)
conn.commit()
conn.close()
def store(file, data): print "store" import whisper try: whisper.create(file, [(1,60*60*24), (10, 60*60*24), (60, 60*60*24*30) ]) except: pass print "update" whisper.update(file, data)
def test_setAggregation(self):
"""
Create a db, change aggregation, xFilesFactor, then use info() to validate
"""
original_lock = whisper.LOCK
original_caching = whisper.CACHE_HEADERS
original_autoflush = whisper.AUTOFLUSH
whisper.LOCK = True
whisper.AUTOFLUSH = True
whisper.CACHE_HEADERS = True
# create a new db with a valid configuration
whisper.create(self.filename, self.retention)
with AssertRaisesException(whisper.InvalidAggregationMethod('Unrecognized aggregation method: yummy beer')):
whisper.setAggregationMethod(self.filename, 'yummy beer')
#set setting every AggregationMethod available
for ag in whisper.aggregationMethods:
for xff in 0.0, 0.2, 0.4, 0.7, 0.75, 1.0:
# original xFilesFactor
info0 = whisper.info(self.filename)
# optional xFilesFactor not passed
old_ag = whisper.setAggregationMethod(self.filename, ag)
# should return old aggregationmethod
self.assertEqual(old_ag, info0['aggregationMethod'])
# original value should not change
info1 = whisper.info(self.filename)
self.assertEqual(info0['xFilesFactor'], info1['xFilesFactor'])
# the selected aggregation method should have applied
self.assertEqual(ag, info1['aggregationMethod'])
# optional xFilesFactor used
old_ag = whisper.setAggregationMethod(self.filename, ag, xff)
# should return old aggregationmethod
self.assertEqual(old_ag, info1['aggregationMethod'])
# new info should match what we just set it to
info2 = whisper.info(self.filename)
# packing and unpacking because
# AssertionError: 0.20000000298023224 != 0.2
target_xff = struct.unpack("!f", struct.pack("!f", xff))[0]
self.assertEqual(info2['xFilesFactor'], target_xff)
# same aggregationMethod assertion again, but double-checking since
# we are playing with packed values and seek()
self.assertEqual(ag, info2['aggregationMethod'])
with SimulatedCorruptWhisperFile():
with AssertRaisesException(whisper.CorruptWhisperFile('Unable to read header', self.filename)):
whisper.setAggregationMethod(self.filename, ag)
whisper.LOCK = original_lock
whisper.AUTOFLUSH = original_autoflush
whisper.CACHE_HEADERS = original_caching
def _create(self):
"""Create the Whisper file on disk"""
if not os.path.exists(settings.SALMON_WHISPER_DB_PATH):
os.makedirs(settings.SALMON_WHISPER_DB_PATH)
archives = [whisper.parseRetentionDef(retentionDef)
for retentionDef in settings.ARCHIVES.split(",")]
whisper.create(self.path, archives,
xFilesFactor=settings.XFILEFACTOR,
aggregationMethod=settings.AGGREGATION_METHOD)
def create_whisper(self, path, gz=False):
path = join(self.test_dir, path)
if not isdir(dirname(path)):
os.makedirs(dirname(path))
whisper.create(path, [(1, 60)])
if gz:
with open(path, 'rb') as f_in, gzip.open("%s.gz" % path, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(path)
def _createdb(self, wsp, schema=[(1, 20)], data=None):
whisper.create(wsp, schema)
if data is None:
tn = time.time() - 20
data = []
for i in range(20):
data.append((tn + 1 + i, random.random() * 10))
whisper.update_many(wsp, data)
return data
def _createdb(self, wsp, schema=[(1, 20)], data=None):
whisper.create(wsp, schema)
if data is None:
tn = time.time() - 20
data = []
for i in range(20):
data.append((tn + 1 + i, random.random() * 10))
whisper.update_many(wsp, data)
return data
def _write_data(self, hostID, plugin, data_json):
# For each plugin
if plugin == None:
return False
data = self.jsonToPython(data_json)
data_timestamp = data.get('TimeStamp')
# TODO dont write data if no infos
# Get data_path
data_path = self._redis_connexion.redis_hget("WSP_PATH", hostID)
if data_path is None:
return False
wspPath = '%s/%s' % (data_path, plugin)
for ds_name, value in data.get("Values", {}).iteritems():
ds_path = '%s/%s.wsp' % (wspPath, ds_name)
# Create wsp file - config wsp here
if not os.path.isfile(ds_path):
self._logger.warning("writewsp host : %s Create wsp file : %s"
% (hostID, ds_path))
# Create directory
if not os.path.exists(wspPath):
try:
os.makedirs(wspPath)
self._logger.info("writewsp host : %s make directory : %s"
% (hostID, wspPath))
except OSError:
self._logger.error("writewsp host : %s can't make directory : %s"
% (hostID, wspPath))
continue
try:
whisper.create(ds_path,
[(60, 1440), # --- Daily (1 Minute Average)
(300, 2016), # --- Weekly (5 Minute Average)
(600, 4608), # --- Monthly (10 Min Average)
(3600, 8784)]) # --- Yearly (1 Hour Average)
except Exception as e:
self._logger.error("writewsp host : %s Create wsp Error %s"
% (hostID, str(e)))
continue
# Update whisper
try:
self._logger.debug("writewsp host : %s Update wsp "
"Timestamp %s For value %s in file %s"
% (hostID, data_timestamp, value, ds_path))
whisper.update(ds_path, str(value), str(data_timestamp) )
except Exception as e:
self._logger.error("writewsp host : %s Update Error %s - %s"
% (hostID, ds_path, e))
continue
self._dataNumber += 1
return True
def test_setAggregation(self):
"""
Create a db, change aggregation, xFilesFactor, then use info() to validate
"""
original_lock = whisper.LOCK
original_caching = whisper.CACHE_HEADERS
original_autoflush = whisper.AUTOFLUSH
whisper.LOCK = True
whisper.AUTOFLUSH = True
whisper.CACHE_HEADERS = True
# create a new db with a valid configuration
whisper.create(self.filename, self.retention)
with AssertRaisesException(
whisper.InvalidAggregationMethod(
'Unrecognized aggregation method: yummy beer')):
whisper.setAggregationMethod(self.filename, 'yummy beer')
#set setting every AggregationMethod available
for ag in whisper.aggregationMethods:
for xff in 0.0, 0.2, 0.4, 0.7, 0.75, 1.0:
# original xFilesFactor
info0 = whisper.info(self.filename)
# optional xFilesFactor not passed
whisper.setAggregationMethod(self.filename, ag)
# original value should not change
info1 = whisper.info(self.filename)
self.assertEqual(info0['xFilesFactor'], info1['xFilesFactor'])
# the selected aggregation method should have applied
self.assertEqual(ag, info1['aggregationMethod'])
# optional xFilesFactor used
whisper.setAggregationMethod(self.filename, ag, xff)
# new info should match what we just set it to
info2 = whisper.info(self.filename)
# packing and unpacking because
# AssertionError: 0.20000000298023224 != 0.2
target_xff = struct.unpack("!f", struct.pack("!f", xff))[0]
self.assertEqual(info2['xFilesFactor'], target_xff)
# same aggregationMethod asssertion again, but double-checking since
# we are playing with packed values and seek()
self.assertEqual(ag, info2['aggregationMethod'])
with SimulatedCorruptWhisperFile():
with AssertRaisesException(
whisper.CorruptWhisperFile('Unable to read header',
self.filename)):
whisper.setAggregationMethod(self.filename, ag)
whisper.LOCK = original_lock
whisper.AUTOFLUSH = original_autoflush
whisper.CACHE_HEADERS = original_caching
def test_resize_with_aggregate(self):
"""resize whisper file with aggregate"""
# 60s per point save two days
retention = [(60, 60 * 24 * 2)]
whisper.create(self.filename, retention)
# insert data
now_timestamp = int(
(datetime.now() - datetime(1970, 1, 1)).total_seconds())
now_timestamp -= now_timestamp % 60 # format timestamp
points = [(now_timestamp - i * 60, i) for i in range(0, 60 * 24 * 2)]
whisper.update_many(self.filename, points)
data = whisper.fetch(self.filename,
fromTime=now_timestamp - 3600 * 25,
untilTime=now_timestamp - 3600 * 25 + 60 * 10)
self.assertEqual(len(data[1]), 10)
self.assertEqual(data[0][2], 60) # high retention == 60
for d in data[1]:
self.assertIsNotNone(d)
# resize from high to low
os.system(
'whisper-resize.py %s 60s:1d 300s:2d --aggregate --nobackup >/dev/null'
% self.filename) # noqa
data_low = whisper.fetch(self.filename,
fromTime=now_timestamp - 3600 * 25,
untilTime=now_timestamp - 3600 * 25 + 60 * 10)
self.assertEqual(len(data_low[1]), 2)
self.assertEqual(data_low[0][2], 300) # low retention == 300
for d in data_low[1]:
self.assertIsNotNone(d)
data_high = whisper.fetch(self.filename,
fromTime=now_timestamp - 60 * 10,
untilTime=now_timestamp)
self.assertEqual(len(data_high[1]), 10)
self.assertEqual(data_high[0][2], 60) # high retention == 60
# resize from low to high
os.system(
'whisper-resize.py %s 60s:2d --aggregate --nobackup >/dev/null' %
self.filename) # noqa
data1 = whisper.fetch(self.filename,
fromTime=now_timestamp - 3600 * 25,
untilTime=now_timestamp - 3600 * 25 + 60 * 10)
self.assertEqual(len(data1[1]), 10)
# noqa data1 looks like ((1588836720, 1588837320, 60), [None, None, 1490.0, None, None, None, None, 1485.0, None, None])
# data1[1] have two not none value
self.assertEqual(len(list(filter(lambda x: x is not None, data1[1]))),
2)
data2 = whisper.fetch(self.filename,
fromTime=now_timestamp - 60 * 15,
untilTime=now_timestamp - 60 * 5)
# noqa data2 looks like ((1588925820, 1588926420, 60), [10.0, 11.0, 10.0, 9.0, 8.0, 5.0, 6.0, 5.0, 4.0, 3.0])
self.assertEqual(len(list(filter(lambda x: x is not None, data2[1]))),
10)
# clean up
self.tearDown()
def create_whisper_files(self):
if self.args.dry_run:
self.log.info(
'Skipping creating whisper files because dry-run flag is set')
return
for metric in METRICS:
try:
whisper.create(self.get_whisper_file(metric), ARCHIVES)
except whisper.InvalidConfiguration:
pass # Already exists.
def test_get_structure(self):
whisper.create(os.path.join(settings.WHISPER_DIR, 'test2.wsp'), [(1, 60)])
# Create a bit of a structure
os.makedirs(os.path.join(settings.WHISPER_DIR, "level1", "level2", "level3"))
level1 = os.path.join(settings.WHISPER_DIR, "level1")
level2 = os.path.join(level1, "level2")
whisper.create(os.path.join(level1, 'level1_test.wsp'), [(1, 60)])
structure = query.get_structure()
self.assertIn("test", structure)
self.assertIsInstance(structure["test2"], LeafNode)
self.assertNotIn("level2", structure["level1"])
def test_merge_bad_archive_config(self):
testdb = "test-%s" % self.filename
# Create 2 whisper databases with different schema
self._update()
whisper.create(testdb, [(100, 1)])
with AssertRaisesException(NotImplementedError('db.wsp and test-db.wsp archive configurations are unalike. Resize the input before merging')):
whisper.merge(self.filename, testdb)
self._remove(testdb)
def create(self, metric, retentions, xfilesfactor, aggregation_method):
path = self.getFilesystemPath(metric)
directory = dirname(path)
try:
if not exists(directory):
os.makedirs(directory)
except OSError as e:
log.err("%s" % e)
whisper.create(path, retentions, xfilesfactor, aggregation_method,
self.sparse_create, self.fallocate_create)
def test_merge_bad_archive_config(self):
testdb = "test-%s" % self.filename
# Create 2 whisper databases with different schema
self._update()
whisper.create(testdb, [(100, 1)])
with AssertRaisesException(NotImplementedError('db.wsp and test-db.wsp archive configurations are unalike. Resize the input before merging')):
whisper.merge(self.filename, testdb)
self._remove(testdb)
def create(self, metric, retentions, xfilesfactor, aggregation_method):
path = self.getFilesystemPath(metric)
directory = dirname(path)
try:
if not exists(directory):
os.makedirs(directory)
except OSError as e:
log.err("%s" % e)
whisper.create(path, retentions, xfilesfactor, aggregation_method,
self.sparse_create, self.fallocate_create)
def test_file_diff_invalid(self):
testdb = "test-%s" % self.filename
whisper.create(testdb, [(120, 10)])
whisper.create(self.filename, self.retention)
# Merging 2 archives with different retentions should fail
with open(testdb, 'rb') as fh_1:
with open(self.filename, 'rb+') as fh_2:
with AssertRaisesException(NotImplementedError('test-db.wsp and db.wsp archive configurations are unalike. Resize the input before diffing')):
whisper.file_diff(fh_1, fh_2)
self._remove(testdb)
def write_series(self, series):
file_name = os.path.join(
WHISPER_DIR,
'{0}.wsp'.format(series.pathExpression.replace('.', os.sep)))
os.makedirs(os.path.dirname(file_name))
whisper.create(file_name, [(1, 180)])
data = []
for index, value in enumerate(series):
if value is None:
continue
data.append((series.start + index * series.step, value))
whisper.update_many(file_name, data)
def _create(self):
"""Create the Whisper file on disk"""
if not os.path.exists(settings.SALMON_WHISPER_DB_PATH):
os.makedirs(settings.SALMON_WHISPER_DB_PATH)
archives = [
whisper.parseRetentionDef(retentionDef)
for retentionDef in settings.ARCHIVES.split(",")
]
whisper.create(self.path,
archives,
xFilesFactor=settings.XFILEFACTOR,
aggregationMethod=settings.AGGREGATION_METHOD)
def test_file_diff_invalid(self):
testdb = "test-%s" % self.filename
whisper.create(testdb, [(120, 10)])
whisper.create(self.filename, self.retention)
# Merging 2 archives with different retentions should fail
with open(testdb, 'rb') as fh_1:
with open(self.filename, 'rb+') as fh_2:
with AssertRaisesException(NotImplementedError('test-db.wsp and db.wsp archive configurations are unalike. Resize the input before diffing')):
whisper.file_diff(fh_1, fh_2)
self._remove(testdb)
def createDs(uuid):
archives = [whisper.parseRetentionDef(retentionDef) for retentionDef in WHISPER_ARCHIVES]
dataFile = os.path.join(WHISPER_DATA,str(uuid) + ".wsp")
try:
os.makedirs(WHISPER_DATA)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
try:
whisper.create(dataFile, archives, xFilesFactor=0.5, aggregationMethod="average")
except whisper.WhisperException, exc:
raise SystemExit('[ERROR] %s' % str(exc))
def test_info_bogus_file(self):
self.assertIsNone(whisper.info('bogus-file'))
# Validate "corrupt" whisper metadata
whisper.create(self.filename, self.retention)
with SimulatedCorruptWhisperFile():
with AssertRaisesException(whisper.CorruptWhisperFile('Unable to read header', self.filename)):
whisper.info(self.filename)
# Validate "corrupt" whisper archive data
with SimulatedCorruptWhisperFile(corrupt_archive=True):
with AssertRaisesException(whisper.CorruptWhisperFile('Unable to read archive0 metadata', self.filename)):
whisper.info(self.filename)
def test_info_bogus_file(self):
self.assertIsNone(whisper.info('bogus-file'))
# Validate "corrupt" whisper metadata
whisper.create(self.filename, self.retention)
with SimulatedCorruptWhisperFile():
with AssertRaisesException(whisper.CorruptWhisperFile('Unable to read header', self.filename)):
whisper.info(self.filename)
# Validate "corrupt" whisper archive data
with SimulatedCorruptWhisperFile(corrupt_archive=True):
with AssertRaisesException(whisper.CorruptWhisperFile('Unable to read archive0 metadata', self.filename)):
whisper.info(self.filename)
def test_raw_data(self):
url = '/render'
whisper.create(self.db, [(1, 60)])
response = self.app.get(url, query_string={'rawData': '1',
'target': 'test'})
info, data = response.data.decode('utf-8').strip().split('|', 1)
path, start, stop, step = info.split(',')
datapoints = data.split(',')
self.assertEqual(datapoints, ['None'] * 60)
self.assertEqual(int(stop) - int(start), 60)
self.assertEqual(path, 'test')
self.assertEqual(int(step), 1)
def update(self, metric, dataline):
data = dataline.split()
metric = os.path.join(self.root, metric)
metric += '.wsp'
if not os.path.exists(metric):
# create this database
mkdirs(os.path.dirname(metric))
whisper.create(metric, [(METRIC_SEC_PER_POINT, METRIC_POINTS_TO_STORE)])
value = float(data[0])
timestamp = float(data[1])
whisper.update(metric, value, timestamp)
def test_merge_empty(self):
"""
test merging from an empty database
"""
testdb_a = "test-a-%s" % self.filename
testdb_b = "test-b-%s" % self.filename
# create two empty databases with same retention
self.addCleanup(self._remove, testdb_a)
whisper.create(testdb_a, self.retention)
self.addCleanup(self._remove, testdb_b)
whisper.create(testdb_b, self.retention)
whisper.merge(testdb_a, testdb_b)
def create_whisper_hosts(self, ts=None):
worker1 = self.hostcpu.replace('hostname', 'worker1')
worker2 = self.hostcpu.replace('hostname', 'worker2')
try:
os.makedirs(worker1.replace('cpu.wsp', ''))
os.makedirs(worker2.replace('cpu.wsp', ''))
except OSError:
pass
whisper.create(worker1, [(1, 60)])
whisper.create(worker2, [(1, 60)])
ts = ts or int(time.time())
whisper.update(worker1, 1, ts)
whisper.update(worker2, 2, ts)
def test_diff(self):
testdb = "test-%s" % self.filename
now = time.time()
whisper.create(testdb, self.retention)
whisper.create(self.filename, self.retention)
whisper.update(testdb, 1.0, now)
whisper.update(self.filename, 2.0, now)
results = whisper.diff(testdb, self.filename)
self._remove(testdb)
expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]
self.assertEqual(results, expected)
class WhisperDatabase(TimeSeriesDatabase):
plugin_name = 'whisper'
def __init__(self, settings):
self.data_dir = settings.LOCAL_DATA_DIR
self.sparse_create = settings.WHISPER_SPARSE_CREATE
self.fallocate_create = settings.WHISPER_FALLOCATE_CREATE
if settings.WHISPER_AUTOFLUSH:
log.msg("Enabling Whisper autoflush")
whisper.AUTOFLUSH = True
if settings.WHISPER_FALLOCATE_CREATE:
if whisper.CAN_FALLOCATE:
log.msg("Enabling Whisper fallocate support")
else:
log.err(
"WHISPER_FALLOCATE_CREATE is enabled but linking failed."
)
if settings.WHISPER_LOCK_WRITES:
if whisper.CAN_LOCK:
log.msg("Enabling Whisper file locking")
whisper.LOCK = True
else:
log.err(
"WHISPER_LOCK_WRITES is enabled but import of fcntl module failed."
)
if settings.WHISPER_FADVISE_RANDOM:
try:
if whisper.CAN_FADVISE:
log.msg("Enabling Whisper fadvise_random support")
whisper.FADVISE_RANDOM = True
else:
log.err(
"WHISPER_FADVISE_RANDOM is enabled but import of ftools module failed."
)
except AttributeError:
log.err(
"WHISPER_FADVISE_RANDOM is enabled but skipped because it is not compatible with the version of Whisper."
)
def write(self, metric, datapoints):
path = self.getFilesystemPath(metric)
whisper.update_many(path, datapoints)
def exists(self, metric):
return exists(self.getFilesystemPath(metric))
def create(self, metric, retentions, xfilesfactor, aggregation_method):
path = self.getFilesystemPath(metric)
directory = dirname(path)
try:
if not exists(directory):
os.makedirs(directory)
except OSError, e:
log.err("%s" % e)
whisper.create(path, retentions, xfilesfactor, aggregation_method,
self.sparse_create, self.fallocate_create)
def handle(self):
points = 0
for metric in self.redis.smembers(METRICS):
values = self.redis.zrange(metric, 0, -1)
points += len(values)
f = target_to_path(self.path, metric)
d = os.path.dirname(f)
if d not in self.dirs:
if not os.path.isdir(d):
os.makedirs(d)
self.dirs.add(d)
if not os.path.exists(f):
whisper.create(f, [(10, 1000)]) # [FIXME] hardcoded values
whisper.update_many(f, [struct.unpack('!ff', a) for a in values])
if len(values):
self.redis.zrem(metric, *values)
self.metric(METRIC_POINTS, points)
def test_fetch_with_archive_to_select(self):
"""
fetch info from database providing the archive to select
"""
# SECOND MINUTE HOUR DAY
retention = [(1, 60), (60, 60), (3600, 24), (86400, 365)]
whisper.create(self.filename, retention)
archives = ["1s", "1m", "1h", "1d"]
for i in range(len(archives)):
fetch = whisper.fetch(self.filename, 0, archiveToSelect=archives[i])
self.assertEqual(fetch[0][2], retention[i][0])
# check time range
self.assertEqual(fetch[0][1] - fetch[0][0], retention[-1][0] * retention[-1][1])
with AssertRaisesException(ValueError("Invalid granularity: 2")):
fetch = whisper.fetch(self.filename, 0, archiveToSelect="2s")
