def test_invalid_retentions(self):
retention_map = (
# From getUnitString
('10x:10', ValueError("Invalid unit 'x'")),
('60:10x', ValueError("Invalid unit 'x'")),
# From parseRetentionDef
('10', ValueError("Invalid retention definition '10'")),
('10X:10', ValueError("Invalid precision specification '10X'")),
('10:10$', ValueError("Invalid retention specification '10$'")),
('60:10', (60, 10)),
)
for retention, expected_exc in retention_map:
try:
results = whisper.parseRetentionDef(retention)
except expected_exc.__class__ as exc:
self.assertEqual(
str(expected_exc),
str(exc),
)
self.assertEqual(
expected_exc.__class__,
exc.__class__,
)
else:
# When there isn't an exception raised
self.assertEqual(results, expected_exc)
def _create(self):
"""Create the Whisper file on disk"""
if not os.path.exists(settings.SALMON_WHISPER_DB_PATH):
os.makedirs(settings.SALMON_WHISPER_DB_PATH)
archives = [whisper.parseRetentionDef(retentionDef)
for retentionDef in settings.ARCHIVES.split(",")]
whisper.create(self.path, archives,
xFilesFactor=settings.XFILEFACTOR,
aggregationMethod=settings.AGGREGATION_METHOD)
def test_valid_retentions(self):
retention_map = (
('60:10', (60, 10)),
('10:60', (10, 60)),
('10s:10h', (10, 3600)),
)
for retention, expected in retention_map:
results = whisper.parseRetentionDef(retention)
self.assertEqual(results, expected)
def test_valid_retentions(self):
retention_map = (
('60:10', (60, 10)),
('10:60', (10, 60)),
('10s:10h', (10, 3600)),
)
for retention, expected in retention_map:
results = whisper.parseRetentionDef(retention)
self.assertEqual(results, expected)
def _create(self):
"""Create the Whisper file on disk"""
if not os.path.exists(settings.SALMON_WHISPER_DB_PATH):
os.makedirs(settings.SALMON_WHISPER_DB_PATH)
archives = [
whisper.parseRetentionDef(retentionDef)
for retentionDef in settings.ARCHIVES.split(",")
]
whisper.create(self.path,
archives,
xFilesFactor=settings.XFILEFACTOR,
aggregationMethod=settings.AGGREGATION_METHOD)
def createDs(uuid):
archives = [whisper.parseRetentionDef(retentionDef) for retentionDef in WHISPER_ARCHIVES]
dataFile = os.path.join(WHISPER_DATA,str(uuid) + ".wsp")
try:
os.makedirs(WHISPER_DATA)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
try:
whisper.create(dataFile, archives, xFilesFactor=0.5, aggregationMethod="average")
except whisper.WhisperException, exc:
raise SystemExit('[ERROR] %s' % str(exc))
def create(path, retentions, overwrite=False, xFilesFactor=0.5, aggregationMethod='average'):
archives = [whisper.parseRetentionDef(str(timePerPoint)+":"+str(timeToStore))
for (timePerPoint, timeToStore) in retentions]
if len(retentions) == 0:
raise Exception("No Retention given")
if overwrite and os.path.exists(path):
print 'Overwriting existing file: %s' % path
os.unlink(path)
p = path.split("/")[:-1]
p = join(p, os.path.sep)
if not os.path.exists(p):
os.makedirs(p)
whisper.create(path + '.wsp', archives, xFilesFactor, aggregationMethod)
return True
config_parser = ConfigParser()
if not config_parser.read(SCHEMAS_FILE):
raise SystemExit("Error: Couldn't read config file: %s" % SCHEMAS_FILE)
errors_found = 0
for section in config_parser.sections():
print("Section '%s':" % section)
options = dict(config_parser.items(section))
retentions = options['retentions'].split(',')
archives = []
section_failed = False
for retention in retentions:
try:
archives.append(whisper.parseRetentionDef(retention))
except ValueError as e:
print(
" - Error: Section '%s' contains an invalid item in its retention definition ('%s')" %
(section, retention)
)
print(" %s" % e)
section_failed = True
if not section_failed:
try:
whisper.validateArchiveList(archives)
except whisper.InvalidConfiguration as e:
print(
" - Error: Section '%s' contains an invalid retention definition ('%s')" %
(section, ','.join(retentions))
help="Change the aggregation function (%s)" % ", ".join(whisper.aggregationMethods),
)
option_parser.add_option("--force", default=False, action="store_true", help="Perform a destructive change")
option_parser.add_option(
"--newfile", default=None, action="store", help="Create a new database file without removing the existing one"
)
option_parser.add_option("--nobackup", action="store_true", help="Delete the .bak file after successful execution")
(options, args) = option_parser.parse_args()
if len(args) < 2:
option_parser.print_usage()
sys.exit(1)
path = args[0]
new_archives = [whisper.parseRetentionDef(retentionDef) for retentionDef in args[1:]]
info = whisper.info(path)
old_archives = info["archives"]
# sort by precision, lowest to highest
old_archives.sort(key=lambda a: a["secondsPerPoint"], reverse=True)
if options.xFilesFactor is None:
xff = info["xFilesFactor"]
else:
xff = options.xFilesFactor
if options.aggregationMethod is None:
aggregationMethod = info["aggregationMethod"]
else:
aggregationMethod = options.aggregationMethod
def fromString(retentionDef): (secondsPerPoint, points) = whisper.parseRetentionDef(retentionDef) return Archive(secondsPerPoint, points)
1h:7d 1 hour per datapoint, 7 days of retention
12h:2y 12 hours per datapoint, 2 years of retention
''')
option_parser.add_option('--xFilesFactor', default=0.5, type='float')
option_parser.add_option('--aggregationMethod', default='average',
type='string', help="Function to use when aggregating values (%s)" %
', '.join(whisper.aggregationMethods))
option_parser.add_option('--overwrite', default=False, action='store_true')
(options, args) = option_parser.parse_args()
if len(args) < 2:
option_parser.print_usage()
sys.exit(1)
path = args[0]
archives = [whisper.parseRetentionDef(retentionDef)
for retentionDef in args[1:]]
if os.path.exists(path) and options.overwrite:
print 'Overwriting existing file: %s' % path
os.unlink(path)
try:
whisper.create(path, archives, xFilesFactor=options.xFilesFactor, aggregationMethod=options.aggregationMethod)
except whisper.WhisperException, exc:
raise SystemExit('[ERROR] %s' % str(exc))
size = os.stat(path).st_size
print 'Created: %s (%d bytes)' % (path,size)
from datetime import timedelta
import glob
import os
import re
import time
import whisper
HOUR = 3600
DAY = timedelta(days=1).total_seconds()
WEEK = timedelta(days=7).total_seconds()
MONTH = timedelta(days=30).total_seconds() # close enough
YEAR = timedelta(days=365).total_seconds()
RETENTION = [ whisper.parseRetentionDef(x)
for x in ["60:70", "10m:150", "1h:8d", "4h:31d", "1d:366d"] ]
def get_period(label):
label = label.lower()
if 'hour' in label:
return HOUR
elif 'week' in label:
return WEEK
elif 'month' in label:
return MONTH
elif 'year' in label:
return YEAR
return DAY #default
class Datastore(object):
if len(args) < 2:
option_parser.print_usage()
sys.exit(1)
path = args[0]
if not os.path.exists(path):
sys.stderr.write("[ERROR] File '%s' does not exist!\n\n" % path)
option_parser.print_usage()
sys.exit(1)
info = whisper.info(path)
new_archives = [
whisper.parseRetentionDef(retentionDef) for retentionDef in args[1:]
]
old_archives = info['archives']
# sort by precision, lowest to highest
old_archives.sort(key=lambda a: a['secondsPerPoint'], reverse=True)
if options.xFilesFactor is None:
xff = info['xFilesFactor']
else:
xff = options.xFilesFactor
if options.aggregationMethod is None:
aggregationMethod = info['aggregationMethod']
else:
aggregationMethod = options.aggregationMethod
def start(self):
# build archives based on retention setting
# NOTE: if retention is changed in config on an existing database
# it will have to be rebuilt.
self.archives = [whisper.parseRetentionDef(x) for x in self.retention]
# -*- coding: utf-8 -*-
"""メッセージ数などを記録する時間別DBを管理する."""
import logging
import os
import threading
from collections import defaultdict
from typing import Dict
import whisper
logger = logging.getLogger(__name__)
WHISPER_ARCHIVES_STRING = '1s:24h 10s:7d 10m:150d 1h:3y'.split(' ')
WHISPER_ARCHIVES = [
whisper.parseRetentionDef(d) for d in WHISPER_ARCHIVES_STRING
]
WHISPER_GLOBAL_LOCK = threading.Lock()
WHISPER_GLOBAL_LOCKS: Dict[str, threading.Lock] = {}
def get_lock(filepath):
with WHISPER_GLOBAL_LOCK:
if filepath not in WHISPER_GLOBAL_LOCKS:
WHISPER_GLOBAL_LOCKS[filepath] = threading.Lock()
return WHISPER_GLOBAL_LOCKS[filepath]
class TimedDBBundle(object):
"""
TimedDBをとりまとめるクラス
def start(self):
# build archives based on retention setting
# NOTE: if retention is changed in config on an existing database
# it will have to be rebuilt.
self.archives = [whisper.parseRetentionDef(x) for x in self.retention]
def fromString(retentionDef):
(secondsPerPoint, points) = whisper.parseRetentionDef(retentionDef)
return Archive(secondsPerPoint, points)
from datetime import timedelta
import glob
import os
import re
import time
import whisper
HOUR = 3600
DAY = timedelta(days=1).total_seconds()
WEEK = timedelta(days=7).total_seconds()
MONTH = timedelta(days=30).total_seconds() # close enough
YEAR = timedelta(days=365).total_seconds()
RETENTION = [whisper.parseRetentionDef(x) for x in ["60:70", "10m:150", "1h:8d", "4h:31d", "1d:366d"]]
def get_period(label):
label = label.lower()
if "hour" in label:
return HOUR
elif "week" in label:
return WEEK
elif "month" in label:
return MONTH
elif "year" in label:
return YEAR
return DAY # default