class MemoryStorageTestCase(AsyncBaseTestCase, UtilsMixin):
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update()
self.app = HTTPApplication(config)
self.app.configure_app(tornado.ioloop.IOLoop.instance())
return self.app
@async_test
@tornado.gen.engine
def test_methods(self):
storage = MemoryStorage(None)
self.assertTrue(isinstance(storage, MemoryStorage))
self.assertTrue(isinstance(storage._store, MagicDict))
self.assertTrue(isinstance(storage._metrics, dict))
self.methods_tests(storage)
self.metric_meta_tests(storage)
self.stop()
@async_test
@tornado.gen.engine
def test_storage(self):
app = self.get_app()
app.configure_storage("gottwall.storages.MemoryStorage")
storage = app.storage
self.storage_tests(storage)
self.stop()
def run(self, port, reload, host, logging, **kwargs):
config = self._commandor_res
if not config:
self.error("You need specify --config\n")
self.exit()
configure_logging(logging)
application = HTTPApplication(config)
ioloop = tornado.ioloop.IOLoop.instance()
application.configure_app(ioloop)
self.http_server = httpserver.HTTPServer(application)
self.http_server.listen(port, host)
if reload:
self.display("Autoreload enabled")
autoreload.start(io_loop=ioloop, check_time=100)
self.display("Server running on http://{0}:{1}".format(host, port))
# Init signals handler
signal.signal(signal.SIGTERM, self.sig_handler)
# This will also catch KeyboardInterrupt exception
signal.signal(signal.SIGINT, self.sig_handler)
ioloop.start()
def run(self, port, reload, host, logging, **kwargs):
config = self._commandor_res
if not config:
self.error("You need specify --config\n")
self.exit()
configure_logging(logging)
application = HTTPApplication(config)
ioloop = tornado.ioloop.IOLoop.instance()
application.configure_app(ioloop)
self.http_server = httpserver.HTTPServer(application)
self.http_server.listen(port, host)
if reload:
self.display("Autoreload enabled")
autoreload.start(io_loop=ioloop, check_time=100)
self.display("Server running on http://{0}:{1}".format(host, port))
# Init signals handler
signal.signal(signal.SIGTERM, self.sig_handler)
# This will also catch KeyboardInterrupt exception
signal.signal(signal.SIGINT, self.sig_handler)
ioloop.start()
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update()
self.app = HTTPApplication(config)
self.app.configure_app(tornado.ioloop.IOLoop.instance())
return self.app
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update({"BACKENDS": []})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update({"BACKENDS": []})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
def test_base_storage(self):
config = Config()
config.from_object(gottwall.default_config)
app = HTTPApplication(config)
app.configure_storage("gottwall.storages.BaseStorage")
self.assertTrue(isinstance(app.storage, BaseStorage))
params = {
"project": "project_name",
"name": "orders",
"timestamp": datetime.datetime.now(),
"filters": {
"clearing": True,
"device": "web"
}
}
self.assertRaises(NotImplementedError, app.storage.incr, **params)
self.assertRaises(NotImplementedError, app.storage.decr, **params)
self.assertRaises(NotImplementedError, app.storage.metrics,
"project_name")
res = app.storage.make_embedded("project", "month",
self.get_embedded_metrics())
self.assertTrue(isinstance(res[0], UUID))
self.assertEquals(res[1]['project'], "project")
self.assertEquals(res[1]['period'], "month")
self.assertEquals(len(res[1]['metrics']),
len(self.get_embedded_metrics()))
self.assertRaises(NotImplementedError, app.storage.query,
"test_project", "test_metric", "month")
self.assertRaises(NotImplementedError, app.storage.metrics,
"test_project")
self.assertRaises(NotImplementedError, app.storage.get_filters,
"test_project", "test_metric")
self.assertRaises(NotImplementedError, app.storage.get_filter_values,
"test_project", "test_metric", "filter1")
def get_app(self):
config = Config()
config.from_object(gottwall.default_config)
config.update({"STORAGE_SETTINGS": {
"HOST": self.redis_settings['HOST']
}})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
class RedisStorageTestCase(AsyncBaseTestCase, RedisTestCaseMixin, UtilsMixin):
def setUp(self):
super(RedisStorageTestCase, self).setUp()
self.client = self._new_client()
self.client.flushdb()
def tearDown(self):
try:
self.client.flushdb()
self.client.connection.disconnect()
del self.client
except AttributeError:
pass
super(RedisStorageTestCase, self).tearDown()
def get_app(self):
config = Config()
config.from_object(gottwall.default_config)
config.update({"STORAGE_SETTINGS": {
"HOST": self.redis_settings['HOST']
}})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
def get_new_ioloop(self):
return tornado.ioloop.IOLoop.instance()
@async_test
@tornado.gen.engine
def test_storage_utils(self):
storage = RedisStorage(self.get_app())
self.assertEquals(storage.make_key("redis_project_name", "metric_name", "week",
filters={"status": "new",
"type": "registered"}),
"redis_project_name;metric_name;week;status|new/type|registered")
self.assertTrue(isinstance(storage, RedisStorage))
filters = {"filter1": "value",
"filter2": ["value1", "value2", "value2"]}
client = self.client
pipe = client.pipeline()
storage.save_metric_meta(pipe, "redis_storage_test", "metric_name",
filters=filters)
(yield Task(pipe.execute))
metrics = yield Task(
client.smembers,
storage.get_metrics_key("redis_storage_test"))
self.assertEquals(len(metrics), 1)
self.assertTrue("metric_name" in metrics)
stored_filters = yield Task(
client.smembers,
storage.get_filters_names_key("redis_storage_test", "metric_name"))
self.assertEquals(len(stored_filters), 2)
for f, values in filters.items():
stored_value = yield Task(
client.smembers,
storage.get_filters_values_key("redis_storage_test", "metric_name", f))
self.assertEquals(set(values)
if isinstance(values, (list, tuple))
else set([values]),
stored_value)
metrics = yield Task(storage.metrics, "redis_storage_test")
self.assertEquals(metrics,
{"metric_name": {"filter1": [filters['filter1']],
"filter2": ["value1", "value2"]}})
self.stop()
@async_test
@tornado.gen.engine
def test_metric_meta(self):
"""Get metrics structure
"""
client = tornadoredis.Client(host=self.redis_settings['HOST'])
app = self.get_app()
app.configure_storage("gw_storage_redis.storage.RedisStorage")
storage = app.storage
pipe = client.pipeline()
(yield Task(storage.save_metric_meta, pipe, "test_metric_meta_project", "metric_name",
filters={"hello": "world",
"test": ["value1", "value2"]}))
self.stop()
@async_test
@tornado.gen.engine
def test_redis_storage(self):
app = self.get_app()
app.configure_storage("gw_storage_redis.storage.RedisStorage")
storage = app.storage
self.storage_tests(storage)
self.stop()
class APITestCase(AsyncHTTPBaseTestCase):
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update({"BACKENDS": []})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
def test_stats(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
project_name = "test_project"
metric_name = "test_stats_metric_name"
now = datetime.datetime.now()
from_date = now - relativedelta(years=3)
to_date = now + relativedelta(years=1)
data = self.build_data(from_date,
to_date,
project_name,
metric_name,
filters={
"filter1": True,
"filter2": ["web", "iphone", "android"]
})
for x in data:
storage.incr(project=x['project'],
name=x['name'],
timestamp=x['timestamp'],
filters={x['filter_name']: x['filter_value']},
value=x['value'])
# Get statistics by weeks
for period in PERIODS:
response = self.fetch(
"/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}"
.format(project_name, from_date.strftime("%Y-%m-%d"),
to_date.strftime("%Y-%m-%d"), period),
method="GET")
self.assertEquals(response.code, 400)
self.assertTrue(
'You need specify name and period' in response.body)
response = self.fetch(
"/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}&name={4}"
.format(project_name, to_date.strftime("%Y-%m-%d"),
from_date.strftime("%Y-%m-%d"), period, metric_name),
method="GET")
self.assertEquals(response.code, 400)
self.assertTrue("Invalid date range" in response.body)
for period in ['month', 'year', "day", "week", "hour", "minute"]:
response = self.fetch(
"/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}&name={4}"
.format(project_name, from_date.strftime("%Y-%m-%d"),
to_date.strftime("%Y-%m-%d"), period, metric_name),
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
self.assertTrue('avg' in response_data)
self.assertEquals(response_data['filter_name'], None)
self.assertEquals(response_data['filter_value'], None)
self.assertTrue('max' in response_data)
self.assertTrue('min' in response_data)
self.assertEquals(response_data['name'], metric_name)
self.assertEquals(response_data['period'], period)
self.assertEquals(response_data['project'], project_name)
self.assertTrue(isinstance(response_data['range'], list))
def test_metrics(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
filters = {
"filter1": "value",
"filter2": ["value1", "value2", "value2"]
}
storage.save_metric_meta("test_project",
"metric_name",
filters=filters)
# Get statistics by weeks
response = self.fetch("/gottwall/api/v1/test_project/metrics",
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
self.assertEquals(
response_data, {
"metric_name": {
"filter1": [filters['filter1']],
"filter2": ["value1", "value2"]
}
})
def test_stats_dataset(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
project_name = "test_project"
metric_name = "test_stats_metric_name"
now = datetime.datetime.now()
from_date = now - relativedelta(years=3)
to_date = now + relativedelta(years=1)
data = self.build_data(from_date,
to_date,
project_name,
metric_name,
filters={
"filter1": True,
"filter2": ["web", "iphone", "android"]
})
for x in data:
storage.incr(project=x['project'],
name=x['name'],
timestamp=x['timestamp'],
filters={x['filter_name']: x['filter_value']},
value=x['value'])
response = self.fetch(
"/gottwall/api/v1/{0}/stats_dataset?from_date={1}&to_date={2}&period={3}&name={4}"
.format(project_name, from_date.strftime("%Y-%m-%d"),
to_date.strftime("%Y-%m-%d"), "month", metric_name),
method="GET")
self.assertEquals(response.code, 400)
for period in ['month', 'year', "day", "week"]:
response = self.fetch(
"/gottwall/api/v1/{0}/stats_dataset?from_date={1}&to_date={2}&period={3}&name={4}&filter_name={5}"
.format(project_name, from_date.strftime("%Y-%m-%d"),
to_date.strftime("%Y-%m-%d"), period, metric_name,
"filter2"),
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
for key, value in list(response_data['data'].items()):
test_etalon_range = list(
self.get_range(data, from_date, to_date, period, "filter2",
key))
for k1, k2 in zip(value['range'], test_etalon_range):
self.assertEquals(k1[0], k2[0])
self.assertEquals(k1[1], k2[1])
self.assertEquals(self.get_max(test_etalon_range),
value['max'])
self.assertEquals(self.get_min(test_etalon_range),
value['min'])
self.assertEquals(self.get_avg(test_etalon_range),
value['avg'])
self.assertEquals(response_data['filter_name'], 'filter2')
self.assertEquals(response_data['name'], 'test_stats_metric_name')
self.assertEquals(response_data['period'], period)
self.assertEquals(response_data['project'], 'test_project')
class APITestCase(AsyncHTTPBaseTestCase):
def get_app(self):
config = Config()
config.from_module(gottwall.default_config)
config.update({"BACKENDS": []})
self.app = HTTPApplication(config)
self.app.configure_app(self.io_loop)
return self.app
def test_stats(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
project_name = "test_project"
metric_name = "test_stats_metric_name"
now = datetime.datetime.now()
from_date = now - relativedelta(years=3)
to_date = now + relativedelta(years=1)
data = self.build_data(from_date, to_date,
project_name, metric_name,
filters={"filter1": True,
"filter2": ["web", "iphone", "android"]})
for x in data:
storage.incr(project=x['project'], name=x['name'], timestamp=x['timestamp'],
filters={x['filter_name']: x['filter_value']}, value=x['value'])
# Get statistics by weeks
for period in PERIODS:
response = self.fetch("/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}".format(
project_name, from_date.strftime("%Y-%m-%d"), to_date.strftime("%Y-%m-%d"), period),
method="GET")
self.assertEquals(response.code, 400)
self.assertTrue('You need specify name and period' in response.body)
response = self.fetch("/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}&name={4}".format(
project_name, to_date.strftime("%Y-%m-%d"), from_date.strftime("%Y-%m-%d"), period, metric_name),
method="GET")
self.assertEquals(response.code, 400)
self.assertTrue("Invalid date range" in response.body)
for period in ['month', 'year', "day", "week", "hour", "minute"]:
response = self.fetch("/gottwall/api/v1/{0}/stats?from_date={1}&to_date={2}&period={3}&name={4}".format(
project_name, from_date.strftime("%Y-%m-%d"), to_date.strftime("%Y-%m-%d"), period, metric_name),
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
self.assertTrue('avg' in response_data)
self.assertEquals(response_data['filter_name'], None)
self.assertEquals(response_data['filter_value'], None)
self.assertTrue('max' in response_data)
self.assertTrue('min' in response_data)
self.assertEquals(response_data['name'], metric_name)
self.assertEquals(response_data['period'], period)
self.assertEquals(response_data['project'], project_name)
self.assertTrue(isinstance(response_data['range'], list))
def test_metrics(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
filters = {"filter1": "value",
"filter2": ["value1", "value2", "value2"]}
storage.save_metric_meta("test_project", "metric_name",
filters=filters)
# Get statistics by weeks
response = self.fetch("/gottwall/api/v1/test_project/metrics",
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
self.assertEquals(response_data,
{"metric_name": {"filter1": [filters['filter1']],
"filter2": ["value1", "value2"]}})
def test_stats_dataset(self):
storage = self.app.storage
storage._metrics = {}
storage._store = MagicDict()
project_name = "test_project"
metric_name = "test_stats_metric_name"
now = datetime.datetime.now()
from_date = now - relativedelta(years=3)
to_date = now + relativedelta(years=1)
data = self.build_data(from_date, to_date,
project_name, metric_name,
filters={"filter1": True,
"filter2": ["web", "iphone", "android"]})
for x in data:
storage.incr(project=x['project'], name=x['name'], timestamp=x['timestamp'],
filters={x['filter_name']: x['filter_value']}, value=x['value'])
response = self.fetch("/gottwall/api/v1/{0}/stats_dataset?from_date={1}&to_date={2}&period={3}&name={4}".format(
project_name, from_date.strftime("%Y-%m-%d"), to_date.strftime("%Y-%m-%d"), "month", metric_name),
method="GET")
self.assertEquals(response.code, 400)
for period in ['month', 'year', "day", "week"]:
response = self.fetch("/gottwall/api/v1/{0}/stats_dataset?from_date={1}&to_date={2}&period={3}&name={4}&filter_name={5}".format(
project_name, from_date.strftime("%Y-%m-%d"), to_date.strftime("%Y-%m-%d"), period, metric_name, "filter2"),
method="GET")
self.assertEquals(response.code, 200)
response_data = json.loads(response.body)
for key, value in response_data['data'].items():
test_etalon_range = list(self.get_range(data, from_date, to_date,
period, "filter2", key))
for k1, k2 in zip(value['range'], test_etalon_range):
self.assertEquals(k1[0], k2[0])
self.assertEquals(k1[1], k2[1])
self.assertEquals(self.get_max(test_etalon_range), value['max'])
self.assertEquals(self.get_min(test_etalon_range), value['min'])
self.assertEquals(self.get_avg(test_etalon_range), value['avg'])
self.assertEquals(response_data['filter_name'], 'filter2')
self.assertEquals(response_data['name'], 'test_stats_metric_name')
self.assertEquals(response_data['period'], period)
self.assertEquals(response_data['project'], 'test_project')
