def test_annotationsingest_make_request(self):
global sleep_time
thread = annotationsingest.AnnotationsIngestThread(0)
thread.slice = [[2, 0]]
thread.position = 0
thread.finish_time = 10000
valid_payload = {
"what":
"annotation int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.0",
"when": 1000,
"tags": "tag",
"data": "data"
}
url, payload = thread.make_request(pp)
#confirm request generates proper URL and payload
self.assertEqual(
url,
'http://qe01.metrics-ingest.api.rackspacecloud.com/v2.0/2/events')
self.assertEqual(eval(payload), valid_payload)
#confirm request increments position if not at end of report interval
self.assertEqual(thread.position, 1)
self.assertEqual(thread.finish_time, 10000)
thread.position = 2
thread.make_request(pp)
#confirm request resets position at end of report interval
self.assertEqual(sleep_time, 9000)
self.assertEqual(thread.position, 1)
self.assertEqual(thread.finish_time, 16000)
def test_generate_annotations_payload(self):
self.tm.create_all_metrics(1)
thread = annotationsingest.AnnotationsIngestThread(0)
payload = json.loads(thread.generate_payload(0, 3))
valid_payload = {'what': 'annotation int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3',
'when': 0,
'tags': 'tag',
'data': 'data'}
self.assertEqual(payload, valid_payload)
def test_init_process(self):
#confirm that threadnum 0 is an ingest thread
t1 = self.tm.setup_thread(0)
self.assertEqual(type(t1), ingest.IngestThread)
#confirm that the threadnum after all ingest threads is EnumIngestThread
t1 = self.tm.setup_thread(ingestenum.default_config['enum_ingest_concurrency'])
self.assertEqual(type(t1), ingestenum.EnumIngestThread)
#confirm that the threadnum after all ingest threads is a query thread
t1 = self.tm.setup_thread(ingest.default_config['ingest_concurrency']+ingestenum.default_config['enum_ingest_concurrency'])
self.assertEqual(type(t1), query.QueryThread)
#confirm that the threadnum after all ingest+query threads is an annotations query thread
t1 = self.tm.setup_thread(ingest.default_config['ingest_concurrency']+ingestenum.default_config['enum_ingest_concurrency']+ingest.default_config['query_concurrency'])
self.assertEqual(type(t1), annotationsingest.AnnotationsIngestThread)
#confirm that a threadnum after all valid thread types raises an exception
tot_threads = (ingest.default_config['ingest_concurrency'] + ingest.default_config['enum_ingest_concurrency'] + ingest.default_config['query_concurrency'] + ingest.default_config['annotations_concurrency'])
self.assertRaises(Exception,self.tm.setup_thread, tot_threads)
#confirm that the correct batches of ingest metrics are created for worker 0
self.tm.create_all_metrics(0)
# confirm annotationsingest
self.assertEqual(annotationsingest.AnnotationsIngestThread.annotations,
[[0, 0], [0, 1], [1, 0], [1, 1]])
thread = annotationsingest.AnnotationsIngestThread(0)
self.assertEqual(thread.slice, [[0, 0], [0, 1]])
thread = annotationsingest.AnnotationsIngestThread(1)
self.assertEqual(thread.slice, [[1, 0], [1, 1]])
#confirm enum metrics ingest
self.assertEqual(ingestenum.EnumIngestThread.metrics,
[
[[0, 0], [0, 1], [1, 0]],
[[1, 1]]
])
thread = ingestenum.EnumIngestThread(0)
self.assertEqual(thread.slice, [[[0, 0], [0, 1], [1, 0]]])
thread = ingestenum.EnumIngestThread(1)
self.assertEqual(thread.slice, [[[1, 1]]])
# confirm metrics ingest
self.assertEqual(ingest.IngestThread.metrics,
[[[0, 0], [0, 1], [0, 2]],
[[0, 3], [0, 4], [0, 5]],
[[0, 6], [1, 0], [1, 1]],
[[1, 2], [1, 3], [1, 4]],
[[1, 5], [1, 6]]])
#confirm that the correct batch slices are created for individual threads
thread = ingest.IngestThread(0)
self.assertEqual(thread.slice,
[[[0, 0], [0, 1], [0, 2]],
[[0, 3], [0, 4], [0, 5]],
[[0, 6], [1, 0], [1, 1]]])
thread = ingest.IngestThread(1)
self.assertEqual(thread.slice,
[[[1, 2], [1, 3], [1, 4]],
[[1, 5], [1, 6]]])
# confirm that the number of queries is correctly distributed across
# each thread in this worker process
num_query_nodes = query.default_config['num_nodes']
single_plot_queries_agent0 = int(math.ceil(query.default_config['singleplot_per_interval']/num_query_nodes))
multi_plot_queries_agent0 = int(math.ceil(query.default_config['multiplot_per_interval']/num_query_nodes))
search_queries_agent0 = int(math.ceil(query.default_config['search_queries_per_interval']/num_query_nodes))
enum_search_queries_agent0 = int(math.ceil(query.default_config['enum_search_queries_per_interval']/num_query_nodes))
enum_single_plot_queries_agent0 = int(math.ceil(query.default_config['enum_single_plot_queries_per_interval']/num_query_nodes))
enum_multi_plot_queries_agent0 = int(math.ceil(query.default_config['enum_multiplot_per_interval']/num_query_nodes))
annotation_queries_agent0 = int(math.ceil(query.default_config['annotations_queries_per_interval']/num_query_nodes))
self.assertEqual(query.QueryThread.queries,
([query.SinglePlotQuery] * single_plot_queries_agent0 + [query.MultiPlotQuery] * multi_plot_queries_agent0 + [query.SearchQuery] * search_queries_agent0 + [query.EnumSearchQuery] * enum_search_queries_agent0 + [query.EnumSinglePlotQuery] * enum_single_plot_queries_agent0 + [query.AnnotationsQuery] * annotation_queries_agent0) + [query.EnumMultiPlotQuery] * enum_multi_plot_queries_agent0)
thread = query.QueryThread(0)
self.assertEqual(thread.slice, [query.SinglePlotQuery] * 2)
thread = query.QueryThread(3)
self.assertEqual(thread.slice, [query.MultiPlotQuery] * 2)
thread = query.QueryThread(6)
self.assertEqual(thread.slice, [query.SearchQuery] * 2)
thread = query.QueryThread(9)
self.assertEqual(thread.slice, [query.EnumSearchQuery] * 2)
thread = query.QueryThread(12)
self.assertEqual(thread.slice, [query.EnumSinglePlotQuery] * 2)
thread = query.QueryThread(14)
self.assertEqual(thread.slice, [query.AnnotationsQuery] * 2)
thread = query.QueryThread(16)
self.assertEqual(thread.slice, [query.EnumMultiPlotQuery] * 1)
#confirm that the correct batches of ingest metrics are created for worker 1
self.tm.create_all_metrics(1)
self.assertEqual(ingest.IngestThread.metrics,
[[[2, 0], [2, 1], [2, 2]],
[[2, 3], [2, 4], [2, 5]],
[[2, 6]]])
self.assertEqual(annotationsingest.AnnotationsIngestThread.annotations,
[[2, 0], [2, 1]])
thread = ingest.IngestThread(0)
self.assertEqual(thread.slice,
[[[2, 0], [2, 1], [2, 2]],
[[2, 3], [2, 4], [2, 5]]])
thread = ingest.IngestThread(1)
self.assertEqual(thread.slice,
[[[2, 6]]])
#confirm that the correct batches of queries are created for worker 1
single_plot_queries_agent1 = query.default_config['singleplot_per_interval'] - single_plot_queries_agent0
multi_plot_queries_agent1 = query.default_config['multiplot_per_interval'] - multi_plot_queries_agent0
search_queries_agent1 = query.default_config['search_queries_per_interval'] - search_queries_agent0
enum_search_queries_agent1 = query.default_config['enum_search_queries_per_interval'] - enum_search_queries_agent0
enum_single_plot_queries_agent1 = query.default_config['enum_single_plot_queries_per_interval'] - enum_single_plot_queries_agent0
annotation_queries_agent1 = query.default_config['annotations_queries_per_interval'] - annotation_queries_agent0
enum_multi_plot_queries_agent1 = query.default_config['enum_multiplot_per_interval'] - enum_multi_plot_queries_agent0
self.assertEqual(query.QueryThread.queries,
([query.SinglePlotQuery] * single_plot_queries_agent1 + [query.MultiPlotQuery] * multi_plot_queries_agent1 + [query.SearchQuery] * search_queries_agent1 + [query.EnumSearchQuery] * enum_search_queries_agent1 + [query.EnumSinglePlotQuery] * enum_single_plot_queries_agent1 + [query.AnnotationsQuery] * annotation_queries_agent1) + [query.EnumMultiPlotQuery] * enum_multi_plot_queries_agent1)
thread = query.QueryThread(0)
self.assertEqual(thread.slice, [query.SinglePlotQuery] * 2)
thread = query.QueryThread(4)
self.assertEqual(thread.slice, [query.MultiPlotQuery] * 2)
thread = query.QueryThread(6)
self.assertEqual(thread.slice, [query.SearchQuery] * 2)
thread = query.QueryThread(8)
self.assertEqual(thread.slice, [query.EnumSearchQuery] * 2)
thread = query.QueryThread(10)
self.assertEqual(thread.slice, [query.EnumSinglePlotQuery] * 2)
thread = query.QueryThread(12)
self.assertEqual(thread.slice, [query.AnnotationsQuery] * 1)
thread = query.QueryThread(16)
self.assertEqual(thread.slice, [query.EnumMultiPlotQuery] * 1)