def test_queryprocessor_2(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE)
self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query'])
qp.query('uuid-test-1', "test query")
self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)
qp.query('uuid-test-2', "bad")
self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)
gevent.sleep(0)
traced_mock.MOCK_QUERIES[0].finish_event.set()
traced_mock.MOCK_QUERIES[1].finish_event.set()
gevent.sleep(0)
qp.stop()
gevent.sleep(0)
gevent.sleep(0)
self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)
self.assertRaises(RuntimeError, traced_mock.MOCK_QUERIES[1].get)
self.assertNotIn('uuid-test-1', store.release_alls)
self.assertIn('uuid-test-2', store.release_alls)
def test_query_pos(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(1*86400*1000, 'log0')
q = minemeld.traced.queryprocessor.Query(
store,
"log",
3*86400*1000, 0,
100,
'uuid-test',
{}
)
q._run()
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 1)
self.assertEqual(eoq, True)
def test_queryprocessor_4(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(
comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE
)
self.assertEqual(
comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query']
)
qp.stop()
gevent.sleep(0)
self.assertRaises(
RuntimeError,
qp.query,
'test-uuid-1', "test"
)
self.assertRaises(
RuntimeError,
qp.kill_query,
'test-uuid-1'
)
def test_queryprocessor_5(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store, {'max_concurrency': 2})
qp.query('uuid-test-1', "test query")
gevent.sleep(0)
qp.query('uuid-test-2', "test query")
gevent.sleep(0)
self.assertRaises(
RuntimeError,
qp.query,
'uuid-test-3', "test query"
)
gevent.sleep(0)
traced_mock.MOCK_QUERIES[0].finish_event.set()
gevent.sleep(0.2)
self.assertEqual(len(qp.queries), 1)
self.assertEqual(
qp.query('uuid-test-4', "test query"),
'OK'
)
gevent.sleep(0)
qp.stop()
gevent.sleep(0)
def test_queryprocessor_3(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(
comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE
)
self.assertEqual(
comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query']
)
qp.query('uuid-test-1', "test query")
gevent.sleep(0)
qp.kill_query('uuid-test-1')
gevent.sleep(0)
self.assertIsInstance(
traced_mock.MOCK_QUERIES[0].get(),
greenlet.GreenletExit
)
self.assertEqual(
len(qp.queries),
0
)
self.assertIn('uuid-test-1', store.release_alls)
qp.stop()
gevent.sleep(0)
def test_queryprocessor_5(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(
comm, store, {'max_concurrency': 2})
qp.query('uuid-test-1', "test query")
gevent.sleep(0)
qp.query('uuid-test-2', "test query")
gevent.sleep(0)
self.assertRaises(RuntimeError, qp.query, 'uuid-test-3', "test query")
gevent.sleep(0)
traced_mock.MOCK_QUERIES[0].finish_event.set()
gevent.sleep(0.2)
self.assertEqual(len(qp.queries), 1)
self.assertEqual(qp.query('uuid-test-4', "test query"), 'OK')
gevent.sleep(0)
qp.stop()
gevent.sleep(0)
def test_query_empty(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
q = minemeld.traced.queryprocessor.Query(
store,
"field3:foo -field4:679",
3*86400*1000, 0,
100,
'uuid-test',
{}
)
q._run()
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 0)
self.assertEqual(eoq, True)
def test_query_empty(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
q = minemeld.traced.queryprocessor.Query(store,
"field3:foo -field4:679",
3 * 86400 * 1000, 0, 100,
'uuid-test', {})
q._run()
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 0)
self.assertEqual(eoq, True)
def test_query_pos(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(1 * 86400 * 1000, 'log0')
q = minemeld.traced.queryprocessor.Query(store, "log",
3 * 86400 * 1000, 0, 100,
'uuid-test', {})
q._run()
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 1)
self.assertEqual(eoq, True)
def test_queryprocessor_4(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE)
self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query'])
qp.stop()
gevent.sleep(0)
self.assertRaises(RuntimeError, qp.query, 'test-uuid-1', "test")
self.assertRaises(RuntimeError, qp.kill_query, 'test-uuid-1')
def test_writer(self):
config = {}
comm = comm_mock.comm_factory(config)
store = traced_mock.store_factory()
writer = minemeld.traced.writer.Writer(comm, store, 'TESTTOPIC')
self.assertEqual(comm.sub_channels[0]['topic'], 'TESTTOPIC')
self.assertEqual(comm.sub_channels[0]['allowed_methods'], ['log'])
writer.log(0, log='testlog')
self.assertEqual(store.writes[0]['timestamp'], 0)
self.assertEqual(store.writes[0]['log'], ujson.dumps({'log': 'testlog'}))
writer.stop()
writer.log(0, log='testlog')
self.assertEqual(len(store.writes), 1)
writer.stop() # just for coverage
def test_query_fs4(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(1*86400*1000, ujson.dumps({
"field1": "foo",
"field2": "bar",
"field3": ["foo", "bar"],
"field4": 12345678
}))
store.write(1*86400*1000, ujson.dumps({
"field1": "bar",
"field2": "foo",
"field3": ["foo", "bar"],
"field4": 12345679
}))
q = minemeld.traced.queryprocessor.Query(
store,
"field3:foo FIELD3:Bar",
3*86400*1000, 0,
100,
'uuid-test',
{}
)
q._run()
LOG.debug(SR_mock.mock_calls)
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 2)
self.assertEqual(eoq, True)
def test_query_wcard(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(1*86400*1000, ujson.dumps({
"field1": "1*oo1",
"field2": "2*ar2",
"field3": ["5foo5", "6bar6"],
"field4": 12345678
}))
store.write(1*86400*1000, ujson.dumps({
"field1": "3bar3",
"field2": "4foo4",
"field3": ["8*oo8", "7bar7"],
"field4": 12345678
}))
q = minemeld.traced.queryprocessor.Query(
store,
"field3:*oo -field4:67?",
3*86400*1000, 0,
100,
'uuid-test',
{}
)
q._run()
LOG.debug(SR_mock.mock_calls)
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 1)
self.assertEqual(eoq, True)
def test_query_fs4(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(
1 * 86400 * 1000,
ujson.dumps({
"field1": "foo",
"field2": "bar",
"field3": ["foo", "bar"],
"field4": 12345678
}))
store.write(
1 * 86400 * 1000,
ujson.dumps({
"field1": "bar",
"field2": "foo",
"field3": ["foo", "bar"],
"field4": 12345679
}))
q = minemeld.traced.queryprocessor.Query(store,
"field3:foo FIELD3:Bar",
3 * 86400 * 1000, 0, 100,
'uuid-test', {})
q._run()
LOG.debug(SR_mock.mock_calls)
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 2)
self.assertEqual(eoq, True)
def test_query_wcard(self, glet_mock, SR_mock):
store = traced_mock.store_factory()
store.write(
1 * 86400 * 1000,
ujson.dumps({
"field1": "1*oo1",
"field2": "2*ar2",
"field3": ["5foo5", "6bar6"],
"field4": 12345678
}))
store.write(
1 * 86400 * 1000,
ujson.dumps({
"field1": "3bar3",
"field2": "4foo4",
"field3": ["8*oo8", "7bar7"],
"field4": 12345678
}))
q = minemeld.traced.queryprocessor.Query(store,
"field3:*oo -field4:67?",
3 * 86400 * 1000, 0, 100,
'uuid-test', {})
q._run()
LOG.debug(SR_mock.mock_calls)
self.assertGreater(len(SR_mock.mock_calls), 1)
num_logs = 0
eoq = False
for call in SR_mock.mock_calls[1:]:
name, args, kwargs = call
self.assertEqual(name, '().publish')
self.assertEqual(args[0], 'mm-traced-q.uuid-test')
if args[1] == '<EOQ>':
eoq = True
else:
line = json.loads(args[1])
if 'log' in line:
num_logs += 1
self.assertEqual(num_logs, 1)
self.assertEqual(eoq, True)
def test_queryprocessor_2(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(
comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE
)
self.assertEqual(
comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query']
)
qp.query('uuid-test-1', "test query")
self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)
qp.query('uuid-test-2', "bad")
self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)
gevent.sleep(0)
traced_mock.MOCK_QUERIES[0].finish_event.set()
traced_mock.MOCK_QUERIES[1].finish_event.set()
gevent.sleep(0)
qp.stop()
gevent.sleep(0)
gevent.sleep(0)
self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)
self.assertRaises(
RuntimeError,
traced_mock.MOCK_QUERIES[1].get
)
self.assertNotIn('uuid-test-1', store.release_alls)
self.assertIn('uuid-test-2', store.release_alls)
def test_queryprocessor_3(self, query_mock):
comm = comm_mock.comm_factory({})
store = traced_mock.store_factory()
qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)
self.assertEqual(comm.rpc_server_channels[0]['name'],
minemeld.traced.queryprocessor.QUERY_QUEUE)
self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
['query', 'kill_query'])
qp.query('uuid-test-1', "test query")
gevent.sleep(0)
qp.kill_query('uuid-test-1')
gevent.sleep(0)
self.assertIsInstance(traced_mock.MOCK_QUERIES[0].get(),
greenlet.GreenletExit)
self.assertEqual(len(qp.queries), 0)
self.assertIn('uuid-test-1', store.release_alls)
qp.stop()
gevent.sleep(0)
