Example #1
0
    def test_queryprocessor_2(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(comm.rpc_server_channels[0]['name'],
                         minemeld.traced.queryprocessor.QUERY_QUEUE)
        self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
                         ['query', 'kill_query'])

        qp.query('uuid-test-1', "test query")
        self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)

        qp.query('uuid-test-2', "bad")
        self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)

        gevent.sleep(0)

        traced_mock.MOCK_QUERIES[0].finish_event.set()
        traced_mock.MOCK_QUERIES[1].finish_event.set()

        gevent.sleep(0)

        qp.stop()
        gevent.sleep(0)
        gevent.sleep(0)

        self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)
        self.assertRaises(RuntimeError, traced_mock.MOCK_QUERIES[1].get)
        self.assertNotIn('uuid-test-1', store.release_alls)
        self.assertIn('uuid-test-2', store.release_alls)
    def test_query_pos(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(1*86400*1000, 'log0')

        q = minemeld.traced.queryprocessor.Query(
            store,
            "log",
            3*86400*1000, 0,
            100,
            'uuid-test',
            {}
        )
        q._run()
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 1)
        self.assertEqual(eoq, True)
    def test_queryprocessor_4(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(
            comm.rpc_server_channels[0]['name'],
            minemeld.traced.queryprocessor.QUERY_QUEUE
        )
        self.assertEqual(
            comm.rpc_server_channels[0]['allowed_methods'],
            ['query', 'kill_query']
        )

        qp.stop()
        gevent.sleep(0)

        self.assertRaises(
            RuntimeError,
            qp.query,
            'test-uuid-1', "test"
        )
        self.assertRaises(
            RuntimeError,
            qp.kill_query,
            'test-uuid-1'
        )
    def test_queryprocessor_5(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store, {'max_concurrency': 2})

        qp.query('uuid-test-1', "test query")
        gevent.sleep(0)

        qp.query('uuid-test-2', "test query")
        gevent.sleep(0)

        self.assertRaises(
            RuntimeError,
            qp.query,
            'uuid-test-3', "test query"
        )
        gevent.sleep(0)

        traced_mock.MOCK_QUERIES[0].finish_event.set()
        gevent.sleep(0.2)
        self.assertEqual(len(qp.queries), 1)

        self.assertEqual(
            qp.query('uuid-test-4', "test query"),
            'OK'
        )
        gevent.sleep(0)

        qp.stop()
        gevent.sleep(0)
    def test_queryprocessor_3(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(
            comm.rpc_server_channels[0]['name'],
            minemeld.traced.queryprocessor.QUERY_QUEUE
        )
        self.assertEqual(
            comm.rpc_server_channels[0]['allowed_methods'],
            ['query', 'kill_query']
        )

        qp.query('uuid-test-1', "test query")
        gevent.sleep(0)

        qp.kill_query('uuid-test-1')
        gevent.sleep(0)

        self.assertIsInstance(
            traced_mock.MOCK_QUERIES[0].get(),
            greenlet.GreenletExit
        )
        self.assertEqual(
            len(qp.queries),
            0
        )
        self.assertIn('uuid-test-1', store.release_alls)

        qp.stop()
        gevent.sleep(0)
Example #6
0
    def test_queryprocessor_5(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(
            comm, store, {'max_concurrency': 2})

        qp.query('uuid-test-1', "test query")
        gevent.sleep(0)

        qp.query('uuid-test-2', "test query")
        gevent.sleep(0)

        self.assertRaises(RuntimeError, qp.query, 'uuid-test-3', "test query")
        gevent.sleep(0)

        traced_mock.MOCK_QUERIES[0].finish_event.set()
        gevent.sleep(0.2)
        self.assertEqual(len(qp.queries), 1)

        self.assertEqual(qp.query('uuid-test-4', "test query"), 'OK')
        gevent.sleep(0)

        qp.stop()
        gevent.sleep(0)
    def test_query_empty(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        q = minemeld.traced.queryprocessor.Query(
            store,
            "field3:foo -field4:679",
            3*86400*1000, 0,
            100,
            'uuid-test',
            {}
        )
        q._run()

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 0)
        self.assertEqual(eoq, True)
Example #8
0
    def test_query_empty(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        q = minemeld.traced.queryprocessor.Query(store,
                                                 "field3:foo -field4:679",
                                                 3 * 86400 * 1000, 0, 100,
                                                 'uuid-test', {})
        q._run()

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 0)
        self.assertEqual(eoq, True)
Example #9
0
    def test_query_pos(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(1 * 86400 * 1000, 'log0')

        q = minemeld.traced.queryprocessor.Query(store, "log",
                                                 3 * 86400 * 1000, 0, 100,
                                                 'uuid-test', {})
        q._run()
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 1)
        self.assertEqual(eoq, True)
Example #10
0
    def test_queryprocessor_4(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(comm.rpc_server_channels[0]['name'],
                         minemeld.traced.queryprocessor.QUERY_QUEUE)
        self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
                         ['query', 'kill_query'])

        qp.stop()
        gevent.sleep(0)

        self.assertRaises(RuntimeError, qp.query, 'test-uuid-1', "test")
        self.assertRaises(RuntimeError, qp.kill_query, 'test-uuid-1')
    def test_writer(self):
        config = {}
        comm = comm_mock.comm_factory(config)
        store = traced_mock.store_factory()

        writer = minemeld.traced.writer.Writer(comm, store, 'TESTTOPIC')
        self.assertEqual(comm.sub_channels[0]['topic'], 'TESTTOPIC')
        self.assertEqual(comm.sub_channels[0]['allowed_methods'], ['log'])

        writer.log(0, log='testlog')
        self.assertEqual(store.writes[0]['timestamp'], 0)
        self.assertEqual(store.writes[0]['log'], ujson.dumps({'log': 'testlog'}))

        writer.stop()
        writer.log(0, log='testlog')
        self.assertEqual(len(store.writes), 1)

        writer.stop()  # just for coverage
    def test_query_fs4(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(1*86400*1000, ujson.dumps({
            "field1": "foo",
            "field2": "bar",
            "field3": ["foo", "bar"],
            "field4": 12345678
        }))
        store.write(1*86400*1000, ujson.dumps({
            "field1": "bar",
            "field2": "foo",
            "field3": ["foo", "bar"],
            "field4": 12345679
        }))

        q = minemeld.traced.queryprocessor.Query(
            store,
            "field3:foo FIELD3:Bar",
            3*86400*1000, 0,
            100,
            'uuid-test',
            {}
        )
        q._run()
        LOG.debug(SR_mock.mock_calls)
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 2)
        self.assertEqual(eoq, True)
    def test_query_wcard(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(1*86400*1000, ujson.dumps({
            "field1": "1*oo1",
            "field2": "2*ar2",
            "field3": ["5foo5", "6bar6"],
            "field4": 12345678
        }))
        store.write(1*86400*1000, ujson.dumps({
            "field1": "3bar3",
            "field2": "4foo4",
            "field3": ["8*oo8", "7bar7"],
            "field4": 12345678
        }))

        q = minemeld.traced.queryprocessor.Query(
            store,
            "field3:*oo -field4:67?",
            3*86400*1000, 0,
            100,
            'uuid-test',
            {}
        )
        q._run()
        LOG.debug(SR_mock.mock_calls)
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 1)
        self.assertEqual(eoq, True)
Example #14
0
    def test_query_fs4(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(
            1 * 86400 * 1000,
            ujson.dumps({
                "field1": "foo",
                "field2": "bar",
                "field3": ["foo", "bar"],
                "field4": 12345678
            }))
        store.write(
            1 * 86400 * 1000,
            ujson.dumps({
                "field1": "bar",
                "field2": "foo",
                "field3": ["foo", "bar"],
                "field4": 12345679
            }))

        q = minemeld.traced.queryprocessor.Query(store,
                                                 "field3:foo FIELD3:Bar",
                                                 3 * 86400 * 1000, 0, 100,
                                                 'uuid-test', {})
        q._run()
        LOG.debug(SR_mock.mock_calls)
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 2)
        self.assertEqual(eoq, True)
Example #15
0
    def test_query_wcard(self, glet_mock, SR_mock):
        store = traced_mock.store_factory()

        store.write(
            1 * 86400 * 1000,
            ujson.dumps({
                "field1": "1*oo1",
                "field2": "2*ar2",
                "field3": ["5foo5", "6bar6"],
                "field4": 12345678
            }))
        store.write(
            1 * 86400 * 1000,
            ujson.dumps({
                "field1": "3bar3",
                "field2": "4foo4",
                "field3": ["8*oo8", "7bar7"],
                "field4": 12345678
            }))

        q = minemeld.traced.queryprocessor.Query(store,
                                                 "field3:*oo -field4:67?",
                                                 3 * 86400 * 1000, 0, 100,
                                                 'uuid-test', {})
        q._run()
        LOG.debug(SR_mock.mock_calls)
        self.assertGreater(len(SR_mock.mock_calls), 1)

        num_logs = 0
        eoq = False
        for call in SR_mock.mock_calls[1:]:
            name, args, kwargs = call
            self.assertEqual(name, '().publish')
            self.assertEqual(args[0], 'mm-traced-q.uuid-test')

            if args[1] == '<EOQ>':
                eoq = True
            else:
                line = json.loads(args[1])
                if 'log' in line:
                    num_logs += 1

        self.assertEqual(num_logs, 1)
        self.assertEqual(eoq, True)
    def test_queryprocessor_2(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(
            comm.rpc_server_channels[0]['name'],
            minemeld.traced.queryprocessor.QUERY_QUEUE
        )
        self.assertEqual(
            comm.rpc_server_channels[0]['allowed_methods'],
            ['query', 'kill_query']
        )

        qp.query('uuid-test-1', "test query")
        self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)

        qp.query('uuid-test-2', "bad")
        self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)

        gevent.sleep(0)

        traced_mock.MOCK_QUERIES[0].finish_event.set()
        traced_mock.MOCK_QUERIES[1].finish_event.set()

        gevent.sleep(0)

        qp.stop()
        gevent.sleep(0)
        gevent.sleep(0)

        self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)
        self.assertRaises(
            RuntimeError,
            traced_mock.MOCK_QUERIES[1].get
        )
        self.assertNotIn('uuid-test-1', store.release_alls)
        self.assertIn('uuid-test-2', store.release_alls)
Example #17
0
    def test_queryprocessor_3(self, query_mock):
        comm = comm_mock.comm_factory({})
        store = traced_mock.store_factory()

        qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)

        self.assertEqual(comm.rpc_server_channels[0]['name'],
                         minemeld.traced.queryprocessor.QUERY_QUEUE)
        self.assertEqual(comm.rpc_server_channels[0]['allowed_methods'],
                         ['query', 'kill_query'])

        qp.query('uuid-test-1', "test query")
        gevent.sleep(0)

        qp.kill_query('uuid-test-1')
        gevent.sleep(0)

        self.assertIsInstance(traced_mock.MOCK_QUERIES[0].get(),
                              greenlet.GreenletExit)
        self.assertEqual(len(qp.queries), 0)
        self.assertIn('uuid-test-1', store.release_alls)

        qp.stop()
        gevent.sleep(0)