Beispiel #1
0
class TestBrokerDB(unittest2.TestCase):

    def setUp(self):
        self.loop = ioloop.IOLoop()
        self.tmp = tempfile.mkdtemp()
        dboptions = {'directory': self.tmp}
        self.db = BrokerDB(self.loop, db='python',
                           dboptions=dboptions)

    def tearDown(self):
        shutil.rmtree(self.db.directory)
        self.db.close()
        self.loop.close()

    def test_brokerdb(self):
        self.assertEqual(list(self.db.get_data('swwqqsw')), [])

        def add_data():
            for line in ONE_RUN:
                data = dict(line)
                data['run_id'] = '1'
                self.db.add(data)
                data['run_id'] = '2'
                self.db.add(data)

        self.loop.add_callback(add_data)
        self.loop.add_callback(add_data)
        self.loop.add_timeout(time.time() + .5, self.loop.stop)
        self.loop.start()

        # let's check if we got the data in the file
        with open(os.path.join(self.db.directory, '1-db.json')) as f:
            data = [json.loads(line) for line in f]
        data.sort()

        with open(os.path.join(self.db.directory, '2-db.json')) as f:
            data2 = [json.loads(line) for line in f]

        self.assertEqual(len(data), 12)
        self.assertEqual(len(data2), 12)
        counts = self.db.get_counts('1')

        for type_ in ('addSuccess', 'stopTestRun', 'stopTest',
                      'startTest', 'startTestRun', 'add_hit'):
            self.assertEqual(dict(counts)[type_], 2)

        # we got 12 lines, let's try batching
        batch = list(self.db.get_data('1', size=2))
        self.assertEqual(len(batch), 2)

        batch = list(self.db.get_data('1', start=2))
        self.assertEqual(len(batch), 10)

        batch = list(self.db.get_data('1', start=2, size=5))
        self.assertEqual(len(batch), 5)

        data3 = list(self.db.get_data('1'))
        data3.sort()
        self.assertEqual(data3, data)

        # filtered
        data3 = list(self.db.get_data('1', data_type='add_hit'))
        self.assertEqual(len(data3), 2)

        # group by
        res = list(self.db.get_data('1', groupby=True))
        self.assertEqual(len(res), 6)
        self.assertEqual(res[0]['count'], 2)

        res = list(self.db.get_data('1', data_type='add_hit', groupby=True))
        self.assertEqual(res[0]['count'], 2)

        self.assertTrue('1' in self.db.get_runs())
        self.assertTrue('2' in self.db.get_runs())

        # len(data) < asked ize
        batch = list(self.db.get_data('1', start=2, size=5000))
        self.assertEqual(len(batch), 10)

    def test_metadata(self):
        self.assertEqual(self.db.get_metadata('1'), {})
        self.db.save_metadata('1', {'hey': 'ho'})
        self.assertEqual(self.db.get_metadata('1'), {'hey': 'ho'})

        self.db.update_metadata('1', one=2)
        meta = self.db.get_metadata('1').items()
        meta.sort()
        self.assertEqual(meta, [('hey', 'ho'), ('one', 2)])

    def test_get_urls(self):
        def add_data():
            for line in ONE_RUN:
                data = dict(line)
                data['run_id'] = '1'
                self.db.add(data)
                data['run_id'] = '2'
                self.db.add(data)

        self.loop.add_callback(add_data)
        self.loop.add_callback(add_data)
        self.loop.add_timeout(time.time() + .5, self.loop.stop)
        self.loop.start()

        self.assertTrue(self.db.ping())
        urls = self.db.get_urls('1')
        self.assertEqual(urls, {'http://127.0.0.1:9200/': 2})
Beispiel #2
0
class TestBrokerDB(unittest2.TestCase):

    def setUp(self):
        self.loop = ioloop.IOLoop()
        self.tmp = tempfile.mkdtemp()
        dboptions = {'directory': self.tmp}
        self.db = BrokerDB(self.loop, db='python',
                           **dboptions)

    def tearDown(self):
        shutil.rmtree(self.db.directory)
        self.db.close()
        self.loop.close()

    def test_brokerdb(self):
        self.assertEqual(list(self.db.get_data('swwqqsw')), [])

        def add_data():
            for line in ONE_RUN:
                data = dict(line)
                data['run_id'] = '1'
                self.db.add(data)
                data['run_id'] = '2'
                self.db.add(data)

        self.loop.add_callback(add_data)
        self.loop.add_callback(add_data)
        self.loop.add_timeout(time.time() + 2.1, self.loop.stop)
        self.loop.start()

        # let's check if we got the data in the file
        db = os.path.join(self.db.directory, '1-db.json')
        data = [record for record, line in read_zfile(db)]
        data.sort()

        db = os.path.join(self.db.directory, '2-db.json')
        data2 = [record for record, line in read_zfile(db)]
        data2.sort()

        self.assertEqual(len(data), 14)
        self.assertEqual(len(data2), 14)
        counts = self.db.get_counts('1')

        for type_ in ('addSuccess', 'stopTestRun', 'stopTest',
                      'startTest', 'startTestRun', 'add_hit'):
            self.assertEqual(counts[type_], 2)

        # we got 12 lines, let's try batching
        batch = list(self.db.get_data('1', size=2))
        self.assertEqual(len(batch), 2)

        batch = list(self.db.get_data('1', start=2))
        self.assertEqual(len(batch), 12)

        batch = list(self.db.get_data('1', start=2, size=5))
        self.assertEqual(len(batch), 5)

        data = [self.db._uncompress_headers('1', line) for line in data]
        data.sort()

        data3 = list(self.db.get_data('1'))
        data3.sort()
        self.assertEqual(data3, data)

        # filtered
        data3 = list(self.db.get_data('1', data_type='add_hit'))
        self.assertEqual(len(data3), 2)

        # group by
        res = list(self.db.get_data('1', groupby=True))
        self.assertEqual(len(res), 7)
        self.assertEqual(res[0]['count'], 2)

        res = list(self.db.get_data('1', data_type='add_hit', groupby=True))
        self.assertEqual(res[0]['count'], 2)

        self.assertTrue('1' in self.db.get_runs())
        self.assertTrue('2' in self.db.get_runs())

        # len(data) < asked ize
        batch = list(self.db.get_data('1', start=2, size=5000))
        self.assertEqual(len(batch), 12)

    def test_metadata(self):
        self.assertEqual(self.db.get_metadata('1'), {})
        self.db.save_metadata('1', {'hey': 'ho'})
        self.assertEqual(self.db.get_metadata('1'), {'hey': 'ho'})

        self.db.update_metadata('1', one=2)
        meta = self.db.get_metadata('1').items()
        meta.sort()
        self.assertEqual(meta, [('hey', 'ho'), ('one', 2)])

    def test_get_urls(self):
        def add_data():
            for line in ONE_RUN:
                data = dict(line)
                data['run_id'] = '1'
                self.db.add(data)
                data['run_id'] = '2'
                self.db.add(data)

        self.loop.add_callback(add_data)
        self.loop.add_callback(add_data)
        self.loop.add_timeout(time.time() + .5, self.loop.stop)
        self.loop.start()

        self.assertTrue(self.db.ping())
        urls = self.db.get_urls('1')
        self.assertEqual(urls, {'http://127.0.0.1:9200/': 2})

    def test_get_errors(self):
        def add_data():
            for line in ONE_RUN:
                data = dict(line)
                data['run_id'] = '1'
                self.db.add(data)
                data['run_id'] = '2'
                self.db.add(data)

        self.loop.add_callback(add_data)
        self.loop.add_callback(add_data)
        self.loop.add_timeout(time.time() + .5, self.loop.stop)
        self.loop.start()

        self.assertTrue(self.db.ping())

        errors = list(self.db.get_errors('2'))
        self.assertEqual(len(errors), 2, errors)

        errors = list(self.db.get_errors('1'))
        self.assertEqual(len(errors), 2, errors)

    def test_compression(self):
        headers_f = os.path.join(self.db.directory, 'run-id-headers.json')
        headers = {"1": 'one', "2": 'two'}

        with open(headers_f, 'w') as f:
            f.write(json.dumps(headers))

        data = {'one': 'ok', 'two': 3, 'three': 'blah'}
        self.db._update_headers('run-id')

        self.db.add({'run_id': 'run-id', 'one': 'ok', 'two': 3,
                     'three': 'blah'})

        result = self.db._compress_headers('run-id', data)
        result = result.items()
        result.sort()
        self.assertEqual(result, [(1, 'ok'), (2, 3), (3, 'blah')])
        self.db.flush()

        with open(headers_f) as f:
            new_headers = json.loads(f.read())

        wanted = [(1, u'one'), (2, u'two'), (3, u'three'), (4, u'run_id')]
        new_headers = [(int(key), value) for key, value in new_headers.items()]
        new_headers.sort()
        self.assertEquals(new_headers, wanted)

    @unittest2.skipIf('TRAVIS' in os.environ, '')
    def test_max_size(self):
        # adding some data for run_1 and run_2
        self.db.prepare_run()

        for run in ('run_1', 'run_2', 'run_3'):
            for i in range(1000):
                self.db.add({'run_id': run, 'one': 'ok', 'two': 3,
                             'three': 'blah'})
            # flushing
            self.db.flush()
            time.sleep(.1)

        self.assertEqual(self.db.get_runs(), ['run_1', 'run_2', 'run_3'])

        # setting the max size to current size
        self.db.max_size = get_dir_size(self.tmp)
        self.db.prepare_run()

        # adding data for run_4
        for i in range(1000):
            self.db.add({'run_id': 'run_4', 'one': 'ok', 'two': 3,
                         'three': 'blah'})

        # run-1 should have been wiped...
        self.db.flush()
        self.assertEqual(self.db.get_runs(), ['run_2', 'run_3', 'run_4'])

    def test_reload(self):
        self.assertEqual(self.db.get_metadata('1'), {})
        self.db.save_metadata('1', {'hey': 'ho'})
        self.assertEqual(self.db.get_metadata('1'), {'hey': 'ho'})
        self.db.update_metadata('1', one=2)
        meta = self.db.get_metadata('1').items()
        meta.sort()
        self.assertEqual(meta, [('hey', 'ho'), ('one', 2)])
        self.db.flush()

        # make sure we don't lose existing data when
        # the db client is started and writes before reads
        dboptions = {'directory': self.tmp}
        db2 = BrokerDB(self.loop, db='python', **dboptions)

        # this used to overwrite any existing data
        db2.update_metadata('1', has_data=1)
        meta = db2.get_metadata('1').items()
        meta.sort()
        wanted = [(u'has_data', 1), (u'hey', u'ho'), (u'one', 2)]
        self.assertEqual(meta, wanted)