def test_creates_virtual_tables_for_partition_with_segment_without_errors(self):
fs = fsopendir('temp://')
def gen():
# generate header
yield ['col1', 'col2']
# generate rows
yield [0, 0]
yield [1, 1]
mprows = MPRowsFile(fs, 'example.com/simple-0.1.3/1.mpr')
mprows.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))
# create virtual tables. This should not raise an error.
#
connection = apsw.Connection(':memory:')
try:
add_partition(connection, mprows, 'vid1')
except Exception as exc:
raise AssertionError('partition adding unexpectadly failed with {} error.'.format(exc))
# check selected rows
#
cursor = connection.cursor()
result = cursor.execute('SELECT * FROM {}'.format('vid1')).fetchall()
self.assertEqual(result, [(0, 0), (1, 1)])
def test_creates_virtual_table_for_simple_fixed_mpr(self):
# build rows reader
cache_fs = fsopendir(self.setup_temp_dir())
sources = self.load_sources()
spec = sources['simple_fixed']
s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))
mprows = MPRowsFile(cache_fs, spec.name).load_rows(s)
# first make sure file not changed.
expected_names = ['id', 'uuid', 'int', 'float']
expected_types = ['int', binary_type.__name__, 'int', 'float']
self.assertEqual([x['name'] for x in mprows.reader.columns],
expected_names)
self.assertEqual([x['type'] for x in mprows.reader.columns],
expected_types)
connection = apsw.Connection(':memory:')
table = 'table1'
add_partition(connection, mprows, table)
# check all columns and some rows.
cursor = connection.cursor()
query = 'SELECT count(*) FROM {};'.format(table)
result = cursor.execute(query).fetchall()
self.assertEqual(result, [(10000, )])
with mprows.reader as r:
expected_first_row = next(iter(r)).row
# query by columns.
query = 'SELECT id, uuid, int, float FROM {} LIMIT 1;'.format(table)
result = cursor.execute(query).fetchall()
self.assertEqual(len(result), 1)
self.assertEqual(result[0], expected_first_row)
def test_creates_virtual_tables_for_partition_with_segment_without_errors(
self):
fs = fsopendir('temp://')
def gen():
# generate header
yield ['col1', 'col2']
# generate rows
yield [0, 0]
yield [1, 1]
mprows = MPRowsFile(fs, 'example.com/simple-0.1.3/1.mpr')
mprows.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))
# create virtual tables. This should not raise an error.
#
connection = apsw.Connection(':memory:')
try:
add_partition(connection, mprows, 'vid1')
except Exception as exc:
raise AssertionError(
'partition adding unexpectadly failed with {} error.'.format(
exc))
# check selected rows
#
cursor = connection.cursor()
result = cursor.execute('SELECT * FROM {}'.format('vid1')).fetchall()
self.assertEqual(result, [(0, 0), (1, 1)])
def test_creates_virtual_table(self, fake_get):
fake_mprows = _get_fake_mprows('int')
fake_connection = Mock()
fake_execute = Mock()
fake_connection.cursor = lambda: AttrDict({'execute': fake_execute})
fake_mprows = _get_fake_mprows('int')
add_partition(fake_connection, fake_mprows, 'vid1')
fake_get.assert_called_once_with()
fake_execute.assert_called_once_with('CREATE VIRTUAL TABLE vid1 using mod_partition(/tmp, tmp);')
def test_creates_virtual_table(self, fake_get):
fake_mprows = _get_fake_mprows('int')
fake_connection = Mock()
fake_execute = Mock()
fake_connection.cursor = lambda: AttrDict({'execute': fake_execute})
fake_mprows = _get_fake_mprows('int')
add_partition(fake_connection, fake_mprows, 'vid1')
fake_get.assert_called_once_with()
fake_execute.assert_called_once_with(
'CREATE VIRTUAL TABLE vid1 using mod_partition(/tmp, tmp);')
def _add_partition(self, connection, partition):
""" Creates sqlite virtual table for mpr file of the given partition.
Args:
connection: connection to the sqlite db who stores mpr data.
partition (orm.Partition):
"""
logger.debug(
'Creating virtual table for partition.\n partition: {}'.format(
partition.name))
sqlite_med.add_partition(connection, partition.datafile,
partition.vid + '_vt')
def test_creates_virtual_table_for_source_with_header_containing_sql_reserved_words(
self):
# build rows reader
cache_fs = fsopendir(self.setup_temp_dir())
spec = SourceSpec('foobar')
def gen():
# yield header
yield ['create', 'index', 'where', 'select', 'distinct']
# yield rows
for i in range(10):
yield [i, i + 1, i + 2, i + 3, i + 4]
s = GeneratorSource(spec, gen())
mprows = MPRowsFile(cache_fs, spec.name).load_rows(s)
connection = apsw.Connection(':memory:')
table = 'table1'
add_partition(connection, mprows, table)
# check all columns and some rows.
cursor = connection.cursor()
query = 'SELECT count(*) FROM {};'.format(table)
result = cursor.execute(query).fetchall()
self.assertEqual(result, [(10, )])
with mprows.reader as r:
expected_first_row = next(iter(r)).row
# query by columns.
query = 'SELECT "create", "index", "where", "select", "distinct" FROM {} LIMIT 1;'.format(
table)
result = cursor.execute(query).fetchall()
self.assertEqual(len(result), 1)
self.assertEqual(result[0], expected_first_row)
def test_creates_virtual_table_for_source_with_header_containing_sql_reserved_words(self):
# build rows reader
cache_fs = fsopendir(self.setup_temp_dir())
spec = SourceSpec('foobar')
def gen():
# yield header
yield ['create', 'index', 'where', 'select', 'distinct']
# yield rows
for i in range(10):
yield [i, i + 1, i + 2, i + 3, i + 4]
s = GeneratorSource(spec, gen())
mprows = MPRowsFile(cache_fs, spec.name).load_rows(s)
connection = apsw.Connection(':memory:')
table = 'table1'
add_partition(connection, mprows, table)
# check all columns and some rows.
cursor = connection.cursor()
query = 'SELECT count(*) FROM {};'.format(table)
result = cursor.execute(query).fetchall()
self.assertEqual(result, [(10,)])
with mprows.reader as r:
expected_first_row = next(iter(r)).row
# query by columns.
query = 'SELECT "create", "index", "where", "select", "distinct" FROM {} LIMIT 1;'.format(table)
result = cursor.execute(query).fetchall()
self.assertEqual(len(result), 1)
self.assertEqual(result[0], expected_first_row)
def test_selects_correct_rows_from_many_mprows(self):
fs = fsopendir('temp://')
header = ['col1', 'col2']
# create 3 mprows files.
#
rows1 = [(0, 0), (1, 1)]
mprows1 = MPRowsFile(fs, 'vid1')
mprows1.load_rows(self._get_generator_source(header, rows1))
rows2 = [(2, 2), (3, 3)]
mprows2 = MPRowsFile(fs, 'vid2')
mprows2.load_rows(self._get_generator_source(header, rows2))
rows3 = [(4, 4), (5, 5)]
mprows3 = MPRowsFile(fs, 'vid3')
mprows3.load_rows(self._get_generator_source(header, rows3))
# create virtual tables for all mprows
#
connection = apsw.Connection(':memory:')
add_partition(connection, mprows1, 'vid1')
add_partition(connection, mprows2, 'vid2')
add_partition(connection, mprows3, 'vid3')
# check rows of all added mprows.
#
cursor = connection.cursor()
query_tmpl = 'SELECT * FROM {};'
# check rows of the first file.
#
query = query_tmpl.format('vid1')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows1)
# check rows of the second mprows file.
#
query = query_tmpl.format('vid2')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows2)
# check rows of the third mprows file.
#
query = query_tmpl.format('vid3')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows3)
def test_selects_correct_rows_from_many_mprows(self):
fs = fsopendir('temp://')
header = ['col1', 'col2']
# create 3 mprows files.
#
rows1 = [(0, 0), (1, 1)]
mprows1 = MPRowsFile(fs, 'vid1')
mprows1.load_rows(self._get_generator_source(header, rows1))
rows2 = [(2, 2), (3, 3)]
mprows2 = MPRowsFile(fs, 'vid2')
mprows2.load_rows(self._get_generator_source(header, rows2))
rows3 = [(4, 4), (5, 5)]
mprows3 = MPRowsFile(fs, 'vid3')
mprows3.load_rows(self._get_generator_source(header, rows3))
# create virtual tables for all mprows
#
connection = apsw.Connection(':memory:')
add_partition(connection, mprows1, 'vid1')
add_partition(connection, mprows2, 'vid2')
add_partition(connection, mprows3, 'vid3')
# check rows of all added mprows.
#
cursor = connection.cursor()
query_tmpl = 'SELECT * FROM {};'
# check rows of the first file.
#
query = query_tmpl.format('vid1')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows1)
# check rows of the second mprows file.
#
query = query_tmpl.format('vid2')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows2)
# check rows of the third mprows file.
#
query = query_tmpl.format('vid3')
result = cursor.execute(query).fetchall()
self.assertEqual(result, rows3)
def test_creates_sqlite_module(self, fake_get):
fake_connection = Mock()
fake_mprows = _get_fake_mprows('int')
add_partition(fake_connection, fake_mprows, 'vid1')
fake_get.assert_called_once_with()
def test_creates_sqlite_module(self, fake_get):
fake_connection = Mock()
fake_mprows = _get_fake_mprows('int')
add_partition(fake_connection, fake_mprows, 'vid1')
fake_get.assert_called_once_with()
