def test_executes_select_query_without_any_error(self, fake_shares):
fake_shares.return_value = True
def gen():
# generate header
yield ['col1', 'col2']
# generate first row
yield [0, 0]
fs = fsopendir('temp://')
datafile = MPRowsFile(fs, 'vid1')
datafile.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))
connection = None
try:
PostgreSQLTestBase._create_postgres_test_db()
connection = psycopg2.connect(**PostgreSQLTestBase.pg_test_db_data)
# create foreign table for partition
with connection.cursor() as cursor:
# we have to close opened transaction.
cursor.execute('COMMIT;')
add_partition(cursor, datafile, 'vid1')
# query just created foreign table.
with connection.cursor() as cursor:
cursor.execute('SELECT * FROM partitions.vid1;')
finally:
if connection:
connection.close()
PostgreSQLTestBase._drop_postgres_test_db()
def test_creates_foreign_server(self, fake_shares, fake_create):
fake_shares.return_value = True
cursor = AttrDict({
'execute': lambda q: None})
mprows = _get_fake_partition()
add_partition(cursor, mprows, 'table1')
self.assertEqual(fake_create.call_count, 1)
def test_executes_select_query_without_any_error(self, fake_shares):
fake_shares.return_value = True
def gen():
# generate header
yield ['col1', 'col2']
# generate first row
yield [0, 0]
fs = fsopendir('temp://')
datafile = MPRowsFile(fs, 'vid1')
datafile.load_rows(GeneratorSource(SourceSpec('foobar'), gen()))
connection = None
try:
PostgreSQLTestBase._create_postgres_test_db()
connection = psycopg2.connect(**PostgreSQLTestBase.pg_test_db_data)
# create foreign table for partition
with connection.cursor() as cursor:
# we have to close opened transaction.
cursor.execute('COMMIT;')
add_partition(cursor, datafile, 'vid1')
# query just created foreign table.
with connection.cursor() as cursor:
cursor.execute('SELECT * FROM partitions.vid1;')
finally:
if connection:
connection.close()
PostgreSQLTestBase._drop_postgres_test_db()
def test_creates_foreign_table(self, fake_shares, fake_create):
fake_shares.return_value = True
fake_execute = Mock()
cursor = AttrDict({'execute': fake_execute})
mprows = _get_fake_partition()
add_partition(cursor, mprows, 'table1')
self.assertEqual(fake_execute.call_count, 1)
self.assertIn('CREATE FOREIGN TABLE', str(fake_execute.mock_calls[0]))
def test_creates_foreign_table(self, fake_shares, fake_create):
fake_shares.return_value = True
fake_execute = Mock()
cursor = AttrDict({
'execute': fake_execute})
mprows = _get_fake_partition()
add_partition(cursor, mprows, 'table1')
self.assertEqual(fake_execute.call_count, 1)
self.assertIn('CREATE FOREIGN TABLE', str(fake_execute.mock_calls[0]))
def test_creates_foreign_data_table_for_simple_fixed_mpr(
self, fake_shares):
fake_shares.return_value = True
# build rows reader
cache_fs = fsopendir(self.setup_temp_dir())
sources = self.load_sources()
spec = sources['simple_fixed']
s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))
mprows = MPRowsFile(cache_fs, spec.name).load_rows(s)
# first make sure file was not changed.
expected_names = ['id', 'uuid', 'int', 'float']
expected_types = ['int', binary_type.__name__, 'int', 'float']
self.assertEqual(sorted([x['name'] for x in mprows.reader.columns]),
sorted(expected_names))
self.assertEqual(sorted([x['type'] for x in mprows.reader.columns]),
sorted(expected_types))
try:
# create foreign data table
PostgreSQLTestBase._create_postgres_test_db()
conn = psycopg2.connect(**PostgreSQLTestBase.pg_test_db_data)
try:
with conn.cursor() as cursor:
# we have to close opened transaction.
cursor.execute('COMMIT;')
add_partition(cursor, mprows, 'table1')
# try to query just added partition foreign data table.
with conn.cursor() as cursor:
table = 'table1'
# count all rows
query = 'SELECT count(*) FROM {}.{};'.format(
POSTGRES_PARTITION_SCHEMA_NAME, table)
cursor.execute(query)
result = cursor.fetchall()
self.assertEqual(result, [(10000, )])
# check first row
cursor.execute(
'SELECT id, uuid, int, float FROM {}.{} LIMIT 1;'.
format(POSTGRES_PARTITION_SCHEMA_NAME, table))
result = cursor.fetchall()
self.assertEqual(len(result), 1)
expected_first_row = (1,
'eb385c36-9298-4427-8925-fe09294dbd',
30, Decimal('99.734691532'))
self.assertEqual(result[0], expected_first_row)
finally:
conn.close()
finally:
PostgreSQLTestBase._drop_postgres_test_db()
def test_raises_exception_if_postgres_user_does_not_have_read_permission(self, fake_shares):
fake_shares.return_value = False
cursor = AttrDict({'execute': lambda q: None})
mprows = _get_fake_partition()
raises = False
try:
add_partition(cursor, mprows, 'table1')
except AssertionError as exc:
raises = True
self.assertIn('postgres user does not have permission to read mpr file.', str(exc))
self.assertTrue(raises)
def _add_partition(self, connection, partition):
""" Creates FDW for the partition.
Args:
connection:
partition (orm.Partition):
"""
logger.debug('Creating foreign table for partition.\n partition: {}'.format(partition.name))
with connection.cursor() as cursor:
postgres_med.add_partition(cursor, partition.datafile, partition.vid)
def test_raises_exception_if_postgres_user_does_not_have_read_permission(
self, fake_shares):
fake_shares.return_value = False
cursor = AttrDict({'execute': lambda q: None})
mprows = _get_fake_partition()
raises = False
try:
add_partition(cursor, mprows, 'table1')
except AssertionError as exc:
raises = True
self.assertIn(
'postgres user does not have permission to read mpr file.',
str(exc))
self.assertTrue(raises)
def test_creates_foreign_data_table_for_simple_fixed_mpr(self, fake_shares):
fake_shares.return_value = True
# build rows reader
cache_fs = fsopendir(self.setup_temp_dir())
sources = self.load_sources()
spec = sources['simple_fixed']
s = get_source(spec, cache_fs, callback=lambda x, y: (x, y))
mprows = MPRowsFile(cache_fs, spec.name).load_rows(s)
# first make sure file was not changed.
expected_names = ['id', 'uuid', 'int', 'float']
expected_types = ['int', binary_type.__name__, 'int', 'float']
self.assertEqual(sorted([x['name'] for x in mprows.reader.columns]), sorted(expected_names))
self.assertEqual(sorted([x['type'] for x in mprows.reader.columns]), sorted(expected_types))
try:
# create foreign data table
PostgreSQLTestBase._create_postgres_test_db()
conn = psycopg2.connect(**PostgreSQLTestBase.pg_test_db_data)
try:
with conn.cursor() as cursor:
# we have to close opened transaction.
cursor.execute('COMMIT;')
add_partition(cursor, mprows, 'table1')
# try to query just added partition foreign data table.
with conn.cursor() as cursor:
table = 'table1'
# count all rows
query = 'SELECT count(*) FROM {}.{};'.format(POSTGRES_PARTITION_SCHEMA_NAME, table)
cursor.execute(query)
result = cursor.fetchall()
self.assertEqual(result, [(10000,)])
# check first row
cursor.execute(
'SELECT id, uuid, int, float FROM {}.{} LIMIT 1;'
.format(POSTGRES_PARTITION_SCHEMA_NAME, table))
result = cursor.fetchall()
self.assertEqual(len(result), 1)
expected_first_row = (
1, 'eb385c36-9298-4427-8925-fe09294dbd', 30, Decimal('99.734691532'))
self.assertEqual(result[0], expected_first_row)
finally:
conn.close()
finally:
PostgreSQLTestBase._drop_postgres_test_db()
def _add_partition(self, connection, partition):
""" Creates FDW for the partition.
Args:
connection:
partition (orm.Partition):
"""
logger.debug(
'Creating foreign table for partition.\n partition: {}'.format(
partition.name))
with connection.cursor() as cursor:
postgres_med.add_partition(cursor, partition.datafile,
partition.vid)
def test_creates_foreign_server(self, fake_shares, fake_create):
fake_shares.return_value = True
cursor = AttrDict({'execute': lambda q: None})
mprows = _get_fake_partition()
add_partition(cursor, mprows, 'table1')
self.assertEqual(fake_create.call_count, 1)