def main(sqs):
pg = None
try:
print("Starting postgres db...")
pg = PostgresContainer("postgres:11.6-alpine")
pg.start()
os.environ['DB_URL'] = pg.get_connection_url()
sqs.create_queue(QueueName="test-queue.fifo",
Attributes={"FifoQueue": "true"})
from app import main
yield main
finally:
if pg is not None:
pg.stop()
def db_session(request):
postgres = PostgresContainer()
postgres.start()
db_url = postgres.get_connection_url()
engine = create_engine(db_url)
session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=engine))
Base.query = session.query_property()
Base.metadata.create_all(engine)
def stop_db():
postgres.stop()
request.addfinalizer(stop_db)
return session
def psql():
"""
We need to modify the Flask Application to accept a seperate
sessionmaker for testing. !TODO
"""
psql_container = PostgresContainer("postgres:9.6")
psql_container.POSTGRES_USER = "******"
psql_container.POSTGRES_PASSWORD = "******"
psql_container.POSTGRES_DB = "test"
psql_container.start()
engine = sqlalchemy.create_engine(psql_container.get_connection_url())
session = sessionmaker(bind=engine)()
# alert webservice to port for this database
port = psql_container.get_connection_url().split(":")[-1].split("/")[0]
header = copy.copy(TEST_HEADER)
header["port"] = port
return {"container": psql_container, "session": session, "header": header}
class CrossLanguageJdbcIOTest(unittest.TestCase):
def setUp(self):
self.start_postgres_container(retries=3)
self.engine = sqlalchemy.create_engine(
self.postgres.get_connection_url())
self.username = '******'
self.password = '******'
self.host = self.postgres.get_container_host_ip()
self.port = self.postgres.get_exposed_port(5432)
self.database_name = 'test'
self.driver_class_name = 'org.postgresql.Driver'
self.jdbc_url = 'jdbc:postgresql://{}:{}/{}'.format(
self.host, self.port, self.database_name)
def tearDown(self):
# Sometimes stopping the container raises ReadTimeout. We can ignore it
# here to avoid the test failure.
try:
self.postgres.stop()
except: # pylint: disable=bare-except
logging.error('Could not stop the postgreSQL container.')
def test_xlang_jdbc_write(self):
table_name = 'jdbc_external_test_write'
self.engine.execute(
"CREATE TABLE {}(f_id INTEGER, f_real REAL, f_string VARCHAR)".
format(table_name))
inserted_rows = [
JdbcWriteTestRow(i, i + 0.1, 'Test{}'.format(i))
for i in range(ROW_COUNT)
]
with TestPipeline() as p:
p.not_use_test_runner_api = True
_ = (
p
|
beam.Create(inserted_rows).with_output_types(JdbcWriteTestRow)
# TODO(BEAM-10750) Add test with overridden write_statement
| 'Write to jdbc' >> WriteToJdbc(
table_name=table_name,
driver_class_name=self.driver_class_name,
jdbc_url=self.jdbc_url,
username=self.username,
password=self.password,
))
fetched_data = self.engine.execute(
"SELECT * FROM {}".format(table_name))
fetched_rows = [
JdbcWriteTestRow(int(row[0]), float(row[1]), str(row[2]))
for row in fetched_data
]
self.assertEqual(
set(fetched_rows),
set(inserted_rows),
'Inserted data does not fit data fetched from table',
)
def test_xlang_jdbc_read(self):
table_name = 'jdbc_external_test_read'
self.engine.execute(
"CREATE TABLE {}(f_int INTEGER)".format(table_name))
for i in range(ROW_COUNT):
self.engine.execute("INSERT INTO {} VALUES({})".format(
table_name, i))
with TestPipeline() as p:
p.not_use_test_runner_api = True
result = (
p
# TODO(BEAM-10750) Add test with overridden read_query
| 'Read from jdbc' >> ReadFromJdbc(
table_name=table_name,
driver_class_name=self.driver_class_name,
jdbc_url=self.jdbc_url,
username=self.username,
password=self.password,
))
assert_that(
result,
equal_to([JdbcReadTestRow(i) for i in range(ROW_COUNT)]))
# Creating a container with testcontainers sometimes raises ReadTimeout
# error. In java there are 2 retries set by default.
def start_postgres_container(self, retries):
for i in range(retries):
try:
self.postgres = PostgresContainer('postgres:12.3')
self.postgres.start()
break
except Exception as e: # pylint: disable=bare-except
if i == retries - 1:
logging.error('Unable to initialize postgreSQL container.')
raise e
class CrossLanguageDebeziumIOTest(unittest.TestCase):
def setUp(self):
self.username = '******'
self.password = '******'
self.database = 'inventory'
self.start_db_container(retries=1)
self.host = self.db.get_container_host_ip()
self.port = self.db.get_exposed_port(5432)
self.connector_class = DriverClassName.POSTGRESQL
self.connection_properties = [
"database.dbname=inventory", "database.server.name=dbserver1",
"database.include.list=inventory", "include.schema.changes=false"
]
def tearDown(self):
# Sometimes stopping the container raises ReadTimeout. We can ignore it
# here to avoid the test failure.
try:
self.db.stop()
except: # pylint: disable=bare-except
logging.error('Could not stop the DB container.')
def test_xlang_debezium_read(self):
expected_response = [{
"metadata": {
"connector": "postgresql",
"version": "1.3.1.Final",
"name": "dbserver1",
"database": "inventory",
"schema": "inventory",
"table": "customers"
},
"before": None,
"after": {
"fields": {
"last_name": "Thomas",
"id": 1001,
"first_name": "Sally",
"email": "*****@*****.**"
}
}
}]
with TestPipeline() as p:
p.not_use_test_runner_api = True
results = (p
| 'Read from debezium' >> ReadFromDebezium(
username=self.username,
password=self.password,
host=self.host,
port=self.port,
max_number_of_records=NUM_RECORDS,
connector_class=self.connector_class,
connection_properties=self.connection_properties))
assert_that(results, equal_to(expected_response))
# Creating a container with testcontainers sometimes raises ReadTimeout
# error. In java there are 2 retries set by default.
def start_db_container(self, retries):
for i in range(retries):
try:
self.db = PostgresContainer('debezium/example-postgres:latest',
user=self.username,
password=self.password,
dbname=self.database)
self.db.start()
break
except Exception as e: # pylint: disable=bare-except
if i == retries - 1:
logging.error('Unable to initialize DB container.')
raise e
import sys
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from testcontainers.postgres import PostgresContainer
SQLALCHEMY_DATABASE_URI = None
# somewhat hacky setup to run a testcontainer when running tests
testing = "pytest" in sys.modules
postgres_container = None
def stop_postgres_container():
if postgres_container != None:
postgres_container.stop()
if testing:
postgres_container = PostgresContainer("ratestask-db:latest")
postgres_container.start()
SQLALCHEMY_DATABASE_URI = postgres_container.get_connection_url()
else:
SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://postgres:ratestask@localhost/postgres"
database = SQLAlchemy()
migrate = Migrate(db=database)
class CrossLanguageJdbcIOTest(unittest.TestCase):
def setUp(self):
self.postgres = PostgresContainer('postgres:latest')
self.postgres.start()
self.engine = sqlalchemy.create_engine(
self.postgres.get_connection_url())
self.username = '******'
self.password = '******'
self.host = self.postgres.get_container_host_ip()
self.port = self.postgres.get_exposed_port(5432)
self.database_name = 'test'
self.driver_class_name = 'org.postgresql.Driver'
self.jdbc_url = 'jdbc:postgresql://{}:{}/{}'.format(
self.host, self.port, self.database_name)
def tearDown(self):
self.postgres.stop()
def test_xlang_jdbc_write(self):
table_name = 'jdbc_external_test_write'
self.engine.execute(
"CREATE TABLE {}(f_id INTEGER, f_real REAL, f_string VARCHAR)".
format(table_name))
inserted_rows = [
JdbcWriteTestRow(i, i + 0.1, 'Test{}'.format(i))
for i in range(ROW_COUNT)
]
with TestPipeline() as p:
p.not_use_test_runner_api = True
_ = (p
|
beam.Create(inserted_rows).with_output_types(JdbcWriteTestRow)
| 'Write to jdbc' >> WriteToJdbc(
driver_class_name=self.driver_class_name,
jdbc_url=self.jdbc_url,
username=self.username,
password=self.password,
statement='INSERT INTO {} VALUES(?, ?, ?)'.format(
table_name),
))
fetched_data = self.engine.execute(
"SELECT * FROM {}".format(table_name))
fetched_rows = [
JdbcWriteTestRow(int(row[0]), float(row[1]), str(row[2]))
for row in fetched_data
]
self.assertEqual(
set(fetched_rows),
set(inserted_rows),
'Inserted data does not fit data fetched from table',
)
def test_xlang_jdbc_read(self):
table_name = 'jdbc_external_test_read'
self.engine.execute(
"CREATE TABLE {}(f_int INTEGER)".format(table_name))
for i in range(ROW_COUNT):
self.engine.execute("INSERT INTO {} VALUES({})".format(
table_name, i))
with TestPipeline() as p:
p.not_use_test_runner_api = True
result = (p
| 'Read from jdbc' >> ReadFromJdbc(
driver_class_name=self.driver_class_name,
jdbc_url=self.jdbc_url,
username=self.username,
password=self.password,
query='SELECT f_int FROM {}'.format(table_name),
))
assert_that(
result,
equal_to([JdbcReadTestRow(i) for i in range(ROW_COUNT)]))