def postgresql_factory(request):
"""
Fixture factory for PostgreSQL.
:param FixtureRequest request: fixture request object
:rtype: psycopg2.connection
:returns: postgresql client
"""
config = get_config(request)
if not psycopg2:
raise ImportError(
'No module named psycopg2. Please install either '
'psycopg2 or psycopg2-binary package for CPython '
'or psycopg2cffi for Pypy.')
proc_fixture = request.getfixturevalue(process_fixture_name)
# _, config = try_import('psycopg2', request)
pg_host = proc_fixture.host
pg_port = proc_fixture.port
pg_user = proc_fixture.user
pg_options = proc_fixture.options
pg_db = db_name or config['dbname']
with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db,
proc_fixture.version):
connection = psycopg2.connect(dbname=pg_db,
user=pg_user,
host=pg_host,
port=pg_port,
options=pg_options)
yield connection
connection.close()
def database(request):
config = get_config(request)
pg_host = config.get("host")
pg_port = config.get("port") or os.environ.get("PGPORT", 5432)
pg_user = config.get("user")
pg_db = config.get("db", "tests")
pg_version = config.get("version", 10.1)
janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, pg_version)
# In case the database already exists, possibly due to an aborted test run,
# attempt to drop it before creating
try:
janitor.drop()
except InvalidCatalogName:
# We can safely ignore this exception as that means there was
# no leftover database
pass
# Create our Database.
janitor.init()
# Ensure our database gets deleted.
@request.addfinalizer
def drop_database():
janitor.drop()
return "postgresql://{}@{}:{}/{}".format(pg_user, pg_host, pg_port, pg_db)
def postgresql_factory(request: FixtureRequest) -> Iterator[connection]:
"""
Fixture factory for PostgreSQL.
:param request: fixture request object
:returns: postgresql client
"""
check_for_psycopg()
proc_fixture: Union[PostgreSQLExecutor,
NoopExecutor] = request.getfixturevalue(
process_fixture_name)
pg_host = proc_fixture.host
pg_port = proc_fixture.port
pg_user = proc_fixture.user
pg_password = proc_fixture.password
pg_options = proc_fixture.options
pg_db = dbname or proc_fixture.dbname
pg_load = load or []
with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db,
proc_fixture.version, pg_password,
isolation_level) as janitor:
db_connection: connection = psycopg.connect(
dbname=pg_db,
user=pg_user,
password=pg_password,
host=pg_host,
port=pg_port,
options=pg_options,
)
for load_element in pg_load:
janitor.load(load_element)
yield db_connection
db_connection.close()
def test_cursor_selects_postgres_database(connect_mock: MagicMock) -> None:
"""Test that the cursor requests the postgres database."""
janitor = DatabaseJanitor("user", "host", "1234", "database_name", 10)
with janitor.cursor():
connect_mock.assert_called_once_with(dbname="postgres",
user="******",
password=None,
host="host",
port="1234")
def test_cursor_selects_postgres_database(connect_mock):
"""Test that the cursor requests the postgres database."""
janitor = DatabaseJanitor('user', 'host', '1234', 'database_name', 9.0)
with janitor.cursor():
connect_mock.assert_called_once_with(
dbname='postgres',
user='******',
host='host',
port='1234'
)
def test_postgresql(postgresql_proc: "PostgreSQLExecutor"):
pg_dict = {
"user": postgresql_proc.user,
"password": postgresql_proc.password,
"host": postgresql_proc.host,
"port": postgresql_proc.port,
"dbname": postgresql_proc.dbname,
}
with DatabaseJanitor(**pg_dict, version=postgresql_proc.version):
do_test_postgresql(pg_dict)
def init_postgresql_database(user, host, port, db_name):
"""
Create database in postgresql.
:param str user: postgresql username
:param str host: postgresql host
:param str port: postgresql port
:param str db_name: database name
"""
warn(
'init_postgresql_database is deprecated, '
'use DatabaseJanitor.init instead.', DeprecationWarning)
DatabaseJanitor(user, host, port, db_name, 0.0).init()
def database(request):
'''
Create a Postgres database for the tests, and drop it when the tests are done.
'''
pg_host = DB_OPTS.get("host")
pg_port = DB_OPTS.get("port")
pg_user = DB_OPTS.get("username")
pg_pass = DB_OPTS.get("password")
pg_db = DB_OPTS["database"]
janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, 9.6, pg_pass)
janitor.init()
yield
janitor.drop()
def drop_postgresql_database(user, host, port, db_name, version):
"""
Drop databse in postgresql.
:param str user: postgresql username
:param str host: postgresql host
:param str port: postgresql port
:param str db_name: database name
:param packaging.version.Version version: postgresql version number
"""
warn(
'drop_postgresql_database is deprecated, '
'use DatabaseJanitor.drop instead.', DeprecationWarning)
DatabaseJanitor(user, host, port, db_name, version).drop()
def database(request):
'''
Create a Postgres database for the tests, and drop it when the tests are done.
'''
pg_host = DB_OPTS.get("host")
pg_port = DB_OPTS.get("port")
pg_user = DB_OPTS.get("username")
pg_pass = DB_OPTS.get("password")
pg_db = DB_OPTS["database"]
db_janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, 12, pg_pass)
db_janitor.init()
@request.addfinalizer
def drop_database():
db_janitor.drop()
def test_janitor_populate(connect_mock: MagicMock, load_database: str) -> None:
"""
Test that the cursor requests the postgres database.
load_database tries to connect to database, which triggers mocks.
"""
call_kwargs = {
"host": "host",
"port": "1234",
"user": "******",
"dbname": "database_name",
"password": "******",
}
janitor = DatabaseJanitor(version=10,
**call_kwargs) # type: ignore[arg-type]
janitor.load(load_database)
assert connect_mock.called
assert connect_mock.call_args.kwargs == call_kwargs
def database(request):
"""
Fixture for starting an ephemeral database instance for the duration
of the test suite
"""
def get_psql_version():
result = subprocess.check_output(["psql", "--version"]).decode("utf-8")
version = result.split(" ")[-1].strip()
major, minor, *_ = version.split(".")
# Get the major and minor version, which are what pytest-postgresql
# wants
return f"{major}.{minor}"
if os.environ.get("POSTGRES_USER"):
# Use separately launched process if environments variables are defined
# This is used in Gitlab CI tests which run in a Docker container
user = os.environ["POSTGRES_USER"]
host = os.environ["POSTGRES_HOST"]
password = os.environ["POSTGRES_PASSWORD"]
# POSTGRES_PORT can also be a value such as "tcp://1.1.1.1:5432"
# This handles that format as well
port = int(os.environ.get("POSTGRES_PORT", "5432").split(":")[-1])
db_name = "passari_test"
version = os.environ["POSTGRES_VERSION"]
create_engine(
f"postgresql://{user}:{password}@{host}:{port}/{db_name}")
yield request.getfixturevalue("postgresql_nooproc")
else:
# Launch PostgreSQL ourselves
postgresql = request.getfixturevalue("postgresql_proc")
user = postgresql.user
host = postgresql.host
port = postgresql.port
db_name = "passari_test"
version = get_psql_version()
with DatabaseJanitor(user, host, port, db_name, version):
create_engine(f"postgresql://{user}@{host}:{port}/{db_name}")
yield postgresql
def config_database(request):
connection_template = 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}'
pg_host = os.environ.get('POSTGRES_HOST', 'db')
pg_port = os.environ.get('POSTGRES_PORT', 5432)
pg_user = os.environ.get('POSTGRES_USER', 'tests')
pg_passsword = os.environ.get('POSTGRES_PASSWORD', 'tests')
pg_db = os.environ.get('POSTGRES_DB', 'tests')
connection_string = connection_template.format(pg_user, pg_passsword,
pg_host, pg_port, pg_db)
# Create the database
janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, '11',
pg_passsword)
janitor.init()
yield connection_string
# Ensure the database gets deleted
janitor.drop()
def database(postgresql_proc):
"""Initialize a test postgres database."""
janitor = DatabaseJanitor(
postgresql_proc.user,
postgresql_proc.host,
postgresql_proc.port,
postgresql_proc.dbname,
postgresql_proc.version,
postgresql_proc.password,
)
janitor.init()
with janitor.cursor() as cur:
cur.execute("""\
CREATE TABLE my_source_1 (id serial PRIMARY KEY, num integer);
CREATE TABLE my_source_2 (id serial PRIMARY KEY, num integer);
""")
yield postgresql_proc
janitor.drop()
def postgresql_factory(request: FixtureRequest) -> connection:
"""
Fixture factory for PostgreSQL.
:param FixtureRequest request: fixture request object
:returns: postgresql client
"""
config = get_config(request)
if not psycopg2:
raise ImportError(
'No module named psycopg2. Please install either '
'psycopg2 or psycopg2-binary package for CPython '
'or psycopg2cffi for Pypy.')
proc_fixture: Union[PostgreSQLExecutor,
NoopExecutor] = request.getfixturevalue(
process_fixture_name)
pg_host = proc_fixture.host
pg_port = proc_fixture.port
pg_user = proc_fixture.user
pg_password = proc_fixture.password
pg_options = proc_fixture.options
pg_db = db_name or config['dbname']
pg_load = load or config['load']
with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db,
proc_fixture.version, pg_password,
isolation_level):
db_connection: connection = psycopg2.connect(dbname=pg_db,
user=pg_user,
password=pg_password,
host=pg_host,
port=pg_port,
options=pg_options)
if pg_load:
for filename in pg_load:
with open(filename, 'r') as _fd:
with db_connection.cursor() as cur:
cur.execute(_fd.read())
yield db_connection
db_connection.close()
def postgresql_factory(request: FixtureRequest) -> connection:
"""
Fixture factory for PostgreSQL.
:param FixtureRequest request: fixture request object
:returns: postgresql client
"""
config = get_config(request)
check_for_psycopg2()
pg_isolation_level = isolation_level or psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
proc_fixture: Union[PostgreSQLExecutor,
NoopExecutor] = request.getfixturevalue(
process_fixture_name)
pg_host = proc_fixture.host
pg_port = proc_fixture.port
pg_user = proc_fixture.user
pg_password = proc_fixture.password
pg_options = proc_fixture.options
pg_db = db_name or config['dbname']
pg_load = load or config['load']
with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db,
proc_fixture.version, pg_password,
pg_isolation_level):
db_connection: connection = psycopg2.connect(dbname=pg_db,
user=pg_user,
password=pg_password,
host=pg_host,
port=pg_port,
options=pg_options)
if pg_load:
for filename in pg_load:
with open(filename, 'r') as _fd:
with db_connection.cursor() as cur:
cur.execute(_fd.read())
yield db_connection
db_connection.close()
def database(request):
config = get_config(request)
pg_host = config.get("host")
pg_port = config.get("port") or os.environ.get("PGPORT", 5432)
pg_user = config.get("user")
pg_db = config.get("db", "tests")
pg_version = config.get("version", 10.1)
janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, pg_version)
# In case the database already exists, possibly due to an aborted test run,
# attempt to drop it before creating
janitor.drop()
# Create our Database.
janitor.init()
# Ensure our database gets deleted.
@request.addfinalizer
def drop_database():
janitor.drop()
return "postgresql://{}@{}:{}/{}".format(pg_user, pg_host, pg_port, pg_db)
def postgresql_noproc_fixture(
request: FixtureRequest) -> Iterator[NoopExecutor]:
"""
Noop Process fixture for PostgreSQL.
:param request: fixture request object
:returns: tcp executor-like object
"""
config = get_config(request)
pg_host = host or config["host"]
pg_port = port or config["port"] or 5432
pg_user = user or config["user"]
pg_password = password or config["password"]
pg_dbname = xdistify_dbname(dbname or config["dbname"])
pg_options = options or config["options"]
pg_load = load or config["load"]
noop_exec = NoopExecutor(
host=pg_host,
port=pg_port,
user=pg_user,
password=pg_password,
dbname=pg_dbname,
options=pg_options,
)
template_dbname = f"{noop_exec.dbname}_tmpl"
with DatabaseJanitor(
user=noop_exec.user,
host=noop_exec.host,
port=noop_exec.port,
dbname=template_dbname,
version=noop_exec.version,
password=noop_exec.password,
) as janitor:
for load_element in pg_load:
janitor.load(load_element)
yield noop_exec
def test_version_cast(version):
"""Test that version is cast to Version object."""
janitor = DatabaseJanitor(None, None, None, None, version)
assert janitor.version == VERSION
def de_server_factory(request):
'''
actually make the fixture
'''
if port:
host_port = (host, port)
else:
host_port = (host, get_random_port())
db_info = {}
proc_fixture = request.getfixturevalue(pg_prog_name)
db_info['host'] = proc_fixture.host
db_info['port'] = proc_fixture.port
db_info['user'] = proc_fixture.user
db_info['password'] = proc_fixture.password
# used to find the version of postgres
conn_fixture = request.getfixturevalue(pg_db_conn_name)
# pseudo random database name for testing
db_info['database'] = DE_DB_NAME + '_test_' + ''.join(
random.choices(string.ascii_uppercase + string.digits, k=5))
# Due to the multi-threaded/connection pooled nature
# of the DE Server, it is cleaner to build out an
# unscoped database. The one created by the `DE_DB`
# fixture is private to a single socket/connection
# and cannot be shared cleanly.
#
# And even if we could share it, then we wouldn't
# be testing the production data path or pooling
# with those tricks
# DatabaseJanitor will create and drop the tablespace for us
with DatabaseJanitor(user=db_info['user'],
password=db_info['password'],
host=db_info['host'],
port=db_info['port'],
dbname=db_info['database'],
version=conn_fixture.server_version):
# if you swap this for the `DE_DB` fixture, it will
# block and changes will not be visable to the connection
# fired up within the DE Server thread.
with psycopg2.connect(**db_info) as connection:
for filename in DE_SCHEMA: # noqa: F405
with open(filename, 'r') as _fd, \
connection.cursor() as cur:
cur.execute(_fd.read())
server_proc = DETestWorker(conf_path, channel_conf_path, host_port,
db_info, conf_override,
channel_conf_override)
server_proc.start()
# The following block only works if there are
# active workers; if it is called before any workers
# exist, then it will return and not block as requested.
# so long as your config contains at least one worker,
# this will work as you'd expect.
server_proc.de_server.block_while(State.BOOT)
if not server_proc.is_alive():
raise RuntimeError('Could not start PrivateDEServer fixture')
yield server_proc
if server_proc.is_alive():
server_proc.de_server.rpc_stop()
server_proc.join()
def test_version_cast(version: Any) -> None:
"""Test that version is cast to Version object."""
janitor = DatabaseJanitor("user", "host", "1234", "database_name", version)
assert janitor.version == VERSION
def postgresql_proc_fixture(
request: FixtureRequest,
tmp_path_factory: TempPathFactory) -> Iterator[PostgreSQLExecutor]:
"""
Process fixture for PostgreSQL.
:param request: fixture request object
:param tmp_path_factory: temporary path object (fixture)
:returns: tcp executor
"""
config = get_config(request)
postgresql_ctl = executable or config["exec"]
logfile_prefix = logs_prefix or config["logsprefix"]
pg_dbname = dbname or config["dbname"]
pg_load = load or config["load"]
# check if that executable exists, as it's no on system PATH
# only replace if executable isn't passed manually
if not os.path.exists(postgresql_ctl) and executable is None:
pg_bindir = subprocess.check_output(
["pg_config", "--bindir"], universal_newlines=True).strip()
postgresql_ctl = os.path.join(pg_bindir, "pg_ctl")
tmpdir = tmp_path_factory.mktemp(
f"pytest-postgresql-{request.fixturename}")
if logfile_prefix:
warn(
f"logfile_prefix and logsprefix config option is deprecated, "
f"and will be dropped in future releases. All fixture related "
f"data resides within {tmpdir}",
DeprecationWarning,
)
pg_port = get_port(port) or get_port(config["port"])
assert pg_port is not None
datadir = tmpdir / f"data-{pg_port}"
datadir.mkdir()
logfile_path = tmpdir / f"{logfile_prefix}postgresql.{pg_port}.log"
if platform.system() == "FreeBSD":
with (datadir / "pg_hba.conf").open(mode="a") as conf_file:
conf_file.write("host all all 0.0.0.0/0 trust\n")
postgresql_executor = PostgreSQLExecutor(
executable=postgresql_ctl,
host=host or config["host"],
port=pg_port,
user=user or config["user"],
password=password or config["password"],
dbname=pg_dbname,
options=options or config["options"],
datadir=str(datadir),
unixsocketdir=unixsocketdir or config["unixsocketdir"],
logfile=str(logfile_path),
startparams=startparams or config["startparams"],
postgres_options=postgres_options or config["postgres_options"],
)
# start server
with postgresql_executor:
postgresql_executor.wait_for_postgres()
template_dbname = f"{postgresql_executor.dbname}_tmpl"
with DatabaseJanitor(
user=postgresql_executor.user,
host=postgresql_executor.host,
port=postgresql_executor.port,
dbname=template_dbname,
version=postgresql_executor.version,
password=postgresql_executor.password,
) as janitor:
for load_element in pg_load:
janitor.load(load_element)
yield postgresql_executor