def postgresql_factory(request): """ Fixture factory for PostgreSQL. :param FixtureRequest request: fixture request object :rtype: psycopg2.connection :returns: postgresql client """ config = get_config(request) if not psycopg2: raise ImportError( 'No module named psycopg2. Please install either ' 'psycopg2 or psycopg2-binary package for CPython ' 'or psycopg2cffi for Pypy.') proc_fixture = request.getfixturevalue(process_fixture_name) # _, config = try_import('psycopg2', request) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_options = proc_fixture.options pg_db = db_name or config['dbname'] with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, proc_fixture.version): connection = psycopg2.connect(dbname=pg_db, user=pg_user, host=pg_host, port=pg_port, options=pg_options) yield connection connection.close()
def database(request): config = get_config(request) pg_host = config.get("host") pg_port = config.get("port") or os.environ.get("PGPORT", 5432) pg_user = config.get("user") pg_db = config.get("db", "tests") pg_version = config.get("version", 10.1) janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, pg_version) # In case the database already exists, possibly due to an aborted test run, # attempt to drop it before creating try: janitor.drop() except InvalidCatalogName: # We can safely ignore this exception as that means there was # no leftover database pass # Create our Database. janitor.init() # Ensure our database gets deleted. @request.addfinalizer def drop_database(): janitor.drop() return "postgresql://{}@{}:{}/{}".format(pg_user, pg_host, pg_port, pg_db)
def postgresql_factory(request: FixtureRequest) -> Iterator[connection]: """ Fixture factory for PostgreSQL. :param request: fixture request object :returns: postgresql client """ check_for_psycopg() proc_fixture: Union[PostgreSQLExecutor, NoopExecutor] = request.getfixturevalue( process_fixture_name) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_password = proc_fixture.password pg_options = proc_fixture.options pg_db = dbname or proc_fixture.dbname pg_load = load or [] with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, proc_fixture.version, pg_password, isolation_level) as janitor: db_connection: connection = psycopg.connect( dbname=pg_db, user=pg_user, password=pg_password, host=pg_host, port=pg_port, options=pg_options, ) for load_element in pg_load: janitor.load(load_element) yield db_connection db_connection.close()
def test_cursor_selects_postgres_database(connect_mock: MagicMock) -> None: """Test that the cursor requests the postgres database.""" janitor = DatabaseJanitor("user", "host", "1234", "database_name", 10) with janitor.cursor(): connect_mock.assert_called_once_with(dbname="postgres", user="******", password=None, host="host", port="1234")
def test_cursor_selects_postgres_database(connect_mock): """Test that the cursor requests the postgres database.""" janitor = DatabaseJanitor('user', 'host', '1234', 'database_name', 9.0) with janitor.cursor(): connect_mock.assert_called_once_with( dbname='postgres', user='******', host='host', port='1234' )
def test_postgresql(postgresql_proc: "PostgreSQLExecutor"): pg_dict = { "user": postgresql_proc.user, "password": postgresql_proc.password, "host": postgresql_proc.host, "port": postgresql_proc.port, "dbname": postgresql_proc.dbname, } with DatabaseJanitor(**pg_dict, version=postgresql_proc.version): do_test_postgresql(pg_dict)
def init_postgresql_database(user, host, port, db_name): """ Create database in postgresql. :param str user: postgresql username :param str host: postgresql host :param str port: postgresql port :param str db_name: database name """ warn( 'init_postgresql_database is deprecated, ' 'use DatabaseJanitor.init instead.', DeprecationWarning) DatabaseJanitor(user, host, port, db_name, 0.0).init()
def database(request): ''' Create a Postgres database for the tests, and drop it when the tests are done. ''' pg_host = DB_OPTS.get("host") pg_port = DB_OPTS.get("port") pg_user = DB_OPTS.get("username") pg_pass = DB_OPTS.get("password") pg_db = DB_OPTS["database"] janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, 9.6, pg_pass) janitor.init() yield janitor.drop()
def drop_postgresql_database(user, host, port, db_name, version): """ Drop databse in postgresql. :param str user: postgresql username :param str host: postgresql host :param str port: postgresql port :param str db_name: database name :param packaging.version.Version version: postgresql version number """ warn( 'drop_postgresql_database is deprecated, ' 'use DatabaseJanitor.drop instead.', DeprecationWarning) DatabaseJanitor(user, host, port, db_name, version).drop()
def database(request): ''' Create a Postgres database for the tests, and drop it when the tests are done. ''' pg_host = DB_OPTS.get("host") pg_port = DB_OPTS.get("port") pg_user = DB_OPTS.get("username") pg_pass = DB_OPTS.get("password") pg_db = DB_OPTS["database"] db_janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, 12, pg_pass) db_janitor.init() @request.addfinalizer def drop_database(): db_janitor.drop()
def test_janitor_populate(connect_mock: MagicMock, load_database: str) -> None: """ Test that the cursor requests the postgres database. load_database tries to connect to database, which triggers mocks. """ call_kwargs = { "host": "host", "port": "1234", "user": "******", "dbname": "database_name", "password": "******", } janitor = DatabaseJanitor(version=10, **call_kwargs) # type: ignore[arg-type] janitor.load(load_database) assert connect_mock.called assert connect_mock.call_args.kwargs == call_kwargs
def database(request): """ Fixture for starting an ephemeral database instance for the duration of the test suite """ def get_psql_version(): result = subprocess.check_output(["psql", "--version"]).decode("utf-8") version = result.split(" ")[-1].strip() major, minor, *_ = version.split(".") # Get the major and minor version, which are what pytest-postgresql # wants return f"{major}.{minor}" if os.environ.get("POSTGRES_USER"): # Use separately launched process if environments variables are defined # This is used in Gitlab CI tests which run in a Docker container user = os.environ["POSTGRES_USER"] host = os.environ["POSTGRES_HOST"] password = os.environ["POSTGRES_PASSWORD"] # POSTGRES_PORT can also be a value such as "tcp://1.1.1.1:5432" # This handles that format as well port = int(os.environ.get("POSTGRES_PORT", "5432").split(":")[-1]) db_name = "passari_test" version = os.environ["POSTGRES_VERSION"] create_engine( f"postgresql://{user}:{password}@{host}:{port}/{db_name}") yield request.getfixturevalue("postgresql_nooproc") else: # Launch PostgreSQL ourselves postgresql = request.getfixturevalue("postgresql_proc") user = postgresql.user host = postgresql.host port = postgresql.port db_name = "passari_test" version = get_psql_version() with DatabaseJanitor(user, host, port, db_name, version): create_engine(f"postgresql://{user}@{host}:{port}/{db_name}") yield postgresql
def config_database(request): connection_template = 'postgresql+psycopg2://{0}:{1}@{2}:{3}/{4}' pg_host = os.environ.get('POSTGRES_HOST', 'db') pg_port = os.environ.get('POSTGRES_PORT', 5432) pg_user = os.environ.get('POSTGRES_USER', 'tests') pg_passsword = os.environ.get('POSTGRES_PASSWORD', 'tests') pg_db = os.environ.get('POSTGRES_DB', 'tests') connection_string = connection_template.format(pg_user, pg_passsword, pg_host, pg_port, pg_db) # Create the database janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, '11', pg_passsword) janitor.init() yield connection_string # Ensure the database gets deleted janitor.drop()
def database(postgresql_proc): """Initialize a test postgres database.""" janitor = DatabaseJanitor( postgresql_proc.user, postgresql_proc.host, postgresql_proc.port, postgresql_proc.dbname, postgresql_proc.version, postgresql_proc.password, ) janitor.init() with janitor.cursor() as cur: cur.execute("""\ CREATE TABLE my_source_1 (id serial PRIMARY KEY, num integer); CREATE TABLE my_source_2 (id serial PRIMARY KEY, num integer); """) yield postgresql_proc janitor.drop()
def postgresql_factory(request: FixtureRequest) -> connection: """ Fixture factory for PostgreSQL. :param FixtureRequest request: fixture request object :returns: postgresql client """ config = get_config(request) if not psycopg2: raise ImportError( 'No module named psycopg2. Please install either ' 'psycopg2 or psycopg2-binary package for CPython ' 'or psycopg2cffi for Pypy.') proc_fixture: Union[PostgreSQLExecutor, NoopExecutor] = request.getfixturevalue( process_fixture_name) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_password = proc_fixture.password pg_options = proc_fixture.options pg_db = db_name or config['dbname'] pg_load = load or config['load'] with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, proc_fixture.version, pg_password, isolation_level): db_connection: connection = psycopg2.connect(dbname=pg_db, user=pg_user, password=pg_password, host=pg_host, port=pg_port, options=pg_options) if pg_load: for filename in pg_load: with open(filename, 'r') as _fd: with db_connection.cursor() as cur: cur.execute(_fd.read()) yield db_connection db_connection.close()
def postgresql_factory(request: FixtureRequest) -> connection: """ Fixture factory for PostgreSQL. :param FixtureRequest request: fixture request object :returns: postgresql client """ config = get_config(request) check_for_psycopg2() pg_isolation_level = isolation_level or psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT proc_fixture: Union[PostgreSQLExecutor, NoopExecutor] = request.getfixturevalue( process_fixture_name) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_password = proc_fixture.password pg_options = proc_fixture.options pg_db = db_name or config['dbname'] pg_load = load or config['load'] with DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, proc_fixture.version, pg_password, pg_isolation_level): db_connection: connection = psycopg2.connect(dbname=pg_db, user=pg_user, password=pg_password, host=pg_host, port=pg_port, options=pg_options) if pg_load: for filename in pg_load: with open(filename, 'r') as _fd: with db_connection.cursor() as cur: cur.execute(_fd.read()) yield db_connection db_connection.close()
def database(request): config = get_config(request) pg_host = config.get("host") pg_port = config.get("port") or os.environ.get("PGPORT", 5432) pg_user = config.get("user") pg_db = config.get("db", "tests") pg_version = config.get("version", 10.1) janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, pg_version) # In case the database already exists, possibly due to an aborted test run, # attempt to drop it before creating janitor.drop() # Create our Database. janitor.init() # Ensure our database gets deleted. @request.addfinalizer def drop_database(): janitor.drop() return "postgresql://{}@{}:{}/{}".format(pg_user, pg_host, pg_port, pg_db)
def postgresql_noproc_fixture( request: FixtureRequest) -> Iterator[NoopExecutor]: """ Noop Process fixture for PostgreSQL. :param request: fixture request object :returns: tcp executor-like object """ config = get_config(request) pg_host = host or config["host"] pg_port = port or config["port"] or 5432 pg_user = user or config["user"] pg_password = password or config["password"] pg_dbname = xdistify_dbname(dbname or config["dbname"]) pg_options = options or config["options"] pg_load = load or config["load"] noop_exec = NoopExecutor( host=pg_host, port=pg_port, user=pg_user, password=pg_password, dbname=pg_dbname, options=pg_options, ) template_dbname = f"{noop_exec.dbname}_tmpl" with DatabaseJanitor( user=noop_exec.user, host=noop_exec.host, port=noop_exec.port, dbname=template_dbname, version=noop_exec.version, password=noop_exec.password, ) as janitor: for load_element in pg_load: janitor.load(load_element) yield noop_exec
def test_version_cast(version): """Test that version is cast to Version object.""" janitor = DatabaseJanitor(None, None, None, None, version) assert janitor.version == VERSION
def de_server_factory(request): ''' actually make the fixture ''' if port: host_port = (host, port) else: host_port = (host, get_random_port()) db_info = {} proc_fixture = request.getfixturevalue(pg_prog_name) db_info['host'] = proc_fixture.host db_info['port'] = proc_fixture.port db_info['user'] = proc_fixture.user db_info['password'] = proc_fixture.password # used to find the version of postgres conn_fixture = request.getfixturevalue(pg_db_conn_name) # pseudo random database name for testing db_info['database'] = DE_DB_NAME + '_test_' + ''.join( random.choices(string.ascii_uppercase + string.digits, k=5)) # Due to the multi-threaded/connection pooled nature # of the DE Server, it is cleaner to build out an # unscoped database. The one created by the `DE_DB` # fixture is private to a single socket/connection # and cannot be shared cleanly. # # And even if we could share it, then we wouldn't # be testing the production data path or pooling # with those tricks # DatabaseJanitor will create and drop the tablespace for us with DatabaseJanitor(user=db_info['user'], password=db_info['password'], host=db_info['host'], port=db_info['port'], dbname=db_info['database'], version=conn_fixture.server_version): # if you swap this for the `DE_DB` fixture, it will # block and changes will not be visable to the connection # fired up within the DE Server thread. with psycopg2.connect(**db_info) as connection: for filename in DE_SCHEMA: # noqa: F405 with open(filename, 'r') as _fd, \ connection.cursor() as cur: cur.execute(_fd.read()) server_proc = DETestWorker(conf_path, channel_conf_path, host_port, db_info, conf_override, channel_conf_override) server_proc.start() # The following block only works if there are # active workers; if it is called before any workers # exist, then it will return and not block as requested. # so long as your config contains at least one worker, # this will work as you'd expect. server_proc.de_server.block_while(State.BOOT) if not server_proc.is_alive(): raise RuntimeError('Could not start PrivateDEServer fixture') yield server_proc if server_proc.is_alive(): server_proc.de_server.rpc_stop() server_proc.join()
def test_version_cast(version: Any) -> None: """Test that version is cast to Version object.""" janitor = DatabaseJanitor("user", "host", "1234", "database_name", version) assert janitor.version == VERSION
def postgresql_proc_fixture( request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Iterator[PostgreSQLExecutor]: """ Process fixture for PostgreSQL. :param request: fixture request object :param tmp_path_factory: temporary path object (fixture) :returns: tcp executor """ config = get_config(request) postgresql_ctl = executable or config["exec"] logfile_prefix = logs_prefix or config["logsprefix"] pg_dbname = dbname or config["dbname"] pg_load = load or config["load"] # check if that executable exists, as it's no on system PATH # only replace if executable isn't passed manually if not os.path.exists(postgresql_ctl) and executable is None: pg_bindir = subprocess.check_output( ["pg_config", "--bindir"], universal_newlines=True).strip() postgresql_ctl = os.path.join(pg_bindir, "pg_ctl") tmpdir = tmp_path_factory.mktemp( f"pytest-postgresql-{request.fixturename}") if logfile_prefix: warn( f"logfile_prefix and logsprefix config option is deprecated, " f"and will be dropped in future releases. All fixture related " f"data resides within {tmpdir}", DeprecationWarning, ) pg_port = get_port(port) or get_port(config["port"]) assert pg_port is not None datadir = tmpdir / f"data-{pg_port}" datadir.mkdir() logfile_path = tmpdir / f"{logfile_prefix}postgresql.{pg_port}.log" if platform.system() == "FreeBSD": with (datadir / "pg_hba.conf").open(mode="a") as conf_file: conf_file.write("host all all 0.0.0.0/0 trust\n") postgresql_executor = PostgreSQLExecutor( executable=postgresql_ctl, host=host or config["host"], port=pg_port, user=user or config["user"], password=password or config["password"], dbname=pg_dbname, options=options or config["options"], datadir=str(datadir), unixsocketdir=unixsocketdir or config["unixsocketdir"], logfile=str(logfile_path), startparams=startparams or config["startparams"], postgres_options=postgres_options or config["postgres_options"], ) # start server with postgresql_executor: postgresql_executor.wait_for_postgres() template_dbname = f"{postgresql_executor.dbname}_tmpl" with DatabaseJanitor( user=postgresql_executor.user, host=postgresql_executor.host, port=postgresql_executor.port, dbname=template_dbname, version=postgresql_executor.version, password=postgresql_executor.password, ) as janitor: for load_element in pg_load: janitor.load(load_element) yield postgresql_executor