def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test") return # TODO move airline demo script_path = script_relative_path( "../../../dagster_examples_tests/airline_demo_tests/") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "test-postgres-db"]) subprocess.check_output( ["docker-compose", "rm", "-f", "test-postgres-db"]) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ["docker-compose", "up", "-d", "test-postgres-db"]) conn_str = get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test") wait_for_connection(conn_str) yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test') return # TODO move airline demo script_path = script_relative_path( '../../../dagster_examples_tests/airline_demo_tests/') if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ['docker-compose', 'stop', 'test-postgres-db']) subprocess.check_output( ['docker-compose', 'rm', '-f', 'test-postgres-db']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', 'up', '-d', 'test-postgres-db']) conn_str = get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test') wait_for_connection(conn_str) yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield return script_path = script_relative_path(".") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "test-postgres-db-airline"]) subprocess.check_output( ["docker-compose", "rm", "-f", "test-postgres-db-airline"]) except Exception: pass subprocess.check_output( ["docker-compose", "up", "-d", "test-postgres-db-airline"]) wait_for_connection( get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test")) yield
def add_data(): con_string = get_conn_string( username=PG_SOURCE_CONFIG["username"], password=PG_SOURCE_CONFIG["password"], hostname=PG_SOURCE_CONFIG["host"], port=str(PG_SOURCE_CONFIG["port"]), db_name=PG_SOURCE_CONFIG["database"], ) users = pd.DataFrame({ "user_id": range(N_USERS), "is_bot": [random.choice([True, False]) for _ in range(N_USERS)], }) users.to_sql("users", con=con_string, if_exists="replace") print("Created users table.") orders = pd.DataFrame({ "user_id": [random.randint(0, N_USERS) for _ in range(N_ORDERS)], "order_time": _random_dates(), "order_value": np.random.normal(loc=100.0, scale=15.0, size=N_ORDERS), }) orders.to_sql("orders", con=con_string, if_exists="replace") print("Created orders table.")
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield return script_path = script_relative_path('.') if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ['docker-compose', 'stop', 'test-postgres-db']) subprocess.check_output( ['docker-compose', 'rm', '-f', 'test-postgres-db']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', 'up', '-d', 'test-postgres-db']) wait_for_connection( get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test')) yield
def conn_string(**kwargs): check.invariant( TestPostgresInstance.dagster_postgres_installed(), "dagster_postgres must be installed to test with postgres", ) from dagster_postgres.utils import get_conn_string # pylint: disable=import-error return get_conn_string(**dict( dict( username="******", password="******", hostname=TestPostgresInstance.get_hostname(), db_name="test", ), **kwargs))
def test_conn_str(): username = "******" password = "******" db_name = "dagster" hostname = "database-city.com" conn_str = get_conn_string( username=username, password=password, db_name=db_name, hostname=hostname, ) assert ( conn_str == r"postgresql://has%40init:full%3Aof%3Ajunk%21%40%[email protected]:5432/dagster" ) parsed = urlparse(conn_str) assert unquote(parsed.username) == username assert unquote(parsed.password) == password assert parsed.hostname == hostname
def postgres(pg_hostname): # pylint: disable=redefined-outer-name conn_string = get_conn_string( username="******", password="******", hostname=pg_hostname, db_name="dbt_example", ) if not BUILDKITE: script_path = file_relative_path(__file__, ".") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "dbt_example_postgresql"]) subprocess.check_output([ "docker-compose", "rm", "-f", "dbt_example_postgresql" ]) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ["docker-compose", "up", "-d", "dbt_example_postgresql"]) wait_for_connection(conn_string) old_env = None if os.getenv("DBT_EXAMPLE_CONN_STRING") is not None: old_env = os.getenv("DBT_EXAMPLE_CONN_STRING") try: os.environ["DBT_EXAMPLE_CONN_STRING"] = conn_string yield finally: if old_env is not None: os.environ["DBT_EXAMPLE_CONN_STRING"] = old_env else: del os.environ["DBT_EXAMPLE_CONN_STRING"]
def _conn_string(**kwargs): return get_conn_string(**dict( dict(username='******', password='******', hostname=get_hostname(), db_name='test'), **kwargs))
PG_SOURCE_CONFIG = { "username": "******", "password": "******", "host": "localhost", "port": 5432, "database": "postgres", } PG_DESTINATION_CONFIG = { "username": "******", "password": "******", "host": "localhost", "port": 5432, "database": "postgres_replica", } AIRBYTE_CONFIG = {"host": "localhost", "port": "8000"} DBT_PROJECT_DIR = file_relative_path(__file__, "../mds_dbt") DBT_PROFILES_DIR = file_relative_path(__file__, "../mds_dbt/config") DBT_CONFIG = {"project_dir": DBT_PROJECT_DIR, "profiles_dir": DBT_PROFILES_DIR} PANDAS_IO_CONFIG = { "con_string": get_conn_string( username=PG_DESTINATION_CONFIG["username"], password=PG_DESTINATION_CONFIG["password"], hostname=PG_DESTINATION_CONFIG["host"], port=str(PG_DESTINATION_CONFIG["port"]), db_name=PG_DESTINATION_CONFIG["database"], ) }