def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test")
return
# TODO move airline demo
script_path = script_relative_path(
"../../../dagster_examples_tests/airline_demo_tests/")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "test-postgres-db"])
subprocess.check_output(
["docker-compose", "rm", "-f", "test-postgres-db"])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
["docker-compose", "up", "-d", "test-postgres-db"])
conn_str = get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test")
wait_for_connection(conn_str)
yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
return
# TODO move airline demo
script_path = script_relative_path(
'../../../dagster_examples_tests/airline_demo_tests/')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
conn_str = get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
wait_for_connection(conn_str)
yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path(".")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "test-postgres-db-airline"])
subprocess.check_output(
["docker-compose", "rm", "-f", "test-postgres-db-airline"])
except Exception:
pass
subprocess.check_output(
["docker-compose", "up", "-d", "test-postgres-db-airline"])
wait_for_connection(
get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test"))
yield
def add_data():
con_string = get_conn_string(
username=PG_SOURCE_CONFIG["username"],
password=PG_SOURCE_CONFIG["password"],
hostname=PG_SOURCE_CONFIG["host"],
port=str(PG_SOURCE_CONFIG["port"]),
db_name=PG_SOURCE_CONFIG["database"],
)
users = pd.DataFrame({
"user_id":
range(N_USERS),
"is_bot": [random.choice([True, False]) for _ in range(N_USERS)],
})
users.to_sql("users", con=con_string, if_exists="replace")
print("Created users table.")
orders = pd.DataFrame({
"user_id": [random.randint(0, N_USERS) for _ in range(N_ORDERS)],
"order_time":
_random_dates(),
"order_value":
np.random.normal(loc=100.0, scale=15.0, size=N_ORDERS),
})
orders.to_sql("orders", con=con_string, if_exists="replace")
print("Created orders table.")
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path('.')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
wait_for_connection(
get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test'))
yield
def conn_string(**kwargs):
check.invariant(
TestPostgresInstance.dagster_postgres_installed(),
"dagster_postgres must be installed to test with postgres",
)
from dagster_postgres.utils import get_conn_string # pylint: disable=import-error
return get_conn_string(**dict(
dict(
username="******",
password="******",
hostname=TestPostgresInstance.get_hostname(),
db_name="test",
), **kwargs))
def test_conn_str():
username = "******"
password = "******"
db_name = "dagster"
hostname = "database-city.com"
conn_str = get_conn_string(
username=username,
password=password,
db_name=db_name,
hostname=hostname,
)
assert (
conn_str ==
r"postgresql://has%40init:full%3Aof%3Ajunk%21%40%[email protected]:5432/dagster"
)
parsed = urlparse(conn_str)
assert unquote(parsed.username) == username
assert unquote(parsed.password) == password
assert parsed.hostname == hostname
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
conn_string = get_conn_string(
username="******",
password="******",
hostname=pg_hostname,
db_name="dbt_example",
)
if not BUILDKITE:
script_path = file_relative_path(__file__, ".")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "dbt_example_postgresql"])
subprocess.check_output([
"docker-compose", "rm", "-f", "dbt_example_postgresql"
])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
["docker-compose", "up", "-d", "dbt_example_postgresql"])
wait_for_connection(conn_string)
old_env = None
if os.getenv("DBT_EXAMPLE_CONN_STRING") is not None:
old_env = os.getenv("DBT_EXAMPLE_CONN_STRING")
try:
os.environ["DBT_EXAMPLE_CONN_STRING"] = conn_string
yield
finally:
if old_env is not None:
os.environ["DBT_EXAMPLE_CONN_STRING"] = old_env
else:
del os.environ["DBT_EXAMPLE_CONN_STRING"]
def _conn_string(**kwargs):
return get_conn_string(**dict(
dict(username='******',
password='******',
hostname=get_hostname(),
db_name='test'), **kwargs))
PG_SOURCE_CONFIG = {
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "postgres",
}
PG_DESTINATION_CONFIG = {
"username": "******",
"password": "******",
"host": "localhost",
"port": 5432,
"database": "postgres_replica",
}
AIRBYTE_CONFIG = {"host": "localhost", "port": "8000"}
DBT_PROJECT_DIR = file_relative_path(__file__, "../mds_dbt")
DBT_PROFILES_DIR = file_relative_path(__file__, "../mds_dbt/config")
DBT_CONFIG = {"project_dir": DBT_PROJECT_DIR, "profiles_dir": DBT_PROFILES_DIR}
PANDAS_IO_CONFIG = {
"con_string": get_conn_string(
username=PG_DESTINATION_CONFIG["username"],
password=PG_DESTINATION_CONFIG["password"],
hostname=PG_DESTINATION_CONFIG["host"],
port=str(PG_DESTINATION_CONFIG["port"]),
db_name=PG_DESTINATION_CONFIG["database"],
)
}
