def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test")
return
# TODO move airline demo
script_path = script_relative_path(
"../../../dagster_examples_tests/airline_demo_tests/")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "test-postgres-db"])
subprocess.check_output(
["docker-compose", "rm", "-f", "test-postgres-db"])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
["docker-compose", "up", "-d", "test-postgres-db"])
conn_str = get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test")
wait_for_connection(conn_str)
yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path(".")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "test-postgres-db-airline"])
subprocess.check_output(
["docker-compose", "rm", "-f", "test-postgres-db-airline"])
except Exception:
pass
subprocess.check_output(
["docker-compose", "up", "-d", "test-postgres-db-airline"])
wait_for_connection(
get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test"))
yield
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
return
# TODO move airline demo
script_path = script_relative_path(
'../../../dagster_examples_tests/airline_demo_tests/')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
conn_str = get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
wait_for_connection(conn_str)
yield conn_str
def docker_service_up(docker_compose_file, service_name):
check.invariant(
TestPostgresInstance.dagster_postgres_installed(),
'dagster_postgres must be installed to test with postgres',
)
check.str_param(service_name, 'service_name')
check.str_param(docker_compose_file, 'docker_compose_file')
check.invariant(
os.path.isfile(docker_compose_file), 'docker_compose_file must specify a valid file'
)
from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error
if BUILDKITE:
yield TestPostgresInstance.conn_string() # buildkite docker is handled in pipeline setup
return
if not is_postgres_running(service_name):
try:
subprocess.check_output(
['docker-compose', '-f', docker_compose_file, 'stop', service_name]
)
subprocess.check_output(
['docker-compose', '-f', docker_compose_file, 'rm', '-f', service_name]
)
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', '-f', docker_compose_file, 'up', '-d', service_name]
)
conn_str = TestPostgresInstance.conn_string()
wait_for_connection(conn_str)
yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path('.')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
wait_for_connection(
get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test'))
yield
def docker_service_up(
docker_compose_file, service_name, conn_args=None, env_name="POSTGRES_TEST_DB_HOST"
):
check.invariant(
TestPostgresInstance.dagster_postgres_installed(),
"dagster_postgres must be installed to test with postgres",
)
check.str_param(service_name, "service_name")
check.str_param(docker_compose_file, "docker_compose_file")
check.invariant(
os.path.isfile(docker_compose_file), "docker_compose_file must specify a valid file"
)
conn_args = check.opt_dict_param(conn_args, "conn_args") if conn_args else {}
from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error
if BUILDKITE:
yield TestPostgresInstance.conn_string(
**conn_args,
env_name=env_name,
) # buildkite docker is handled in pipeline setup
return
try:
subprocess.check_output(
["docker-compose", "-f", docker_compose_file, "stop", service_name]
)
subprocess.check_output(
["docker-compose", "-f", docker_compose_file, "rm", "-f", service_name]
)
except subprocess.CalledProcessError:
pass
try:
subprocess.check_output(
["docker-compose", "-f", docker_compose_file, "up", "-d", service_name],
stderr=subprocess.STDOUT, # capture STDERR for error handling
)
except subprocess.CalledProcessError as ex:
err_text = ex.output.decode("utf-8")
raise PostgresDockerError(
"Failed to launch docker container(s) via docker-compose: {}".format(err_text),
ex,
)
conn_str = TestPostgresInstance.conn_string(**conn_args)
wait_for_connection(conn_str, retry_limit=10, retry_wait=3)
yield conn_str
try:
subprocess.check_output(
["docker-compose", "-f", docker_compose_file, "stop", service_name]
)
subprocess.check_output(
["docker-compose", "-f", docker_compose_file, "rm", "-f", service_name]
)
except subprocess.CalledProcessError:
pass
def docker_service_up(docker_compose_file, service_name, conn_args=None):
check.invariant(
TestPostgresInstance.dagster_postgres_installed(),
'dagster_postgres must be installed to test with postgres',
)
check.str_param(service_name, 'service_name')
check.str_param(docker_compose_file, 'docker_compose_file')
check.invariant(os.path.isfile(docker_compose_file),
'docker_compose_file must specify a valid file')
conn_args = check.opt_dict_param(conn_args,
'conn_args') if conn_args else {}
from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error
if BUILDKITE:
yield TestPostgresInstance.conn_string(
**conn_args) # buildkite docker is handled in pipeline setup
return
if not is_postgres_running(service_name):
try:
subprocess.check_output([
'docker-compose', '-f', docker_compose_file, 'stop',
service_name
])
subprocess.check_output([
'docker-compose', '-f', docker_compose_file, 'rm', '-f',
service_name
])
except subprocess.CalledProcessError:
pass
try:
subprocess.check_output(
[
'docker-compose', '-f', docker_compose_file, 'up',
'-d', service_name
],
stderr=subprocess.
STDOUT, # capture STDERR for error handling
)
except subprocess.CalledProcessError as ex:
err_text = ex.output.decode()
raise PostgresDockerError(
'Failed to launch docker container(s) via docker-compose: {}'
.format(err_text),
ex,
)
conn_str = TestPostgresInstance.conn_string(**conn_args)
wait_for_connection(conn_str)
yield conn_str
def postgres():
if BUILDKITE:
yield
return
if not is_postgres_running():
with pushd(script_relative_path('.')):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
wait_for_connection(_conn_string())
yield
def multi_postgres(): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
# It's not clear why none of this works -- we can debug after 0.6.0
# See, maybe, https://success.docker.com/article/multiple-docker-networks for
# further debug strategy
# https://github.com/dagster-io/dagster/issues/1791
# event_log_storage_conn_string = _conn_string(
# hostname=get_hostname('POSTGRES_TEST_EVENT_LOG_STORAGE_DB_HOST'), port='5433'
# )
# run_storage_conn_string = _conn_string(
# hostname=get_hostname('POSTGRES_TEST_RUN_STORAGE_DB_HOST'), port='5434'
# )
# print(subprocess.check_output(['docker', 'ps']))
# wait_for_connection(event_log_storage_conn_string)
# wait_for_connection(run_storage_conn_string)
# yield (run_storage_conn_string, event_log_storage_conn_string)
# return
with pushd(script_relative_path('.')):
try:
subprocess.check_output(
['docker-compose', '-f', 'docker-compose-multi.yml', 'stop'])
subprocess.check_output(
['docker-compose', '-f', 'docker-compose-multi.yml', 'rm'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', '-f', 'docker-compose-multi.yml', 'up', '-d'])
event_log_storage_conn_string = _conn_string(port='5433')
run_storage_conn_string = _conn_string(port='5434')
wait_for_connection(event_log_storage_conn_string)
wait_for_connection(run_storage_conn_string)
yield (run_storage_conn_string, event_log_storage_conn_string)
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
conn_string = get_conn_string(
username="******",
password="******",
hostname=pg_hostname,
db_name="dbt_example",
)
if not BUILDKITE:
script_path = file_relative_path(__file__, ".")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "dbt_example_postgresql"])
subprocess.check_output([
"docker-compose", "rm", "-f", "dbt_example_postgresql"
])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
["docker-compose", "up", "-d", "dbt_example_postgresql"])
wait_for_connection(conn_string)
old_env = None
if os.getenv("DBT_EXAMPLE_CONN_STRING") is not None:
old_env = os.getenv("DBT_EXAMPLE_CONN_STRING")
try:
os.environ["DBT_EXAMPLE_CONN_STRING"] = conn_string
yield
finally:
if old_env is not None:
os.environ["DBT_EXAMPLE_CONN_STRING"] = old_env
else:
del os.environ["DBT_EXAMPLE_CONN_STRING"]
