def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test") return # TODO move airline demo script_path = script_relative_path( "../../../dagster_examples_tests/airline_demo_tests/") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "test-postgres-db"]) subprocess.check_output( ["docker-compose", "rm", "-f", "test-postgres-db"]) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ["docker-compose", "up", "-d", "test-postgres-db"]) conn_str = get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test") wait_for_connection(conn_str) yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield return script_path = script_relative_path(".") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "test-postgres-db-airline"]) subprocess.check_output( ["docker-compose", "rm", "-f", "test-postgres-db-airline"]) except Exception: pass subprocess.check_output( ["docker-compose", "up", "-d", "test-postgres-db-airline"]) wait_for_connection( get_conn_string(username="******", password="******", hostname=pg_hostname, db_name="test")) yield
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test') return # TODO move airline demo script_path = script_relative_path( '../../../dagster_examples_tests/airline_demo_tests/') if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ['docker-compose', 'stop', 'test-postgres-db']) subprocess.check_output( ['docker-compose', 'rm', '-f', 'test-postgres-db']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', 'up', '-d', 'test-postgres-db']) conn_str = get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test') wait_for_connection(conn_str) yield conn_str
def docker_service_up(docker_compose_file, service_name): check.invariant( TestPostgresInstance.dagster_postgres_installed(), 'dagster_postgres must be installed to test with postgres', ) check.str_param(service_name, 'service_name') check.str_param(docker_compose_file, 'docker_compose_file') check.invariant( os.path.isfile(docker_compose_file), 'docker_compose_file must specify a valid file' ) from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error if BUILDKITE: yield TestPostgresInstance.conn_string() # buildkite docker is handled in pipeline setup return if not is_postgres_running(service_name): try: subprocess.check_output( ['docker-compose', '-f', docker_compose_file, 'stop', service_name] ) subprocess.check_output( ['docker-compose', '-f', docker_compose_file, 'rm', '-f', service_name] ) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', '-f', docker_compose_file, 'up', '-d', service_name] ) conn_str = TestPostgresInstance.conn_string() wait_for_connection(conn_str) yield conn_str
def postgres(pg_hostname): # pylint: disable=redefined-outer-name if BUILDKITE: yield return script_path = script_relative_path('.') if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ['docker-compose', 'stop', 'test-postgres-db']) subprocess.check_output( ['docker-compose', 'rm', '-f', 'test-postgres-db']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', 'up', '-d', 'test-postgres-db']) wait_for_connection( get_conn_string(username='******', password='******', hostname=pg_hostname, db_name='test')) yield
def docker_service_up( docker_compose_file, service_name, conn_args=None, env_name="POSTGRES_TEST_DB_HOST" ): check.invariant( TestPostgresInstance.dagster_postgres_installed(), "dagster_postgres must be installed to test with postgres", ) check.str_param(service_name, "service_name") check.str_param(docker_compose_file, "docker_compose_file") check.invariant( os.path.isfile(docker_compose_file), "docker_compose_file must specify a valid file" ) conn_args = check.opt_dict_param(conn_args, "conn_args") if conn_args else {} from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error if BUILDKITE: yield TestPostgresInstance.conn_string( **conn_args, env_name=env_name, ) # buildkite docker is handled in pipeline setup return try: subprocess.check_output( ["docker-compose", "-f", docker_compose_file, "stop", service_name] ) subprocess.check_output( ["docker-compose", "-f", docker_compose_file, "rm", "-f", service_name] ) except subprocess.CalledProcessError: pass try: subprocess.check_output( ["docker-compose", "-f", docker_compose_file, "up", "-d", service_name], stderr=subprocess.STDOUT, # capture STDERR for error handling ) except subprocess.CalledProcessError as ex: err_text = ex.output.decode("utf-8") raise PostgresDockerError( "Failed to launch docker container(s) via docker-compose: {}".format(err_text), ex, ) conn_str = TestPostgresInstance.conn_string(**conn_args) wait_for_connection(conn_str, retry_limit=10, retry_wait=3) yield conn_str try: subprocess.check_output( ["docker-compose", "-f", docker_compose_file, "stop", service_name] ) subprocess.check_output( ["docker-compose", "-f", docker_compose_file, "rm", "-f", service_name] ) except subprocess.CalledProcessError: pass
def docker_service_up(docker_compose_file, service_name, conn_args=None): check.invariant( TestPostgresInstance.dagster_postgres_installed(), 'dagster_postgres must be installed to test with postgres', ) check.str_param(service_name, 'service_name') check.str_param(docker_compose_file, 'docker_compose_file') check.invariant(os.path.isfile(docker_compose_file), 'docker_compose_file must specify a valid file') conn_args = check.opt_dict_param(conn_args, 'conn_args') if conn_args else {} from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error if BUILDKITE: yield TestPostgresInstance.conn_string( **conn_args) # buildkite docker is handled in pipeline setup return if not is_postgres_running(service_name): try: subprocess.check_output([ 'docker-compose', '-f', docker_compose_file, 'stop', service_name ]) subprocess.check_output([ 'docker-compose', '-f', docker_compose_file, 'rm', '-f', service_name ]) except subprocess.CalledProcessError: pass try: subprocess.check_output( [ 'docker-compose', '-f', docker_compose_file, 'up', '-d', service_name ], stderr=subprocess. STDOUT, # capture STDERR for error handling ) except subprocess.CalledProcessError as ex: err_text = ex.output.decode() raise PostgresDockerError( 'Failed to launch docker container(s) via docker-compose: {}' .format(err_text), ex, ) conn_str = TestPostgresInstance.conn_string(**conn_args) wait_for_connection(conn_str) yield conn_str
def postgres(): if BUILDKITE: yield return if not is_postgres_running(): with pushd(script_relative_path('.')): try: subprocess.check_output( ['docker-compose', 'stop', 'test-postgres-db']) subprocess.check_output( ['docker-compose', 'rm', '-f', 'test-postgres-db']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', 'up', '-d', 'test-postgres-db']) wait_for_connection(_conn_string()) yield
def multi_postgres(): # pylint: disable=redefined-outer-name if BUILDKITE: yield return # It's not clear why none of this works -- we can debug after 0.6.0 # See, maybe, https://success.docker.com/article/multiple-docker-networks for # further debug strategy # https://github.com/dagster-io/dagster/issues/1791 # event_log_storage_conn_string = _conn_string( # hostname=get_hostname('POSTGRES_TEST_EVENT_LOG_STORAGE_DB_HOST'), port='5433' # ) # run_storage_conn_string = _conn_string( # hostname=get_hostname('POSTGRES_TEST_RUN_STORAGE_DB_HOST'), port='5434' # ) # print(subprocess.check_output(['docker', 'ps'])) # wait_for_connection(event_log_storage_conn_string) # wait_for_connection(run_storage_conn_string) # yield (run_storage_conn_string, event_log_storage_conn_string) # return with pushd(script_relative_path('.')): try: subprocess.check_output( ['docker-compose', '-f', 'docker-compose-multi.yml', 'stop']) subprocess.check_output( ['docker-compose', '-f', 'docker-compose-multi.yml', 'rm']) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ['docker-compose', '-f', 'docker-compose-multi.yml', 'up', '-d']) event_log_storage_conn_string = _conn_string(port='5433') run_storage_conn_string = _conn_string(port='5434') wait_for_connection(event_log_storage_conn_string) wait_for_connection(run_storage_conn_string) yield (run_storage_conn_string, event_log_storage_conn_string)
def postgres(pg_hostname): # pylint: disable=redefined-outer-name conn_string = get_conn_string( username="******", password="******", hostname=pg_hostname, db_name="dbt_example", ) if not BUILDKITE: script_path = file_relative_path(__file__, ".") if not is_postgres_running(): with pushd(script_path): try: subprocess.check_output( ["docker-compose", "stop", "dbt_example_postgresql"]) subprocess.check_output([ "docker-compose", "rm", "-f", "dbt_example_postgresql" ]) except Exception: # pylint: disable=broad-except pass subprocess.check_output( ["docker-compose", "up", "-d", "dbt_example_postgresql"]) wait_for_connection(conn_string) old_env = None if os.getenv("DBT_EXAMPLE_CONN_STRING") is not None: old_env = os.getenv("DBT_EXAMPLE_CONN_STRING") try: os.environ["DBT_EXAMPLE_CONN_STRING"] = conn_string yield finally: if old_env is not None: os.environ["DBT_EXAMPLE_CONN_STRING"] = old_env else: del os.environ["DBT_EXAMPLE_CONN_STRING"]