def postgres_extra_cmds_fn(_): return [ "pushd python_modules/libraries/dagster-postgres/dagster_postgres_tests/", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, "docker-compose -f docker-compose-multi.yml up -d", # clean up in hooks/pre-exit, network_buildkite_container('postgres'), connect_sibling_docker_container('postgres', 'test-postgres-db', 'POSTGRES_TEST_DB_HOST'), network_buildkite_container('postgres_multi'), connect_sibling_docker_container( 'postgres_multi', 'test-run-storage-db', 'POSTGRES_TEST_RUN_STORAGE_DB_HOST', ), connect_sibling_docker_container( 'postgres_multi', 'test-event-log-storage-db', 'POSTGRES_TEST_EVENT_LOG_STORAGE_DB_HOST', ), connect_sibling_docker_container( 'postgres_multi', 'test-schedule-storage-db', 'POSTGRES_TEST_SCHEDULE_STORAGE_DB_HOST', ), "popd", ]
def celery_docker_extra_cmds_fn(version): return celery_extra_cmds_fn(version) + [ "pushd python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/", "docker-compose up -d --remove-orphans", network_buildkite_container("postgres"), connect_sibling_docker_container( "postgres", "test-postgres-db-celery-docker", "POSTGRES_TEST_DB_HOST", ), "popd", ]
def legacy_examples_extra_cmds_fn(_): return [ "pushd examples/legacy_examples", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("postgres"), connect_sibling_docker_container("postgres", "test-postgres-db", "POSTGRES_TEST_DB_HOST"), "popd", ]
def dask_extra_cmds_fn(version): return [ "pushd python_modules/libraries/dagster-dask/dagster_dask_tests/dask-docker", "./build.sh " + version, # Run the docker-compose dask cluster "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit network_buildkite_container('dask'), connect_sibling_docker_container('dask', 'dask-scheduler', 'DASK_ADDRESS'), "popd", ]
def dbt_extra_cmds_fn(_): return [ "pushd python_modules/libraries/dagster-dbt/dagster_dbt_tests", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("postgres"), connect_sibling_docker_container("postgres", "test-postgres-db-dbt", "POSTGRES_TEST_DB_DBT_HOST"), "popd", ]
def deploy_docker_example_extra_cmds_fn(_): return [ "pushd examples/deploy_docker/from_source", "./build.sh", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit network_buildkite_container("docker_example_network"), connect_sibling_docker_container( "docker_example_network", "docker_example_dagit", "DEPLOY_DOCKER_DAGIT_HOST", ), "popd", ]
def graphql_pg_extra_cmds_fn(_): return [ "pushd python_modules/dagster-graphql/dagster_graphql_tests/graphql/", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container('postgres'), connect_sibling_docker_container('postgres', 'test-postgres-db-graphql', 'POSTGRES_TEST_DB_HOST'), "popd", ]
def docker_extra_cmds_fn(version): return [ "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"', "pushd python_modules/libraries/dagster-docker/dagster_docker_tests/", "docker-compose up -d --remove-orphans", network_buildkite_container("postgres"), connect_sibling_docker_container( "postgres", "test-postgres-db-docker", "POSTGRES_TEST_DB_HOST", ), "popd", ]
def dagster_extra_cmds_fn(version): return [ "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"', "aws ecr get-login --no-include-email --region us-west-1 | sh", "export IMAGE_NAME=$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com/dagster-core-docker-buildkite:$${BUILDKITE_BUILD_ID}-" + version, "pushd python_modules/dagster/dagster_tests", "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit network_buildkite_container("dagster"), connect_sibling_docker_container("dagster", "dagster-grpc-server", "GRPC_SERVER_HOST"), "popd", ]
def airline_demo_extra_cmds_fn(_): return [ "pushd examples/airline_demo", # Run the postgres db. We are in docker running docker # so this will be a sibling container. "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("postgres"), connect_sibling_docker_container("postgres", "test-postgres-db-airline", "POSTGRES_TEST_DB_HOST"), "popd", ]
def celery_extra_cmds_fn(version): return [ "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"', "pushd python_modules/libraries/dagster-celery", # Run the rabbitmq db. We are in docker running docker # so this will be a sibling container. "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("rabbitmq"), connect_sibling_docker_container("rabbitmq", "test-rabbitmq", "DAGSTER_CELERY_BROKER_HOST"), "popd", ]
def airline_demo_extra_cmds_fn(_): return [ 'pushd examples/legacy_examples', # Build the image we use for airflow in the demo tests './build_airline_demo_image.sh', 'mkdir -p /home/circleci/airflow', # Run the postgres db. We are in docker running docker # so this will be a sibling container. 'docker-compose up -d --remove-orphans', # clean up in hooks/pre-exit # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container('postgres'), connect_sibling_docker_container('postgres', 'test-postgres-db', 'POSTGRES_TEST_DB_HOST'), 'popd', ]
def dbt_example_extra_cmds_fn(_): return [ "pushd examples/dbt_example", # Run the postgres db. We are in docker running docker # so this will be a sibling container. "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("postgres"), connect_sibling_docker_container("postgres", "dbt_example_postgresql", "DAGSTER_DBT_EXAMPLE_PGHOST"), "mkdir -p ~/.dbt/", "touch ~/.dbt/profiles.yml", "cat dbt_example_project/profiles.yml >> ~/.dbt/profiles.yml", "popd", ]
def airflow_extra_cmds_fn(version): return [ 'export AIRFLOW_HOME="/airflow"', "mkdir -p $${AIRFLOW_HOME}", "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"', "aws ecr get-login --no-include-email --region us-west-1 | sh", r"aws s3 cp s3://\${BUILDKITE_SECRETS_BUCKET}/gcp-key-elementl-dev.json " + GCP_CREDS_LOCAL_FILE, "export GOOGLE_APPLICATION_CREDENTIALS=" + GCP_CREDS_LOCAL_FILE, "pushd python_modules/libraries/dagster-airflow/dagster_airflow_tests/", "docker-compose up -d --remove-orphans", network_buildkite_container("postgres"), connect_sibling_docker_container( "postgres", "test-postgres-db-airflow", "POSTGRES_TEST_DB_HOST", ), "popd", ]
def integration_suite_extra_cmds_fn(version): return [ 'export AIRFLOW_HOME="/airflow"', "mkdir -p $${AIRFLOW_HOME}", "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"', "aws ecr get-login --no-include-email --region us-west-1 | sh", r"aws s3 cp s3://\${BUILDKITE_SECRETS_BUCKET}/gcp-key-elementl-dev.json " + GCP_CREDS_LOCAL_FILE, "export GOOGLE_APPLICATION_CREDENTIALS=" + GCP_CREDS_LOCAL_FILE, "pushd python_modules/libraries/dagster-celery", # Run the rabbitmq db. We are in docker running docker # so this will be a sibling container. "docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit, # Can't use host networking on buildkite and communicate via localhost # between these sibling containers, so pass along the ip. network_buildkite_container("rabbitmq"), connect_sibling_docker_container("rabbitmq", "test-rabbitmq", "DAGSTER_CELERY_BROKER_HOST"), "popd", ]