def postgres_extra_cmds_fn(_):
return [
"pushd python_modules/libraries/dagster-postgres/dagster_postgres_tests/",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
"docker-compose -f docker-compose-multi.yml up -d", # clean up in hooks/pre-exit,
network_buildkite_container('postgres'),
connect_sibling_docker_container('postgres', 'test-postgres-db',
'POSTGRES_TEST_DB_HOST'),
network_buildkite_container('postgres_multi'),
connect_sibling_docker_container(
'postgres_multi',
'test-run-storage-db',
'POSTGRES_TEST_RUN_STORAGE_DB_HOST',
),
connect_sibling_docker_container(
'postgres_multi',
'test-event-log-storage-db',
'POSTGRES_TEST_EVENT_LOG_STORAGE_DB_HOST',
),
connect_sibling_docker_container(
'postgres_multi',
'test-schedule-storage-db',
'POSTGRES_TEST_SCHEDULE_STORAGE_DB_HOST',
),
"popd",
]
def celery_docker_extra_cmds_fn(version):
return celery_extra_cmds_fn(version) + [
"pushd python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/",
"docker-compose up -d --remove-orphans",
network_buildkite_container("postgres"),
connect_sibling_docker_container(
"postgres", "test-postgres-db-celery-docker", "POSTGRES_TEST_DB_HOST",
),
"popd",
]
def legacy_examples_extra_cmds_fn(_):
return [
"pushd examples/legacy_examples",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("postgres"),
connect_sibling_docker_container("postgres", "test-postgres-db", "POSTGRES_TEST_DB_HOST"),
"popd",
]
def dask_extra_cmds_fn(version):
return [
"pushd python_modules/libraries/dagster-dask/dagster_dask_tests/dask-docker",
"./build.sh " + version,
# Run the docker-compose dask cluster
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit
network_buildkite_container('dask'),
connect_sibling_docker_container('dask', 'dask-scheduler', 'DASK_ADDRESS'),
"popd",
]
def dbt_extra_cmds_fn(_):
return [
"pushd python_modules/libraries/dagster-dbt/dagster_dbt_tests",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("postgres"),
connect_sibling_docker_container("postgres", "test-postgres-db-dbt",
"POSTGRES_TEST_DB_DBT_HOST"),
"popd",
]
def deploy_docker_example_extra_cmds_fn(_):
return [
"pushd examples/deploy_docker/from_source",
"./build.sh",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit
network_buildkite_container("docker_example_network"),
connect_sibling_docker_container(
"docker_example_network", "docker_example_dagit", "DEPLOY_DOCKER_DAGIT_HOST",
),
"popd",
]
def graphql_pg_extra_cmds_fn(_):
return [
"pushd python_modules/dagster-graphql/dagster_graphql_tests/graphql/",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container('postgres'),
connect_sibling_docker_container('postgres',
'test-postgres-db-graphql',
'POSTGRES_TEST_DB_HOST'),
"popd",
]
def docker_extra_cmds_fn(version):
return [
"export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"',
"pushd python_modules/libraries/dagster-docker/dagster_docker_tests/",
"docker-compose up -d --remove-orphans",
network_buildkite_container("postgres"),
connect_sibling_docker_container(
"postgres", "test-postgres-db-docker", "POSTGRES_TEST_DB_HOST",
),
"popd",
]
def dagster_extra_cmds_fn(version):
return [
"export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"',
"aws ecr get-login --no-include-email --region us-west-1 | sh",
"export IMAGE_NAME=$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com/dagster-core-docker-buildkite:$${BUILDKITE_BUILD_ID}-"
+ version,
"pushd python_modules/dagster/dagster_tests",
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit
network_buildkite_container("dagster"),
connect_sibling_docker_container("dagster", "dagster-grpc-server", "GRPC_SERVER_HOST"),
"popd",
]
def airline_demo_extra_cmds_fn(_):
return [
"pushd examples/airline_demo",
# Run the postgres db. We are in docker running docker
# so this will be a sibling container.
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("postgres"),
connect_sibling_docker_container("postgres",
"test-postgres-db-airline",
"POSTGRES_TEST_DB_HOST"),
"popd",
]
def celery_extra_cmds_fn(version):
return [
"export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"',
"pushd python_modules/libraries/dagster-celery",
# Run the rabbitmq db. We are in docker running docker
# so this will be a sibling container.
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("rabbitmq"),
connect_sibling_docker_container("rabbitmq", "test-rabbitmq", "DAGSTER_CELERY_BROKER_HOST"),
"popd",
]
def airline_demo_extra_cmds_fn(_):
return [
'pushd examples/legacy_examples',
# Build the image we use for airflow in the demo tests
'./build_airline_demo_image.sh',
'mkdir -p /home/circleci/airflow',
# Run the postgres db. We are in docker running docker
# so this will be a sibling container.
'docker-compose up -d --remove-orphans', # clean up in hooks/pre-exit
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container('postgres'),
connect_sibling_docker_container('postgres', 'test-postgres-db', 'POSTGRES_TEST_DB_HOST'),
'popd',
]
def dbt_example_extra_cmds_fn(_):
return [
"pushd examples/dbt_example",
# Run the postgres db. We are in docker running docker
# so this will be a sibling container.
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("postgres"),
connect_sibling_docker_container("postgres", "dbt_example_postgresql",
"DAGSTER_DBT_EXAMPLE_PGHOST"),
"mkdir -p ~/.dbt/",
"touch ~/.dbt/profiles.yml",
"cat dbt_example_project/profiles.yml >> ~/.dbt/profiles.yml",
"popd",
]
def airflow_extra_cmds_fn(version):
return [
'export AIRFLOW_HOME="/airflow"',
"mkdir -p $${AIRFLOW_HOME}",
"export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"',
"aws ecr get-login --no-include-email --region us-west-1 | sh",
r"aws s3 cp s3://\${BUILDKITE_SECRETS_BUCKET}/gcp-key-elementl-dev.json "
+ GCP_CREDS_LOCAL_FILE,
"export GOOGLE_APPLICATION_CREDENTIALS=" + GCP_CREDS_LOCAL_FILE,
"pushd python_modules/libraries/dagster-airflow/dagster_airflow_tests/",
"docker-compose up -d --remove-orphans",
network_buildkite_container("postgres"),
connect_sibling_docker_container(
"postgres", "test-postgres-db-airflow", "POSTGRES_TEST_DB_HOST",
),
"popd",
]
def integration_suite_extra_cmds_fn(version):
return [
'export AIRFLOW_HOME="/airflow"',
"mkdir -p $${AIRFLOW_HOME}",
"export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version,
'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-1.amazonaws.com"',
"aws ecr get-login --no-include-email --region us-west-1 | sh",
r"aws s3 cp s3://\${BUILDKITE_SECRETS_BUCKET}/gcp-key-elementl-dev.json "
+ GCP_CREDS_LOCAL_FILE,
"export GOOGLE_APPLICATION_CREDENTIALS=" + GCP_CREDS_LOCAL_FILE,
"pushd python_modules/libraries/dagster-celery",
# Run the rabbitmq db. We are in docker running docker
# so this will be a sibling container.
"docker-compose up -d --remove-orphans", # clean up in hooks/pre-exit,
# Can't use host networking on buildkite and communicate via localhost
# between these sibling containers, so pass along the ip.
network_buildkite_container("rabbitmq"),
connect_sibling_docker_container("rabbitmq", "test-rabbitmq", "DAGSTER_CELERY_BROKER_HOST"),
"popd",
]
