def test_execute_celery_docker(): docker_image = test_project_docker_image() docker_config = { "image": docker_image, "env_vars": ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], } if IS_BUILDKITE: ecr_client = boto3.client("ecr", region_name="us-west-1") token = ecr_client.get_authorization_token() username, password = (base64.b64decode( token["authorizationData"][0] ["authorizationToken"]).decode().split(":")) registry = token["authorizationData"][0]["proxyEndpoint"] docker_config["registry"] = { "url": registry, "username": username, "password": password, } else: try: client = docker.from_env() client.images.get(docker_image) print( # pylint: disable=print-call "Found existing image tagged {image}, skipping image build. To rebuild, first run: " "docker rmi {image}".format(image=docker_image)) except docker.errors.ImageNotFound: build_and_tag_test_image(docker_image) with seven.TemporaryDirectory() as temp_dir: run_config = merge_dicts( merge_yamls([ os.path.join(test_project_environments_path(), "env.yaml"), os.path.join(test_project_environments_path(), "env_s3.yaml"), ]), { "execution": { "celery-docker": { "config": { "docker": docker_config, "config_source": { "task_always_eager": True }, } } }, }, ) result = execute_pipeline( get_test_project_recon_pipeline("docker_celery_pipeline"), run_config=run_config, instance=DagsterInstance.local_temp(temp_dir), ) assert result.success
def test_execute_celery_docker(): docker_image = test_project_docker_image() docker_config = { 'image': docker_image, 'env_vars': ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'], } if IS_BUILDKITE: ecr_client = boto3.client('ecr', region_name='us-west-1') token = ecr_client.get_authorization_token() username, password = (base64.b64decode( token['authorizationData'][0] ['authorizationToken']).decode().split(':')) registry = token['authorizationData'][0]['proxyEndpoint'] docker_config['registry'] = { 'url': registry, 'username': username, 'password': password, } else: try: client = docker.from_env() client.images.get(docker_image) print( # pylint: disable=print-call 'Found existing image tagged {image}, skipping image build. To rebuild, first run: ' 'docker rmi {image}'.format(image=docker_image)) except docker.errors.ImageNotFound: build_and_tag_test_image(docker_image) with seven.TemporaryDirectory() as temp_dir: run_config = merge_dicts( merge_yamls([ os.path.join(test_project_environments_path(), 'env.yaml'), os.path.join(test_project_environments_path(), 'env_s3.yaml'), ]), { 'execution': { 'celery-docker': { 'config': { 'docker': docker_config, 'config_source': { 'task_always_eager': True }, } } }, }, ) result = execute_pipeline( get_test_project_recon_pipeline('docker_celery_pipeline'), run_config=run_config, instance=DagsterInstance.local_temp(temp_dir), ) assert result.success
def _cluster_provider(request): from .kind import kind_cluster_exists, kind_cluster, kind_load_images if IS_BUILDKITE: print("Installing ECR credentials...") check_output( "aws ecr get-login --no-include-email --region us-west-2 | sh", shell=True) provider = request.config.getoption("--cluster-provider") # Use a kind cluster if provider == "kind": cluster_name = request.config.getoption("--kind-cluster") # Cluster will be deleted afterwards unless this is set. # This is to allow users to reuse an existing cluster in local test by running # `pytest --kind-cluster my-cluster --no-cleanup` -- this avoids the per-test run # overhead of cluster setup and teardown should_cleanup = True if IS_BUILDKITE else not request.config.getoption( "--no-cleanup") existing_cluster = kind_cluster_exists(cluster_name) with kind_cluster(cluster_name, should_cleanup=should_cleanup) as cluster_config: if not IS_BUILDKITE and not existing_cluster: docker_image = get_test_project_docker_image() try: client = docker.from_env() client.images.get(docker_image) print( # pylint: disable=print-call "Found existing image tagged {image}, skipping image build. To rebuild, first run: " "docker rmi {image}".format(image=docker_image)) except docker.errors.ImageNotFound: build_and_tag_test_image(docker_image) kind_load_images( cluster_name=cluster_config.name, local_dagster_test_image=docker_image, additional_images=additional_kind_images, ) yield cluster_config # Use cluster from kubeconfig elif provider == "kubeconfig": kubeconfig_file = os.getenv( "KUBECONFIG", os.path.expandvars("${HOME}/.kube/config")) kubernetes.config.load_kube_config(config_file=kubeconfig_file) yield ClusterConfig(name="from_system_kubeconfig", kubeconfig_file=kubeconfig_file) else: raise Exception("unknown cluster provider %s" % provider)
def dagster_docker_image(): docker_image = test_project_docker_image() if not IS_BUILDKITE: try: client = docker.from_env() client.images.get(docker_image) print( # pylint: disable=print-call "Found existing image tagged {image}, skipping image build. To rebuild, first run: " "docker rmi {image}".format(image=docker_image)) except docker.errors.ImageNotFound: build_and_tag_test_image(docker_image) return docker_image
def _cluster_provider(request): from .kind import kind_cluster_exists, kind_cluster, kind_load_images if IS_BUILDKITE: print('Installing ECR credentials...') check_output( 'aws ecr get-login --no-include-email --region us-west-1 | sh', shell=True) provider = request.config.getoption('--cluster-provider') # Use a kind cluster if provider == 'kind': cluster_name = request.config.getoption('--kind-cluster') # Cluster will be deleted afterwards unless this is set. # This is to allow users to reuse an existing cluster in local test by running # `pytest --kind-cluster my-cluster --no-cleanup` -- this avoids the per-test run # overhead of cluster setup and teardown should_cleanup = True if IS_BUILDKITE else not request.config.getoption( '--no-cleanup') existing_cluster = kind_cluster_exists(cluster_name) with kind_cluster(cluster_name, should_cleanup=should_cleanup) as cluster_config: if not IS_BUILDKITE and not existing_cluster: docker_image = test_project_docker_image() build_and_tag_test_image(docker_image) kind_load_images( cluster_name=cluster_config.name, local_dagster_test_image=docker_image, additional_images=additional_kind_images, ) yield cluster_config # Use cluster from kubeconfig elif provider == 'kubeconfig': kubeconfig_file = os.getenv( 'KUBECONFIG', os.path.expandvars('${HOME}/.kube/config')) kubernetes.config.load_kube_config(config_file=kubeconfig_file) yield ClusterConfig(name='from_system_kubeconfig', kubeconfig_file=kubeconfig_file) else: raise Exception('unknown cluster provider %s' % provider)