def test_k8s_run_launcher_default(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image):
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace":
helm_namespace_for_k8s_run_launcher,
"job_image":
dagster_docker_image,
"image_pull_policy":
image_pull_policy(),
"env_config_maps": ["dagster-pipeline-env"] +
([TEST_AWS_CONFIGMAP_NAME]
if not IS_BUILDKITE else []),
}
}
},
},
)
_launch_executor_run(
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
)
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
_launch_executor_run(
dagit_url_for_k8s_run_launcher,
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
)
def test_k8s_executor_get_config_from_run_launcher(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
# Verify that if you do not specify executor config it is delegated by the run launcher
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_image": dagster_docker_image
}
}
},
},
)
_launch_executor_run(
dagit_url_for_k8s_run_launcher,
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
)
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher, dagster_docker_image
):
# sanity check that we have a K8sRunLauncher
check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher)
pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher
)
celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
"env_config_maps": ["dagster-pipeline-env"]
+ ([TEST_AWS_CONFIGMAP_NAME] if not IS_BUILDKITE else []),
}
}
},
},
)
pipeline_name = "demo_k8s_executor_pipeline"
tags = {"key": "value"}
with get_test_project_location_and_external_pipeline(pipeline_name) as (
location,
external_pipeline,
):
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=location.get_external_execution_plan(
external_pipeline, run_config, "default", None, None
).execution_plan_snapshot,
)
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(external_pipeline),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run.run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image()
def test_k8s_run_launcher_image_from_origin(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
# Like the previous test, but the executor doesn't supply an image - it's pulled
# from the origin on the run instead
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml")),
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "demo_k8s_executor_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_k8s_run_monitoring(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
_launch_run_and_wait_for_resume(
dagit_url_for_k8s_run_launcher,
run_config,
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
)
def test_valid_job_format(kubeconfig, docker_image, image_pull_policy): # pylint: disable=redefined-outer-name
run_id = uuid.uuid4().hex
environment_dict = load_yaml_from_path(
os.path.join(environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun.create_empty_run(pipeline_name, run_id, environment_dict)
run_launcher = K8sRunLauncher(
image_pull_policy=image_pull_policy,
image_pull_secrets=[{
'name': 'element-dev-key'
}],
service_account_name='dagit-admin',
instance_config_map='dagster-instance',
job_image=docker_image,
load_kubeconfig=True,
kubeconfig_file=kubeconfig,
)
job = run_launcher.construct_job(run)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy,
dagster_version=dagster_version,
).strip())
def test_failing_k8s_run_launcher(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "always_fail_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" not in result, "no match, result: {}".format(
result)
event_records = dagster_instance_for_k8s_run_launcher.all_logs(run_id)
assert any(
["Op Exception Message" in str(event) for event in event_records])
def load_workspace_from_yaml_paths(yaml_paths, python_user_process_api):
check.list_param(yaml_paths, "yaml_paths", str)
check.inst_param(python_user_process_api, "python_user_process_api", UserProcessApi)
workspace_configs = [load_yaml_from_path(yaml_path) for yaml_path in yaml_paths]
workspaces = []
for workspace_config, yaml_path in zip(workspace_configs, yaml_paths):
check.invariant(
workspace_config is not None,
(
"Could not parse a workspace config from the yaml file at {yaml_path}. Check that "
"the file contains valid yaml."
).format(yaml_path=os.path.abspath(yaml_path)),
)
workspaces.append(
load_workspace_from_config(workspace_config, yaml_path, python_user_process_api)
)
repository_location_handles_dict = {}
for workspace in workspaces:
for repository_location_name in workspace.repository_location_names:
repository_location_handles_dict[
repository_location_name
] = workspace.get_repository_location_handle(repository_location_name)
repository_location_handles = list(repository_location_handles_dict.values())
merged_workspace = Workspace(repository_location_handles)
return merged_workspace
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagit_url_for_k8s_run_launcher,
):
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run_id = launch_run_over_graphql(dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_valid_job_format(run_launcher):
docker_image = test_project_docker_image()
environment_dict = load_yaml_from_path(
os.path.join(test_project_environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun(pipeline_name=pipeline_name,
environment_dict=environment_dict)
job_name = 'dagster-run-%s' % run.run_id
pod_name = 'dagster-run-%s' % run.run_id
job = construct_dagster_graphql_k8s_job(
run_launcher.job_config,
args=[
'-p',
'startPipelineExecutionForCreatedRun',
'-v',
seven.json.dumps({'runId': run.run_id}),
],
job_name=job_name,
pod_name=pod_name,
component='runmaster',
)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run.run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy(),
dagster_version=dagster_version,
).strip())
def test_valid_job_format(run_launcher):
docker_image = test_project_docker_image()
run_config = load_yaml_from_path(
os.path.join(test_project_environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun(pipeline_name=pipeline_name, run_config=run_config)
job_name = 'dagster-run-%s' % run.run_id
pod_name = 'dagster-run-%s' % run.run_id
job = construct_dagster_k8s_job(
job_config=run_launcher.job_config,
command=['dagster-graphql'],
args=[
'-p',
'executeRunInProcess',
'-v',
seven.json.dumps({'runId': run.run_id}),
],
job_name=job_name,
pod_name=pod_name,
component='run_coordinator',
)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run.run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy(),
dagster_version=dagster_version,
resources='',
).strip())
def test_valid_job_format(run_launcher):
docker_image = test_project_docker_image()
run_config = load_yaml_from_path(
os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run = PipelineRun(pipeline_name=pipeline_name, run_config=run_config)
job_name = "dagster-run-%s" % run.run_id
pod_name = "dagster-run-%s" % run.run_id
job = construct_dagster_k8s_job(
job_config=run_launcher.job_config,
command=["dagster"],
args=["api", "execute_run_with_structured_logs"],
job_name=job_name,
pod_name=pod_name,
component="run_coordinator",
)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run.run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy(),
dagster_version=dagster_version,
resources="",
).strip())
def test_k8s_run_launcher_default(
dagster_instance_for_k8s_run_launcher, helm_namespace_for_k8s_run_launcher
):
# sanity check that we have a K8sRunLauncher
check.inst(dagster_instance_for_k8s_run_launcher.run_launcher, K8sRunLauncher)
pods = DagsterKubernetesClient.production_client().core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher
)
celery_pod_names = [p.metadata.name for p in pods.items if "celery-workers" in p.metadata.name]
check.invariant(not celery_pod_names)
run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
tags = {"key": "value"}
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
tags=tags,
mode="default",
)
dagster_instance_for_k8s_run_launcher.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
def load_workspace_from_yaml_paths(yaml_paths, python_user_process_api):
check.list_param(yaml_paths, 'yaml_paths', str)
check.inst_param(python_user_process_api, 'python_user_process_api',
UserProcessApi)
workspace_configs = [
load_yaml_from_path(yaml_path) for yaml_path in yaml_paths
]
workspaces = [
load_workspace_from_config(workspace_config, yaml_path,
python_user_process_api)
for workspace_config, yaml_path in zip(workspace_configs, yaml_paths)
]
repository_location_handles_dict = {}
for workspace in workspaces:
for repository_location_name in workspace.repository_location_names:
repository_location_handles_dict[
repository_location_name] = workspace.get_repository_location_handle(
repository_location_name)
repository_location_handles = list(
repository_location_handles_dict.values())
merged_workspace = Workspace(repository_location_handles)
return merged_workspace
def test_k8s_run_launcher(dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher):
run_config = load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
with get_test_project_workspace_and_external_pipeline(
dagster_instance_for_k8s_run_launcher, pipeline_name) as (
workspace,
external_pipeline,
):
reoriginated_pipeline = ReOriginatedExternalPipelineForTest(
external_pipeline)
run = create_run_for_test(
dagster_instance_for_k8s_run_launcher,
pipeline_name=pipeline_name,
run_config=run_config,
mode="default",
external_pipeline_origin=reoriginated_pipeline.get_external_origin(
),
pipeline_code_origin=reoriginated_pipeline.get_python_origin(),
)
dagster_instance_for_k8s_run_launcher.launch_run(run.run_id, workspace)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(
result)
def from_yaml(cls, file_path):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
scheduler_pointer, partitions_pointer = _handle_backcompat_pointers(
config, file_path)
if module_name:
pointer = ModuleCodePointer(module_name, fn_name)
else:
# rebase file in config off of the path in the config file
file_name = os.path.join(
os.path.dirname(os.path.abspath(file_path)), file_name)
pointer = FileCodePointer(file_name, fn_name)
return cls(
pointer=pointer,
yaml_path=file_path,
scheduler_pointer=scheduler_pointer,
partitions_pointer=partitions_pointer,
)
def test_k8s_run_launcher_terminate(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
pipeline_name = "slow_pipeline"
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
wait_for_job(job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher)
timeout = datetime.timedelta(0, 30)
start_time = datetime.datetime.now()
while True:
assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for can_terminate"
if can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id):
break
time.sleep(5)
terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id=run_id)
start_time = datetime.datetime.now()
pipeline_run = None
while True:
assert datetime.datetime.now() < start_time + timeout, "Timed out waiting for termination"
pipeline_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
if pipeline_run.status == PipelineRunStatus.CANCELED:
break
time.sleep(5)
# useful to have logs here, because the worker pods get deleted
print(dagster_instance_for_k8s_run_launcher.all_logs(run_id)) # pylint: disable=print-call
assert pipeline_run.status == PipelineRunStatus.CANCELED
assert not can_terminate_run_over_graphql(dagit_url_for_k8s_run_launcher, run_id)
def test_execute_on_k8s_retry_pipeline( # pylint: disable=redefined-outer-name
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
run_config = merge_dicts(
load_yaml_from_path(os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "retry_pipeline"
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id, namespace=helm_namespace_for_k8s_run_launcher
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
stats = dagster_instance_for_k8s_run_launcher.get_run_stats(run_id)
assert stats.steps_succeeded == 1
all_logs = dagster_instance_for_k8s_run_launcher.all_logs(run_id)
assert DagsterEventType.STEP_START in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_UP_FOR_RETRY in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_RESTARTED in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
assert DagsterEventType.STEP_SUCCESS in [
event.dagster_event.event_type for event in all_logs if event.is_dagster_event
]
def test_valid_job_format_with_backcompat_resources(run_launcher):
docker_image = test_project_docker_image()
run_config = load_yaml_from_path(
os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run = PipelineRun(pipeline_name=pipeline_name, run_config=run_config)
tags = validate_tags({
K8S_RESOURCE_REQUIREMENTS_KEY: ({
"requests": {
"cpu": "250m",
"memory": "64Mi"
},
"limits": {
"cpu": "500m",
"memory": "2560Mi"
},
})
})
user_defined_k8s_config = get_user_defined_k8s_config(tags)
job_name = "dagster-run-%s" % run.run_id
pod_name = "dagster-run-%s" % run.run_id
job = construct_dagster_k8s_job(
job_config=run_launcher.job_config,
command=["dagster-graphql"],
args=[
"-p",
"executeRunInProcess",
"-v",
seven.json.dumps({"runId": run.run_id}),
],
job_name=job_name,
user_defined_k8s_config=user_defined_k8s_config,
pod_name=pod_name,
component="run_coordinator",
)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run.run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy(),
dagster_version=dagster_version,
resources="""
resources:
limits:
cpu: 500m
memory: 2560Mi
requests:
cpu: 250m
memory: 64Mi""",
).strip())
def test_valid_job_format_with_resources(run_launcher):
docker_image = test_project_docker_image()
run_config = load_yaml_from_path(
os.path.join(test_project_environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun(pipeline_name=pipeline_name, run_config=run_config)
tags = validate_tags({
K8S_RESOURCE_REQUIREMENTS_KEY: ({
'requests': {
'cpu': '250m',
'memory': '64Mi'
},
'limits': {
'cpu': '500m',
'memory': '2560Mi'
},
})
})
resources = get_k8s_resource_requirements(tags)
job_name = 'dagster-run-%s' % run.run_id
pod_name = 'dagster-run-%s' % run.run_id
job = construct_dagster_k8s_job(
job_config=run_launcher.job_config,
command=['dagster-graphql'],
args=[
'-p',
'executeRunInProcess',
'-v',
seven.json.dumps({'runId': run.run_id}),
],
job_name=job_name,
resources=resources,
pod_name=pod_name,
component='runmaster',
)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run.run_id,
job_image=docker_image,
image_pull_policy=image_pull_policy(),
dagster_version=dagster_version,
resources='''
resources:
limits:
cpu: 500m
memory: 2560Mi
requests:
cpu: 250m
memory: 64Mi''',
).strip())
def get_pipelines():
pipelines = {}
for filename in os.listdir(file_relative_path(__file__, PIPELINES_DIR)):
if filename.endswith('.yml') or filename.endswith('.yaml'):
yaml_config = load_yaml_from_path(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}'))
pipeline_name = yaml_config['name']
pipelines[pipeline_name] = partial(define_pipeline_from_yaml, yaml_config)
elif filename.endswith('.py') and filename != '__init__.py':
targets = loadable_targets_from_python_file(file_relative_path(__file__, f'{PIPELINES_DIR}/{filename}'))
for target in targets:
pipelines[target.attribute] = target.target_definition
return pipelines
def from_legacy_repository_yaml(file_path):
check.str_param(file_path, "file_path")
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, "repository")
module_name = check.opt_str_elem(repository_config, "module")
file_name = check.opt_str_elem(repository_config, "file")
fn_name = check.str_elem(repository_config, "fn")
return (CodePointer.from_module(module_name, fn_name) if module_name
# rebase file in config off of the path in the config file
else CodePointer.from_python_file(
rebase_file(file_name, file_path), fn_name, None))
def test_valid_job_format(run_launcher, docker_image, environments_path): # pylint: disable=redefined-outer-name
run_id = uuid.uuid4().hex
environment_dict = load_yaml_from_path(
os.path.join(environments_path, 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun.create_empty_run(pipeline_name, run_id, environment_dict)
job = run_launcher.construct_job(run)
assert (yaml.dump(
remove_none_recursively(job.to_dict()),
default_flow_style=False).strip() == EXPECTED_JOB_SPEC.format(
run_id=run_id, job_image=docker_image).strip())
def do_execute_command(pipeline, env, printer):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_str_param(env, 'env')
check.callable_param(printer, 'printer')
if env:
env_config = load_yaml_from_path(env)
environment = config.construct_environment(env_config)
else:
environment = config.Environment()
pipeline_iter = execute_pipeline_iterator(pipeline, environment)
process_results_for_console(pipeline_iter)
def test_k8s_executor_resource_requirements(
dagster_instance_for_k8s_run_launcher,
helm_namespace_for_k8s_run_launcher,
dagster_docker_image,
dagit_url_for_k8s_run_launcher,
):
# sanity check that we have a K8sRunLauncher
pods = DagsterKubernetesClient.production_client(
).core_api.list_namespaced_pod(
namespace=helm_namespace_for_k8s_run_launcher)
celery_pod_names = [
p.metadata.name for p in pods.items
if "celery-workers" in p.metadata.name
]
check.invariant(not celery_pod_names)
run_config = merge_dicts(
load_yaml_from_path(
os.path.join(get_test_project_environments_path(), "env_s3.yaml")),
{
"execution": {
"k8s": {
"config": {
"job_namespace": helm_namespace_for_k8s_run_launcher,
"job_image": dagster_docker_image,
"image_pull_policy": image_pull_policy(),
}
}
},
},
)
pipeline_name = "resources_limit_pipeline"
run_id = launch_run_over_graphql(
dagit_url_for_k8s_run_launcher,
run_config=run_config,
pipeline_name=pipeline_name,
mode="k8s",
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run_id,
namespace=helm_namespace_for_k8s_run_launcher)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
updated_run = dagster_instance_for_k8s_run_launcher.get_run_by_id(run_id)
assert updated_run.tags[DOCKER_IMAGE_TAG] == get_test_project_docker_image(
)
def test_k8s_run_launcher(dagster_instance): # pylint: disable=redefined-outer-name
run_id = uuid.uuid4().hex
environment_dict = load_yaml_from_path(os.path.join(environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = PipelineRun.create_empty_run(pipeline_name, run_id, environment_dict)
dagster_instance.launch_run(run)
success, raw_logs = wait_for_job_success('dagster-job-%s' % run_id)
result = parse_raw_res(raw_logs.split('\n'))
assert success
assert not result.get('errors')
assert result['data']
assert result['data']['startPipelineExecution']['__typename'] == 'StartPipelineExecutionSuccess'
def from_yaml(file_path, from_handle=None):
check.str_param(file_path, 'file_path')
config = load_yaml_from_path(file_path)
repository_config = check.dict_elem(config, 'repository')
module_name = check.opt_str_elem(repository_config, 'module')
file_name = check.opt_str_elem(repository_config, 'file')
fn_name = check.str_elem(repository_config, 'fn')
if module_name:
return LoaderEntrypoint.from_module_target(module_name, fn_name, from_handle)
else:
# rebase file in config off of the path in the config file
file_name = os.path.join(os.path.dirname(os.path.abspath(file_path)), file_name)
return LoaderEntrypoint.from_file_target(file_name, fn_name, from_handle)
def test_k8s_run_launcher(dagster_instance, helm_namespace):
environment_dict = load_yaml_from_path(
os.path.join(test_project_environments_path(), 'env.yaml'))
pipeline_name = 'demo_pipeline'
run = dagster_instance.create_run(pipeline_name=pipeline_name,
environment_dict=environment_dict,
mode='default')
dagster_instance.launch_run(run.run_id)
result = wait_for_job_and_get_logs(job_name='dagster-run-%s' % run.run_id,
namespace=helm_namespace)
assert not result.get('errors')
assert result['data']
assert (result['data']['startPipelineExecutionForCreatedRun']['__typename']
== 'StartPipelineRunSuccess')
def test_k8s_run_launcher(dagster_instance, helm_namespace):
run_config = load_yaml_from_path(os.path.join(test_project_environments_path(), "env.yaml"))
pipeline_name = "demo_pipeline"
run = create_run_for_test(
dagster_instance, pipeline_name=pipeline_name, run_config=run_config, mode="default",
)
dagster_instance.launch_run(
run.run_id,
ReOriginatedExternalPipelineForTest(get_test_project_external_pipeline(pipeline_name)),
)
result = wait_for_job_and_get_raw_logs(
job_name="dagster-run-%s" % run.run_id, namespace=helm_namespace
)
assert "PIPELINE_SUCCESS" in result, "no match, result: {}".format(result)
