Пример #1
0
def test_custom_run_coordinator_config(template: HelmTemplate):
    module = "a_module"
    class_ = "Class"
    config_field_one = "1"
    config_field_two = "two"
    config = {
        "config_field_one": config_field_one,
        "config_field_two": config_field_two
    }
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        runCoordinator=RunCoordinator.construct(
            enabled=True,
            type=RunCoordinatorType.CUSTOM,
            config=RunCoordinatorConfig.construct(
                customRunCoordinator=ConfigurableClass.construct(
                    module=module,
                    class_=class_,
                    config=config,
                )),
        )))
    configmaps = template.render(helm_values)
    assert len(configmaps) == 1

    instance = yaml.full_load(configmaps[0].data["dagster.yaml"])

    assert instance["run_coordinator"]["module"] == module
    assert instance["run_coordinator"]["class"] == class_
    assert instance["run_coordinator"]["config"] == config
Пример #2
0
def test_queued_run_coordinator(instance_template: HelmTemplate, ):  # pylint: disable=redefined-outer-name
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        runCoordinator=RunCoordinator.construct(
            enabled=True,
            type=RunCoordinatorType.QUEUED,
            config=RunCoordinatorConfig.construct(
                queuedRunCoordinator=QueuedRunCoordinatorConfig.construct(
                    tagConcurrencyLimits=[
                        TagConcurrencyLimit.construct(
                            key="foo", value="hi", limit=1)
                    ]), ),
        )))

    configmaps = instance_template.render(helm_values)

    assert len(configmaps) == 1

    instance = yaml.full_load(configmaps[0].data["dagster.yaml"])

    assert instance["run_coordinator"]["class"] == "QueuedRunCoordinator"
    assert instance["run_coordinator"]["config"]["tag_concurrency_limits"] == [
        {
            "key": "foo",
            "value": "hi",
            "limit": 1
        }
    ]
Пример #3
0
def test_run_monitoring(instance_template: HelmTemplate, ):  # pylint: disable=redefined-outer-name
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        runMonitoring={"enabled": True}))

    configmaps = instance_template.render(helm_values)

    assert len(configmaps) == 1

    instance = yaml.full_load(configmaps[0].data["dagster.yaml"])

    assert instance["run_monitoring"]["enabled"] == True
Пример #4
0
def test_startup_probe_enabled(template: HelmTemplate, enabled: bool):
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        startupProbe=kubernetes.StartupProbe(enabled=enabled)))

    daemon = template.render(helm_values)
    assert len(daemon) == 1
    daemon = daemon[0]

    assert len(daemon.spec.template.spec.containers) == 1
    container = daemon.spec.template.spec.containers[0]

    assert (container.startup_probe is not None) == enabled
Пример #5
0
def test_daemon_labels(template: HelmTemplate):
    deployment_labels = {"deployment_label": "label"}
    pod_labels = {"pod_label": "label"}
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        deploymentLabels=deployment_labels,
        labels=pod_labels,
    ))

    [daemon_deployment] = template.render(helm_values)

    assert set(deployment_labels.items()).issubset(
        daemon_deployment.metadata.labels.items())
    assert set(pod_labels.items()).issubset(
        daemon_deployment.spec.template.metadata.labels.items())
Пример #6
0
def test_dagit_image(template: HelmTemplate):
    repository = "repository"
    tag = "tag"
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        image=kubernetes.Image.construct(repository=repository, tag=tag)))

    daemon_deployments = template.render(helm_values)

    assert len(daemon_deployments) == 1

    image = daemon_deployments[0].spec.template.spec.containers[0].image
    image_name, image_tag = image.split(":")

    assert image_name == repository
    assert image_tag == tag
Пример #7
0
def test_queued_run_coordinator_config(template: HelmTemplate, enabled: bool):
    max_concurrent_runs = 50
    tag_concurrency_limits = [
        TagConcurrencyLimit(key="key", value="value", limit=10)
    ]
    dequeue_interval_seconds = 50
    helm_values = DagsterHelmValues.construct(dagsterDaemon=Daemon.construct(
        runCoordinator=RunCoordinator.construct(
            enabled=enabled,
            type=RunCoordinatorType.QUEUED,
            config=RunCoordinatorConfig.construct(
                queuedRunCoordinator=QueuedRunCoordinatorConfig.construct(
                    maxConcurrentRuns=max_concurrent_runs,
                    tagConcurrencyLimits=tag_concurrency_limits,
                    dequeueIntervalSeconds=dequeue_interval_seconds,
                )),
        )))
    configmaps = template.render(helm_values)
    assert len(configmaps) == 1

    instance = yaml.full_load(configmaps[0].data["dagster.yaml"])

    assert ("run_coordinator" in instance) == enabled
    if enabled:
        assert instance["run_coordinator"][
            "module"] == "dagster.core.run_coordinator"
        assert instance["run_coordinator"]["class"] == "QueuedRunCoordinator"
        assert instance["run_coordinator"]["config"]

        run_coordinator_config = instance["run_coordinator"]["config"]

        assert run_coordinator_config[
            "max_concurrent_runs"] == max_concurrent_runs
        assert run_coordinator_config[
            "dequeue_interval_seconds"] == dequeue_interval_seconds

        assert len(run_coordinator_config["tag_concurrency_limits"]) == len(
            tag_concurrency_limits)
        assert run_coordinator_config["tag_concurrency_limits"] == [
            tag_concurrency_limit.dict()
            for tag_concurrency_limit in tag_concurrency_limits
        ]
Пример #8
0
def test_daemon_command_without_user_deployments(template: HelmTemplate):
    repository = "repository"
    tag = "tag"
    helm_values = DagsterHelmValues.construct(
        dagsterDaemon=Daemon.construct(
            image=kubernetes.Image.construct(repository=repository, tag=tag)),
        dagsterUserDeployments=UserDeployments(
            enabled=False,
            enableSubchart=False,
            deployments=[],
        ),
    )
    daemon_deployments = template.render(helm_values)

    assert len(daemon_deployments) == 1

    command = daemon_deployments[0].spec.template.spec.containers[0].command
    assert command == [
        "/bin/bash",
        "-c",
        "dagster-daemon run",
    ]
Пример #9
0
def test_daemon_command_with_user_deployments(template: HelmTemplate):
    repository = "repository"
    tag = "tag"
    helm_values = DagsterHelmValues.construct(
        dagsterDaemon=Daemon.construct(
            image=kubernetes.Image.construct(repository=repository, tag=tag)),
        dagsterUserDeployments=UserDeployments(
            enabled=True,
            enableSubchart=True,
            deployments=[
                create_simple_user_deployment("simple-deployment-one")
            ],
        ),
    )
    daemon_deployments = template.render(helm_values)

    assert len(daemon_deployments) == 1

    command = daemon_deployments[0].spec.template.spec.containers[0].command
    assert command == [
        "/bin/bash",
        "-c",
        "dagster-daemon run -w /dagster-workspace/workspace.yaml",
    ]