def test_custom_compute_log_manager_config(template: HelmTemplate): module = "a_module" class_ = "Class" config_field_one = "1" config_field_two = "two" config = {"config_field_one": config_field_one, "config_field_two": config_field_two} helm_values = DagsterHelmValues.construct( computeLogManager=ComputeLogManager.construct( type=ComputeLogManagerType.CUSTOM, config=ComputeLogManagerConfig.construct( customComputeLogManager=ConfigurableClass.construct( module=module, class_=class_, config=config, ) ), ) ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) compute_logs_config = instance["compute_logs"] assert compute_logs_config["module"] == module assert compute_logs_config["class"] == class_ assert compute_logs_config["config"] == config
def test_gcs_compute_log_manager(template: HelmTemplate): bucket = "bucket" local_dir = "/dir" prefix = "prefix" helm_values = DagsterHelmValues.construct( computeLogManager=ComputeLogManager.construct( type=ComputeLogManagerType.GCS, config=ComputeLogManagerConfig.construct( gcsComputeLogManager=GCSComputeLogManagerModel( bucket=bucket, localDir=local_dir, prefix=prefix ) ), ) ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) compute_logs_config = instance["compute_logs"] assert compute_logs_config["module"] == "dagster_gcp.gcs.compute_log_manager" assert compute_logs_config["class"] == "GCSComputeLogManager" assert compute_logs_config["config"] == { "bucket": bucket, "local_dir": local_dir, "prefix": prefix, } # Test all config fields in configurable class assert compute_logs_config["config"].keys() == GCSComputeLogManager.config_type().keys()
def test_noop_compute_log_manager( template: HelmTemplate, compute_log_manager_type: ComputeLogManagerType ): helm_values = DagsterHelmValues.construct( computeLogManager=ComputeLogManager.construct(type=compute_log_manager_type) ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) compute_logs_config = instance["compute_logs"] assert compute_logs_config["module"] == "dagster.core.storage.noop_compute_log_manager" assert compute_logs_config["class"] == "NoOpComputeLogManager"
def test_s3_compute_log_manager(template: HelmTemplate): bucket = "bucket" local_dir = "/dir" prefix = "prefix" use_ssl = True verify = True verify_cert_path = "/path" endpoint_url = "endpoint.com" skip_empty_files = True helm_values = DagsterHelmValues.construct( computeLogManager=ComputeLogManager.construct( type=ComputeLogManagerType.S3, config=ComputeLogManagerConfig.construct( s3ComputeLogManager=S3ComputeLogManagerModel( bucket=bucket, localDir=local_dir, prefix=prefix, useSsl=use_ssl, verify=verify, verifyCertPath=verify_cert_path, endpointUrl=endpoint_url, skipEmptyFiles=skip_empty_files, ) ), ) ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) compute_logs_config = instance["compute_logs"] assert compute_logs_config["module"] == "dagster_aws.s3.compute_log_manager" assert compute_logs_config["class"] == "S3ComputeLogManager" assert compute_logs_config["config"] == { "bucket": bucket, "local_dir": local_dir, "prefix": prefix, "use_ssl": use_ssl, "verify": verify, "verify_cert_path": verify_cert_path, "endpoint_url": endpoint_url, "skip_empty_files": skip_empty_files, } # Test all config fields in configurable class assert compute_logs_config["config"].keys() == S3ComputeLogManager.config_type().keys()
def test_azure_blob_compute_log_manager(template: HelmTemplate): storage_account = "account" container = "container" secret_key = "secret_key" local_dir = "/dir" prefix = "prefix" helm_values = DagsterHelmValues.construct( computeLogManager=ComputeLogManager.construct( type=ComputeLogManagerType.AZURE, config=ComputeLogManagerConfig.construct( azureBlobComputeLogManager=AzureBlobComputeLogManagerModel( storageAccount=storage_account, container=container, secretKey=secret_key, localDir=local_dir, prefix=prefix, ) ), ) ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) compute_logs_config = instance["compute_logs"] assert compute_logs_config["module"] == "dagster_azure.blob.compute_log_manager" assert compute_logs_config["class"] == "AzureBlobComputeLogManager" assert compute_logs_config["config"] == { "storage_account": storage_account, "container": container, "secret_key": secret_key, "local_dir": local_dir, "prefix": prefix, } # Test all config fields in configurable class assert compute_logs_config["config"].keys() == AzureBlobComputeLogManager.config_type().keys()