def configurable_class_data(config_field):
return ConfigurableClassData(
check.str_elem(config_field, "module"),
check.str_elem(config_field, "class"),
yaml.dump(check.opt_dict_elem(config_field, "config"),
default_flow_style=False),
)
def configurable_class_data_or_default(config_value, field_name, default):
if config_value.get(field_name):
return ConfigurableClassData(
config_value[field_name]['module'],
config_value[field_name]['class'],
yaml.dump(config_value[field_name].get('config') or {}, default_flow_style=False),
)
return default
def config_defaults(base_dir):
return {
"local_artifact_storage":
ConfigurableClassData(
"dagster.core.storage.root",
"LocalArtifactStorage",
yaml.dump({"base_dir": base_dir}, default_flow_style=False),
),
"run_storage":
ConfigurableClassData(
"dagster.core.storage.runs",
"SqliteRunStorage",
yaml.dump({"base_dir": _runs_directory(base_dir)},
default_flow_style=False),
),
"event_log_storage":
ConfigurableClassData(
"dagster.core.storage.event_log",
"SqliteEventLogStorage",
yaml.dump({"base_dir": _event_logs_directory(base_dir)},
default_flow_style=False),
),
"compute_logs":
ConfigurableClassData(
"dagster.core.storage.local_compute_log_manager",
"LocalComputeLogManager",
yaml.dump({"base_dir": compute_logs_directory(base_dir)},
default_flow_style=False),
),
"schedule_storage":
ConfigurableClassData(
"dagster.core.storage.schedules",
"SqliteScheduleStorage",
yaml.dump({"base_dir": _schedule_directory(base_dir)},
default_flow_style=False),
),
"scheduler":
ConfigurableClassData(
"dagster.core.scheduler",
"DagsterDaemonScheduler",
yaml.dump({}),
),
"run_coordinator":
ConfigurableClassData("dagster.core.run_coordinator",
"DefaultRunCoordinator", yaml.dump({})),
"run_launcher":
ConfigurableClassData(
"dagster",
"DefaultRunLauncher",
yaml.dump({}),
),
}
def from_dir(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
config_value = dagster_instance_config(
base_dir, config_filename=config_filename, overrides=overrides
)
local_artifact_storage_data = configurable_class_data_or_default(
config_value,
'local_artifact_storage',
ConfigurableClassData(
'dagster.core.storage.root',
'LocalArtifactStorage',
yaml.dump({'base_dir': base_dir}, default_flow_style=False),
),
)
run_storage_data = configurable_class_data_or_default(
config_value,
'run_storage',
ConfigurableClassData(
'dagster.core.storage.runs',
'SqliteRunStorage',
yaml.dump({'base_dir': _runs_directory(base_dir)}, default_flow_style=False),
),
)
event_storage_data = configurable_class_data_or_default(
config_value,
'event_log_storage',
ConfigurableClassData(
'dagster.core.storage.event_log',
'SqliteEventLogStorage',
yaml.dump({'base_dir': _event_logs_directory(base_dir)}, default_flow_style=False),
),
)
compute_logs_data = configurable_class_data_or_default(
config_value,
'compute_logs',
ConfigurableClassData(
'dagster.core.storage.local_compute_log_manager',
'LocalComputeLogManager',
yaml.dump({'base_dir': compute_logs_directory(base_dir)}, default_flow_style=False),
),
)
schedule_storage_data = configurable_class_data_or_default(
config_value,
'schedule_storage',
ConfigurableClassData(
'dagster.core.storage.schedules',
'SqliteScheduleStorage',
yaml.dump({'base_dir': _schedule_directory(base_dir)}, default_flow_style=False),
),
)
scheduler_data = configurable_class_data_or_default(config_value, 'scheduler', None)
run_launcher_data = configurable_class_data_or_default(config_value, 'run_launcher', None)
return InstanceRef(
local_artifact_storage_data=local_artifact_storage_data,
run_storage_data=run_storage_data,
event_storage_data=event_storage_data,
compute_logs_data=compute_logs_data,
schedule_storage_data=schedule_storage_data,
scheduler_data=scheduler_data,
run_launcher_data=run_launcher_data,
dagit_settings=config_value.get('dagit'),
telemetry_settings=config_value.get('telemetry'),
)
def value_for_ref_item(k, v):
if v is None:
return None
if k in ['dagit_settings', 'telemetry_settings']:
return v
return ConfigurableClassData(*v)
def value_for_ref_item(k, v):
if v is None:
return None
if k == "settings":
return v
return ConfigurableClassData(*v)
def from_dir(base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None):
overrides = check.opt_dict_param(overrides, "overrides")
config_value, custom_instance_class = dagster_instance_config(
base_dir, config_filename=config_filename, overrides=overrides)
if custom_instance_class:
config_keys = set(custom_instance_class.config_schema().keys())
custom_instance_class_config = {
key: val
for key, val in config_value.items() if key in config_keys
}
custom_instance_class_data = ConfigurableClassData(
config_value["instance_class"]["module"],
config_value["instance_class"]["class"],
yaml.dump(custom_instance_class_config,
default_flow_style=False),
)
defaults = custom_instance_class.config_defaults(base_dir)
else:
custom_instance_class_data = None
defaults = InstanceRef.config_defaults(base_dir)
local_artifact_storage_data = configurable_class_data_or_default(
config_value, "local_artifact_storage",
defaults["local_artifact_storage"])
run_storage_data = configurable_class_data_or_default(
config_value, "run_storage", defaults["run_storage"])
event_storage_data = configurable_class_data_or_default(
config_value, "event_log_storage", defaults["event_log_storage"])
compute_logs_data = configurable_class_data_or_default(
config_value,
"compute_logs",
defaults["compute_logs"],
)
schedule_storage_data = configurable_class_data_or_default(
config_value, "schedule_storage", defaults["schedule_storage"])
scheduler_data = configurable_class_data_or_default(
config_value, "scheduler", defaults["scheduler"])
run_coordinator_data = configurable_class_data_or_default(
config_value,
"run_coordinator",
defaults["run_coordinator"],
)
run_launcher_data = configurable_class_data_or_default(
config_value,
"run_launcher",
defaults["run_launcher"],
)
settings_keys = {
"telemetry", "python_logs", "run_monitoring", "code_servers"
}
settings = {
key: config_value.get(key)
for key in settings_keys if config_value.get(key)
}
return InstanceRef(
local_artifact_storage_data=local_artifact_storage_data,
run_storage_data=run_storage_data,
event_storage_data=event_storage_data,
compute_logs_data=compute_logs_data,
schedule_storage_data=schedule_storage_data,
scheduler_data=scheduler_data,
run_coordinator_data=run_coordinator_data,
run_launcher_data=run_launcher_data,
settings=settings,
custom_instance_class_data=custom_instance_class_data,
)
def from_dir(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None):
overrides = check.opt_dict_param(overrides, "overrides")
config_value = dagster_instance_config(
base_dir, config_filename=config_filename, overrides=overrides
)
local_artifact_storage_data = configurable_class_data_or_default(
config_value,
"local_artifact_storage",
ConfigurableClassData(
"dagster.core.storage.root",
"LocalArtifactStorage",
yaml.dump({"base_dir": base_dir}, default_flow_style=False),
),
)
run_storage_data = configurable_class_data_or_default(
config_value,
"run_storage",
ConfigurableClassData(
"dagster.core.storage.runs",
"SqliteRunStorage",
yaml.dump({"base_dir": _runs_directory(base_dir)}, default_flow_style=False),
),
)
event_storage_data = configurable_class_data_or_default(
config_value,
"event_log_storage",
ConfigurableClassData(
"dagster.core.storage.event_log",
"SqliteEventLogStorage",
yaml.dump({"base_dir": _event_logs_directory(base_dir)}, default_flow_style=False),
),
)
compute_logs_data = configurable_class_data_or_default(
config_value,
"compute_logs",
ConfigurableClassData(
"dagster.core.storage.local_compute_log_manager",
"LocalComputeLogManager",
yaml.dump({"base_dir": compute_logs_directory(base_dir)}, default_flow_style=False),
),
)
schedule_storage_data = configurable_class_data_or_default(
config_value,
"schedule_storage",
ConfigurableClassData(
"dagster.core.storage.schedules",
"SqliteScheduleStorage",
yaml.dump({"base_dir": _schedule_directory(base_dir)}, default_flow_style=False),
),
)
scheduler_data = configurable_class_data_or_default(config_value, "scheduler", None,)
run_coordinator_data = configurable_class_data_or_default(
config_value,
"run_coordinator",
ConfigurableClassData(
"dagster.core.run_coordinator", "DefaultRunCoordinator", yaml.dump({})
),
)
run_launcher_data = configurable_class_data_or_default(
config_value,
"run_launcher",
ConfigurableClassData("dagster", "DefaultRunLauncher", yaml.dump({}),),
)
settings_keys = {"telemetry"}
settings = {key: config_value.get(key) for key in settings_keys}
return InstanceRef(
local_artifact_storage_data=local_artifact_storage_data,
run_storage_data=run_storage_data,
event_storage_data=event_storage_data,
compute_logs_data=compute_logs_data,
schedule_storage_data=schedule_storage_data,
scheduler_data=scheduler_data,
run_coordinator_data=run_coordinator_data,
run_launcher_data=run_launcher_data,
settings=settings,
)
def configurable_class_data(config_field):
return ConfigurableClassData(
config_field["module"],
config_field["class"],
yaml.dump(config_field.get("config") or {}, default_flow_style=False),
)