def from_dir(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
config_value = dagster_instance_config(
base_dir, config_filename=config_filename, overrides=overrides
)
local_artifact_storage_data = configurable_class_data_or_default(
config_value,
'local_artifact_storage',
ConfigurableClassData(
'dagster.core.storage.root',
'LocalArtifactStorage',
yaml.dump({'base_dir': base_dir}, default_flow_style=False),
),
)
run_storage_data = configurable_class_data_or_default(
config_value,
'run_storage',
ConfigurableClassData(
'dagster.core.storage.runs',
'SqliteRunStorage',
yaml.dump({'base_dir': _runs_directory(base_dir)}, default_flow_style=False),
),
)
event_storage_data = configurable_class_data_or_default(
config_value,
'event_log_storage',
ConfigurableClassData(
'dagster.core.storage.event_log',
'SqliteEventLogStorage',
yaml.dump({'base_dir': _event_logs_directory(base_dir)}, default_flow_style=False),
),
)
compute_logs_data = configurable_class_data_or_default(
config_value,
'compute_logs',
ConfigurableClassData(
'dagster.core.storage.local_compute_log_manager',
'LocalComputeLogManager',
yaml.dump(
{'base_dir': _compute_logs_directory(base_dir)}, default_flow_style=False
),
),
)
run_launcher_data = configurable_class_data_or_default(config_value, 'run_launcher', None)
return InstanceRef(
local_artifact_storage_data=local_artifact_storage_data,
run_storage_data=run_storage_data,
event_storage_data=event_storage_data,
compute_logs_data=compute_logs_data,
run_launcher_data=run_launcher_data,
dagit_settings=config_value.get('dagit'),
)
def from_dir(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME):
config_value = dagster_instance_config(base_dir,
config_filename=config_filename)
local_artifact_storage_data = configurable_class_data_or_default(
config_value,
'local_artifact_storage',
ConfigurableClassData(
'dagster.core.storage.root',
'LocalArtifactStorage',
yaml.dump({'base_dir': base_dir}, default_flow_style=False),
),
)
run_storage_data = configurable_class_data_or_default(
config_value,
'run_storage',
ConfigurableClassData(
'dagster.core.storage.sqlite_run_storage',
'SqliteRunStorage',
yaml.dump({'base_dir': _runs_directory(base_dir)},
default_flow_style=False),
),
)
event_storage_data = configurable_class_data_or_default(
config_value,
'event_log_storage',
ConfigurableClassData(
'dagster.core.storage.event_log',
'SqliteEventLogStorage',
yaml.dump({'base_dir': _event_logs_directory(base_dir)},
default_flow_style=False),
),
)
compute_logs_data = configurable_class_data_or_default(
config_value,
'compute_logs',
ConfigurableClassData(
'dagster.core.storage.local_compute_log_manager',
'LocalComputeLogManager',
yaml.dump({'base_dir': _compute_logs_directory(base_dir)},
default_flow_style=False),
),
)
return InstanceRef(
feature_set=dagster_feature_set(base_dir),
local_artifact_storage_data=local_artifact_storage_data,
run_storage_data=run_storage_data,
event_storage_data=event_storage_data,
compute_logs_data=compute_logs_data,
)
def from_dict(instance_ref_dict):
return InstanceRef(
**{
k: (ConfigurableClassData(*v) if v is not None else None)
for k, v in instance_ref_dict.items()
}
)
def configurable_class_data_or_default(config_value, field_name, default):
if config_value.get(field_name):
return ConfigurableClassData(
config_value[field_name]['module'],
config_value[field_name]['class'],
yaml.dump(config_value[field_name]['config'], default_flow_style=False),
)
return default
def from_root_storage_dir(root_storage_dir):
return LocalInstanceRef(
feature_set=dagster_feature_set(root_storage_dir),
root_storage_data=ConfigurableClassData(
'dagster.core.storage.root',
'RootStorage',
yaml.dump({'root_storage_dir': root_storage_dir},
default_flow_style=False),
),
run_storage_data=ConfigurableClassData(
'dagster.core.storage.sqlite_run_storage',
'SqliteRunStorage',
yaml.dump({'base_dir': _runs_directory(root_storage_dir)},
default_flow_style=False),
),
event_storage_data=ConfigurableClassData(
'dagster.core.storage.event_log',
'SqliteEventLogStorage',
yaml.dump(
{'base_dir': _event_logs_directory(root_storage_dir)},
default_flow_style=False),
),
)
def value_for_ref_item(k, v):
if v is None:
return None
if k == 'dagit_settings':
return v
return ConfigurableClassData(*v)
