def event_metadata_entries(metadata_entry_datas):
if not metadata_entry_datas:
return
for metadata_entry_data in metadata_entry_datas:
typename = metadata_entry_data['__typename']
label = metadata_entry_data['label']
description = metadata_entry_data.get('description')
if typename == 'EventPathMetadataEntry':
yield EventMetadataEntry.path(label=label,
description=description,
path=metadata_entry_data['path'])
elif typename == 'EventJsonMetadataEntry':
yield EventMetadataEntry.json(
label=label,
description=description,
data=json.loads(metadata_entry_data.get('jsonString', '')),
)
elif typename == 'EventTextMetadataEntry':
yield EventMetadataEntry.text(label=label,
description=description,
text=metadata_entry_data['text'])
elif typename == 'EventUrlMetadataEntry':
yield EventMetadataEntry.url(label=label,
description=description,
url=metadata_entry_data['url'])
else:
check.not_implemented('TODO for type {}'.format(typename))
def check_for_unsupported_composite_overrides(self, runtime_type):
composite_overrides = {t.name for t in runtime_type.inner_types if t.name in self._registry}
if composite_overrides:
outer_type = 'composite type'
if runtime_type.is_list:
if runtime_type.is_nullable:
outer_type = 'Optional List'
else:
outer_type = 'List'
elif runtime_type.is_nullable:
outer_type = 'Optional'
if len(composite_overrides) > 1:
plural = 's'
this = 'These'
has = 'have'
else:
plural = ''
this = 'This'
has = 'has'
check.not_implemented(
'You are attempting to store a {outer_type} containing type{plural} '
'{type_names} in a object store. {this} type{plural} {has} specialized storage '
'behavior (configured in the TYPE_STORAGE_PLUGIN_REGISTRY). We do not '
'currently support storing Nullables or Lists of types with customized '
'storage. See https://github.com/dagster-io/dagster/issues/1190 for '
'details.'.format(
outer_type=outer_type,
plural=plural,
this=this,
has=has,
type_names=', '.join([str(x) for x in composite_overrides]),
)
)
def iterate_metadata_entries(metadata_entries):
check.list_param(metadata_entries, 'metadata_entries', of_type=EventMetadataEntry)
for metadata_entry in metadata_entries:
if isinstance(metadata_entry.entry_data, PathMetadataEntryData):
yield DauphinEventPathMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
path=metadata_entry.entry_data.path,
)
elif isinstance(metadata_entry.entry_data, JsonMetadataEntryData):
yield DauphinEventJsonMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
jsonString=json.dumps(metadata_entry.entry_data.data),
)
elif isinstance(metadata_entry.entry_data, TextMetadataEntryData):
yield DauphinEventTextMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
text=metadata_entry.entry_data.text,
)
elif isinstance(metadata_entry.entry_data, UrlMetadataEntryData):
yield DauphinEventUrlMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
url=metadata_entry.entry_data.url,
)
else:
# skip rest for now
check.not_implemented(
'{} unsupported metadata entry for now'.format(type(metadata_entry.entry_data))
)
def workspace_from_load_target(load_target, instance):
check.inst_param(load_target, 'load_target', WorkspaceLoadTarget)
check.inst_param(instance, 'instance', DagsterInstance)
opt_in_settings = instance.get_settings('opt_in')
python_user_process_api = (UserProcessApi.GRPC if
(opt_in_settings
and opt_in_settings['local_servers']) else
UserProcessApi.CLI)
if isinstance(load_target, WorkspaceFileTarget):
return load_workspace_from_yaml_paths(load_target.paths,
python_user_process_api)
elif isinstance(load_target, PythonFileTarget):
return Workspace([
location_handle_from_python_file(
load_target.python_file,
load_target.attribute,
user_process_api=python_user_process_api,
)
])
elif isinstance(load_target, ModuleTarget):
return Workspace([
location_handle_from_module_name(
load_target.module_name,
load_target.attribute,
user_process_api=python_user_process_api,
)
])
elif isinstance(load_target, EmptyWorkspaceTarget):
return Workspace([])
else:
check.not_implemented('Unsupported: {}'.format(load_target))
def workspace_from_load_target(load_target):
check.inst_param(load_target, 'load_target', WorkspaceLoadTarget)
if isinstance(load_target, WorkspaceFileTarget):
return load_workspace_from_yaml_paths(load_target.paths)
elif isinstance(load_target, PythonFileTarget):
return Workspace([
location_handle_from_python_file(
load_target.python_file,
load_target.attribute,
user_process_api=load_target.user_process_api,
)
])
elif isinstance(load_target, ModuleTarget):
return Workspace([
location_handle_from_module_name(
load_target.module_name,
load_target.attribute,
user_process_api=load_target.user_process_api,
)
])
elif isinstance(load_target, EmptyWorkspaceTarget):
return Workspace([])
else:
check.not_implemented('Unsupported: {}'.format(load_target))
def recon_repo_for_cli_args(kwargs):
"""Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
"""
check.dict_param(kwargs, "kwargs")
_cli_load_invariant(kwargs.get("pipeline_name") is None)
if kwargs.get("workspace"):
check.not_implemented(
"Workspace not supported yet in this cli command")
elif kwargs.get("module_name") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("python_file") is None)
return ReconstructableRepository.for_module(
kwargs["module_name"],
kwargs["fn_name"],
get_working_directory_from_kwargs(kwargs),
)
elif kwargs.get("python_file") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("module_name") is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs["python_file"]),
kwargs["fn_name"],
get_working_directory_from_kwargs(kwargs),
)
else:
_cli_load_invariant(False)
def location_origins_from_load_target(load_target):
if isinstance(load_target, WorkspaceFileTarget):
return location_origins_from_yaml_paths(load_target.paths,)
elif isinstance(load_target, PythonFileTarget):
return [
location_origin_from_python_file(
python_file=load_target.python_file,
attribute=load_target.attribute,
working_directory=load_target.working_directory,
)
]
elif isinstance(load_target, ModuleTarget):
return [location_origin_from_module_name(load_target.module_name, load_target.attribute,)]
elif isinstance(load_target, PackageTarget):
return [location_origin_from_package_name(load_target.package_name, load_target.attribute,)]
elif isinstance(load_target, GrpcServerTarget):
return [
GrpcServerRepositoryLocationOrigin(
port=load_target.port, socket=load_target.socket, host=load_target.host,
)
]
elif isinstance(load_target, EmptyWorkspaceTarget):
return []
else:
check.not_implemented("Unsupported: {}".format(load_target))
def iterate_metadata_entries(metadata_entries):
check.list_param(metadata_entries,
"metadata_entries",
of_type=EventMetadataEntry)
for metadata_entry in metadata_entries:
if isinstance(metadata_entry.entry_data, PathMetadataEntryData):
yield DauphinEventPathMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
path=metadata_entry.entry_data.path,
)
elif isinstance(metadata_entry.entry_data, JsonMetadataEntryData):
yield DauphinEventJsonMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
jsonString=seven.json.dumps(metadata_entry.entry_data.data),
)
elif isinstance(metadata_entry.entry_data, TextMetadataEntryData):
yield DauphinEventTextMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
text=metadata_entry.entry_data.text,
)
elif isinstance(metadata_entry.entry_data, UrlMetadataEntryData):
yield DauphinEventUrlMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
url=metadata_entry.entry_data.url,
)
elif isinstance(metadata_entry.entry_data, MarkdownMetadataEntryData):
yield DauphinEventMarkdownMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
md_str=metadata_entry.entry_data.md_str,
)
elif isinstance(metadata_entry.entry_data,
PythonArtifactMetadataEntryData):
yield DauphinEventPythonArtifactMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
module=metadata_entry.entry_data.module,
name=metadata_entry.entry_data.name,
)
elif isinstance(metadata_entry.entry_data, FloatMetadataEntryData):
yield DauphinEventFloatMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
floatValue=metadata_entry.entry_data.value,
)
elif isinstance(metadata_entry.entry_data, IntMetadataEntryData):
yield DauphinEventIntMetadataEntry(
label=metadata_entry.label,
description=metadata_entry.description,
intValue=metadata_entry.entry_data.value,
)
else:
# skip rest for now
check.not_implemented(
"{} unsupported metadata entry for now".format(
type(metadata_entry.entry_data)))
def copy_for_configured(
self,
name: str,
description: Optional[str],
config_schema: Any,
config_or_config_fn: Any,
):
check.not_implemented("@graph does not yet implement configured")
def _location_handle_from_location_config(location_config, yaml_path):
check.dict_param(location_config, 'location_config')
check.str_param(yaml_path, 'yaml_path')
if is_target_config(location_config):
return _location_handle_from_target_config(location_config, yaml_path)
elif 'python_environment' in location_config:
return _location_handle_from_python_environment_config(
location_config['python_environment'], yaml_path)
else:
check.not_implemented(
'Unsupported location config: {}'.format(location_config))
def evaluate_value(self, _value):
'''Subclasses can implement this method. Check if the value is a valid one
and return a processed version of it. If value is invalid,
raise `DagsterEvaluateValueError`.
This class provides a default implementation of this method
Args:
value: The value to check
Returns:
value: A transformed value
'''
check.not_implemented('Must implement in subclass')
def _location_origin_from_location_config(location_config, yaml_path):
check.dict_param(location_config, "location_config")
check.str_param(yaml_path, "yaml_path")
if is_target_config(location_config):
return _location_origin_from_target_config(location_config, yaml_path)
elif "grpc_server" in location_config:
return _location_origin_from_grpc_server_config(
location_config["grpc_server"], yaml_path)
else:
check.not_implemented(
"Unsupported location config: {}".format(location_config))
def to_dauphin_config_type(config_type):
check.inst_param(config_type, 'config_type', ConfigType)
# all types inherit from the DauphinConfigType interface
# which require the same set of fields. Passing them
# as kwargs into each derived type.
type_kwargs = _dauphin_config_type_kwargs_args_from_type(config_type)
if config_type.kind == ConfigTypeKind.ENUM:
return DauphinEnumConfigType(values=[
DauphinEnumConfigValue(value=ev.config_value,
description=ev.description)
for ev in config_type.enum_values
],
**type_kwargs)
elif ConfigTypeKind.has_fields(config_type.kind):
return DauphinCompositeConfigType(
fields=sorted(
[
to_dauphin_config_type_field(name, field)
for name, field in config_type.fields.items()
],
key=lambda field: field.name,
),
inner_types=_resolve_inner_types(config_type),
**type_kwargs)
elif config_type.kind == ConfigTypeKind.LIST:
return DauphinListConfigType(
of_type=to_dauphin_config_type(config_type.inner_type),
inner_types=_resolve_inner_types(config_type),
**type_kwargs)
elif config_type.kind == ConfigTypeKind.NULLABLE:
return DauphinNullableConfigType(
of_type=to_dauphin_config_type(config_type.inner_type),
inner_types=_resolve_inner_types(config_type),
**type_kwargs)
elif config_type.kind == ConfigTypeKind.SCALAR or config_type.kind == ConfigTypeKind.REGULAR:
return DauphinRegularConfigType(**type_kwargs)
else:
# Set and Tuple unsupported in the graphql layer
# https://github.com/dagster-io/dagster/issues/1925
check.not_implemented(
'Unsupported kind {kind} in config_type {key}'.format(
kind=config_type.kind, key=config_type.key))
def event_metadata_entries(metadata_entry_datas):
if not metadata_entry_datas:
return
for metadata_entry_data in metadata_entry_datas:
typename = metadata_entry_data["__typename"]
label = metadata_entry_data["label"]
description = metadata_entry_data.get("description")
if typename == "EventPathMetadataEntry":
yield EventMetadataEntry.path(label=label,
description=description,
path=metadata_entry_data["path"])
elif typename == "EventJsonMetadataEntry":
yield EventMetadataEntry.json(
label=label,
description=description,
data=seven.json.loads(metadata_entry_data.get(
"jsonString", "")),
)
elif typename == "EventMarkdownMetadataEntry":
yield EventMetadataEntry.md(label=label,
description=description,
md_str=metadata_entry_data.get(
"md_str", ""))
elif typename == "EventTextMetadataEntry":
yield EventMetadataEntry.text(label=label,
description=description,
text=metadata_entry_data["text"])
elif typename == "EventUrlMetadataEntry":
yield EventMetadataEntry.url(label=label,
description=description,
url=metadata_entry_data["url"])
elif typename == "EventPythonArtifactMetadataEntry":
yield EventMetadataEntry(
label=label,
description=description,
entry_data=PythonArtifactMetadataEntryData(
metadata_entry_data["module"],
metadata_entry_data["name"]),
)
else:
check.not_implemented("TODO for type {}".format(typename))
def recon_repo_for_cli_args(kwargs):
"""Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
"""
check.dict_param(kwargs, "kwargs")
_cli_load_invariant(kwargs.get("pipeline_name") is None)
if kwargs.get("workspace"):
check.not_implemented(
"Workspace not supported yet in this cli command")
if kwargs.get("repository_yaml") or all_none(kwargs):
_cli_load_invariant(kwargs.get("module_name") is None)
_cli_load_invariant(kwargs.get("python_file") is None)
_cli_load_invariant(kwargs.get("fn_name") is None)
repo_yaml = (os.path.abspath(kwargs.get("repository_yaml"))
if kwargs.get("repository_yaml") else
DEFAULT_REPOSITORY_YAML_FILENAME)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
"Verify your current working directory or CLI arguments.".format(
repo_yaml),
)
return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
elif kwargs.get("module_name") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("python_file") is None)
return ReconstructableRepository.for_module(kwargs["module_name"],
kwargs["fn_name"])
elif kwargs.get("python_file") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("module_name") is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs["python_file"]),
kwargs["fn_name"],
kwargs.get("working_directory")
if kwargs.get("working_directory") else os.getcwd(),
)
else:
_cli_load_invariant(False)
def workspace_from_load_target(load_target, instance):
check.inst_param(load_target, "load_target", WorkspaceLoadTarget)
check.inst_param(instance, "instance", DagsterInstance)
opt_in_settings = instance.get_settings("opt_in")
python_user_process_api = (UserProcessApi.GRPC if
(opt_in_settings
and opt_in_settings["local_servers"]) else
UserProcessApi.CLI)
if isinstance(load_target, WorkspaceFileTarget):
return load_workspace_from_yaml_paths(load_target.paths,
python_user_process_api)
elif isinstance(load_target, PythonFileTarget):
return Workspace([
location_handle_from_python_file(
python_file=load_target.python_file,
attribute=load_target.attribute,
working_directory=load_target.working_directory,
user_process_api=python_user_process_api,
)
])
elif isinstance(load_target, ModuleTarget):
return Workspace([
location_handle_from_module_name(
load_target.module_name,
load_target.attribute,
user_process_api=python_user_process_api,
)
])
elif isinstance(load_target, GrpcServerTarget):
return Workspace([
RepositoryLocationHandle.create_grpc_server_location(
port=load_target.port,
socket=load_target.socket,
host=load_target.host,
)
])
elif isinstance(load_target, EmptyWorkspaceTarget):
return Workspace([])
else:
check.not_implemented("Unsupported: {}".format(load_target))
def _location_handle_from_location_config(location_config, yaml_path, python_user_process_api):
check.dict_param(location_config, "location_config")
check.str_param(yaml_path, "yaml_path")
check.inst_param(python_user_process_api, "python_user_process_api", UserProcessApi)
if is_target_config(location_config):
return _location_handle_from_target_config(
location_config, yaml_path, python_user_process_api
)
elif "grpc_server" in location_config:
return _location_handle_from_grpc_server_config(location_config["grpc_server"], yaml_path)
elif "python_environment" in location_config:
return _location_handle_from_python_environment_config(
location_config["python_environment"], yaml_path, python_user_process_api
)
else:
check.not_implemented("Unsupported location config: {}".format(location_config))
def check_for_unsupported_composite_overrides(self, dagster_type):
from dagster.core.types.dagster_type import DagsterTypeKind
composite_overrides = {
t.unique_name
for t in dagster_type.inner_types
if (t.has_unique_name and t.unique_name in self._registry)
}
if composite_overrides:
outer_type = "composite type"
if dagster_type.kind == DagsterTypeKind.LIST:
if dagster_type.kind == DagsterTypeKind.NULLABLE:
outer_type = "Optional List"
else:
outer_type = "List"
elif dagster_type.kind == DagsterTypeKind.NULLABLE:
outer_type = "Optional"
if len(composite_overrides) > 1:
plural = "s"
this = "These"
has = "have"
else:
plural = ""
this = "This"
has = "has"
check.not_implemented(
"You are attempting to store a {outer_type} containing type{plural} "
"{type_names} in a object store. {this} type{plural} {has} specialized storage "
"behavior (configured in the TYPE_STORAGE_PLUGIN_REGISTRY). We do not "
"currently support storing Nullables or Lists of types with customized "
"storage. See https://github.com/dagster-io/dagster/issues/1190 for "
"details.".format(
outer_type=outer_type,
plural=plural,
this=this,
has=has,
type_names=", ".join([str(x) for x in composite_overrides]),
)
)
def _location_origin_from_location_config(location_config, yaml_path):
check.dict_param(location_config, "location_config")
check.str_param(yaml_path, "yaml_path")
if is_target_config(location_config):
return _location_origin_from_target_config(location_config, yaml_path)
elif "grpc_server" in location_config:
return _location_origin_from_grpc_server_config(location_config["grpc_server"], yaml_path)
elif "python_environment" in location_config:
warnings.warn(
"The `python_environment` key is deprecated. Use `python_file`, `python_package`, or "
"`python_module` with the `executable_path` attribute set if you want to load a "
"repository in a different python environment."
)
return _location_origin_from_python_environment_config(
location_config["python_environment"], yaml_path
)
else:
check.not_implemented("Unsupported location config: {}".format(location_config))
def recon_repo_for_cli_args(kwargs):
'''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
'''
check.dict_param(kwargs, 'kwargs')
_cli_load_invariant(kwargs.get('pipeline_name') is None)
if kwargs.get('workspace'):
check.not_implemented('Workspace not supported yet in this cli command')
if kwargs.get('repository_yaml') or all_none(kwargs):
_cli_load_invariant(kwargs.get('module_name') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (
os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml')
else DEFAULT_REPOSITORY_YAML_FILENAME
)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(repo_yaml),
)
return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
elif kwargs.get('module_name') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name'])
elif kwargs.get('python_file') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs['python_file']),
kwargs['fn_name'],
kwargs.get('working_directory') if kwargs.get('working_directory') else os.getcwd(),
)
else:
_cli_load_invariant(False)
def _location_handle_from_location_config(location_config, yaml_path, opt_ins):
check.dict_param(location_config, 'location_config')
check.str_param(yaml_path, 'yaml_path')
check.set_param(opt_ins, 'opt_ins')
user_process_api = UserProcessApi.GRPC if 'grpc' in opt_ins else UserProcessApi.CLI
if is_target_config(location_config):
return _location_handle_from_target_config(location_config, yaml_path,
user_process_api)
elif 'grpc_server' in location_config:
return _location_handle_from_grpc_server_config(
location_config['grpc_server'], yaml_path)
elif 'python_environment' in location_config:
return _location_handle_from_python_environment_config(
location_config['python_environment'], yaml_path, user_process_api)
else:
check.not_implemented(
'Unsupported location config: {}'.format(location_config))
def _location_handle_from_location_config(location_config, yaml_path,
python_user_process_api):
check.dict_param(location_config, 'location_config')
check.str_param(yaml_path, 'yaml_path')
check.inst_param(python_user_process_api, 'python_user_process_api',
UserProcessApi)
if is_target_config(location_config):
return _location_handle_from_target_config(location_config, yaml_path,
python_user_process_api)
elif 'grpc_server' in location_config:
return _location_handle_from_grpc_server_config(
location_config['grpc_server'], yaml_path)
elif 'python_environment' in location_config:
return _location_handle_from_python_environment_config(
location_config['python_environment'], yaml_path,
python_user_process_api)
else:
check.not_implemented(
'Unsupported location config: {}'.format(location_config))
def test_not_implemented():
with pytest.raises(NotImplementedCheckError, match='some string'):
check.not_implemented('some string')
def terminate(self, run_id):
check.not_implemented("Termination not supported.")
def materialize_runtime_value(self, _context, _config_value,
_runtime_value):
'''
How to materialize a runtime value given configuration.
'''
check.not_implemented('Must implement')
def schema_type(self):
check.not_implemented('Must override schema_type in {klass}'.format(
klass=type(self).__name__))
def test_not_implemented():
with pytest.raises(NotImplementedCheckError, match="some string"):
check.not_implemented("some string")
with pytest.raises(CheckError, match="desc argument must be a string"):
check.not_implemented(None)
def query_text(self):
check.not_implemented('table cannot be a standalone query')
def from_target(self):
check.not_implemented('must implemented in subclass')
def terminate(self, run_id):
check.str_param(run_id, 'run_id')
check.not_implemented('Termination not yet implemented')
