def event_metadata_entries(metadata_entry_datas): if not metadata_entry_datas: return for metadata_entry_data in metadata_entry_datas: typename = metadata_entry_data['__typename'] label = metadata_entry_data['label'] description = metadata_entry_data.get('description') if typename == 'EventPathMetadataEntry': yield EventMetadataEntry.path(label=label, description=description, path=metadata_entry_data['path']) elif typename == 'EventJsonMetadataEntry': yield EventMetadataEntry.json( label=label, description=description, data=json.loads(metadata_entry_data.get('jsonString', '')), ) elif typename == 'EventTextMetadataEntry': yield EventMetadataEntry.text(label=label, description=description, text=metadata_entry_data['text']) elif typename == 'EventUrlMetadataEntry': yield EventMetadataEntry.url(label=label, description=description, url=metadata_entry_data['url']) else: check.not_implemented('TODO for type {}'.format(typename))
def check_for_unsupported_composite_overrides(self, runtime_type): composite_overrides = {t.name for t in runtime_type.inner_types if t.name in self._registry} if composite_overrides: outer_type = 'composite type' if runtime_type.is_list: if runtime_type.is_nullable: outer_type = 'Optional List' else: outer_type = 'List' elif runtime_type.is_nullable: outer_type = 'Optional' if len(composite_overrides) > 1: plural = 's' this = 'These' has = 'have' else: plural = '' this = 'This' has = 'has' check.not_implemented( 'You are attempting to store a {outer_type} containing type{plural} ' '{type_names} in a object store. {this} type{plural} {has} specialized storage ' 'behavior (configured in the TYPE_STORAGE_PLUGIN_REGISTRY). We do not ' 'currently support storing Nullables or Lists of types with customized ' 'storage. See https://github.com/dagster-io/dagster/issues/1190 for ' 'details.'.format( outer_type=outer_type, plural=plural, this=this, has=has, type_names=', '.join([str(x) for x in composite_overrides]), ) )
def iterate_metadata_entries(metadata_entries): check.list_param(metadata_entries, 'metadata_entries', of_type=EventMetadataEntry) for metadata_entry in metadata_entries: if isinstance(metadata_entry.entry_data, PathMetadataEntryData): yield DauphinEventPathMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, path=metadata_entry.entry_data.path, ) elif isinstance(metadata_entry.entry_data, JsonMetadataEntryData): yield DauphinEventJsonMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, jsonString=json.dumps(metadata_entry.entry_data.data), ) elif isinstance(metadata_entry.entry_data, TextMetadataEntryData): yield DauphinEventTextMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, text=metadata_entry.entry_data.text, ) elif isinstance(metadata_entry.entry_data, UrlMetadataEntryData): yield DauphinEventUrlMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, url=metadata_entry.entry_data.url, ) else: # skip rest for now check.not_implemented( '{} unsupported metadata entry for now'.format(type(metadata_entry.entry_data)) )
def workspace_from_load_target(load_target, instance): check.inst_param(load_target, 'load_target', WorkspaceLoadTarget) check.inst_param(instance, 'instance', DagsterInstance) opt_in_settings = instance.get_settings('opt_in') python_user_process_api = (UserProcessApi.GRPC if (opt_in_settings and opt_in_settings['local_servers']) else UserProcessApi.CLI) if isinstance(load_target, WorkspaceFileTarget): return load_workspace_from_yaml_paths(load_target.paths, python_user_process_api) elif isinstance(load_target, PythonFileTarget): return Workspace([ location_handle_from_python_file( load_target.python_file, load_target.attribute, user_process_api=python_user_process_api, ) ]) elif isinstance(load_target, ModuleTarget): return Workspace([ location_handle_from_module_name( load_target.module_name, load_target.attribute, user_process_api=python_user_process_api, ) ]) elif isinstance(load_target, EmptyWorkspaceTarget): return Workspace([]) else: check.not_implemented('Unsupported: {}'.format(load_target))
def workspace_from_load_target(load_target): check.inst_param(load_target, 'load_target', WorkspaceLoadTarget) if isinstance(load_target, WorkspaceFileTarget): return load_workspace_from_yaml_paths(load_target.paths) elif isinstance(load_target, PythonFileTarget): return Workspace([ location_handle_from_python_file( load_target.python_file, load_target.attribute, user_process_api=load_target.user_process_api, ) ]) elif isinstance(load_target, ModuleTarget): return Workspace([ location_handle_from_module_name( load_target.module_name, load_target.attribute, user_process_api=load_target.user_process_api, ) ]) elif isinstance(load_target, EmptyWorkspaceTarget): return Workspace([]) else: check.not_implemented('Unsupported: {}'.format(load_target))
def recon_repo_for_cli_args(kwargs): """Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. """ check.dict_param(kwargs, "kwargs") _cli_load_invariant(kwargs.get("pipeline_name") is None) if kwargs.get("workspace"): check.not_implemented( "Workspace not supported yet in this cli command") elif kwargs.get("module_name") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("python_file") is None) return ReconstructableRepository.for_module( kwargs["module_name"], kwargs["fn_name"], get_working_directory_from_kwargs(kwargs), ) elif kwargs.get("python_file") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("module_name") is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs["python_file"]), kwargs["fn_name"], get_working_directory_from_kwargs(kwargs), ) else: _cli_load_invariant(False)
def location_origins_from_load_target(load_target): if isinstance(load_target, WorkspaceFileTarget): return location_origins_from_yaml_paths(load_target.paths,) elif isinstance(load_target, PythonFileTarget): return [ location_origin_from_python_file( python_file=load_target.python_file, attribute=load_target.attribute, working_directory=load_target.working_directory, ) ] elif isinstance(load_target, ModuleTarget): return [location_origin_from_module_name(load_target.module_name, load_target.attribute,)] elif isinstance(load_target, PackageTarget): return [location_origin_from_package_name(load_target.package_name, load_target.attribute,)] elif isinstance(load_target, GrpcServerTarget): return [ GrpcServerRepositoryLocationOrigin( port=load_target.port, socket=load_target.socket, host=load_target.host, ) ] elif isinstance(load_target, EmptyWorkspaceTarget): return [] else: check.not_implemented("Unsupported: {}".format(load_target))
def iterate_metadata_entries(metadata_entries): check.list_param(metadata_entries, "metadata_entries", of_type=EventMetadataEntry) for metadata_entry in metadata_entries: if isinstance(metadata_entry.entry_data, PathMetadataEntryData): yield DauphinEventPathMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, path=metadata_entry.entry_data.path, ) elif isinstance(metadata_entry.entry_data, JsonMetadataEntryData): yield DauphinEventJsonMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, jsonString=seven.json.dumps(metadata_entry.entry_data.data), ) elif isinstance(metadata_entry.entry_data, TextMetadataEntryData): yield DauphinEventTextMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, text=metadata_entry.entry_data.text, ) elif isinstance(metadata_entry.entry_data, UrlMetadataEntryData): yield DauphinEventUrlMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, url=metadata_entry.entry_data.url, ) elif isinstance(metadata_entry.entry_data, MarkdownMetadataEntryData): yield DauphinEventMarkdownMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, md_str=metadata_entry.entry_data.md_str, ) elif isinstance(metadata_entry.entry_data, PythonArtifactMetadataEntryData): yield DauphinEventPythonArtifactMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, module=metadata_entry.entry_data.module, name=metadata_entry.entry_data.name, ) elif isinstance(metadata_entry.entry_data, FloatMetadataEntryData): yield DauphinEventFloatMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, floatValue=metadata_entry.entry_data.value, ) elif isinstance(metadata_entry.entry_data, IntMetadataEntryData): yield DauphinEventIntMetadataEntry( label=metadata_entry.label, description=metadata_entry.description, intValue=metadata_entry.entry_data.value, ) else: # skip rest for now check.not_implemented( "{} unsupported metadata entry for now".format( type(metadata_entry.entry_data)))
def copy_for_configured( self, name: str, description: Optional[str], config_schema: Any, config_or_config_fn: Any, ): check.not_implemented("@graph does not yet implement configured")
def _location_handle_from_location_config(location_config, yaml_path): check.dict_param(location_config, 'location_config') check.str_param(yaml_path, 'yaml_path') if is_target_config(location_config): return _location_handle_from_target_config(location_config, yaml_path) elif 'python_environment' in location_config: return _location_handle_from_python_environment_config( location_config['python_environment'], yaml_path) else: check.not_implemented( 'Unsupported location config: {}'.format(location_config))
def evaluate_value(self, _value): '''Subclasses can implement this method. Check if the value is a valid one and return a processed version of it. If value is invalid, raise `DagsterEvaluateValueError`. This class provides a default implementation of this method Args: value: The value to check Returns: value: A transformed value ''' check.not_implemented('Must implement in subclass')
def _location_origin_from_location_config(location_config, yaml_path): check.dict_param(location_config, "location_config") check.str_param(yaml_path, "yaml_path") if is_target_config(location_config): return _location_origin_from_target_config(location_config, yaml_path) elif "grpc_server" in location_config: return _location_origin_from_grpc_server_config( location_config["grpc_server"], yaml_path) else: check.not_implemented( "Unsupported location config: {}".format(location_config))
def to_dauphin_config_type(config_type): check.inst_param(config_type, 'config_type', ConfigType) # all types inherit from the DauphinConfigType interface # which require the same set of fields. Passing them # as kwargs into each derived type. type_kwargs = _dauphin_config_type_kwargs_args_from_type(config_type) if config_type.kind == ConfigTypeKind.ENUM: return DauphinEnumConfigType(values=[ DauphinEnumConfigValue(value=ev.config_value, description=ev.description) for ev in config_type.enum_values ], **type_kwargs) elif ConfigTypeKind.has_fields(config_type.kind): return DauphinCompositeConfigType( fields=sorted( [ to_dauphin_config_type_field(name, field) for name, field in config_type.fields.items() ], key=lambda field: field.name, ), inner_types=_resolve_inner_types(config_type), **type_kwargs) elif config_type.kind == ConfigTypeKind.LIST: return DauphinListConfigType( of_type=to_dauphin_config_type(config_type.inner_type), inner_types=_resolve_inner_types(config_type), **type_kwargs) elif config_type.kind == ConfigTypeKind.NULLABLE: return DauphinNullableConfigType( of_type=to_dauphin_config_type(config_type.inner_type), inner_types=_resolve_inner_types(config_type), **type_kwargs) elif config_type.kind == ConfigTypeKind.SCALAR or config_type.kind == ConfigTypeKind.REGULAR: return DauphinRegularConfigType(**type_kwargs) else: # Set and Tuple unsupported in the graphql layer # https://github.com/dagster-io/dagster/issues/1925 check.not_implemented( 'Unsupported kind {kind} in config_type {key}'.format( kind=config_type.kind, key=config_type.key))
def event_metadata_entries(metadata_entry_datas): if not metadata_entry_datas: return for metadata_entry_data in metadata_entry_datas: typename = metadata_entry_data["__typename"] label = metadata_entry_data["label"] description = metadata_entry_data.get("description") if typename == "EventPathMetadataEntry": yield EventMetadataEntry.path(label=label, description=description, path=metadata_entry_data["path"]) elif typename == "EventJsonMetadataEntry": yield EventMetadataEntry.json( label=label, description=description, data=seven.json.loads(metadata_entry_data.get( "jsonString", "")), ) elif typename == "EventMarkdownMetadataEntry": yield EventMetadataEntry.md(label=label, description=description, md_str=metadata_entry_data.get( "md_str", "")) elif typename == "EventTextMetadataEntry": yield EventMetadataEntry.text(label=label, description=description, text=metadata_entry_data["text"]) elif typename == "EventUrlMetadataEntry": yield EventMetadataEntry.url(label=label, description=description, url=metadata_entry_data["url"]) elif typename == "EventPythonArtifactMetadataEntry": yield EventMetadataEntry( label=label, description=description, entry_data=PythonArtifactMetadataEntryData( metadata_entry_data["module"], metadata_entry_data["name"]), ) else: check.not_implemented("TODO for type {}".format(typename))
def recon_repo_for_cli_args(kwargs): """Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. """ check.dict_param(kwargs, "kwargs") _cli_load_invariant(kwargs.get("pipeline_name") is None) if kwargs.get("workspace"): check.not_implemented( "Workspace not supported yet in this cli command") if kwargs.get("repository_yaml") or all_none(kwargs): _cli_load_invariant(kwargs.get("module_name") is None) _cli_load_invariant(kwargs.get("python_file") is None) _cli_load_invariant(kwargs.get("fn_name") is None) repo_yaml = (os.path.abspath(kwargs.get("repository_yaml")) if kwargs.get("repository_yaml") else DEFAULT_REPOSITORY_YAML_FILENAME) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' "Verify your current working directory or CLI arguments.".format( repo_yaml), ) return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) elif kwargs.get("module_name") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("python_file") is None) return ReconstructableRepository.for_module(kwargs["module_name"], kwargs["fn_name"]) elif kwargs.get("python_file") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("module_name") is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs["python_file"]), kwargs["fn_name"], kwargs.get("working_directory") if kwargs.get("working_directory") else os.getcwd(), ) else: _cli_load_invariant(False)
def workspace_from_load_target(load_target, instance): check.inst_param(load_target, "load_target", WorkspaceLoadTarget) check.inst_param(instance, "instance", DagsterInstance) opt_in_settings = instance.get_settings("opt_in") python_user_process_api = (UserProcessApi.GRPC if (opt_in_settings and opt_in_settings["local_servers"]) else UserProcessApi.CLI) if isinstance(load_target, WorkspaceFileTarget): return load_workspace_from_yaml_paths(load_target.paths, python_user_process_api) elif isinstance(load_target, PythonFileTarget): return Workspace([ location_handle_from_python_file( python_file=load_target.python_file, attribute=load_target.attribute, working_directory=load_target.working_directory, user_process_api=python_user_process_api, ) ]) elif isinstance(load_target, ModuleTarget): return Workspace([ location_handle_from_module_name( load_target.module_name, load_target.attribute, user_process_api=python_user_process_api, ) ]) elif isinstance(load_target, GrpcServerTarget): return Workspace([ RepositoryLocationHandle.create_grpc_server_location( port=load_target.port, socket=load_target.socket, host=load_target.host, ) ]) elif isinstance(load_target, EmptyWorkspaceTarget): return Workspace([]) else: check.not_implemented("Unsupported: {}".format(load_target))
def _location_handle_from_location_config(location_config, yaml_path, python_user_process_api): check.dict_param(location_config, "location_config") check.str_param(yaml_path, "yaml_path") check.inst_param(python_user_process_api, "python_user_process_api", UserProcessApi) if is_target_config(location_config): return _location_handle_from_target_config( location_config, yaml_path, python_user_process_api ) elif "grpc_server" in location_config: return _location_handle_from_grpc_server_config(location_config["grpc_server"], yaml_path) elif "python_environment" in location_config: return _location_handle_from_python_environment_config( location_config["python_environment"], yaml_path, python_user_process_api ) else: check.not_implemented("Unsupported location config: {}".format(location_config))
def check_for_unsupported_composite_overrides(self, dagster_type): from dagster.core.types.dagster_type import DagsterTypeKind composite_overrides = { t.unique_name for t in dagster_type.inner_types if (t.has_unique_name and t.unique_name in self._registry) } if composite_overrides: outer_type = "composite type" if dagster_type.kind == DagsterTypeKind.LIST: if dagster_type.kind == DagsterTypeKind.NULLABLE: outer_type = "Optional List" else: outer_type = "List" elif dagster_type.kind == DagsterTypeKind.NULLABLE: outer_type = "Optional" if len(composite_overrides) > 1: plural = "s" this = "These" has = "have" else: plural = "" this = "This" has = "has" check.not_implemented( "You are attempting to store a {outer_type} containing type{plural} " "{type_names} in a object store. {this} type{plural} {has} specialized storage " "behavior (configured in the TYPE_STORAGE_PLUGIN_REGISTRY). We do not " "currently support storing Nullables or Lists of types with customized " "storage. See https://github.com/dagster-io/dagster/issues/1190 for " "details.".format( outer_type=outer_type, plural=plural, this=this, has=has, type_names=", ".join([str(x) for x in composite_overrides]), ) )
def _location_origin_from_location_config(location_config, yaml_path): check.dict_param(location_config, "location_config") check.str_param(yaml_path, "yaml_path") if is_target_config(location_config): return _location_origin_from_target_config(location_config, yaml_path) elif "grpc_server" in location_config: return _location_origin_from_grpc_server_config(location_config["grpc_server"], yaml_path) elif "python_environment" in location_config: warnings.warn( "The `python_environment` key is deprecated. Use `python_file`, `python_package`, or " "`python_module` with the `executable_path` attribute set if you want to load a " "repository in a different python environment." ) return _location_origin_from_python_environment_config( location_config["python_environment"], yaml_path ) else: check.not_implemented("Unsupported location config: {}".format(location_config))
def recon_repo_for_cli_args(kwargs): '''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. ''' check.dict_param(kwargs, 'kwargs') _cli_load_invariant(kwargs.get('pipeline_name') is None) if kwargs.get('workspace'): check.not_implemented('Workspace not supported yet in this cli command') if kwargs.get('repository_yaml') or all_none(kwargs): _cli_load_invariant(kwargs.get('module_name') is None) _cli_load_invariant(kwargs.get('python_file') is None) _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = ( os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME ) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format(repo_yaml), ) return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) elif kwargs.get('module_name') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name']) elif kwargs.get('python_file') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name'], kwargs.get('working_directory') if kwargs.get('working_directory') else os.getcwd(), ) else: _cli_load_invariant(False)
def _location_handle_from_location_config(location_config, yaml_path, opt_ins): check.dict_param(location_config, 'location_config') check.str_param(yaml_path, 'yaml_path') check.set_param(opt_ins, 'opt_ins') user_process_api = UserProcessApi.GRPC if 'grpc' in opt_ins else UserProcessApi.CLI if is_target_config(location_config): return _location_handle_from_target_config(location_config, yaml_path, user_process_api) elif 'grpc_server' in location_config: return _location_handle_from_grpc_server_config( location_config['grpc_server'], yaml_path) elif 'python_environment' in location_config: return _location_handle_from_python_environment_config( location_config['python_environment'], yaml_path, user_process_api) else: check.not_implemented( 'Unsupported location config: {}'.format(location_config))
def _location_handle_from_location_config(location_config, yaml_path, python_user_process_api): check.dict_param(location_config, 'location_config') check.str_param(yaml_path, 'yaml_path') check.inst_param(python_user_process_api, 'python_user_process_api', UserProcessApi) if is_target_config(location_config): return _location_handle_from_target_config(location_config, yaml_path, python_user_process_api) elif 'grpc_server' in location_config: return _location_handle_from_grpc_server_config( location_config['grpc_server'], yaml_path) elif 'python_environment' in location_config: return _location_handle_from_python_environment_config( location_config['python_environment'], yaml_path, python_user_process_api) else: check.not_implemented( 'Unsupported location config: {}'.format(location_config))
def test_not_implemented(): with pytest.raises(NotImplementedCheckError, match='some string'): check.not_implemented('some string')
def terminate(self, run_id): check.not_implemented("Termination not supported.")
def materialize_runtime_value(self, _context, _config_value, _runtime_value): ''' How to materialize a runtime value given configuration. ''' check.not_implemented('Must implement')
def schema_type(self): check.not_implemented('Must override schema_type in {klass}'.format( klass=type(self).__name__))
def test_not_implemented(): with pytest.raises(NotImplementedCheckError, match="some string"): check.not_implemented("some string") with pytest.raises(CheckError, match="desc argument must be a string"): check.not_implemented(None)
def query_text(self): check.not_implemented('table cannot be a standalone query')
def from_target(self): check.not_implemented('must implemented in subclass')
def terminate(self, run_id): check.str_param(run_id, 'run_id') check.not_implemented('Termination not yet implemented')