def byte_key_value_sink_from_params( params: Parameters, *, output_namespace: str = "output", eval_context: Optional[Dict] = None, ) -> KeyValueSink[str, bytes]: """ Get a binary key-value sink based on parameters. This should be passed a parameter namespace. If the "type" field is present, it should be the name of a class or method. If a class, its static `from_parameters` method will be called with these parameters and should return a `KeyValueSink[str, bytes]`. If a callable, it will be called with these parameters (and should also return a KeyValueSink[str, bytes]. The type 'zip' is a shortcut for a key-value zip file. 'directory' is a shortcut for writing the output files to the specified directory. If additional imports are needed to resolve 'type', they can be specified as a Python list in the `import` field. If no type is specified, a 'directory' sink will be created. """ # to be sure the default special values can be evaluated, we want to include this module # itself in the evaluation context. We combine it with eval_context, giving priority to # the context specified by the user effective_context = dict(globals()) effective_context.update(eval_context or {}) return params.object_from_parameters( # type: ignore output_namespace, KeyValueSink, special_factories=_BYTE_KEY_VALUE_SINK_SPECIAL_VALUES, default_factory=_DirectoryBytesKeyValueSink, context=effective_context, factory_namespace_param_name="type", )
def byte_key_value_linear_source_from_params( params: Parameters, *, input_namespace: str = "input", eval_context: Optional[Dict] = None, ) -> KeyValueLinearSource[str, bytes]: """ Get a key-value source based on parameters. This should be passed a parameter namespace. If the "type" field is present, it should be the name of a class or method. If a class, its static `from_parameters` method will be called with these parameters and should return a `KeyValueLinearSource[str, bytes]`. If a callable, it will be called with these parameters (and should also return a KeyValueLinearSource[str, bytes]). The type 'zip' is a shortcut for a key-value zip file. 'file-map' is a shortcut for a docID to file map. If additional imports are needed to resolve 'type', they can be specified as a Python list in the `import` field. If no type is specified, a source will be constructed from the doc-id-to-file map specified by the `docIdToFileMap` parameter. This differs from "byte_key_value_source_from_params" only in that is relaxes the guarantee on what is returned to only requiring iterability over mappings and not random access. """ # to be sure the default special values can be evaluated, we want to include this module # itself in the evaluation context. We combine it with eval_context, giving priority to # the context specified by the user effective_context = dict(globals()) effective_context.update(eval_context or {}) return params.object_from_parameters( input_namespace, KeyValueLinearSource, special_factories=_BYTE_KEY_VALUE_SOURCE_SPECIAL_VALUES, default_factory=_doc_id_binary_source_from_params, context=effective_context, factory_namespace_param_name="type", )