def byte_key_value_sink_from_params(
params: Parameters,
*,
output_namespace: str = "output",
eval_context: Optional[Dict] = None,
) -> KeyValueSink[str, bytes]:
"""
Get a binary key-value sink based on parameters.
This should be passed a parameter namespace. If the "type" field is present, it should
be the name of a class or method. If a class, its static `from_parameters` method will be
called with these parameters and should return a `KeyValueSink[str, bytes]`. If a callable,
it will be called with these parameters (and should also return a KeyValueSink[str, bytes].
The type 'zip' is a shortcut for a key-value zip file. 'directory' is a shortcut for
writing the output files to the specified directory.
If additional imports are needed to resolve 'type', they can be specified as a Python list in
the `import` field.
If no type is specified, a 'directory' sink will be created.
"""
# to be sure the default special values can be evaluated, we want to include this module
# itself in the evaluation context. We combine it with eval_context, giving priority to
# the context specified by the user
effective_context = dict(globals())
effective_context.update(eval_context or {})
return params.object_from_parameters( # type: ignore
output_namespace,
KeyValueSink,
special_factories=_BYTE_KEY_VALUE_SINK_SPECIAL_VALUES,
default_factory=_DirectoryBytesKeyValueSink,
context=effective_context,
factory_namespace_param_name="type",
)
def byte_key_value_linear_source_from_params(
params: Parameters,
*,
input_namespace: str = "input",
eval_context: Optional[Dict] = None,
) -> KeyValueLinearSource[str, bytes]:
"""
Get a key-value source based on parameters.
This should be passed a parameter namespace. If the "type" field is present, it should
be the name of a class or method. If a class, its static `from_parameters` method will be
called with these parameters and should return a `KeyValueLinearSource[str, bytes]`. If a
callable, it will be called with these parameters (and should also return a
KeyValueLinearSource[str, bytes]).
The type 'zip' is a shortcut for a key-value zip file. 'file-map' is a shortcut for a
docID to file map.
If additional imports are needed to resolve 'type', they can be specified as a Python
list in
the `import` field.
If no type is specified, a source will be constructed from the doc-id-to-file map specified
by the `docIdToFileMap` parameter.
This differs from "byte_key_value_source_from_params" only in that is relaxes the guarantee
on what is returned to only requiring iterability over mappings and not random access.
"""
# to be sure the default special values can be evaluated, we want to include this module
# itself in the evaluation context. We combine it with eval_context, giving priority to
# the context specified by the user
effective_context = dict(globals())
effective_context.update(eval_context or {})
return params.object_from_parameters(
input_namespace,
KeyValueLinearSource,
special_factories=_BYTE_KEY_VALUE_SOURCE_SPECIAL_VALUES,
default_factory=_doc_id_binary_source_from_params,
context=effective_context,
factory_namespace_param_name="type",
)
