def convert(
self,
value: typing.Any,
python_type_hint: typing.Optional[typing.Type] = None) -> Scalar:
if self.primitive_type:
return Scalar(primitive=Primitive(**{self.primitive_type: value}))
if self.scalar_type:
return Scalar(**{self.scalar_type: value})
raise NotImplementedError("Not implemented yet!")
def convert_to_blob(
self,
ctx: typing.Optional[click.Context],
param: typing.Optional[click.Parameter],
value: typing.Union[Directory, FileParam],
) -> Literal:
if isinstance(value, Directory):
uri = self.get_uri_for_dir(value)
else:
uri = value.filepath
if self._remote and value.local:
fp = pathlib.Path(value.filepath)
md5, _ = script_mode.hash_file(value.filepath)
df_remote_location = self._create_upload_fn(filename=fp.name,
content_md5=md5)
self._flyte_ctx.file_access.put_data(
fp, df_remote_location.signed_url)
uri = df_remote_location.native_url
lit = Literal(scalar=Scalar(blob=Blob(
metadata=BlobMetadata(type=self._literal_type.blob),
uri=uri,
), ), )
return lit
def to_literal(
self,
ctx: FlyteContext,
python_val: PipelineModel,
python_type: Type[PipelineModel],
expected: LiteralType,
) -> Literal:
local_path = ctx.file_access.get_random_local_path()
pathlib.Path(local_path).parent.mkdir(parents=True, exist_ok=True)
python_val.save(local_path)
remote_dir = ctx.file_access.get_random_remote_directory()
ctx.file_access.upload_directory(local_path, remote_dir)
return Literal(scalar=Scalar(blob=Blob(
uri=remote_dir, metadata=BlobMetadata(type=self._TYPE_INFO))))
def to_literal(
self,
ctx: FlyteContext,
python_val: MyDataset,
python_type: Type[MyDataset],
expected: LiteralType,
) -> Literal:
"""
This method is used to convert from the given python type object ``MyDataset`` to the Literal representation.
"""
# Step 1: let's upload all the data into a remote place recommended by Flyte
remote_dir = ctx.file_access.get_random_remote_directory()
ctx.file_access.upload_directory(python_val.base_dir, remote_dir)
# Step 2: let's return a pointer to this remote_dir in the form of a Literal
return Literal(scalar=Scalar(blob=Blob(
uri=remote_dir, metadata=BlobMetadata(type=self._TYPE_INFO))))
def convert_to_structured_dataset(self,
ctx: typing.Optional[click.Context],
param: typing.Optional[click.Parameter],
value: Directory) -> Literal:
uri = self.get_uri_for_dir(value, "00000.parquet")
lit = Literal(scalar=Scalar(
structured_dataset=literals.StructuredDataset(
uri=uri,
metadata=literals.StructuredDatasetMetadata(
structured_dataset_type=self._literal_type.
structured_dataset_type),
), ), )
return lit
def transform_inputs_to_parameters(
ctx: context_manager.FlyteContext,
interface: Interface) -> _interface_models.ParameterMap:
"""
Transforms the given interface (with inputs) to a Parameter Map with defaults set
:param interface: the interface object
"""
if interface is None or interface.inputs_with_defaults is None:
return _interface_models.ParameterMap({})
if interface.docstring is None:
inputs_vars = transform_variable_map(interface.inputs)
else:
inputs_vars = transform_variable_map(
interface.inputs, interface.docstring.input_descriptions)
params = {}
inputs_with_def = interface.inputs_with_defaults
for k, v in inputs_vars.items():
val, _default = inputs_with_def[k]
if _default is None and get_origin(val) is typing.Union and type(
None) in get_args(val):
from flytekit import Literal, Scalar
literal = Literal(scalar=Scalar(none_type=Void()))
params[k] = _interface_models.Parameter(var=v,
default=literal,
required=False)
else:
required = _default is None
default_lv = None
if _default is not None:
default_lv = TypeEngine.to_literal(
ctx,
_default,
python_type=interface.inputs[k],
expected=v.type)
params[k] = _interface_models.Parameter(var=v,
default=default_lv,
required=required)
return _interface_models.ParameterMap(params)