def __init__(
self,
column: AutoMapperColumnOrColumnLikeType,
check: Union[AutoMapperAnyDataType, List[AutoMapperAnyDataType]],
value: _TAutoMapperDataType,
else_: Optional[_TAutoMapperDataType] = None,
):
super().__init__()
self.column: AutoMapperColumnOrColumnLikeType = column
if isinstance(check, list):
self.check: Union[AutoMapperDataTypeBase,
List[AutoMapperDataTypeBase]] = [
a if isinstance(a, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(
value=a) for a in check
]
else:
self.check = (check if isinstance(check, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(value=check))
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
if else_:
self.else_: AutoMapperDataTypeBase = (
cast(AutoMapperDataTypeBase, else_) if isinstance(
value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
else:
self.else_ = AutoMapperDataTypeLiteral(None)
def __init__(self,
value: Optional[Union[List[_T], AutoMapperDataTypeBase,
List[AutoMapperDataTypeBase],
List[AutoMapperTextLikeBase]]],
remove_nulls: bool = True,
include_null_properties: bool = True) -> None:
"""
Generates a list (array) in Spark
:param value: items to make into an array
:param remove_nulls: whether to remove nulls from the array
"""
super().__init__()
# can a single mapper or a list of mappers
self.remove_nulls: bool = remove_nulls
self.value: Union[AutoMapperDataTypeBase, List[AutoMapperDataTypeBase]]
if not value:
self.value = []
if isinstance(value, str):
self.value = AutoMapperValueParser.parse_value(value=value)
elif isinstance(value, AutoMapperDataTypeBase):
self.value = value
elif isinstance(value, List):
self.value = [AutoMapperValueParser.parse_value(v) for v in value]
# if there are more than two items we have to maintain the same schema in children or Spark errors
if include_null_properties:
self.include_null_properties(
include_null_properties=include_null_properties)
else:
raise ValueError(f"{type(value)} is not supported")
def set_value_from_kwargs(self, kwargs: Dict[str, Any]) -> None:
self.value = {
PythonKeywordCleaner.from_python_safe(name=parameter_name):
AutoMapperValueParser.parse_value(column_name=parameter_name,
value=parameter_value)
for parameter_name, parameter_value in kwargs.items()
}
def __init__(self, value: Dict[str, Any]) -> None:
super().__init__()
assert isinstance(value, dict)
self.value: Dict[str, AutoMapperDataTypeBase] = {
key: AutoMapperValueParser.parse_value(value)
for key, value in value.items()
}
def __init__(self, value: AutoMapperDataTypeBase):
super().__init__()
self.value: AutoMapperDataTypeBase = (
value
if isinstance(value, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(value)
)
def __init__(self,
check: AutoMapperColumnOrColumnLikeType,
value: _TAutoMapperDataType,
when_null_or_empty: Optional[_TAutoMapperDataType] = None):
super().__init__()
self.check: AutoMapperColumnOrColumnLikeType = check
self.value: AutoMapperDataTypeBase = value \
if isinstance(value, AutoMapperDataTypeBase) \
else AutoMapperValueParser.parse_value(value)
if when_null_or_empty:
self.when_null: AutoMapperDataTypeBase = cast(AutoMapperDataTypeBase, when_null_or_empty) \
if isinstance(value, AutoMapperDataTypeBase) \
else AutoMapperValueParser.parse_value(value)
else:
self.when_null = AutoMapperDataTypeLiteral(None)
def __init__(
self,
dst_column: str,
value: AutoMapperAnyDataType,
column_schema: Optional[StructField],
include_null_properties: bool,
enable_schema_pruning: bool,
skip_if_columns_null_or_empty: Optional[List[str]] = None,
) -> None:
"""
This class handles assigning to a single column
"""
super().__init__()
# should only have one parameter
self.dst_column: str = dst_column
self.column_schema: Optional[StructField] = column_schema
self.value: AutoMapperDataTypeBase = (
AutoMapperValueParser.parse_value(column_name=dst_column, value=value)
if not isinstance(value, AutoMapperDataTypeBase)
else value
)
self.skip_if_columns_null_or_empty: Optional[
List[str]
] = skip_if_columns_null_or_empty
if include_null_properties:
self.value.include_null_properties(
include_null_properties=include_null_properties
)
self.enable_schema_pruning: bool = enable_schema_pruning
def __init__(
self,
*args: _T,
):
super().__init__()
self.value: List[AutoMapperDataTypeBase] = [
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value) for value in args
]
def __init__(
self,
*columns: AutoMapperColumnOrColumnLikeType,
):
super().__init__()
self.columns: List[AutoMapperColumnOrColumnLikeType] = [
AutoMapperValueParser.parse_value(column) for column in columns
]
def __init__(self,
column: AutoMapperColumnOrColumnLikeType,
check: Union[str, List[str]],
value: _TAutoMapperDataType,
else_: Optional[_TAutoMapperDataType] = None):
super().__init__()
self.column: AutoMapperColumnOrColumnLikeType = column
self.check: Union[str, List[str]] = check
self.value: AutoMapperDataTypeBase = value \
if isinstance(value, AutoMapperDataTypeBase) \
else AutoMapperValueParser.parse_value(value)
if else_:
self.else_: AutoMapperDataTypeBase = cast(AutoMapperDataTypeBase, else_) \
if isinstance(value, AutoMapperDataTypeBase) \
else AutoMapperValueParser.parse_value(value)
else:
self.else_ = AutoMapperDataTypeLiteral(None)
def __init__(self,
*args: Union[AutoMapperNativeTextType, AutoMapperWrapperType,
AutoMapperTextLikeBase, AutoMapperDataTypeBase]):
super().__init__()
self.value: List[AutoMapperDataTypeBase] = [
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value) for value in args
]
def __init__(
self,
column: AutoMapperColumnOrColumnLikeType,
mapping: Dict[Optional[AutoMapperTextInputType],
AutoMapperAnyDataType],
default: Optional[AutoMapperAnyDataType] = None,
):
super().__init__(value="")
self.column: AutoMapperColumnOrColumnLikeType = column
self.mapping: Dict[AutoMapperAnyDataType, AutoMapperDataTypeBase] = {
key: (value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value))
for key, value in mapping.items()
}
assert self.mapping
self.default: AutoMapperDataTypeBase = (
default if isinstance(default, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(default))
def __init__(
self,
check: AutoMapperColumnOrColumnLikeType,
value: _TAutoMapperDataType,
when_null: Optional[Union[AutoMapperTextLikeBase,
_TAutoMapperDataType]] = None,
):
super().__init__()
self.check: AutoMapperColumnOrColumnLikeType = check
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
if when_null:
self.when_null: AutoMapperDataTypeBase = (
cast(AutoMapperDataTypeBase, when_null) if isinstance(
value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
else:
self.when_null = AutoMapperDataTypeLiteral(None)
def __init__(self, value: AutoMapperNumberInputType) -> None:
"""
Converts the value to a timestamp type in Spark
:param value: value
:param formats: (Optional) formats to use for trying to parse the value otherwise uses Spark defaults
"""
super().__init__()
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
def __init__(
self,
column: AutoMapperColumnOrColumnLikeType,
if_exists: Optional[_TAutoMapperDataType],
if_not_exists: Optional[_TAutoMapperDataType],
):
super().__init__()
self.column: AutoMapperColumnOrColumnLikeType = (
AutoMapperValueParser.parse_value(value=column))
self.if_exists_column: Optional[AutoMapperDataTypeBase] = None
if if_exists:
self.if_exists_column = (
if_exists if isinstance(if_exists, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=if_exists))
self.if_not_exists: Optional[AutoMapperDataTypeBase] = None
if if_not_exists:
self.if_not_exists = (if_not_exists if isinstance(
if_not_exists, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(
value=if_not_exists))
def __init__(self,
*args: Union[AutoMapperNativeTextType, AutoMapperWrapperType,
AutoMapperTextLikeBase,
AutoMapperDataTypeBase, ]):
super().__init__()
self.value: List[AutoMapperDataTypeBase] = [
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value) for value in args
]
# always include null properties in a concat operation
self.include_null_properties(include_null_properties=True)
def __init__(
self,
column: AutoMapperColumnOrColumnLikeType,
check: Union[AutoMapperAnyDataType, List[AutoMapperAnyDataType]],
value: _TAutoMapperDataType,
):
super().__init__()
self.column: AutoMapperColumnOrColumnLikeType = column
if isinstance(check, list):
self.check: Union[AutoMapperDataTypeBase,
List[AutoMapperDataTypeBase]] = [
a if isinstance(a, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(a)
for a in check
]
else:
self.check = (check if isinstance(check, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(check))
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value))
def __init__(self, value: AutoMapperAnyDataType, format_: str) -> None:
"""
Converts the date or timestamp to a string in Spark
:param value: value
:param format_: (Optional) formats to use for trying to format the value otherwise uses Spark defaults
"""
super().__init__()
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
self.format_: str = format_
def __init__(self, value: AutoMapperAmountInputType, precision: int, scale: int):
"""
Specifies the value should be used as a decimal
:param value:
:param precision: the maximum total number of digits (on both sides of dot)
:param scale: the number of digits on right side of dot
"""
super().__init__()
self.precision = precision
self.scale = scale
self.value: AutoMapperDataTypeBase = (
value
if isinstance(value, AutoMapperDataTypeBase)
else AutoMapperValueParser.parse_value(value=value)
)
def __init__(self, **kwargs: Any) -> None:
"""
base class for complex types
:param kwargs:
"""
super().__init__()
# this flag specifies that we should include all values in the column_spec event NULLs
self.include_nulls: bool = "include_nulls" in kwargs
self.value: Dict[str, AutoMapperDataTypeBase] = {
parameter_name if not parameter_name.endswith(
"_"
) # some property names are python keywords so we have to append with _
else parameter_name[:-1]:
AutoMapperValueParser.parse_value(parameter_value)
for parameter_name, parameter_value in kwargs.items()
}
def __init__(
self,
dst_column: str,
value: AutoMapperAnyDataType,
column_schema: Optional[StructField],
include_null_properties: bool,
skip_if_columns_null_or_empty: Optional[List[str]] = None) -> None:
super().__init__()
# should only have one parameter
self.dst_column: str = dst_column
self.column_schema: Optional[StructField] = column_schema
self.value: AutoMapperDataTypeBase = AutoMapperValueParser.parse_value(value) \
if not isinstance(value, AutoMapperDataTypeBase) \
else value
self.skip_if_columns_null_or_empty: Optional[
List[str]] = skip_if_columns_null_or_empty
if include_null_properties:
self.value.include_null_properties(
include_null_properties=include_null_properties)
def __init__(self,
value: AutoMapperDateInputType,
formats: Optional[List[str]] = None) -> None:
"""
Converts a value to date only
For datetime use the datetime mapper type
:param value: value
:param formats: (Optional) formats to use for trying to parse the value otherwise uses:
y-M-d
yyyyMMdd
M/d/y
"""
super().__init__()
# keep string separate so we can parse it to date
self.value: AutoMapperDataTypeBase = (
value if isinstance(value, AutoMapperDataTypeBase) else
AutoMapperValueParser.parse_value(value=value))
self.formats: Optional[List[str]] = formats
def __init__(self, value: AutoMapperAmountInputType):
super().__init__()
self.value: AutoMapperDataTypeBase = value \
if isinstance(value, AutoMapperDataTypeBase) \
else AutoMapperValueParser.parse_value(value)
def __init__(self, **kwargs: AutoMapperAnyDataType) -> None:
super().__init__()
self.value: Dict[str, AutoMapperDataTypeBase] = {
key: AutoMapperValueParser.parse_value(value)
for key, value in kwargs.items()
}