def test_cast_from_decimal_to_decimal(self):
self.assertEqual(
cast_to_decimal(
cast_to_decimal(1.526,
FloatType(),
DecimalType(scale=2),
options=BASE_OPTIONS),
DecimalType(scale=2),
DecimalType(scale=3),
options=BASE_OPTIONS,
),
1.53,
)
def guess_type_from_values_as_string(values, options):
# Reproduces inferences available in Spark
# PartitioningUtils.inferPartitionColumnValue()
# located in org.apache.spark.sql.execution.datasources
tested_types = (
IntegerType(),
LongType(),
DecimalType(),
DoubleType(),
TimestampType(),
StringType(),
)
string_type = StringType()
for tested_type in tested_types:
type_caster = get_caster(from_type=string_type,
to_type=tested_type,
options=options)
try:
for value in values:
casted_value = type_caster(value)
if casted_value is None and value not in ('null', None):
raise ValueError
return tested_type
except ValueError:
pass
# Should never happen
raise AnalysisException(
'Unable to find a matching type for some fields, even StringType did not work'
)
def test_cast_float_to_decimal_with_scale_and_other_rounding(self):
self.assertEqual(
cast_to_decimal(
10.987654321,
FloatType(),
DecimalType(precision=10, scale=8),
options=BASE_OPTIONS,
),
10.98765432,
)
def test_cast_float_to_decimal_with_scale(self):
self.assertEqual(
cast_to_decimal(
10.123456789,
FloatType(),
DecimalType(precision=10, scale=8),
options=BASE_OPTIONS,
),
10.12345679,
)
def test_cast_timestamp_to_decimal_with_scale(self):
self.assertEqual(
cast_to_decimal(
datetime.datetime(2019, 8, 28),
TimestampType(),
DecimalType(precision=11, scale=1),
options=BASE_OPTIONS,
),
1566943200.0,
)
def test_cast_timestamp_to_decimal_with_too_small_precision(self):
self.assertEqual(
cast_to_decimal(
datetime.datetime(2019, 8, 28),
TimestampType(),
DecimalType(precision=10, scale=1),
options=BASE_OPTIONS,
),
None,
)
def test_cast_date_to_decimal(self):
self.assertEqual(
cast_to_decimal(
datetime.date(2019, 8, 28),
DateType(),
DecimalType(),
options=BASE_OPTIONS,
),
None,
)
def test_cast_decimal_to_timestamp(self):
self.assertEqual(
cast_to_timestamp(147.58, DecimalType(), options=BASE_OPTIONS),
datetime.datetime(1970, 1, 1, 1, 2, 27, 580000),
)