def test_cast_from_decimal_to_decimal(self):
self.assertEqual(
cast_to_decimal(cast_to_decimal(1.526,
FloatType(),
DecimalType(scale=2),
options=BASE_OPTIONS),
DecimalType(scale=2),
DecimalType(scale=3),
options=BASE_OPTIONS), 1.53)
def test_cast_timestamp_to_decimal_with_scale(self):
self.assertEqual(
cast_to_decimal(datetime.datetime(2019, 8, 28),
TimestampType(),
DecimalType(precision=11, scale=1),
options=BASE_OPTIONS),
1566939600.0 + self.tz_diff.seconds)
def guess_type_from_values_as_string(values, options):
# Reproduces inferences available in Spark
# PartitioningUtils.inferPartitionColumnValue()
# located in org.apache.spark.sql.execution.datasources
tested_types = (
IntegerType(),
LongType(),
DecimalType(),
DoubleType(),
TimestampType(),
StringType()
)
string_type = StringType()
for tested_type in tested_types:
type_caster = get_caster(from_type=string_type, to_type=tested_type, options=options)
try:
for value in values:
casted_value = type_caster(value)
if casted_value is None and value not in ("null", None):
raise ValueError
return tested_type
except ValueError:
pass
# Should never happen
raise AnalysisException(
"Unable to find a matching type for some fields, even StringType did not work"
)
def test_cast_float_to_decimal_with_scale_and_other_rounding(self):
self.assertEqual(
cast_to_decimal(10.987654321,
FloatType(),
DecimalType(precision=10, scale=8),
options=BASE_OPTIONS), 10.98765432)
def test_cast_float_to_decimal_with_scale(self):
self.assertEqual(
cast_to_decimal(10.123456789,
FloatType(),
DecimalType(precision=10, scale=8),
options=BASE_OPTIONS), 10.12345679)
def test_cast_timestamp_to_decimal_with_too_small_precision(self):
self.assertEqual(
cast_to_decimal(datetime.datetime(2019, 8, 28),
TimestampType(),
DecimalType(precision=10, scale=1),
options=BASE_OPTIONS), None)
def test_cast_date_to_decimal(self):
self.assertEqual(
cast_to_decimal(datetime.date(2019, 8, 28),
DateType(),
DecimalType(),
options=BASE_OPTIONS), None)
def test_cast_decimal_to_timestamp(self):
self.assertEqual(
cast_to_timestamp(147.58, DecimalType(), options=BASE_OPTIONS),
datetime.datetime(1970, 1, 1, 0, 2, 27, 580000) + self.tz_diff)
def test_cast_timestamp_to_decimal_without_scale(self):
self.assertEqual(
cast_to_decimal(datetime.datetime(2019, 8, 28),
TimestampType(),
DecimalType(),
options=BASE_OPTIONS), 1566943200.0)