def test_decimal_different_precisions(self):
data = [
decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
]
type = pa.decimal(precision=13, scale=3)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_different_precisions(self):
data = [
decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
]
type = pa.decimal(precision=13, scale=3)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_type_schema_pickling():
cases = [
pa.int8(),
pa.string(),
pa.binary(),
pa.binary(10),
pa.list_(pa.string()),
pa.struct([
pa.field('a', 'int8'),
pa.field('b', 'string')
]),
pa.time32('s'),
pa.time64('us'),
pa.date32(),
pa.date64(),
pa.timestamp('ms'),
pa.timestamp('ns'),
pa.decimal(12, 2),
pa.field('a', 'string', metadata={b'foo': b'bar'})
]
for val in cases:
roundtripped = pickle.loads(pickle.dumps(val))
assert val == roundtripped
fields = []
for i, f in enumerate(cases):
if isinstance(f, pa.Field):
fields.append(f)
else:
fields.append(pa.field('_f{}'.format(i), f))
schema = pa.schema(fields, metadata={b'foo': b'bar'})
roundtripped = pickle.loads(pickle.dumps(schema))
assert schema == roundtripped
def test_decimal_large_integer(self):
data = [
decimal.Decimal('-394029506937548693.42983'),
decimal.Decimal('32358695912932.01033')
]
type = pa.decimal(precision=23, scale=5)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_64_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('-129934.123331'),
decimal.Decimal('129534.123731'),
]
})
converted = pa.Table.from_pandas(expected, preserve_index=False)
field = pa.field('decimals', pa.decimal(12, 6))
schema = pa.schema([field])
assert converted.schema.equals(schema)
def test_decimal_128_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('394092382910493.12341234678'),
-decimal.Decimal('314292388910493.12343437128'),
]
})
converted = pa.Table.from_pandas(expected, preserve_index=False)
field = pa.field('decimals', pa.decimal(26, 11))
schema = pa.schema([field])
assert converted.schema.equals(schema)
def test_decimal_128_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('394092382910493.12341234678'),
-decimal.Decimal('314292388910493.12343437128'),
]
})
converted = pa.Table.from_pandas(expected, preserve_index=False)
field = pa.field('decimals', pa.decimal(26, 11))
schema = pa.schema([field])
assert converted.schema.equals(schema)
def test_decimal_64_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('-129934.123331'),
decimal.Decimal('129534.123731'),
]
})
converted = pa.Table.from_pandas(expected)
field = pa.Field.from_py('decimals', pa.decimal(12, 6))
schema = pa.Schema.from_fields([field])
assert converted.schema.equals(schema)
def test_decimal_32_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('-1234.123'),
decimal.Decimal('1234.439'),
]
})
converted = pa.Table.from_pandas(expected)
field = pa.Field.from_py('decimals', pa.decimal(7, 3))
schema = pa.Schema.from_fields([field])
assert converted.schema.equals(schema)
def test_decimal_64_from_pandas(self):
expected = pd.DataFrame({
'decimals': [
decimal.Decimal('-129934.123331'),
decimal.Decimal('129534.123731'),
]
})
converted = pa.Table.from_pandas(expected, preserve_index=False)
field = pa.field('decimals', pa.decimal(12, 6))
schema = pa.schema([field])
assert converted.schema.equals(schema)
('type', 'expected'),
[
(pa.null(), 'float64'),
(pa.bool_(), 'bool'),
(pa.int8(), 'int8'),
(pa.int16(), 'int16'),
(pa.int32(), 'int32'),
(pa.int64(), 'int64'),
(pa.uint8(), 'uint8'),
(pa.uint16(), 'uint16'),
(pa.uint32(), 'uint32'),
(pa.uint64(), 'uint64'),
(pa.float16(), 'float16'),
(pa.float32(), 'float32'),
(pa.float64(), 'float64'),
(pa.date32(), 'date'),
(pa.date64(), 'date'),
(pa.binary(), 'bytes'),
(pa.binary(length=4), 'bytes'),
(pa.string(), 'unicode'),
(pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
(pa.decimal(18, 3), 'decimal'),
(pa.timestamp('ms'), 'datetime'),
(pa.timestamp('us', 'UTC'), 'datetimetz'),
pytest.mark.xfail((pa.time32('s'), None), raises=NotImplementedError),
pytest.mark.xfail((pa.time64('us'), None), raises=NotImplementedError),
]
)
def test_logical_type(type, expected):
assert get_logical_type(type) == expected
def test_decimal(self):
data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
type = pa.decimal(precision=7, scale=3)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
safe_cases = [(np.array([0, 1, 2, 3], dtype='i1'), pa.uint8(),
np.array([0, 1, 2, 3], dtype='u1'), pa.uint8()),
(np.array([0, 1, 2, 3], dtype='i2'), pa.uint16(),
np.array([0, 1, 2, 3], dtype='u2'), pa.uint16())]
for case in safe_cases:
_check_cast_case(case)
def test_simple_type_construction():
result = pa.lib.TimestampType()
with pytest.raises(TypeError):
str(result)
@pytest.mark.parametrize(
('type', 'expected'),
[(pa.null(), 'float64'), (pa.bool_(), 'bool'), (pa.int8(), 'int8'),
(pa.int16(), 'int16'), (pa.int32(), 'int32'), (pa.int64(), 'int64'),
(pa.uint8(), 'uint8'), (pa.uint16(), 'uint16'), (pa.uint32(), 'uint32'),
(pa.uint64(), 'uint64'), (pa.float16(), 'float16'),
(pa.float32(), 'float32'), (pa.float64(), 'float64'),
(pa.date32(), 'date'), (pa.date64(), 'date'), (pa.binary(), 'bytes'),
(pa.binary(length=4), 'bytes'), (pa.string(), 'unicode'),
(pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
(pa.decimal(18, 3), 'decimal'), (pa.timestamp('ms'), 'datetime'),
(pa.timestamp('us', 'UTC'), 'datetimetz'), (pa.time32('s'), 'time'),
(pa.time64('us'), 'time')])
def test_logical_type(type, expected):
assert get_logical_type(type) == expected
def test_decimal_no_scale(self):
data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
type = pa.decimal(precision=10)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_no_whole_part(self):
data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
type = pa.decimal(precision=7, scale=7)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_negative(self):
data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
type = pa.decimal(precision=10, scale=6)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_large_integer(self):
data = [decimal.Decimal('-394029506937548693.42983'),
decimal.Decimal('32358695912932.01033')]
type = pa.decimal(precision=23, scale=5)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal(self):
data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
type = pa.decimal(precision=7, scale=3)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_decimal_negative(self):
data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
type = pa.decimal(precision=10, scale=6)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
def test_is_decimal():
assert types.is_decimal(pa.decimal(19, 4))
assert not types.is_decimal(pa.int32())
def test_is_decimal():
assert types.is_decimal(pa.decimal(19, 4))
assert not types.is_decimal(pa.int32())
def test_decimal_no_scale(self):
data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
type = pa.decimal(precision=10)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data
(pa.int32(), 'int32'),
(pa.int64(), 'int64'),
(pa.uint8(), 'uint8'),
(pa.uint16(), 'uint16'),
(pa.uint32(), 'uint32'),
(pa.uint64(), 'uint64'),
(pa.float16(), 'float16'),
(pa.float32(), 'float32'),
(pa.float64(), 'float64'),
(pa.date32(), 'date'),
(pa.date64(), 'date'),
(pa.binary(), 'bytes'),
(pa.binary(length=4), 'bytes'),
(pa.string(), 'unicode'),
(pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
(pa.decimal(18, 3), 'decimal'),
(pa.timestamp('ms'), 'datetime'),
(pa.timestamp('us', 'UTC'), 'datetimetz'),
(pa.time32('s'), 'time'),
(pa.time64('us'), 'time')
]
)
def test_logical_type(type, expected):
assert get_logical_type(type) == expected
def test_array_conversions_no_sentinel_values():
arr = np.array([1, 2, 3, 4], dtype='int8')
refcount = sys.getrefcount(arr)
arr2 = pa.array(arr) # noqa
assert sys.getrefcount(arr) == (refcount + 1)
def test_decimal_no_whole_part(self):
data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
type = pa.decimal(precision=7, scale=7)
arr = pa.array(data, type=type)
assert arr.to_pylist() == data