def test_cast_from_null():
in_data = [None] * 3
in_type = pa.null()
out_types = [
pa.null(),
pa.uint8(),
pa.float16(),
pa.utf8(),
pa.binary(),
pa.binary(10),
pa.list_(pa.int16()),
pa.decimal128(19, 4),
pa.timestamp('us'),
pa.timestamp('us', tz='UTC'),
pa.timestamp('us', tz='Europe/Paris'),
pa.struct([pa.field('a', pa.int32()),
pa.field('b', pa.list_(pa.int8())),
pa.field('c', pa.string())]),
]
for out_type in out_types:
_check_cast_case((in_data, in_type, in_data, out_type))
out_types = [
pa.dictionary(pa.int32(), pa.string()),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
]
in_arr = pa.array(in_data, type=pa.null())
for out_type in out_types:
with pytest.raises(NotImplementedError):
in_arr.cast(out_type)
def test_empty_cast():
types = [
pa.null(),
pa.bool_(),
pa.int8(),
pa.int16(),
pa.int32(),
pa.int64(),
pa.uint8(),
pa.uint16(),
pa.uint32(),
pa.uint64(),
pa.float16(),
pa.float32(),
pa.float64(),
pa.date32(),
pa.date64(),
pa.binary(),
pa.binary(length=4),
pa.string(),
]
for (t1, t2) in itertools.product(types, types):
try:
# ARROW-4766: Ensure that supported types conversion don't segfault
# on empty arrays of common types
pa.array([], type=t1).cast(t2)
except pa.lib.ArrowNotImplementedError:
continue
def get_arrow_type(self, dt, is_list):
"""get_arrow_type"""
if dt == dtypes.bool:
arrow_type = pa.bool_()
elif dt == dtypes.int8:
arrow_type = pa.int8()
elif dt == dtypes.int16:
arrow_type = pa.int16()
elif dt == dtypes.int32:
arrow_type = pa.int32()
elif dt == dtypes.int64:
arrow_type = pa.int64()
elif dt == dtypes.uint8:
arrow_type = pa.uint8()
elif dt == dtypes.uint16:
arrow_type = pa.uint16()
elif dt == dtypes.uint32:
arrow_type = pa.uint32()
elif dt == dtypes.uint64:
arrow_type = pa.uint64()
elif dt == dtypes.float16:
arrow_type = pa.float16()
elif dt == dtypes.float32:
arrow_type = pa.float32()
elif dt == dtypes.float64:
arrow_type = pa.float64()
else:
raise TypeError("Unsupported dtype for Arrow" + str(dt))
if is_list:
arrow_type = pa.list_(arrow_type)
return arrow_type
def test_schema_pyarrow_from_decimal_and_floating_types():
field_name = "decimal_test"
metadata = {b"metadata_k": b"metadata_v"}
precision = 20
scale = 2
pyarrow_field = pyarrow_field_from_dict(
{
"name": field_name,
"nullable": False,
"metadata": metadata,
"type": {"name": "decimal", "precision": precision, "scale": scale},
}
)
assert pyarrow_field.name == field_name
assert pyarrow_field.type == pyarrow.decimal128(precision=precision, scale=scale)
assert dict(pyarrow_field.metadata) == metadata
assert pyarrow_field.nullable is False
field_name = "floating_test"
metadata = {b"metadata_k": b"metadata_v"}
pyarrow_field = pyarrow_field_from_dict(
{
"name": field_name,
"nullable": False,
"metadata": metadata,
"type": {"name": "floatingpoint", "precision": "HALF"},
}
)
assert pyarrow_field.name == field_name
assert pyarrow_field.type == pyarrow.float16()
assert dict(pyarrow_field.metadata) == metadata
assert pyarrow_field.nullable is False
def get_many_types():
# returning them from a function is required because of pa.dictionary
# type holds a pyarrow array and test_array.py::test_toal_bytes_allocated
# checks that the default memory pool has zero allocated bytes
return (pa.null(), pa.bool_(), pa.int32(), pa.time32('s'), pa.time64('us'),
pa.date32(), pa.timestamp('us'), pa.timestamp('us', tz='UTC'),
pa.timestamp('us', tz='Europe/Paris'), pa.float16(), pa.float32(),
pa.float64(), pa.decimal128(19, 4), pa.string(), pa.binary(),
pa.binary(10), pa.list_(pa.int32()),
pa.struct([
pa.field('a', pa.int32()),
pa.field('b', pa.int8()),
pa.field('c', pa.string())
]),
pa.struct([
pa.field('a', pa.int32(), nullable=False),
pa.field('b', pa.int8(), nullable=False),
pa.field('c', pa.string())
]),
pa.union(
[pa.field('a', pa.binary(10)),
pa.field('b', pa.string())],
mode=pa.lib.UnionMode_DENSE),
pa.union(
[pa.field('a', pa.binary(10)),
pa.field('b', pa.string())],
mode=pa.lib.UnionMode_SPARSE),
pa.union([
pa.field('a', pa.binary(10), nullable=False),
pa.field('b', pa.string())
],
mode=pa.lib.UnionMode_SPARSE),
pa.dictionary(pa.int32(), pa.array(['a', 'b', 'c'])))
def _dtype_to_arrow_type(dtype: np.dtype) -> pa.DataType:
if dtype == np.int8:
return pa.int8()
elif dtype == np.int16:
return pa.int16()
elif dtype == np.int32:
return pa.int32()
elif dtype == np.int64:
return pa.int64()
elif dtype == np.uint8:
return pa.uint8()
elif dtype == np.uint16:
return pa.uint16()
elif dtype == np.uint32:
return pa.uint32()
elif dtype == np.uint64:
return pa.uint64()
elif dtype == np.float16:
return pa.float16()
elif dtype == np.float32:
return pa.float32()
elif dtype == np.float64:
return pa.float64()
elif dtype.kind == "M":
# [2019-09-17] Pandas only allows "ns" unit -- as in, datetime64[ns]
# https://github.com/pandas-dev/pandas/issues/7307#issuecomment-224180563
assert dtype.str.endswith("[ns]")
return pa.timestamp(unit="ns", tz=None)
elif dtype == np.object_:
return pa.string()
else:
raise RuntimeError("Unhandled dtype %r" % dtype)
def test_from_numpy_dtype():
cases = [(np.dtype('bool'), pa.bool_()), (np.dtype('int8'), pa.int8()),
(np.dtype('int16'), pa.int16()), (np.dtype('int32'), pa.int32()),
(np.dtype('int64'), pa.int64()), (np.dtype('uint8'), pa.uint8()),
(np.dtype('uint16'), pa.uint16()),
(np.dtype('uint32'), pa.uint32()),
(np.dtype('float16'), pa.float16()),
(np.dtype('float32'), pa.float32()),
(np.dtype('float64'), pa.float64()), (np.dtype('U'), pa.string()),
(np.dtype('S'), pa.binary()),
(np.dtype('datetime64[s]'), pa.timestamp('s')),
(np.dtype('datetime64[ms]'), pa.timestamp('ms')),
(np.dtype('datetime64[us]'), pa.timestamp('us')),
(np.dtype('datetime64[ns]'), pa.timestamp('ns'))]
for dt, pt in cases:
result = pa.from_numpy_dtype(dt)
assert result == pt
# Things convertible to numpy dtypes work
assert pa.from_numpy_dtype('U') == pa.string()
assert pa.from_numpy_dtype(np.unicode) == pa.string()
assert pa.from_numpy_dtype('int32') == pa.int32()
assert pa.from_numpy_dtype(bool) == pa.bool_()
with pytest.raises(NotImplementedError):
pa.from_numpy_dtype(np.dtype('O'))
with pytest.raises(TypeError):
pa.from_numpy_dtype('not_convertible_to_dtype')
def test_cast_from_null():
in_data = [None] * 3
in_type = pa.null()
out_types = [
pa.null(),
pa.uint8(),
pa.float16(),
pa.utf8(),
pa.binary(),
pa.binary(10),
pa.list_(pa.int16()),
pa.large_list(pa.uint8()),
pa.decimal128(19, 4),
pa.timestamp('us'),
pa.timestamp('us', tz='UTC'),
pa.timestamp('us', tz='Europe/Paris'),
pa.struct([pa.field('a', pa.int32()),
pa.field('b', pa.list_(pa.int8())),
pa.field('c', pa.string())]),
]
for out_type in out_types:
_check_cast_case((in_data, in_type, in_data, out_type))
out_types = [
pa.dictionary(pa.int32(), pa.string()),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
]
in_arr = pa.array(in_data, type=pa.null())
for out_type in out_types:
with pytest.raises(NotImplementedError):
in_arr.cast(out_type)
def test_type_to_pandas_dtype():
M8_ns = np.dtype('datetime64[ns]')
cases = [
(pa.null(), np.float64),
(pa.bool_(), np.bool_),
(pa.int8(), np.int8),
(pa.int16(), np.int16),
(pa.int32(), np.int32),
(pa.int64(), np.int64),
(pa.uint8(), np.uint8),
(pa.uint16(), np.uint16),
(pa.uint32(), np.uint32),
(pa.uint64(), np.uint64),
(pa.float16(), np.float16),
(pa.float32(), np.float32),
(pa.float64(), np.float64),
(pa.date32(), M8_ns),
(pa.date64(), M8_ns),
(pa.timestamp('ms'), M8_ns),
(pa.binary(), np.object_),
(pa.binary(12), np.object_),
(pa.string(), np.object_),
(pa.list_(pa.int8()), np.object_),
]
for arrow_type, numpy_type in cases:
assert arrow_type.to_pandas_dtype() == numpy_type
def test_type_ids():
# Having this fixed is very important because internally we rely on this id
# to parse from python
for idx, arrow_type in [
(0, pa.null()),
(1, pa.bool_()),
(2, pa.uint8()),
(3, pa.int8()),
(4, pa.uint16()),
(5, pa.int16()),
(6, pa.uint32()),
(7, pa.int32()),
(8, pa.uint64()),
(9, pa.int64()),
(10, pa.float16()),
(11, pa.float32()),
(12, pa.float64()),
(13, pa.string()),
(13, pa.utf8()),
(14, pa.binary()),
(16, pa.date32()),
(17, pa.date64()),
(18, pa.timestamp("us")),
(19, pa.time32("s")),
(20, pa.time64("us")),
(23, pa.decimal128(8, 1)),
(34, pa.large_utf8()),
(35, pa.large_binary()),
]:
assert idx == arrow_type.id
def test_empty_cast():
types = [
pa.null(),
pa.bool_(),
pa.int8(),
pa.int16(),
pa.int32(),
pa.int64(),
pa.uint8(),
pa.uint16(),
pa.uint32(),
pa.uint64(),
pa.float16(),
pa.float32(),
pa.float64(),
pa.date32(),
pa.date64(),
pa.binary(),
pa.binary(length=4),
pa.string(),
]
for (t1, t2) in itertools.product(types, types):
try:
# ARROW-4766: Ensure that supported types conversion don't segfault
# on empty arrays of common types
pa.array([], type=t1).cast(t2)
except pa.lib.ArrowNotImplementedError:
continue
def test_bit_width():
for ty, expected in [(pa.bool_(), 1), (pa.int8(), 8), (pa.uint32(), 32),
(pa.float16(), 16), (pa.decimal128(19, 4), 128),
(pa.binary(42), 42 * 8)]:
assert ty.bit_width == expected
for ty in [pa.binary(), pa.string(), pa.list_(pa.int16())]:
with pytest.raises(ValueError, match="fixed width"):
ty.bit_width
def test_table(n, types=None, offset=None, length=None, nullable=True):
if types is None:
types = [
pyarrow.null(),
pyarrow.bool_(),
pyarrow.int8(),
pyarrow.int16(),
pyarrow.int32(),
pyarrow.int64(),
pyarrow.uint8(),
pyarrow.uint16(),
pyarrow.uint32(),
pyarrow.uint64(),
pyarrow.float16(),
pyarrow.float32(),
pyarrow.float64(),
pyarrow.date32(),
pyarrow.date64(),
pyarrow.timestamp('s'),
pyarrow.timestamp('ms'),
pyarrow.timestamp('us'),
pyarrow.timestamp('ns'),
pyarrow.time32('s'),
pyarrow.time32('ms'),
pyarrow.time64('us'),
pyarrow.time64('ns'),
pyarrow.string(),
pyarrow.binary(),
pyarrow.binary(4),
pyarrow.dictionary(pyarrow.int32(), pyarrow.string(), True),
pyarrow.dictionary(pyarrow.int64(), pyarrow.int64(), True),
pyarrow.dictionary(pyarrow.int32(), pyarrow.string(), False),
pyarrow.dictionary(pyarrow.int64(), pyarrow.int64(), False),
pyarrow.list_(pyarrow.int32()),
pyarrow.struct([pyarrow.field('int32', pyarrow.int32())]),
pyarrow.list_(
pyarrow.struct([pyarrow.field('int32', pyarrow.int32())])),
pyarrow.struct(
[pyarrow.field('int32', pyarrow.list_(pyarrow.int32()))]),
]
data = list()
for t in types:
name = str(t)
array = TestArrayGenerator(n, t, False).array
if offset is not None:
array = array.slice(offset, length)
data.append(pyarrow.column(name, array))
if nullable:
name = str(t) + ' (null)'
array = TestArrayGenerator(n, t, True).array
if offset is not None:
array = array.slice(offset, length)
data.append(pyarrow.column(name, array))
return pyarrow.Table.from_arrays(data)
def test_arrow_schema_convertion_fail():
arrow_schema = pa.schema([
pa.field('list_of_int', pa.float16()),
])
mock_dataset = _mock_parquet_dataset([], arrow_schema)
with pytest.raises(ValueError, match='Cannot auto-create unischema due to unsupported column type'):
Unischema.from_arrow_schema(mock_dataset)
def test_half_float(self):
arr = pa.array([np.float16(1.5), None], type=pa.float16())
v = arr[0]
assert isinstance(v, pa.HalfFloatValue)
assert repr(v) == "1.5"
assert v.as_py() == 1.5
assert v == 1.5
assert arr[1] is pa.NA
def test_half_float(self):
arr = pa.array([np.float16(1.5), None], type=pa.float16())
v = arr[0]
assert isinstance(v, pa.HalfFloatValue)
assert repr(v) == "1.5"
assert v.as_py() == 1.5
assert v == 1.5
assert arr[1] is pa.NA
def test_arrow_schema_convertion_ignore():
arrow_schema = pa.schema([
pa.field('list_of_int', pa.float16()),
pa.field('struct', pa.struct([pa.field('a', pa.string()), pa.field('b', pa.int32())])),
])
mock_dataset = _mock_parquet_dataset([], arrow_schema)
unischema = Unischema.from_arrow_schema(mock_dataset, omit_unsupported_fields=True)
assert not hasattr(unischema, 'list_of_int')
def test_bit_width():
for ty, expected in [(pa.bool_(), 1),
(pa.int8(), 8),
(pa.uint32(), 32),
(pa.float16(), 16),
(pa.decimal128(19, 4), 128),
(pa.binary(42), 42 * 8)]:
assert ty.bit_width == expected
for ty in [pa.binary(), pa.string(), pa.list_(pa.int16())]:
with pytest.raises(ValueError, match="fixed width"):
ty.bit_width
def test_arrow_schema_convertion_fail(self):
arrow_schema = pa.schema([
pa.field('list_of_int', pa.float16()),
])
mock_dataset = _mock_parquet_dataset([], arrow_schema)
with self.assertRaises(ValueError) as ex:
Unischema.from_arrow_schema(mock_dataset)
assert 'Cannot auto-create unischema due to unsupported column type' in str(ex.exception)
def test_floats():
table = pa.table({
"float16":
pa.array([np.float16(1), np.float16(2),
np.float16(3)], pa.float16()),
"float32":
pa.array([1, 2, 3], pa.float32()),
"float64":
pa.array([1, 2, 3], pa.float64()),
})
with arrow_file(table) as path:
assert validate(path, ALL_CHECKS) is None
def _map_arrow_type(arrow_type):
arrow_to_dh = {
pa.null(): '',
pa.bool_(): '',
pa.int8(): 'byte',
pa.int16(): 'short',
pa.int32(): 'int',
pa.int64(): 'long',
pa.uint8(): '',
pa.uint16(): 'char',
pa.uint32(): '',
pa.uint64(): '',
pa.float16(): '',
pa.float32(): 'float',
pa.float64(): 'double',
pa.time32('s'): '',
pa.time32('ms'): '',
pa.time64('us'): '',
pa.time64('ns'): 'io.deephaven.time.DateTime',
pa.timestamp('us', tz=None): '',
pa.timestamp('ns', tz=None): '',
pa.date32(): 'java.time.LocalDate',
pa.date64(): 'java.time.LocalDate',
pa.binary(): '',
pa.string(): 'java.lang.String',
pa.utf8(): 'java.lang.String',
pa.large_binary(): '',
pa.large_string(): '',
pa.large_utf8(): '',
# decimal128(int precision, int scale=0)
# list_(value_type, int list_size=-1)
# large_list(value_type)
# map_(key_type, item_type[, keys_sorted])
# struct(fields)
# dictionary(index_type, value_type, …)
# field(name, type, bool nullable = True[, metadata])
# schema(fields[, metadata])
# from_numpy_dtype(dtype)
}
dh_type = arrow_to_dh.get(arrow_type)
if not dh_type:
# if this is a case of timestamp with tz specified
if isinstance(arrow_type, pa.TimestampType):
dh_type = "io.deephaven.time.DateTime"
if not dh_type:
raise DHError(f'unsupported arrow data type : {arrow_type}')
return {"deephaven:type": dh_type}
def generate_pyarray(csv_contents, memory):
"""
Generate a (pseudo-) random pyarrays object based on schema, data is stored in memory
Note:
Generating csv does not currently support writing half floats yet. Workaround is to write
halffloat as regular float, while still pass the halffloat version for reading.
This works for now because we dont need to actually write a halffloat to test reading one.
"""
assert isinstance(csv_contents, dict)
assert isinstance(csv_contents.get('columns'), list)
assert isinstance(csv_contents.get('rows_no'), int)
arrow_datatypes = [
arrow.pyarrow_dtypes_from_str(column['datatype'])
for column in csv_contents.get('columns')
if isinstance(column.get('datatype'), str)
and isinstance(column.get('min_value'), int)
and isinstance(column.get('max_value'), int)
and isinstance(column.get('name'), str)
]
# assure no items have been skipped
assert arrow_datatypes.__len__() == csv_contents.get('columns').__len__()
import pyarrow as pa
arrow_datatypes = [
pa.float32() if arrow_type == pa.float16() else arrow_type
for arrow_type in arrow_datatypes
]
dtypes_dict = arrow.pyarray_dtypes_dict()
def _retrieve_updated_column(number):
column = csv_contents['columns'][number]
column['datatype'] = dtypes_dict[arrow_datatypes[number]]
return column
# Convert datatype to a pyarray known value, pass columns that contain all required values
updated_columns = [
_retrieve_updated_column(c)
for c in range(0,
csv_contents.get('columns').__len__())
]
# Generate
return libio.generate_arrays(memory.retrieve_ptr(),
csv_contents.get('rows_no'), updated_columns)
def _create_parquet_schema(dtypes):
"""Create parquet schema from Pandas dtypes
Args:
dtypes: A dict or Series of dtypes
Returns:
pyarrow.Schema
"""
import pyarrow as pa
dtypes = dict(dtypes)
fields = []
for varname, vartype in dtypes.items():
if vartype == np.float16:
fields.append(pa.field(varname, pa.float16()))
elif vartype == np.float32:
fields.append(pa.field(varname, pa.float32()))
elif vartype == np.float64:
fields.append(pa.field(varname, pa.float64()))
elif vartype == np.int8:
fields.append(pa.field(varname, pa.int8()))
elif vartype == np.int16:
fields.append(pa.field(varname, pa.int16()))
elif vartype == np.int32:
fields.append(pa.field(varname, pa.int32()))
elif vartype == np.int64:
fields.append(pa.field(varname, pa.int64()))
elif vartype == np.uint8:
fields.append(pa.field(varname, pa.uint8()))
elif vartype == np.uint16:
fields.append(pa.field(varname, pa.uint16()))
elif vartype == np.uint32:
fields.append(pa.field(varname, pa.uint32()))
elif vartype == np.uint64:
fields.append(pa.field(varname, pa.uint64()))
elif vartype == np.bool_:
fields.append(pa.field(varname, pa.bool_()))
elif (vartype == object) | (vartype.name == 'category'):
fields.append(pa.field(varname, pa.string()))
elif np.issubdtype(vartype, np.datetime64):
fields.append(pa.field(varname, pa.timestamp('ns')))
assert len(dtypes) == len(fields)
schema = pa.schema(fields)
return schema
def _from_jvm_float_type(jvm_type):
"""
Convert a JVM float type to its Python equivalent.
Parameters
----------
jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$FloatingPoint
Returns
-------
typ: pyarrow.DataType
"""
precision = jvm_type.getPrecision().toString()
if precision == 'HALF':
return pa.float16()
elif precision == 'SINGLE':
return pa.float32()
elif precision == 'DOUBLE':
return pa.float64()
def _from_jvm_float_type(jvm_type):
"""
Convert a JVM float type to its Python equivalent.
Parameters
----------
jvm_type: org.apache.arrow.vector.types.pojo.ArrowType$FloatingPoint
Returns
-------
typ: pyarrow.DataType
"""
precision = jvm_type.getPrecision().toString()
if precision == 'HALF':
return pa.float16()
elif precision == 'SINGLE':
return pa.float32()
elif precision == 'DOUBLE':
return pa.float64()
def normalize_arrow_dtype(dtype): # noqa: C901
if dtype in ['bool']:
return pa.bool_()
if dtype in ['int8_t', 'int8', 'byte']:
return pa.int8()
if dtype in ['uint8_t', 'uint8', 'char']:
return pa.uint8()
if dtype in ['int16_t', 'int16', 'short']:
return pa.int16()
if dtype in ['uint16_t', 'uint16']:
return pa.uint16()
if dtype in ['int32_t', 'int32', 'int']:
return pa.int32()
if dtype in ['uint32_t', 'uint32']:
return pa.uint32()
if dtype in ['int64_t', 'int64', 'long']:
return pa.int64()
if dtype in ['uint64_t', 'uint64']:
return pa.uint64()
if dtype in ['half']:
return pa.float16()
if dtype in ['float', 'float32']:
return pa.float32()
if dtype in ['double', 'float64']:
return pa.float64()
if dtype in ['string', 'std::string', 'std::__1::string', 'str']:
return pa.large_string()
if dtype in ['large_list<item: int32>']:
return pa.large_list(pa.int32())
if dtype in ['large_list<item: uint32>']:
return pa.large_list(pa.uint32())
if dtype in ['large_list<item: int64>']:
return pa.large_list(pa.int64())
if dtype in ['large_list<item: uint64>']:
return pa.large_list(pa.uint64())
if dtype in ['large_list<item: float>']:
return pa.large_list(pa.float())
if dtype in ['large_list<item: double>']:
return pa.large_list(pa.double())
if dtype in ['null', 'NULL', 'None', None]:
return pa.null()
raise ValueError('Unsupported data type: %s' % dtype)
def get_many_types():
# returning them from a function is required because of pa.dictionary
# type holds a pyarrow array and test_array.py::test_toal_bytes_allocated
# checks that the default memory pool has zero allocated bytes
return (
pa.null(),
pa.bool_(),
pa.int32(),
pa.time32('s'),
pa.time64('us'),
pa.date32(),
pa.timestamp('us'),
pa.timestamp('us', tz='UTC'),
pa.timestamp('us', tz='Europe/Paris'),
pa.float16(),
pa.float32(),
pa.float64(),
pa.decimal128(19, 4),
pa.string(),
pa.binary(),
pa.binary(10),
pa.list_(pa.int32()),
pa.struct([pa.field('a', pa.int32()),
pa.field('b', pa.int8()),
pa.field('c', pa.string())]),
pa.struct([pa.field('a', pa.int32(), nullable=False),
pa.field('b', pa.int8(), nullable=False),
pa.field('c', pa.string())]),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE),
pa.union([pa.field('a', pa.binary(10)),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
pa.union([pa.field('a', pa.binary(10), nullable=False),
pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE),
pa.dictionary(pa.int32(), pa.string())
)
def test_from_numpy_dtype():
cases = [
(np.dtype('bool'), pa.bool_()),
(np.dtype('int8'), pa.int8()),
(np.dtype('int16'), pa.int16()),
(np.dtype('int32'), pa.int32()),
(np.dtype('int64'), pa.int64()),
(np.dtype('uint8'), pa.uint8()),
(np.dtype('uint16'), pa.uint16()),
(np.dtype('uint32'), pa.uint32()),
(np.dtype('float16'), pa.float16()),
(np.dtype('float32'), pa.float32()),
(np.dtype('float64'), pa.float64()),
(np.dtype('U'), pa.string()),
(np.dtype('S'), pa.binary()),
(np.dtype('datetime64[s]'), pa.timestamp('s')),
(np.dtype('datetime64[ms]'), pa.timestamp('ms')),
(np.dtype('datetime64[us]'), pa.timestamp('us')),
(np.dtype('datetime64[ns]'), pa.timestamp('ns'))
]
for dt, pt in cases:
result = pa.from_numpy_dtype(dt)
assert result == pt
# Things convertible to numpy dtypes work
assert pa.from_numpy_dtype('U') == pa.string()
assert pa.from_numpy_dtype(np.unicode) == pa.string()
assert pa.from_numpy_dtype('int32') == pa.int32()
assert pa.from_numpy_dtype(bool) == pa.bool_()
with pytest.raises(NotImplementedError):
pa.from_numpy_dtype(np.dtype('O'))
with pytest.raises(TypeError):
pa.from_numpy_dtype('not_convertible_to_dtype')
def test_is_floating():
for t in [pa.float16(), pa.float32(), pa.float64()]:
assert types.is_floating(t)
assert not types.is_floating(pa.int32())
assert wr() is None
@pytest.mark.parametrize('t,check_func', [(pa.date32(), types.is_date32),
(pa.date64(), types.is_date64),
(pa.time32('s'), types.is_time32),
(pa.time64('ns'), types.is_time64),
(pa.int8(), types.is_int8),
(pa.int16(), types.is_int16),
(pa.int32(), types.is_int32),
(pa.int64(), types.is_int64),
(pa.uint8(), types.is_uint8),
(pa.uint16(), types.is_uint16),
(pa.uint32(), types.is_uint32),
(pa.uint64(), types.is_uint64),
(pa.float16(), types.is_float16),
(pa.float32(), types.is_float32),
(pa.float64(), types.is_float64)])
def test_exact_primitive_types(t, check_func):
assert check_func(t)
def test_type_id():
# enum values are not exposed publicly
for ty in get_many_types():
assert isinstance(ty.id, int)
def test_bit_width():
for ty, expected in [(pa.bool_(), 1), (pa.int8(), 8), (pa.uint32(), 32),
(pa.float16(), 16), (pa.decimal128(19, 4), 128),
])
def test_to_numpy_roundtrip(narr):
arr = pa.array(narr)
assert narr.dtype == arr.to_numpy().dtype
np.testing.assert_array_equal(narr, arr.to_numpy())
np.testing.assert_array_equal(narr[:6], arr[:6].to_numpy())
np.testing.assert_array_equal(narr[2:], arr[2:].to_numpy())
np.testing.assert_array_equal(narr[2:6], arr[2:6].to_numpy())
@pytest.mark.parametrize(
('type', 'expected'),
[(pa.null(), 'empty'), (pa.bool_(), 'bool'), (pa.int8(), 'int8'),
(pa.int16(), 'int16'), (pa.int32(), 'int32'), (pa.int64(), 'int64'),
(pa.uint8(), 'uint8'), (pa.uint16(), 'uint16'), (pa.uint32(), 'uint32'),
(pa.uint64(), 'uint64'), (pa.float16(), 'float16'),
(pa.float32(), 'float32'), (pa.float64(), 'float64'),
(pa.date32(), 'date'), (pa.date64(), 'date'), (pa.binary(), 'bytes'),
(pa.binary(length=4), 'bytes'), (pa.string(), 'unicode'),
(pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
(pa.decimal128(18, 3), 'decimal'), (pa.timestamp('ms'), 'datetime'),
(pa.timestamp('us', 'UTC'), 'datetimetz'), (pa.time32('s'), 'time'),
(pa.time64('us'), 'time')])
def test_logical_type(type, expected):
assert get_logical_type(type) == expected
def test_array_uint64_from_py_over_range():
arr = pa.array([2**63], type=pa.uint64())
expected = pa.array(np.array([2**63], dtype='u8'))
assert arr.equals(expected)
def test_mixed_sequence_errors():
with pytest.raises(ValueError, match="tried to convert to boolean"):
pa.array([True, 'foo'], type=pa.bool_())
with pytest.raises(ValueError, match="tried to convert to float32"):
pa.array([1.5, 'foo'], type=pa.float32())
with pytest.raises(ValueError, match="tried to convert to double"):
pa.array([1.5, 'foo'])
@parametrize_with_iterable_types
@pytest.mark.parametrize("np_scalar,pa_type", [
(np.float16, pa.float16()),
(np.float32, pa.float32()),
(np.float64, pa.float64())
])
@pytest.mark.parametrize("from_pandas", [True, False])
def test_sequence_numpy_double(seq, np_scalar, pa_type, from_pandas):
data = [np_scalar(1.5), np_scalar(1), None, np_scalar(2.5), None, np.nan]
arr = pa.array(seq(data), from_pandas=from_pandas)
assert len(arr) == 6
if from_pandas:
assert arr.null_count == 3
else:
assert arr.null_count == 2
if from_pandas:
# The NaN is skipped in type inference, otherwise it forces a
# float64 promotion
pa.field('b', pa.int8()),
pa.field('c', pa.string())])
]
in_dict = {}
for i, type_ in enumerate(types):
assert hash(type_) == hash(type_)
in_dict[type_] = i
assert in_dict[type_] == i
@pytest.mark.parametrize('t,check_func', [
(pa.date32(), types.is_date32),
(pa.date64(), types.is_date64),
(pa.time32('s'), types.is_time32),
(pa.time64('ns'), types.is_time64),
(pa.int8(), types.is_int8),
(pa.int16(), types.is_int16),
(pa.int32(), types.is_int32),
(pa.int64(), types.is_int64),
(pa.uint8(), types.is_uint8),
(pa.uint16(), types.is_uint16),
(pa.uint32(), types.is_uint32),
(pa.uint64(), types.is_uint64),
(pa.float16(), types.is_float16),
(pa.float32(), types.is_float32),
(pa.float64(), types.is_float64)
])
def test_exact_primitive_types(t, check_func):
assert check_func(t)
null_type = st.just(pa.null())
bool_type = st.just(pa.bool_())
binary_type = st.just(pa.binary())
string_type = st.just(pa.string())
signed_integer_types = st.sampled_from(
[pa.int8(), pa.int16(), pa.int32(),
pa.int64()])
unsigned_integer_types = st.sampled_from(
[pa.uint8(), pa.uint16(),
pa.uint32(), pa.uint64()])
integer_types = st.one_of(signed_integer_types, unsigned_integer_types)
floating_types = st.sampled_from([pa.float16(), pa.float32(), pa.float64()])
decimal_type = st.builds(pa.decimal128,
precision=st.integers(min_value=0, max_value=38),
scale=st.integers(min_value=0, max_value=38))
numeric_types = st.one_of(integer_types, floating_types, decimal_type)
date_types = st.sampled_from([pa.date32(), pa.date64()])
time_types = st.sampled_from(
[pa.time32('s'),
pa.time32('ms'),
pa.time64('us'),
pa.time64('ns')])
timestamp_types = st.builds(pa.timestamp,
unit=st.sampled_from(['s', 'ms', 'us', 'ns']),
tz=tzst.timezones())
temporal_types = st.one_of(date_types, time_types, timestamp_types)
pa.array(invalid_values2, type=pa.float32())
def test_mixed_sequence_errors():
with pytest.raises(ValueError, match="tried to convert to boolean"):
pa.array([True, 'foo'], type=pa.bool_())
with pytest.raises(ValueError, match="tried to convert to float32"):
pa.array([1.5, 'foo'], type=pa.float32())
with pytest.raises(ValueError, match="tried to convert to double"):
pa.array([1.5, 'foo'])
@parametrize_with_iterable_types
@pytest.mark.parametrize("np_scalar,pa_type", [(np.float16, pa.float16()),
(np.float32, pa.float32()),
(np.float64, pa.float64())])
@pytest.mark.parametrize("from_pandas", [True, False])
def test_sequence_numpy_double(seq, np_scalar, pa_type, from_pandas):
data = [np_scalar(1.5), np_scalar(1), None, np_scalar(2.5), None, np.nan]
arr = pa.array(seq(data), from_pandas=from_pandas)
assert len(arr) == 6
if from_pandas:
assert arr.null_count == 3
else:
assert arr.null_count == 2
if from_pandas:
# The NaN is skipped in type inference, otherwise it forces a
# float64 promotion
assert arr.type == pa_type
#
# om = jpype.JClass('com.fasterxml.jackson.databind.ObjectMapper')()
# field = … # Code to instantiate the field
# jvm_spec = om.writeValueAsString(field)
@pytest.mark.parametrize('pa_type,jvm_spec', [
(pa.null(), '{"name":"null"}'),
(pa.bool_(), '{"name":"bool"}'),
(pa.int8(), '{"name":"int","bitWidth":8,"isSigned":true}'),
(pa.int16(), '{"name":"int","bitWidth":16,"isSigned":true}'),
(pa.int32(), '{"name":"int","bitWidth":32,"isSigned":true}'),
(pa.int64(), '{"name":"int","bitWidth":64,"isSigned":true}'),
(pa.uint8(), '{"name":"int","bitWidth":8,"isSigned":false}'),
(pa.uint16(), '{"name":"int","bitWidth":16,"isSigned":false}'),
(pa.uint32(), '{"name":"int","bitWidth":32,"isSigned":false}'),
(pa.uint64(), '{"name":"int","bitWidth":64,"isSigned":false}'),
(pa.float16(), '{"name":"floatingpoint","precision":"HALF"}'),
(pa.float32(), '{"name":"floatingpoint","precision":"SINGLE"}'),
(pa.float64(), '{"name":"floatingpoint","precision":"DOUBLE"}'),
(pa.time32('s'), '{"name":"time","unit":"SECOND","bitWidth":32}'),
(pa.time32('ms'), '{"name":"time","unit":"MILLISECOND","bitWidth":32}'),
(pa.time64('us'), '{"name":"time","unit":"MICROSECOND","bitWidth":64}'),
(pa.time64('ns'), '{"name":"time","unit":"NANOSECOND","bitWidth":64}'),
(pa.timestamp('s'), '{"name":"timestamp","unit":"SECOND",'
'"timezone":null}'),
(pa.timestamp('ms'), '{"name":"timestamp","unit":"MILLISECOND",'
'"timezone":null}'),
(pa.timestamp('us'), '{"name":"timestamp","unit":"MICROSECOND",'
'"timezone":null}'),
(pa.timestamp('ns'), '{"name":"timestamp","unit":"NANOSECOND",'
'"timezone":null}'),
(pa.timestamp('ns', tz='UTC'), '{"name":"timestamp","unit":"NANOSECOND"'
import pytest
import numpy as np
import pyarrow as pa
tensor_type_pairs = [
('i1', pa.int8()),
('i2', pa.int16()),
('i4', pa.int32()),
('i8', pa.int64()),
('u1', pa.uint8()),
('u2', pa.uint16()),
('u4', pa.uint32()),
('u8', pa.uint64()),
('f2', pa.float16()),
('f4', pa.float32()),
('f8', pa.float64())
]
def test_tensor_attrs():
data = np.random.randn(10, 4)
tensor = pa.Tensor.from_numpy(data)
assert tensor.ndim == 2
assert tensor.dim_names == []
assert tensor.size == 40
assert tensor.shape == data.shape
assert tensor.strides == data.strides
signed_integer_types = st.sampled_from([
pa.int8(),
pa.int16(),
pa.int32(),
pa.int64()
])
unsigned_integer_types = st.sampled_from([
pa.uint8(),
pa.uint16(),
pa.uint32(),
pa.uint64()
])
integer_types = st.one_of(signed_integer_types, unsigned_integer_types)
floating_types = st.sampled_from([
pa.float16(),
pa.float32(),
pa.float64()
])
decimal_type = st.builds(
pa.decimal128,
precision=st.integers(min_value=1, max_value=38),
scale=st.integers(min_value=1, max_value=38)
)
numeric_types = st.one_of(integer_types, floating_types, decimal_type)
date_types = st.sampled_from([
pa.date32(),
pa.date64()
])
time_types = st.sampled_from([
tensor = pa.Tensor.from_numpy(np.random.randn(10, 4))
n = sys.getrefcount(tensor)
array = tensor.to_numpy()
assert sys.getrefcount(tensor) == n + 1
@pytest.mark.parametrize('dtype_str,arrow_type', [
('i1', pa.int8()),
('i2', pa.int16()),
('i4', pa.int32()),
('i8', pa.int64()),
('u1', pa.uint8()),
('u2', pa.uint16()),
('u4', pa.uint32()),
('u8', pa.uint64()),
('f2', pa.float16()),
('f4', pa.float32()),
('f8', pa.float64())
])
def test_tensor_numpy_roundtrip(dtype_str, arrow_type):
dtype = np.dtype(dtype_str)
data = (100 * np.random.randn(10, 4)).astype(dtype)
tensor = pa.Tensor.from_numpy(data)
assert tensor.type == arrow_type
repr(tensor)
result = tensor.to_numpy()
assert (data == result).all()
"INT64",
pyarrow.int16().id:
"INT64",
pyarrow.int32().id:
"INT64",
pyarrow.int64().id:
"INT64",
pyarrow.uint8().id:
"INT64",
pyarrow.uint16().id:
"INT64",
pyarrow.uint32().id:
"INT64",
pyarrow.uint64().id:
"INT64",
pyarrow.float16().id:
"FLOAT64",
pyarrow.float32().id:
"FLOAT64",
pyarrow.float64().id:
"FLOAT64",
pyarrow.time32("ms").id:
"TIME",
pyarrow.time64("ns").id:
"TIME",
pyarrow.timestamp("ns").id:
"TIMESTAMP",
pyarrow.date32().id:
"DATE",
pyarrow.date64().id:
"DATETIME", # because millisecond resolution
@pytest.mark.parametrize(
('type', 'expected'),
[
(pa.null(), 'empty'),
(pa.bool_(), 'bool'),
(pa.int8(), 'int8'),
(pa.int16(), 'int16'),
(pa.int32(), 'int32'),
(pa.int64(), 'int64'),
(pa.uint8(), 'uint8'),
(pa.uint16(), 'uint16'),
(pa.uint32(), 'uint32'),
(pa.uint64(), 'uint64'),
(pa.float16(), 'float16'),
(pa.float32(), 'float32'),
(pa.float64(), 'float64'),
(pa.date32(), 'date'),
(pa.date64(), 'date'),
(pa.binary(), 'bytes'),
(pa.binary(length=4), 'bytes'),
(pa.string(), 'unicode'),
(pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'),
(pa.decimal128(18, 3), 'decimal'),
(pa.timestamp('ms'), 'datetime'),
(pa.timestamp('us', 'UTC'), 'datetimetz'),
(pa.time32('s'), 'time'),
(pa.time64('us'), 'time')
]
)
@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
(False, None, pa.BooleanScalar, pa.BooleanValue),
(True, None, pa.BooleanScalar, pa.BooleanValue),
(1, None, pa.Int64Scalar, pa.Int64Value),
(-1, None, pa.Int64Scalar, pa.Int64Value),
(1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
(1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
(1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
(1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
(1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
(1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
(1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
(1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
(1.0, None, pa.DoubleScalar, pa.DoubleValue),
(np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
(1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
(decimal.Decimal("1.123"), None, pa.Decimal128Scalar, pa.Decimal128Value),
("string", None, pa.StringScalar, pa.StringValue),
(b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
("largestring", pa.large_string(), pa.LargeStringScalar,
pa.LargeStringValue),
(b"largebytes", pa.large_binary(), pa.LargeBinaryScalar,
pa.LargeBinaryValue),
(b"abc", pa.binary(3), pa.FixedSizeBinaryScalar, pa.FixedSizeBinaryValue),
([1, 2, 3], None, pa.ListScalar, pa.ListValue),
([1, 2, 3, 4], pa.large_list(
pa.int8()), pa.LargeListScalar, pa.LargeListValue),
([1, 2, 3, 4, 5], pa.list_(
pa.int8(), 5), pa.FixedSizeListScalar, pa.FixedSizeListValue),
(datetime.date.today(), None, pa.Date32Scalar, pa.Date32Value),
def test_is_floating():
for t in [pa.float16(), pa.float32(), pa.float64()]:
assert types.is_floating(t)
assert not types.is_floating(pa.int32())
