def orc_type(field): if pa.types.is_boolean(field): return pyorc.Boolean() elif pa.types.is_int8(field): return pyorc.TinyInt() elif pa.types.is_int16(field): return pyorc.SmallInt() elif pa.types.is_int32(field): return pyorc.Int() elif pa.types.is_int64(field): return pyorc.BigInt() elif pa.types.is_float32(field): return pyorc.Float() elif pa.types.is_float64(field): return pyorc.Double() elif pa.types.is_decimal(field): return pyorc.Decimal(field.precision, field.scale) elif pa.types.is_list(field): return pyorc.Array(orc_type(field.value_type)) elif pa.types.is_timestamp(field): return pyorc.Timestamp() elif pa.types.is_date(field): return pyorc.Date() elif pa.types.is_binary(field): return pyorc.Binary() elif pa.types.is_string(field): return pyorc.String() else: raise ValueError('Cannot Convert %s' % field)
def test_empty_statistics(): buff = BytesIO() orc_schema = po.Struct( a=po.BigInt(), b=po.Double(), c=po.String(), d=po.Decimal(11, 2), e=po.Date(), f=po.Timestamp(), g=po.Boolean(), h=po.Binary(), i=po.BigInt(), # One column with non null value, else cudf/pyorc readers crash ) data = tuple([None] * (len(orc_schema.fields) - 1) + [1]) with po.Writer(buff, orc_schema) as writer: writer.write(data) got = cudf.io.orc.read_orc_statistics([buff]) # Check for both file and stripe stats for stats in got: # Similar expected stats for the first 6 columns in this case for col_name in ascii_lowercase[:6]: assert stats[0][col_name].get("number_of_values") == 0 assert stats[0][col_name].get("has_null") is True assert stats[0][col_name].get("minimum") is None assert stats[0][col_name].get("maximum") is None for col_name in ascii_lowercase[:3]: assert stats[0][col_name].get("sum") == 0 # Sum for decimal column is a string assert stats[0]["d"].get("sum") == "0" assert stats[0]["g"].get("number_of_values") == 0 assert stats[0]["g"].get("has_null") is True assert stats[0]["g"].get("true_count") == 0 assert stats[0]["g"].get("false_count") == 0 assert stats[0]["h"].get("number_of_values") == 0 assert stats[0]["h"].get("has_null") is True assert stats[0]["h"].get("sum") == 0 assert stats[0]["i"].get("number_of_values") == 1 assert stats[0]["i"].get("has_null") is False assert stats[0]["i"].get("minimum") == 1 assert stats[0]["i"].get("maximum") == 1 assert stats[0]["i"].get("sum") == 1
def test_orc_read_skiprows(tmpdir): buff = BytesIO() df = pd.DataFrame( {"a": [1, 0, 1, 0, None, 1, 1, 1, 0, None, 0, 0, 1, 1, 1, 1]}, dtype=pd.BooleanDtype(), ) writer = pyorc.Writer(buff, pyorc.Struct(a=pyorc.Boolean())) tuples = list( map( lambda x: (None, ) if x[0] is pd.NA else x, list(df.itertuples(index=False, name=None)), )) writer.writerows(tuples) writer.close() skiprows = 10 expected = cudf.read_orc(buff)[skiprows::].reset_index(drop=True) got = cudf.read_orc(buff, skiprows=skiprows) assert_eq(expected, got)
cudf.dtype("O"): "string", cudf.dtype("str"): "string", cudf.dtype("float32"): "float", cudf.dtype("float64"): "double", cudf.dtype("<M8[ns]"): {"type": "long", "logicalType": "timestamp-millis"}, cudf.dtype("<M8[ms]"): {"type": "long", "logicalType": "timestamp-millis"}, cudf.dtype("<M8[us]"): {"type": "long", "logicalType": "timestamp-micros"}, } PANDAS_TO_ORC_TYPES = { cudf.dtype("int8"): pyorc.TinyInt(), pd.Int8Dtype(): pyorc.TinyInt(), pd.Int16Dtype(): pyorc.SmallInt(), pd.Int32Dtype(): pyorc.Int(), pd.Int64Dtype(): pyorc.BigInt(), pd.BooleanDtype(): pyorc.Boolean(), cudf.dtype("bool_"): pyorc.Boolean(), cudf.dtype("int16"): pyorc.SmallInt(), cudf.dtype("int32"): pyorc.Int(), cudf.dtype("int64"): pyorc.BigInt(), cudf.dtype("O"): pyorc.String(), pd.StringDtype(): pyorc.String(), cudf.dtype("float32"): pyorc.Float(), cudf.dtype("float64"): pyorc.Double(), cudf.dtype("<M8[ns]"): pyorc.Timestamp(), cudf.dtype("<M8[ms]"): pyorc.Timestamp(), cudf.dtype("<M8[us]"): pyorc.Timestamp(), } ORC_TO_PANDAS_TYPES = { pyorc.TinyInt().name: pd.Int8Dtype(),
"type": "long", "logicalType": "timestamp-millis" }, np.dtype("<M8[us]"): { "type": "long", "logicalType": "timestamp-micros" }, } PANDAS_TO_ORC_TYPES = { np.dtype("int8"): pyorc.TinyInt(), pd.Int8Dtype(): pyorc.TinyInt(), pd.Int16Dtype(): pyorc.SmallInt(), pd.Int32Dtype(): pyorc.Int(), pd.Int64Dtype(): pyorc.BigInt(), pd.BooleanDtype(): pyorc.Boolean(), np.dtype("bool_"): pyorc.Boolean(), np.dtype("int16"): pyorc.SmallInt(), np.dtype("int32"): pyorc.Int(), np.dtype("int64"): pyorc.BigInt(), np.dtype("O"): pyorc.String(), pd.StringDtype(): pyorc.String(), np.dtype("float32"): pyorc.Float(), np.dtype("float64"): pyorc.Double(), np.dtype("<M8[ns]"): pyorc.Timestamp(), np.dtype("<M8[ms]"): pyorc.Timestamp(), np.dtype("<M8[us]"): pyorc.Timestamp(), } def _generate_rand_meta(obj, dtypes_list, null_frequency_override=None):