Example #1
0
def orc_type(field):
    if pa.types.is_boolean(field):
        return pyorc.Boolean()
    elif pa.types.is_int8(field):
        return pyorc.TinyInt()
    elif pa.types.is_int16(field):
        return pyorc.SmallInt()
    elif pa.types.is_int32(field):
        return pyorc.Int()
    elif pa.types.is_int64(field):
        return pyorc.BigInt()
    elif pa.types.is_float32(field):
        return pyorc.Float()
    elif pa.types.is_float64(field):
        return pyorc.Double()
    elif pa.types.is_decimal(field):
        return pyorc.Decimal(field.precision, field.scale)
    elif pa.types.is_list(field):
        return pyorc.Array(orc_type(field.value_type))
    elif pa.types.is_timestamp(field):
        return pyorc.Timestamp()
    elif pa.types.is_date(field):
        return pyorc.Date()
    elif pa.types.is_binary(field):
        return pyorc.Binary()
    elif pa.types.is_string(field):
        return pyorc.String()
    else:
        raise ValueError('Cannot Convert %s' % field)
Example #2
0
def test_empty_statistics():
    buff = BytesIO()
    orc_schema = po.Struct(
        a=po.BigInt(),
        b=po.Double(),
        c=po.String(),
        d=po.Decimal(11, 2),
        e=po.Date(),
        f=po.Timestamp(),
        g=po.Boolean(),
        h=po.Binary(),
        i=po.BigInt(),
        # One column with non null value, else cudf/pyorc readers crash
    )
    data = tuple([None] * (len(orc_schema.fields) - 1) + [1])
    with po.Writer(buff, orc_schema) as writer:
        writer.write(data)

    got = cudf.io.orc.read_orc_statistics([buff])

    # Check for both file and stripe stats
    for stats in got:
        # Similar expected stats for the first 6 columns in this case
        for col_name in ascii_lowercase[:6]:
            assert stats[0][col_name].get("number_of_values") == 0
            assert stats[0][col_name].get("has_null") is True
            assert stats[0][col_name].get("minimum") is None
            assert stats[0][col_name].get("maximum") is None
        for col_name in ascii_lowercase[:3]:
            assert stats[0][col_name].get("sum") == 0
        # Sum for decimal column is a string
        assert stats[0]["d"].get("sum") == "0"

        assert stats[0]["g"].get("number_of_values") == 0
        assert stats[0]["g"].get("has_null") is True
        assert stats[0]["g"].get("true_count") == 0
        assert stats[0]["g"].get("false_count") == 0

        assert stats[0]["h"].get("number_of_values") == 0
        assert stats[0]["h"].get("has_null") is True
        assert stats[0]["h"].get("sum") == 0

        assert stats[0]["i"].get("number_of_values") == 1
        assert stats[0]["i"].get("has_null") is False
        assert stats[0]["i"].get("minimum") == 1
        assert stats[0]["i"].get("maximum") == 1
        assert stats[0]["i"].get("sum") == 1