def test_string_to_decimal_literal(): decimal_str = Literal.of("34.560") decimal_lit = decimal_str.to(DecimalType.of(9, 3)) assert 3 == abs(decimal_lit.value.as_tuple().exponent) assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() assert decimal_str.to(DecimalType.of(9, 2)) is None assert decimal_str.to(DecimalType.of(9, 4)) is None
# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from iceberg.api.expressions import (Literal, StringLiteral) import pytest @pytest.mark.parametrize( "input_vals", [ (Literal.of("abc"), Literal.of(u'abc')), # unicode and str are same (StringLiteral(None), StringLiteral(None)) # None literals are equal ]) def test_special_equality(input_vals): assert input_vals[0] == input_vals[1] @pytest.mark.parametrize( "input_vals", [ (Literal.of("abc"), Literal.of('abcd')), # test_seq_length, longer is greater (Literal.of('abcd'), Literal.of("adc") ), # test_char_order, first difference takes precedence over length (None, Literal.of('abc') ) # test_null_handling, null comes before non-null
params=[ Expressions.equal("id", 5), Expressions.equal("id", 29), Expressions.equal("id", 30), Expressions.equal("id", 75), Expressions.equal("id", 79), Expressions.equal("id", 80), Expressions.equal("id", 85) ]) def not_eq_rewrite(request): yield request.param @pytest.fixture(scope="session", params=[ Literal.of(False), Literal.of(34), Literal.of(35), Literal.of(36.75), Literal.of(8.75), Literal.of("2017-11-29").to(DateType.get()), Literal.of("11:30:0").to(TimeType.get()), Literal.of("2017-11-29T11:30:07.123").to( TimestampType.without_timezone()), Literal.of("2017-11-29T11:30:07.123+01:00").to( TimestampType.with_timezone()), Literal.of("abc"), Literal.of(uuid.uuid4()), Literal.of(bytes([0x01, 0x02, 0x03])).to(FixedType.of_length(3)), Literal.of(bytes([0x03, 0x04, 0x05,
def test_binary_unsigned_comparator(): b1 = bytearray([0x01, 0x01, 0x02]) b2 = bytearray([0x01, 0xFF, 0x01]) assert Literal.of(b2) > Literal.of(b1)
def test_double_to_float(): lit = Literal.of(34.56).to(DoubleType.get()) float_lit = lit.to(FloatType.get()) assert math.isclose(lit.value, float_lit.value)
def test_float_to_double(): lit = Literal.of(34.56) dbl_lit = lit.to(DoubleType.get()) assert math.isclose(lit.value, dbl_lit.value)
def test_long_to_double_conversion(): lit = Literal.of(34).to(LongType.get()) dbl_lit = lit.to(DoubleType.get()) assert math.isclose(lit.value, dbl_lit.value)
def test_long_to_integer(): lit = Literal.of(34).to(LongType.get()) int_lit = lit.to(IntegerType.get()) assert lit.value == int_lit.value
def test_string_to_string_literal(): assert Literal.of("abc") == Literal.of("abc").to(StringType.get())
def test_to_bytes(self): self.assertEqual(b'\x00\x00', Literal.of(False).to_byte_buffer()) self.assertEqual(b'\x01\x00', Literal.of(True).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00\x00\x00\x00\x00', Literal.of(1234).to(LongType.get()).to_byte_buffer()) self.assertEqual(b'\x19\x04\x9e?', Literal.of(1.2345).to_byte_buffer()) self.assertEqual( b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f', Literal.of(1.2345).to(DoubleType.get()).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to(DateType.get()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to(TimeType.get()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to( TimestampType.with_timezone()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to( TimestampType.without_timezone()).to_byte_buffer()) self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) self.assertEqual( b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7', Literal.of(uuid.UUID( "f79c3e09-677c-4bbd-a479-3f349cb785e7")).to_byte_buffer()) self.assertEqual(b'foo', Literal.of(bytes(b'foo')).to_byte_buffer()) self.assertEqual(b'foo', Literal.of(bytearray(b'foo')).to_byte_buffer())
def test_timestamp_with_zone_human_string(): ts_tz = TimestampType.with_timezone() identity = Transforms.identity(ts_tz) ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(ts_tz) assert identity.to_human_string(ts.value) == "2017-12-01T18:12:55.038194Z"
def test_natural_order_eq(input_val): assert Literal.of(input_val) == Literal.of(input_val)
def test_natural_order(larger, smaller, op): assert op(Literal.of(larger), Literal.of(smaller))
def test_fixed_unsigned_comparator(): b1 = bytearray([0x01, 0x01, 0x02]) b2 = bytearray([0x01, 0xFF, 0x01]) assert Literal.of(b2) > Literal.of(b1).to(FixedType.of_length(3))
def test_integer_to_long_conversion(): lit = Literal.of(34) long_lit = lit.to(LongType.get()) assert lit.value == long_lit.value
def test_integer_to_decimal_conversion(type_val_tuples): lit = Literal.of(34) assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple()
def test_timestamp_without_zone_with_zone_in_literal(): with raises(RuntimeError): timestamp_str = Literal.of("2017-08-18T14:21:01.919+07:00") timestamp_str.to(TimestampType.without_timezone())
def test_long_to_float_conversion(): lit = Literal.of(34).to(LongType.get()) float_lit = lit.to(FloatType.get()) assert math.isclose(lit.value, float_lit.value)
def test_string_to_uuid_literal(): expected = uuid.uuid4() uuid_str = Literal.of(str(expected)) uuid_lit = uuid_str.to(UUIDType.get()) assert expected == uuid_lit.value
def test_long_to_decimal_conversion(type_val_tuples): lit = Literal.of(34).to(LongType.get()) assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple()
def test_byte_buffer_conversions(self): # booleans are stored as 0x00 for 'false' and a non-zero byte for 'true' self.assertConversion(False, BooleanType.get(), b'\x00') self.assertConversion(True, BooleanType.get(), b'\x01') self.assertEqual(b'\x00', Literal.of(False).to_byte_buffer()) self.assertEqual(b'\x01', Literal.of(True).to_byte_buffer()) # integers are stored as 4 bytes in little-endian order # 84202 is 0...01|01001000|11101010 in binary # 11101010 -> 234 (-22), 01001000 -> 72, 00000001 -> 1, 00000000 -> 0 self.assertConversion(84202, IntegerType.get(), bytes([234, 72, 1, 0])) self.assertEqual(bytes([234, 72, 1, 0]), Literal.of(84202).to_byte_buffer()) # longs are stored as 8 bytes in little-endian order # 200L is 0...0|11001000 in binary # 11001000 -> 200 (-56), 00000000 -> 0, ... , 00000000 -> 0 self.assertConversion(200, LongType.get(), bytes([200, 0, 0, 0, 0, 0, 0, 0])) self.assertEqual(bytes([200, 0, 0, 0, 0, 0, 0, 0]), Literal.of(200).to(LongType.get()).to_byte_buffer()) # floats are stored as 4 bytes in little-endian order # floating point numbers are represented as sign * 2ˆexponent * mantissa # -4.5F is -1 * 2ˆ2 * 1.125 and encoded as 11000000|10010000|0...0 in binary # 00000000 -> 0, 00000000 -> 0, 10010000 -> 144 (-112), 11000000 -> 192 (-64), self.assertConversion(-4.5, FloatType.get(), bytes([0, 0, 144, 192])) self.assertEqual(bytes([0, 0, 144, 192]), Literal.of(-4.5).to_byte_buffer()) # doubles are stored as 8 bytes in little-endian order # floating point numbers are represented as sign * 2ˆexponent * mantissa # 6.0 is 1 * 2ˆ4 * 1.5 and encoded as 01000000|00011000|0...0 # 00000000 -> 0, ... , 00011000 -> 24, 01000000 -> 64 self.assertConversion(6.0, DoubleType.get(), bytes([0, 0, 0, 0, 0, 0, 24, 64])) self.assertEqual(bytes([0, 0, 0, 0, 0, 0, 24, 64]), Literal.of(6.0).to(DoubleType.get()).to_byte_buffer()) # dates are stored as days from 1970-01-01 in a 4-byte little-endian int # 1000 is 0...0|00000011|11101000 in binary # 11101000 -> 232 (-24), 00000011 -> 3, ... , 00000000 -> 0 self.assertConversion(1000, DateType.get(), bytes([232, 3, 0, 0])) self.assertEqual(bytes([232, 3, 0, 0]), Literal.of(1000).to(DateType.get()).to_byte_buffer()) # time is stored as microseconds from midnight in an 8-byte little-endian long # 10000L is 0...0|00100111|00010000 in binary # 00010000 -> 16, 00100111 -> 39, ... , 00000000 -> 0 self.assertConversion(10000, TimeType.get(), bytes([16, 39, 0, 0, 0, 0, 0, 0])) self.assertEqual( bytes([16, 39, 0, 0, 0, 0, 0, 0]), Literal.of(10000).to(LongType.get()).to( TimeType.get()).to_byte_buffer()) # timestamps are stored as microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian long # 400000L is 0...110|00011010|10000000 in binary # 10000000 -> 128 (-128), 00011010 -> 26, 00000110 -> 6, ... , 00000000 -> 0 self.assertConversion(400000, TimestampType.without_timezone(), bytes([128, 26, 6, 0, 0, 0, 0, 0])) self.assertConversion(400000, TimestampType.with_timezone(), bytes([128, 26, 6, 0, 0, 0, 0, 0])) self.assertEqual( bytes([128, 26, 6, 0, 0, 0, 0, 0]), Literal.of(400000).to(LongType.get()).to( TimestampType.without_timezone()).to_byte_buffer()) self.assertEqual( bytes([128, 26, 6, 0, 0, 0, 0, 0]), Literal.of(400000).to(LongType.get()).to( TimestampType.with_timezone()).to_byte_buffer()) # strings are stored as UTF-8 bytes (without length) # 'A' -> 65, 'B' -> 66, 'C' -> 67 self.assertConversion("ABC", StringType.get(), bytes([65, 66, 67])) self.assertEqual(bytes([65, 66, 67]), Literal.of("ABC").to_byte_buffer()) # uuids are stored as 16-byte big-endian values # f79c3e09-677c-4bbd-a479-3f349cb785e7 is encoded as F7 9C 3E 09 67 7C 4B BD A4 79 3F 34 9C B7 85 E7 # 0xF7 -> 11110111 -> 247 (-9), 0x9C -> 10011100 -> 156 (-100), 0x3E -> 00111110 -> 62, # 0x09 -> 00001001 -> 9, 0x67 -> 01100111 -> 103, 0x7C -> 01111100 -> 124, # 0x4B -> 01001011 -> 75, 0xBD -> 10111101 -> 189 (-67), 0xA4 -> 10100100 -> 164 (-92), # 0x79 -> 01111001 -> 121, 0x3F -> 00111111 -> 63, 0x34 -> 00110100 -> 52, # 0x9C -> 10011100 -> 156 (-100), 0xB7 -> 10110111 -> 183 (-73), 0x85 -> 10000101 -> 133 (-123), # 0xE7 -> 11100111 -> 231 (-25) self.assertConversion( uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), UUIDType.get(), bytes([ 247, 156, 62, 9, 103, 124, 75, 189, 164, 121, 63, 52, 156, 183, 133, 231 ])) self.assertEqual( bytes([ 247, 156, 62, 9, 103, 124, 75, 189, 164, 121, 63, 52, 156, 183, 133, 231 ]), Literal.of(uuid.UUID( "f79c3e09-677c-4bbd-a479-3f349cb785e7")).to_byte_buffer()) # fixed values are stored directly # 'a' -> 97, 'b' -> 98 self.assertConversion(bytes("ab", "utf8"), FixedType.of_length(2), bytes([97, 98])) self.assertEqual(bytes([97, 98]), Literal.of(bytes("ab", "utf8")).to_byte_buffer()) # binary values are stored directly # 'Z' -> 90 self.assertConversion(bytearray("Z", "utf8"), BinaryType.get(), bytes([90])) self.assertEqual(bytes([90]), Literal.of(bytearray("Z", "utf8")).to_byte_buffer()) # decimals are stored as unscaled values in the form of two's-complement big-endian binary, # using the minimum number of bytes for the values # 345 is 0...1|01011001 in binary # 00000001 -> 1, 01011001 -> 89 self.assertConversion( Decimal(3.45).quantize(Decimal(".01")), DecimalType.of(3, 2), bytes([1, 89])) self.assertEqual( bytes([1, 89]), Literal.of(3.45).to(DecimalType.of(3, 2)).to_byte_buffer()) # decimal on 3-bytes to test that we use the minimum number of bytes and not a power of 2 # 1234567 is 00010010|11010110|10000111 in binary # 00010010 -> 18, 11010110 -> 214, 10000111 -> 135 self.assertConversion( Decimal(123.4567).quantize(Decimal(".0001")), DecimalType.of(7, 4), bytes([18, 214, 135])) self.assertEqual( bytes([18, 214, 135]), Literal.of(123.4567).to(DecimalType.of(7, 4)).to_byte_buffer()) # negative decimal to test two's complement # -1234567 is 11101101|00101001|01111001 in binary # 11101101 -> 237, 00101001 -> 41, 01111001 -> 121 self.assertConversion( Decimal(-123.4567).quantize(Decimal(".0001")), DecimalType.of(7, 4), bytes([237, 41, 121])) self.assertEqual( bytes([237, 41, 121]), Literal.of(-123.4567).to(DecimalType.of(7, 4)).to_byte_buffer()) # test empty byte in decimal # 11 is 00001011 in binary # 00001011 -> 11 self.assertConversion( Decimal(0.011).quantize(Decimal(".001")), DecimalType.of(10, 3), bytes([11])) self.assertEqual( bytes([11]), Literal.of(0.011).to(DecimalType.of(10, 3)).to_byte_buffer())
def test_float_to_decimal_conversion(float_type_val_tuples): lit = Literal.of(34.56) assert lit.to(float_type_val_tuples[0]).value.as_tuple() == Decimal(float_type_val_tuples[1]).as_tuple()
def test_to_bytes(self): self.assertEqual(b'\x00', Literal.of(False).to_byte_buffer()) self.assertEqual(b'\x01', Literal.of(True).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00\x00\x00\x00\x00', Literal.of(1234).to(LongType.get()).to_byte_buffer()) self.assertEqual(b'\x19\x04\x9e?', Literal.of(1.2345).to_byte_buffer()) self.assertEqual( b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f', Literal.of(1.2345).to(DoubleType.get()).to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to(DateType.get()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to(TimeType.get()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to( TimestampType.with_timezone()).to_byte_buffer()) self.assertEqual( b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to( TimestampType.without_timezone()).to_byte_buffer()) self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) self.assertEqual( b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7', Literal.of(uuid.UUID( "f79c3e09-677c-4bbd-a479-3f349cb785e7")).to_byte_buffer()) self.assertEqual(b'foo', Literal.of(bytes(b'foo')).to_byte_buffer()) self.assertEqual(b'foo', Literal.of(bytearray(b'foo')).to_byte_buffer()) # Decimal on 2-bytes self.assertEqual( b'\x30\x39', Literal.of(123.45).to(DecimalType.of(5, 2)).to_byte_buffer()) # Decimal on 3-bytes to test that we use the minimum number of bytes self.assertEqual( b'\x12\xd6\x87', Literal.of(123.4567).to(DecimalType.of(7, 4)).to_byte_buffer()) # Negative decimal to test two's complement self.assertEqual( b'\xed\x29\x79', Literal.of(-123.4567).to(DecimalType.of(7, 4)).to_byte_buffer())
@pytest.mark.parametrize("test_input,test_type,expected", [(1, BucketFloat(100), -142385009), (1, BucketDouble(100), -142385009)]) def test_spec_values_dbl(test_input, test_type, expected): assert test_type.hash(test_input) == expected @pytest.mark.parametrize( "test_input,test_type,scale_factor,expected", [(Decimal("14.20"), DecimalType.of(9, 2), Decimal(10)**-2, -500754589), (Decimal("137302769811943318102518958871258.37580"), DecimalType.of( 38, 5), Decimal(10)**-5, -32334285)]) def test_spec_values_dec(test_input, test_type, scale_factor, expected): getcontext().prec = 38 assert Bucket.get(test_type, 100).hash(test_input.quantize(scale_factor)) == expected @pytest.mark.parametrize("test_input,test_type,expected", [ (Literal.of("2017-11-16").to(DateType.get()), DateType.get(), -653330422), (Literal.of("22:31:08").to(TimeType.get()), TimeType.get(), -662762989), (Literal.of("2017-11-16T22:31:08").to(TimestampType.without_timezone()), TimestampType.without_timezone(), -2047944441), (Literal.of("2017-11-16T14:31:08-08:00").to(TimestampType.with_timezone()), TimestampType.with_timezone(), -2047944441) ]) def test_spec_values_datetime_uuid(test_input, test_type, expected): assert Bucket.get(test_type, 100).hash(test_input.value) == expected
# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from iceberg.api.expressions import Literal from iceberg.api.transforms import Transforms from iceberg.api.types import TimestampType import pytest @pytest.mark.parametrize("lit,type_var", [(Literal.of("2017-12-01T10:12:55.038194-08:00"), TimestampType.with_timezone()), (Literal.of("2017-12-01T18:12:55.038194"), TimestampType.without_timezone())]) @pytest.mark.parametrize("transform_gran,expected", [(Transforms.year, "2017"), (Transforms.month, "2017-12"), (Transforms.day, "2017-12-01"), (Transforms.hour, "2017-12-01-18")]) def test_ts_to_human_string(lit, type_var, transform_gran, expected): date_var = lit.to(type_var) assert (transform_gran(type_var).to_human_string( transform_gran(type_var).apply(date_var.value))) == expected @pytest.mark.parametrize(