def _create_orc_basic_row_and_data() -> Tuple[RowType, RowTypeInfo, List[Row]]: row_type = DataTypes.ROW([ DataTypes.FIELD('char', DataTypes.CHAR(10)), DataTypes.FIELD('varchar', DataTypes.VARCHAR(10)), DataTypes.FIELD('bytes', DataTypes.BYTES()), DataTypes.FIELD('boolean', DataTypes.BOOLEAN()), DataTypes.FIELD('decimal', DataTypes.DECIMAL(2, 0)), DataTypes.FIELD('int', DataTypes.INT()), DataTypes.FIELD('bigint', DataTypes.BIGINT()), DataTypes.FIELD('double', DataTypes.DOUBLE()), DataTypes.FIELD('date', DataTypes.DATE().bridged_to('java.sql.Date')), DataTypes.FIELD('timestamp', DataTypes.TIMESTAMP(3).bridged_to('java.sql.Timestamp')), ]) row_type_info = Types.ROW_NAMED( ['char', 'varchar', 'bytes', 'boolean', 'decimal', 'int', 'bigint', 'double', 'date', 'timestamp'], [Types.STRING(), Types.STRING(), Types.PRIMITIVE_ARRAY(Types.BYTE()), Types.BOOLEAN(), Types.BIG_DEC(), Types.INT(), Types.LONG(), Types.DOUBLE(), Types.SQL_DATE(), Types.SQL_TIMESTAMP()] ) data = [Row( char='char', varchar='varchar', bytes=b'varbinary', boolean=True, decimal=Decimal(1.5), int=2147483647, bigint=-9223372036854775808, double=2e-308, date=date(1970, 1, 1), timestamp=datetime(1970, 1, 2, 3, 4, 5, 600000), )] return row_type, row_type_info, data
def test_from_collection_with_data_types(self): # verify from_collection for the collection with single object. ds = self.env.from_collection(['Hi', 'Hello'], type_info=Types.STRING()) ds.add_sink(self.test_sink) self.env.execute("test from collection with single object") results = self.test_sink.get_results(False) expected = ['Hello', 'Hi'] results.sort() expected.sort() self.assertEqual(expected, results) # verify from_collection for the collection with multiple objects like tuple. ds = self.env.from_collection([(1, None, 1, True, 32767, -2147483648, 1.23, 1.98932, bytearray(b'flink'), 'pyflink', datetime.date(2014, 9, 13), datetime.time(hour=12, minute=0, second=0, microsecond=123000), datetime.datetime(2018, 3, 11, 3, 0, 0, 123000), [1, 2, 3], decimal.Decimal('1000000000000000000.05'), decimal.Decimal('1000000000000000000.0599999999999' '9999899999999999')), (2, None, 2, True, 43878, 9147483648, 9.87, 2.98936, bytearray(b'flink'), 'pyflink', datetime.date(2015, 10, 14), datetime.time(hour=11, minute=2, second=2, microsecond=234500), datetime.datetime(2020, 4, 15, 8, 2, 6, 235000), [2, 4, 6], decimal.Decimal('2000000000000000000.74'), decimal.Decimal('2000000000000000000.061111111111111' '11111111111111'))], type_info=Types.ROW( [Types.LONG(), Types.LONG(), Types.SHORT(), Types.BOOLEAN(), Types.SHORT(), Types.INT(), Types.FLOAT(), Types.DOUBLE(), Types.PICKLED_BYTE_ARRAY(), Types.STRING(), Types.SQL_DATE(), Types.SQL_TIME(), Types.SQL_TIMESTAMP(), Types.BASIC_ARRAY(Types.LONG()), Types.BIG_DEC(), Types.BIG_DEC()])) ds.add_sink(self.test_sink) self.env.execute("test from collection with tuple object") results = self.test_sink.get_results(False) # if user specifies data types of input data, the collected result should be in row format. expected = [ '+I[1, null, 1, true, 32767, -2147483648, 1.23, 1.98932, [102, 108, 105, 110, 107], ' 'pyflink, 2014-09-13, 12:00:00, 2018-03-11 03:00:00.123, [1, 2, 3], ' '1000000000000000000.05, 1000000000000000000.05999999999999999899999999999]', '+I[2, null, 2, true, -21658, 557549056, 9.87, 2.98936, [102, 108, 105, 110, 107], ' 'pyflink, 2015-10-14, 11:02:02, 2020-04-15 08:02:06.235, [2, 4, 6], ' '2000000000000000000.74, 2000000000000000000.06111111111111111111111111111]'] results.sort() expected.sort() self.assertEqual(expected, results)
def test_tuple_type(self): self.assertEqual(TupleTypeInfo([Types.STRING(), Types.INT()]), TupleTypeInfo([Types.STRING(), Types.INT()]), True) self.assertEqual(TupleTypeInfo([Types.STRING(), Types.INT()]).__str__(), "TupleTypeInfo(String, Integer)") self.assertNotEqual(TupleTypeInfo([Types.STRING(), Types.INT()]), TupleTypeInfo([Types.STRING(), Types.BOOLEAN()])) self.assertEqual(Types.TUPLE([Types.STRING(), Types.INT()]), TupleTypeInfo([Types.STRING(), Types.INT()])) self.assertEqual(Types.TUPLE([Types.STRING(), Types.INT()]).get_field_types(), [Types.STRING(), Types.INT()])
def _create_parquet_basic_row_and_data() -> Tuple[RowType, RowTypeInfo, List[Row]]: row_type = DataTypes.ROW([ DataTypes.FIELD('char', DataTypes.CHAR(10)), DataTypes.FIELD('varchar', DataTypes.VARCHAR(10)), DataTypes.FIELD('binary', DataTypes.BINARY(10)), DataTypes.FIELD('varbinary', DataTypes.VARBINARY(10)), DataTypes.FIELD('boolean', DataTypes.BOOLEAN()), DataTypes.FIELD('decimal', DataTypes.DECIMAL(2, 0)), DataTypes.FIELD('int', DataTypes.INT()), DataTypes.FIELD('bigint', DataTypes.BIGINT()), DataTypes.FIELD('double', DataTypes.DOUBLE()), DataTypes.FIELD('date', DataTypes.DATE().bridged_to('java.sql.Date')), DataTypes.FIELD('time', DataTypes.TIME().bridged_to('java.sql.Time')), DataTypes.FIELD('timestamp', DataTypes.TIMESTAMP(3).bridged_to('java.sql.Timestamp')), DataTypes.FIELD('timestamp_ltz', DataTypes.TIMESTAMP_LTZ(3)), ]) row_type_info = Types.ROW_NAMED( ['char', 'varchar', 'binary', 'varbinary', 'boolean', 'decimal', 'int', 'bigint', 'double', 'date', 'time', 'timestamp', 'timestamp_ltz'], [Types.STRING(), Types.STRING(), Types.PRIMITIVE_ARRAY(Types.BYTE()), Types.PRIMITIVE_ARRAY(Types.BYTE()), Types.BOOLEAN(), Types.BIG_DEC(), Types.INT(), Types.LONG(), Types.DOUBLE(), Types.SQL_DATE(), Types.SQL_TIME(), Types.SQL_TIMESTAMP(), Types.INSTANT()] ) datetime_ltz = datetime.datetime(1970, 2, 3, 4, 5, 6, 700000, tzinfo=pytz.timezone('UTC')) timestamp_ltz = Instant.of_epoch_milli( ( calendar.timegm(datetime_ltz.utctimetuple()) + calendar.timegm(time.localtime(0)) ) * 1000 + datetime_ltz.microsecond // 1000 ) data = [Row( char='char', varchar='varchar', binary=b'binary', varbinary=b'varbinary', boolean=True, decimal=Decimal(1.5), int=2147483647, bigint=-9223372036854775808, double=2e-308, date=datetime.date(1970, 1, 1), time=datetime.time(1, 1, 1), timestamp=datetime.datetime(1970, 1, 2, 3, 4, 5, 600000), timestamp_ltz=timestamp_ltz )] return row_type, row_type_info, data
def to_java_typeinfo(type_info: TypeInformation): if isinstance(type_info, BasicTypeInfo): basic_type = type_info._basic_type if basic_type == BasicType.STRING: j_typeinfo = JTypes.STRING elif basic_type == BasicType.BYTE: j_typeinfo = JTypes.LONG elif basic_type == BasicType.BOOLEAN: j_typeinfo = JTypes.BOOLEAN elif basic_type == BasicType.SHORT: j_typeinfo = JTypes.LONG elif basic_type == BasicType.INT: j_typeinfo = JTypes.LONG elif basic_type == BasicType.LONG: j_typeinfo = JTypes.LONG elif basic_type == BasicType.FLOAT: j_typeinfo = JTypes.DOUBLE elif basic_type == BasicType.DOUBLE: j_typeinfo = JTypes.DOUBLE elif basic_type == BasicType.CHAR: j_typeinfo = JTypes.STRING elif basic_type == BasicType.BIG_INT: j_typeinfo = JTypes.BIG_INT elif basic_type == BasicType.BIG_DEC: j_typeinfo = JTypes.BIG_DEC elif basic_type == BasicType.INSTANT: j_typeinfo = JTypes.INSTANT else: raise TypeError("Invalid BasicType %s." % basic_type) elif isinstance(type_info, PrimitiveArrayTypeInfo): element_type = type_info._element_type if element_type == Types.BOOLEAN(): j_typeinfo = JPrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.BYTE(): j_typeinfo = JPrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.SHORT(): j_typeinfo = JPrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.INT(): j_typeinfo = JPrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.LONG(): j_typeinfo = JPrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.FLOAT(): j_typeinfo = JPrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.DOUBLE(): j_typeinfo = JPrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO elif element_type == Types.CHAR(): j_typeinfo = JPrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO else: raise TypeError("Invalid element type for a primitive array.") elif isinstance(type_info, BasicArrayTypeInfo): element_type = type_info._element_type if element_type == Types.BOOLEAN(): j_typeinfo = JBasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO elif element_type == Types.BYTE(): j_typeinfo = JBasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO elif element_type == Types.SHORT(): j_typeinfo = JBasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO elif element_type == Types.INT(): j_typeinfo = JBasicArrayTypeInfo.INT_ARRAY_TYPE_INFO elif element_type == Types.LONG(): j_typeinfo = JBasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO elif element_type == Types.FLOAT(): j_typeinfo = JBasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO elif element_type == Types.DOUBLE(): j_typeinfo = JBasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO elif element_type == Types.CHAR(): j_typeinfo = JBasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO elif element_type == Types.STRING(): j_typeinfo = JBasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO else: raise TypeError("Invalid element type for a basic array.") elif isinstance(type_info, ObjectArrayTypeInfo): element_type = type_info._element_type j_typeinfo = JTypes.OBJECT_ARRAY(to_java_typeinfo(element_type)) elif isinstance(type_info, MapTypeInfo): j_key_typeinfo = to_java_typeinfo(type_info._key_type_info) j_value_typeinfo = to_java_typeinfo(type_info._value_type_info) j_typeinfo = JMapTypeInfo(j_key_typeinfo, j_value_typeinfo) else: j_typeinfo = JPickledByteArrayTypeInfo.PICKLED_BYTE_ARRAY_TYPE_INFO return j_typeinfo