def from_proto(field_type): """ Creates the corresponding :class:`Coder` given the protocol representation of the field type. :param field_type: the protocol representation of the field type :return: :class:`Coder` """ field_type_name = field_type.type_name coder = _type_name_mappings.get(field_type_name) if coder is not None: return coder if field_type_name == type_name.ROW: return RowCoder([from_proto(f.type) for f in field_type.row_schema.fields]) if field_type_name == type_name.TIMESTAMP: return TimestampCoder(field_type.timestamp_info.precision) if field_type_name == type_name.LOCAL_ZONED_TIMESTAMP: timezone = pytz.timezone(pipeline_options.view_as(DebugOptions).lookup_experiment( "table.exec.timezone")) return LocalZonedTimestampCoder(field_type.local_zoned_timestamp_info.precision, timezone) elif field_type_name == type_name.ARRAY: return ArrayCoder(from_proto(field_type.collection_element_type)) elif field_type_name == type_name.MAP: return MapCoder(from_proto(field_type.map_info.key_type), from_proto(field_type.map_info.value_type)) elif field_type_name == type_name.DECIMAL: return DecimalCoder(field_type.decimal_info.precision, field_type.decimal_info.scale) else: raise ValueError("field_type %s is not supported." % field_type)
def _pickle_from_runner_api_parameter(schema_proto, unused_components, unused_context): def _to_arrow_schema(row_type): return pa.schema([ pa.field(n, to_arrow_type(t), t._nullable) for n, t in zip(row_type.field_names(), row_type.field_types()) ]) def _to_data_type(field): if field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.TINYINT: return TinyIntType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.SMALLINT: return SmallIntType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.INT: return IntType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.BIGINT: return BigIntType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.BOOLEAN: return BooleanType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.FLOAT: return FloatType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.DOUBLE: return DoubleType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.VARCHAR: return VarCharType(0x7fffffff, field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.VARBINARY: return VarBinaryType(0x7fffffff, field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.DECIMAL: return DecimalType(field.type.decimal_info.precision, field.type.decimal_info.scale, field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.DATE: return DateType(field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.TIME: return TimeType(field.type.time_info.precision, field.type.nullable) elif field.type.type_name == \ flink_fn_execution_pb2.Schema.TypeName.LOCAL_ZONED_TIMESTAMP: return LocalZonedTimestampType( field.type.local_zoned_timestamp_info.precision, field.type.nullable) elif field.type.type_name == flink_fn_execution_pb2.Schema.TypeName.TIMESTAMP: return TimestampType(field.type.timestamp_info.precision, field.type.nullable) else: raise ValueError("field_type %s is not supported." % field.type) def _to_row_type(row_schema): return RowType([ RowField(f.name, _to_data_type(f)) for f in row_schema.fields ]) timezone = pytz.timezone( pipeline_options.view_as(DebugOptions).lookup_experiment( "table.exec.timezone")) row_type = _to_row_type(schema_proto) return ArrowCoder(_to_arrow_schema(row_type), row_type, timezone)