def test_convert_with_table_metadata(self, converter): pkey_col1 = RedshiftSQLColumn( 'pkey_col_one', redshift_data_types.RedshiftInteger(), primary_key_order=1, sort_key_order=2, is_dist_key=True ) pkey_col2 = RedshiftSQLColumn( 'pkey_col_two', redshift_data_types.RedshiftInteger(), primary_key_order=2, sort_key_order=1 ) col = RedshiftSQLColumn('col', redshift_data_types.RedshiftInteger()) sql_table = RedshiftSQLTable( self.table_name, [pkey_col2, pkey_col1, col], diststyle='all' ) expected_schema = { 'type': 'record', 'name': self.table_name, 'namespace': '', 'fields': [ { 'name': pkey_col2.name, 'type': ['null', 'int'], 'default': None, AvroMetaDataKeys.PRIMARY_KEY: 2, AvroMetaDataKeys.SORT_KEY: 1 }, { 'name': pkey_col1.name, 'type': ['null', 'int'], 'default': None, AvroMetaDataKeys.PRIMARY_KEY: 1, AvroMetaDataKeys.DIST_KEY: True, AvroMetaDataKeys.SORT_KEY: 2 }, { 'name': col.name, 'type': ['null', 'int'], 'default': None }, ], AvroMetaDataKeys.PRIMARY_KEY: [pkey_col1.name, pkey_col2.name], AvroMetaDataKeys.SORT_KEY: [pkey_col2.name, pkey_col1.name], AvroMetaDataKeys.DIST_KEY: pkey_col1.name, AvroMetaDataKeys.DISTSTYLE: 'all' } actual_schema = converter.convert(sql_table) assert expected_schema == actual_schema
def test_convert_with_table_namespace_and_aliases(self, converter): doc = 'I am doc' aliases = ['foo'] metadata = { MetaDataKey.NAMESPACE: self.namespace, MetaDataKey.ALIASES: aliases } col_name = 'col' sql_table = RedshiftSQLTable( self.table_name, [ RedshiftSQLColumn( col_name, redshift_data_types.RedshiftInteger() ) ], doc=doc, **metadata ) expected_schema = { 'type': 'record', 'name': self.table_name, 'namespace': self.namespace, 'fields': [ {'name': col_name, 'type': ['null', 'int'], 'default': None} ], 'doc': doc, 'aliases': aliases } actual_schema = converter.convert(sql_table) assert expected_schema == actual_schema
def test_convert_with_col_integer(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_integer', redshift_data_types.RedshiftInteger() ), {'name': 'col_integer', 'type': ['null', 'int'], 'default': None} )
def test_convert_with_col_bigint(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_bigint', redshift_data_types.RedshiftBigInt() ), {'name': 'col_bigint', 'type': ['null', 'long'], 'default': None} )
def test_convert_with_col_real(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn('col_real', redshift_data_types.RedshiftReal()), { 'name': 'col_real', 'type': ['null', 'float'], 'default': None, } )
def test_convert_with_column_default_value(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col', redshift_data_types.RedshiftInteger(), default_value=10 ), {'name': 'col', 'type': ['int', 'null'], 'default': 10} )
def test_convert_with_unsupported_type(self, converter): class RedshiftUnsupportedType(SQLColumnDataType): """Dummy class to test error handling code for unsupported datatypes """ type_name = 'redshift_unsupported_type' with pytest.raises(UnsupportedTypeException): column = RedshiftSQLColumn('col', RedshiftUnsupportedType()) sql_table = RedshiftSQLTable(self.table_name, [column]) converter.convert(sql_table)
def test_convert_with_col_boolean(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_boolean', redshift_data_types.RedshiftBoolean() ), {'name': 'col_boolean', 'type': ['null', 'boolean'], 'default': None} )
def test_convert_with_col_date(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn('col_date', redshift_data_types.RedshiftDate()), { 'name': 'col_date', 'type': ['null', 'int'], 'default': None, AvroMetaDataKeys.DATE: True } )
def test_convert_with_non_nullable_column(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col', redshift_data_types.RedshiftInteger(), is_nullable=False, default_value=0 ), {'name': 'col', 'type': 'int', 'default': 0} )
def test_convert_with_col_text(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn('col_text', redshift_data_types.RedshiftText()), { 'name': 'col_text', 'type': ['null', 'string'], 'default': None, AvroMetaDataKeys.MAX_LEN: 256 }, )
def test_convert_with_col_timestamp(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_ts', redshift_data_types.RedshiftTimestamp() ), { 'name': 'col_ts', 'type': ['null', 'long'], 'default': None, AvroMetaDataKeys.TIMESTAMP: True } )
def test_convert_with_col_nchar(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_nchar', redshift_data_types.RedshiftNChar(16) ), { 'name': 'col_nchar', 'type': ['null', 'string'], 'default': None, AvroMetaDataKeys.FIX_LEN: 16 } )
def test_convert_with_col_double(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_double', redshift_data_types.RedshiftDouble() ), { 'name': 'col_double', 'type': ['null', 'double'], 'default': None, } )
def test_convert_with_col_charactervarying(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_charactervarying', redshift_data_types.RedshiftCharacterVarying(20) ), { 'name': 'col_charactervarying', 'type': ['null', 'string'], 'default': None, AvroMetaDataKeys.MAX_LEN: 20 }, )
def test_convert_with_primary_key_column(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col', redshift_data_types.RedshiftInteger(), primary_key_order=1 ), { 'name': 'col', 'type': ['null', 'int'], 'default': None, AvroMetaDataKeys.PRIMARY_KEY: True } )
def test_convert_with_col_numeric(self, converter): self._convert_and_assert_with_one_column( converter, RedshiftSQLColumn( 'col_numeric', redshift_data_types.RedshiftNumeric(8, 0) ), { 'name': 'col_numeric', 'type': [ 'null', { 'logicalType': 'decimal', 'scale': 0, 'type': 'bytes', 'precision': 8 } ], 'default': None, } )