def test_convert_with_table_metadata(self, converter):
pkey_col1 = RedshiftSQLColumn(
'pkey_col_one',
redshift_data_types.RedshiftInteger(),
primary_key_order=1,
sort_key_order=2,
is_dist_key=True
)
pkey_col2 = RedshiftSQLColumn(
'pkey_col_two',
redshift_data_types.RedshiftInteger(),
primary_key_order=2,
sort_key_order=1
)
col = RedshiftSQLColumn('col', redshift_data_types.RedshiftInteger())
sql_table = RedshiftSQLTable(
self.table_name, [pkey_col2, pkey_col1, col], diststyle='all'
)
expected_schema = {
'type': 'record',
'name': self.table_name,
'namespace': '',
'fields': [
{
'name': pkey_col2.name,
'type': ['null', 'int'],
'default': None,
AvroMetaDataKeys.PRIMARY_KEY: 2,
AvroMetaDataKeys.SORT_KEY: 1
},
{
'name': pkey_col1.name,
'type': ['null', 'int'],
'default': None,
AvroMetaDataKeys.PRIMARY_KEY: 1,
AvroMetaDataKeys.DIST_KEY: True,
AvroMetaDataKeys.SORT_KEY: 2
},
{
'name': col.name,
'type': ['null', 'int'],
'default': None
},
],
AvroMetaDataKeys.PRIMARY_KEY: [pkey_col1.name, pkey_col2.name],
AvroMetaDataKeys.SORT_KEY: [pkey_col2.name, pkey_col1.name],
AvroMetaDataKeys.DIST_KEY: pkey_col1.name,
AvroMetaDataKeys.DISTSTYLE: 'all'
}
actual_schema = converter.convert(sql_table)
assert expected_schema == actual_schema
def test_convert_with_table_namespace_and_aliases(self, converter):
doc = 'I am doc'
aliases = ['foo']
metadata = {
MetaDataKey.NAMESPACE: self.namespace,
MetaDataKey.ALIASES: aliases
}
col_name = 'col'
sql_table = RedshiftSQLTable(
self.table_name,
[
RedshiftSQLColumn(
col_name, redshift_data_types.RedshiftInteger()
)
],
doc=doc,
**metadata
)
expected_schema = {
'type': 'record',
'name': self.table_name,
'namespace': self.namespace,
'fields': [
{'name': col_name, 'type': ['null', 'int'], 'default': None}
],
'doc': doc,
'aliases': aliases
}
actual_schema = converter.convert(sql_table)
assert expected_schema == actual_schema
def test_convert_with_col_integer(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_integer', redshift_data_types.RedshiftInteger()
),
{'name': 'col_integer', 'type': ['null', 'int'], 'default': None}
)
def test_convert_with_col_bigint(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_bigint', redshift_data_types.RedshiftBigInt()
),
{'name': 'col_bigint', 'type': ['null', 'long'], 'default': None}
)
def test_convert_with_col_real(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn('col_real', redshift_data_types.RedshiftReal()),
{
'name': 'col_real',
'type': ['null', 'float'],
'default': None,
}
)
def test_convert_with_column_default_value(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col',
redshift_data_types.RedshiftInteger(),
default_value=10
),
{'name': 'col', 'type': ['int', 'null'], 'default': 10}
)
def test_convert_with_unsupported_type(self, converter):
class RedshiftUnsupportedType(SQLColumnDataType):
"""Dummy class to test error handling code for
unsupported datatypes
"""
type_name = 'redshift_unsupported_type'
with pytest.raises(UnsupportedTypeException):
column = RedshiftSQLColumn('col', RedshiftUnsupportedType())
sql_table = RedshiftSQLTable(self.table_name, [column])
converter.convert(sql_table)
def test_convert_with_col_boolean(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_boolean', redshift_data_types.RedshiftBoolean()
),
{'name': 'col_boolean',
'type': ['null', 'boolean'],
'default': None}
)
def test_convert_with_col_date(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn('col_date', redshift_data_types.RedshiftDate()),
{
'name': 'col_date',
'type': ['null', 'int'],
'default': None,
AvroMetaDataKeys.DATE: True
}
)
def test_convert_with_non_nullable_column(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col',
redshift_data_types.RedshiftInteger(),
is_nullable=False,
default_value=0
),
{'name': 'col', 'type': 'int', 'default': 0}
)
def test_convert_with_col_text(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn('col_text', redshift_data_types.RedshiftText()),
{
'name': 'col_text',
'type': ['null', 'string'],
'default': None,
AvroMetaDataKeys.MAX_LEN: 256
},
)
def test_convert_with_col_timestamp(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_ts', redshift_data_types.RedshiftTimestamp()
),
{
'name': 'col_ts',
'type': ['null', 'long'],
'default': None,
AvroMetaDataKeys.TIMESTAMP: True
}
)
def test_convert_with_col_nchar(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_nchar', redshift_data_types.RedshiftNChar(16)
),
{
'name': 'col_nchar',
'type': ['null', 'string'],
'default': None,
AvroMetaDataKeys.FIX_LEN: 16
}
)
def test_convert_with_col_double(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_double',
redshift_data_types.RedshiftDouble()
),
{
'name': 'col_double',
'type': ['null', 'double'],
'default': None,
}
)
def test_convert_with_col_charactervarying(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_charactervarying',
redshift_data_types.RedshiftCharacterVarying(20)
),
{
'name': 'col_charactervarying',
'type': ['null', 'string'],
'default': None,
AvroMetaDataKeys.MAX_LEN: 20
},
)
def test_convert_with_primary_key_column(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col',
redshift_data_types.RedshiftInteger(),
primary_key_order=1
),
{
'name': 'col',
'type': ['null', 'int'],
'default': None,
AvroMetaDataKeys.PRIMARY_KEY: True
}
)
def test_convert_with_col_numeric(self, converter):
self._convert_and_assert_with_one_column(
converter,
RedshiftSQLColumn(
'col_numeric',
redshift_data_types.RedshiftNumeric(8, 0)
),
{
'name': 'col_numeric',
'type':
[
'null',
{
'logicalType': 'decimal',
'scale': 0,
'type': 'bytes',
'precision': 8
}
],
'default': None,
}
)