def test_execute_empty_description_field(self, mock_hive_hook,
mock_mssql_hook, mock_tmp_file,
mock_csv):
type(mock_tmp_file).name = PropertyMock(return_value='tmp_file')
mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file)
mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__
mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__
mock_mssql_hook_cursor.return_value.description = [('', '')]
mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs)
mssql_to_hive_transfer.execute(context={})
field_dict = OrderedDict()
col_count = 0
for field in mock_mssql_hook_cursor.return_value.description:
col_count += 1
col_position = "Column{position}".format(position=col_count)
field_dict[col_position] = mssql_to_hive_transfer.type_map(
field[1])
mock_hive_hook.return_value.load_file.assert_called_once_with(
mock_tmp_file.name,
mssql_to_hive_transfer.hive_table,
field_dict=field_dict,
create=mssql_to_hive_transfer.create,
partition=mssql_to_hive_transfer.partition,
delimiter=mssql_to_hive_transfer.delimiter,
recreate=mssql_to_hive_transfer.recreate,
tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file,
mock_csv):
type(mock_tmp_file).name = PropertyMock(return_value='tmp_file')
mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file)
mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__
mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__
mock_mssql_hook_cursor.return_value.description = [('te', 'st')]
mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs)
mssql_to_hive_transfer.execute(context={})
mock_mssql_hook_cursor.return_value.execute.assert_called_once_with(
mssql_to_hive_transfer.sql)
mock_csv.writer.assert_called_once_with(
mock_tmp_file,
delimiter=mssql_to_hive_transfer.delimiter,
encoding='utf-8')
field_dict = OrderedDict()
for field in mock_mssql_hook_cursor.return_value.description:
field_dict[field[0]] = mssql_to_hive_transfer.type_map(field[1])
mock_csv.writer.return_value.writerows.assert_called_once_with(
mock_mssql_hook_cursor.return_value)
mock_hive_hook.return_value.load_file.assert_called_once_with(
mock_tmp_file.name,
mssql_to_hive_transfer.hive_table,
field_dict=field_dict,
create=mssql_to_hive_transfer.create,
partition=mssql_to_hive_transfer.partition,
delimiter=mssql_to_hive_transfer.delimiter,
recreate=mssql_to_hive_transfer.recreate,
tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute_empty_description_field(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv):
type(mock_tmp_file).name = PropertyMock(return_value='tmp_file')
mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file)
mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__
mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__
mock_mssql_hook_cursor.return_value.description = [('', '')]
mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs)
mssql_to_hive_transfer.execute(context={})
field_dict = OrderedDict()
col_count = 0
for field in mock_mssql_hook_cursor.return_value.description:
col_count += 1
col_position = "Column{position}".format(position=col_count)
field_dict[col_position] = mssql_to_hive_transfer.type_map(field[1])
mock_hive_hook.return_value.load_file.assert_called_once_with(
mock_tmp_file.name,
mssql_to_hive_transfer.hive_table,
field_dict=field_dict,
create=mssql_to_hive_transfer.create,
partition=mssql_to_hive_transfer.partition,
delimiter=mssql_to_hive_transfer.delimiter,
recreate=mssql_to_hive_transfer.recreate,
tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv):
type(mock_tmp_file).name = PropertyMock(return_value='tmp_file')
mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file)
mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__
mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__
mock_mssql_hook_cursor.return_value.description = [('te', 'st')]
mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs)
mssql_to_hive_transfer.execute(context={})
mock_mssql_hook_cursor.return_value.execute.assert_called_once_with(mssql_to_hive_transfer.sql)
mock_csv.writer.assert_called_once_with(
mock_tmp_file, delimiter=mssql_to_hive_transfer.delimiter, encoding='utf-8')
field_dict = OrderedDict()
for field in mock_mssql_hook_cursor.return_value.description:
field_dict[field[0]] = mssql_to_hive_transfer.type_map(field[1])
mock_csv.writer.return_value.writerows.assert_called_once_with(mock_mssql_hook_cursor.return_value)
mock_hive_hook.return_value.load_file.assert_called_once_with(
mock_tmp_file.name,
mssql_to_hive_transfer.hive_table,
field_dict=field_dict,
create=mssql_to_hive_transfer.create,
partition=mssql_to_hive_transfer.partition,
delimiter=mssql_to_hive_transfer.delimiter,
recreate=mssql_to_hive_transfer.recreate,
tblproperties=mssql_to_hive_transfer.tblproperties)
def match_datatransfer_operater(self):
source_db_type = self.get_conn_db_type(self.source_conn_id)
destination_db_type = self.get_conn_db_type(self.destination_conn_id)
if source_db_type == 'mysql' and destination_db_type == 'hiveserver2':
return MySqlToHiveTransfer(
sql=self.sql,
hive_table=self.destination_table,
create=False,
recreate=False,
partition=None,
delimiter=chr(1),
mysql_conn_id=self.source_conn_id,
hive_cli_conn_id=self.destination_conn_id,
tblproperties=None)
if source_db_type == 'mssql' and destination_db_type == 'hiveserver2':
return MsSqlToHiveTransfer(
sql=self.sql,
hive_table=self.destination_table,
create=False,
recreate=False,
partition=None,
delimiter=chr(1),
mysql_conn_id=self.source_conn_id,
hive_cli_conn_id=self.destination_conn_id,
tblproperties=None)
if source_db_type == 'hiveserver2' and destination_db_type == 'mysql':
return HiveToMySqlTransfer(sql=self.sql,
mysql_table=self.destination_table,
hiveserver2_conn_id=self.source_conn_id,
mysql_conn_id=self.destination_conn_id,
mysql_preoperator=None,
mysql_postoperator=None,
bulk_load=False)
if source_db_type == 'oracle' and destination_db_type == 'oracle':
return OracleToOracleTransfer(
source_sql=self.sql,
destination_table=self.destination_table,
oracle_source_conn_id=self.source_conn_id,
oracle_destination_conn_id=self.destination_conn_id,
source_sql_params=None,
rows_chunk=5000)
return GenericTransfer(sql=self.sql,
destination_table=self.destination_table,
source_conn_id=self.source_conn_id,
destination_conn_id=self.destination_conn_id,
preoperator=None)
def test_type_map_string(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(None)
self.assertEqual(mapped_type, 'STRING')
def test_type_map_number(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(
pymssql.NUMBER.value)
self.assertEqual(mapped_type, 'INT')
def test_type_map_decimal(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(
pymssql.DECIMAL.value)
self.assertEqual(mapped_type, 'FLOAT')
def test_type_map_binary(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(
pymssql.BINARY.value)
self.assertEqual(mapped_type, 'INT')
def test_type_map_number(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(
pymssql.NUMBER.value) # pylint: disable=c-extension-no-member
self.assertEqual(mapped_type, 'INT')
def test_type_map_decimal(self):
mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(
pymssql.DECIMAL.value) # pylint: disable=c-extension-no-member
self.assertEqual(mapped_type, 'FLOAT')
