def test_execute_empty_description_field(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv): type(mock_tmp_file).name = PropertyMock(return_value='tmp_file') mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file) mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__ mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__ mock_mssql_hook_cursor.return_value.description = [('', '')] mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs) mssql_to_hive_transfer.execute(context={}) field_dict = OrderedDict() col_count = 0 for field in mock_mssql_hook_cursor.return_value.description: col_count += 1 col_position = "Column{position}".format(position=col_count) field_dict[col_position] = mssql_to_hive_transfer.type_map( field[1]) mock_hive_hook.return_value.load_file.assert_called_once_with( mock_tmp_file.name, mssql_to_hive_transfer.hive_table, field_dict=field_dict, create=mssql_to_hive_transfer.create, partition=mssql_to_hive_transfer.partition, delimiter=mssql_to_hive_transfer.delimiter, recreate=mssql_to_hive_transfer.recreate, tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv): type(mock_tmp_file).name = PropertyMock(return_value='tmp_file') mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file) mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__ mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__ mock_mssql_hook_cursor.return_value.description = [('te', 'st')] mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs) mssql_to_hive_transfer.execute(context={}) mock_mssql_hook_cursor.return_value.execute.assert_called_once_with( mssql_to_hive_transfer.sql) mock_csv.writer.assert_called_once_with( mock_tmp_file, delimiter=mssql_to_hive_transfer.delimiter, encoding='utf-8') field_dict = OrderedDict() for field in mock_mssql_hook_cursor.return_value.description: field_dict[field[0]] = mssql_to_hive_transfer.type_map(field[1]) mock_csv.writer.return_value.writerows.assert_called_once_with( mock_mssql_hook_cursor.return_value) mock_hive_hook.return_value.load_file.assert_called_once_with( mock_tmp_file.name, mssql_to_hive_transfer.hive_table, field_dict=field_dict, create=mssql_to_hive_transfer.create, partition=mssql_to_hive_transfer.partition, delimiter=mssql_to_hive_transfer.delimiter, recreate=mssql_to_hive_transfer.recreate, tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute_empty_description_field(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv): type(mock_tmp_file).name = PropertyMock(return_value='tmp_file') mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file) mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__ mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__ mock_mssql_hook_cursor.return_value.description = [('', '')] mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs) mssql_to_hive_transfer.execute(context={}) field_dict = OrderedDict() col_count = 0 for field in mock_mssql_hook_cursor.return_value.description: col_count += 1 col_position = "Column{position}".format(position=col_count) field_dict[col_position] = mssql_to_hive_transfer.type_map(field[1]) mock_hive_hook.return_value.load_file.assert_called_once_with( mock_tmp_file.name, mssql_to_hive_transfer.hive_table, field_dict=field_dict, create=mssql_to_hive_transfer.create, partition=mssql_to_hive_transfer.partition, delimiter=mssql_to_hive_transfer.delimiter, recreate=mssql_to_hive_transfer.recreate, tblproperties=mssql_to_hive_transfer.tblproperties)
def test_execute(self, mock_hive_hook, mock_mssql_hook, mock_tmp_file, mock_csv): type(mock_tmp_file).name = PropertyMock(return_value='tmp_file') mock_tmp_file.return_value.__enter__ = Mock(return_value=mock_tmp_file) mock_mssql_hook_get_conn = mock_mssql_hook.return_value.get_conn.return_value.__enter__ mock_mssql_hook_cursor = mock_mssql_hook_get_conn.return_value.cursor.return_value.__enter__ mock_mssql_hook_cursor.return_value.description = [('te', 'st')] mssql_to_hive_transfer = MsSqlToHiveTransfer(**self.kwargs) mssql_to_hive_transfer.execute(context={}) mock_mssql_hook_cursor.return_value.execute.assert_called_once_with(mssql_to_hive_transfer.sql) mock_csv.writer.assert_called_once_with( mock_tmp_file, delimiter=mssql_to_hive_transfer.delimiter, encoding='utf-8') field_dict = OrderedDict() for field in mock_mssql_hook_cursor.return_value.description: field_dict[field[0]] = mssql_to_hive_transfer.type_map(field[1]) mock_csv.writer.return_value.writerows.assert_called_once_with(mock_mssql_hook_cursor.return_value) mock_hive_hook.return_value.load_file.assert_called_once_with( mock_tmp_file.name, mssql_to_hive_transfer.hive_table, field_dict=field_dict, create=mssql_to_hive_transfer.create, partition=mssql_to_hive_transfer.partition, delimiter=mssql_to_hive_transfer.delimiter, recreate=mssql_to_hive_transfer.recreate, tblproperties=mssql_to_hive_transfer.tblproperties)
def match_datatransfer_operater(self): source_db_type = self.get_conn_db_type(self.source_conn_id) destination_db_type = self.get_conn_db_type(self.destination_conn_id) if source_db_type == 'mysql' and destination_db_type == 'hiveserver2': return MySqlToHiveTransfer( sql=self.sql, hive_table=self.destination_table, create=False, recreate=False, partition=None, delimiter=chr(1), mysql_conn_id=self.source_conn_id, hive_cli_conn_id=self.destination_conn_id, tblproperties=None) if source_db_type == 'mssql' and destination_db_type == 'hiveserver2': return MsSqlToHiveTransfer( sql=self.sql, hive_table=self.destination_table, create=False, recreate=False, partition=None, delimiter=chr(1), mysql_conn_id=self.source_conn_id, hive_cli_conn_id=self.destination_conn_id, tblproperties=None) if source_db_type == 'hiveserver2' and destination_db_type == 'mysql': return HiveToMySqlTransfer(sql=self.sql, mysql_table=self.destination_table, hiveserver2_conn_id=self.source_conn_id, mysql_conn_id=self.destination_conn_id, mysql_preoperator=None, mysql_postoperator=None, bulk_load=False) if source_db_type == 'oracle' and destination_db_type == 'oracle': return OracleToOracleTransfer( source_sql=self.sql, destination_table=self.destination_table, oracle_source_conn_id=self.source_conn_id, oracle_destination_conn_id=self.destination_conn_id, source_sql_params=None, rows_chunk=5000) return GenericTransfer(sql=self.sql, destination_table=self.destination_table, source_conn_id=self.source_conn_id, destination_conn_id=self.destination_conn_id, preoperator=None)
def test_type_map_string(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map(None) self.assertEqual(mapped_type, 'STRING')
def test_type_map_number(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map( pymssql.NUMBER.value) self.assertEqual(mapped_type, 'INT')
def test_type_map_decimal(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map( pymssql.DECIMAL.value) self.assertEqual(mapped_type, 'FLOAT')
def test_type_map_binary(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map( pymssql.BINARY.value) self.assertEqual(mapped_type, 'INT')
def test_type_map_number(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map( pymssql.NUMBER.value) # pylint: disable=c-extension-no-member self.assertEqual(mapped_type, 'INT')
def test_type_map_decimal(self): mapped_type = MsSqlToHiveTransfer(**self.kwargs).type_map( pymssql.DECIMAL.value) # pylint: disable=c-extension-no-member self.assertEqual(mapped_type, 'FLOAT')