def test_execute():
oracle_destination_conn_id = 'oracle_destination_conn_id'
destination_table = 'destination_table'
oracle_source_conn_id = 'oracle_source_conn_id'
source_sql = "select sysdate from dual where trunc(sysdate) = :p_data"
source_sql_params = {':p_data': "2018-01-01"}
rows_chunk = 5000
cursor_description = [('id', "<class 'cx_Oracle.NUMBER'>", 39, None,
38, 0, 0),
('description', "<class 'cx_Oracle.STRING'>", 60,
240, None, None, 1)]
cursor_rows = [[1, 'description 1'], [2, 'description 2']]
mock_dest_hook = MagicMock()
mock_src_hook = MagicMock()
mock_src_conn = mock_src_hook.get_conn.return_value.__enter__.return_value
mock_cursor = mock_src_conn.cursor.return_value
mock_cursor.description.__iter__.return_value = cursor_description
mock_cursor.fetchmany.side_effect = [cursor_rows, []]
op = OracleToOracleTransfer(
task_id='copy_data',
oracle_destination_conn_id=oracle_destination_conn_id,
destination_table=destination_table,
oracle_source_conn_id=oracle_source_conn_id,
source_sql=source_sql,
source_sql_params=source_sql_params,
rows_chunk=rows_chunk)
op._execute(mock_src_hook, mock_dest_hook, None)
assert mock_src_hook.get_conn.called
assert mock_src_conn.cursor.called
mock_cursor.execute.assert_called_once_with(source_sql,
source_sql_params)
calls = [
mock.call(rows_chunk),
mock.call(rows_chunk),
]
mock_cursor.fetchmany.assert_has_calls(calls)
mock_dest_hook.bulk_insert_rows.assert_called_once_with(
destination_table,
cursor_rows,
commit_every=rows_chunk,
target_fields=['id', 'description'])
def test_execute():
oracle_destination_conn_id = 'oracle_destination_conn_id'
destination_table = 'destination_table'
oracle_source_conn_id = 'oracle_source_conn_id'
source_sql = "select sysdate from dual where trunc(sysdate) = :p_data"
source_sql_params = {':p_data': "2018-01-01"}
rows_chunk = 5000
cursor_description = [
('id', "<class 'cx_Oracle.NUMBER'>", 39, None, 38, 0, 0),
('description', "<class 'cx_Oracle.STRING'>", 60, 240, None, None, 1)
]
cursor_rows = [[1, 'description 1'], [2, 'description 2']]
mock_dest_hook = MagicMock()
mock_src_hook = MagicMock()
mock_src_conn = mock_src_hook.get_conn.return_value.__enter__.return_value
mock_cursor = mock_src_conn.cursor.return_value
mock_cursor.description.__iter__.return_value = cursor_description
mock_cursor.fetchmany.side_effect = [cursor_rows, []]
op = OracleToOracleTransfer(
task_id='copy_data',
oracle_destination_conn_id=oracle_destination_conn_id,
destination_table=destination_table,
oracle_source_conn_id=oracle_source_conn_id,
source_sql=source_sql,
source_sql_params=source_sql_params,
rows_chunk=rows_chunk)
op._execute(mock_src_hook, mock_dest_hook, None)
assert mock_src_hook.get_conn.called
assert mock_src_conn.cursor.called
mock_cursor.execute.assert_called_with(source_sql, source_sql_params)
mock_cursor.fetchmany.assert_called_with(rows_chunk)
mock_dest_hook.bulk_insert_rows.assert_called_once_with(
destination_table,
cursor_rows,
commit_every=rows_chunk,
target_fields=['id', 'description'])
def match_datatransfer_operater(self):
source_db_type = self.get_conn_db_type(self.source_conn_id)
destination_db_type = self.get_conn_db_type(self.destination_conn_id)
if source_db_type == 'mysql' and destination_db_type == 'hiveserver2':
return MySqlToHiveTransfer(
sql=self.sql,
hive_table=self.destination_table,
create=False,
recreate=False,
partition=None,
delimiter=chr(1),
mysql_conn_id=self.source_conn_id,
hive_cli_conn_id=self.destination_conn_id,
tblproperties=None)
if source_db_type == 'mssql' and destination_db_type == 'hiveserver2':
return MsSqlToHiveTransfer(
sql=self.sql,
hive_table=self.destination_table,
create=False,
recreate=False,
partition=None,
delimiter=chr(1),
mysql_conn_id=self.source_conn_id,
hive_cli_conn_id=self.destination_conn_id,
tblproperties=None)
if source_db_type == 'hiveserver2' and destination_db_type == 'mysql':
return HiveToMySqlTransfer(sql=self.sql,
mysql_table=self.destination_table,
hiveserver2_conn_id=self.source_conn_id,
mysql_conn_id=self.destination_conn_id,
mysql_preoperator=None,
mysql_postoperator=None,
bulk_load=False)
if source_db_type == 'oracle' and destination_db_type == 'oracle':
return OracleToOracleTransfer(
source_sql=self.sql,
destination_table=self.destination_table,
oracle_source_conn_id=self.source_conn_id,
oracle_destination_conn_id=self.destination_conn_id,
source_sql_params=None,
rows_chunk=5000)
return GenericTransfer(sql=self.sql,
destination_table=self.destination_table,
source_conn_id=self.source_conn_id,
destination_conn_id=self.destination_conn_id,
preoperator=None)