def _load(config, data_pool_item, connection_to_master):
"""
Loads the data using separate process.
:param config: dict
:param data_pool_item: dict
:param connection_to_master: multiprocessing.connection.PipeConnection
:return: None
"""
log_title = 'DataLoader::_load'
conversion = Conversion(config)
msg = '\t--[%s] Loading the data into "%s"."%s" table...' \
% (log_title, conversion.schema, data_pool_item['_tableName'])
FsOps.log(conversion, msg)
is_recovery_mode = DataLoader.data_transferred(conversion, data_pool_item['_id'])
if is_recovery_mode:
pg_client = DBAccess.get_db_client(conversion, DBVendors.PG)
DataLoader.delete_data_pool_item(conversion, data_pool_item['_id'], pg_client)
else:
DataLoader.populate_table_worker(
conversion,
data_pool_item['_tableName'],
data_pool_item['_selectFieldList'],
data_pool_item['_rowsCnt'],
data_pool_item['_id'],
connection_to_master
)
def _arrange_and_load_batch(
conversion_config,
table_name,
batch,
rows_cnt,
rows_to_insert,
number_of_inserted_rows
):
"""
Formats a batch of data as csv, and passes it to COPY.
:param conversion_config: dict
:param table_name: str
:param batch: list
:param rows_cnt: int
:param rows_to_insert: int
:param number_of_inserted_rows: int
:return: None
"""
conversion = Conversion(conversion_config)
pg_client = DBAccess.get_db_client(conversion, DBVendors.PG)
pg_cursor = pg_client.cursor()
if conversion.should_migrate_only_data():
# TODO: how to pass original_session_replication_role to the parent?
original_session_replication_role = DataLoader.disable_triggers(conversion, pg_client)
rows = ColumnsDataArranger.prepare_batch_for_copy(batch)
text_stream = io.StringIO()
text_stream.write(rows)
text_stream.seek(0)
pg_cursor.copy_from(text_stream, '"%s"."%s"' % (conversion.schema, table_name))
pg_client.commit()
number_of_inserted_rows += rows_to_insert
msg = '\t--[{0}] For now inserted: {4} rows, Total rows to insert into "{2}"."{3}": {1}' \
.format(log_title, rows_cnt, conversion.schema, table_name, number_of_inserted_rows)
print(msg) # TODO: check why FsOps.log() below doesn't work as expected.
FsOps.log(conversion, msg)
