def _load(config, data_pool_item, connection_to_master): """ Loads the data using separate process. :param config: dict :param data_pool_item: dict :param connection_to_master: multiprocessing.connection.PipeConnection :return: None """ log_title = 'DataLoader::_load' conversion = Conversion(config) msg = '\t--[%s] Loading the data into "%s"."%s" table...' \ % (log_title, conversion.schema, data_pool_item['_tableName']) FsOps.log(conversion, msg) is_recovery_mode = DataLoader.data_transferred(conversion, data_pool_item['_id']) if is_recovery_mode: pg_client = DBAccess.get_db_client(conversion, DBVendors.PG) DataLoader.delete_data_pool_item(conversion, data_pool_item['_id'], pg_client) else: DataLoader.populate_table_worker( conversion, data_pool_item['_tableName'], data_pool_item['_selectFieldList'], data_pool_item['_rowsCnt'], data_pool_item['_id'], connection_to_master )
def _arrange_and_load_batch( conversion_config, table_name, batch, rows_cnt, rows_to_insert, number_of_inserted_rows ): """ Formats a batch of data as csv, and passes it to COPY. :param conversion_config: dict :param table_name: str :param batch: list :param rows_cnt: int :param rows_to_insert: int :param number_of_inserted_rows: int :return: None """ conversion = Conversion(conversion_config) pg_client = DBAccess.get_db_client(conversion, DBVendors.PG) pg_cursor = pg_client.cursor() if conversion.should_migrate_only_data(): # TODO: how to pass original_session_replication_role to the parent? original_session_replication_role = DataLoader.disable_triggers(conversion, pg_client) rows = ColumnsDataArranger.prepare_batch_for_copy(batch) text_stream = io.StringIO() text_stream.write(rows) text_stream.seek(0) pg_cursor.copy_from(text_stream, '"%s"."%s"' % (conversion.schema, table_name)) pg_client.commit() number_of_inserted_rows += rows_to_insert msg = '\t--[{0}] For now inserted: {4} rows, Total rows to insert into "{2}"."{3}": {1}' \ .format(log_title, rows_cnt, conversion.schema, table_name, number_of_inserted_rows) print(msg) # TODO: check why FsOps.log() below doesn't work as expected. FsOps.log(conversion, msg)