def _make_sum_rows(table, idx_str): """Populate the summary table This method inserts rows into the compare table from the original table then forms the summary table by combining a prefix of the primary key hash (group by). table[in] Table instance idx_str[in] string representation of primary key columns Returns result from """ from mysql.utilities.common.lock import Lock col_str = ", ".join(table.get_col_names()) # Lock table first tbl_lock_list = [(table.table, "READ"), ("%s.compare_%s" % (table.db_name, table.tbl_name), "WRITE")] my_lock = Lock(table.server, tbl_lock_list) table.server.exec_query( _COMPARE_INSERT.format( db=table.db_name, table=table.tbl_name, colstr=col_str.strip(", "), pkstr=idx_str.strip(", ") ) ) res = table.server.exec_query(_COMPARE_SUM.format(db=table.db_name, table=table.tbl_name)) # Unlock table my_lock.unlock() return res
def run(self): from mysql.utilities.common.lock import Lock self.server1 = self.servers.get_server(0) test_num = 1 # Here we test the locking and unlocking of tables (if applicable) # for the locking types. comment = "Test case {0} - test locking type '{1}'" for lock_test in _LOCKTESTS: lock = None comment_str = comment.format(test_num, lock_test[0]) res_entry = { 'test_case': comment_str, 'lock_res': (0, ""), 'unlock_res': (0, ""), 'lock_fail': lock_test[2], 'unlock_fail': lock_test[3], } if lock_test[0] == 'SKIP_UNLOCK': self.options['locking'] = 'lock-all' else: self.options['locking'] = lock_test[0] try: lock = Lock(self.server1, lock_test[1], self.options) except UtilError as err: res_entry['lock_res'] = (1, err.errmsg) if lock is not None: if lock_test[0] != 'SKIP_UNLOCK': try: lock.unlock() except UtilError as err: res_entry['unlock_res'] = (1, err.errmsg) else: res = lock.__del__() if res[0:7] == 'WARNING': res_entry['unlock_res'] = (1, res) else: res_entry['unlock_res'] = (1, "Wrong result: " "{0}".format(res)) else: res_entry['unlock_res'] = (1, "Unlock() skipped.") self.results.append(res_entry) if self.debug: print(comment_str) print(" expected to fail:", res_entry['lock_fail'], res_entry['unlock_fail']) print(" locking step:", res_entry['lock_res'][0], res_entry['lock_res'][1]) print(" unlocking step:", res_entry['unlock_res'][0], res_entry['unlock_res'][1]) test_num += 1 return True
def run(self): from mysql.utilities.common.lock import Lock self.server1 = self.servers.get_server(0) test_num = 1 # Here we test the locking and unlocking of tables (if applicable) # for the locking types. comment = "Test case {0} - test locking type '{1}'" for lock_test in _LOCKTESTS: lock = None comment_str = comment.format(test_num, lock_test[0]) res_entry = {'test_case': comment_str, 'lock_res': (0, ""), 'unlock_res': (0, ""), 'lock_fail': lock_test[2], 'unlock_fail': lock_test[3], } if lock_test[0] == 'SKIP_UNLOCK': self.options['locking'] = 'lock-all' else: self.options['locking'] = lock_test[0] try: lock = Lock(self.server1, lock_test[1], self.options) except UtilError as err: res_entry['lock_res'] = (1, err.errmsg) if lock is not None: if lock_test[0] != 'SKIP_UNLOCK': try: lock.unlock() except UtilError as err: res_entry['unlock_res'] = (1, err.errmsg) else: res = lock.__del__() if res[0:7] == 'WARNING': res_entry['unlock_res'] = (1, res) else: res_entry['unlock_res'] = (1, "Wrong result: " "{0}".format(res)) else: res_entry['unlock_res'] = (1, "Unlock() skipped.") self.results.append(res_entry) if self.debug: print(comment_str) print(" expected to fail:", res_entry['lock_fail'], res_entry['unlock_fail']) print(" locking step:", res_entry['lock_res'][0], res_entry['lock_res'][1]) print (" unlocking step:", res_entry['unlock_res'][0], res_entry['unlock_res'][1]) test_num += 1 return True
def run(self): from mysql.utilities.common.lock import Lock self.server1 = self.servers.get_server(0) test_num = 1 # Here we test the locking and unlocking of tables (if applicable) # for the locking types. comment = "Test case %s - test locking type '%s'" for lock_test in _LOCKTESTS: lock = None comment_str = comment % (test_num, lock_test[0]) res_entry = { 'test_case' : comment_str, 'lock_res' : (0,""), 'unlock_res' : (0,""), 'lock_fail' : lock_test[2], 'unlock_fail' : lock_test[3], } if lock_test[0] == 'SKIP_UNLOCK': self.options['locking'] = 'lock-all' else: self.options['locking'] = lock_test[0] try: lock = Lock(self.server1, lock_test[1], self.options) except UtilError, e: res_entry['lock_res'] = (1, e.errmsg) if lock is not None: if lock_test[0] != 'SKIP_UNLOCK': try: lock.unlock() except UtilError, e: res_entry['unlock_res'] = (1, e.errmsg) else: res = lock.__del__() if res[0:7] == 'WARNING': res_entry['unlock_res'] = (1, res) else: res_entry['unlock_res'] = (1, "Wrong result: %s" % res)
def _make_sum_rows(table, idx_str): """Populate the summary table This method inserts rows into the compare table from the original table then forms the summary table by combining a prefix of the primary key hash (group by). table[in] Table instance idx_str[in] string representation of primary key columns Returns result from """ from mysql.utilities.common.lock import Lock col_str = ", ".join(table.get_col_names(True)) # Lock table first tbl_lock_list = [(table.table, 'READ'), ("%s.compare_%s" % (table.db_name, table.tbl_name), 'WRITE')] my_lock = Lock(table.server, tbl_lock_list) # Quote compare table appropriately with backticks q_tbl_name = quote_with_backticks( _COMPARE_TABLE_NAME.format(tbl=table.tbl_name)) table.server.exec_query( _COMPARE_INSERT.format(db=table.q_db_name, compare_tbl=q_tbl_name, colstr=col_str.strip(", "), pkstr=idx_str.strip(", "), table=table.q_tbl_name)) res = table.server.exec_query( _COMPARE_SUM.format(db=table.q_db_name, compare_tbl=q_tbl_name)) # Unlock table my_lock.unlock() return res
def _bulk_insert(self, rows, new_db, destination=None): """Import data using bulk insert Reads data from a table and builds group INSERT statements for writing to the destination server specified (new_db.name). This method is designed to be used in a thread for parallel inserts. As such, it requires its own connection to the destination server. Note: This method does not print any information to stdout. rows[in] a list of rows to process new_db[in] new database name destination[in] the destination server """ if self.dest_vals is None: self.dest_vals = self.get_dest_values(destination) # Spawn a new connection server_options = { 'conn_info': self.dest_vals, 'role': "thread", } dest = Server(server_options) dest.connect() # Issue the write lock lock_list = [("%s.%s" % (new_db, self.q_tbl_name), 'WRITE')] my_lock = Lock(dest, lock_list, {'locking': 'lock-all', }) # First, turn off foreign keys if turned on dest.disable_foreign_key_checks(True) if self.column_format is None: self.get_column_metadata() data_lists = self.make_bulk_insert(rows, new_db) insert_data = data_lists[0] blob_data = data_lists[1] # Insert the data first for data_insert in insert_data: try: dest.exec_query(data_insert, self.query_options) except UtilError, e: raise UtilError("Problem inserting data. " "Error = %s" % e.errmsg)
def get_copy_lock(server, db_list, options, include_mysql=False, cloning=False): """Get an instance of the Lock class with a standard copy (read) lock This method creates an instance of the Lock class using the lock type specified in the options. It is used to initiate the locks for the copy and related operations. server[in] Server instance for locking calls db_list[in] list of database names options[in] option dictionary Must include the skip_* options for copy and export include_mysql[in] if True, include the mysql tables for copy operation cloning[in] if True, create lock tables with WRITE on dest db Default = False Returns Lock - Lock class instance """ rpl_mode = options.get("rpl_mode", None) locking = options.get('locking', 'snapshot') # Determine if we need to use FTWRL. There are two conditions: # - running on master (rpl_mode = 'master') # - using locking = 'lock-all' and rpl_mode present if (rpl_mode in ["master", "both"]) or \ (rpl_mode and locking == 'lock-all'): new_opts = options.copy() new_opts['locking'] = 'flush' lock = Lock(server, [], new_opts) # if this is a lock-all type and not replication operation, # find all tables and lock them # pylint: disable=R0101 elif locking == 'lock-all': table_lock_list = [] # Build table lock list for db_name in db_list: db = db_name[0] if isinstance(db_name, tuple) else db_name source_db = Database(server, db) tables = source_db.get_db_objects("TABLE") for table in tables: table_lock_list.append(("{0}.{1}".format(db, table[0]), 'READ')) # Cloning requires issuing WRITE locks because we use same # conn. # Non-cloning will issue WRITE lock on a new destination conn. if cloning: if db_name[1] is None: db_clone = db_name[0] else: db_clone = db_name[1] # For cloning, we use the same connection so we need to # lock the destination tables with WRITE. table_lock_list.append( ("{0}.{1}".format(db_clone, table[0]), 'WRITE')) # We must include views for server version 5.5.3 and higher if server.check_version_compat(5, 5, 3): tables = source_db.get_db_objects("VIEW") for table in tables: table_lock_list.append( ("{0}.{1}".format(db, table[0]), 'READ')) # Cloning requires issuing WRITE locks because we use same # conn. # Non-cloning will issue WRITE lock on a new destination # conn. if cloning: if db_name[1] is None: db_clone = db_name[0] else: db_clone = db_name[1] # For cloning, we use the same connection so we need to # lock the destination tables with WRITE. table_lock_list.append( ("{0}.{1}".format(db_clone, table[0]), 'WRITE')) # Now add mysql tables if include_mysql: # Don't lock proc tables if no procs of funcs are being read if not options.get('skip_procs', False) and \ not options.get('skip_funcs', False): table_lock_list.append(("mysql.proc", 'READ')) table_lock_list.append(("mysql.procs_priv", 'READ')) # Don't lock event table if events are skipped if not options.get('skip_events', False): table_lock_list.append(("mysql.event", 'READ')) lock = Lock(server, table_lock_list, options) # Use default or no locking option else: lock = Lock(server, [], options) return lock