def _make_sum_rows(table, idx_str):
"""Populate the summary table
This method inserts rows into the compare table from the original table
then forms the summary table by combining a prefix of the primary key
hash (group by).
table[in] Table instance
idx_str[in] string representation of primary key columns
Returns result from
"""
from mysql.utilities.common.lock import Lock
col_str = ", ".join(table.get_col_names())
# Lock table first
tbl_lock_list = [(table.table, "READ"), ("%s.compare_%s" % (table.db_name, table.tbl_name), "WRITE")]
my_lock = Lock(table.server, tbl_lock_list)
table.server.exec_query(
_COMPARE_INSERT.format(
db=table.db_name, table=table.tbl_name, colstr=col_str.strip(", "), pkstr=idx_str.strip(", ")
)
)
res = table.server.exec_query(_COMPARE_SUM.format(db=table.db_name, table=table.tbl_name))
# Unlock table
my_lock.unlock()
return res
def run(self):
from mysql.utilities.common.lock import Lock
self.server1 = self.servers.get_server(0)
test_num = 1
# Here we test the locking and unlocking of tables (if applicable)
# for the locking types.
comment = "Test case {0} - test locking type '{1}'"
for lock_test in _LOCKTESTS:
lock = None
comment_str = comment.format(test_num, lock_test[0])
res_entry = {
'test_case': comment_str,
'lock_res': (0, ""),
'unlock_res': (0, ""),
'lock_fail': lock_test[2],
'unlock_fail': lock_test[3],
}
if lock_test[0] == 'SKIP_UNLOCK':
self.options['locking'] = 'lock-all'
else:
self.options['locking'] = lock_test[0]
try:
lock = Lock(self.server1, lock_test[1], self.options)
except UtilError as err:
res_entry['lock_res'] = (1, err.errmsg)
if lock is not None:
if lock_test[0] != 'SKIP_UNLOCK':
try:
lock.unlock()
except UtilError as err:
res_entry['unlock_res'] = (1, err.errmsg)
else:
res = lock.__del__()
if res[0:7] == 'WARNING':
res_entry['unlock_res'] = (1, res)
else:
res_entry['unlock_res'] = (1, "Wrong result: "
"{0}".format(res))
else:
res_entry['unlock_res'] = (1, "Unlock() skipped.")
self.results.append(res_entry)
if self.debug:
print(comment_str)
print(" expected to fail:", res_entry['lock_fail'],
res_entry['unlock_fail'])
print(" locking step:", res_entry['lock_res'][0],
res_entry['lock_res'][1])
print(" unlocking step:", res_entry['unlock_res'][0],
res_entry['unlock_res'][1])
test_num += 1
return True
def run(self):
from mysql.utilities.common.lock import Lock
self.server1 = self.servers.get_server(0)
test_num = 1
# Here we test the locking and unlocking of tables (if applicable)
# for the locking types.
comment = "Test case {0} - test locking type '{1}'"
for lock_test in _LOCKTESTS:
lock = None
comment_str = comment.format(test_num, lock_test[0])
res_entry = {'test_case': comment_str, 'lock_res': (0, ""),
'unlock_res': (0, ""), 'lock_fail': lock_test[2],
'unlock_fail': lock_test[3], }
if lock_test[0] == 'SKIP_UNLOCK':
self.options['locking'] = 'lock-all'
else:
self.options['locking'] = lock_test[0]
try:
lock = Lock(self.server1, lock_test[1], self.options)
except UtilError as err:
res_entry['lock_res'] = (1, err.errmsg)
if lock is not None:
if lock_test[0] != 'SKIP_UNLOCK':
try:
lock.unlock()
except UtilError as err:
res_entry['unlock_res'] = (1, err.errmsg)
else:
res = lock.__del__()
if res[0:7] == 'WARNING':
res_entry['unlock_res'] = (1, res)
else:
res_entry['unlock_res'] = (1, "Wrong result: "
"{0}".format(res))
else:
res_entry['unlock_res'] = (1, "Unlock() skipped.")
self.results.append(res_entry)
if self.debug:
print(comment_str)
print(" expected to fail:", res_entry['lock_fail'],
res_entry['unlock_fail'])
print(" locking step:", res_entry['lock_res'][0],
res_entry['lock_res'][1])
print (" unlocking step:", res_entry['unlock_res'][0],
res_entry['unlock_res'][1])
test_num += 1
return True
def run(self):
from mysql.utilities.common.lock import Lock
self.server1 = self.servers.get_server(0)
test_num = 1
# Here we test the locking and unlocking of tables (if applicable)
# for the locking types.
comment = "Test case %s - test locking type '%s'"
for lock_test in _LOCKTESTS:
lock = None
comment_str = comment % (test_num, lock_test[0])
res_entry = {
'test_case' : comment_str,
'lock_res' : (0,""),
'unlock_res' : (0,""),
'lock_fail' : lock_test[2],
'unlock_fail' : lock_test[3],
}
if lock_test[0] == 'SKIP_UNLOCK':
self.options['locking'] = 'lock-all'
else:
self.options['locking'] = lock_test[0]
try:
lock = Lock(self.server1, lock_test[1], self.options)
except UtilError, e:
res_entry['lock_res'] = (1, e.errmsg)
if lock is not None:
if lock_test[0] != 'SKIP_UNLOCK':
try:
lock.unlock()
except UtilError, e:
res_entry['unlock_res'] = (1, e.errmsg)
else:
res = lock.__del__()
if res[0:7] == 'WARNING':
res_entry['unlock_res'] = (1, res)
else:
res_entry['unlock_res'] = (1, "Wrong result: %s" % res)
def run(self):
from mysql.utilities.common.lock import Lock
self.server1 = self.servers.get_server(0)
test_num = 1
# Here we test the locking and unlocking of tables (if applicable)
# for the locking types.
comment = "Test case %s - test locking type '%s'"
for lock_test in _LOCKTESTS:
lock = None
comment_str = comment % (test_num, lock_test[0])
res_entry = {
'test_case' : comment_str,
'lock_res' : (0,""),
'unlock_res' : (0,""),
'lock_fail' : lock_test[2],
'unlock_fail' : lock_test[3],
}
if lock_test[0] == 'SKIP_UNLOCK':
self.options['locking'] = 'lock-all'
else:
self.options['locking'] = lock_test[0]
try:
lock = Lock(self.server1, lock_test[1], self.options)
except UtilError, e:
res_entry['lock_res'] = (1, e.errmsg)
if lock is not None:
if lock_test[0] != 'SKIP_UNLOCK':
try:
lock.unlock()
except UtilError, e:
res_entry['unlock_res'] = (1, e.errmsg)
else:
res = lock.__del__()
if res[0:7] == 'WARNING':
res_entry['unlock_res'] = (1, res)
else:
res_entry['unlock_res'] = (1, "Wrong result: %s" % res)
def _make_sum_rows(table, idx_str):
"""Populate the summary table
This method inserts rows into the compare table from the original table
then forms the summary table by combining a prefix of the primary key
hash (group by).
table[in] Table instance
idx_str[in] string representation of primary key columns
Returns result from
"""
from mysql.utilities.common.lock import Lock
col_str = ", ".join(table.get_col_names(True))
# Lock table first
tbl_lock_list = [(table.table, 'READ'),
("%s.compare_%s" % (table.db_name, table.tbl_name),
'WRITE')]
my_lock = Lock(table.server, tbl_lock_list)
# Quote compare table appropriately with backticks
q_tbl_name = quote_with_backticks(
_COMPARE_TABLE_NAME.format(tbl=table.tbl_name))
table.server.exec_query(
_COMPARE_INSERT.format(db=table.q_db_name,
compare_tbl=q_tbl_name,
colstr=col_str.strip(", "),
pkstr=idx_str.strip(", "),
table=table.q_tbl_name))
res = table.server.exec_query(
_COMPARE_SUM.format(db=table.q_db_name, compare_tbl=q_tbl_name))
# Unlock table
my_lock.unlock()
return res
def _bulk_insert(self, rows, new_db, destination=None):
"""Import data using bulk insert
Reads data from a table and builds group INSERT statements for writing
to the destination server specified (new_db.name).
This method is designed to be used in a thread for parallel inserts.
As such, it requires its own connection to the destination server.
Note: This method does not print any information to stdout.
rows[in] a list of rows to process
new_db[in] new database name
destination[in] the destination server
"""
if self.dest_vals is None:
self.dest_vals = self.get_dest_values(destination)
# Spawn a new connection
server_options = {
'conn_info': self.dest_vals,
'role': "thread",
}
dest = Server(server_options)
dest.connect()
# Issue the write lock
lock_list = [("%s.%s" % (new_db, self.q_tbl_name), 'WRITE')]
my_lock = Lock(dest, lock_list, {'locking': 'lock-all', })
# First, turn off foreign keys if turned on
dest.disable_foreign_key_checks(True)
if self.column_format is None:
self.get_column_metadata()
data_lists = self.make_bulk_insert(rows, new_db)
insert_data = data_lists[0]
blob_data = data_lists[1]
# Insert the data first
for data_insert in insert_data:
try:
dest.exec_query(data_insert, self.query_options)
except UtilError, e:
raise UtilError("Problem inserting data. "
"Error = %s" % e.errmsg)
def get_copy_lock(server,
db_list,
options,
include_mysql=False,
cloning=False):
"""Get an instance of the Lock class with a standard copy (read) lock
This method creates an instance of the Lock class using the lock type
specified in the options. It is used to initiate the locks for the copy
and related operations.
server[in] Server instance for locking calls
db_list[in] list of database names
options[in] option dictionary
Must include the skip_* options for copy and export
include_mysql[in] if True, include the mysql tables for copy operation
cloning[in] if True, create lock tables with WRITE on dest db
Default = False
Returns Lock - Lock class instance
"""
rpl_mode = options.get("rpl_mode", None)
locking = options.get('locking', 'snapshot')
# Determine if we need to use FTWRL. There are two conditions:
# - running on master (rpl_mode = 'master')
# - using locking = 'lock-all' and rpl_mode present
if (rpl_mode in ["master", "both"]) or \
(rpl_mode and locking == 'lock-all'):
new_opts = options.copy()
new_opts['locking'] = 'flush'
lock = Lock(server, [], new_opts)
# if this is a lock-all type and not replication operation,
# find all tables and lock them
# pylint: disable=R0101
elif locking == 'lock-all':
table_lock_list = []
# Build table lock list
for db_name in db_list:
db = db_name[0] if isinstance(db_name, tuple) else db_name
source_db = Database(server, db)
tables = source_db.get_db_objects("TABLE")
for table in tables:
table_lock_list.append(("{0}.{1}".format(db,
table[0]), 'READ'))
# Cloning requires issuing WRITE locks because we use same
# conn.
# Non-cloning will issue WRITE lock on a new destination conn.
if cloning:
if db_name[1] is None:
db_clone = db_name[0]
else:
db_clone = db_name[1]
# For cloning, we use the same connection so we need to
# lock the destination tables with WRITE.
table_lock_list.append(
("{0}.{1}".format(db_clone, table[0]), 'WRITE'))
# We must include views for server version 5.5.3 and higher
if server.check_version_compat(5, 5, 3):
tables = source_db.get_db_objects("VIEW")
for table in tables:
table_lock_list.append(
("{0}.{1}".format(db, table[0]), 'READ'))
# Cloning requires issuing WRITE locks because we use same
# conn.
# Non-cloning will issue WRITE lock on a new destination
# conn.
if cloning:
if db_name[1] is None:
db_clone = db_name[0]
else:
db_clone = db_name[1]
# For cloning, we use the same connection so we need to
# lock the destination tables with WRITE.
table_lock_list.append(
("{0}.{1}".format(db_clone, table[0]), 'WRITE'))
# Now add mysql tables
if include_mysql:
# Don't lock proc tables if no procs of funcs are being read
if not options.get('skip_procs', False) and \
not options.get('skip_funcs', False):
table_lock_list.append(("mysql.proc", 'READ'))
table_lock_list.append(("mysql.procs_priv", 'READ'))
# Don't lock event table if events are skipped
if not options.get('skip_events', False):
table_lock_list.append(("mysql.event", 'READ'))
lock = Lock(server, table_lock_list, options)
# Use default or no locking option
else:
lock = Lock(server, [], options)
return lock
