def check_connect(port, name="cloned_server", conn_dict=None):
"""Check connection.
port[in] Server port.
name[in] Server name.
"""
conn = {"user": "******", "passwd": "root", "host": "localhost",
"port": port}
if conn_dict is not None:
conn.update(conn_dict)
server_options = {'conn_info': conn, 'role': name, }
new_server = Server(server_options)
if new_server is None:
return None
# Connect to the new instance
try:
new_server.connect()
except UtilError as err:
raise MUTLibError("Cannot connect to spawned server: {0}".format(
err.errmsg))
return new_server
def check_connect(self, port, full_datadir, name="cloned_server"):
new_server = None
# Create a new instance
conn = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": port,
"unix_socket": full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info': conn,
'role': name,
}
new_server = Server(server_options)
if new_server is None:
return None
# Connect to the new instance
try:
new_server.connect()
except UtilError, e:
raise MUTLibError("Cannot connect to spawned server.")
def check_connect(self, port, full_datadir, name="cloned_server"):
new_server = None
# Create a new instance
conn = {
"user" : "root",
"passwd" : "root",
"host" : "localhost",
"port" : port,
"unix_socket" : full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info' : conn,
'role' : name,
}
new_server = Server(server_options)
if new_server is None:
return None
# Connect to the new instance
try:
new_server.connect()
except UtilError, e:
raise MUTLibError("Cannot connect to spawned server.")
def check_connect(port, name="cloned_server", conn_dict=None):
"""Check connection.
port[in] Server port.
name[in] Server name.
"""
conn = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": port
}
if conn_dict is not None:
conn.update(conn_dict)
server_options = {
'conn_info': conn,
'role': name,
}
new_server = Server(server_options)
if new_server is None:
return None
# Connect to the new instance
try:
new_server.connect()
except UtilError as err:
raise MUTLibError("Cannot connect to spawned server: {0}".format(
err.errmsg))
return new_server
class test(mutlib.System_test):
"""clone server parameters
This test exercises the parameters for mysqlserverclone
"""
def check_prerequisites(self):
return self.check_num_servers(1)
def setup(self):
# No setup needed
self.new_server = None
return True
def _test_server_clone(self, cmd_str, comment, kill=True, capture_all=False):
self.results.append(comment+"\n")
port1 = int(self.servers.get_next_port())
cmd_str += " --new-port=%d " % port1
full_datadir = os.path.join(os.getcwd(), "temp_%s" % port1)
cmd_str += " --new-data=%s --delete " % full_datadir
res = self.exec_util(cmd_str, "start.txt")
for line in open("start.txt").readlines():
# Don't save lines that have [Warning] or don't start with #
index = line.find("[Warning]")
if capture_all or (index <= 0 and line[0] == '#'):
self.results.append(line)
if res:
raise MUTLibError("%s: failed" % comment)
# Create a new instance
conn = {
"user" : "root",
"passwd" : "root",
"host" : "localhost",
"port" : port1,
"unix_socket" : full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info' : conn,
'role' : "cloned_server_2",
}
self.new_server = Server(server_options)
if self.new_server is None:
return False
if kill:
# Connect to the new instance
try:
self.new_server.connect()
except UtilError, e:
self.new_server = None
raise MUTLibError("Cannot connect to spawned server.")
self.servers.stop_server(self.new_server)
self.servers.clear_last_port()
return True
def show_options(self):
""" Show all audit log variables.
"""
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
rows = server.show_server_variable("audit%")
server.disconnect()
if rows:
print "#\n# Audit Log Variables and Options\n#"
print_list(sys.stdout, "GRID", ['Variable_name', 'Value'], rows)
print
else:
raise UtilError("No audit log variables found.")
def _bulk_insert(self, rows, new_db, destination=None):
"""Import data using bulk insert
Reads data from a table and builds group INSERT statements for writing
to the destination server specified (new_db.name).
This method is designed to be used in a thread for parallel inserts.
As such, it requires its own connection to the destination server.
Note: This method does not print any information to stdout.
rows[in] a list of rows to process
new_db[in] new database name
destination[in] the destination server
"""
from mysql.utilities.common.lock import Lock
from mysql.utilities.common.server import Server
if self.dest_vals is None:
self.dest_vals = self.get_dest_values(destination)
# Spawn a new connection
server_options = {
'conn_info' : self.dest_vals,
'role' : "thread",
}
dest = Server(server_options)
dest.connect()
# Issue the write lock
lock_list = [("`%s`.`%s`" % (new_db, self.tbl_name), 'WRITE')]
my_lock = Lock(dest, lock_list, {'locking':'lock-all',})
# First, turn off foreign keys if turned on
dest.disable_foreign_key_checks(True)
if self.column_format is None:
self.get_column_metadata()
data_lists = self.make_bulk_insert(rows, new_db)
insert_data = data_lists[0]
blob_data = data_lists[1]
# Insert the data first
for data_insert in insert_data:
try:
res = dest.exec_query(data_insert, self.query_options)
except UtilError, e:
raise UtilError("Problem inserting data. "
"Error = %s" % e.errmsg)
def check_audit_log(self):
"""Verify if the audit log plugin is installed on the server.
Return the message error if not, or None.
"""
error = None
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
# Check to see if the plug-in is installed
if not server.supports_plugin("audit"):
error = "The audit log plug-in is not installed on this " + \
"server or is not enabled."
server.disconnect()
return error
def show_options(self):
""" Show all audit log variables.
"""
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
rows = server.show_server_variable("audit%")
server.disconnect()
if rows:
print "#\n# Audit Log Variables and Options\n#"
print_list(sys.stdout, "GRID", ['Variable_name', 'Value'],
rows)
print
else:
raise UtilError("No audit log variables found.")
def check_audit_log(self):
"""Verify if the audit log plugin is installed on the server.
Return the message error if not, or None.
"""
error = None
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
# Check to see if the plug-in is installed
if not server.supports_plugin("audit"):
error = "The audit log plug-in is not installed on this " + \
"server or is not enabled."
server.disconnect()
return error
def _bulk_insert(self, rows, new_db, destination=None):
"""Import data using bulk insert
Reads data from a table and builds group INSERT statements for writing
to the destination server specified (new_db.name).
This method is designed to be used in a thread for parallel inserts.
As such, it requires its own connection to the destination server.
Note: This method does not print any information to stdout.
rows[in] a list of rows to process
new_db[in] new database name
destination[in] the destination server
"""
if self.dest_vals is None:
self.dest_vals = self.get_dest_values(destination)
# Spawn a new connection
server_options = {
'conn_info': self.dest_vals,
'role': "thread",
}
dest = Server(server_options)
dest.connect()
# Issue the write lock
lock_list = [("%s.%s" % (new_db, self.q_tbl_name), 'WRITE')]
my_lock = Lock(dest, lock_list, {'locking': 'lock-all', })
# First, turn off foreign keys if turned on
dest.disable_foreign_key_checks(True)
if self.column_format is None:
self.get_column_metadata()
data_lists = self.make_bulk_insert(rows, new_db)
insert_data = data_lists[0]
blob_data = data_lists[1]
# Insert the data first
for data_insert in insert_data:
try:
dest.exec_query(data_insert, self.query_options)
except UtilError, e:
raise UtilError("Problem inserting data. "
"Error = %s" % e.errmsg)
def do_command(self):
""" Check and execute the audit log command (previously set by the the
options of the object constructor).
"""
# Check for valid command
command = self.options.get("command", None)
if not command in VALID_COMMANDS:
raise UtilError("Invalid command.")
command_value = self.options.get("value", None)
# Check for valid value if needed
if (command_requires_value(command)
and not check_command_value(command, command_value)):
raise UtilError("Please provide the correct value for the %s "
"command." % command)
# Copy command does not need the server
if command == "COPY":
self._copy_log()
return True
# Connect to server
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
# Now execute the command
print "#\n# Executing %s command.\n#\n" % command
try:
if command == "POLICY":
server.exec_query("SET @@GLOBAL.audit_log_policy = %s" %
command_value)
elif command == "ROTATE":
self._rotate_log(server)
else: # "ROTATE_ON_SIZE":
server.exec_query(
"SET @@GLOBAL.audit_log_rotate_on_size = %s" %
command_value)
finally:
server.disconnect()
return True
def do_command(self):
""" Check and execute the audit log command (previously set by the the
options of the object constructor).
"""
# Check for valid command
command = self.options.get("command", None)
if not command in VALID_COMMANDS:
raise UtilError("Invalid command.")
command_value = self.options.get("value", None)
# Check for valid value if needed
if (command_requires_value(command)
and not check_command_value(command, command_value)):
raise UtilError("Please provide the correct value for the %s "
"command." % command)
# Copy command does not need the server
if command == "COPY":
self._copy_log()
return True
# Connect to server
server = Server({'conn_info': self.options.get("server_vals", None)})
server.connect()
# Now execute the command
print "#\n# Executing %s command.\n#\n" % command
try:
if command == "POLICY":
server.exec_query("SET @@GLOBAL.audit_log_policy = %s" %
command_value)
elif command == "ROTATE":
self._rotate_log(server)
else: # "ROTATE_ON_SIZE":
server.exec_query("SET @@GLOBAL.audit_log_rotate_on_size = %s"
% command_value)
finally:
server.disconnect()
return True
def load_test_data(server, db_num=1):
"""Load/insert data into the test databases.
A considerable amount of data should be considered in order to take some
time to load, allowing mysqlrplsync to be executed at the same time the
data is still being inserted.
server[in] Target server to load the test data.
db_num[in] Number of databases to load the data (by default: 1).
It is assumed that a matching number of test databases
have been previously created.
Note: method prepared to be invoked by a different thread.
"""
# Create a new server instance with a new connection (for multithreading).
srv = Server({'conn_info': server})
srv.connect()
for db_index in xrange(db_num):
db_name = '`test_rplsync_db{0}`'.format(
'' if db_num == 1 else db_index
)
# Insert random data on all tables.
random_values = string.letters + string.digits
for _ in xrange(TEST_DB_NUM_ROWS):
columns = []
values = []
for table_index in xrange(TEST_DB_NUM_TABLES):
columns.append('rnd_txt{0}'.format(table_index))
rnd_text = "".join(
[random.choice(random_values) for _ in xrange(20)]
)
values.append("'{0}'".format(rnd_text))
insert = ("INSERT INTO {0}.`t{1}` ({2}) VALUES ({3})"
"").format(db_name, table_index, ', '.join(columns),
', '.join(values))
srv.exec_query(insert)
srv.commit()
def _start_server(server_val, basedir, datadir, options={}):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
from mysql.utilities.common.tools import get_tool_path, get_mysqld_version
from mysql.utilities.common.server import Server
import time
verbosity = options.get("verbosity", 0)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
" -uroot",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
proc = subprocess.Popen(args, executable=mysqld_path)
else:
out = open(os.devnull, 'w')
proc = subprocess.Popen(args, executable=mysqld_path,
stdout=out, stderr=out)
server_options = {
'conn_info' : server_val,
'role' : "read_only",
}
server = Server(server_options)
# Now wait for the server to become ready - could be up to 10 seconds
# for Windows machines.
if os.name == "nt":
time.sleep(10)
else:
time.sleep(1)
server.connect()
print "done."
return server
def _shutdown_running_servers(server_list, processes, basedir):
"""Shutdown any running servers.
processes[in] The list of processes to shutdown with the form:
(process_id, [datadir|port])
basedir[in] The starting path to search for mysqladmin tool
Returns bool - True - servers shutdown attempted
False - no servers to shutdown
"""
if len(processes) < 1:
return False
for process in processes:
datadir = os.getcwd()
connection = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": None,
}
if os.name == "posix":
try:
connection["port"] = get_port(process[0])
except MUTLibError:
connection["port"] = -1 # try to kill it forcefully later
elif os.name == "nt":
connection["port"] = process[1]
if os.name == "posix":
print(" Process id: {0:>6}, Data path: {1}".format(process[0],
process[1]))
elif os.name == "nt":
print(" Process id: {0:>6}, Port: {1}".format(process[0],
process[1]))
# 1) connect to the server.
server_options = {
'conn_info': connection,
}
svr = Server(server_options)
ok_to_shutdown = True
try:
svr.connect()
except UtilError: # if we cannot connect, don't try to shut it down.
ok_to_shutdown = False
print(" WARNING: shutdown failed - cannot connect.")
if not ok_to_shutdown:
if os.name == "posix":
# Attempt kill
ret_code = subprocess.call(["kill", "-9",
" {0}".format(process[0])])
# wait until connection is closed
for _ in range(CONNECTION_CLOSE_WAIT_TIMEOUT):
try:
sock = socket.create_connection((svr.host, svr.port))
sock.close()
time.sleep(1)
except socket.error:
# could not connect, connection is closed
break
else:
print(" WARNING: timeout waiting for connection to "
"process {0} to close".format(process[0]))
if ret_code:
print(" WARNING: shutdown failed: Killing process {0}"
"returned error code {1}".format(process[0],
ret_code))
else:
ret_code = subprocess.call("taskkill /F /T /PID {0}".format(
process[0]))
if ret_code not in (0, 128):
print(" WARNING: shutdown failed: Killing process {0} "
"returned error code {1}".format(process[0],
ret_code))
# 2) if nt, verify datadirectory
if os.name == "nt" and ok_to_shutdown:
res = svr.show_server_variable("datadir")
server_datadir = res[0][1]
ok_to_shutdown = (server_datadir.find(datadir) >= 0)
# 3) call shutdown method from mutlib Server_list class
if ok_to_shutdown and svr:
try:
server_list.stop_server(svr)
except MUTLibError:
_, e, _ = sys.exc_info()
print(" WARNING: shutdown failed: " + e.errmsg)
return True
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
force = options.get('force', False)
quote_char = "'" if os.name == "posix" else '"'
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Check if path to database files is greater than MAX_DIR_SIZE char,
if len(new_data) > MAX_DATADIR_SIZE and not force:
raise UtilError("The --new-data path '{0}' is too long "
"(> {1} characters). Please use a smaller one. "
"You can use the --force option to skip this "
"check".format(new_data, MAX_DATADIR_SIZE))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
new_data_deleted = False
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
new_data_deleted = True
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except OSError as err:
raise UtilError("Unable to create directory '{0}', reason: {1}"
"".format(new_data, err.strerror))
# After create the new data directory, check for free space, so the errors
# regarding invalid or inaccessible path had been dismissed already.
# If not force specified verify and stop if there is not enough free space
if not force and os.path.exists(new_data) and \
estimate_free_space(new_data) < REQ_FREE_SPACE:
# Don't leave empty folders, delete new_data if was previously deleted
if os.path.exists(new_data) and new_data_deleted:
shutil.rmtree(new_data, True)
raise UtilError(
LOW_SPACE_ERRR_MSG.format(directory=new_data,
megabytes=REQ_FREE_SPACE))
# Check for warning of using --skip-innodb
mysqld_path = get_tool_path(basedir, "mysqld")
version_str = get_mysqld_version(mysqld_path)
# convert version_str from str tuple to integer tuple if possible
if version_str is not None:
version = tuple([int(digit) for digit in version_str])
else:
version = None
if mysqld_options is not None and ("--skip-innodb" in mysqld_options or
"--innodb" in mysqld_options) and version is not None and \
version >= (5, 7, 5):
print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB))
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
]
# From 5.7.6 version onwards, bootstrap is done via mysqld with the
# --initialize-insecure option, so no need to get information about the
# sql system tables that need to be loaded.
if version < (5, 7, 6):
system_tables = get_tool_path(basedir, "mysql_system_tables.sql",
False)
system_tables_data = get_tool_path(basedir,
"mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir,
"mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
locations.extend([
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
])
if verbosity >= 3 and not quiet:
print "# Location of files:"
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
fnull = open(os.devnull, 'w')
# For MySQL versions before 5.7.6, use regular bootstrap procedure.
if version < (5, 7, 6):
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [
system_tables, system_tables_data, test_data_timezone,
help_data
]:
lines = open(sqlfile, 'r').readlines()
# On MySQL 5.7.5, the root@localhost account creation was
# moved from the system_tables_data sql file into the
# mysql_install_db binary. Since we don't use mysql_install_db
# directly we need to create the root user account ourselves.
if (version is not None and version == (5, 7, 5)
and sqlfile == system_tables_data):
lines.extend(_CREATE_ROOT_USER)
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955)
# (Ugly hack)
if (sqlfile == system_tables
and "SET @sql_mode_orig==@@SES" in line
and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
proc.communicate('\n'.join(sql))
# From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and
# the --initialize option
else:
cmd = [
mysqld_path, "--initialize-insecure=on",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir))
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid),
shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid, ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
socket_path = os.path.join(new_data, 'mysql.sock')
# If socket path is too long, use mkdtemp to create a tmp dir and
# use it instead to store the socket
if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE:
socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock')
if not quiet:
print(
"# WARNING: The socket file path '{0}' is too long (>{1}), "
"using '{2}' instead".format(
os.path.join(new_data, 'mysql.sock'), MAX_SOCKET_PATH_SIZE,
socket_path))
cmd = {
'datadir': '--datadir={0}'.format(new_data),
'tmpdir': '--tmpdir={0}'.format(new_data),
'pid-file':
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'port': '--port={0}'.format(new_port),
'server': '--server-id={0}'.format(options.get('new_id', 2)),
'basedir': '--basedir={0}'.format(mysql_basedir),
'socket': '--socket={0}'.format(socket_path),
}
if user:
cmd.update({'user': '******'.format(user)})
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.update(dict(zip(mysqld_options, mysqld_options)))
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
list_ = shlex.split(new_opts.strip('"'))
cmd.update(dict(zip(list_, list_)))
elif new_opts.startswith("'") and new_opts.endswith("'"):
list_ = shlex.split(new_opts.strip("'"))
cmd.update(dict(zip(list_, list_)))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.update({mysqld_options: mysqld_options})
else:
list_ = shlex.split(new_opts)
cmd.update(dict(zip(list_, list_)))
# set of options that must be surrounded with quotes
options_to_quote = set(
["datadir", "tmpdir", "basedir", "socket", "pid-file"])
# Strip spaces from each option
for key in cmd:
cmd[key] = cmd[key].strip(' ')
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
comment = " Startup command generated by mysqlserverclone.\n"
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("#{0}".format(comment))
elif os.name == 'nt' and cmd_file.endswith('.bat'):
cfile.write("REM{0}".format(comment))
else:
cfile.write("#{0}".format(comment))
start_cmd_lst = [
"{0}{1}{0} --no-defaults".format(quote_char, mysqld_path)
]
# build start command
for key, val in cmd.iteritems():
if key in options_to_quote:
val = "{0}{1}{0}".format(quote_char, val)
start_cmd_lst.append(val)
cfile.write("{0}\n".format(" ".join(start_cmd_lst)))
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.update({"console": "--console"})
start_cmd_lst = [mysqld_path, "--no-defaults"]
sorted_keys = sorted(cmd.keys())
start_cmd_lst.extend([cmd[val] for val in sorted_keys])
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(start_cmd_lst)))
proc = subprocess.Popen(start_cmd_lst, shell=False)
else:
proc = subprocess.Popen(start_cmd_lst,
shell=False,
stdout=fnull,
stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
if os.name == "posix":
new_sock = socket_path
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def _spawn_server(options):
"""Spawn a server to use for reading .frm files
This method spawns a new server instance on the port specified by the
user in the options dictionary.
options[in] Options from user
Returns tuple - (Server instance, new datdir) or raises exception on error
"""
verbosity = int(options.get("verbosity", 0))
quiet = options.get("quiet", False)
new_port = options.get("port", 3310)
user = options.get("user", None)
start_timeout = int(options.get("start_timeout", 10))
# 1) create a directory to use for new datadir
# If the user is not the same as the user running the script...
if user_change_as_root(options):
# Since Python libraries correctly restrict temporary folders to
# the user who runs the script and /tmp is protected on some
# platforms, we must create the folder in the current folder
temp_datadir = os.path.join(os.getcwd(), str(uuid.uuid4()))
os.mkdir(temp_datadir)
else:
temp_datadir = tempfile.mkdtemp()
if verbosity > 1 and not quiet:
print "# Creating a temporary datadir =", temp_datadir
# 2) spawn a server pointed to temp
if not quiet:
if user:
print("# Spawning server with --user={0}.".format(user))
print "# Starting the spawned server on port %s ..." % new_port,
sys.stdout.flush()
bootstrap_options = {
'new_data': temp_datadir,
'new_port': new_port,
'new_id': 101,
'root_pass': "******",
'mysqld_options': None,
'verbosity': verbosity if verbosity > 1 else 0,
'basedir': options.get("basedir"),
'delete': True,
'quiet': True if verbosity <= 1 else False,
'user': user,
'start_timeout': start_timeout,
}
if verbosity > 1 and not quiet:
print
try:
serverclone.clone_server(None, bootstrap_options)
except UtilError as error:
if error.errmsg.startswith("Unable to communicate"):
err = ". Clone server error: {0}".format(error.errmsg)
proc_id = int(error.errmsg.split("=")[1].strip('.'))
print("ERROR Attempting to stop failed spawned server. "
" Process id = {0}.".format(proc_id))
if os.name == "posix":
try:
os.kill(proc_id, subprocess.signal.SIGTERM)
except OSError:
pass
else:
try:
subprocess.Popen("taskkill /F /T /PID %i" %
proc_id, shell=True)
except:
pass
raise UtilError(_SPAWN_SERVER_ERROR.format(err))
else:
raise
if verbosity > 1 and not quiet:
print "# Connecting to spawned server"
conn = {
"user": "******",
"passwd": "root",
"host": "127.0.0.1",
"port": options.get("port"),
}
server_options = {
'conn_info': conn,
'role': "frm_reader_bootstrap",
}
server = Server(server_options)
try:
server.connect()
except UtilError:
raise UtilError(_SPAWN_SERVER_ERROR.format(""))
if not quiet:
print "done."
return (server, temp_datadir)
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
force = options.get('force', False)
quote_char = "'" if os.name == "posix" else '"'
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Check if path to database files is greater than MAX_DIR_SIZE char,
if len(new_data) > MAX_DATADIR_SIZE and not force:
raise UtilError("The --new-data path '{0}' is too long "
"(> {1} characters). Please use a smaller one. "
"You can use the --force option to skip this "
"check".format(new_data, MAX_DATADIR_SIZE))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
new_data_deleted = False
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
new_data_deleted = True
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except OSError as err:
raise UtilError("Unable to create directory '{0}', reason: {1}"
"".format(new_data, err.strerror))
# After create the new data directory, check for free space, so the errors
# regarding invalid or inaccessible path had been dismissed already.
# If not force specified verify and stop if there is not enough free space
if not force and os.path.exists(new_data) and \
estimate_free_space(new_data) < REQ_FREE_SPACE:
# Don't leave empty folders, delete new_data if was previously deleted
if os.path.exists(new_data) and new_data_deleted:
shutil.rmtree(new_data, True)
raise UtilError(LOW_SPACE_ERRR_MSG.format(directory=new_data,
megabytes=REQ_FREE_SPACE))
# Check for warning of using --skip-innodb
mysqld_path = get_tool_path(basedir, "mysqld")
version_str = get_mysqld_version(mysqld_path)
# convert version_str from str tuple to integer tuple if possible
if version_str is not None:
version = tuple([int(digit) for digit in version_str])
else:
version = None
if mysqld_options is not None and ("--skip-innodb" in mysqld_options or
"--innodb" in mysqld_options) and version is not None and \
version >= (5, 7, 5):
print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB))
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
]
# From 5.7.6 version onwards, bootstrap is done via mysqld with the
# --initialize-insecure option, so no need to get information about the
# sql system tables that need to be loaded.
if version < (5, 7, 6):
system_tables = get_tool_path(basedir, "mysql_system_tables.sql",
False)
system_tables_data = get_tool_path(basedir,
"mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir,
"mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
locations.extend([("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
])
if verbosity >= 3 and not quiet:
print "# Location of files:"
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
fnull = open(os.devnull, 'w')
# For MySQL versions before 5.7.6, use regular bootstrap procedure.
if version < (5, 7, 6):
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [system_tables, system_tables_data, test_data_timezone,
help_data]:
lines = open(sqlfile, 'r').readlines()
# On MySQL 5.7.5, the root@localhost account creation was
# moved from the system_tables_data sql file into the
# mysql_install_db binary. Since we don't use mysql_install_db
# directly we need to create the root user account ourselves.
if (version is not None and version == (5, 7, 5) and
sqlfile == system_tables_data):
lines.extend(_CREATE_ROOT_USER)
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955)
# (Ugly hack)
if (sqlfile == system_tables and
"SET @sql_mode_orig==@@SES" in line and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
proc.communicate('\n'.join(sql))
# From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and
# the --initialize option
else:
cmd = [
mysqld_path,
"--initialize-insecure=on",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir))
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid), shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid,
ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
socket_path = os.path.join(new_data, 'mysql.sock')
# If socket path is too long, use mkdtemp to create a tmp dir and
# use it instead to store the socket
if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE:
socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock')
if not quiet:
print("# WARNING: The socket file path '{0}' is too long (>{1}), "
"using '{2}' instead".format(
os.path.join(new_data, 'mysql.sock'),
MAX_SOCKET_PATH_SIZE, socket_path))
cmd = {
'datadir': '--datadir={0}'.format(new_data),
'tmpdir': '--tmpdir={0}'.format(new_data),
'pid-file': '--pid-file={0}'.format(
os.path.join(new_data, "clone.pid")),
'port': '--port={0}'.format(new_port),
'server': '--server-id={0}'.format(options.get('new_id', 2)),
'basedir': '--basedir={0}'.format(mysql_basedir),
'socket': '--socket={0}'.format(socket_path),
}
if user:
cmd.update({'user': '******'.format(user)})
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.update(dict(zip(mysqld_options, mysqld_options)))
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
list_ = shlex.split(new_opts.strip('"'))
cmd.update(dict(zip(list_, list_)))
elif new_opts.startswith("'") and new_opts.endswith("'"):
list_ = shlex.split(new_opts.strip("'"))
cmd.update(dict(zip(list_, list_)))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.update({mysqld_options: mysqld_options})
else:
list_ = shlex.split(new_opts)
cmd.update(dict(zip(list_, list_)))
# set of options that must be surrounded with quotes
options_to_quote = set(["datadir", "tmpdir", "basedir", "socket",
"pid-file"])
# Strip spaces from each option
for key in cmd:
cmd[key] = cmd[key].strip(' ')
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
comment = " Startup command generated by mysqlserverclone.\n"
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("#{0}".format(comment))
elif os.name == 'nt' and cmd_file.endswith('.bat'):
cfile.write("REM{0}".format(comment))
else:
cfile.write("#{0}".format(comment))
start_cmd_lst = ["{0}{1}{0} --no-defaults".format(quote_char,
mysqld_path)]
# build start command
for key, val in cmd.iteritems():
if key in options_to_quote:
val = "{0}{1}{0}".format(quote_char, val)
start_cmd_lst.append(val)
cfile.write("{0}\n".format(" ".join(start_cmd_lst)))
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.update({"console": "--console"})
start_cmd_lst = [mysqld_path, "--no-defaults"]
sorted_keys = sorted(cmd.keys())
start_cmd_lst.extend([cmd[val] for val in sorted_keys])
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(start_cmd_lst)))
proc = subprocess.Popen(start_cmd_lst, shell=False)
else:
proc = subprocess.Popen(start_cmd_lst, shell=False, stdout=fnull,
stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
if os.name == "posix":
new_sock = socket_path
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
# Parse source connection values
try:
conn = parse_connection(opt.server)
except:
parser.error("Server connection values invalid or cannot be parsed.")
# Get a server class instance
print "# Connecting to server..."
server_options = {
'conn_info' : conn,
'role' : "source",
}
server1 = Server(server_options)
try:
server1.connect()
except UtilError, e:
print "ERROR:", e.errmsg
# Get list of databases from the server if not specified in options
print "# Getting databases..."
db_list = []
if opt.dbs_to_copy is None:
for db in server1.get_all_databases():
db_list.append((db[0], None))
else:
for db in opt.dbs_to_copy.split(","):
db_list.append((db, None))
# Get list of all users from the server
print "# Getting users..."
def move_binlogs_from_server(server_cnx_val, destination, options,
bin_basename=None, bin_index=None,
relay_basename=None):
"""Relocate binary logs from the given server to a new location.
This function relocate the binary logs from a MySQL server to the specified
destination directory, attending to the specified options.
server_cnx_val[in] Dictionary with the connection values for the server.
destination[in] Path of the destination directory for the binary log
files.
options[in] Dictionary of options (log_type, skip_flush_binlogs,
modified_before, sequence, verbosity).
bin_basename[in] Base name for the binlog files, i.e., same as the
value for the server option --log-bin. It replaces
the server variable 'log_bin_basename' for versions
< 5.6.2, otherwise it is ignored.
bin_index[in] Path of the binlog index file. It replaces the server
variable 'log_bin_index' for versions < 5.6.4,
otherwise it is ignored.
relay_basename[in] Base name for the relay log files, i.e., filename
without the extension (sequence number). Same as the
value for the server option --relay-log. It replaces
the server variable 'relay_log_basename' for versions
< 5.6.2, otherwise it is ignored.
"""
log_type = options.get('log_type', LOG_TYPE_BIN)
skip_flush = options['skip_flush_binlogs']
verbosity = options['verbosity']
# Connect to server
server_options = {
'conn_info': server_cnx_val,
}
srv = Server(server_options)
srv.connect()
# Check if the server is running locally (not remote server).
if not srv.is_alias('localhost'):
raise UtilError("You are using a remote server. This utility must be "
"run on the local server. It does not support remote "
"access to the binary log files.")
# Check privileges.
_check_privileges_to_move_binlogs(srv, options)
# Process binlog files.
if log_type in (LOG_TYPE_BIN, LOG_TYPE_ALL):
# Get log_bin_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if bin_basename:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin basename', min_version='5.6.2',
var_name='log_bin_basename'))
binlog_basename = srv.select_variable('log_bin_basename')
if verbosity > 0:
print("#")
print("# log_bin_basename: {0}".format(binlog_basename))
binlog_source, binlog_file = os.path.split(binlog_basename)
# Get log_bin_index (available since MySQL 5.6.4).
if srv.check_version_compat(5, 6, 4):
if bin_index:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin index', min_version='5.6.4',
var_name='log_bin_index'))
binlog_index = srv.select_variable('log_bin_index')
else:
binlog_index = None
action = _ACTION_SEARCH_INDEX.format(file_type='bin-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename', host=srv.host, port=srv.port,
min_version='5.6.4', action=action))
if verbosity > 0:
print("# log_bin_index: {0}".format(binlog_index))
else:
if bin_basename:
binlog_source, binlog_file = os.path.split(bin_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='bin-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename', host=srv.host, port=srv.port,
min_version='5.6.2', action=action))
# Get datadir value.
binlog_source = srv.select_variable('datadir')
binlog_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(binlog_source))
binlog_index = bin_index
# Move binlog files.
num_files = _move_binlogs(
binlog_source, destination, LOG_TYPE_BIN, options,
basename=binlog_file, index_file=binlog_index, skip_latest=True)
print("#")
# Flush binary logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='binary'))
srv.flush_logs(log_type='BINARY')
else:
print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='binary',
host=srv.host,
port=srv.port))
print("#")
if log_type in (LOG_TYPE_RELAY, LOG_TYPE_ALL):
# Get relay_log_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if relay_basename:
print(_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='relay basename', min_version='5.6.2',
var_name='relay_log_basename'))
relay_log_basename = srv.select_variable('relay_log_basename')
if verbosity > 0:
print("#")
print("# relay_log_basename: {0}".format(relay_log_basename))
relay_source, relay_file = os.path.split(relay_log_basename)
else:
if relay_basename:
relay_source, relay_file = os.path.split(relay_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='relay-log')
print(_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='relay_log_basename', host=srv.host,
port=srv.port, min_version='5.6.2', action=action))
# Get datadir value.
relay_source = srv.select_variable('datadir')
relay_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(relay_source))
# Get relay_log_index (available for all supported versions).
relay_log_index = srv.select_variable('relay_log_index')
if verbosity > 0:
print("# relay_log_index: {0}".format(relay_log_index))
# Move relay log files.
num_files = _move_binlogs(
relay_source, destination, LOG_TYPE_RELAY, options,
basename=relay_file, index_file=relay_log_index, skip_latest=True)
print("#")
# Flush relay logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='relay'))
srv.flush_logs(log_type='RELAY')
else:
print(_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='relay',
host=srv.host,
port=srv.port))
print("#")
print("#...done.\n#")
def _start_server(server_val, basedir, datadir, options={}):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
from mysql.utilities.common.tools import get_tool_path, get_mysqld_version
from mysql.utilities.common.server import Server
import time
verbosity = options.get("verbosity", 0)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
" -uroot",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
proc = subprocess.Popen(args, executable=mysqld_path)
else:
out = open(os.devnull, 'w')
proc = subprocess.Popen(args, executable=mysqld_path,
stdout=out, stderr=out)
server_options = {
'conn_info' : server_val,
'role' : "read_only",
}
server = Server(server_options)
# Now wait for the server to become ready - could be up to 10 seconds
# for Windows machines.
if os.name == "nt":
time.sleep(10)
else:
time.sleep(1)
server.connect()
print "done."
def _shutdown_running_servers(server_list, processes, basedir):
"""Shutdown any running servers.
processes[in] The list of processes to shutdown with the form:
(process_id, [datadir|port])
basedir[in] The starting path to search for mysqladmin tool
Returns bool - True - servers shutdown attempted
False - no servers to shutdown
"""
if len(processes) < 1:
return False
for process in processes:
datadir = os.getcwd()
connection = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": None,
}
if os.name == "posix":
try:
connection["port"] = get_port(process[0])
except MUTLibError:
connection["port"] = -1 # try to kill it forcefully later
elif os.name == "nt":
connection["port"] = process[1]
if os.name == "posix":
print(" Process id: {0:>6}, Data path: {1}".format(process[0],
process[1]))
elif os.name == "nt":
print(" Process id: {0:>6}, Port: {1}".format(process[0],
process[1]))
# 1) connect to the server.
server_options = {
'conn_info': connection,
}
svr = Server(server_options)
ok_to_shutdown = True
try:
svr.connect()
except UtilError: # if we cannot connect, don't try to shut it down.
ok_to_shutdown = False
print(" WARNING: shutdown failed - cannot connect.")
if not ok_to_shutdown:
if os.name == "posix":
# Attempt kill
ret_code = subprocess.call(["kill", "-9",
" {0}".format(process[0])])
# wait until connection is closed
for _ in range(CONNECTION_CLOSE_WAIT_TIMEOUT):
try:
sock = socket.create_connection((svr.host, svr.port))
sock.close()
time.sleep(1)
except socket.error:
# could not connect, connection is closed
break
else:
print(" WARNING: timeout waiting for connection to "
"process {0} to close".format(process[0]))
if ret_code:
print(" WARNING: shutdown failed: Killing process {0}"
"returned error code {1}".format(process[0],
ret_code))
else:
ret_code = subprocess.call("taskkill /F /T /PID {0}".format(
process[0]))
if ret_code not in (0, 128):
print(" WARNING: shutdown failed: Killing process {0} "
"returned error code {1}".format(process[0],
ret_code))
# 2) if nt, verify datadirectory
if os.name == "nt" and ok_to_shutdown:
res = svr.show_server_variable("datadir")
server_datadir = res[0][1]
ok_to_shutdown = (server_datadir.find(datadir) >= 0)
# 3) call shutdown method from mutlib Server_list class
if ok_to_shutdown and svr:
try:
server_list.stop_server(svr)
except MUTLibError:
_, e, _ = sys.exc_info()
print(" WARNING: shutdown failed: " + e.errmsg)
return True
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
"""
from mysql.utilities.common.server import Server
from mysql.utilities.exception import UtilError
from mysql.utilities.common.tools import get_tool_path
new_data = options.get("new_data", None)
new_port = options.get("new_port", "3307")
root_pass = options.get("root_pass", None)
verbosity = options.get("verbosity", 0)
quiet = options.get("quiet", False)
cmd_file = options.get("cmd_file", None)
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {"conn_info": conn_val, "role": "source"}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = rows[0][1]
# Cloning downed or offline server
else:
basedir = options.get("basedir", None)
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, delete it
if os.path.exists(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not quiet:
print "# Creating new data directory..."
if not os.path.exists(new_data):
try:
res = os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Create the bootstrap file
f_boot = open("bootstrap.sql", "w")
f_boot.write("CREATE DATABASE mysql;\n")
f_boot.write("USE mysql;\n")
f_boot.writelines(open(system_tables).readlines())
f_boot.writelines(open(system_tables_data).readlines())
f_boot.writelines(open(test_data_timezone).readlines())
f_boot.writelines(open(help_data).readlines())
f_boot.close()
# Bootstap to setup mysql tables
fnull = open(os.devnull, "w")
cmd = (
mysqld_path
+ " --no-defaults --bootstrap "
+ " --datadir=%s --basedir=%s " % (new_data, mysql_basedir)
+ " < bootstrap.sql"
)
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
cmd = mysqld_path + " --no-defaults "
if options.get("mysqld_options", None):
cmd += options.get("mysqld_options") + " --user=root "
cmd += "--datadir=%s " % (new_data)
cmd += "--tmpdir=%s " % (new_data)
cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid")
cmd += "--port=%s " % (new_port)
cmd += "--server-id=%s " % (options.get("new_id", 2))
cmd += "--basedir=%s " % (mysql_basedir)
cmd += "--socket=%s/mysql.sock " % (new_data)
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, "w")
if os.name == "posix" and cmd_file.endswith(".sh"):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print "# Startup command for new server:\n%s" % cmd
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock,
}
server2_options = {"conn_info": conn, "role": "clone"}
server2 = Server(server2_options)
stop = 10 # stop after 10 attempts
i = 0
while i < stop:
i += 1
time.sleep(1)
try:
server2.connect()
i = stop + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == stop:
raise UtilError("Unable to communicate with new instance.")
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
if os.name == "posix":
cmd = mysqladmin_path + " --no-defaults -v -uroot " + "--socket=%s password %s " % (new_sock, root_pass)
else:
cmd = mysqladmin_path + " --no-defaults -v -uroot " + "password %s --port=%s" % (root_pass, int(new_port))
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys",
False, False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [system_tables, system_tables_data, test_data_timezone,
help_data]:
lines = open(sqlfile, 'r').readlines()
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack)
if (sqlfile == system_tables and
"SET @sql_mode_orig==@@SES" in line and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
proc.communicate('\n'.join(sql))
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid), shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid,
ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
cmd = [mysqld_path, '--no-defaults']
cmd.extend([
'--datadir={0}'.format(new_data),
'--tmpdir={0}'.format(new_data),
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'--port={0}'.format(new_port),
'--server-id={0}'.format(options.get('new_id', 2)),
'--basedir={0}'.format(mysql_basedir),
'--socket={0}'.format(os.path.join(new_data, 'mysql.sock')),
])
if user:
cmd.append('--user={0}'.format(user))
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.extend(mysqld_options)
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
cmd.extend(shlex.split(new_opts.strip('"')))
elif new_opts.startswith("'") and new_opts.endswith("'"):
cmd.extend(shlex.split(new_opts.strip("'")))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.append(mysqld_options)
else:
cmd.extend(shlex.split(new_opts))
# Strip spaces from each option
cmd = [opt.strip(' ') for opt in cmd]
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.append("--console")
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
"""
from mysql.utilities.common.server import Server
from mysql.utilities.exception import UtilError
from mysql.utilities.common.tools import get_tool_path
new_data = options.get('new_data', None)
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = rows[0][1]
# Cloning downed or offline server
else:
basedir = options.get("basedir", None)
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
res = os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False,
False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Create the bootstrap file
f_boot = open("bootstrap.sql", 'w')
f_boot.write("CREATE DATABASE mysql;\n")
f_boot.write("USE mysql;\n")
f_boot.writelines(open(system_tables).readlines())
f_boot.writelines(open(system_tables_data).readlines())
f_boot.writelines(open(test_data_timezone).readlines())
f_boot.writelines(open(help_data).readlines())
f_boot.close()
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = mysqld_path + " --no-defaults --bootstrap " + \
" --datadir=%s --basedir=%s " % (new_data, mysql_basedir) + \
" < bootstrap.sql"
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
pass
else:
try:
retval = subprocess.Popen("taskkill /F /T /PID %i" % proc.pid,
shell=True)
except:
pass
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
cmd = mysqld_path + " --no-defaults "
if options.get('mysqld_options', None):
cmd += options.get('mysqld_options') + " --user=root "
cmd += "--datadir=%s " % (new_data)
cmd += "--tmpdir=%s " % (new_data)
cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid")
cmd += "--port=%s " % (new_port)
cmd += "--server-id=%s " % (options.get('new_id', 2))
cmd += "--basedir=%s " % (mysql_basedir)
cmd += "--socket=%s/mysql.sock " % (new_data)
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print "# Startup command for new server:\n%s" % cmd
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
stop = 10 # stop after 10 attempts
i = 0
while i < stop:
i += 1
time.sleep(1)
try:
server2.connect()
i = stop + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == stop:
raise UtilError("Unable to communicate with new instance.")
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
if os.name == "posix":
cmd = mysqladmin_path + " --no-defaults -v -uroot " + \
"--socket=%s password %s " % (new_sock, root_pass)
else:
cmd = mysqladmin_path + " --no-defaults -v -uroot " + \
"password %s --port=%s" % (root_pass, int(new_port))
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd,
shell=True,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
class BinaryLogRotate(object):
"""The BinaryLogRotate Class, it represents a binary log rotation task.
The rotate method performs the following tasks:
- Retrieves the active binary log and file size.
- If the minimum size is given, evaluate active binlog file size, and
if this is greater than the minimum size rotation will occur.
rotation occurs.
"""
def __init__(self, server_cnx_val, options):
"""Initiator.
server_cnx_val[in] Dictionary with the connection values for the
server.
options[in] options for controlling behavior:
logging If logging is active or not.
verbose print extra data during operations (optional)
default value = False
min_size minimum size that the active binlog must have prior
to rotate it.
dry_run Don't actually rotate the active binlog, instead
it will print information about file name and size.
"""
# Connect to server
self.server = Server({'conn_info': server_cnx_val})
self.server.connect()
self.options = options
self.verbosity = self.options.get("verbosity", 0)
self.quiet = self.options.get("quiet", False)
self.logging = self.options.get("logging", False)
self.dry_run = self.options.get("dry_run", 0)
self.binlog_min_size = self.options.get("min_size", False)
def _report(self, message, level=logging.INFO, print_msg=True):
"""Log message if logging is on.
This method will log the message presented if the log is turned on.
Specifically, if options['log_file'] is not None. It will also
print the message to stdout.
message[in] Message to be printed.
level[in] Level of message to log. Default = INFO.
print_msg[in] If True, print the message to stdout. Default = True.
"""
# First, print the message.
if print_msg and not self.quiet:
print(message)
# Now log message if logging turned on
if self.logging:
msg = message.strip("#").strip(" ")
logging.log(int(level), msg)
def rotate(self):
"""This Method runs the rotation.
This method will use the methods from the common library to rotate the
binary log.
"""
# Check required privileges
check_privileges(self.server, BINLOG_OP_ROTATE,
["RELOAD", "REPLICATION CLIENT"],
BINLOG_OP_ROTATE_DESC, self.verbosity, self._report)
active_binlog, binlog_size = get_active_binlog_and_size(self.server)
if self.verbosity:
self._report("# Active binlog file: '{0}' (size: {1} bytes)'"
"".format(active_binlog, binlog_size))
if self.binlog_min_size:
rotated = rotate(self.server, self.binlog_min_size,
reporter=self._report)
else:
rotated = rotate(self.server, reporter=self._report)
if rotated:
new_active_binlog, _ = get_active_binlog_and_size(self.server)
if active_binlog == new_active_binlog:
raise UtilError("Unable to rotate binlog file.")
else:
self._report("# The binlog file has been rotated.")
if self.verbosity:
self._report("# New active binlog file: '{0}'"
"".format(new_active_binlog))
# Parse source connection values
try:
conn = parse_connection(opt.server)
except:
parser.error("Server connection values invalid or cannot be parsed.")
# Get a server class instance
print "# Connecting to server..."
server_options = {
'conn_info': conn,
'role': "source",
}
server1 = Server(server_options)
try:
server1.connect()
except UtilError, e:
print "ERROR:", e.errmsg
# Get list of databases from the server if not specified in options
print "# Getting databases..."
db_list = []
if opt.dbs_to_copy is None:
for db in server1.get_all_databases():
db_list.append((db[0], None))
else:
for db in opt.dbs_to_copy.split(","):
db_list.append((db, None))
# Get list of all users from the server
print "# Getting users..."
def move_binlogs_from_server(server_cnx_val,
destination,
options,
bin_basename=None,
bin_index=None,
relay_basename=None):
"""Relocate binary logs from the given server to a new location.
This function relocate the binary logs from a MySQL server to the specified
destination directory, attending to the specified options.
server_cnx_val[in] Dictionary with the connection values for the server.
destination[in] Path of the destination directory for the binary log
files.
options[in] Dictionary of options (log_type, skip_flush_binlogs,
modified_before, sequence, verbosity).
bin_basename[in] Base name for the binlog files, i.e., same as the
value for the server option --log-bin. It replaces
the server variable 'log_bin_basename' for versions
< 5.6.2, otherwise it is ignored.
bin_index[in] Path of the binlog index file. It replaces the server
variable 'log_bin_index' for versions < 5.6.4,
otherwise it is ignored.
relay_basename[in] Base name for the relay log files, i.e., filename
without the extension (sequence number). Same as the
value for the server option --relay-log. It replaces
the server variable 'relay_log_basename' for versions
< 5.6.2, otherwise it is ignored.
"""
log_type = options.get('log_type', LOG_TYPE_BIN)
skip_flush = options['skip_flush_binlogs']
verbosity = options['verbosity']
# Connect to server
server_options = {
'conn_info': server_cnx_val,
}
srv = Server(server_options)
srv.connect()
# Check if the server is running locally (not remote server).
if not srv.is_alias('localhost'):
raise UtilError("You are using a remote server. This utility must be "
"run on the local server. It does not support remote "
"access to the binary log files.")
# Check privileges.
_check_privileges(srv, options)
# Process binlog files.
if log_type in (LOG_TYPE_BIN, LOG_TYPE_ALL):
# Get log_bin_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if bin_basename:
print(
_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin basename',
min_version='5.6.2',
var_name='log_bin_basename'))
binlog_basename = srv.select_variable('log_bin_basename')
if verbosity > 0:
print("#")
print("# log_bin_basename: {0}".format(binlog_basename))
binlog_source, binlog_file = os.path.split(binlog_basename)
# Get log_bin_index (available since MySQL 5.6.4).
if srv.check_version_compat(5, 6, 4):
if bin_index:
print(
_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='bin index',
min_version='5.6.4',
var_name='log_bin_index'))
binlog_index = srv.select_variable('log_bin_index')
else:
binlog_index = None
action = _ACTION_SEARCH_INDEX.format(file_type='bin-log')
print(
_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename',
host=srv.host,
port=srv.port,
min_version='5.6.4',
action=action))
if verbosity > 0:
print("# log_bin_index: {0}".format(binlog_index))
else:
if bin_basename:
binlog_source, binlog_file = os.path.split(bin_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='bin-log')
print(
_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='log_bin_basename',
host=srv.host,
port=srv.port,
min_version='5.6.2',
action=action))
# Get datadir value.
binlog_source = srv.select_variable('datadir')
binlog_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(binlog_source))
binlog_index = bin_index
# Move binlog files.
num_files = _move_binlogs(binlog_source,
destination,
LOG_TYPE_BIN,
options,
basename=binlog_file,
index_file=binlog_index,
skip_latest=True)
print("#")
# Flush binary logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='binary'))
srv.flush_logs(log_type='BINARY')
else:
print(
_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='binary',
host=srv.host,
port=srv.port))
print("#")
if log_type in (LOG_TYPE_RELAY, LOG_TYPE_ALL):
# Get relay_log_basename (available since MySQL 5.6.2).
if srv.check_version_compat(5, 6, 2):
if relay_basename:
print(
_WARN_MSG_VAL_NOT_REQ_FOR_SERVER.format(
value='relay basename',
min_version='5.6.2',
var_name='relay_log_basename'))
relay_log_basename = srv.select_variable('relay_log_basename')
if verbosity > 0:
print("#")
print("# relay_log_basename: {0}".format(relay_log_basename))
relay_source, relay_file = os.path.split(relay_log_basename)
else:
if relay_basename:
relay_source, relay_file = os.path.split(relay_basename)
else:
action = _ACTION_DATADIR_USED.format(file_type='relay-log')
print(
_WARN_MSG_VAR_NOT_AVAILABLE.format(
var_name='relay_log_basename',
host=srv.host,
port=srv.port,
min_version='5.6.2',
action=action))
# Get datadir value.
relay_source = srv.select_variable('datadir')
relay_file = None
if verbosity > 0:
print("#")
print("# datadir: {0}".format(relay_source))
# Get relay_log_index (available for all supported versions).
relay_log_index = srv.select_variable('relay_log_index')
if verbosity > 0:
print("# relay_log_index: {0}".format(relay_log_index))
# Move relay log files.
num_files = _move_binlogs(relay_source,
destination,
LOG_TYPE_RELAY,
options,
basename=relay_file,
index_file=relay_log_index,
skip_latest=True)
print("#")
# Flush relay logs to reload server's cache after move.
if not skip_flush and num_files > 0:
# Note: log_type for FLUSH available since MySQL 5.5.3.
if srv.check_version_compat(5, 5, 3):
print(_INFO_MSG_FLUSH_LOGS.format(log_type='relay'))
srv.flush_logs(log_type='RELAY')
else:
print(
_WARN_MSG_FLUSH_LOG_TYPE.format(log_type='relay',
host=srv.host,
port=srv.port))
print("#")
print("#...done.\n#")
class Server_list(object):
"""The Server_list class is used by the MySQL Utilities Test (MUT)
facility to gather all the servers used by the tests.
The following utilities are provided:
- start/stop a new server
- shutdown all new servers
- manage ports and server_ids
"""
def __init__(self, servers, startport, utildir, verbose=False):
"""Constructor
servers[in] List of existing servers (may be None)
startport[in] Starting port for spawned servers
util_dir[in] Path to utilities directory
verbose[in] print extra data during operations (optional)
default value = False
"""
self.utildir = utildir # Location of utilities being tested
self.new_port = startport # Starting port for spawned servers
self.verbose = verbose # Option for verbosity
self.new_id = 100 # Starting server id for spawned servers
self.server_list = servers # List of servers available
self.cleanup_list = [] # List of files to remove at shutdown
if servers is None:
self.server_list = []
def view_next_port(self):
"""View the next available server port but don't consume it.
"""
return self.new_port
def get_next_port(self):
"""Get the next available server port.
"""
new_port = self.new_port
self.new_port += 1
return new_port
def clear_last_port(self):
"""Return last port used to available status.
"""
self.new_port -= 1
def get_next_id(self):
"""Get the next available server id.
"""
new_id = self.new_id
self.new_id += 1
return new_id
def find_server_by_name(self, name):
"""Retrieve index of the server with the name indicated.
name[in] Name of the server (also used as role)
Note: This finds the first server with the name. Server names are
not unique.
Returns -1 if not found, index if found.
"""
stop = len(self.server_list)
for index in range(0, stop):
if self.server_list[index][0].role == name:
return index
return -1
def get_server(self, index):
"""Retrieve the server located at index.
index[in] Index (starting at 0)
Returns - None if index > maximum servers in list or
Server class for server at position index
"""
if index > len(self.server_list):
return None
else:
return self.server_list[index][0]
def start_new_server(self,
cur_server,
port,
server_id,
passwd,
role="server",
parameters=None):
"""Start a new server with optional parameters
This method will start a new server with the supplied optional
parameters using the mysqlserverclone.py utility by cloning the
server passed. It will also connect to the new server.
cur_server[in] Server instance to clone
port[in] Port
server_id[in] Server id
password[in] Root password for new server
role[in] Name to give for new server
parameters[in] Parameters to use on startup
Returns tuple (server, msg) [(server, None) | (None, error_str)]:
server = Server class instance or None if error
msg = None or error message if error
"""
from mysql.utilities.common.server import Server
new_server = (None, None)
# Set data directory for new server so that it is unique
full_datadir = os.getcwd() + "/temp_%s" % port
# Attempt to clone existing server
cmd = "mysqlserverclone.py --delete-data --server="
cmd += self.get_connection_string(cur_server)
if passwd:
cmd += " --root-password=%s " % passwd
cmd += " --new-port=%s " % port
cmd += "--new-id=%s " % server_id
cmd += "--new-data=%s " % os.path.normpath(full_datadir)
if parameters is not None:
cmd += "--mysqld=%s" % parameters
res = _exec_util(cmd, "cmd.txt", self.utildir)
# Create a new instance
conn = {
"user": "******",
"passwd": passwd,
"host": "localhost",
"port": port,
"unix_socket": full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info': conn,
'role': role,
}
self.new_server = Server(server_options)
server = (self.new_server, None)
# Connect to the new instance
try:
self.new_server.connect()
except UtilError, e:
raise MUTLibError("Cannot connect to spawned server: %s" % \
e.errmsg)
# If connected user is not root, clone it to the new instance.
conn_val = self.get_connection_values(cur_server)
if conn_val["user"].lower() != "root":
user_str = conn_val["user"]
if conn_val.get("passwd") is not None:
user_str += ":%s" % conn_val["passwd"]
user_str += "@%s" % conn_val["host"]
cmd = "mysqluserclone.py -s --source=%s --destination=%s" % \
(self.get_connection_string(cur_server),
self.get_connection_string(self.new_server)) + \
"%s %s" % (user_str, user_str)
res = _exec_util(cmd, "cmd.txt", self.utildir)
if res != 0:
raise MUTLibError("Cannot clone connected user.")
return server
def _test_server_clone(self, cmd_str, comment, kill=True,
capture_all=False, restart=None):
"""Test server clone.
cmd_str[in] Command to be executed.
comment[in] Test comment.
kill[in] True for kill process.
capture_all[in] True for capture all rows.
restart[in] True for restart server.
"""
quote_char = "'" if os.name == "posix" else '"'
self.results.append(comment + "\n")
port1 = int(self.servers.get_next_port())
cmd_str = "{0} --new-port={1} ".format(cmd_str, port1)
full_datadir = os.path.join(os.getcwd(),
"temp with spaces".format(port1))
cmd_str = "{0} --new-data={2}{1}{2} --delete ".format(cmd_str,
full_datadir,
quote_char)
res = self.exec_util(cmd_str, "start.txt")
with open("start.txt") as f:
for line in f:
# Don't save lines that have [Warning] or don't start with #
index = line.find("[Warning]")
if capture_all or (index <= 0 and line[0] == '#'):
self.results.append(line)
if res:
raise MUTLibError("{0}: failed".format(comment))
# Create a new instance
conn = {"user": "******", "passwd": "root", "host": "localhost",
"port": port1}
server_options = {'conn_info': conn, 'role': "cloned_server_2"}
new_server = Server(server_options)
if new_server is None:
return False
if kill:
# Connect to the new instance
try:
new_server.connect()
except UtilError:
new_server = None
raise MUTLibError("Cannot connect to spawned server.")
drop = False if restart else True
self.servers.stop_server(new_server, drop=drop)
# if restart, try to use the generated script to restart the
# server.
if restart:
if os.name == 'posix':
# Change file permissions
fl_st = os.stat(self.start_cmd_fl)
os.chmod(self.start_cmd_fl, fl_st.st_mode | stat.S_IEXEC)
# This sleep gives a chance to the OS to unlock the server
# files before restart the server.
time.sleep(3)
run_script = restart
with open("restart.txt", 'a+') as f_out:
if self.debug:
print
print("executing script: {0}".format(run_script))
subprocess.Popen(run_script)
else:
subprocess.Popen(run_script, stdout=f_out,
stderr=f_out)
# Reconnect to the instance after restart it
max_tries, attempt = 3, 0
while attempt < max_tries:
try:
time.sleep(3)
new_server.connect()
break
except UtilError:
attempt += 1
else: # if while is exhausted
new_server = None
raise MUTLibError(CANNOT_START_SERVER_ERR)
self.servers.stop_server(new_server)
return True
class Server_list(object):
"""The Server_list class is used by the MySQL Utilities Test (MUT)
facility to gather all the servers used by the tests.
The following utilities are provided:
- start/stop a new server
- shutdown all new servers
- manage ports and server_ids
"""
def __init__(self, servers, startport, utildir, verbose=False):
"""Constructor
servers[in] List of existing servers (may be None)
startport[in] Starting port for spawned servers
util_dir[in] Path to utilities directory
verbose[in] print extra data during operations (optional)
default value = False
"""
self.utildir = utildir # Location of utilities being tested
self.new_port = startport # Starting port for spawned servers
self.verbose = verbose # Option for verbosity
self.new_id = 100 # Starting server id for spawned servers
self.server_list = servers # List of servers available
self.cleanup_list = [] # List of files to remove at shutdown
if servers is None:
self.server_list = []
def view_next_port(self):
"""View the next available server port but don't consume it.
"""
return self.new_port
def get_next_port(self):
"""Get the next available server port.
"""
new_port = self.new_port
self.new_port += 1
return new_port
def clear_last_port(self):
"""Return last port used to available status.
"""
self.new_port -= 1
def get_next_id(self):
"""Get the next available server id.
"""
new_id = self.new_id
self.new_id += 1
return new_id
def find_server_by_name(self, name):
"""Retrieve index of the server with the name indicated.
name[in] Name of the server (also used as role)
Note: This finds the first server with the name. Server names are
not unique.
Returns -1 if not found, index if found.
"""
stop = len(self.server_list)
for index in range(0, stop):
if self.server_list[index][0].role == name:
return index
return -1
def get_server(self, index):
"""Retrieve the server located at index.
index[in] Index (starting at 0)
Returns - None if index > maximum servers in list or
Server class for server at position index
"""
if index > len(self.server_list):
return None
else:
return self.server_list[index][0]
def start_new_server(self, cur_server, port, server_id, passwd,
role="server", parameters=None):
"""Start a new server with optional parameters
This method will start a new server with the supplied optional
parameters using the mysqlserverclone.py utility by cloning the
server passed. It will also connect to the new server.
cur_server[in] Server instance to clone
port[in] Port
server_id[in] Server id
password[in] Root password for new server
role[in] Name to give for new server
parameters[in] Parameters to use on startup
Returns tuple (server, msg) [(server, None) | (None, error_str)]:
server = Server class instance or None if error
msg = None or error message if error
"""
from mysql.utilities.common.server import Server
new_server = (None, None)
# Set data directory for new server so that it is unique
full_datadir = os.getcwd() + "/temp_%s" % port
# Attempt to clone existing server
cmd = "mysqlserverclone.py --delete-data --server="
cmd += self.get_connection_string(cur_server)
if passwd:
cmd += " --root-password=%s " % passwd
cmd += " --new-port=%s " % port
cmd += "--new-id=%s " % server_id
cmd += "--new-data=%s " % os.path.normpath(full_datadir)
if parameters is not None:
cmd += "--mysqld=%s" % parameters
res = _exec_util(cmd, "cmd.txt", self.utildir)
# Create a new instance
conn = {
"user" : "root",
"passwd" : passwd,
"host" : "localhost",
"port" : port,
"unix_socket" : full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info' : conn,
'role' : role,
}
self.new_server = Server(server_options)
server = (self.new_server, None)
# Connect to the new instance
try:
self.new_server.connect()
except UtilError, e:
raise MUTLibError("Cannot connect to spawned server: %s" % \
e.errmsg)
# If connected user is not root, clone it to the new instance.
conn_val = self.get_connection_values(cur_server)
if conn_val["user"].lower() != "root":
user_str = conn_val["user"]
if conn_val.get("passwd") is not None:
user_str += ":%s" % conn_val["passwd"]
user_str += "@%s" % conn_val["host"]
cmd = "mysqluserclone.py -s --source=%s --destination=%s" % \
(self.get_connection_string(cur_server),
self.get_connection_string(self.new_server)) + \
"%s %s" % (user_str, user_str)
res = _exec_util(cmd, "cmd.txt", self.utildir)
if res != 0:
raise MUTLibError("Cannot clone connected user.")
return server
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
args.insert(0, mysqld_path)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
print "done."
return server
def _spawn_server(options):
"""Spawn a server to use for reading .frm files
This method spawns a new server instance on the port specified by the
user in the options dictionary.
options[in] Options from user
Returns tuple - (Server instance, new datdir) or raises exception on error
"""
verbosity = int(options.get("verbosity", 0))
quiet = options.get("quiet", False)
new_port = options.get("port", 3310)
user = options.get("user", None)
start_timeout = int(options.get("start_timeout", 10))
# 1) create a directory to use for new datadir
# If the user is not the same as the user running the script...
if user_change_as_root(options):
# Since Python libraries correctly restrict temporary folders to
# the user who runs the script and /tmp is protected on some
# platforms, we must create the folder in the current folder
temp_datadir = os.path.join(os.getcwd(), str(uuid.uuid4()))
os.mkdir(temp_datadir)
else:
temp_datadir = tempfile.mkdtemp()
if verbosity > 1 and not quiet:
print "# Creating a temporary datadir =", temp_datadir
# 2) spawn a server pointed to temp
if not quiet:
if user:
print("# Spawning server with --user={0}.".format(user))
print "# Starting the spawned server on port %s ..." % new_port,
sys.stdout.flush()
bootstrap_options = {
'new_data': temp_datadir,
'new_port': new_port,
'new_id': 101,
'root_pass': "******",
'mysqld_options': None,
'verbosity': verbosity if verbosity > 1 else 0,
'basedir': options.get("basedir"),
'delete': True,
'quiet': True if verbosity <= 1 else False,
'user': user,
'start_timeout': start_timeout,
}
if verbosity > 1 and not quiet:
print
try:
serverclone.clone_server(None, bootstrap_options)
except UtilError as error:
if error.errmsg.startswith("Unable to communicate"):
err = ". Clone server error: {0}".format(error.errmsg)
proc_id = int(error.errmsg.split("=")[1].strip('.'))
print(
"ERROR Attempting to stop failed spawned server. "
" Process id = {0}.".format(proc_id))
if os.name == "posix":
try:
os.kill(proc_id, subprocess.signal.SIGTERM)
except OSError:
pass
else:
try:
subprocess.Popen("taskkill /F /T /PID %i" % proc_id,
shell=True)
except:
pass
raise UtilError(_SPAWN_SERVER_ERROR.format(err))
else:
raise
if verbosity > 1 and not quiet:
print "# Connecting to spawned server"
conn = {
"user": "******",
"passwd": "root",
"host": "127.0.0.1",
"port": options.get("port"),
}
server_options = {
'conn_info': conn,
'role': "frm_reader_bootstrap",
}
server = Server(server_options)
try:
server.connect()
except UtilError:
raise UtilError(_SPAWN_SERVER_ERROR.format(""))
if not quiet:
print "done."
return (server, temp_datadir)
def _shutdown_running_servers(server_list, processes, basedir):
"""Shutdown any running servers.
processes[in] The list of processes to shutdown with the form:
(process_id, [datadir|port])
basedir[in] The starting path to search for mysqladmin tool
Returns bool - True - servers shutdown attempted
False - no servers to shutdown
"""
if len(processes) < 1:
return False
for process in processes:
datadir = os.getcwd()
connection = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": None,
"unix_socket": None
}
if os.name == "posix":
connection["unix_socket"] = os.path.join(process[1], "mysql.sock")
elif os.name == "nt":
connection["port"] = process[1]
if os.name == "posix":
print(" Process id: %6d, Data path: %s" % \
(int(process[0]), process[1]))
elif os.name == "nt":
print(" Process id: %6d, Port: %s" % \
(int(process[0]), process[1]))
# 1) connect to the server.
server_options = {
'conn_info': connection,
}
svr = Server(server_options)
ok_to_shutdown = True
try:
svr.connect()
except: # if we cannot connect, don't try to shut it down.
ok_to_shutdown = False
print(" WARNING: shutdown failed - cannot connect.")
if not ok_to_shutdown and os.name == "posix":
# Attempt kill
os.system("kill -9 %s" % process[0])
# 2) if nt, verify datadirectory
if os.name == "nt" and ok_to_shutdown:
res = svr.show_server_variable("datadir")
server_datadir = res[0][1]
ok_to_shudown = (server_datadir.find(datadir) >= 0)
# 3) call shutdown method from mutlib Server_list class
if ok_to_shutdown and svr:
try:
server_list.stop_server(svr)
except MUTLibError:
_, e, _ = sys.exc_info()
print(" WARNING: shutdown failed: " + e.errmsg)
return True
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False,
False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [
system_tables, system_tables_data, test_data_timezone, help_data
]:
lines = open(sqlfile, 'r').readlines()
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack)
if (sqlfile == system_tables
and "SET @sql_mode_orig==@@SES" in line
and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
proc.communicate('\n'.join(sql))
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid),
shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid, ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
cmd = [mysqld_path, '--no-defaults']
cmd.extend([
'--datadir={0}'.format(new_data),
'--tmpdir={0}'.format(new_data),
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'--port={0}'.format(new_port),
'--server-id={0}'.format(options.get('new_id', 2)),
'--basedir={0}'.format(mysql_basedir),
'--socket={0}'.format(os.path.join(new_data, 'mysql.sock')),
])
if user:
cmd.append('--user={0}'.format(user))
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.extend(mysqld_options)
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
cmd.extend(shlex.split(new_opts.strip('"')))
elif new_opts.startswith("'") and new_opts.endswith("'"):
cmd.extend(shlex.split(new_opts.strip("'")))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.append(mysqld_options)
else:
cmd.extend(shlex.split(new_opts))
# Strip spaces from each option
cmd = [opt.strip(' ') for opt in cmd]
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.append("--console")
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld", quote=True)
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
if version is not None and int(version[0]) >= 5:
post_5_5 = int(version[1]) >= 5
post_5_6 = int(version[1]) >= 6
post_5_7_4 = int(version[1]) >= 7 and int(version[2]) > 4
else:
print("# Warning: cannot get server version.")
post_5_5 = False
post_5_6 = False
post_5_7_4 = False
# Get the user executing the utility to use in the mysqld options.
# Note: the option --user=user_name is mandatory to start mysqld as root.
user_name = getpass.getuser()
# Start the instance
if verbosity > 0:
print "# Starting read-only instance of the server ..."
print "# --- BEGIN (server output) ---"
else:
print "# Starting read-only instance of the server ...",
args = shlex.split(mysqld_path)
args.extend([
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
"--user={0}".format(user_name),
])
# It the server is 5.6 or later, we must use additional parameters
if post_5_5:
server_args = [
"--skip-slave-start",
"--default-storage-engine=MYISAM",
"--server-id=0",
]
if post_5_6:
server_args.append("--default-tmp-storage-engine=MYISAM")
if not post_5_7_4:
server_args.append("--skip-innodb")
args.extend(server_args)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Indicate end of the server output.
if verbosity > 0:
print "# --- END (server output) ---"
# Raise last known exception (if unable to connect to the server)
if error:
# See: http://www.logilab.org/ticket/3207
# pylint: disable=E0702
raise error
if verbosity > 0:
print "# done (server started)."
else:
print "done."
return server
def _test_server_clone(self,
cmd_str,
comment,
kill=True,
capture_all=False,
restart=None):
"""Test server clone.
cmd_str[in] Command to be executed.
comment[in] Test comment.
kill[in] True for kill process.
capture_all[in] True for capture all rows.
restart[in] True for restart server.
"""
quote_char = "'" if os.name == "posix" else '"'
self.results.append(comment + "\n")
port1 = int(self.servers.get_next_port())
cmd_str = "{0} --new-port={1} ".format(cmd_str, port1)
full_datadir = os.path.join(os.getcwd(),
"temp with spaces".format(port1))
cmd_str = "{0} --new-data={2}{1}{2} --delete ".format(
cmd_str, full_datadir, quote_char)
res = self.exec_util(cmd_str, "start.txt")
with open("start.txt") as f:
for line in f:
# Don't save lines that have [Warning] or don't start with #
index = line.find("[Warning]")
if capture_all or (index <= 0 and line[0] == '#'):
self.results.append(line)
if res:
raise MUTLibError("{0}: failed".format(comment))
# Create a new instance
conn = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": port1
}
server_options = {'conn_info': conn, 'role': "cloned_server_2"}
new_server = Server(server_options)
if new_server is None:
return False
if kill:
# Connect to the new instance
try:
new_server.connect()
except UtilError:
new_server = None
raise MUTLibError("Cannot connect to spawned server.")
drop = False if restart else True
self.servers.stop_server(new_server, drop=drop)
time.sleep(5)
# if restart, try to use the generated script to restart the
# server.
if restart:
if os.name == 'posix':
# Change file permissions
fl_st = os.stat(self.start_cmd_fl)
os.chmod(self.start_cmd_fl, fl_st.st_mode | stat.S_IEXEC)
# This sleep gives a chance to the OS to unlock the server
# files before restart the server.
time.sleep(3)
run_script = restart
with open("restart.txt", 'a+') as f_out:
if self.debug:
print
print("executing script: {0}".format(run_script))
subprocess.Popen(run_script)
else:
subprocess.Popen(run_script,
stdout=f_out,
stderr=f_out)
# Reconnect to the instance after restart it
max_tries, attempt = 3, 0
while attempt < max_tries:
try:
time.sleep(3)
new_server.connect()
break
except UtilError:
attempt += 1
else: # if while is exhausted
new_server = None
raise MUTLibError(CANNOT_START_SERVER_ERR)
self.servers.stop_server(new_server)
return True
(conn_val["host"], conn_val["user"],
conn_val["port"]))
sys.stdout.flush()
if conn_val["port"] is not None:
conn_val["port"] = int(conn_val["port"])
else:
conn_val["port"] = 0
server_options = {
'conn_info' : conn_val,
'role' : "server%d" % i,
}
conn = Server(server_options)
try:
conn.connect()
server_list.add_new_server(conn)
print("CONNECTED")
res = conn.show_server_variable("basedir")
#print res
basedir = res[0][1]
# Here we capture any exception and print the error message.
# Since all util errors (exceptions) derive from Exception, this is
# safe.
except Exception:
_, err, _ = sys.exc_info()
print("%sFAILED%s" % (BOLD_ON, BOLD_OFF))
if conn.connect_error is not None:
print(conn.connect_error)
print("ERROR: %s" % str(err))
if server_list.num_servers() == 0:
class BinaryLogPurge(object):
"""BinaryLogPurge
"""
def __init__(self, server_cnx_val, options):
"""Initiator.
server_cnx_val[in] Server connection dictionary.
options[in] Options dictionary.
"""
self.server_cnx_val = server_cnx_val
self.server = None
self.options = options
self.verbosity = self.options.get("verbosity", 0)
self.quiet = self.options.get("quiet", False)
self.logging = self.options.get("logging", False)
self.dry_run = self.options.get("dry_run", 0)
self.to_binlog_name = self.options.get("to_binlog_name", False)
def _report(self, message, level=logging.INFO, print_msg=True):
"""Log message if logging is on.
This method will log the message presented if the log is turned on.
Specifically, if options['log_file'] is not None. It will also
print the message to stdout.
message[in] Message to be printed.
level[in] Level of message to log. Default = INFO.
print_msg[in] If True, print the message to stdout. Default = True.
"""
# First, print the message.
if print_msg and not self.quiet:
print(message)
# Now log message if logging turned on
if self.logging:
logging.log(int(level), message.strip("#").strip(" "))
def get_target_binlog_index(self, binlog_file_name):
"""Retrieves the target binlog file index.
Retrieves the target binlog file index that will used in the purge
query, by the fault the latest log not in use unless the user
specifies a different target which is validated against the server's
binlog base name.
binlog_file_name[in] the binlog base file name used by the server.
Returns the target index binlog file
"""
if self.to_binlog_name:
to_binlog_name = self.to_binlog_name.split('.')[0]
if to_binlog_name != binlog_file_name:
raise UtilError(
"The given binlog file name: '{0}' differs "
"from the used by the server: '{1}'"
"".
format(to_binlog_name, binlog_file_name))
else:
to_binlog_index = int(self.to_binlog_name.split('.')[1])
return to_binlog_index
return None
def _purge(self, index_last_in_use, active_binlog_file, binlog_file_name,
target_binlog_index=None, server=None, server_is_master=False):
"""The inner purge method.
Purges the binary logs from the given server, it will purge all of the
binlogs older than the active_binlog_file ot to target_binlog_index.
index_last_in_use[in] The index of the latest binary log not in
use. in case of a Master, must be the latest
binlog caought by all the slaves.
active_binlog_file[in] Current active binlog file.
binlog_file_name[in] Binlog base file name.
target_binlog_index[in] The target binlog index, in case doesn't want
to use the index_last_in_use by default None.
server[in] Server object where to purge the binlogs from,
by default self.server is used.
server_is_master[in] Indicates if the given server is a Master,
used for report purposes by default False.
"""
if server is None:
server = self.server
if server_is_master:
server_name = "master"
else:
server_name = "server"
# The purge_to_binlog file used to purge query based on earliest log
# not in use
z_len = len(active_binlog_file.split('.')[1])
purge_to_binlog = (
"{0}.{1}".format(binlog_file_name,
repr(index_last_in_use).zfill(z_len))
)
server_binlogs_list = server.get_server_binlogs_list()
if self.verbosity >= 1:
_report_binlogs(server_binlogs_list, self._report)
# The last_binlog_not_in_use used for information purposes
index_last_not_in_use = index_last_in_use - 1
last_binlog_not_in_use = (
"{0}.{1}".format(binlog_file_name,
repr(index_last_not_in_use).zfill(z_len))
)
if server_is_master:
self._report("# Latest binlog file replicated by all slaves: "
"{0}".format(last_binlog_not_in_use))
if target_binlog_index is None:
# Purge to latest binlog not in use
if self.verbosity > 0:
self._report("# Latest not active binlog"
" file: {0}".format(last_binlog_not_in_use))
# last_binlog_not_in_use
purge(server, purge_to_binlog, server_binlogs_list,
reporter=self._report, dryrun=self.dry_run,
verbosity=self.verbosity)
else:
purge_to_binlog = (
"{0}.{1}".format(binlog_file_name,
repr(target_binlog_index).zfill(z_len))
)
if purge_to_binlog not in server_binlogs_list:
self._report(
_COULD_NOT_FIND_BINLOG.format(bin_name=self.to_binlog_name,
server_name=server_name,
host=server.host,
port=server.port))
return
if target_binlog_index > index_last_in_use:
self._report("WARNING: The given binlog name: '{0}' is "
"required for one or more slaves, the Utilitiy "
"will purge to binlog '{1}' instead."
"".format(self.to_binlog_name,
last_binlog_not_in_use))
target_binlog_index = last_binlog_not_in_use
# last_binlog_not_in_use
purge(server, purge_to_binlog, server_binlogs_list,
reporter=self._report, dryrun=self.dry_run,
verbosity=self.verbosity)
server_binlogs_list_after = server.get_server_binlogs_list()
if self.verbosity >= 1:
_report_binlogs(server_binlogs_list_after, self._report)
for binlog in server_binlogs_list_after:
if binlog in server_binlogs_list:
server_binlogs_list.remove(binlog)
if self.verbosity >= 1 and server_binlogs_list:
_report_binlogs(server_binlogs_list, self._report, removed=True)
def purge(self):
"""The purge method for a standalone server.
Determines the latest log file to purge, which becomes the target
file to purge binary logs to in case no other file is specified.
"""
# Connect to server
self.server = Server({'conn_info': self.server_cnx_val})
self.server.connect()
# Check required privileges
check_privileges(self.server, BINLOG_OP_PURGE,
["SUPER", "REPLICATION SLAVE"],
BINLOG_OP_PURGE_DESC, self.verbosity, self._report)
# retrieve active binlog info
binlog_file_name, active_binlog_file, index_last_in_use = (
get_binlog_info(self.server, reporter=self._report,
server_name="server", verbosity=self.verbosity)
)
# Verify this server is not a Master.
processes = self.server.exec_query("SHOW PROCESSLIST")
binlog_dump = False
for process in processes:
if process[4] == "Binlog Dump":
binlog_dump = True
break
hosts = self.server.exec_query("SHOW SLAVE HOSTS")
if binlog_dump or hosts:
if hosts and not self.verbosity:
msg_v = " For more info use verbose option."
else:
msg_v = ""
if self.verbosity >= 1:
for host in hosts:
self._report("# WARNING: Slave with id:{0} at {1}:{2} "
"is connected to this server."
"".format(host[0], host[1], host[2]))
raise UtilError("The given server is acting as a master and has "
"slaves connected to it. To proceed please use the"
" --master option.{0}".format(msg_v))
target_binlog_index = self.get_target_binlog_index(binlog_file_name)
self._purge(index_last_in_use, active_binlog_file, binlog_file_name,
target_binlog_index)
def _shutdown_running_servers(server_list, processes, basedir):
"""Shutdown any running servers.
processes[in] The list of processes to shutdown with the form:
(process_id, [datadir|port])
basedir[in] The starting path to search for mysqladmin tool
Returns bool - True - servers shutdown attempted
False - no servers to shutdown
"""
if len(processes) < 1:
return False
for process in processes:
datadir = os.getcwd()
connection = {
"user" : "root",
"passwd" : "root",
"host" : "localhost",
"port" : None,
"unix_socket" : None
}
if os.name == "posix":
connection["unix_socket"] = os.path.join(process[1], "mysql.sock")
elif os.name == "nt":
connection["port"] = process[1]
if os.name == "posix":
print(" Process id: %6d, Data path: %s" % \
(int(process[0]), process[1]))
elif os.name == "nt":
print(" Process id: %6d, Port: %s" % \
(int(process[0]), process[1]))
# 1) connect to the server.
server_options = {
'conn_info' : connection,
}
svr = Server(server_options)
ok_to_shutdown = True
try:
svr.connect()
except: # if we cannot connect, don't try to shut it down.
ok_to_shutdown = False
print(" WARNING: shutdown failed - cannot connect.")
if not ok_to_shutdown and os.name == "posix":
# Attempt kill
os.system("kill -9 %s" % process[0])
# 2) if nt, verify datadirectory
if os.name == "nt" and ok_to_shutdown:
res = svr.show_server_variable("datadir")
server_datadir = res[0][1]
ok_to_shudown = (server_datadir.find(datadir) >= 0)
# 3) call shutdown method from mutlib Server_list class
if ok_to_shutdown and svr:
try:
server_list.stop_server(svr)
except MUTLibError:
_, e, _ = sys.exc_info()
print(" WARNING: shutdown failed: " + e.errmsg)
return True
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld", quote=True)
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
if version is not None and int(version[0]) >= 5:
post_5_5 = int(version[1]) >= 5
post_5_6 = int(version[1]) >= 6
post_5_7_4 = int(version[1]) >= 7 and int(version[2]) > 4
else:
print("# Warning: cannot get server version.")
post_5_5 = False
post_5_6 = False
post_5_7_4 = False
# Get the user executing the utility to use in the mysqld options.
# Note: the option --user=user_name is mandatory to start mysqld as root.
user_name = getpass.getuser()
# Start the instance
if verbosity > 0:
print "# Starting read-only instance of the server ..."
print "# --- BEGIN (server output) ---"
else:
print "# Starting read-only instance of the server ...",
args = shlex.split(mysqld_path)
args.extend([
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
"--user={0}".format(user_name),
])
# It the server is 5.6 or later, we must use additional parameters
if post_5_5:
server_args = [
"--skip-slave-start",
"--default-storage-engine=MYISAM",
"--server-id=0",
]
if post_5_6:
server_args.append("--default-tmp-storage-engine=MYISAM")
if not post_5_7_4:
server_args.append("--skip-innodb")
args.extend(server_args)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Indicate end of the server output.
if verbosity > 0:
print "# --- END (server output) ---"
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
if verbosity > 0:
print "# done (server started)."
else:
print "done."
return server
class BinaryLogPurge(object):
"""BinaryLogPurge
"""
def __init__(self, server_cnx_val, options):
"""Initiator.
server_cnx_val[in] Server connection dictionary.
options[in] Options dictionary.
"""
self.server_cnx_val = server_cnx_val
self.server = None
self.options = options
self.verbosity = self.options.get("verbosity", 0)
self.quiet = self.options.get("quiet", False)
self.logging = self.options.get("logging", False)
self.dry_run = self.options.get("dry_run", 0)
self.to_binlog_name = self.options.get("to_binlog_name", False)
def _report(self, message, level=logging.INFO, print_msg=True):
"""Log message if logging is on.
This method will log the message presented if the log is turned on.
Specifically, if options['log_file'] is not None. It will also
print the message to stdout.
message[in] Message to be printed.
level[in] Level of message to log. Default = INFO.
print_msg[in] If True, print the message to stdout. Default = True.
"""
# First, print the message.
if print_msg and not self.quiet:
print(message)
# Now log message if logging turned on
if self.logging:
logging.log(int(level), message.strip("#").strip(" "))
def get_target_binlog_index(self, binlog_file_name):
"""Retrieves the target binlog file index.
Retrieves the target binlog file index that will used in the purge
query, by the fault the latest log not in use unless the user
specifies a different target which is validated against the server's
binlog base name.
binlog_file_name[in] the binlog base file name used by the server.
Returns the target index binlog file
"""
if self.to_binlog_name:
to_binlog_name = self.to_binlog_name.split('.')[0]
if to_binlog_name != binlog_file_name:
raise UtilError("The given binlog file name: '{0}' differs "
"from the used by the server: '{1}'"
"".format(to_binlog_name, binlog_file_name))
else:
to_binlog_index = int(self.to_binlog_name.split('.')[1])
return to_binlog_index
return None
def _purge(self,
index_last_in_use,
active_binlog_file,
binlog_file_name,
target_binlog_index=None,
server=None,
server_is_master=False):
"""The inner purge method.
Purges the binary logs from the given server, it will purge all of the
binlogs older than the active_binlog_file ot to target_binlog_index.
index_last_in_use[in] The index of the latest binary log not in
use. in case of a Master, must be the latest
binlog caought by all the slaves.
active_binlog_file[in] Current active binlog file.
binlog_file_name[in] Binlog base file name.
target_binlog_index[in] The target binlog index, in case doesn't want
to use the index_last_in_use by default None.
server[in] Server object where to purge the binlogs from,
by default self.server is used.
server_is_master[in] Indicates if the given server is a Master,
used for report purposes by default False.
"""
if server is None:
server = self.server
if server_is_master:
server_name = "master"
else:
server_name = "server"
# The purge_to_binlog file used to purge query based on earliest log
# not in use
z_len = len(active_binlog_file.split('.')[1])
purge_to_binlog = ("{0}.{1}".format(
binlog_file_name,
repr(index_last_in_use).zfill(z_len)))
server_binlogs_list = server.get_server_binlogs_list()
if self.verbosity >= 1:
_report_binlogs(server_binlogs_list, self._report)
# The last_binlog_not_in_use used for information purposes
index_last_not_in_use = index_last_in_use - 1
last_binlog_not_in_use = ("{0}.{1}".format(
binlog_file_name,
repr(index_last_not_in_use).zfill(z_len)))
if server_is_master:
self._report("# Latest binlog file replicated by all slaves: "
"{0}".format(last_binlog_not_in_use))
if target_binlog_index is None:
# Purge to latest binlog not in use
if self.verbosity > 0:
self._report("# Latest not active binlog"
" file: {0}".format(last_binlog_not_in_use))
# last_binlog_not_in_use
purge(server,
purge_to_binlog,
server_binlogs_list,
reporter=self._report,
dryrun=self.dry_run,
verbosity=self.verbosity)
else:
purge_to_binlog = ("{0}.{1}".format(
binlog_file_name,
repr(target_binlog_index).zfill(z_len)))
if purge_to_binlog not in server_binlogs_list:
self._report(
_COULD_NOT_FIND_BINLOG.format(bin_name=self.to_binlog_name,
server_name=server_name,
host=server.host,
port=server.port))
return
if target_binlog_index > index_last_in_use:
self._report("WARNING: The given binlog name: '{0}' is "
"required for one or more slaves, the Utilitiy "
"will purge to binlog '{1}' instead."
"".format(self.to_binlog_name,
last_binlog_not_in_use))
target_binlog_index = last_binlog_not_in_use
# last_binlog_not_in_use
purge(server,
purge_to_binlog,
server_binlogs_list,
reporter=self._report,
dryrun=self.dry_run,
verbosity=self.verbosity)
server_binlogs_list_after = server.get_server_binlogs_list()
if self.verbosity >= 1:
_report_binlogs(server_binlogs_list_after, self._report)
for binlog in server_binlogs_list_after:
if binlog in server_binlogs_list:
server_binlogs_list.remove(binlog)
if self.verbosity >= 1 and server_binlogs_list:
_report_binlogs(server_binlogs_list, self._report, removed=True)
def purge(self):
"""The purge method for a standalone server.
Determines the latest log file to purge, which becomes the target
file to purge binary logs to in case no other file is specified.
"""
# Connect to server
self.server = Server({'conn_info': self.server_cnx_val})
self.server.connect()
# Check required privileges
check_privileges(self.server, BINLOG_OP_PURGE,
["SUPER", "REPLICATION SLAVE"], BINLOG_OP_PURGE_DESC,
self.verbosity, self._report)
# retrieve active binlog info
binlog_file_name, active_binlog_file, index_last_in_use = (
get_binlog_info(self.server,
reporter=self._report,
server_name="server",
verbosity=self.verbosity))
# Verify this server is not a Master.
processes = self.server.exec_query("SHOW PROCESSLIST")
binlog_dump = False
for process in processes:
if process[4] == "Binlog Dump":
binlog_dump = True
break
hosts = self.server.exec_query("SHOW SLAVE HOSTS")
if binlog_dump or hosts:
if hosts and not self.verbosity:
msg_v = " For more info use verbose option."
else:
msg_v = ""
if self.verbosity >= 1:
for host in hosts:
self._report("# WARNING: Slave with id:{0} at {1}:{2} "
"is connected to this server."
"".format(host[0], host[1], host[2]))
raise UtilError("The given server is acting as a master and has "
"slaves connected to it. To proceed please use the"
" --master option.{0}".format(msg_v))
target_binlog_index = self.get_target_binlog_index(binlog_file_name)
self._purge(index_last_in_use, active_binlog_file, binlog_file_name,
target_binlog_index)
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
args.insert(0, mysqld_path)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
print "done."
return server
class BinaryLogRotate(object):
"""The BinaryLogRotate Class, it represents a binary log rotation task.
The rotate method performs the following tasks:
- Retrieves the active binary log and file size.
- If the minimum size is given, evaluate active binlog file size, and
if this is greater than the minimum size rotation will occur.
rotation occurs.
"""
def __init__(self, server_cnx_val, options):
"""Initiator.
server_cnx_val[in] Dictionary with the connection values for the
server.
options[in] options for controlling behavior:
logging If logging is active or not.
verbose print extra data during operations (optional)
default value = False
min_size minimum size that the active binlog must have prior
to rotate it.
dry_run Don't actually rotate the active binlog, instead
it will print information about file name and size.
"""
# Connect to server
self.server = Server({'conn_info': server_cnx_val})
self.server.connect()
self.options = options
self.verbosity = self.options.get("verbosity", 0)
self.quiet = self.options.get("quiet", False)
self.logging = self.options.get("logging", False)
self.dry_run = self.options.get("dry_run", 0)
self.binlog_min_size = self.options.get("min_size", False)
def _report(self, message, level=logging.INFO, print_msg=True):
"""Log message if logging is on.
This method will log the message presented if the log is turned on.
Specifically, if options['log_file'] is not None. It will also
print the message to stdout.
message[in] Message to be printed.
level[in] Level of message to log. Default = INFO.
print_msg[in] If True, print the message to stdout. Default = True.
"""
# First, print the message.
if print_msg and not self.quiet:
print(message)
# Now log message if logging turned on
if self.logging:
msg = message.strip("#").strip(" ")
logging.log(int(level), msg)
def rotate(self):
"""This Method runs the rotation.
This method will use the methods from the common library to rotate the
binary log.
"""
# Check required privileges
check_privileges(self.server, BINLOG_OP_ROTATE,
["RELOAD", "REPLICATION CLIENT"],
BINLOG_OP_ROTATE_DESC, self.verbosity, self._report)
active_binlog, binlog_size = get_active_binlog_and_size(self.server)
if self.verbosity:
self._report("# Active binlog file: '{0}' (size: {1} bytes)'"
"".format(active_binlog, binlog_size))
if self.binlog_min_size:
rotated = rotate(self.server,
self.binlog_min_size,
reporter=self._report)
else:
rotated = rotate(self.server, reporter=self._report)
if rotated:
new_active_binlog, _ = get_active_binlog_and_size(self.server)
if active_binlog == new_active_binlog:
raise UtilError("Unable to rotate binlog file.")
else:
self._report("# The binlog file has been rotated.")
if self.verbosity:
self._report("# New active binlog file: '{0}'"
"".format(new_active_binlog))
" Connecting to %s as user %s on port %s: " %
(conn_val["host"], conn_val["user"], conn_val["port"]))
sys.stdout.flush()
if conn_val["port"] is not None:
conn_val["port"] = int(conn_val["port"])
else:
conn_val["port"] = 0
server_options = {
'conn_info': conn_val,
'role': "server%d" % i,
}
conn = Server(server_options)
try:
conn.connect()
server_list.add_new_server(conn)
print("CONNECTED")
res = conn.show_server_variable("basedir")
#print res
basedir = res[0][1]
# Here we capture any exception and print the error message.
# Since all util errors (exceptions) derive from Exception, this is
# safe.
except Exception:
_, err, _ = sys.exc_info()
print("%sFAILED%s" % (BOLD_ON, BOLD_OFF))
if conn.connect_error is not None:
print(conn.connect_error)
print("ERROR: %s" % str(err))
if server_list.num_servers() == 0:
class test(mutlib.System_test):
"""clone server parameters
This test exercises the parameters for mysqlserverclone
"""
def check_prerequisites(self):
return self.check_num_servers(1)
def setup(self):
# No setup needed
self.new_server = None
return True
def _test_server_clone(self,
cmd_str,
comment,
kill=True,
capture_all=False):
self.results.append(comment + "\n")
port1 = int(self.servers.get_next_port())
cmd_str += " --new-port=%d " % port1
full_datadir = os.path.join(os.getcwd(), "temp_%s" % port1)
cmd_str += " --new-data=%s --delete " % full_datadir
res = self.exec_util(cmd_str, "start.txt")
for line in open("start.txt").readlines():
# Don't save lines that have [Warning] or don't start with #
index = line.find("[Warning]")
if capture_all or (index <= 0 and line[0] == '#'):
self.results.append(line)
if res:
raise MUTLibError("%s: failed" % comment)
# Create a new instance
conn = {
"user": "******",
"passwd": "root",
"host": "localhost",
"port": port1,
"unix_socket": full_datadir + "/mysql.sock"
}
if os.name != "posix":
conn["unix_socket"] = None
server_options = {
'conn_info': conn,
'role': "cloned_server_2",
}
self.new_server = Server(server_options)
if self.new_server is None:
return False
if kill:
# Connect to the new instance
try:
self.new_server.connect()
except UtilError, e:
self.new_server = None
raise MUTLibError("Cannot connect to spawned server.")
self.servers.stop_server(self.new_server)
self.servers.clear_last_port()
return True
