def _bulk_insert(self, rows, new_db, destination=None): """Import data using bulk insert Reads data from a table and builds group INSERT statements for writing to the destination server specified (new_db.name). This method is designed to be used in a thread for parallel inserts. As such, it requires its own connection to the destination server. Note: This method does not print any information to stdout. rows[in] a list of rows to process new_db[in] new database name destination[in] the destination server """ if self.dest_vals is None: self.dest_vals = self.get_dest_values(destination) # Spawn a new connection server_options = { 'conn_info': self.dest_vals, 'role': "thread", } dest = Server(server_options) dest.connect() # Issue the write lock lock_list = [("%s.%s" % (new_db, self.q_tbl_name), 'WRITE')] my_lock = Lock(dest, lock_list, {'locking': 'lock-all', }) # First, turn off foreign keys if turned on dest.disable_foreign_key_checks(True) if self.column_format is None: self.get_column_metadata() data_lists = self.make_bulk_insert(rows, new_db) insert_data = data_lists[0] blob_data = data_lists[1] # Insert the data first for data_insert in insert_data: try: dest.exec_query(data_insert, self.query_options) except UtilError, e: raise UtilError("Problem inserting data. " "Error = %s" % e.errmsg)
def do_command(self): """ Check and execute the audit log command (previously set by the the options of the object constructor). """ # Check for valid command command = self.options.get("command", None) if not command in VALID_COMMANDS: raise UtilError("Invalid command.") command_value = self.options.get("value", None) # Check for valid value if needed if (command_requires_value(command) and not check_command_value(command, command_value)): raise UtilError("Please provide the correct value for the %s " "command." % command) # Copy command does not need the server if command == "COPY": self._copy_log() return True # Connect to server server = Server({'conn_info': self.options.get("server_vals", None)}) server.connect() # Now execute the command print "#\n# Executing %s command.\n#\n" % command try: if command == "POLICY": server.exec_query("SET @@GLOBAL.audit_log_policy = %s" % command_value) elif command == "ROTATE": self._rotate_log(server) else: # "ROTATE_ON_SIZE": server.exec_query( "SET @@GLOBAL.audit_log_rotate_on_size = %s" % command_value) finally: server.disconnect() return True
def do_command(self): """ Check and execute the audit log command (previously set by the the options of the object constructor). """ # Check for valid command command = self.options.get("command", None) if not command in VALID_COMMANDS: raise UtilError("Invalid command.") command_value = self.options.get("value", None) # Check for valid value if needed if (command_requires_value(command) and not check_command_value(command, command_value)): raise UtilError("Please provide the correct value for the %s " "command." % command) # Copy command does not need the server if command == "COPY": self._copy_log() return True # Connect to server server = Server({'conn_info': self.options.get("server_vals", None)}) server.connect() # Now execute the command print "#\n# Executing %s command.\n#\n" % command try: if command == "POLICY": server.exec_query("SET @@GLOBAL.audit_log_policy = %s" % command_value) elif command == "ROTATE": self._rotate_log(server) else: # "ROTATE_ON_SIZE": server.exec_query("SET @@GLOBAL.audit_log_rotate_on_size = %s" % command_value) finally: server.disconnect() return True
def load_test_data(server, db_num=1): """Load/insert data into the test databases. A considerable amount of data should be considered in order to take some time to load, allowing mysqlrplsync to be executed at the same time the data is still being inserted. server[in] Target server to load the test data. db_num[in] Number of databases to load the data (by default: 1). It is assumed that a matching number of test databases have been previously created. Note: method prepared to be invoked by a different thread. """ # Create a new server instance with a new connection (for multithreading). srv = Server({'conn_info': server}) srv.connect() for db_index in xrange(db_num): db_name = '`test_rplsync_db{0}`'.format( '' if db_num == 1 else db_index ) # Insert random data on all tables. random_values = string.letters + string.digits for _ in xrange(TEST_DB_NUM_ROWS): columns = [] values = [] for table_index in xrange(TEST_DB_NUM_TABLES): columns.append('rnd_txt{0}'.format(table_index)) rnd_text = "".join( [random.choice(random_values) for _ in xrange(20)] ) values.append("'{0}'".format(rnd_text)) insert = ("INSERT INTO {0}.`t{1}` ({2}) VALUES ({3})" "").format(db_name, table_index, ', '.join(columns), ', '.join(values)) srv.exec_query(insert) srv.commit()
class BinaryLogPurge(object): """BinaryLogPurge """ def __init__(self, server_cnx_val, options): """Initiator. server_cnx_val[in] Server connection dictionary. options[in] Options dictionary. """ self.server_cnx_val = server_cnx_val self.server = None self.options = options self.verbosity = self.options.get("verbosity", 0) self.quiet = self.options.get("quiet", False) self.logging = self.options.get("logging", False) self.dry_run = self.options.get("dry_run", 0) self.to_binlog_name = self.options.get("to_binlog_name", False) def _report(self, message, level=logging.INFO, print_msg=True): """Log message if logging is on. This method will log the message presented if the log is turned on. Specifically, if options['log_file'] is not None. It will also print the message to stdout. message[in] Message to be printed. level[in] Level of message to log. Default = INFO. print_msg[in] If True, print the message to stdout. Default = True. """ # First, print the message. if print_msg and not self.quiet: print(message) # Now log message if logging turned on if self.logging: logging.log(int(level), message.strip("#").strip(" ")) def get_target_binlog_index(self, binlog_file_name): """Retrieves the target binlog file index. Retrieves the target binlog file index that will used in the purge query, by the fault the latest log not in use unless the user specifies a different target which is validated against the server's binlog base name. binlog_file_name[in] the binlog base file name used by the server. Returns the target index binlog file """ if self.to_binlog_name: to_binlog_name = self.to_binlog_name.split('.')[0] if to_binlog_name != binlog_file_name: raise UtilError( "The given binlog file name: '{0}' differs " "from the used by the server: '{1}'" "". format(to_binlog_name, binlog_file_name)) else: to_binlog_index = int(self.to_binlog_name.split('.')[1]) return to_binlog_index return None def _purge(self, index_last_in_use, active_binlog_file, binlog_file_name, target_binlog_index=None, server=None, server_is_master=False): """The inner purge method. Purges the binary logs from the given server, it will purge all of the binlogs older than the active_binlog_file ot to target_binlog_index. index_last_in_use[in] The index of the latest binary log not in use. in case of a Master, must be the latest binlog caought by all the slaves. active_binlog_file[in] Current active binlog file. binlog_file_name[in] Binlog base file name. target_binlog_index[in] The target binlog index, in case doesn't want to use the index_last_in_use by default None. server[in] Server object where to purge the binlogs from, by default self.server is used. server_is_master[in] Indicates if the given server is a Master, used for report purposes by default False. """ if server is None: server = self.server if server_is_master: server_name = "master" else: server_name = "server" # The purge_to_binlog file used to purge query based on earliest log # not in use z_len = len(active_binlog_file.split('.')[1]) purge_to_binlog = ( "{0}.{1}".format(binlog_file_name, repr(index_last_in_use).zfill(z_len)) ) server_binlogs_list = server.get_server_binlogs_list() if self.verbosity >= 1: _report_binlogs(server_binlogs_list, self._report) # The last_binlog_not_in_use used for information purposes index_last_not_in_use = index_last_in_use - 1 last_binlog_not_in_use = ( "{0}.{1}".format(binlog_file_name, repr(index_last_not_in_use).zfill(z_len)) ) if server_is_master: self._report("# Latest binlog file replicated by all slaves: " "{0}".format(last_binlog_not_in_use)) if target_binlog_index is None: # Purge to latest binlog not in use if self.verbosity > 0: self._report("# Latest not active binlog" " file: {0}".format(last_binlog_not_in_use)) # last_binlog_not_in_use purge(server, purge_to_binlog, server_binlogs_list, reporter=self._report, dryrun=self.dry_run, verbosity=self.verbosity) else: purge_to_binlog = ( "{0}.{1}".format(binlog_file_name, repr(target_binlog_index).zfill(z_len)) ) if purge_to_binlog not in server_binlogs_list: self._report( _COULD_NOT_FIND_BINLOG.format(bin_name=self.to_binlog_name, server_name=server_name, host=server.host, port=server.port)) return if target_binlog_index > index_last_in_use: self._report("WARNING: The given binlog name: '{0}' is " "required for one or more slaves, the Utilitiy " "will purge to binlog '{1}' instead." "".format(self.to_binlog_name, last_binlog_not_in_use)) target_binlog_index = last_binlog_not_in_use # last_binlog_not_in_use purge(server, purge_to_binlog, server_binlogs_list, reporter=self._report, dryrun=self.dry_run, verbosity=self.verbosity) server_binlogs_list_after = server.get_server_binlogs_list() if self.verbosity >= 1: _report_binlogs(server_binlogs_list_after, self._report) for binlog in server_binlogs_list_after: if binlog in server_binlogs_list: server_binlogs_list.remove(binlog) if self.verbosity >= 1 and server_binlogs_list: _report_binlogs(server_binlogs_list, self._report, removed=True) def purge(self): """The purge method for a standalone server. Determines the latest log file to purge, which becomes the target file to purge binary logs to in case no other file is specified. """ # Connect to server self.server = Server({'conn_info': self.server_cnx_val}) self.server.connect() # Check required privileges check_privileges(self.server, BINLOG_OP_PURGE, ["SUPER", "REPLICATION SLAVE"], BINLOG_OP_PURGE_DESC, self.verbosity, self._report) # retrieve active binlog info binlog_file_name, active_binlog_file, index_last_in_use = ( get_binlog_info(self.server, reporter=self._report, server_name="server", verbosity=self.verbosity) ) # Verify this server is not a Master. processes = self.server.exec_query("SHOW PROCESSLIST") binlog_dump = False for process in processes: if process[4] == "Binlog Dump": binlog_dump = True break hosts = self.server.exec_query("SHOW SLAVE HOSTS") if binlog_dump or hosts: if hosts and not self.verbosity: msg_v = " For more info use verbose option." else: msg_v = "" if self.verbosity >= 1: for host in hosts: self._report("# WARNING: Slave with id:{0} at {1}:{2} " "is connected to this server." "".format(host[0], host[1], host[2])) raise UtilError("The given server is acting as a master and has " "slaves connected to it. To proceed please use the" " --master option.{0}".format(msg_v)) target_binlog_index = self.get_target_binlog_index(binlog_file_name) self._purge(index_last_in_use, active_binlog_file, binlog_file_name, target_binlog_index)
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command start_timeout[in] Number of seconds to wait for server to start """ new_data = os.path.abspath(options.get('new_data', None)) new_port = options.get('new_port', '3307') root_pass = options.get('root_pass', None) verbosity = options.get('verbosity', 0) user = options.get('user', 'root') quiet = options.get('quiet', False) cmd_file = options.get('cmd_file', None) start_timeout = int(options.get('start_timeout', 10)) mysqld_options = options.get('mysqld_options', '') force = options.get('force', False) quote_char = "'" if os.name == "posix" else '"' if not check_port_in_use('localhost', int(new_port)): raise UtilError("Port {0} in use. Please choose an " "available port.".format(new_port)) # Check if path to database files is greater than MAX_DIR_SIZE char, if len(new_data) > MAX_DATADIR_SIZE and not force: raise UtilError("The --new-data path '{0}' is too long " "(> {1} characters). Please use a smaller one. " "You can use the --force option to skip this " "check".format(new_data, MAX_DATADIR_SIZE)) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = { 'conn_info': conn_val, 'role': "source", } server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % \ conn_val["host"] # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = os.path.normpath(rows[0][1]) # Cloning downed or offline server else: basedir = os.path.abspath(options.get("basedir", None)) if not quiet: print "# Cloning the MySQL server located at %s." % basedir new_data_deleted = False # If datadir exists, has data, and user said it was Ok, delete it if os.path.exists(new_data) and options.get("delete", False) and \ os.listdir(new_data): new_data_deleted = True shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not os.path.exists(new_data): if not quiet: print "# Creating new data directory..." try: os.mkdir(new_data) except OSError as err: raise UtilError("Unable to create directory '{0}', reason: {1}" "".format(new_data, err.strerror)) # After create the new data directory, check for free space, so the errors # regarding invalid or inaccessible path had been dismissed already. # If not force specified verify and stop if there is not enough free space if not force and os.path.exists(new_data) and \ estimate_free_space(new_data) < REQ_FREE_SPACE: # Don't leave empty folders, delete new_data if was previously deleted if os.path.exists(new_data) and new_data_deleted: shutil.rmtree(new_data, True) raise UtilError( LOW_SPACE_ERRR_MSG.format(directory=new_data, megabytes=REQ_FREE_SPACE)) # Check for warning of using --skip-innodb mysqld_path = get_tool_path(basedir, "mysqld") version_str = get_mysqld_version(mysqld_path) # convert version_str from str tuple to integer tuple if possible if version_str is not None: version = tuple([int(digit) for digit in version_str]) else: version = None if mysqld_options is not None and ("--skip-innodb" in mysqld_options or "--innodb" in mysqld_options) and version is not None and \ version >= (5, 7, 5): print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB)) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ] # From 5.7.6 version onwards, bootstrap is done via mysqld with the # --initialize-insecure option, so no need to get information about the # sql system tables that need to be loaded. if version < (5, 7, 6): system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) locations.extend([ ("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ]) if verbosity >= 3 and not quiet: print "# Location of files:" if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." fnull = open(os.devnull, 'w') # For MySQL versions before 5.7.6, use regular bootstrap procedure. if version < (5, 7, 6): # Get bootstrap SQL statements sql = list() sql.append("CREATE DATABASE mysql;") sql.append("USE mysql;") innodb_disabled = False if mysqld_options: innodb_disabled = '--innodb=OFF' in mysqld_options for sqlfile in [ system_tables, system_tables_data, test_data_timezone, help_data ]: lines = open(sqlfile, 'r').readlines() # On MySQL 5.7.5, the root@localhost account creation was # moved from the system_tables_data sql file into the # mysql_install_db binary. Since we don't use mysql_install_db # directly we need to create the root user account ourselves. if (version is not None and version == (5, 7, 5) and sqlfile == system_tables_data): lines.extend(_CREATE_ROOT_USER) for line in lines: line = line.strip() # Don't fail when InnoDB is turned off (Bug#16369955) # (Ugly hack) if (sqlfile == system_tables and "SET @sql_mode_orig==@@SES" in line and innodb_disabled): for line in lines: if 'SET SESSION sql_mode=@@sql' in line: break sql.append(line) # Bootstap to setup mysql tables cmd = [ mysqld_path, "--no-defaults", "--bootstrap", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)), ] if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) proc.communicate('\n'.join(sql)) # From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and # the --initialize option else: cmd = [ mysqld_path, "--initialize-insecure=on", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)) ] if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() # Kill subprocess just in case it didn't finish - Ok if proc doesn't exist if int(res) != 0: if os.name == "posix": try: os.kill(proc.pid, subprocess.signal.SIGTERM) except OSError: raise UtilError("Failed to kill process with pid '{0}'" "".format(proc.pid)) else: ret_code = subprocess.call("taskkill /F /T /PID " "{0}".format(proc.pid), shell=True) # return code 0 means it was successful and 128 means it tried # to kill a process that doesn't exist if ret_code not in (0, 128): raise UtilError("Failed to kill process with pid '{0}'. " "Return code {1}".format(proc.pid, ret_code)) # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." # If the user is not the same as the user running the script... # and this is a Posix system... and we are running as root if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_data]) subprocess.call(['chgrp', '-R', user, new_data]) socket_path = os.path.join(new_data, 'mysql.sock') # If socket path is too long, use mkdtemp to create a tmp dir and # use it instead to store the socket if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE: socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock') if not quiet: print( "# WARNING: The socket file path '{0}' is too long (>{1}), " "using '{2}' instead".format( os.path.join(new_data, 'mysql.sock'), MAX_SOCKET_PATH_SIZE, socket_path)) cmd = { 'datadir': '--datadir={0}'.format(new_data), 'tmpdir': '--tmpdir={0}'.format(new_data), 'pid-file': '--pid-file={0}'.format(os.path.join(new_data, "clone.pid")), 'port': '--port={0}'.format(new_port), 'server': '--server-id={0}'.format(options.get('new_id', 2)), 'basedir': '--basedir={0}'.format(mysql_basedir), 'socket': '--socket={0}'.format(socket_path), } if user: cmd.update({'user': '******'.format(user)}) if mysqld_options: if isinstance(mysqld_options, (list, tuple)): cmd.update(dict(zip(mysqld_options, mysqld_options))) else: new_opts = mysqld_options.strip(" ") # Drop the --mysqld= if new_opts.startswith("--mysqld="): new_opts = new_opts[9:] if new_opts.startswith('"') and new_opts.endswith('"'): list_ = shlex.split(new_opts.strip('"')) cmd.update(dict(zip(list_, list_))) elif new_opts.startswith("'") and new_opts.endswith("'"): list_ = shlex.split(new_opts.strip("'")) cmd.update(dict(zip(list_, list_))) # Special case where there is only 1 option elif len(new_opts.split("--")) == 1: cmd.update({mysqld_options: mysqld_options}) else: list_ = shlex.split(new_opts) cmd.update(dict(zip(list_, list_))) # set of options that must be surrounded with quotes options_to_quote = set( ["datadir", "tmpdir", "basedir", "socket", "pid-file"]) # Strip spaces from each option for key in cmd: cmd[key] = cmd[key].strip(' ') # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, 'w') comment = " Startup command generated by mysqlserverclone.\n" if os.name == 'posix' and cmd_file.endswith('.sh'): cfile.write("#!/bin/sh\n") cfile.write("#{0}".format(comment)) elif os.name == 'nt' and cmd_file.endswith('.bat'): cfile.write("REM{0}".format(comment)) else: cfile.write("#{0}".format(comment)) start_cmd_lst = [ "{0}{1}{0} --no-defaults".format(quote_char, mysqld_path) ] # build start command for key, val in cmd.iteritems(): if key in options_to_quote: val = "{0}{1}{0}".format(quote_char, val) start_cmd_lst.append(val) cfile.write("{0}\n".format(" ".join(start_cmd_lst))) cfile.close() if os.name == "nt" and verbosity >= 1: cmd.update({"console": "--console"}) start_cmd_lst = [mysqld_path, "--no-defaults"] sorted_keys = sorted(cmd.keys()) start_cmd_lst.extend([cmd[val] for val in sorted_keys]) if verbosity >= 1 and not quiet: if verbosity >= 2: print("# Startup command for new server:\n" "{0}".format(" ".join(start_cmd_lst))) proc = subprocess.Popen(start_cmd_lst, shell=False) else: proc = subprocess.Popen(start_cmd_lst, shell=False, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None if os.name == "posix": new_sock = socket_path port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock } server2_options = { 'conn_info': conn, 'role': "clone", } server2 = Server(server2_options) i = 0 while i < start_timeout: i += 1 time.sleep(1) try: server2.connect() i = start_timeout + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == start_timeout: raise UtilError("Unable to communicate with new instance. " "Process id = {0}.".format(proc.pid)) elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot'] if os.name == "posix": cmd.append("--socket={0}".format(new_sock)) else: cmd.append("--port={0}".format(int(new_port))) cmd.extend(["password", root_pass]) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command start_timeout[in] Number of seconds to wait for server to start """ new_data = os.path.abspath(options.get('new_data', None)) new_port = options.get('new_port', '3307') root_pass = options.get('root_pass', None) verbosity = options.get('verbosity', 0) user = options.get('user', 'root') quiet = options.get('quiet', False) cmd_file = options.get('cmd_file', None) start_timeout = int(options.get('start_timeout', 10)) mysqld_options = options.get('mysqld_options', '') force = options.get('force', False) quote_char = "'" if os.name == "posix" else '"' if not check_port_in_use('localhost', int(new_port)): raise UtilError("Port {0} in use. Please choose an " "available port.".format(new_port)) # Check if path to database files is greater than MAX_DIR_SIZE char, if len(new_data) > MAX_DATADIR_SIZE and not force: raise UtilError("The --new-data path '{0}' is too long " "(> {1} characters). Please use a smaller one. " "You can use the --force option to skip this " "check".format(new_data, MAX_DATADIR_SIZE)) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = { 'conn_info': conn_val, 'role': "source", } server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % \ conn_val["host"] # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = os.path.normpath(rows[0][1]) # Cloning downed or offline server else: basedir = os.path.abspath(options.get("basedir", None)) if not quiet: print "# Cloning the MySQL server located at %s." % basedir new_data_deleted = False # If datadir exists, has data, and user said it was Ok, delete it if os.path.exists(new_data) and options.get("delete", False) and \ os.listdir(new_data): new_data_deleted = True shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not os.path.exists(new_data): if not quiet: print "# Creating new data directory..." try: os.mkdir(new_data) except OSError as err: raise UtilError("Unable to create directory '{0}', reason: {1}" "".format(new_data, err.strerror)) # After create the new data directory, check for free space, so the errors # regarding invalid or inaccessible path had been dismissed already. # If not force specified verify and stop if there is not enough free space if not force and os.path.exists(new_data) and \ estimate_free_space(new_data) < REQ_FREE_SPACE: # Don't leave empty folders, delete new_data if was previously deleted if os.path.exists(new_data) and new_data_deleted: shutil.rmtree(new_data, True) raise UtilError(LOW_SPACE_ERRR_MSG.format(directory=new_data, megabytes=REQ_FREE_SPACE)) # Check for warning of using --skip-innodb mysqld_path = get_tool_path(basedir, "mysqld") version_str = get_mysqld_version(mysqld_path) # convert version_str from str tuple to integer tuple if possible if version_str is not None: version = tuple([int(digit) for digit in version_str]) else: version = None if mysqld_options is not None and ("--skip-innodb" in mysqld_options or "--innodb" in mysqld_options) and version is not None and \ version >= (5, 7, 5): print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB)) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ] # From 5.7.6 version onwards, bootstrap is done via mysqld with the # --initialize-insecure option, so no need to get information about the # sql system tables that need to be loaded. if version < (5, 7, 6): system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) locations.extend([("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ]) if verbosity >= 3 and not quiet: print "# Location of files:" if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." fnull = open(os.devnull, 'w') # For MySQL versions before 5.7.6, use regular bootstrap procedure. if version < (5, 7, 6): # Get bootstrap SQL statements sql = list() sql.append("CREATE DATABASE mysql;") sql.append("USE mysql;") innodb_disabled = False if mysqld_options: innodb_disabled = '--innodb=OFF' in mysqld_options for sqlfile in [system_tables, system_tables_data, test_data_timezone, help_data]: lines = open(sqlfile, 'r').readlines() # On MySQL 5.7.5, the root@localhost account creation was # moved from the system_tables_data sql file into the # mysql_install_db binary. Since we don't use mysql_install_db # directly we need to create the root user account ourselves. if (version is not None and version == (5, 7, 5) and sqlfile == system_tables_data): lines.extend(_CREATE_ROOT_USER) for line in lines: line = line.strip() # Don't fail when InnoDB is turned off (Bug#16369955) # (Ugly hack) if (sqlfile == system_tables and "SET @sql_mode_orig==@@SES" in line and innodb_disabled): for line in lines: if 'SET SESSION sql_mode=@@sql' in line: break sql.append(line) # Bootstap to setup mysql tables cmd = [ mysqld_path, "--no-defaults", "--bootstrap", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)), ] if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) proc.communicate('\n'.join(sql)) # From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and # the --initialize option else: cmd = [ mysqld_path, "--initialize-insecure=on", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)) ] if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() # Kill subprocess just in case it didn't finish - Ok if proc doesn't exist if int(res) != 0: if os.name == "posix": try: os.kill(proc.pid, subprocess.signal.SIGTERM) except OSError: raise UtilError("Failed to kill process with pid '{0}'" "".format(proc.pid)) else: ret_code = subprocess.call("taskkill /F /T /PID " "{0}".format(proc.pid), shell=True) # return code 0 means it was successful and 128 means it tried # to kill a process that doesn't exist if ret_code not in (0, 128): raise UtilError("Failed to kill process with pid '{0}'. " "Return code {1}".format(proc.pid, ret_code)) # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." # If the user is not the same as the user running the script... # and this is a Posix system... and we are running as root if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_data]) subprocess.call(['chgrp', '-R', user, new_data]) socket_path = os.path.join(new_data, 'mysql.sock') # If socket path is too long, use mkdtemp to create a tmp dir and # use it instead to store the socket if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE: socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock') if not quiet: print("# WARNING: The socket file path '{0}' is too long (>{1}), " "using '{2}' instead".format( os.path.join(new_data, 'mysql.sock'), MAX_SOCKET_PATH_SIZE, socket_path)) cmd = { 'datadir': '--datadir={0}'.format(new_data), 'tmpdir': '--tmpdir={0}'.format(new_data), 'pid-file': '--pid-file={0}'.format( os.path.join(new_data, "clone.pid")), 'port': '--port={0}'.format(new_port), 'server': '--server-id={0}'.format(options.get('new_id', 2)), 'basedir': '--basedir={0}'.format(mysql_basedir), 'socket': '--socket={0}'.format(socket_path), } if user: cmd.update({'user': '******'.format(user)}) if mysqld_options: if isinstance(mysqld_options, (list, tuple)): cmd.update(dict(zip(mysqld_options, mysqld_options))) else: new_opts = mysqld_options.strip(" ") # Drop the --mysqld= if new_opts.startswith("--mysqld="): new_opts = new_opts[9:] if new_opts.startswith('"') and new_opts.endswith('"'): list_ = shlex.split(new_opts.strip('"')) cmd.update(dict(zip(list_, list_))) elif new_opts.startswith("'") and new_opts.endswith("'"): list_ = shlex.split(new_opts.strip("'")) cmd.update(dict(zip(list_, list_))) # Special case where there is only 1 option elif len(new_opts.split("--")) == 1: cmd.update({mysqld_options: mysqld_options}) else: list_ = shlex.split(new_opts) cmd.update(dict(zip(list_, list_))) # set of options that must be surrounded with quotes options_to_quote = set(["datadir", "tmpdir", "basedir", "socket", "pid-file"]) # Strip spaces from each option for key in cmd: cmd[key] = cmd[key].strip(' ') # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, 'w') comment = " Startup command generated by mysqlserverclone.\n" if os.name == 'posix' and cmd_file.endswith('.sh'): cfile.write("#!/bin/sh\n") cfile.write("#{0}".format(comment)) elif os.name == 'nt' and cmd_file.endswith('.bat'): cfile.write("REM{0}".format(comment)) else: cfile.write("#{0}".format(comment)) start_cmd_lst = ["{0}{1}{0} --no-defaults".format(quote_char, mysqld_path)] # build start command for key, val in cmd.iteritems(): if key in options_to_quote: val = "{0}{1}{0}".format(quote_char, val) start_cmd_lst.append(val) cfile.write("{0}\n".format(" ".join(start_cmd_lst))) cfile.close() if os.name == "nt" and verbosity >= 1: cmd.update({"console": "--console"}) start_cmd_lst = [mysqld_path, "--no-defaults"] sorted_keys = sorted(cmd.keys()) start_cmd_lst.extend([cmd[val] for val in sorted_keys]) if verbosity >= 1 and not quiet: if verbosity >= 2: print("# Startup command for new server:\n" "{0}".format(" ".join(start_cmd_lst))) proc = subprocess.Popen(start_cmd_lst, shell=False) else: proc = subprocess.Popen(start_cmd_lst, shell=False, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None if os.name == "posix": new_sock = socket_path port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock } server2_options = { 'conn_info': conn, 'role': "clone", } server2 = Server(server2_options) i = 0 while i < start_timeout: i += 1 time.sleep(1) try: server2.connect() i = start_timeout + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == start_timeout: raise UtilError("Unable to communicate with new instance. " "Process id = {0}.".format(proc.pid)) elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot'] if os.name == "posix": cmd.append("--socket={0}".format(new_sock)) else: cmd.append("--port={0}".format(int(new_port))) cmd.extend(["password", root_pass]) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
# Get list of databases from the server if not specified in options print "# Getting databases..." db_list = [] if opt.dbs_to_copy is None: for db in server1.get_all_databases(): db_list.append((db[0], None)) else: for db in opt.dbs_to_copy.split(","): db_list.append((db, None)) # Get list of all users from the server print "# Getting users..." user_list=[] if opt.users_to_copy is None: users = server1.exec_query("SELECT user, host " "FROM mysql.user " "WHERE user != 'root' and user != ''") for user in users: user_list.append(user[0]+'@'+user[1]) else: for user in opt.users_to_copy.split(","): user_list.append(user) # Clone the server print "# Cloning server instance..." try: res = serverclone.clone_server(conn, opt.new_data, opt.new_port, opt.new_id, "root", None, False, True) except exception.UtilError, e: print "ERROR:", e.errmsg exit(1)
# Get list of databases from the server if not specified in options print "# Getting databases..." db_list = [] if opt.dbs_to_copy is None: for db in server1.get_all_databases(): db_list.append((db[0], None)) else: for db in opt.dbs_to_copy.split(","): db_list.append((db, None)) # Get list of all users from the server print "# Getting users..." user_list = [] if opt.users_to_copy is None: users = server1.exec_query("SELECT user, host " "FROM mysql.user " "WHERE user != 'root' and user != ''") for user in users: user_list.append(user[0] + '@' + user[1]) else: for user in opt.users_to_copy.split(","): user_list.append(user) # Clone the server print "# Cloning server instance..." try: res = serverclone.clone_server(conn, opt.new_data, opt.new_port, opt.new_id, "root", None, False, True) except exception.UtilError, e: print "ERROR:", e.errmsg exit(1)
class BinaryLogPurge(object): """BinaryLogPurge """ def __init__(self, server_cnx_val, options): """Initiator. server_cnx_val[in] Server connection dictionary. options[in] Options dictionary. """ self.server_cnx_val = server_cnx_val self.server = None self.options = options self.verbosity = self.options.get("verbosity", 0) self.quiet = self.options.get("quiet", False) self.logging = self.options.get("logging", False) self.dry_run = self.options.get("dry_run", 0) self.to_binlog_name = self.options.get("to_binlog_name", False) def _report(self, message, level=logging.INFO, print_msg=True): """Log message if logging is on. This method will log the message presented if the log is turned on. Specifically, if options['log_file'] is not None. It will also print the message to stdout. message[in] Message to be printed. level[in] Level of message to log. Default = INFO. print_msg[in] If True, print the message to stdout. Default = True. """ # First, print the message. if print_msg and not self.quiet: print(message) # Now log message if logging turned on if self.logging: logging.log(int(level), message.strip("#").strip(" ")) def get_target_binlog_index(self, binlog_file_name): """Retrieves the target binlog file index. Retrieves the target binlog file index that will used in the purge query, by the fault the latest log not in use unless the user specifies a different target which is validated against the server's binlog base name. binlog_file_name[in] the binlog base file name used by the server. Returns the target index binlog file """ if self.to_binlog_name: to_binlog_name = self.to_binlog_name.split('.')[0] if to_binlog_name != binlog_file_name: raise UtilError("The given binlog file name: '{0}' differs " "from the used by the server: '{1}'" "".format(to_binlog_name, binlog_file_name)) else: to_binlog_index = int(self.to_binlog_name.split('.')[1]) return to_binlog_index return None def _purge(self, index_last_in_use, active_binlog_file, binlog_file_name, target_binlog_index=None, server=None, server_is_master=False): """The inner purge method. Purges the binary logs from the given server, it will purge all of the binlogs older than the active_binlog_file ot to target_binlog_index. index_last_in_use[in] The index of the latest binary log not in use. in case of a Master, must be the latest binlog caought by all the slaves. active_binlog_file[in] Current active binlog file. binlog_file_name[in] Binlog base file name. target_binlog_index[in] The target binlog index, in case doesn't want to use the index_last_in_use by default None. server[in] Server object where to purge the binlogs from, by default self.server is used. server_is_master[in] Indicates if the given server is a Master, used for report purposes by default False. """ if server is None: server = self.server if server_is_master: server_name = "master" else: server_name = "server" # The purge_to_binlog file used to purge query based on earliest log # not in use z_len = len(active_binlog_file.split('.')[1]) purge_to_binlog = ("{0}.{1}".format( binlog_file_name, repr(index_last_in_use).zfill(z_len))) server_binlogs_list = server.get_server_binlogs_list() if self.verbosity >= 1: _report_binlogs(server_binlogs_list, self._report) # The last_binlog_not_in_use used for information purposes index_last_not_in_use = index_last_in_use - 1 last_binlog_not_in_use = ("{0}.{1}".format( binlog_file_name, repr(index_last_not_in_use).zfill(z_len))) if server_is_master: self._report("# Latest binlog file replicated by all slaves: " "{0}".format(last_binlog_not_in_use)) if target_binlog_index is None: # Purge to latest binlog not in use if self.verbosity > 0: self._report("# Latest not active binlog" " file: {0}".format(last_binlog_not_in_use)) # last_binlog_not_in_use purge(server, purge_to_binlog, server_binlogs_list, reporter=self._report, dryrun=self.dry_run, verbosity=self.verbosity) else: purge_to_binlog = ("{0}.{1}".format( binlog_file_name, repr(target_binlog_index).zfill(z_len))) if purge_to_binlog not in server_binlogs_list: self._report( _COULD_NOT_FIND_BINLOG.format(bin_name=self.to_binlog_name, server_name=server_name, host=server.host, port=server.port)) return if target_binlog_index > index_last_in_use: self._report("WARNING: The given binlog name: '{0}' is " "required for one or more slaves, the Utilitiy " "will purge to binlog '{1}' instead." "".format(self.to_binlog_name, last_binlog_not_in_use)) target_binlog_index = last_binlog_not_in_use # last_binlog_not_in_use purge(server, purge_to_binlog, server_binlogs_list, reporter=self._report, dryrun=self.dry_run, verbosity=self.verbosity) server_binlogs_list_after = server.get_server_binlogs_list() if self.verbosity >= 1: _report_binlogs(server_binlogs_list_after, self._report) for binlog in server_binlogs_list_after: if binlog in server_binlogs_list: server_binlogs_list.remove(binlog) if self.verbosity >= 1 and server_binlogs_list: _report_binlogs(server_binlogs_list, self._report, removed=True) def purge(self): """The purge method for a standalone server. Determines the latest log file to purge, which becomes the target file to purge binary logs to in case no other file is specified. """ # Connect to server self.server = Server({'conn_info': self.server_cnx_val}) self.server.connect() # Check required privileges check_privileges(self.server, BINLOG_OP_PURGE, ["SUPER", "REPLICATION SLAVE"], BINLOG_OP_PURGE_DESC, self.verbosity, self._report) # retrieve active binlog info binlog_file_name, active_binlog_file, index_last_in_use = ( get_binlog_info(self.server, reporter=self._report, server_name="server", verbosity=self.verbosity)) # Verify this server is not a Master. processes = self.server.exec_query("SHOW PROCESSLIST") binlog_dump = False for process in processes: if process[4] == "Binlog Dump": binlog_dump = True break hosts = self.server.exec_query("SHOW SLAVE HOSTS") if binlog_dump or hosts: if hosts and not self.verbosity: msg_v = " For more info use verbose option." else: msg_v = "" if self.verbosity >= 1: for host in hosts: self._report("# WARNING: Slave with id:{0} at {1}:{2} " "is connected to this server." "".format(host[0], host[1], host[2])) raise UtilError("The given server is acting as a master and has " "slaves connected to it. To proceed please use the" " --master option.{0}".format(msg_v)) target_binlog_index = self.get_target_binlog_index(binlog_file_name) self._purge(index_last_in_use, active_binlog_file, binlog_file_name, target_binlog_index)
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command start_timeout[in] Number of seconds to wait for server to start """ new_data = os.path.abspath(options.get('new_data', None)) new_port = options.get('new_port', '3307') root_pass = options.get('root_pass', None) verbosity = options.get('verbosity', 0) user = options.get('user', 'root') quiet = options.get('quiet', False) cmd_file = options.get('cmd_file', None) start_timeout = int(options.get('start_timeout', 10)) mysqld_options = options.get('mysqld_options', '') if not check_port_in_use('localhost', int(new_port)): raise UtilError("Port {0} in use. Please choose an " "available port.".format(new_port)) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = { 'conn_info': conn_val, 'role': "source", } server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % \ conn_val["host"] basedir = "" # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = os.path.normpath(rows[0][1]) # Cloning downed or offline server else: basedir = os.path.abspath(options.get("basedir", None)) if not quiet: print "# Cloning the MySQL server located at %s." % basedir # If datadir exists, has data, and user said it was Ok, delete it if os.path.exists(new_data) and options.get("delete", False) and \ os.listdir(new_data): shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not os.path.exists(new_data): if not quiet: print "# Creating new data directory..." try: os.mkdir(new_data) except: raise UtilError("Unable to create directory '%s'" % new_data) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqld_path = get_tool_path(basedir, "mysqld") mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False) mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) if verbosity >= 3 and not quiet: print "# Location of files:" locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ] if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." # Get bootstrap SQL statements sql = list() sql.append("CREATE DATABASE mysql;") sql.append("USE mysql;") innodb_disabled = False if mysqld_options: innodb_disabled = '--innodb=OFF' in mysqld_options for sqlfile in [ system_tables, system_tables_data, test_data_timezone, help_data ]: lines = open(sqlfile, 'r').readlines() for line in lines: line = line.strip() # Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack) if (sqlfile == system_tables and "SET @sql_mode_orig==@@SES" in line and innodb_disabled): for line in lines: if 'SET SESSION sql_mode=@@sql' in line: break sql.append(line) # Bootstap to setup mysql tables fnull = open(os.devnull, 'w') cmd = [ mysqld_path, "--no-defaults", "--bootstrap", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)), ] proc = None if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) proc.communicate('\n'.join(sql)) # Wait for subprocess to finish res = proc.wait() # Kill subprocess just in case it didn't finish - Ok if proc doesn't exist if int(res) != 0: if os.name == "posix": try: os.kill(proc.pid, subprocess.signal.SIGTERM) except OSError: raise UtilError("Failed to kill process with pid '{0}'" "".format(proc.pid)) else: ret_code = subprocess.call("taskkill /F /T /PID " "{0}".format(proc.pid), shell=True) # return code 0 means it was successful and 128 means it tried # to kill a process that doesn't exist if ret_code not in (0, 128): raise UtilError("Failed to kill process with pid '{0}'. " "Return code {1}".format(proc.pid, ret_code)) # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." # If the user is not the same as the user running the script... # and this is a Posix system... and we are running as root if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_data]) subprocess.call(['chgrp', '-R', user, new_data]) cmd = [mysqld_path, '--no-defaults'] cmd.extend([ '--datadir={0}'.format(new_data), '--tmpdir={0}'.format(new_data), '--pid-file={0}'.format(os.path.join(new_data, "clone.pid")), '--port={0}'.format(new_port), '--server-id={0}'.format(options.get('new_id', 2)), '--basedir={0}'.format(mysql_basedir), '--socket={0}'.format(os.path.join(new_data, 'mysql.sock')), ]) if user: cmd.append('--user={0}'.format(user)) if mysqld_options: if isinstance(mysqld_options, (list, tuple)): cmd.extend(mysqld_options) else: new_opts = mysqld_options.strip(" ") # Drop the --mysqld= if new_opts.startswith("--mysqld="): new_opts = new_opts[9:] if new_opts.startswith('"') and new_opts.endswith('"'): cmd.extend(shlex.split(new_opts.strip('"'))) elif new_opts.startswith("'") and new_opts.endswith("'"): cmd.extend(shlex.split(new_opts.strip("'"))) # Special case where there is only 1 option elif len(new_opts.split("--")) == 1: cmd.append(mysqld_options) else: cmd.extend(shlex.split(new_opts)) # Strip spaces from each option cmd = [opt.strip(' ') for opt in cmd] # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, 'w') if os.name == 'posix' and cmd_file.endswith('.sh'): cfile.write("#!/bin/sh\n") cfile.write("# Startup command generated by mysqlserverclone.\n") cfile.write("%s\n" % cmd) cfile.close() if os.name == "nt" and verbosity >= 1: cmd.append("--console") if verbosity >= 1 and not quiet: if verbosity >= 2: print("# Startup command for new server:\n" "{0}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None port_int = None if os.name == "posix": new_sock = os.path.join(new_data, "mysql.sock") port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock } server2_options = { 'conn_info': conn, 'role': "clone", } server2 = Server(server2_options) i = 0 while i < start_timeout: i += 1 time.sleep(1) try: server2.connect() i = start_timeout + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == start_timeout: raise UtilError("Unable to communicate with new instance. " "Process id = {0}.".format(proc.pid)) elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot'] if os.name == "posix": cmd.append("--socket={0}".format(new_sock)) else: cmd.append("--port={0}".format(int(new_port))) cmd.extend(["password", root_pass]) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command """ from mysql.utilities.common.server import Server from mysql.utilities.exception import UtilError from mysql.utilities.common.tools import get_tool_path new_data = options.get("new_data", None) new_port = options.get("new_port", "3307") root_pass = options.get("root_pass", None) verbosity = options.get("verbosity", 0) quiet = options.get("quiet", False) cmd_file = options.get("cmd_file", None) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = {"conn_info": conn_val, "role": "source"} server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % conn_val["host"] basedir = "" # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = rows[0][1] # Cloning downed or offline server else: basedir = options.get("basedir", None) if not quiet: print "# Cloning the MySQL server located at %s." % basedir # If datadir exists, delete it if os.path.exists(new_data): shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not quiet: print "# Creating new data directory..." if not os.path.exists(new_data): try: res = os.mkdir(new_data) except: raise UtilError("Unable to create directory '%s'" % new_data) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqld_path = get_tool_path(basedir, "mysqld") mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False) mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) if verbosity >= 3 and not quiet: print "# Location of files:" locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ] if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." # Create the bootstrap file f_boot = open("bootstrap.sql", "w") f_boot.write("CREATE DATABASE mysql;\n") f_boot.write("USE mysql;\n") f_boot.writelines(open(system_tables).readlines()) f_boot.writelines(open(system_tables_data).readlines()) f_boot.writelines(open(test_data_timezone).readlines()) f_boot.writelines(open(help_data).readlines()) f_boot.close() # Bootstap to setup mysql tables fnull = open(os.devnull, "w") cmd = ( mysqld_path + " --no-defaults --bootstrap " + " --datadir=%s --basedir=%s " % (new_data, mysql_basedir) + " < bootstrap.sql" ) proc = None if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." cmd = mysqld_path + " --no-defaults " if options.get("mysqld_options", None): cmd += options.get("mysqld_options") + " --user=root " cmd += "--datadir=%s " % (new_data) cmd += "--tmpdir=%s " % (new_data) cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid") cmd += "--port=%s " % (new_port) cmd += "--server-id=%s " % (options.get("new_id", 2)) cmd += "--basedir=%s " % (mysql_basedir) cmd += "--socket=%s/mysql.sock " % (new_data) # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, "w") if os.name == "posix" and cmd_file.endswith(".sh"): cfile.write("#!/bin/sh\n") cfile.write("# Startup command generated by mysqlserverclone.\n") cfile.write("%s\n" % cmd) cfile.close() if verbosity >= 1 and not quiet: if verbosity >= 2: print "# Startup command for new server:\n%s" % cmd proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None port_int = None if os.name == "posix": new_sock = os.path.join(new_data, "mysql.sock") port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock, } server2_options = {"conn_info": conn, "role": "clone"} server2 = Server(server2_options) stop = 10 # stop after 10 attempts i = 0 while i < stop: i += 1 time.sleep(1) try: server2.connect() i = stop + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == stop: raise UtilError("Unable to communicate with new instance.") elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." if os.name == "posix": cmd = mysqladmin_path + " --no-defaults -v -uroot " + "--socket=%s password %s " % (new_sock, root_pass) else: cmd = mysqladmin_path + " --no-defaults -v -uroot " + "password %s --port=%s" % (root_pass, int(new_port)) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command """ from mysql.utilities.common.server import Server from mysql.utilities.exception import UtilError from mysql.utilities.common.tools import get_tool_path new_data = options.get('new_data', None) new_port = options.get('new_port', '3307') root_pass = options.get('root_pass', None) verbosity = options.get('verbosity', 0) quiet = options.get('quiet', False) cmd_file = options.get('cmd_file', None) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = { 'conn_info': conn_val, 'role': "source", } server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % conn_val["host"] basedir = "" # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = rows[0][1] # Cloning downed or offline server else: basedir = options.get("basedir", None) if not quiet: print "# Cloning the MySQL server located at %s." % basedir # If datadir exists, has data, and user said it was Ok, delete it if os.path.exists(new_data) and options.get("delete", False) and \ os.listdir(new_data): shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not os.path.exists(new_data): if not quiet: print "# Creating new data directory..." try: res = os.mkdir(new_data) except: raise UtilError("Unable to create directory '%s'" % new_data) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqld_path = get_tool_path(basedir, "mysqld") mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False) mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) if verbosity >= 3 and not quiet: print "# Location of files:" locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ] if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." # Create the bootstrap file f_boot = open("bootstrap.sql", 'w') f_boot.write("CREATE DATABASE mysql;\n") f_boot.write("USE mysql;\n") f_boot.writelines(open(system_tables).readlines()) f_boot.writelines(open(system_tables_data).readlines()) f_boot.writelines(open(test_data_timezone).readlines()) f_boot.writelines(open(help_data).readlines()) f_boot.close() # Bootstap to setup mysql tables fnull = open(os.devnull, 'w') cmd = mysqld_path + " --no-defaults --bootstrap " + \ " --datadir=%s --basedir=%s " % (new_data, mysql_basedir) + \ " < bootstrap.sql" proc = None if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() # Kill subprocess just in case it didn't finish - Ok if proc doesn't exist if int(res) != 0: if os.name == "posix": try: os.kill(proc.pid, subprocess.signal.SIGTERM) except OSError: pass else: try: retval = subprocess.Popen("taskkill /F /T /PID %i" % proc.pid, shell=True) except: pass # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." cmd = mysqld_path + " --no-defaults " if options.get('mysqld_options', None): cmd += options.get('mysqld_options') + " --user=root " cmd += "--datadir=%s " % (new_data) cmd += "--tmpdir=%s " % (new_data) cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid") cmd += "--port=%s " % (new_port) cmd += "--server-id=%s " % (options.get('new_id', 2)) cmd += "--basedir=%s " % (mysql_basedir) cmd += "--socket=%s/mysql.sock " % (new_data) # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, 'w') if os.name == 'posix' and cmd_file.endswith('.sh'): cfile.write("#!/bin/sh\n") cfile.write("# Startup command generated by mysqlserverclone.\n") cfile.write("%s\n" % cmd) cfile.close() if verbosity >= 1 and not quiet: if verbosity >= 2: print "# Startup command for new server:\n%s" % cmd proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None port_int = None if os.name == "posix": new_sock = os.path.join(new_data, "mysql.sock") port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock } server2_options = { 'conn_info': conn, 'role': "clone", } server2 = Server(server2_options) stop = 10 # stop after 10 attempts i = 0 while i < stop: i += 1 time.sleep(1) try: server2.connect() i = stop + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == stop: raise UtilError("Unable to communicate with new instance.") elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." if os.name == "posix": cmd = mysqladmin_path + " --no-defaults -v -uroot " + \ "--socket=%s password %s " % (new_sock, root_pass) else: cmd = mysqladmin_path + " --no-defaults -v -uroot " + \ "password %s --port=%s" % (root_pass, int(new_port)) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=True) else: proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
def clone_server(conn_val, options): """Clone an existing server This method creates a new instance of a running server using a datadir set to the new_data parametr, with a port set to new_port, server_id set to new_id and a root password of root_pass. You can also specify additional parameters for the mysqld command line as well as turn on verbosity mode to display more diagnostic information during the clone process. The method will build a new base database installation from the .sql files used to construct a new installation. Once the database is created, the server will be started. dest_val[in] a dictionary containing connection information including: (user, password, host, port, socket) options[in] dictionary of options: new_data[in] An existing path to create the new database and use as datadir for new instance (default = None) new_port[in] Port number for new instance (default = 3307) new_id[in] Server_id for new instance (default = 2) root_pass[in] Password for root user on new instance (optional) mysqld_options[in] Additional command line options for mysqld verbosity[in] Print additional information during operation (default is 0) quiet[in] If True, do not print messages. (default is False) cmd_file[in] file name to write startup command start_timeout[in] Number of seconds to wait for server to start """ new_data = os.path.abspath(options.get('new_data', None)) new_port = options.get('new_port', '3307') root_pass = options.get('root_pass', None) verbosity = options.get('verbosity', 0) user = options.get('user', 'root') quiet = options.get('quiet', False) cmd_file = options.get('cmd_file', None) start_timeout = int(options.get('start_timeout', 10)) mysqld_options = options.get('mysqld_options', '') if not check_port_in_use('localhost', int(new_port)): raise UtilError("Port {0} in use. Please choose an " "available port.".format(new_port)) # Clone running server if conn_val is not None: # Try to connect to the MySQL database server. server1_options = { 'conn_info': conn_val, 'role': "source", } server1 = Server(server1_options) server1.connect() if not quiet: print "# Cloning the MySQL server running on %s." % \ conn_val["host"] basedir = "" # Get basedir rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = os.path.normpath(rows[0][1]) # Cloning downed or offline server else: basedir = os.path.abspath(options.get("basedir", None)) if not quiet: print "# Cloning the MySQL server located at %s." % basedir # If datadir exists, has data, and user said it was Ok, delete it if os.path.exists(new_data) and options.get("delete", False) and \ os.listdir(new_data): shutil.rmtree(new_data, True) # Create new data directory if it does not exist if not os.path.exists(new_data): if not quiet: print "# Creating new data directory..." try: os.mkdir(new_data) except: raise UtilError("Unable to create directory '%s'" % new_data) if not quiet: print "# Configuring new instance..." print "# Locating mysql tools..." mysqld_path = get_tool_path(basedir, "mysqld") mysqladmin_path = get_tool_path(basedir, "mysqladmin") mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False) mysql_basedir = basedir if os.path.exists(os.path.join(basedir, "local/mysql/share/")): mysql_basedir = os.path.join(mysql_basedir, "local/mysql/") # for source trees elif os.path.exists(os.path.join(basedir, "/sql/share/english/")): mysql_basedir = os.path.join(mysql_basedir, "/sql/") system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False) system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False) test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False) help_data = get_tool_path(basedir, "fill_help_tables.sql", False) if verbosity >= 3 and not quiet: print "# Location of files:" locations = [ ("mysqld", mysqld_path), ("mysqladmin", mysqladmin_path), ("mysql_system_tables.sql", system_tables), ("mysql_system_tables_data.sql", system_tables_data), ("mysql_test_data_timezone.sql", test_data_timezone), ("fill_help_tables.sql", help_data), ] if cmd_file is not None: locations.append(("write startup command to", cmd_file)) for location in locations: print "# % 28s: %s" % location # Create the new mysql data with mysql_import_db-like process if not quiet: print "# Setting up empty database and mysql tables..." # Get bootstrap SQL statements sql = list() sql.append("CREATE DATABASE mysql;") sql.append("USE mysql;") innodb_disabled = False if mysqld_options: innodb_disabled = '--innodb=OFF' in mysqld_options for sqlfile in [system_tables, system_tables_data, test_data_timezone, help_data]: lines = open(sqlfile, 'r').readlines() for line in lines: line = line.strip() # Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack) if (sqlfile == system_tables and "SET @sql_mode_orig==@@SES" in line and innodb_disabled): for line in lines: if 'SET SESSION sql_mode=@@sql' in line: break sql.append(line) # Bootstap to setup mysql tables fnull = open(os.devnull, 'w') cmd = [ mysqld_path, "--no-defaults", "--bootstrap", "--datadir={0}".format(new_data), "--basedir={0}".format(os.path.abspath(mysql_basedir)), ] proc = None if verbosity >= 1 and not quiet: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE) else: proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=fnull, stderr=fnull) proc.communicate('\n'.join(sql)) # Wait for subprocess to finish res = proc.wait() # Kill subprocess just in case it didn't finish - Ok if proc doesn't exist if int(res) != 0: if os.name == "posix": try: os.kill(proc.pid, subprocess.signal.SIGTERM) except OSError: raise UtilError("Failed to kill process with pid '{0}'" "".format(proc.pid)) else: ret_code = subprocess.call("taskkill /F /T /PID " "{0}".format(proc.pid), shell=True) # return code 0 means it was successful and 128 means it tried # to kill a process that doesn't exist if ret_code not in (0, 128): raise UtilError("Failed to kill process with pid '{0}'. " "Return code {1}".format(proc.pid, ret_code)) # Drop the bootstrap file if os.path.isfile("bootstrap.sql"): os.unlink("bootstrap.sql") # Start the instance if not quiet: print "# Starting new instance of the server..." # If the user is not the same as the user running the script... # and this is a Posix system... and we are running as root if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_data]) subprocess.call(['chgrp', '-R', user, new_data]) cmd = [mysqld_path, '--no-defaults'] cmd.extend([ '--datadir={0}'.format(new_data), '--tmpdir={0}'.format(new_data), '--pid-file={0}'.format(os.path.join(new_data, "clone.pid")), '--port={0}'.format(new_port), '--server-id={0}'.format(options.get('new_id', 2)), '--basedir={0}'.format(mysql_basedir), '--socket={0}'.format(os.path.join(new_data, 'mysql.sock')), ]) if user: cmd.append('--user={0}'.format(user)) if mysqld_options: if isinstance(mysqld_options, (list, tuple)): cmd.extend(mysqld_options) else: new_opts = mysqld_options.strip(" ") # Drop the --mysqld= if new_opts.startswith("--mysqld="): new_opts = new_opts[9:] if new_opts.startswith('"') and new_opts.endswith('"'): cmd.extend(shlex.split(new_opts.strip('"'))) elif new_opts.startswith("'") and new_opts.endswith("'"): cmd.extend(shlex.split(new_opts.strip("'"))) # Special case where there is only 1 option elif len(new_opts.split("--")) == 1: cmd.append(mysqld_options) else: cmd.extend(shlex.split(new_opts)) # Strip spaces from each option cmd = [opt.strip(' ') for opt in cmd] # Write startup command if specified if cmd_file is not None: if verbosity >= 0 and not quiet: print "# Writing startup command to file." cfile = open(cmd_file, 'w') if os.name == 'posix' and cmd_file.endswith('.sh'): cfile.write("#!/bin/sh\n") cfile.write("# Startup command generated by mysqlserverclone.\n") cfile.write("%s\n" % cmd) cfile.close() if os.name == "nt" and verbosity >= 1: cmd.append("--console") if verbosity >= 1 and not quiet: if verbosity >= 2: print("# Startup command for new server:\n" "{0}".format(" ".join(cmd))) proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Try to connect to the new MySQL instance if not quiet: print "# Testing connection to new instance..." new_sock = None port_int = None if os.name == "posix": new_sock = os.path.join(new_data, "mysql.sock") port_int = int(new_port) conn = { "user": "******", "passwd": "", "host": conn_val["host"] if conn_val is not None else "localhost", "port": port_int, "unix_socket": new_sock } server2_options = { 'conn_info': conn, 'role': "clone", } server2 = Server(server2_options) i = 0 while i < start_timeout: i += 1 time.sleep(1) try: server2.connect() i = start_timeout + 1 except: pass finally: if verbosity >= 1 and not quiet: print "# trying again..." if i == start_timeout: raise UtilError("Unable to communicate with new instance. " "Process id = {0}.".format(proc.pid)) elif not quiet: print "# Success!" # Set the root password if root_pass: if not quiet: print "# Setting the root password..." cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot'] if os.name == "posix": cmd.append("--socket={0}".format(new_sock)) else: cmd.append("--port={0}".format(int(new_port))) cmd.extend(["password", root_pass]) if verbosity > 0 and not quiet: proc = subprocess.Popen(cmd, shell=False) else: proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull) # Wait for subprocess to finish res = proc.wait() if not quiet: conn_str = "# Connection Information:\n" conn_str += "# -uroot" if root_pass: conn_str += " -p%s" % root_pass if os.name == "posix": conn_str += " --socket=%s" % new_sock else: conn_str += " --port=%s" % new_port print conn_str print "#...done." fnull.close()
# MUT. if "unix_socket" in conn_val: conn_val.pop("unix_socket") server_options = { 'conn_info': conn_val, 'role': "server{0}".format(i), } conn = Server(server_options) try: conn.connect() server_list.add_new_server(conn) print("CONNECTED") res = conn.show_server_variable("basedir") basedir = res[0][1] res = conn.exec_query("SELECT @@version") print(" MySQL Version: {0}".format(res[0][0])) # Here we capture any exception and print the error message. # Since all util errors (exceptions) derive from Exception, this is # safe. except: _, err, _ = sys.exc_info() print("{0}FAILED{1}".format(BOLD_ON, BOLD_OFF)) if conn.connect_error is not None: print(conn.connect_error) print("ERROR: {0!s}".format(err)) if server_list.num_servers() == 0: print("ERROR: Failed to connect to any servers listed.") sys.exit(1) # Check for running servers