def backup_everything(run_type, twindb_config, binlogs_only=False): """ Run backup job :param run_type: hourly, daily, etc :type run_type: str :param twindb_config: ConfigParser instance :type twindb_config: TwinDBBackupConfig :param binlogs_only: If True copy only MySQL binary logs. :type binlogs_only: bool """ set_open_files_limit() try: if not binlogs_only: backup_start = time.time() backup_files(run_type, twindb_config) backup_mysql(run_type, twindb_config) backup_binlogs(run_type, twindb_config) end = time.time() save_measures(backup_start, end) else: backup_binlogs(run_type, twindb_config) except ConfigParser.NoSectionError as err: LOG.debug(traceback.format_exc()) LOG.error(err) exit(1)
def get_connection(self): """ Connect to MySQL host and yield a connection. :return: MySQL connection :raise MySQLSourceError: if can't connect to server """ connection = None try: connection = pymysql.connect( host=self.hostname, read_default_file=self.defaults_file, connect_timeout=self.connect_timeout, cursorclass=pymysql.cursors.DictCursor ) yield connection except OperationalError: LOG.error( "Can't connect to MySQL server on %s", self.hostname) raise MySQLSourceError( "Can't connect to MySQL server on %s" % self.hostname) finally: if connection: connection.close()
def disable_wsrep_desync(self): """ Wait till wsrep_local_recv_queue is zero and disable wsrep_local_recv_queue then """ max_time = time.time() + 900 try: with self.get_connection() as connection: with connection.cursor() as cursor: while time.time() < max_time: cursor.execute("SHOW GLOBAL STATUS LIKE " "'wsrep_local_recv_queue'") res = {r['Variable_name'].lower(): r['Value'].lower() for r in cursor.fetchall()} if not res.get('wsrep_local_recv_queue'): raise Exception('Unknown status variable ' '"wsrep_local_recv_queue"') if int(res['wsrep_local_recv_queue']) == 0: break time.sleep(1) LOG.debug('Disabling wsrep_desync') cursor.execute("SET GLOBAL wsrep_desync=OFF") except pymysql.Error as err: LOG.error(err)
def revert_stream(self): """ Un-Apply modifier and return output stream. The Base modifier does nothing, so it will return the input stream without modifications :return: output stream handle """ with self._input as input_stream: LOG.debug('Running %s', ' '.join(self._unmodifier_cmd)) proc = Popen( self._unmodifier_cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE ) yield proc.stdout _, cerr = proc.communicate() if proc.returncode: msg = '%s exited with non-zero code.' \ % ' '.join(self._unmodifier_cmd) LOG.error(msg) LOG.error(cerr) raise ModifierException(msg)
def _handle_failure_exec(self, err, stderr_file): """Cleanup on failure exec""" LOG.error(err) LOG.error('Failed to run xtrabackup. ' 'Check error output in %s', stderr_file.name) self.dst.delete(self.get_name()) exit(1)
def get_stream(self): """ Get a PIPE handler with content of the source :return: """ cmd = [ self._xtrabackup, "--defaults-file=%s" % self._connect_info.defaults_file, "--stream=xbstream", "--host=127.0.0.1", "--backup" ] cmd += ["--target-dir", "."] if self.is_galera(): cmd.append("--galera-info") cmd.append("--no-backup-locks") if self.incremental: cmd += [ "--incremental-basedir", ".", "--incremental-lsn=%d" % self._parent_lsn ] # If this is a Galera node then additional step needs to be taken to # prevent the backups from locking up the cluster. wsrep_desynced = False LOG.debug('Running %s', ' '.join(cmd)) stderr_file = tempfile.NamedTemporaryFile(delete=False) try: if self.is_galera(): wsrep_desynced = self.enable_wsrep_desync() LOG.debug('Running %s', ' '.join(cmd)) proc_xtrabackup = Popen(cmd, stderr=stderr_file, stdout=PIPE) yield proc_xtrabackup.stdout proc_xtrabackup.communicate() if proc_xtrabackup.returncode: LOG.error('Failed to run xtrabackup. ' 'Check error output in %s', stderr_file.name) try: if LOG.debug_enabled: with open(stderr_file.name) as xb_out: for line in xb_out: print(line, end='', file=sys.stderr) except AttributeError: pass self.dst.delete(self.get_name()) exit(1) else: LOG.debug('Successfully streamed xtrabackup output') self._update_backup_info(stderr_file) except OSError as err: LOG.error('Failed to run %s: %s', cmd, err) exit(1) finally: if wsrep_desynced: self.disable_wsrep_desync()
def backup_mysql(self): """FLag to backup MySQL or not""" try: return self.__cfg.getboolean('source', 'backup_mysql') except NoOptionError: return False except NoSectionError as err: LOG.error("Section 'source' is mandatory") raise ConfigurationError(err)
def restore_mysql(ctx, dst, backup_copy, cache): """Restore from mysql backup""" LOG.debug('mysql: %r', ctx.obj['twindb_config']) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(ctx.obj['twindb_config']) exit(1) try: ensure_empty(dst) incomplete_copy = MySQLCopy( path=backup_copy ) dst_storage = ctx.obj['twindb_config'].destination( backup_source=incomplete_copy.host ) mysql_status = MySQLStatus(dst=dst_storage) copies = [ cp for cp in mysql_status if backup_copy.endswith(cp.name) ] try: copy = copies.pop(0) except IndexError: raise TwinDBBackupError( 'Can not find copy %s in MySQL status. ' 'Inspect output of `twindb-backup status` and verify ' 'that correct copy is specified.' % backup_copy ) if copies: raise TwinDBBackupError( 'Multiple copies match pattern %s. Make sure you give unique ' 'copy name for restore.' ) if cache: restore_from_mysql( ctx.obj['twindb_config'], copy, dst, cache=Cache(cache) ) else: restore_from_mysql(ctx.obj['twindb_config'], copy, dst) except (TwinDBBackupError, CacheException) as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except (OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1)
def backup_dirs(self): """Directories to backup""" try: dirs = self.__cfg.get('source', 'backup_dirs') return split(dirs) except NoOptionError: return [] except NoSectionError as err: LOG.error("Section 'source' is mandatory") raise ConfigurationError(err)
def verify_mysql_backup(twindb_config, dst_path, backup_file, hostname=None): """ Restore mysql backup and measure time :param hostname: :param backup_file: :param dst_path: :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig """ dst = twindb_config.destination(backup_source=hostname) status = MySQLStatus(dst=dst) copy = None if backup_file == "latest": copy = status.latest_backup else: for copy in status: if backup_file.endswith(copy.key): break if copy is None: return json.dumps({ 'backup_copy': backup_file, 'restore_time': 0, 'success': False }, indent=4, sort_keys=True) start_restore_time = time.time() success = True tmp_dir = tempfile.mkdtemp() try: LOG.debug('Verifying backup copy in %s', tmp_dir) restore_from_mysql(twindb_config, copy, dst_path, tmp_dir) edit_backup_my_cnf(dst_path) except (TwinDBBackupError, OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) success = False finally: shutil.rmtree(tmp_dir, ignore_errors=True) end_restore_time = time.time() restore_time = end_restore_time - start_restore_time return json.dumps({ 'backup_copy': copy.key, 'restore_time': restore_time, 'success': success }, indent=4, sort_keys=True)
def share_backup(ctx, s3_url): """Share backup copy for download""" if not s3_url: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(ctx.obj['twindb_config']) exit(1) try: share(ctx.obj['twindb_config'], s3_url) except TwinDBBackupError as err: LOG.error(err) exit(1)
def run_command(command, ok_non_zero=False): """ Run shell command locally :param command: Command to run :type command: list :param ok_non_zero: Don't consider non-zero exit code as an error. :type ok_non_zero: bool :return: file object with stdout as generator to use with ``with`` """ try: LOG.debug('Running %s', " ".join(command)) proc = Popen(command, stderr=PIPE, stdout=PIPE) yield proc.stdout _, cerr = proc.communicate() if proc.returncode and not ok_non_zero: LOG.error('Command %s exited with error code %d', ' '.join(command), proc.returncode) LOG.error(cerr) exit(1) else: LOG.debug('Exited with zero code') except OSError as err: LOG.error('Failed to run %s', ' '.join(command)) LOG.error(err) exit(1)
def _load(self, status_as_json): status = [] try: status_as_obj = json.loads(status_as_json) except ValueError: raise CorruptedStatus( 'Could not load status from a bad JSON string %s' % (status_as_json, ) ) for run_type in INTERVALS: for key, value in status_as_obj[run_type].iteritems(): try: host = key.split('/')[0] file_name = key.split('/')[3] kwargs = { 'type': value['type'], 'config': self.__serialize_config(value) } keys = [ 'backup_started', 'backup_finished', 'binlog', 'parent', 'lsn', 'position', 'wsrep_provider_version', ] for copy_key in keys: if copy_key in value: kwargs[copy_key] = value[copy_key] copy = MySQLCopy( host, run_type, file_name, **kwargs ) status.append(copy) except IndexError as err: LOG.error(err) raise CorruptedStatus('Unexpected key %s' % key) return status
def _get_config(self, cfg_path): """ Return parsed config :param cfg_path: Path to config :type cfg_path: str :return: Path and config :rtype: ConfigParser.ConfigParser """ cfg = ConfigParser.ConfigParser(allow_no_value=True) try: cmd = "cat %s" % cfg_path with self._ssh_client.get_remote_handlers(cmd) as (_, cout, _): cfg.readfp(cout) except ConfigParser.ParsingError as err: LOG.error(err) raise return cfg
def find_files(self, prefix, run_type): """ Find files with common prefix and given run type. :param prefix: Common prefix. :type prefix: str :param run_type: daily, hourly, etc :type run_type: str :return: list of file names :rtype: list(str) :raise S3DestinationError: if failed to find files. """ s3client = boto3.resource('s3') bucket = s3client.Bucket(self.bucket) LOG.debug('Listing %s in bucket %s', prefix, bucket) # Try to list the bucket several times # because of intermittent error NoSuchBucket: # https://travis-ci.org/twindb/backup/jobs/204066704 retry_timeout = time.time() + S3_READ_TIMEOUT retry_interval = 2 while time.time() < retry_timeout: try: files = [] all_objects = bucket.objects.filter(Prefix='') for file_object in all_objects: if "/" + run_type + "/" in file_object.key: files.append("s3://%s/%s" % (self.bucket, file_object.key)) return sorted(files) except ClientError as err: LOG.warning('%s. Will retry in %d seconds.', err, retry_interval) time.sleep(retry_interval) retry_interval *= 2 except Exception as err: LOG.error('Failed to list objects in bucket %s: %s', self.bucket, err) raise raise S3DestinationError('Failed to find files.')
def backup(ctx, run_type, lock_file, binlogs_only): """Run backup job""" try: run_backup_job( ctx.obj['twindb_config'], run_type, lock_file=lock_file, binlogs_only=binlogs_only ) except TwinDBBackupError as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def get_remote_handlers(self, cmd): """ Get remote stdin, stdout and stderr handler :param cmd: Command for execution :type cmd: str :return: Remote stdin, stdout and stderr handler :rtype: tuple(generator, generator, generator) :raise SshDestinationError: if any error """ try: with self._shell() as shell: LOG.debug("Try to get remote handlers: %s", cmd) stdin_, stdout_, stderr_ = shell.exec_command(cmd) yield stdin_, stdout_, stderr_ except SSHException as err: LOG.error('Failed to execute %s', cmd) raise SshClientException(err)
def _run_remote_netcat( compress, datadir, # pylint: disable=too-many-arguments destination, dst, netcat_port, src, xbstream_path, ): netcat_cmd = "{xbstream_binary} -x -C {datadir}".format( xbstream_binary=xbstream_path, datadir=datadir) if compress: netcat_cmd = "gunzip -c - | %s" % netcat_cmd # find unused port while netcat_port < 64000: if dst.ensure_tcp_port_listening(netcat_port, wait_timeout=1): netcat_port += 1 else: LOG.debug("Will use port %d for streaming", netcat_port) break proc_netcat = Process(target=dst.netcat, args=(netcat_cmd, ), kwargs={"port": netcat_port}) LOG.debug("Starting netcat on the destination") proc_netcat.start() nc_wait_timeout = 10 if not dst.ensure_tcp_port_listening(netcat_port, wait_timeout=nc_wait_timeout): LOG.error( "netcat on the destination " "is not ready after %d seconds", nc_wait_timeout, ) proc_netcat.terminate() exit(1) src.clone( dest_host=split_host_port(destination)[0], port=netcat_port, compress=compress, ) proc_netcat.join()
def restore_file(ctx, dst, backup_copy): """Restore from file backup""" LOG.debug('file: %r', ctx.obj['twindb_config']) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(ctx.obj['twindb_config']) exit(1) try: ensure_empty(dst) copy = FileCopy(path=backup_copy) restore_from_file(ctx.obj['twindb_config'], copy, dst) except TwinDBBackupError as err: LOG.error(err) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def backup_everything(run_type, config): """ Run backup job :param run_type: hourly, daily, etc :type run_type: str :param config: ConfigParser instance :type config: ConfigParser.ConfigParser """ set_open_files_limit() try: backup_start = time.time() backup_files(run_type, config) backup_mysql(run_type, config) end = time.time() save_measures(backup_start, end) except ConfigParser.NoSectionError as err: LOG.error(err) exit(1)
def get_stream(self): """ Encrypt the input stream and return it as the output stream :return: output stream handle :raise: OSError if failed to call the gpg command """ with self.input as input_stream: proc = Popen([ 'gpg', '--no-default-keyring', '--trust-model', 'always', '--keyring', self.keyring, '--recipient', self.recipient, '--encrypt', '--yes', '--batch' ], stdin=input_stream, stdout=PIPE, stderr=PIPE) yield proc.stdout cerr = proc.communicate() if proc.returncode: LOG.error('gpg exited with non-zero code.') LOG.error(cerr)
def _extract_xbstream(input_stream, working_dir, xbstream_binary=XBSTREAM_BINARY): """ Extract xbstream stream in directory :param input_stream: The stream in xbstream format :param working_dir: directory :param xbstream_binary: Path to xbstream :return: True if extracted successfully """ try: cmd = [xbstream_binary, "-x"] LOG.debug("Running %s", " ".join(cmd)) LOG.debug("Working directory: %s", working_dir) LOG.debug("Xbstream binary: %s", xbstream_binary) proc = Popen(cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE, cwd=working_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error("%s exited with code %d", " ".join(cmd), ret) if cout: LOG.error("STDOUT: %s", cout) if cerr: LOG.error("STDERR: %s", cerr) return ret == 0 except OSError as err: raise TwinDBBackupError("Failed to extract xbstream: %s" % err)
def _load(self, status_as_json): status = [] try: status_as_obj = json.loads(status_as_json) except ValueError: raise CorruptedStatus( "Could not load status from a bad JSON string %s" % (status_as_json,) ) for run_type in INTERVALS: for key, value in status_as_obj[run_type].items(): try: host = key.split("/")[0] file_name = key.split("/")[3] kwargs = { "type": value["type"], "config": self.__serialize_config(value), } keys = [ "backup_started", "backup_finished", "binlog", "parent", "lsn", "position", "wsrep_provider_version", ] for copy_key in keys: if copy_key in value: kwargs[copy_key] = value[copy_key] copy = MySQLCopy(host, run_type, file_name, **kwargs) status.append(copy) except IndexError as err: LOG.error(err) raise CorruptedStatus("Unexpected key %s" % key) return status
def execute(self, cmd, quiet=False): """Execute a command on a remote SSH server. :param cmd: Command for execution. :type cmd: str :param quiet: if quiet is True don't print error messages :return: Handlers of stdin, stdout and stderr :rtype: tuple :raise SshDestinationError: if any error """ try: with self._shell() as shell: LOG.debug('Executing %s', cmd) stdin_, stdout_, stderr_ = shell.exec_command(cmd) # while not stdout_.channel.exit_status_ready(): # LOG.debug('%s: waiting', cmd) # time.sleep(1) exit_code = stdout_.channel.recv_exit_status() if exit_code != 0: if not quiet: LOG.error("Failed while execute command %s", cmd) LOG.error(stderr_.read()) raise SshClientException('%s exited with code %d' % (cmd, exit_code)) return stdin_, stdout_, stderr_ except (SSHException, IOError) as err: if not quiet: LOG.error('Failed to execute %s: %s', cmd, err) raise SshClientException('Failed to execute %s: %s' % (cmd, err))
def _load(self, status_as_json): status = [] try: status_as_obj = json.loads(status_as_json) except ValueError: raise CorruptedStatus( 'Could not load status from a bad JSON string %s' % (status_as_json, )) for run_type in INTERVALS: for key, value in status_as_obj[run_type].iteritems(): try: host = key.split('/')[0] file_name = key.split('/')[3] kwargs = { 'type': value['type'], 'config': self.__serialize_config(value) } keys = [ 'backup_started', 'backup_finished', 'binlog', 'parent', 'lsn', 'position', 'wsrep_provider_version', ] for copy_key in keys: if copy_key in value: kwargs[copy_key] = value[copy_key] copy = MySQLCopy(host, run_type, file_name, **kwargs) status.append(copy) except IndexError as err: LOG.error(err) raise CorruptedStatus('Unexpected key %s' % key) return status
def restore_mysql(cfg, dst, backup_copy, cache): """Restore from mysql backup""" LOG.debug('mysql: %r', cfg) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(cfg) exit(1) try: ensure_empty(dst) if cache: restore_from_mysql(cfg, backup_copy, dst, cache=Cache(cache)) else: restore_from_mysql(cfg, backup_copy, dst) except (TwinDBBackupError, CacheException) as err: LOG.error(err) exit(1) except (OSError, IOError) as err: LOG.error(err) exit(1)
def restore_from_file(config, backup_copy, dst_dir): """ Restore a directory from a backup copy in the directory :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param backup_copy: Backup name. :type backup_copy: str :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str """ LOG.info('Restoring %s in %s', backup_copy, dst_dir) mkdir_p(dst_dir) if os.path.exists(backup_copy): dst = Local(backup_copy) stream = dst.get_stream(backup_copy) else: dst = get_destination(config) stream = dst.get_stream(backup_copy) # GPG modifier try: gpg = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring'), secret_keyring=config.get('gpg', 'secret_keyring')) LOG.debug('Decrypting stream') stream = gpg.revert_stream() except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): LOG.debug('Not decrypting the stream') with stream as handler: try: cmd = ["tar", "zvxf", "-"] LOG.debug('Running %s', ' '.join(cmd)) proc = Popen(cmd, stdin=handler, cwd=dst_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', cmd, ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return LOG.info('Successfully restored %s in %s', backup_copy, dst_dir) except OSError as err: LOG.error('Failed to decompress %s: %s', backup_copy, err) exit(1)
def _run_remote_netcat(compress, datadir, # pylint: disable=too-many-arguments destination, dst, netcat_port, src, xbstream_path): netcat_cmd = "{xbstream_binary} -x -C {datadir}".format( xbstream_binary=xbstream_path, datadir=datadir ) if compress: netcat_cmd = "gunzip -c - | %s" % netcat_cmd # find unused port while netcat_port < 64000: if dst.ensure_tcp_port_listening(netcat_port, wait_timeout=1): netcat_port += 1 else: LOG.debug('Will use port %d for streaming', netcat_port) break proc_netcat = Process( target=dst.netcat, args=(netcat_cmd,), kwargs={ 'port': netcat_port } ) LOG.debug('Starting netcat on the destination') proc_netcat.start() nc_wait_timeout = 10 if not dst.ensure_tcp_port_listening(netcat_port, wait_timeout=nc_wait_timeout): LOG.error('netcat on the destination ' 'is not ready after %d seconds', nc_wait_timeout) proc_netcat.terminate() exit(1) src.clone( dest_host=split_host_port(destination)[0], port=netcat_port, compress=compress ) proc_netcat.join()
def run_backup_job(twindb_config, run_type, lock_file=LOCK_FILE, binlogs_only=False): """ Grab a lock waiting up to allowed timeout and start backup jobs :param twindb_config: Tool configuration :type twindb_config: TwinDBBackupConfig :param run_type: Run type :type run_type: str :param lock_file: File used as a lock :type lock_file: str :param binlogs_only: If True copy only binlogs. :type binlogs_only: bool """ with timeout(get_timeout(run_type)): try: file_desriptor = open(lock_file, 'w') fcntl.flock(file_desriptor, fcntl.LOCK_EX) LOG.debug(run_type) if getattr(twindb_config.run_intervals, run_type): backup_everything( run_type, twindb_config, binlogs_only=binlogs_only ) else: LOG.debug('Not running because run_%s is no', run_type) except IOError as err: if err.errno != errno.EINTR: LOG.debug(traceback.format_exc()) raise LockWaitTimeoutError(err) msg = 'Another instance of twindb-backup is running?' if run_type == 'hourly': LOG.debug(msg) else: LOG.error(msg)
def is_galera(self): """Check if local MySQL instance is a Galera cluster :return: True if it's a Galera. :rtype: bool """ try: with self._cursor() as cursor: cursor.execute("SELECT @@wsrep_on as wsrep_on") row = cursor.fetchone() return (str(row['wsrep_on']).lower() == "1" or str(row['wsrep_on']).lower() == 'on') except pymysql.InternalError as err: error_code = err.args[0] error_message = err.args[1] if error_code == 1193: LOG.debug('Galera is not supported or not enabled') return False else: LOG.error(error_message) raise
def _save(cmd, handler): with handler as input_handler: LOG.debug('Running %s', ' '.join(cmd)) try: proc = Popen(cmd, stdin=input_handler, stdout=PIPE, stderr=PIPE) cout_ssh, cerr_ssh = proc.communicate() ret = proc.returncode if ret: if cout_ssh: LOG.info(cout_ssh) if cerr_ssh: LOG.error(cerr_ssh) raise DestinationError('%s exited with error code %d' % (' '.join(cmd), ret)) LOG.debug('Exited with code %d', ret) except OSError as err: raise DestinationError('Failed to run %s: %s' % (' '.join(cmd), err))
def _download_object(s3_client, bucket_name, key, read_fd, write_fd): # The read end of the pipe must be closed in the child process # before we start writing to it. os.close(read_fd) with os.fdopen(write_fd, 'wb') as w_pipe: try: retry_interval = 2 for _ in xrange(10): try: s3_client.download_fileobj(bucket_name, key, w_pipe) return except ClientError as err: LOG.warning(err) LOG.warning('Will retry in %d seconds', retry_interval) time.sleep(retry_interval) retry_interval *= 2 except IOError as err: LOG.error(err) exit(1)
def get_stream(self): """ Get a PIPE handler with content of the source :return: """ cmd = "tar cf - %s" % self.path try: LOG.debug('Running %s', cmd) proc = Popen(shlex.split(cmd), stderr=PIPE, stdout=PIPE) yield proc.stdout _, cerr = proc.communicate() if proc.returncode: LOG.error('Failed to read from %s: %s', self.path, cerr) exit(1) else: LOG.debug('Successfully streamed %s', self.path) except OSError as err: LOG.error('Failed to run %s: %s', cmd, err) exit(1)
def _mysql_service(dst, action): """Start or stop MySQL service :param dst: Destination server :type dst: Ssh :param action: string start or stop :type action: str """ for service in ['mysqld', 'mysql']: try: return dst.execute_command( "PATH=$PATH:/sbin sudo service %s %s" % (service, action), quiet=True ) except SshClientException as err: LOG.debug(err) try: LOG.warning('Failed to %s MySQL with an init script. ' 'Will try to %s mysqld.', action, action) if action == "start": ret = dst.execute_command( "PATH=$PATH:/sbin sudo bash -c 'nohup mysqld &'", background=True ) time.sleep(10) return ret elif action == "stop": return dst.execute_command( "PATH=$PATH:/sbin sudo kill $(pidof mysqld)" ) except SshClientException as err: LOG.error(err) raise OperationError( 'Failed to %s MySQL on %r' % (action, dst) )
def _mysql_service(dst, action): """Start or stop MySQL service :param dst: Destination server :type dst: Ssh :param action: string start or stop :type action: str """ for service in ["mysqld", "mysql"]: try: return dst.execute_command( "PATH=$PATH:/sbin sudo service %s %s" % (service, action), quiet=True, ) except SshClientException as err: LOG.debug(err) try: LOG.warning( "Failed to %s MySQL with an init script. " "Will try to %s mysqld.", action, action, ) if action == "start": ret = dst.execute_command( "PATH=$PATH:/sbin sudo bash -c 'nohup mysqld &'", background=True, ) time.sleep(10) return ret elif action == "stop": return dst.execute_command( "PATH=$PATH:/sbin sudo kill $(pidof mysqld)") except SshClientException as err: LOG.error(err) raise OperationError("Failed to %s MySQL on %r" % (action, dst))
def get_full_copy_name(self, file_path): """ For a given backup copy find a parent. If it's a full copy then return itself :param file_path: :return: """ try: for run_type in INTERVALS: for key in self.status()[run_type].keys(): if file_path.endswith(key): if self.status()[run_type][key]['type'] == "full": return file_path else: remote_part = file_path.replace(key, '') parent = self.status()[run_type][key]['parent'] result = "%s%s" % (remote_part, parent) return result except (TypeError, KeyError) as err: LOG.error('Failed to find parent of %s', file_path) raise DestinationError(err) raise DestinationError('Failed to find parent of %s' % file_path)
def run_backup_job(twindb_config, run_type, lock_file=LOCK_FILE, binlogs_only=False): """ Grab a lock waiting up to allowed timeout and start backup jobs :param twindb_config: Tool configuration :type twindb_config: TwinDBBackupConfig :param run_type: Run type :type run_type: str :param lock_file: File used as a lock :type lock_file: str :param binlogs_only: If True copy only binlogs. :type binlogs_only: bool """ with timeout(get_timeout(run_type)): try: file_desriptor = open(lock_file, "w") fcntl.flock(file_desriptor, fcntl.LOCK_EX) LOG.debug(run_type) if getattr(twindb_config.run_intervals, run_type): backup_everything(run_type, twindb_config, binlogs_only=binlogs_only) else: LOG.debug("Not running because run_%s is no", run_type) except IOError as err: if err.errno != errno.EINTR: LOG.debug(traceback.format_exc()) raise LockWaitTimeoutError(err) msg = "Another instance of twindb-backup is running?" if run_type == "hourly": LOG.debug(msg) else: LOG.error(msg)
def get_connection(self): """ Connect to MySQL host and yield a connection. :return: MySQL connection :raise MySQLSourceError: if can't connect to server """ connection = None try: connection = pymysql.connect( host=self.hostname, read_default_file=self.defaults_file, connect_timeout=self.connect_timeout, cursorclass=pymysql.cursors.DictCursor, ) yield connection except OperationalError: LOG.error("Can't connect to MySQL server on %s", self.hostname) raise MySQLSourceError("Can't connect to MySQL server on %s" % self.hostname) finally: if connection: connection.close()
def restore_file(cfg, dst, backup_copy): """Restore from file backup""" LOG.debug('file: %r', cfg) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(cfg) exit(1) try: ensure_empty(dst) copy = FileCopy( get_hostname_from_backup_copy(backup_copy), basename(backup_copy), get_run_type_from_backup_copy(backup_copy) ) restore_from_file(cfg, copy, dst) except TwinDBBackupError as err: LOG.error(err) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def execute(self, cmd, quiet=False, background=False): """Execute a command on a remote SSH server. :param cmd: Command for execution. :type cmd: str :param quiet: if quiet is True don't print error messages :param background: Don't wait until the command exits. :type background: bool :return: Strings with stdout and stderr. If command is executed in background the method will return None. :rtype: tuple :raise SshClientException: if any error or non-zero exit code """ max_chunk_size = 1024 * 1024 try: with self._shell() as shell: if not background: LOG.debug('Executing command: %s', cmd) stdin_, stdout_, _ = shell.exec_command(cmd) channel = stdout_.channel stdin_.close() channel.shutdown_write() stdout_chunks = [] stderr_chunks = [] while not channel.closed \ or channel.recv_ready() \ or channel.recv_stderr_ready(): if channel.recv_ready(): stdout_chunks.append( channel.recv(max_chunk_size).decode("utf-8")) if channel.recv_stderr_ready(): stderr_chunks.append( channel.recv_stderr(max_chunk_size).decode( "utf-8")) exit_code = channel.recv_exit_status() if exit_code != 0: if not quiet: LOG.error("Failed to execute command %s", cmd) LOG.error(''.join(stderr_chunks)) raise SshClientException('%s exited with code %d' % (cmd, exit_code)) return ''.join(stdout_chunks), ''.join(stderr_chunks) else: LOG.debug('Executing in background: %s', cmd) transport = shell.get_transport() channel = transport.open_session() channel.exec_command(cmd) LOG.debug('Ran %s in background', cmd) except (SSHException, IOError) as err: if not quiet: LOG.error('Failed to execute %s: %s', cmd, err) raise SshClientException('Failed to execute %s: %s' % (cmd, err))
def _extract_xbstream( input_stream, working_dir, xbstream_binary=XBSTREAM_BINARY): """ Extract xbstream stream in directory :param input_stream: The stream in xbstream format :param working_dir: directory :param xbstream_binary: Path to xbstream :return: True if extracted successfully """ try: cmd = [xbstream_binary, '-x'] LOG.debug('Running %s', ' '.join(cmd)) LOG.debug('Working directory: %s', working_dir) LOG.debug('Xbstream binary: %s', xbstream_binary) proc = Popen( cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE, cwd=working_dir ) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', ' '.join(cmd), ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return ret == 0 except OSError as err: LOG.error('Failed to extract xbstream: %s', err) return False
def test___find_all_cnf(mock_get_text_content, mock_list, tmpdir, mycnfs, expected_result_template): mycnf_root = Path(tmpdir) # Prepare steps (writing config files with content) for key in mycnfs.keys(): mycnf_root.joinpath(key).parent.mkdir(exist_ok=True) with open(str(mycnf_root.joinpath(key)), "w") as fp: fp.write(mycnfs[key]) # mock helper functions def get_text_content(full_path): LOG.debug("Getting content of %s", full_path) # cut mysql_root prefix from the full path and lookup for content in the mycnfs dictionary. return mycnfs["/".join( PurePath(full_path).parts[len(mycnf_root.parts):])] def get_list(path, recursive=False, files_only=True): return os.listdir(path) mock_get_text_content.side_effect = get_text_content mock_list.side_effect = get_list # rmt_sql = RemoteMySQLSource({ "run_type": INTERVALS[0], "backup_type": 'full', "mysql_connect_info": MySQLConnectInfo("/"), "ssh_connection_info": None }) expected_result = sorted( [osp.join(str(mycnf_root), item) for item in expected_result_template]) actual_result = sorted(rmt_sql._find_all_cnf( mycnf_root.joinpath("my.cnf"))) assert (actual_result == expected_result), LOG.error( "Expected: %s\nActual: %s" % (pformat(expected_result), pformat(actual_result)))
def backup(cfg, run_type, lock_file): """Run backup job""" try: run_backup_job(cfg, run_type, lock_file=lock_file) except IOError as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except ModifierException as err: LOG.error('Error in modifier class') LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def restore_from_mysql_incremental(stream, dst_dir, config, tmp_dir=None, xtrabackup_binary=XTRABACKUP_BINARY, xbstream_binary=XBSTREAM_BINARY): """ Restore MySQL datadir from an incremental copy. :param stream: Generator that provides backup copy :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str :param config: Tool configuration. :type config: TwinDBBackupConfig :param tmp_dir: Path to temp dir :type tmp_dir: str :param xtrabackup_binary: Path to xtrabackup binary. :param xbstream_binary: Path to xbstream binary :return: If success, return True :rtype: bool """ if tmp_dir is None: try: inc_dir = tempfile.mkdtemp() except (IOError, OSError): try: empty_dir(dst_dir) except (IOError, OSError): raise raise else: inc_dir = tmp_dir # GPG modifier if config.gpg: gpg = Gpg( stream, config.gpg.recipient, config.gpg.keyring, secret_keyring=config.gpg.secret_keyring ) LOG.debug('Decrypting stream') stream = gpg.revert_stream() else: LOG.debug('Not decrypting the stream') stream = config.compression.get_modifier(stream).revert_stream() with stream as handler: if not _extract_xbstream(handler, inc_dir, xbstream_binary): return False try: mem_usage = psutil.virtual_memory() try: xtrabackup_cmd = [ xtrabackup_binary, '--use-memory=%d' % (mem_usage.available / 2), '--prepare', '--apply-log-only', '--target-dir=%s' % dst_dir ] LOG.debug('Running %s', ' '.join(xtrabackup_cmd)) xtrabackup_proc = Popen( xtrabackup_cmd, stdout=None, stderr=None ) xtrabackup_proc.communicate() ret = xtrabackup_proc.returncode if ret: LOG.error( '%s exited with code %d', " ".join(xtrabackup_cmd), ret) return False xtrabackup_cmd = [ xtrabackup_binary, '--use-memory=%d' % (mem_usage.available / 2), '--prepare', "--target-dir=%s" % dst_dir, "--incremental-dir=%s" % inc_dir ] LOG.debug('Running %s', ' '.join(xtrabackup_cmd)) xtrabackup_proc = Popen( xtrabackup_cmd, stdout=None, stderr=None ) xtrabackup_proc.communicate() ret = xtrabackup_proc.returncode if ret: LOG.error('%s exited with code %d', " ".join(xtrabackup_cmd), ret) return ret == 0 except OSError as err: LOG.error('Failed to prepare backup in %s: %s', dst_dir, err) return False finally: try: pass except OSError as exc: if exc.errno != errno.ENOENT: # ENOENT - no such file or directory raise # re-raise exception
def get_stream(self, copy): """ Get a PIPE handler with content of the backup copy streamed from the destination. :param copy: Backup copy :type copy: BaseCopy :return: Stream with backup copy :rtype: generator :raise S3DestinationError: if failed to stream a backup copy. """ path = "%s/%s" % (self.remote_path, copy.key) object_key = urlparse(path).path.lstrip('/') def _download_object(s3_client, bucket_name, key, read_fd, write_fd): # The read end of the pipe must be closed in the child process # before we start writing to it. os.close(read_fd) with os.fdopen(write_fd, 'wb') as w_pipe: try: retry_interval = 2 for _ in xrange(10): try: s3_client.download_fileobj(bucket_name, key, w_pipe) return except ClientError as err: LOG.warning(err) LOG.warning('Will retry in %d seconds', retry_interval) time.sleep(retry_interval) retry_interval *= 2 except IOError as err: LOG.error(err) exit(1) download_proc = None try: LOG.debug('Fetching object %s from bucket %s', object_key, self._bucket) read_pipe, write_pipe = os.pipe() download_proc = Process(target=_download_object, args=(self.s3_client, self._bucket, object_key, read_pipe, write_pipe), name='_download_object') download_proc.start() # The write end of the pipe must be closed in this process before # we start reading from it. os.close(write_pipe) LOG.debug('read_pipe type: %s', type(read_pipe)) yield read_pipe os.close(read_pipe) download_proc.join() if download_proc.exitcode: LOG.error('Failed to download %s', path) # exit(1) LOG.debug('Successfully streamed %s', path) finally: if download_proc: download_proc.join()
def get_stream(self, copy): """ Get a PIPE handler with content of the backup copy streamed from the destination. :param copy: Backup copy :type copy: BaseCopy :return: Stream with backup copy :rtype: generator :raise S3DestinationError: if failed to stream a backup copy. """ path = "%s/%s" % (self.remote_path, copy.key) object_key = urlparse(path).path.lstrip('/') def _download_object(s3_client, bucket_name, key, read_fd, write_fd): # The read end of the pipe must be closed in the child process # before we start writing to it. os.close(read_fd) with os.fdopen(write_fd, 'wb') as w_pipe: try: retry_interval = 2 for _ in xrange(10): try: s3_client.download_fileobj(bucket_name, key, w_pipe) return except ClientError as err: LOG.warning(err) LOG.warning('Will retry in %d seconds', retry_interval) time.sleep(retry_interval) retry_interval *= 2 except IOError as err: LOG.error(err) exit(1) download_proc = None try: LOG.debug('Fetching object %s from bucket %s', object_key, self.bucket) read_pipe, write_pipe = os.pipe() download_proc = Process(target=_download_object, args=(self.s3_client, self.bucket, object_key, read_pipe, write_pipe), name='_download_object') download_proc.start() # The write end of the pipe must be closed in this process before # we start reading from it. os.close(write_pipe) LOG.debug('read_pipe type: %s', type(read_pipe)) yield read_pipe os.close(read_pipe) download_proc.join() if download_proc.exitcode: LOG.error('Failed to download %s', path) # exit(1) LOG.debug('Successfully streamed %s', path) finally: if download_proc: download_proc.join()
def test_clone( runner, master1, slave, docker_client, config_content_clone, client_my_cnf, rsa_private_key): twindb_config_dir = get_twindb_config_dir(docker_client, runner['Id']) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' my_cnf_path = "%s/my.cnf" % twindb_config_dir private_key_host = "%s/private_key" % twindb_config_dir private_key_guest = "/etc/twindb/private_key" with open(my_cnf_path, "w") as my_cnf: my_cnf.write(client_my_cnf) with open(private_key_host, "w") as key_fd: key_fd.write(rsa_private_key) with open(twindb_config_host, 'w') as fp: content = config_content_clone.format( PRIVATE_KEY=private_key_guest, MY_CNF='/etc/twindb/my.cnf' ) fp.write(content) cmd = '/usr/sbin/sshd' LOG.info('Run SSH daemon on master1_1') ret, cout = docker_execute(docker_client, master1['Id'], cmd) print(cout) cmd = [ 'twindb-backup', '--debug', '--config', twindb_config_guest, 'clone', 'mysql', '%s:3306' % master1['ip'], '%s:3306' % slave['ip'] ] pause_test(' '.join(cmd)) ret, cout = docker_execute(docker_client, runner['Id'], cmd) print(cout) assert ret == 0 sql_master_2 = RemoteMySQLSource({ "ssh_host": slave['ip'], "ssh_user": '******', "ssh_key": private_key_guest, "mysql_connect_info": MySQLConnectInfo( my_cnf_path, hostname=slave['ip'] ), "run_type": INTERVALS[0], "backup_type": 'full' }) timeout = time.time() + 30 while time.time() < timeout: with sql_master_2.get_connection() as conn: with conn.cursor() as cursor: cursor.execute('SHOW SLAVE STATUS') row = cursor.fetchone() if row['Slave_IO_Running'] == 'Yes' \ and row['Slave_SQL_Running'] == 'Yes': LOG.info('Replication is up and running') return LOG.error('Replication is not running after 30 seconds timeout') assert False
def test_clone(runner, master1, slave, docker_client, config_content_clone): twindb_config_dir = get_twindb_config_dir(docker_client, runner['Id']) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' my_cnf_path = "%s/my.cnf" % twindb_config_dir private_key_host = "%s/private_key" % twindb_config_dir private_key_guest = "/etc/twindb/private_key" contents = """ [client] user=dba password=qwerty """ with open(my_cnf_path, "w") as my_cnf: my_cnf.write(contents) private_key = """-----BEGIN RSA PRIVATE KEY----- MIIEoAIBAAKCAQEAyXxAjPShNGAedbaEtltFI6A7RlsyI+4evxTq6uQrgbJ6Hm+p HBXshXQYXDyVjvytaM+6GKF+r+6+C+6Wc5Xz4lLO/ZiSCdPbyEgqw1JoHrgPNpc6 wmCtjJExxjzvpwSVgbZg3xOdqW1y+TyqeUkXEg/Lm4VZhN1Q/KyGCgBlWuAXoOYR GhaNWqcnr/Wn5YzVHAx2yJNrurtKLVYVMIkGcN/6OUaPpWqKZLaXiK/28PSZ5GdT DmxRg4W0pdyGEYQndpPlpLF4w5gNUEhVZM8hWVE29+DIW3XXVYGYchxmkhU7wrGx xZR+k5AT+7g8VspVS8zNMXM9Z27w55EQuluNMQIBIwKCAQAzz35QIaXLo7APo/Y9 hS8JKTPQQ1YJPTsbMUO4vlRUjPrUoF6vc1oTsCOFbqoddCyXS1u9MNdvEYFThn51 flSn6WhtGJqU0BPxrChA2q0PNqTThfkqqyVQCBQdCFrhzfqPEaPhl1RtZUlzSh01 IWxVGgEn/bfu9xTTQk5aV9+MZQ2XKe4BGzpOZMI/B7ivRCcthEwMTx92opr52bre 4t7DahVLN/2Wu4lxajDzCaKXpjMuL76lFov0mZZN7S8whH5xSx1tpapHqsCAwfLL k49lDdR8aN6oqoeK0e9w//McIaKxN2FVxD4bcuXiQTjihx+QwQOLmlHSRDKhTsYg 4Q5bAoGBAOgVZM2eqC8hNl5UH//uuxOeBKqwz7L/FtGemNr9m0XG8N9yE/K7A5iX 6EDvDyVI51IlIXdxfK8re5yxfbJ4YevenwdEZZ2O8YRrVByJ53PV9CcVeWL4p6f/ I56sYyDfXcnDTEOVYY0mCfYUfUcSb1ExpuIU4RvuQJg6tvbdxD9FAoGBAN4/pVCT krRd6PJmt6Dbc2IF6N09OrAnLB3fivGztF5cp+RpyqZK4ve+akLoe1laTg7vNtnF l/PZtM9v/VT45hb70MFEHO+sKvGa5Yimxkb6YCriJOcLxTysSgFHKz7v+8BqqoHi qY4fORGwPVDv28I8jKRvcuNHendV/Rdcuk79AoGAd1t1q5NscAJzu3u4r4IXEWc1 mZzClpHROJq1AujTgviZInUu1JqxZGthgHrx2KkmggR3nIOB86/2bdefut7TRhq4 L5+Et24VzxKgSTD6sJnrR0zfV3iQvMxbdizFRBsaSoGyMWLEdHn2fo4xzMem9o6Q VwNsdMOsMQhA1rsxuiMCgYBr8wcnIxte68jqxC1OIXKOsmnKi3RG7nSDidXF2vE1 JbCiJMGD+Hzeu5KyyLDw4rgzI7uOWKjkJ+obnMuBCy3t6AZPPlcylXPxsaKwFn2Q MHfaUJWUyzPqRQ4AnukekdINAJv18cAR1Kaw0fHle9Ej1ERP3lxfw6HiMRSHsLJD nwKBgCIXVhXCDaXOOn8M4ky6k27bnGJrTkrRjHaq4qWiQhzizOBTb+7MjCrJIV28 8knW8+YtEOfl5R053SKQgVsmRjjDfvCirGgqC4kSAN4A6MD+GNVXZVUUjAUBVUbU 8Wt4BxW6kFA7+Su7n8o4DxCqhZYmK9ZUhNjE+uUhxJCJaGr4 -----END RSA PRIVATE KEY----- """ with open(private_key_host, "w") as key_fd: key_fd.write(private_key) with open(twindb_config_host, 'w') as fp: content = config_content_clone.format( PRIVATE_KEY=private_key_guest, MY_CNF='/etc/twindb/my.cnf' ) fp.write(content) cmd = '/usr/sbin/sshd' # Run SSH daemon on master1_1 ret, cout = docker_execute(docker_client, master1['Id'], cmd) print(cout) cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, 'clone', 'mysql', "%s:3306" % master1['ip'], "%s:3306" % slave['ip'] ] ret, cout = docker_execute(docker_client, runner['Id'], cmd) print(cout) assert ret == 0 sql_master_2 = RemoteMySQLSource({ "ssh_host": slave['ip'], "ssh_user": '******', "ssh_key": private_key_guest, "mysql_connect_info": MySQLConnectInfo( my_cnf_path, hostname=slave['ip'] ), "run_type": INTERVALS[0], "backup_type": 'full' }) timeout = time.time() + 30 while time.time() < timeout: with sql_master_2.get_connection() as conn: with conn.cursor() as cursor: cursor.execute('SHOW SLAVE STATUS') row = cursor.fetchone() if row['Slave_IO_Running'] == 'Yes' and row['Slave_SQL_Running'] == 'Yes': LOG.info('Relication is up and running') return LOG.error('Replication is not running after 30 seconds timeout') assert False
def get_container(name, client, network, datadir=None, bootstrap_script=None, last_n=1, twindb_config_dir=None, image=NODE_IMAGE): api = client.api api.pull(image) cwd = os.getcwd() LOG.debug('Current directory: %s', cwd) binds = { cwd: { 'bind': '/twindb-backup', 'mode': 'rw', } } if twindb_config_dir: LOG.debug('TwinDB config directory: %s', twindb_config_dir) mkdir_p(twindb_config_dir, mode=0755) binds[twindb_config_dir] = { 'bind': '/etc/twindb', 'mode': 'rw', } if datadir: binds[datadir] = { 'bind': '/var/lib/mysql', 'mode': 'rw', } host_config = api.create_host_config( binds=binds, dns=['8.8.8.8', '208.67.222.222', '208.67.220.220']) ip = '172.%d.3.%d' % (network['second_octet'], last_n) networking_config = api.create_networking_config( {network['NAME']: api.create_endpoint_config(ipv4_address=ip)}) LOG.debug(networking_config) container_hostname = '%s_%d' % (name, last_n) kwargs = { 'image': image, 'name': container_hostname, 'ports': [22, 3306], 'hostname': container_hostname, 'host_config': host_config, 'networking_config': networking_config, 'volumes': ['/twindb-backup'], 'environment': {} } try: kwargs['environment'] = {'DEV': os.environ['DEV']} except KeyError: pass if bootstrap_script: kwargs['command'] = 'bash %s' % bootstrap_script container = api.create_container(**kwargs) container['ip'] = ip LOG.info('Created container %r', container) try: api.start(container['Id']) LOG.info('Started %r', container) return container except APIError as err: LOG.error(err) client.api.remove_container(container=container['Id'], force=True)
def clone_mysql( cfg, source, destination, # pylint: disable=too-many-arguments replication_user, replication_password, netcat_port=9990, compress=False): """Clone mysql backup of remote machine and stream it to slave""" try: LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0]) LOG.debug('MySQL defaults: %s', cfg.get('mysql', 'mysql_defaults_file')) LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user')) LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key')) src = RemoteMySQLSource({ "ssh_host": split_host_port(source)[0], "ssh_user": cfg.get('ssh', 'ssh_user'), "ssh_key": cfg.get('ssh', 'ssh_key'), "mysql_connect_info": MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'), hostname=split_host_port(source)[0]), "run_type": INTERVALS[0], "backup_type": 'full' }) xbstream_binary = cfg.get('mysql', 'xbstream_binary') LOG.debug('SSH destination: %s', split_host_port(destination)[0]) LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user')) LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key')) dst = Ssh('/tmp', ssh_host=split_host_port(destination)[0], ssh_user=cfg.get('ssh', 'ssh_user'), ssh_key=cfg.get('ssh', 'ssh_key')) datadir = src.datadir LOG.debug('datadir: %s', datadir) if dst.list_files(datadir): LOG.error("Destination datadir is not empty: %s", datadir) exit(1) _run_remote_netcat(compress, datadir, destination, dst, netcat_port, src, xbstream_binary) LOG.debug('Copying MySQL config to the destination') src.clone_config(dst) LOG.debug('Remote MySQL destination: %s', split_host_port(destination)[0]) LOG.debug('MySQL defaults: %s', cfg.get('mysql', 'mysql_defaults_file')) LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user')) LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key')) dst_mysql = RemoteMySQLSource({ "ssh_host": split_host_port(destination)[0], "ssh_user": cfg.get('ssh', 'ssh_user'), "ssh_key": cfg.get('ssh', 'ssh_key'), "mysql_connect_info": MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'), hostname=split_host_port(destination)[0]), "run_type": INTERVALS[0], "backup_type": 'full' }) binlog, position = dst_mysql.apply_backup(datadir) LOG.debug('Binlog coordinates: (%s, %d)', binlog, position) try: LOG.debug('Starting MySQL on the destination') _mysql_service(dst, action='start') LOG.debug('MySQL started') except TwinDBBackupError as err: LOG.error(err) exit(1) LOG.debug('Setting up replication.') LOG.debug('Master host: %s', source) LOG.debug('Replication user: %s', replication_user) LOG.debug('Replication password: %s', replication_password) dst_mysql.setup_slave( MySQLMasterInfo(host=split_host_port(source)[0], port=split_host_port(source)[1], user=replication_user, password=replication_password, binlog=binlog, binlog_pos=position)) except (ConfigParser.NoOptionError, OperationalError) as err: LOG.error(err) exit(1)
def clone_mysql(cfg, source, destination, # pylint: disable=too-many-arguments replication_user, replication_password, netcat_port=9990, compress=False): """Clone mysql backup of remote machine and stream it to slave :param cfg: TwinDB Backup tool config :type cfg: TwinDBBackupConfig """ LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0]) LOG.debug( 'MySQL defaults: %s', cfg.mysql.defaults_file ) LOG.debug( 'SSH username: %s', cfg.ssh.user ) LOG.debug( 'SSH key: %s', cfg.ssh.key ) src = RemoteMySQLSource( { "ssh_host": split_host_port(source)[0], "ssh_user": cfg.ssh.user, "ssh_key": cfg.ssh.key, "mysql_connect_info": MySQLConnectInfo( cfg.mysql.defaults_file, hostname=split_host_port(source)[0]), "run_type": INTERVALS[0], "backup_type": 'full' } ) xbstream_binary = cfg.mysql.xbstream_binary LOG.debug('SSH destination: %s', split_host_port(destination)[0]) LOG.debug('SSH username: %s', cfg.ssh.user) LOG.debug('SSH key: %s', cfg.ssh.key) dst = Ssh( '/tmp', ssh_host=split_host_port(destination)[0], ssh_user=cfg.ssh.user, ssh_key=cfg.ssh.key ) datadir = src.datadir LOG.debug('datadir: %s', datadir) if dst.list_files(datadir): LOG.error("Destination datadir is not empty: %s", datadir) exit(1) _run_remote_netcat( compress, datadir, destination, dst, netcat_port, src, xbstream_binary ) LOG.debug('Copying MySQL config to the destination') src.clone_config(dst) LOG.debug('Remote MySQL destination: %s', split_host_port(destination)[0]) LOG.debug( 'MySQL defaults: %s', cfg.mysql.defaults_file ) LOG.debug('SSH username: %s', cfg.ssh.user) LOG.debug('SSH key: %s', cfg.ssh.key) dst_mysql = RemoteMySQLSource({ "ssh_host": split_host_port(destination)[0], "ssh_user": cfg.ssh.user, "ssh_key": cfg.ssh.key, "mysql_connect_info": MySQLConnectInfo( cfg.mysql.defaults_file, hostname=split_host_port(destination)[0] ), "run_type": INTERVALS[0], "backup_type": 'full' }) binlog, position = dst_mysql.apply_backup(datadir) LOG.debug('Binlog coordinates: (%s, %d)', binlog, position) LOG.debug('Starting MySQL on the destination') _mysql_service(dst, action='start') LOG.debug('MySQL started') LOG.debug('Setting up replication.') LOG.debug('Master host: %s', source) LOG.debug('Replication user: %s', replication_user) LOG.debug('Replication password: %s', replication_password) dst_mysql.setup_slave( MySQLMasterInfo( host=split_host_port(source)[0], port=split_host_port(source)[1], user=replication_user, password=replication_password, binlog=binlog, binlog_pos=position ) )
def get_container( name, client, network, datadir=None, bootstrap_script=None, last_n=1, twindb_config_dir=None, image=NODE_IMAGE, ): api = client.api api.pull(image) cwd = os.getcwd() LOG.debug("Current directory: %s", cwd) binds = { cwd: { "bind": "/twindb-backup", "mode": "rw", } } if twindb_config_dir: LOG.debug("TwinDB config directory: %s", twindb_config_dir) mkdir_p(twindb_config_dir, mode=0o755) binds[twindb_config_dir] = { "bind": "/etc/twindb", "mode": "rw", } if datadir: binds[datadir] = { "bind": "/var/lib/mysql", "mode": "rw", } host_config = api.create_host_config( binds=binds, dns=["8.8.8.8", "208.67.222.222", "208.67.220.220"]) ip = "172.%d.3.%d" % (network["second_octet"], last_n) networking_config = api.create_networking_config( {network["NAME"]: api.create_endpoint_config(ipv4_address=ip)}) LOG.debug(networking_config) container_hostname = "%s_%d" % (name, last_n) kwargs = { "image": image, "name": container_hostname, "ports": [22, 3306], "hostname": container_hostname, "host_config": host_config, "networking_config": networking_config, "volumes": ["/twindb-backup"], "environment": {}, } try: kwargs["environment"] = {"DEV": os.environ["DEV"]} except KeyError: pass if bootstrap_script: kwargs["command"] = "bash %s" % bootstrap_script container = api.create_container(**kwargs) container["ip"] = ip LOG.info("Created container %r", container) try: api.start(container["Id"]) LOG.info("Started %r", container) return container except APIError as err: LOG.error(err) client.api.remove_container(container=container["Id"], force=True)
def verify_mysql_backup(twindb_config, dst_path, backup_file, hostname=None): """ Restore mysql backup and measure time :param hostname: :param backup_file: :param dst_path: :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig """ dst = twindb_config.destination(backup_source=hostname) status = MySQLStatus(dst=dst) copy = None if backup_file == "latest": copy = status.latest_backup else: for copy in status: if backup_file.endswith(copy.key): break if copy is None: return json.dumps( { "backup_copy": backup_file, "restore_time": 0, "success": False }, indent=4, sort_keys=True, ) start_restore_time = time.time() success = True tmp_dir = tempfile.mkdtemp() try: LOG.debug("Verifying backup copy in %s", tmp_dir) restore_from_mysql(twindb_config, copy, dst_path, tmp_dir) edit_backup_my_cnf(dst_path) except (TwinDBBackupError, OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) success = False finally: shutil.rmtree(tmp_dir, ignore_errors=True) end_restore_time = time.time() restore_time = end_restore_time - start_restore_time return json.dumps( { "backup_copy": copy.key, "restore_time": restore_time, "success": success, }, indent=4, sort_keys=True, )
def restore_from_file(twindb_config, copy, dst_dir): """ Restore a directory from a backup copy in the directory :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig :param copy: Instance of BaseCopy or and inheriting classes. :type copy: BaseCopy :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str """ LOG.info('Restoring %s in %s', copy.key, dst_dir) mkdir_p(dst_dir) restore_start = time.time() keep_local_path = twindb_config.keep_local_path if keep_local_path and os.path.exists(osp.join(keep_local_path, copy.key)): dst = Local(osp.join(keep_local_path, copy.key)) stream = dst.get_stream(copy) else: dst = twindb_config.destination() stream = dst.get_stream(copy) # GPG modifier if twindb_config.gpg: gpg = Gpg( stream, twindb_config.gpg.recipient, twindb_config.gpg.keyring, secret_keyring=twindb_config.gpg.secret_keyring ) LOG.debug('Decrypting stream') stream = gpg.revert_stream() else: LOG.debug('Not decrypting the stream') with stream as handler: try: LOG.debug('handler type: %s', type(handler)) LOG.debug('stream type: %s', type(stream)) cmd = ["tar", "zvxf", "-"] LOG.debug('Running %s', ' '.join(cmd)) proc = Popen(cmd, stdin=handler, cwd=dst_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', cmd, ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return LOG.info('Successfully restored %s in %s', copy.key, dst_dir) except (OSError, DestinationError) as err: LOG.error('Failed to decompress %s: %s', copy.key, err) exit(1) export_info( twindb_config, data=time.time() - restore_start, category=ExportCategory.files, measure_type=ExportMeasureType.restore )
def restore_from_mysql_full(stream, dst_dir, config, redo_only=False, xtrabackup_binary=XTRABACKUP_BINARY, xbstream_binary=XBSTREAM_BINARY): """ Restore MySQL datadir from a backup copy :param stream: Generator that provides backup copy :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str :param config: Tool configuration. :type config: TwinDBBackupConfig :param redo_only: True if the function has to do final apply of the redo log. For example, if you restore backup from a full copy it should be False. If you restore from incremental copy and you restore base full copy redo_only should be True. :type redo_only: bool :param xtrabackup_binary: path to xtrabackup binary. :param xbstream_binary: Path to xbstream binary :return: If success, return True :rtype: bool """ # GPG modifier if config.gpg: gpg = Gpg( stream, config.gpg.recipient, config.gpg.keyring, secret_keyring=config.gpg.secret_keyring ) LOG.debug('Decrypting stream') stream = gpg.revert_stream() else: LOG.debug('Not decrypting the stream') stream = config.compression.get_modifier(stream).revert_stream() with stream as handler: if not _extract_xbstream(handler, dst_dir, xbstream_binary): return False mem_usage = psutil.virtual_memory() try: xtrabackup_cmd = [xtrabackup_binary, '--use-memory=%d' % (mem_usage.available/2), '--prepare'] if redo_only: xtrabackup_cmd += ['--apply-log-only'] xtrabackup_cmd += ["--target-dir", dst_dir] LOG.debug('Running %s', ' '.join(xtrabackup_cmd)) xtrabackup_proc = Popen(xtrabackup_cmd, stdout=None, stderr=None) xtrabackup_proc.communicate() ret = xtrabackup_proc.returncode if ret: LOG.error('%s exited with code %d', " ".join(xtrabackup_cmd), ret) return ret == 0 except OSError as err: LOG.error('Failed to prepare backup in %s: %s', dst_dir, err) return False