def _run_remote_netcat( compress, datadir, # pylint: disable=too-many-arguments destination, dst, netcat_port, src): netcat_cmd = "xbstream -x -C {datadir}".format(datadir=datadir) if compress: netcat_cmd = "gunzip -c - | %s" % netcat_cmd # find unused port while netcat_port < 64000: if dst.ensure_tcp_port_listening(netcat_port, wait_timeout=1): netcat_port += 1 else: LOG.debug('Will use port %d for streaming', netcat_port) break proc_netcat = Process(target=dst.netcat, args=(netcat_cmd, ), kwargs={'port': netcat_port}) LOG.debug('Starting netcat on the destination') proc_netcat.start() nc_wait_timeout = 10 if not dst.ensure_tcp_port_listening(netcat_port, wait_timeout=nc_wait_timeout): LOG.error('netcat on the destination ' 'is not ready after %d seconds', nc_wait_timeout) proc_netcat.terminate() exit(1) src.clone(dest_host=split_host_port(destination)[0], port=netcat_port, compress=compress) proc_netcat.join()
def backup_everything(run_type, twindb_config, binlogs_only=False): """ Run backup job :param run_type: hourly, daily, etc :type run_type: str :param twindb_config: ConfigParser instance :type twindb_config: TwinDBBackupConfig :param binlogs_only: If True copy only MySQL binary logs. :type binlogs_only: bool """ set_open_files_limit() try: if not binlogs_only: backup_start = time.time() backup_files(run_type, twindb_config) backup_mysql(run_type, twindb_config) backup_binlogs(run_type, twindb_config) end = time.time() save_measures(backup_start, end) else: backup_binlogs(run_type, twindb_config) except configparser.NoSectionError as err: LOG.debug(traceback.format_exc()) LOG.error(err) exit(1)
def setup_slave(self, master_info): # noqa # pylint: disable=too-many-arguments """ Change master :param master_info: Master details. :type master_info: MySQLMasterInfo """ try: with self._cursor() as cursor: query = "CHANGE MASTER TO " \ "MASTER_HOST = '{master}', " \ "MASTER_USER = '******', " \ "MASTER_PORT = {port}, " \ "MASTER_PASSWORD = '******', " \ "MASTER_LOG_FILE = '{binlog}', " \ "MASTER_LOG_POS = {binlog_pos}"\ .format( master=master_info.host, user=master_info.user, password=master_info.password, binlog=master_info.binlog, binlog_pos=master_info.binlog_position, port=master_info.port ) cursor.execute(query) cursor.execute("START SLAVE") return True except pymysql.Error as err: LOG.debug(err) return False
def revert_stream(self): """ Un-Apply modifier and return output stream. The Base modifier does nothing, so it will return the input stream without modifications :return: output stream handle """ with self._input as input_stream: LOG.debug('Running %s', ' '.join(self._unmodifier_cmd)) proc = Popen( self._unmodifier_cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE ) yield proc.stdout _, cerr = proc.communicate() if proc.returncode: msg = '%s exited with non-zero code.' \ % ' '.join(self._unmodifier_cmd) LOG.error(msg) LOG.error(cerr) raise ModifierException(msg)
def run_command(command, ok_non_zero=False): """ Run shell command locally :param command: Command to run :type command: list :param ok_non_zero: Don't consider non-zero exit code as an error. :type ok_non_zero: bool :return: file object with stdout as generator to use with ``with`` """ try: LOG.debug('Running %s', " ".join(command)) proc = Popen(command, stderr=PIPE, stdout=PIPE) yield proc.stdout _, cerr = proc.communicate() if proc.returncode and not ok_non_zero: LOG.error('Command %s exited with error code %d', ' '.join(command), proc.returncode) LOG.error(cerr) exit(1) else: LOG.debug('Exited with zero code') except OSError as err: LOG.error('Failed to run %s', ' '.join(command)) LOG.error(err) exit(1)
def verify_mysql_backup(twindb_config, dst_path, backup_file, hostname=None): """ Restore mysql backup and measure time :param hostname: :param backup_file: :param dst_path: :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig """ dst = twindb_config.destination(backup_source=hostname) status = MySQLStatus(dst=dst) copy = None if backup_file == "latest": copy = status.latest_backup else: for copy in status: if backup_file.endswith(copy.key): break if copy is None: return json.dumps( { 'backup_copy': backup_file, 'restore_time': 0, 'success': False }, indent=4, sort_keys=True) start_restore_time = time.time() success = True tmp_dir = tempfile.mkdtemp() try: LOG.debug('Verifying backup copy in %s', tmp_dir) restore_from_mysql(twindb_config, copy, dst_path, tmp_dir) edit_backup_my_cnf(dst_path) except (TwinDBBackupError, OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) success = False finally: shutil.rmtree(tmp_dir, ignore_errors=True) end_restore_time = time.time() restore_time = end_restore_time - start_restore_time return json.dumps( { 'backup_copy': copy.key, 'restore_time': restore_time, 'success': success }, indent=4, sort_keys=True)
def _find_all_cnf(self, root_path): """ Return list of embed cnf files :param root_path: Path to the originating my.cnf config. (/etc/my.cnf, or /etc/mysql/my.cnf) :type root_path: Path :return: List of all included my.cnf files :rtype: list """ files = [str(root_path)] cfg_content = self._ssh_client.get_text_content(str(root_path)) for line in cfg_content.splitlines(): if "!includedir" in line: rel_path = line.split()[1] file_list = self._ssh_client.list_files( root_path.parent.joinpath(rel_path), recursive=False, files_only=True, ) LOG.debug(file_list) for sub_file in file_list: files.extend( self._find_all_cnf( root_path.parent.joinpath(rel_path).joinpath( sub_file ) ) ) elif "!include" in line: rel_path = line.split()[1] files.extend( self._find_all_cnf(root_path.parent.joinpath(rel_path)) ) return files
def disable_wsrep_desync(self): """ Wait till wsrep_local_recv_queue is zero and disable wsrep_local_recv_queue then """ max_time = time.time() + 900 try: with self.get_connection() as connection: with connection.cursor() as cursor: while time.time() < max_time: cursor.execute("SHOW GLOBAL STATUS LIKE " "'wsrep_local_recv_queue'") res = {r['Variable_name'].lower(): r['Value'].lower() for r in cursor.fetchall()} if not res.get('wsrep_local_recv_queue'): raise Exception('Unknown status variable ' '"wsrep_local_recv_queue"') if int(res['wsrep_local_recv_queue']) == 0: break time.sleep(1) LOG.debug('Disabling wsrep_desync') cursor.execute("SET GLOBAL wsrep_desync=OFF") except pymysql.Error as err: LOG.error(err)
def _shell(self): """ Create SSHClient instance and connect to the destination host. :return: Connected to the remote destination host shell. :rtype: generator(SSHClient) :raise SshDestinationError: if the ssh client fails to connect. """ shell = SSHClient() shell.set_missing_host_key_policy(AutoAddPolicy()) try: LOG.debug("Connecting to %s:%d as %s with key %s", self._host, self._port, self._user, self._key) shell.connect(hostname=self._host, key_filename=self._key, port=self._port, username=self._user) yield shell except FileNotFoundError: raise except (AuthenticationException, SSHException, socket.error) as err: # print(type(err)) raise SshClientException(err) finally: shell.close()
def _mysql_service(dst, action): """Start or stop MySQL service :param dst: Destination server :type dst: Ssh :param action: string start or stop :type action: str """ for service in ['mysqld', 'mysql']: try: return dst.execute_command("PATH=$PATH:/sbin sudo service %s %s" % (service, action), quiet=True) except SshClientException as err: LOG.debug(err) try: LOG.warning( 'Failed to %s MySQL with an init script. ' 'Will try to %s mysqld.', action, action) if action == "start": ret = dst.execute_command( "PATH=$PATH:/sbin sudo bash -c 'nohup mysqld &'", background=True) time.sleep(10) return ret elif action == "stop": return dst.execute_command( "PATH=$PATH:/sbin sudo kill $(pidof mysqld)") except SshClientException as err: LOG.error(err) raise TwinDBBackupError('Failed to %s MySQL on %r' % (action, dst))
def backup_everything(run_type, twindb_config, binlogs_only=False): """ Run backup job :param run_type: hourly, daily, etc :type run_type: str :param twindb_config: ConfigParser instance :type twindb_config: TwinDBBackupConfig :param binlogs_only: If True copy only MySQL binary logs. :type binlogs_only: bool """ set_open_files_limit() try: if not binlogs_only: backup_start = time.time() backup_files(run_type, twindb_config) backup_mysql(run_type, twindb_config) backup_binlogs(run_type, twindb_config) end = time.time() save_measures(backup_start, end) else: backup_binlogs(run_type, twindb_config) except ConfigParser.NoSectionError as err: LOG.debug(traceback.format_exc()) LOG.error(err) exit(1)
def execute(self, cmd, quiet=False): """Execute a command on a remote SSH server. :param cmd: Command for execution. :type cmd: str :param quiet: if quiet is True don't print error messages :return: Handlers of stdin, stdout and stderr :rtype: tuple :raise SshDestinationError: if any error """ try: with self._shell() as shell: LOG.debug('Executing %s', cmd) stdin_, stdout_, stderr_ = shell.exec_command(cmd) # while not stdout_.channel.exit_status_ready(): # LOG.debug('%s: waiting', cmd) # time.sleep(1) exit_code = stdout_.channel.recv_exit_status() if exit_code != 0: if not quiet: LOG.error("Failed while execute command %s", cmd) LOG.error(stderr_.read()) raise SshClientException('%s exited with code %d' % (cmd, exit_code)) return stdin_, stdout_, stderr_ except (SSHException, IOError) as err: if not quiet: LOG.error('Failed to execute %s: %s', cmd, err) raise SshClientException('Failed to execute %s: %s' % (cmd, err))
def test__take_mysql_backup(s3_client, config_content_mysql_only, tmpdir): config = tmpdir.join('twindb-backup.cfg') content = config_content_mysql_only.format( AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], BUCKET=s3_client.bucket, daily_copies=1, hourly_copies=2) config.write(content) cmd = [ 'twindb-backup', '--debug', '--config', str(config), 'backup', 'hourly' ] assert call(cmd) == 0 cmd = ['twindb-backup', '--config', str(config), 'status'] proc = Popen(cmd, stdout=PIPE, stderr=PIPE) cout, cerr = proc.communicate() LOG.debug('STDOUT: %s', cout) LOG.debug('STDERR: %s', cerr) key = json.loads(cout)['hourly'].keys()[0] assert key.endswith('.xbstream.gz')
def backup_files(run_type, config): """Backup local directories :param run_type: Run type :type run_type: str :param config: Configuration :type config: TwinDBBackupConfig """ backup_start = time.time() try: for directory in config.backup_dirs: LOG.debug('copying %s', directory) src = FileSource(directory, run_type) dst = config.destination() _backup_stream(config, src, dst) src.apply_retention_policy(dst, config, run_type) except ( DestinationError, SourceError, SshClientException ) as err: raise OperationError(err) export_info(config, data=time.time() - backup_start, category=ExportCategory.files, measure_type=ExportMeasureType.backup)
def apply_backup(self, datadir): """ Apply backup of destination server :param datadir: Path to datadir :return: Binlog file name and position :rtype: tuple :raise RemoteMySQLSourceError: if any error. """ try: use_memory = "--use-memory %d" % int(self._mem_available() / 2) except OSError: use_memory = "" logfile_path = "/tmp/xtrabackup-apply-log.log" cmd = "sudo {xtrabackup} --prepare --apply-log-only " \ "--target-dir {target_dir} {use_memory} " \ "> {logfile} 2>&1" \ "".format( xtrabackup=self._xtrabackup, target_dir=datadir, use_memory=use_memory, logfile=logfile_path ) try: self._ssh_client.execute(cmd) self._ssh_client.execute("sudo chown -R mysql %s" % datadir) return self._get_binlog_info(datadir) except SshClientException as err: LOG.debug("Logfile is:") LOG.debug(self._ssh_client.get_text_content(logfile_path)) raise RemoteMySQLSourceError(err)
def restore_mysql(cfg, dst, backup_copy, cache): """Restore from mysql backup""" LOG.debug('mysql: %r', cfg) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(cfg) exit(1) try: ensure_empty(dst) dst_storage = get_destination( cfg, get_hostname_from_backup_copy(backup_copy) ) key = dst_storage.basename(backup_copy) copy = dst_storage.status()[key] if cache: restore_from_mysql(cfg, copy, dst, cache=Cache(cache)) else: restore_from_mysql(cfg, copy, dst) except (TwinDBBackupError, CacheException) as err: LOG.error(err) exit(1) except (OSError, IOError) as err: LOG.error(err) exit(1)
def setup_slave(self, host, user, password, binlog, binlog_position): # noqa # pylint: disable=too-many-arguments """ Change master :param host: Master host name. :type host: str :param user: Replication user. :param password: Replication password :param binlog: Binlog file on the master :param binlog_position: Binlog position """ try: with self._cursor() as cursor: query = "CHANGE MASTER TO " \ "MASTER_HOST = '{master}', " \ "MASTER_USER = '******', " \ "MASTER_PASSWORD = '******', " \ "MASTER_LOG_FILE = '{binlog}', " \ "MASTER_LOG_POS = {binlog_pos}"\ .format( master=host, user=user, password=password, binlog=binlog, binlog_pos=binlog_position) cursor.execute(query) cursor.execute("START SLAVE") return True except pymysql.Error as err: LOG.debug(err) return False
def apply_retention_policy(self, dst, config, run_type, status): """ Delete old backup copies. :param dst: Destination where the backups are stored. :type dst: BaseDestination :param config: Tool configuration :type config: ConfigParser.ConfigParser :param run_type: Run type. :type run_type: str :param status: Backups status. :type status: dict :return: Updated status. :rtype: dict """ prefix = "{remote_path}/{prefix}/mysql/mysql-".format( remote_path=dst.remote_path, prefix=self.get_prefix()) keep_copies = config.getint('retention', '%s_copies' % run_type) objects = dst.list_files(prefix) for backup_copy in get_files_to_delete(objects, keep_copies): LOG.debug('Deleting remote file %s', backup_copy) dst.delete(backup_copy) status = self._delete_from_status(status, dst.remote_path, backup_copy) self._delete_local_files('mysql', config) return status
def _extract_xbstream(input_stream, working_dir): """ Extract xbstream stream in directory :param input_stream: The stream in xbstream format :param working_dir: directory :return: True if extracted successfully """ try: cmd = ['xbstream', '-x'] LOG.debug('Running %s', ' '.join(cmd)) LOG.debug('Working directory: %s', working_dir) proc = Popen(cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE, cwd=working_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', ' '.join(cmd), ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return ret == 0 except OSError as err: LOG.error('Failed to extract xbstream: %s', err) return False
def run_command(command, ok_non_zero=False): """ Run shell command locally :param command: Command to run :type command: list :param ok_non_zero: Don't consider non-zero exit code as an error. :type ok_non_zero: bool :return: file object with stdout as generator to use with ``with`` """ try: LOG.debug("Running %s", " ".join(command)) proc = Popen(command, stderr=PIPE, stdout=PIPE) yield proc.stdout _, cerr = proc.communicate() if proc.returncode and not ok_non_zero: LOG.error( "Command %s exited with error code %d", " ".join(command), proc.returncode, ) LOG.error(cerr) sys.exit(1) else: LOG.debug("Exited with zero code") except OSError as err: LOG.error("Failed to run %s", " ".join(command)) LOG.error(err) sys.exit(1)
def get_stream(self): """ Stream content of one binary file. :return: stream of bytes with the binlog content. """ with self._mysql_client.cursor() as cursor: cursor.execute("SELECT @@log_bin_basename AS log_bin_basename") row = cursor.fetchone() log_bin_basename = row['log_bin_basename'] log_bin_dirname = osp.dirname(log_bin_basename) log_bin_file = osp.join(log_bin_dirname, self._binlog_file) cmd = [ "cat", log_bin_file, ] try: LOG.debug('Running %s', ' '.join(cmd)) proc = Popen(cmd, stderr=PIPE, stdout=PIPE) yield proc.stdout _, cerr = proc.communicate() if proc.returncode: LOG.error('Failed to read from %s: %s', log_bin_file, cerr) exit(1) else: LOG.debug('Successfully streamed %s', log_bin_file) except OSError as err: LOG.error('Failed to run %s: %s', cmd, err) exit(1)
def list_files(self, path, recursive=False, files_only=False): """ Get list of file by prefix :param path: Path :param recursive: Recursive return list of files :type path: str :type recursive: bool :param files_only: Don't list directories if True. Default is False. :type files_only: bool :return: List of files :rtype: list """ rec_cond = "" if recursive else " -maxdepth 1" fil_cond = " -type f" if files_only else "" cmd = "bash -c 'if test -d {path} ; " \ "then find {path}{recursive}{files_only}; fi'" cmd = cmd.format(path=path, recursive=rec_cond, files_only=fil_cond) cout, cerr = self.execute(cmd) LOG.debug("COUT:\n%s", cout) LOG.debug("CERR:\n%s", cerr) if files_only: return cout.split() else: return cout.split()[1:]
def revert_stream(self): """ Un-Apply modifier and return output stream. The Base modifier does nothing, so it will return the input stream without modifications :return: output stream handle """ with self._input as input_stream: LOG.debug("Running %s", " ".join(self._unmodifier_cmd)) proc = Popen( self._unmodifier_cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE, ) yield proc.stdout _, cerr = proc.communicate() if proc.returncode: msg = "%s exited with non-zero code." % " ".join( self._unmodifier_cmd) LOG.error(msg) LOG.error(cerr) raise ModifierException(msg)
def get_container(name, bootstrap_script, client, network, last_n=1): api = client.api api.pull(NODE_IMAGE) cwd = os.getcwd() host_config = api.create_host_config( binds={cwd: { 'bind': '/twindb-backup', 'mode': 'rw', }}, dns=['8.8.8.8']) ip = '172.%d.3.%d' % (network['second_octet'], last_n) networking_config = api.create_networking_config( {network['NAME']: api.create_endpoint_config(ipv4_address=ip)}) LOG.debug(networking_config) container = api.create_container(image=NODE_IMAGE, name='%s_%d' % (name, last_n), ports=[22, 3306], host_config=host_config, networking_config=networking_config, volumes=['/twindb-backup'], command='bash %s' % bootstrap_script) container['ip'] = ip LOG.info('Created container %r', container) try: api.start(container['Id']) LOG.info('Started %r', container) return container except APIError as err: LOG.error(err) client.api.remove_container(container=container['Id'], force=True)
def container_network(docker_client): api = docker_client.api network = None network_params = { 'NAME': NETWORK_NAME, 'subnet': None, 'second_octet': None } ipam_config = _ipam_config() subnet = ipam_config['Config'][0]['Subnet'] network_params['subnet'] = subnet network_params['second_octet'] = int(subnet.split('.')[1]) try: network = api.create_network(name=NETWORK_NAME, driver="bridge", ipam=ipam_config, check_duplicate=True) LOG.info('Created subnet %s', network_params['subnet']) LOG.debug(network) except APIError as err: if err.status_code == 500: LOG.info('Network %r already exists', network) else: raise yield network_params if network: api.remove_network(net_id=network['Id'])
def container_network(docker_client): api = docker_client.api network = None network_params = { "NAME": NETWORK_NAME, "subnet": None, "second_octet": None } ipam_config = _ipam_config() subnet = ipam_config["Config"][0]["Subnet"] network_params["subnet"] = subnet network_params["second_octet"] = int(subnet.split(".")[1]) try: network = api.create_network(name=NETWORK_NAME, driver="bridge", ipam=ipam_config, check_duplicate=True) LOG.info("Created subnet %s", network_params["subnet"]) LOG.debug(network) except APIError as err: if err.status_code == 500: LOG.info("Network %r already exists", network) else: raise yield network_params if network: api.remove_network(net_id=network["Id"])
def _backup_stream(config, src, dst, callbacks=None): stream = src.get_stream() # Gzip modifier stream = Gzip(stream).get_stream() src.suffix += '.gz' # KeepLocal modifier try: keep_local_path = config.get('destination', 'keep_local_path') kl_modifier = KeepLocal(stream, os.path.join(keep_local_path, src.get_name())) stream = kl_modifier.get_stream() if callbacks is not None: callbacks.append((kl_modifier, { 'keep_local_path': keep_local_path, 'dst': dst })) except ConfigParser.NoOptionError: LOG.debug('keep_local_path is not present in the config file') # GPG modifier try: stream = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring')).get_stream() src.suffix += '.gpg' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass except ModifierException as err: LOG.warning(err) LOG.warning('Will skip encryption') if not dst.save(stream, src.get_name()): LOG.error('Failed to save backup copy %s', src.get_name()) exit(1)
def get_stream(self, copy): """ Get a PIPE handler with content of the backup copy streamed from the destination. :param copy: Backup copy. :type copy: BaseCopy :return: Standard output. :rtype: file """ path = "%s/%s" % (self.remote_path, copy.key) cmd = "cat %s" % path def _read_write_chunk(channel, write_fd, size=1024): while channel.recv_ready(): chunk = channel.recv(size) LOG.debug('read %d bytes', len(chunk)) if chunk: os.write(write_fd, chunk) def _write_to_pipe(read_fd, write_fd): try: os.close(read_fd) with self._ssh_client.session() as channel: LOG.debug('Executing %s', cmd) channel.exec_command(cmd) while not channel.exit_status_ready(): _read_write_chunk(channel, write_fd) LOG.debug('closing channel') _read_write_chunk(channel, write_fd) channel.recv_exit_status() except KeyboardInterrupt: return read_process = None try: read_pipe, write_pipe = os.pipe() read_process = Process(target=_write_to_pipe, args=(read_pipe, write_pipe), name='_write_to_pipe') read_process.start() os.close(write_pipe) yield read_pipe os.close(read_pipe) read_process.join() if read_process.exitcode: raise SshDestinationError('Failed to download %s' % path) LOG.debug('Successfully streamed %s', path) finally: if read_process: read_process.join()
def restore_from_file(twindb_config, copy, dst_dir): """ Restore a directory from a backup copy in the directory :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig :param copy: Instance of BaseCopy or and inheriting classes. :type copy: BaseCopy :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str """ LOG.info('Restoring %s in %s', copy.key, dst_dir) mkdir_p(dst_dir) restore_start = time.time() keep_local_path = twindb_config.keep_local_path if keep_local_path and os.path.exists(osp.join(keep_local_path, copy.key)): dst = Local(osp.join(keep_local_path, copy.key)) stream = dst.get_stream(copy) else: dst = twindb_config.destination() stream = dst.get_stream(copy) # GPG modifier if twindb_config.gpg: gpg = Gpg(stream, twindb_config.gpg.recipient, twindb_config.gpg.keyring, secret_keyring=twindb_config.gpg.secret_keyring) LOG.debug('Decrypting stream') stream = gpg.revert_stream() else: LOG.debug('Not decrypting the stream') with stream as handler: try: LOG.debug('handler type: %s', type(handler)) LOG.debug('stream type: %s', type(stream)) cmd = ["tar", "zvxf", "-"] LOG.debug('Running %s', ' '.join(cmd)) proc = Popen(cmd, stdin=handler, cwd=dst_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', cmd, ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return LOG.info('Successfully restored %s in %s', copy.key, dst_dir) except (OSError, DestinationError) as err: LOG.error('Failed to decompress %s: %s', copy.key, err) exit(1) export_info(twindb_config, data=time.time() - restore_start, category=ExportCategory.files, measure_type=ExportMeasureType.restore)
def get_destination(config, hostname=socket.gethostname()): """ Read config and return instance of Destination class. :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param hostname: Local hostname. :type hostname: str :return: Instance of destination class. :rtype: BaseDestination """ destination = None try: destination = config.get('destination', 'backup_destination') LOG.debug('Destination in the config %s', destination) destination = destination.strip('"\'') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): LOG.critical("Backup destination must be specified " "in the config file") exit(-1) if destination == "ssh": host = config.get('ssh', 'backup_host') try: port = int(config.get('ssh', 'port')) except ConfigParser.NoOptionError: port = 22 try: ssh_key = config.get('ssh', 'ssh_key') except ConfigParser.NoOptionError: ssh_key = '/root/.ssh/id_rsa' LOG.debug('ssh_key is not defined in config. ' 'Will use default %s', ssh_key) user = config.get('ssh', 'ssh_user') remote_path = config.get('ssh', 'backup_dir') return Ssh( remote_path, SshConnectInfo( host=host, port=port, user=user, key=ssh_key), hostname=hostname) elif destination == "s3": bucket = config.get('s3', 'BUCKET').strip('"\'') access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'') secret_access_key = config.get('s3', 'AWS_SECRET_ACCESS_KEY').strip('"\'') default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'') return S3(bucket, AWSAuthOptions(access_key_id, secret_access_key, default_region=default_region), hostname=hostname) else: LOG.critical('Destination %s is not supported', destination) exit(-1)
def _update_backup_info(self, stderr_file): """Update backup_info from stderr""" LOG.debug("xtrabackup error log file %s", stderr_file.name) self._backup_info.lsn = self._get_lsn(stderr_file.name) self._backup_info.binlog_coordinate = self.get_binlog_coordinates( stderr_file.name) os.unlink(stderr_file.name)
def _get_name(self, filename_prefix): LOG.debug('Suffix = %s', self.suffix) self._name = osp.join( self.get_prefix(), self._media_type, "{file}-{time}.{suffix}".format(file=filename_prefix, time=self._created_at, suffix=self._suffix)) return self._name
def verify_mysql(cfg, hostname, dst, backup_copy): """Verify backup""" LOG.debug('mysql: %r', cfg) if not backup_copy: list_available_backups(cfg) exit(1) print(verify_mysql_backup(cfg, dst, backup_copy, hostname))
def restore_from_file(config, backup_copy, dst_dir): """ Restore a directory from a backup copy in the directory :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param backup_copy: Backup name. :type backup_copy: str :param dst_dir: Path to destination directory. Must exist and be empty. :type dst_dir: str """ LOG.info('Restoring %s in %s', backup_copy, dst_dir) mkdir_p(dst_dir) restore_start = time.time() if os.path.exists(backup_copy): dst = Local(backup_copy) stream = dst.get_stream(backup_copy) else: dst = get_destination(config) stream = dst.get_stream(backup_copy) # GPG modifier try: gpg = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring'), secret_keyring=config.get('gpg', 'secret_keyring')) LOG.debug('Decrypting stream') stream = gpg.revert_stream() except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): LOG.debug('Not decrypting the stream') with stream as handler: try: LOG.debug('handler type: %s', type(handler)) LOG.debug('stream type: %s', type(stream)) cmd = ["tar", "zvxf", "-"] LOG.debug('Running %s', ' '.join(cmd)) proc = Popen(cmd, stdin=handler, cwd=dst_dir) cout, cerr = proc.communicate() ret = proc.returncode if ret: LOG.error('%s exited with code %d', cmd, ret) if cout: LOG.error('STDOUT: %s', cout) if cerr: LOG.error('STDERR: %s', cerr) return LOG.info('Successfully restored %s in %s', backup_copy, dst_dir) except (OSError, DestinationError) as err: LOG.error('Failed to decompress %s: %s', backup_copy, err) exit(1) export_info(config, data=time.time() - restore_start, category=ExportCategory.files, measure_type=ExportMeasureType.restore)
def set_open_files_limit(): """Detect maximum supported number of open file and set it""" max_files = getrlimit(RLIMIT_NOFILE)[0] while True: try: setrlimit(RLIMIT_NOFILE, (max_files, max_files)) max_files += 1 except ValueError: break LOG.debug('Setting max files limit to %d', max_files)
def _update_backup_info(self, stderr_file): """Update backup_info from stderr""" LOG.debug('xtrabackup error log file %s', stderr_file.name) self._backup_info.lsn = self._get_lsn(stderr_file.name) self._backup_info.binlog_coordinate = self.get_binlog_coordinates( stderr_file.name ) os.unlink(stderr_file.name)
def save(self, handler, filepath): """ Read from handler and save it to Amazon S3 :param filepath: save backup copy in a file with this name :param handler: stdout handler from backup source """ with handler as file_obj: ret = self._upload_object(file_obj, filepath) LOG.debug('Returning code %d', ret)
def clone_config(self, dst): """ Clone config to destination server :param dst: Destination server :type dst: Ssh """ cfg_path = self._get_root_my_cnf() LOG.debug("Root my.cnf is: %s", cfg_path) self._save_cfg(dst, cfg_path)
def test_get_stream(gs): status = MySQLStatus(dst=gs) copy = status['master1/daily/mysql/mysql-2019-04-04_05_29_05.xbstream.gz'] with gs.get_stream(copy) as stream: LOG.debug('starting reading from pipe') content = stream.read() LOG.debug('finished reading from pipe') assert len(content), 'Failed to read from GS' LOG.info('Read %d bytes', len(content))
def verify_mysql_backup(twindb_config, dst_path, backup_file, hostname=None): """ Restore mysql backup and measure time :param hostname: :param backup_file: :param dst_path: :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig """ dst = twindb_config.destination(backup_source=hostname) status = MySQLStatus(dst=dst) copy = None if backup_file == "latest": copy = status.latest_backup else: for copy in status: if backup_file.endswith(copy.key): break if copy is None: return json.dumps({ 'backup_copy': backup_file, 'restore_time': 0, 'success': False }, indent=4, sort_keys=True) start_restore_time = time.time() success = True tmp_dir = tempfile.mkdtemp() try: LOG.debug('Verifying backup copy in %s', tmp_dir) restore_from_mysql(twindb_config, copy, dst_path, tmp_dir) edit_backup_my_cnf(dst_path) except (TwinDBBackupError, OSError, IOError) as err: LOG.error(err) LOG.debug(traceback.format_exc()) success = False finally: shutil.rmtree(tmp_dir, ignore_errors=True) end_restore_time = time.time() restore_time = end_restore_time - start_restore_time return json.dumps({ 'backup_copy': copy.key, 'restore_time': restore_time, 'success': success }, indent=4, sort_keys=True)
def bucket_name(): travis_job_number = os.environ.get('TRAVIS_JOB_NUMBER') LOG.debug('TRAVIS_JOB_NUMBER=%s' % travis_job_number) number = random.randint(0, 1000000) LOG.debug('Default job number %d' % number) if travis_job_number: bucket = 'twindb-backup-test-travis-%s' % travis_job_number else: bucket = 'twindb-backup-test-travis-%d' % number return '%s-%s' % (bucket, time.time())
def candidate_parent(self, run_type): """ Find a backup copy that can be a parent :param run_type: See :func:`~get_backup_type`. :return: Backup copy or None :rtype: MySQLCopy """ full_backup_index = INTERVALS.index(run_type) LOG.debug('Looking a parent candidate for %s run', run_type) for i in xrange(full_backup_index, len(INTERVALS)): period_copies = getattr(self, INTERVALS[i]) LOG.debug( 'Checking %d %s copies', len(period_copies), INTERVALS[i] ) for _, value in period_copies.iteritems(): try: if value.type == 'full': LOG.debug('Found parent %r', value) return value except KeyError: return None LOG.debug('No eligible parents') return None
def enable_wsrep_desync(self): """ Try to enable wsrep_desync :return: True if wsrep_desync was enabled. False if not supported """ try: with self.get_connection() as connection: with connection.cursor() as cursor: cursor.execute('SET GLOBAL wsrep_desync=ON') return True except pymysql.Error as err: LOG.debug(err) return False
def _get_name(self, filename_prefix): LOG.debug('Suffix = %s', self.suffix) self._name = osp.join( self.get_prefix(), self._media_type, "{file}-{time}.{suffix}".format( file=filename_prefix, time=self._created_at, suffix=self._suffix ) ) return self._name
def s3_client(bucket_name): LOG.debug('Bucket: %s' % bucket_name) client = S3( bucket=bucket_name, aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] ) try: assert client.create_bucket() yield client finally: client.delete_bucket(force=True)
def get_stream(self): """ Compress the input stream and return it as the output stream :return: output stream handle """ with self._input as input_stream: LOG.debug('Running %s', ' '.join(self._modifier_cmd)) proc = Popen( self._modifier_cmd, stdin=input_stream, stdout=PIPE, stderr=PIPE ) yield proc.stdout proc.communicate()
def _backup_stream(config, src, dst, callbacks=None): """ :param config: Tool config :type config: TwinDBBackupConfig :param src: :param dst: :param callbacks: :return: """ stream = src.get_stream() # Compression modifier cmp_modifier = config.compression.get_modifier(stream) stream = cmp_modifier.get_stream() src.suffix += cmp_modifier.suffix # KeepLocal modifier if config.keep_local_path: keep_local_path = config.keep_local_path kl_modifier = KeepLocal( stream, osp.join( keep_local_path, src.get_name() ) ) stream = kl_modifier.get_stream() if callbacks is not None: callbacks.append((kl_modifier, { 'keep_local_path': keep_local_path, 'dst': dst })) else: LOG.debug('keep_local_path is not present in the config file') # GPG modifier if config.gpg: gpg_modifier = Gpg( stream, config.gpg.recipient, config.gpg.keyring ) stream = gpg_modifier.get_stream() src.suffix += '.gpg' dst.save(stream, src.get_name())
def apply_retention_policy(self, dst, config, run_type): """Apply retention policy """ prefix = "{remote_path}/{prefix}/files/{file}".format( remote_path=dst.remote_path, prefix=self.get_prefix(), file=self._sanitize_filename() ) keep_copies = getattr(config.retention, run_type) backups_list = dst.list_files(prefix) LOG.debug('Remote copies: %r', backups_list) for backup_copy in get_files_to_delete(backups_list, keep_copies): LOG.debug('Deleting remote file %s', backup_copy) dst.delete(backup_copy) self._delete_local_files(self._sanitize_filename(), config)
def backup(ctx, run_type, lock_file, binlogs_only): """Run backup job""" try: run_backup_job( ctx.obj['twindb_config'], run_type, lock_file=lock_file, binlogs_only=binlogs_only ) except TwinDBBackupError as err: LOG.error(err) LOG.debug(traceback.format_exc()) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def run_intervals(self): """ Run intervals config. When to run or not the backup. :return: Configuration with data on whether to run the backup tool now. :rtype: RunIntervals """ kwargs = {} try: kwargs = { i: self.__cfg.getboolean('intervals', 'run_%s' % i) for i in INTERVALS } except (NoOptionError, NoSectionError) as err: LOG.debug(err) LOG.debug('Will use default retention policy') return RunIntervals(**kwargs)
def delete(self, path): """Deletes an S3 object. :param path: Key of S3 object. :type path: str :raise S3DestinationError: if failed to delete object. """ key = path.replace( 's3://%s/' % self._bucket, '' ) if path.startswith('s3://') else path s3client = boto3.resource('s3') bucket = s3client.Bucket(self._bucket) s3obj = s3client.Object(bucket.name, key) LOG.debug('deleting s3://%s/%s', bucket.name, key) return s3obj.delete()
def restore_file(ctx, dst, backup_copy): """Restore from file backup""" LOG.debug('file: %r', ctx.obj['twindb_config']) if not backup_copy: LOG.info('No backup copy specified. Choose one from below:') list_available_backups(ctx.obj['twindb_config']) exit(1) try: ensure_empty(dst) copy = FileCopy(path=backup_copy) restore_from_file(ctx.obj['twindb_config'], copy, dst) except TwinDBBackupError as err: LOG.error(err) exit(1) except KeyboardInterrupt: LOG.info('Exiting...') kill_children() exit(1)
def verify_mysql(ctx, hostname, dst, backup_copy): """Verify backup""" LOG.debug('mysql: %r', ctx.obj['twindb_config']) try: if not backup_copy: list_available_backups(ctx.obj['twindb_config']) exit(1) print( verify_mysql_backup( ctx.obj['twindb_config'], dst, backup_copy, hostname ) ) finally: shutil.rmtree(dst, ignore_errors=True)
def test_init_creates_instance_from_new(status_raw_content): status = MySQLStatus(status_raw_content) assert status.version == STATUS_FORMAT_VERSION key = 'master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz' copy = MySQLCopy( 'master1', 'hourly', 'mysql-2018-03-28_04_11_16.xbstream.gz', backup_started=1522210276, backup_finished=1522210295, binlog='mysql-bin.000001', parent='master1/daily/mysql/mysql-2018-03-28_04_09_53.xbstream.gz', lsn=19903207, config={ '/etc/my.cnf': """[mysqld] datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock user=mysql # Disabling symbolic-links is recommended to prevent assorted security risks symbolic-links=0 server_id=100 gtid_mode=ON log-bin=mysql-bin log-slave-updates enforce-gtid-consistency [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid """ }, position=46855, type='incremental' ) assert key in status.hourly LOG.debug("Copy %s: %r", copy.key, copy) LOG.debug("Copy from status %s: %r", key, status[key]) assert status[key] == copy
def get_stream(self): """ Get a PIPE handler with content of the source :return: """ cmd = [ self._xtrabackup, "--defaults-file=%s" % self._connect_info.defaults_file, "--stream=xbstream", "--host=127.0.0.1", "--backup" ] cmd += ["--target-dir", "."] if self.is_galera(): cmd.append("--galera-info") cmd.append("--no-backup-locks") if self.incremental: cmd += [ "--incremental-basedir", ".", "--incremental-lsn=%d" % self._parent_lsn ] # If this is a Galera node then additional step needs to be taken to # prevent the backups from locking up the cluster. wsrep_desynced = False LOG.debug('Running %s', ' '.join(cmd)) stderr_file = tempfile.NamedTemporaryFile(delete=False) try: if self.is_galera(): wsrep_desynced = self.enable_wsrep_desync() LOG.debug('Running %s', ' '.join(cmd)) proc_xtrabackup = Popen(cmd, stderr=stderr_file, stdout=PIPE) yield proc_xtrabackup.stdout proc_xtrabackup.communicate() if proc_xtrabackup.returncode: LOG.error('Failed to run xtrabackup. ' 'Check error output in %s', stderr_file.name) try: if LOG.debug_enabled: with open(stderr_file.name) as xb_out: for line in xb_out: print(line, end='', file=sys.stderr) except AttributeError: pass self.dst.delete(self.get_name()) exit(1) else: LOG.debug('Successfully streamed xtrabackup output') self._update_backup_info(stderr_file) except OSError as err: LOG.error('Failed to run %s: %s', cmd, err) exit(1) finally: if wsrep_desynced: self.disable_wsrep_desync()
def _upload_object(self, file_obj, object_key): """Upload objects to S3 in streaming fashion. :param file file_obj: A file like object to upload. At a minimum, it must implement the read method, and must return bytes. :param str object_key: The destination key where to upload the object. :raise S3DestinationError: if failed to upload object. """ remote_name = "s3://{bucket}/{name}".format( bucket=self._bucket, name=object_key ) LOG.debug("Generating S3 transfer config") s3_transfer_config = self.get_transfer_config() LOG.debug("Starting to stream to %s", remote_name) try: self.s3_client.upload_fileobj(file_obj, self._bucket, object_key, Config=s3_transfer_config) LOG.debug("Successfully streamed to %s", remote_name) except ClientError as err: raise S3DestinationError(err) return self._validate_upload(object_key)
def _validate_upload(self, object_key): """ Validates that upload of an object was successful. Raises an exception if the response code is not 200. :raise S3DestinationError: if object is not available on the destination. """ remote_name = "s3://{bucket}/{name}".format( bucket=self._bucket, name=object_key ) LOG.debug("Validating upload to %s", remote_name) response = self.s3_client.get_object(Bucket=self._bucket, Key=object_key) self.validate_client_response(response) LOG.debug("Upload successfully validated") return 0