def apply_retention_policy(self, dst, config, run_type, status): """ Delete old backup copies. :param dst: Destination where the backups are stored. :type dst: BaseDestination :param config: Tool configuration :type config: TwinDBBackupConfig :param run_type: Run type. :type run_type: str :param status: Backups status. :type status: Status :return: Updated status. :rtype: Status """ prefix = osp.join(dst.remote_path, self.get_prefix(), "mysql") keep_copies = getattr(config.retention, run_type) backups_list = dst.list_files(prefix, files_only=True) LOG.debug("Remote copies: %r", backups_list) for backup_file in get_files_to_delete(backups_list, keep_copies): LOG.debug("Deleting remote file %s", backup_file) dst.delete(backup_file) try: status.remove(backup_file) except StatusKeyNotFound as err: LOG.warning(err) LOG.debug("Status: %r", status) self._delete_local_files("mysql", config) return status
def apply_retention_policy(self, dst, config, run_type, status): """ Delete old backup copies. :param dst: Destination where the backups are stored. :type dst: BaseDestination :param config: Tool configuration :type config: ConfigParser.ConfigParser :param run_type: Run type. :type run_type: str :param status: Backups status. :type status: Status :return: Updated status. :rtype: Status """ prefix = "{remote_path}/{prefix}/mysql/mysql-".format( remote_path=dst.remote_path, prefix=self.get_prefix()) keep_copies = config.getint('retention', '%s_copies' % run_type) backups_list = dst.list_files(prefix) LOG.debug('Remote copies: %r', backups_list) for backup_copy in get_files_to_delete(backups_list, keep_copies): LOG.debug('Deleting remote file %s', backup_copy) dst.delete(backup_copy) try: status.remove(run_type, dst.basename(backup_copy)) except StatusKeyNotFound as err: LOG.warning(err) LOG.debug('Status: %r', status) self._delete_local_files('mysql', config) return status
def _mysql_service(dst, action): """Start or stop MySQL service :param dst: Destination server :type dst: Ssh :param action: string start or stop :type action: str """ for service in ['mysqld', 'mysql']: try: return dst.execute_command("PATH=$PATH:/sbin sudo service %s %s" % (service, action), quiet=True) except SshClientException as err: LOG.debug(err) try: LOG.warning( 'Failed to %s MySQL with an init script. ' 'Will try to %s mysqld.', action, action) if action == "start": ret = dst.execute_command( "PATH=$PATH:/sbin sudo bash -c 'nohup mysqld &'", background=True) time.sleep(10) return ret elif action == "stop": return dst.execute_command( "PATH=$PATH:/sbin sudo kill $(pidof mysqld)") except SshClientException as err: LOG.error(err) raise TwinDBBackupError('Failed to %s MySQL on %r' % (action, dst))
def main(ctx, cfg, debug, config, version): """ Main entry point :param ctx: context (See Click docs (http://click.pocoo.org/6/) for explanation) :param cfg: instance of ConfigParser :type cfg: ConfigParser.ConfigParser :param debug: if True enabled debug logging :type debug: bool :param config: path to configuration file :type config: str :param version: If True print version string :type version: bool """ if not ctx.invoked_subcommand: if version: print(__version__) exit(0) else: print(ctx.get_help()) exit(-1) setup_logging(LOG, debug=debug) if os.path.exists(config): cfg.read(config) else: LOG.warning("Config file %s doesn't exist", config)
def _backup_stream(config, src, dst, callbacks=None): stream = src.get_stream() # Gzip modifier stream = Gzip(stream).get_stream() src.suffix += '.gz' # KeepLocal modifier try: keep_local_path = config.get('destination', 'keep_local_path') kl_modifier = KeepLocal(stream, os.path.join(keep_local_path, src.get_name())) stream = kl_modifier.get_stream() if callbacks is not None: callbacks.append((kl_modifier, { 'keep_local_path': keep_local_path, 'dst': dst })) except ConfigParser.NoOptionError: LOG.debug('keep_local_path is not present in the config file') # GPG modifier try: stream = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring')).get_stream() src.suffix += '.gpg' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass except ModifierException as err: LOG.warning(err) LOG.warning('Will skip encryption') if not dst.save(stream, src.get_name()): LOG.error('Failed to save backup copy %s', src.get_name()) exit(1)
def list_files(self, prefix, recursive=False, pattern=None, files_only=False): """ List files in the destination that have common prefix. :param prefix: Common prefix. May include the bucket name. (e.g. ``s3://my_bucket/foo/``) or simply a prefix in the bucket (e.g. ``foo/``). :type prefix: str :param recursive: Does nothing for this class. :return: sorted list of file names. :param pattern: files must match with this regexp if specified. :type pattern: str :param files_only: Does nothing for this class. :return: Full S3 url in form ``s3://bucket/path/to/file``. :rtype: list(str) :raise S3DestinationError: if failed to list files. """ s3client = boto3.resource('s3') bucket = s3client.Bucket(self.bucket) LOG.debug('Listing bucket %s', self.bucket) LOG.debug('prefix = %s', prefix) norm_prefix = prefix.replace('s3://%s' % bucket.name, '') norm_prefix = norm_prefix.lstrip('/') LOG.debug('normal prefix = %s', norm_prefix) # Try to list the bucket several times # because of intermittent error NoSuchBucket: # https://travis-ci.org/twindb/backup/jobs/204053690 expire = time.time() + S3_READ_TIMEOUT retry_interval = 2 while time.time() < expire: try: files = [] all_objects = bucket.objects.filter(Prefix=norm_prefix) for file_object in all_objects: if pattern: if re.search(pattern, file_object.key): files.append('s3://{bucket}/{key}'.format( bucket=self.bucket, key=file_object.key)) else: files.append('s3://{bucket}/{key}'.format( bucket=self.bucket, key=file_object.key)) return sorted(files) except ClientError as err: LOG.warning('%s. Will retry in %d seconds.', err, retry_interval) time.sleep(retry_interval) retry_interval *= 2 raise S3DestinationError('Failed to list files.')
def _retention(self, section): kwargs = {} for i in INTERVALS: option = "%s_copies" % i try: kwargs[i] = self.__cfg.getint(section, option) except (NoOptionError, NoSectionError): LOG.warning("Option %s is not defined in section %s", option, section) return RetentionPolicy(**kwargs)
def _get_status_key(status, key, variable): LOG.debug('status = %s', json.dumps(status, indent=4, sort_keys=True)) LOG.debug('key = %s', key) try: for run_type in INTERVALS: if key in status[run_type]: return status[run_type][key][variable] except KeyError: pass LOG.warning('key %s is not found', key) return None
def _retention(self, section): kwargs = {} for i in INTERVALS: option = '%s_copies' % i try: kwargs[i] = self.__cfg.getint(section, option) except (NoOptionError, NoSectionError): LOG.warning( 'Option %s is not defined in section %s', option, section ) return RetentionPolicy(**kwargs)
def backup_files(run_type, config): """Backup local directories :param run_type: Run type :type run_type: str :param config: Configuration :type config: ConfigParser.ConfigParser """ for directory in get_directories_to_backup(config): LOG.debug('copying %s', directory) src = FileSource(directory, run_type) dst = get_destination(config) stream = src.get_stream() # Gzip modifier stream = Gzip(stream).get_stream() src.suffix += '.gz' # KeepLocal modifier try: keep_local_path = config.get('destination', 'keep_local_path') # src.suffix = 'tar.gz.aaa' dst_name = src.get_name() kl_modifier = KeepLocal(stream, os.path.join(keep_local_path, dst_name)) stream = kl_modifier.get_stream() except ConfigParser.NoOptionError: pass # GPG modifier try: keyring = config.get('gpg', 'keyring') recipient = config.get('gpg', 'recipient') gpg = Gpg(stream, recipient, keyring) stream = gpg.get_stream() src.suffix += '.gpg' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass except ModifierException as err: LOG.warning(err) LOG.warning('Will skip encryption') dst.save(stream, src.get_name()) src.apply_retention_policy(dst, config, run_type)
def main( ctx, debug, # pylint: disable=too-many-arguments config, version, xtrabackup_binary, xbstream_binary): """ Main entry point :param ctx: context (See Click docs (http://click.pocoo.org/6/) for explanation) :param debug: if True enabled debug logging :type debug: bool :param config: path to configuration file :type config: str :param version: If True print version string :type version: bool :param xtrabackup_binary: Path to xtrabackup binary. :type xtrabackup_binary: str :param xbstream_binary: Path to xbstream binary. :type xbstream_binary: str """ if not ctx.invoked_subcommand: if version: print(__version__) exit(0) else: print(ctx.get_help()) exit(-1) setup_logging(LOG, debug=debug) if os.path.exists(config): ctx.obj = {'twindb_config': TwinDBBackupConfig(config_file=config)} if xtrabackup_binary is not None: ctx.obj['twindb_config'].mysql.xtrabackup_binary = \ xtrabackup_binary if xbstream_binary is not None: ctx.obj['twindb_config'].mysql.xbstream_binary = \ xbstream_binary else: LOG.warning("Config file %s doesn't exist", config) exit(os.EX_CONFIG)
def clone(self, dest_host, port, compress=False): """ Send backup to destination host :param dest_host: Destination host :type dest_host: str :param port: Port to sending backup :type port: int :param compress: If True compress stream :type compress: bool :raise RemoteMySQLSourceError: if any error """ retry = 1 retry_time = 2 error_log = "/tmp/{src}_{src_port}-{dst}_{dst_port}.log".format( src=self._ssh_client.host, src_port=self._ssh_client.port, dst=dest_host, dst_port=port, ) if compress: compress_cmd = "| gzip -c - " else: compress_cmd = "" cmd = ( 'bash -c "sudo %s ' "--stream=xbstream " "--host=127.0.0.1 " "--backup " "--target-dir ./ 2> %s" ' %s | ncat %s %d --send-only"' % (self._xtrabackup, error_log, compress_cmd, dest_host, port) ) while retry < 3: try: return self._ssh_client.execute(cmd) except SshClientException as err: LOG.warning(err) LOG.info("Will try again in after %d seconds", retry_time) time.sleep(retry_time) retry_time *= 2 retry += 1
def find_files(self, prefix, run_type): """ Find files with common prefix and given run type. :param prefix: Common prefix. :type prefix: str :param run_type: daily, hourly, etc :type run_type: str :return: list of file names :rtype: list(str) :raise S3DestinationError: if failed to find files. """ s3client = boto3.resource('s3') bucket = s3client.Bucket(self.bucket) LOG.debug('Listing %s in bucket %s', prefix, bucket) # Try to list the bucket several times # because of intermittent error NoSuchBucket: # https://travis-ci.org/twindb/backup/jobs/204066704 retry_timeout = time.time() + S3_READ_TIMEOUT retry_interval = 2 while time.time() < retry_timeout: try: files = [] all_objects = bucket.objects.filter(Prefix='') for file_object in all_objects: if "/" + run_type + "/" in file_object.key: files.append("s3://%s/%s" % (self.bucket, file_object.key)) return sorted(files) except ClientError as err: LOG.warning('%s. Will retry in %d seconds.', err, retry_interval) time.sleep(retry_interval) retry_interval *= 2 except Exception as err: LOG.error('Failed to list objects in bucket %s: %s', self.bucket, err) raise raise S3DestinationError('Failed to find files.')
def main(ctx, cfg, debug, # pylint: disable=too-many-arguments config, version, xtrabackup_binary=XTRABACKUP_BINARY, xbstream_binary=XBSTREAM_BINARY): """ Main entry point :param ctx: context (See Click docs (http://click.pocoo.org/6/) for explanation) :param cfg: instance of ConfigParser :type cfg: ConfigParser.ConfigParser :param debug: if True enabled debug logging :type debug: bool :param config: path to configuration file :type config: str :param version: If True print version string :type version: bool :param xtrabackup_binary: Path to xtrabackup binary. :type xtrabackup_binary: str :param xbstream_binary: Path to xbstream binary. :type xbstream_binary: str """ if not ctx.invoked_subcommand: if version: print(__version__) exit(0) else: print(ctx.get_help()) exit(-1) setup_logging(LOG, debug=debug) if os.path.exists(config): cfg.read(config) try: cfg.set('mysql', 'xtrabackup_binary', xtrabackup_binary) cfg.set('mysql', 'xbstream_binary', xbstream_binary) except NoSectionError: # if there is no mysql section, we will not backup mysql pass else: LOG.warning("Config file %s doesn't exist", config)
def _get_file_content(self, path): attempts = 10 # up to 1024 seconds sleep_time = 2 while sleep_time <= 2**attempts: try: response = self.s3_client.get_object(Bucket=self.bucket, Key=path) self.validate_client_response(response) content = response['Body'].read() return content except ClientError as err: LOG.warning('Failed to read s3://%s/%s', self.bucket, path) LOG.warning(err) LOG.info('Will try again in %d seconds', sleep_time) time.sleep(sleep_time) sleep_time *= 2 msg = 'Failed to read s3://%s/%s after %d attempts' \ % (self.bucket, path, attempts) raise TwinDBBackupError(msg)
def apply_retention_policy(self, dst, config, run_type, status): """ Delete old backup copies. :param dst: Destination where the backups are stored. :type dst: BaseDestination :param config: Tool configuration :type config: TwinDBBackupConfig :param run_type: Run type. :type run_type: str :param status: Backups status. :type status: Status :return: Updated status. :rtype: Status """ prefix = osp.join( dst.remote_path, self.get_prefix(), 'mysql' ) keep_copies = getattr(config.retention, run_type) backups_list = dst.list_files( prefix, files_only=True ) LOG.debug('Remote copies: %r', backups_list) for backup_file in get_files_to_delete(backups_list, keep_copies): LOG.debug('Deleting remote file %s', backup_file) dst.delete(backup_file) try: status.remove(backup_file) except StatusKeyNotFound as err: LOG.warning(err) LOG.debug('Status: %r', status) self._delete_local_files('mysql', config) return status
def main(ctx, debug, # pylint: disable=too-many-arguments config, version, xtrabackup_binary, xbstream_binary): """ Main entry point :param ctx: context (See Click docs (http://click.pocoo.org/6/) for explanation) :param debug: if True enabled debug logging :type debug: bool :param config: path to configuration file :type config: str :param version: If True print version string :type version: bool :param xtrabackup_binary: Path to xtrabackup binary. :type xtrabackup_binary: str :param xbstream_binary: Path to xbstream binary. :type xbstream_binary: str """ if not ctx.invoked_subcommand: if version: print(__version__) exit(0) else: print(ctx.get_help()) exit(-1) setup_logging(LOG, debug=debug) if os.path.exists(config): ctx.obj = { 'twindb_config': TwinDBBackupConfig(config_file=config) } if xtrabackup_binary is not None: ctx.obj['twindb_config'].mysql.xtrabackup_binary = \ xtrabackup_binary if xbstream_binary is not None: ctx.obj['twindb_config'].mysql.xbstream_binary = \ xbstream_binary else: LOG.warning("Config file %s doesn't exist", config)
def clone(self, dest_host, port, compress=False): """ Send backup to destination host :param dest_host: Destination host :type dest_host: str :param port: Port to sending backup :type port: int :param compress: If True compress stream :type compress: bool :raise RemoteMySQLSourceError: if any error """ retry = 1 retry_time = 2 error_log = "/tmp/{src}_{src_port}-{dst}_{dst_port}.log".format( src=self._ssh_client.host, src_port=self._ssh_client.port, dst=dest_host, dst_port=port ) if compress: compress_cmd = "| gzip -c - " else: compress_cmd = "" cmd = "bash -c \"sudo %s " \ "--stream=xbstream " \ "--host=127.0.0.1 " \ "--backup " \ "--target-dir ./ 2> %s" \ " %s | ncat %s %d --send-only\"" \ % (self._xtrabackup, error_log, compress_cmd, dest_host, port) while retry < 3: try: return self._ssh_client.execute(cmd) except SshClientException as err: LOG.warning(err) LOG.info('Will try again in after %d seconds', retry_time) time.sleep(retry_time) retry_time *= 2 retry += 1
def _get_file_content(self, path): attempts = 10 # up to 1024 seconds sleep_time = 2 while sleep_time <= 2**attempts: try: response = self.s3_client.get_object( Bucket=self._bucket, Key=path ) self.validate_client_response(response) content = response['Body'].read() return content except ClientError as err: LOG.warning('Failed to read s3://%s/%s', self._bucket, path) LOG.warning(err) LOG.info('Will try again in %d seconds', sleep_time) time.sleep(sleep_time) sleep_time *= 2 msg = 'Failed to read s3://%s/%s after %d attempts' \ % (self._bucket, path, attempts) raise OperationError(msg)
def list_files(self, prefix, recursive=False): """ List files in the destination that have common prefix. :param prefix: Common prefix. :type prefix: str :param recursive: Does nothing for this class. :return: sorted list of file names :rtype: list(str) :raise S3DestinationError: if failed to list files. """ s3client = boto3.resource('s3') bucket = s3client.Bucket(self.bucket) LOG.debug('Listing %s in bucket %s', prefix, self.bucket) norm_prefix = prefix.replace('s3://%s/' % bucket.name, '') LOG.debug('norm_prefix = %s', norm_prefix) # Try to list the bucket several times # because of intermittent error NoSuchBucket: # https://travis-ci.org/twindb/backup/jobs/204053690 retry_timeout = time.time() + S3_READ_TIMEOUT retry_interval = 2 while time.time() < retry_timeout: try: files = [] all_objects = bucket.objects.filter(Prefix=norm_prefix) for file_object in all_objects: files.append(file_object.key) return sorted(files) except ClientError as err: LOG.warning('%s. Will retry in %d seconds.', err, retry_interval) time.sleep(retry_interval) retry_interval *= 2 raise S3DestinationError('Failed to list files.')
def _download_object(s3_client, bucket_name, key, read_fd, write_fd): # The read end of the pipe must be closed in the child process # before we start writing to it. os.close(read_fd) with os.fdopen(write_fd, 'wb') as w_pipe: try: retry_interval = 2 for _ in xrange(10): try: s3_client.download_fileobj(bucket_name, key, w_pipe) return except ClientError as err: LOG.warning(err) LOG.warning('Will retry in %d seconds', retry_interval) time.sleep(retry_interval) retry_interval *= 2 except IOError as err: LOG.error(err) exit(1)
def _get_file_content(self, path): attempts = 10 # up to 1024 seconds sleep_time = 2 while sleep_time <= 2**attempts: try: response = self.s3_client.get_object(Bucket=self._bucket, Key=path) self.validate_client_response(response) content = response["Body"].read() return content except ClientError as err: LOG.warning("Failed to read s3://%s/%s", self._bucket, path) LOG.warning(err) LOG.info("Will try again in %d seconds", sleep_time) time.sleep(sleep_time) sleep_time *= 2 msg = "Failed to read s3://%s/%s after %d attempts" % ( self._bucket, path, attempts, ) raise OperationError(msg)
def _mysql_service(dst, action): """Start or stop MySQL service :param dst: Destination server :type dst: Ssh :param action: string start or stop :type action: str """ for service in ['mysqld', 'mysql']: try: return dst.execute_command( "PATH=$PATH:/sbin sudo service %s %s" % (service, action), quiet=True ) except SshClientException as err: LOG.debug(err) try: LOG.warning('Failed to %s MySQL with an init script. ' 'Will try to %s mysqld.', action, action) if action == "start": ret = dst.execute_command( "PATH=$PATH:/sbin sudo bash -c 'nohup mysqld &'", background=True ) time.sleep(10) return ret elif action == "stop": return dst.execute_command( "PATH=$PATH:/sbin sudo kill $(pidof mysqld)" ) except SshClientException as err: LOG.error(err) raise OperationError( 'Failed to %s MySQL on %r' % (action, dst) )
def clone(self, dest_host, port, compress=False): """ Send backup to destination host :param dest_host: Destination host :type dest_host: str :param port: Port to sending backup :type port: int :param compress: If True compress stream :type compress: bool :raise RemoteMySQLSourceError: if any error """ retry = 1 retry_time = 2 error_log = "/tmp/{src}_{src_port}-{dst}_{dst_port}.log".format( src=self._ssh_client.ssh_connect_info.host, src_port=self._ssh_client.ssh_connect_info.port, dst=dest_host, dst_port=port ) if compress: compress_cmd = "| gzip -c - " else: compress_cmd = "" cmd = "bash -c \"sudo innobackupex --stream=xbstream ./ 2> %s" \ " %s | nc %s %d\"" \ % (error_log, compress_cmd, dest_host, port) while retry < 3: try: return self._ssh_client.execute(cmd) except SshClientException as err: LOG.warning(err) LOG.info('Will try again in after %d seconds', retry_time) time.sleep(retry_time) retry_time *= 2 retry += 1
def backup_mysql(run_type, config): """Take backup of local MySQL instance :param run_type: Run type :type run_type: str :param config: Tool configuration :type config: ConfigParser.ConfigParser :return: None """ try: if not config.getboolean('source', 'backup_mysql'): raise TwinDBBackupError('MySQL backups are not enabled in config') except (ConfigParser.NoOptionError, TwinDBBackupError) as err: LOG.debug(err) LOG.debug('Not backing up MySQL') return dst = get_destination(config) try: full_backup = config.get('mysql', 'full_backup') except ConfigParser.NoOptionError: full_backup = 'daily' backup_start = time.time() src = MySQLSource( MySQLConnectInfo(config.get('mysql', 'mysql_defaults_file')), run_type, full_backup, dst) callbacks = [] stream = src.get_stream() src_name = src.get_name() # Gzip modifier stream = Gzip(stream).get_stream() src_name += '.gz' # KeepLocal modifier try: keep_local_path = config.get('destination', 'keep_local_path') kl_modifier = KeepLocal(stream, os.path.join(keep_local_path, src_name)) stream = kl_modifier.get_stream() callbacks.append((kl_modifier, { 'keep_local_path': keep_local_path, 'dst': dst })) except ConfigParser.NoOptionError: LOG.debug('keep_local_path is not present in the config file') # GPG modifier try: stream = Gpg(stream, config.get('gpg', 'recipient'), config.get('gpg', 'keyring')).get_stream() src_name += '.gpg' except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): pass except ModifierException as err: LOG.warning(err) LOG.warning('Will skip encryption') if not dst.save(stream, src_name): LOG.error('Failed to save backup copy %s', src_name) exit(1) status = prepare_status(dst, src, run_type, src_name, backup_start) src.apply_retention_policy(dst, config, run_type, status) dst.status(status) LOG.debug('Callbacks are %r', callbacks) for callback in callbacks: callback[0].callback(**callback[1])
def list_files(self, prefix=None, recursive=False, pattern=None, files_only=False): """ List files in the destination that have common prefix. :param prefix: Common prefix. May include the bucket name. (e.g. ``s3://my_bucket/foo/``) or simply a prefix in the bucket (e.g. ``foo/``). :type prefix: str :param recursive: Does nothing for this class. :return: sorted list of file names. :param pattern: files must match with this regexp if specified. :type pattern: str :param files_only: Does nothing for this class. :return: Full S3 url in form ``s3://bucket/path/to/file``. :rtype: list(str) :raise S3DestinationError: if failed to list files. """ s3client = boto3.resource('s3') bucket = s3client.Bucket(self._bucket) LOG.debug('Listing bucket %s', self._bucket) LOG.debug('prefix = %s', prefix) norm_prefix = prefix.replace('s3://%s' % bucket.name, '') norm_prefix = norm_prefix.lstrip('/') LOG.debug('normal prefix = %s', norm_prefix) # Try to list the bucket several times # because of intermittent error NoSuchBucket: # https://travis-ci.org/twindb/backup/jobs/204053690 expire = time.time() + S3_READ_TIMEOUT retry_interval = 2 while time.time() < expire: try: files = [] all_objects = bucket.objects.filter(Prefix=norm_prefix) for file_object in all_objects: if pattern: if re.search(pattern, file_object.key): files.append( 's3://{bucket}/{key}'.format( bucket=self._bucket, key=file_object.key ) ) else: files.append( 's3://{bucket}/{key}'.format( bucket=self._bucket, key=file_object.key ) ) return sorted(files) except ClientError as err: LOG.warning( '%s. Will retry in %d seconds.', err, retry_interval ) time.sleep(retry_interval) retry_interval *= 2 raise S3DestinationError('Failed to list files.')