def _write_retry(status_content, attempts=3): for _ in xrange(attempts): self._write_status(status_content, cls=cls) try: return self._read_status(cls=cls) except CorruptedStatus: pass raise DestinationError("Can't write status")
def _save(cmd, handler): with handler as input_handler: LOG.debug('Running %s', ' '.join(cmd)) try: proc = Popen(cmd, stdin=input_handler, stdout=PIPE, stderr=PIPE) cout_ssh, cerr_ssh = proc.communicate() ret = proc.returncode if ret: if cout_ssh: LOG.info(cout_ssh) if cerr_ssh: LOG.error(cerr_ssh) raise DestinationError('%s exited with error code %d' % (' '.join(cmd), ret)) LOG.debug('Exited with code %d', ret) except OSError as err: raise DestinationError('Failed to run %s: %s' % (' '.join(cmd), err))
def __init__(self, remote_path, status_path=None, status_tmp_path=None): if not remote_path: raise DestinationError( 'remote path must be defined and cannot be %r' % remote_path ) self.remote_path = remote_path.rstrip('/') if status_path: self.status_path = status_path else: self.status_path = '%s/status' % remote_path if status_tmp_path: self.status_tmp_path = status_tmp_path else: self.status_tmp_path = '%s.tmp' % self.status_path
def get_full_copy_name(self, file_path): """ For a given backup copy find a parent. If it's a full copy then return itself :param file_path: :return: """ try: for run_type in INTERVALS: for key in self.status()[run_type].keys(): if file_path.endswith(key): if self.status()[run_type][key]['type'] == "full": return file_path else: remote_part = file_path.replace(key, '') parent = self.status()[run_type][key]['parent'] result = "%s%s" % (remote_part, parent) return result except (TypeError, KeyError) as err: LOG.error('Failed to find parent of %s', file_path) raise DestinationError(err) raise DestinationError('Failed to find parent of %s' % file_path)
def get_backup_type(status, key): """ Get backup type - full or incremental - from the backup status. :param status: backup status :type status: dict :param key: backup name :type key: str :return: Backup type or None if backup name is not found :rtype: str """ backup_type = _get_status_key(status, key, 'type') if backup_type: return backup_type raise DestinationError('Unknown backup type for backup copy %s' % key)
def status(self, status=None): """ Read or save backup status. Status is an instance of Status class. If status is None the function will read status from the remote storage. Otherwise it will store the status remotely. :param status: instance of Status class :type status: Status :return: instance of Status class :rtype: Status """ if status: for _ in xrange(3): self._write_status(status) try: return self._read_status() except CorruptedStatus: pass raise DestinationError("Can't write status") else: return self._read_status()
def __init__(self, remote_path): if not remote_path: raise DestinationError( 'remote path must be defined and cannot be %r' % remote_path ) self.remote_path = remote_path.rstrip('/')
def restore_from_mysql(config, copy, dst_dir, tmp_dir=None, cache=None, hostname=None): """ Restore MySQL datadir in a given directory :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param copy: Backup copy instance. :type copy: MySQLCopy :param dst_dir: Destination directory. Must exist and be empty. :type dst_dir: str :param tmp_dir: Path to temp directory :type tmp_dir: str :param cache: Local cache object. :type cache: Cache :param hostname: Hostname :type hostname: str """ LOG.info('Restoring %s in %s', copy, dst_dir) mkdir_p(dst_dir) dst = None restore_start = time.time() try: xtrabackup_binary = config.get('mysql', 'xtrabackup_binary') except ConfigParser.NoOptionError: xtrabackup_binary = XTRABACKUP_BINARY try: xbstream_binary = config.get('mysql', 'xbstream_binary') except ConfigParser.NoOptionError: xbstream_binary = XBSTREAM_BINARY try: keep_local_path = config.get('destination', 'keep_local_path') if osp.exists(osp.join(keep_local_path, copy.key)): dst = Local(keep_local_path) except ConfigParser.NoOptionError: pass if not dst: if not hostname: hostname = copy.host if not hostname: raise DestinationError('Failed to get hostname from %s' % copy) dst = get_destination(config, hostname=hostname) key = copy.key status = dst.status() stream = dst.get_stream(copy) if status[key].type == "full": cache_key = os.path.basename(key) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(stream, dst_dir, config, redo_only=False, xtrabackup_binary=xtrabackup_binary, xbstream_binary=xbstream_binary) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(stream, dst_dir, config, redo_only=False, xtrabackup_binary=xtrabackup_binary, xbstream_binary=xbstream_binary) else: full_copy = status.candidate_parent(copy.run_type) full_stream = dst.get_stream(full_copy) LOG.debug("Full parent copy is %s", full_copy.key) cache_key = os.path.basename(full_copy.key) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(full_stream, dst_dir, config, redo_only=True, xtrabackup_binary=xtrabackup_binary, xbstream_binary=xbstream_binary) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(full_stream, dst_dir, config, redo_only=True, xtrabackup_binary=xtrabackup_binary, xbstream_binary=xbstream_binary) restore_from_mysql_incremental(stream, dst_dir, config, tmp_dir, xtrabackup_binary=xtrabackup_binary, xbstream_binary=xbstream_binary) config_dir = os.path.join(dst_dir, "_config") for path, content in get_my_cnf(status, key): config_sub_dir = os.path.join(config_dir, os.path.dirname(path).lstrip('/')) mkdir_p(config_sub_dir, mode=0755) with open(os.path.join(config_sub_dir, os.path.basename(path)), 'w') as mysql_config: mysql_config.write(content) update_grastate(dst_dir, status, key) export_info(config, data=time.time() - restore_start, category=ExportCategory.mysql, measure_type=ExportMeasureType.restore) LOG.info('Successfully restored %s in %s.', copy.key, dst_dir) LOG.info( 'Now copy content of %s to MySQL datadir: ' 'cp -R %s /var/lib/mysql/', dst_dir, osp.join(dst_dir, '*')) LOG.info('Fix permissions: chown -R mysql:mysql /var/lib/mysql/') LOG.info( 'Make sure innodb_log_file_size and innodb_log_files_in_group ' 'in %s/backup-my.cnf and in /etc/my.cnf are same.', dst_dir) if osp.exists(config_dir): LOG.info('Original my.cnf is restored in %s.', config_dir) LOG.info('Then you can start MySQL normally.')
def restore_from_mysql(config, backup_copy, dst_dir, cache=None): """ Restore MySQL datadir in a given directory :param config: Tool configuration. :type config: ConfigParser.ConfigParser :param backup_copy: Backup copy name. :type backup_copy: str :param dst_dir: Destination directory. Must exist and be empty. :type dst_dir: str :param cache: Local cache object. :type cache: Cache """ LOG.info('Restoring %s in %s', backup_copy, dst_dir) mkdir_p(dst_dir) dst = None try: keep_local_path = config.get('destination', 'keep_local_path') if os.path.exists(backup_copy) \ and backup_copy.startswith(keep_local_path): dst = Local(keep_local_path) except ConfigParser.NoOptionError: pass if not dst: hostname = get_hostname_from_backup_copy(backup_copy) if not hostname: raise DestinationError('Failed to get hostname from %s' % backup_copy) dst = get_destination(config, hostname=hostname) key = dst.basename(backup_copy) status = dst.status() stream = dst.get_stream(backup_copy) if get_backup_type(status, key) == "full": cache_key = os.path.basename(key) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(stream, dst_dir, config) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(stream, dst_dir, config) else: full_copy = dst.get_full_copy_name(backup_copy) full_stream = dst.get_stream(full_copy) cache_key = os.path.basename(full_copy) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(full_stream, dst_dir, config, redo_only=True) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(full_stream, dst_dir, config, redo_only=True) restore_from_mysql_incremental(stream, dst_dir, config) config_dir = os.path.join(dst_dir, "_config") for path, content in get_my_cnf(status, key): config_sub_dir = os.path.join(config_dir, os.path.dirname(path).lstrip('/')) os.makedirs(config_sub_dir) with open(os.path.join(config_sub_dir, os.path.basename(path)), 'w') as mysql_config: mysql_config.write(content) update_grastate(dst_dir, status, key) LOG.info('Successfully restored %s in %s.', backup_copy, dst_dir) LOG.info('Now copy content of %s to MySQL datadir: ' 'cp -R %s/* /var/lib/mysql/', dst_dir, dst_dir) LOG.info('Fix permissions: chown -R mysql:mysql /var/lib/mysql/') LOG.info('Make sure innodb_log_file_size and innodb_log_files_in_group ' 'in %s/backup-my.cnf and in /etc/my.cnf are same.', dst_dir) if os.path.exists(config_dir): LOG.info('Original my.cnf is restored in %s.', config_dir) LOG.info('Then you can start MySQL normally.')
def restore_from_mysql(twindb_config, copy, dst_dir, tmp_dir=None, cache=None, hostname=None): """ Restore MySQL datadir in a given directory :param twindb_config: tool configuration :type twindb_config: TwinDBBackupConfig :param copy: Backup copy instance. :type copy: MySQLCopy :param dst_dir: Destination directory. Must exist and be empty. :type dst_dir: str :param tmp_dir: Path to temp directory :type tmp_dir: str :param cache: Local cache object. :type cache: Cache :param hostname: Hostname :type hostname: str """ LOG.info("Restoring %s in %s", copy, dst_dir) mkdir_p(dst_dir) dst = None restore_start = time.time() keep_local_path = twindb_config.keep_local_path if keep_local_path and osp.exists(osp.join(keep_local_path, copy.key)): dst = Local(twindb_config.keep_local_path) if not dst: if not hostname: hostname = copy.host if not hostname: raise DestinationError("Failed to get hostname from %s" % copy) dst = twindb_config.destination(backup_source=hostname) key = copy.key status = MySQLStatus(dst=dst, status_directory=hostname) stream = dst.get_stream(copy) if status[key].type == "full": cache_key = os.path.basename(key) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(stream, dst_dir, twindb_config, redo_only=False) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(stream, dst_dir, twindb_config, redo_only=False) else: full_copy = status.candidate_parent(copy.run_type) full_stream = dst.get_stream(full_copy) LOG.debug("Full parent copy is %s", full_copy.key) cache_key = os.path.basename(full_copy.key) if cache: if cache_key in cache: # restore from cache cache.restore_in(cache_key, dst_dir) else: restore_from_mysql_full(full_stream, dst_dir, twindb_config, redo_only=True) cache.add(dst_dir, cache_key) else: restore_from_mysql_full(full_stream, dst_dir, twindb_config, redo_only=True) restore_from_mysql_incremental(stream, dst_dir, twindb_config, tmp_dir) config_dir = os.path.join(dst_dir, "_config") for path, content in get_my_cnf(status, key): config_sub_dir = os.path.join(config_dir, os.path.dirname(path).lstrip("/")) mkdir_p(config_sub_dir, mode=0o755) with open(os.path.join(config_sub_dir, os.path.basename(path)), "w") as mysql_config: mysql_config.write(content) update_grastate(dst_dir, status, key) export_info( twindb_config, data=time.time() - restore_start, category=ExportCategory.mysql, measure_type=ExportMeasureType.restore, ) LOG.info("Successfully restored %s in %s.", copy.key, dst_dir) LOG.info( "Now copy content of %s to MySQL datadir: " "cp -R %s /var/lib/mysql/", dst_dir, osp.join(dst_dir, "*"), ) LOG.info("Fix permissions: chown -R mysql:mysql /var/lib/mysql/") LOG.info( "Make sure innodb_log_file_size and innodb_log_files_in_group " "in %s/backup-my.cnf and in /etc/my.cnf are same.", dst_dir, ) if osp.exists(config_dir): LOG.info("Original my.cnf is restored in %s.", config_dir) LOG.info("Then you can start MySQL normally.")