def backup_files(run_type, config):
"""Backup local directories
:param run_type: Run type
:type run_type: str
:param config: Configuration
:type config: TwinDBBackupConfig
"""
backup_start = time.time()
try:
for directory in config.backup_dirs:
LOG.debug('copying %s', directory)
src = FileSource(directory, run_type)
dst = config.destination()
_backup_stream(config, src, dst)
src.apply_retention_policy(dst, config, run_type)
except (
DestinationError,
SourceError,
SshClientException
) as err:
raise OperationError(err)
export_info(config, data=time.time() - backup_start,
category=ExportCategory.files,
measure_type=ExportMeasureType.backup)
def backup_everything(run_type, twindb_config, binlogs_only=False):
"""
Run backup job
:param run_type: hourly, daily, etc
:type run_type: str
:param twindb_config: ConfigParser instance
:type twindb_config: TwinDBBackupConfig
:param binlogs_only: If True copy only MySQL binary logs.
:type binlogs_only: bool
"""
set_open_files_limit()
try:
if not binlogs_only:
backup_start = time.time()
backup_files(run_type, twindb_config)
backup_mysql(run_type, twindb_config)
backup_binlogs(run_type, twindb_config)
end = time.time()
save_measures(backup_start, end)
else:
backup_binlogs(run_type, twindb_config)
except ConfigParser.NoSectionError as err:
LOG.debug(traceback.format_exc())
LOG.error(err)
exit(1)
def get_connection(self):
"""
Connect to MySQL host and yield a connection.
:return: MySQL connection
:raise MySQLSourceError: if can't connect to server
"""
connection = None
try:
connection = pymysql.connect(
host=self.hostname,
read_default_file=self.defaults_file,
connect_timeout=self.connect_timeout,
cursorclass=pymysql.cursors.DictCursor
)
yield connection
except OperationalError:
LOG.error(
"Can't connect to MySQL server on %s",
self.hostname)
raise MySQLSourceError(
"Can't connect to MySQL server on %s"
% self.hostname)
finally:
if connection:
connection.close()
def _handle_failure_exec(self, err, stderr_file):
"""Cleanup on failure exec"""
LOG.error(err)
LOG.error('Failed to run xtrabackup. '
'Check error output in %s', stderr_file.name)
self.dst.delete(self.get_name())
exit(1)
def disable_wsrep_desync(self):
"""
Wait till wsrep_local_recv_queue is zero
and disable wsrep_local_recv_queue then
"""
max_time = time.time() + 900
try:
with self.get_connection() as connection:
with connection.cursor() as cursor:
while time.time() < max_time:
cursor.execute("SHOW GLOBAL STATUS LIKE "
"'wsrep_local_recv_queue'")
res = {r['Variable_name'].lower(): r['Value'].lower()
for r in cursor.fetchall()}
if not res.get('wsrep_local_recv_queue'):
raise Exception('Unknown status variable '
'"wsrep_local_recv_queue"')
if int(res['wsrep_local_recv_queue']) == 0:
break
time.sleep(1)
LOG.debug('Disabling wsrep_desync')
cursor.execute("SET GLOBAL wsrep_desync=OFF")
except pymysql.Error as err:
LOG.error(err)
def storage_server(docker_client, container_network):
bootstrap_script = '/twindb-backup/support/bootstrap/storage_server.sh'
container = get_container(
'storage_server',
docker_client,
container_network,
bootstrap_script=bootstrap_script,
image="centos:centos7",
last_n=2
)
timeout = time.time() + 30 * 60
while time.time() < timeout:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if sock.connect_ex((container['ip'], 22)) == 0:
break
time.sleep(1)
yield container
if container:
LOG.info('Removing container %s', container['Id'])
docker_client.api.remove_container(container=container['Id'],
force=True)
def get_stream(self):
"""
Get a PIPE handler with content of the source
:return:
"""
cmd = [
self._xtrabackup,
"--defaults-file=%s" % self._connect_info.defaults_file,
"--stream=xbstream",
"--host=127.0.0.1",
"--backup"
]
cmd += ["--target-dir", "."]
if self.is_galera():
cmd.append("--galera-info")
cmd.append("--no-backup-locks")
if self.incremental:
cmd += [
"--incremental-basedir",
".",
"--incremental-lsn=%d" % self._parent_lsn
]
# If this is a Galera node then additional step needs to be taken to
# prevent the backups from locking up the cluster.
wsrep_desynced = False
LOG.debug('Running %s', ' '.join(cmd))
stderr_file = tempfile.NamedTemporaryFile(delete=False)
try:
if self.is_galera():
wsrep_desynced = self.enable_wsrep_desync()
LOG.debug('Running %s', ' '.join(cmd))
proc_xtrabackup = Popen(cmd,
stderr=stderr_file,
stdout=PIPE)
yield proc_xtrabackup.stdout
proc_xtrabackup.communicate()
if proc_xtrabackup.returncode:
LOG.error('Failed to run xtrabackup. '
'Check error output in %s', stderr_file.name)
try:
if LOG.debug_enabled:
with open(stderr_file.name) as xb_out:
for line in xb_out:
print(line, end='', file=sys.stderr)
except AttributeError:
pass
self.dst.delete(self.get_name())
exit(1)
else:
LOG.debug('Successfully streamed xtrabackup output')
self._update_backup_info(stderr_file)
except OSError as err:
LOG.error('Failed to run %s: %s', cmd, err)
exit(1)
finally:
if wsrep_desynced:
self.disable_wsrep_desync()
def setup_slave(self, master_info): # noqa # pylint: disable=too-many-arguments
"""
Change master
:param master_info: Master details.
:type master_info: MySQLMasterInfo
"""
try:
with self._cursor() as cursor:
query = "CHANGE MASTER TO " \
"MASTER_HOST = '{master}', " \
"MASTER_USER = '******', " \
"MASTER_PORT = {port}, " \
"MASTER_PASSWORD = '******', " \
"MASTER_LOG_FILE = '{binlog}', " \
"MASTER_LOG_POS = {binlog_pos}"\
.format(
master=master_info.host,
user=master_info.user,
password=master_info.password,
binlog=master_info.binlog,
binlog_pos=master_info.binlog_position,
port=master_info.port
)
cursor.execute(query)
cursor.execute("START SLAVE")
return True
except pymysql.Error as err:
LOG.debug(err)
return False
def apply_backup(self, datadir):
"""
Apply backup of destination server
:param datadir: Path to datadir
:return: Binlog file name and position
:rtype: tuple
:raise RemoteMySQLSourceError: if any error.
"""
try:
use_memory = "--use-memory %d" % int(self._mem_available() / 2)
except OSError:
use_memory = ""
logfile_path = "/tmp/xtrabackup-apply-log.log"
cmd = "sudo {xtrabackup} --prepare --apply-log-only " \
"--target-dir {target_dir} {use_memory} " \
"> {logfile} 2>&1" \
"".format(
xtrabackup=self._xtrabackup,
target_dir=datadir,
use_memory=use_memory,
logfile=logfile_path
)
try:
self._ssh_client.execute(cmd)
self._ssh_client.execute("sudo chown -R mysql %s" % datadir)
return self._get_binlog_info(datadir)
except SshClientException as err:
LOG.debug("Logfile is:")
LOG.debug(self._ssh_client.get_text_content(logfile_path))
raise RemoteMySQLSourceError(err)
def backup_mysql(self):
"""FLag to backup MySQL or not"""
try:
return self.__cfg.getboolean('source', 'backup_mysql')
except NoOptionError:
return False
except NoSectionError as err:
LOG.error("Section 'source' is mandatory")
raise ConfigurationError(err)
def backup_dirs(self):
"""Directories to backup"""
try:
dirs = self.__cfg.get('source', 'backup_dirs')
return split(dirs)
except NoOptionError:
return []
except NoSectionError as err:
LOG.error("Section 'source' is mandatory")
raise ConfigurationError(err)
def _update_backup_info(self, stderr_file):
"""Update backup_info from stderr"""
LOG.debug('xtrabackup error log file %s',
stderr_file.name)
self._backup_info.lsn = self._get_lsn(stderr_file.name)
self._backup_info.binlog_coordinate = self.get_binlog_coordinates(
stderr_file.name
)
os.unlink(stderr_file.name)
def clone_config(self, dst):
"""
Clone config to destination server
:param dst: Destination server
:type dst: Ssh
"""
cfg_path = self._get_root_my_cnf()
LOG.debug("Root my.cnf is: %s", cfg_path)
self._save_cfg(dst, cfg_path)
def set_open_files_limit():
"""Detect maximum supported number of open file and set it"""
max_files = getrlimit(RLIMIT_NOFILE)[0]
while True:
try:
setrlimit(RLIMIT_NOFILE, (max_files, max_files))
max_files += 1
except ValueError:
break
LOG.debug('Setting max files limit to %d', max_files)
def save(self, handler, filepath):
"""
Read from handler and save it to Amazon S3
:param filepath: save backup copy in a file with this name
:param handler: stdout handler from backup source
"""
with handler as file_obj:
ret = self._upload_object(file_obj, filepath)
LOG.debug('Returning code %d', ret)
def kill_children():
"""
Kill child process
"""
for proc in multiprocessing.active_children():
LOG.info('Terminating %r [%d] ...', proc, proc.pid)
proc.terminate()
parent = psutil.Process(os.getpid())
for child in parent.children(recursive=True):
LOG.info('Terminating process %r', child)
child.kill()
def share_backup(ctx, s3_url):
"""Share backup copy for download"""
if not s3_url:
LOG.info('No backup copy specified. Choose one from below:')
list_available_backups(ctx.obj['twindb_config'])
exit(1)
try:
share(ctx.obj['twindb_config'], s3_url)
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
def _print_binlog(dst):
dst_files = dst.list_files(
dst.remote_path,
pattern='/binlog/',
recursive=True,
files_only=True
)
if dst_files:
LOG.info('Binary logs:')
for copy in dst_files:
print(copy)
def _print_media_type(dst, media_type):
for run_type in INTERVALS:
pattern = "/%s/%s/" % (run_type, media_type)
dst_files = dst.list_files(
dst.remote_path,
pattern=pattern,
recursive=True,
files_only=True
)
if dst_files:
LOG.info('%s %s copies:', media_type, run_type)
for copy in dst_files:
print(copy)
def create_bucket(self):
"""Creates the bucket in gcs that will store the backups.
:raises GCSDestinationError: if failed to create the bucket.
:raises GCSDestinationError: If authentication error.
"""
try:
self._gcs_client.create_bucket(bucket_name=self.bucket)
except (GoogleAPIError, GoogleAuthError) as err:
raise GCSDestinationError(err)
LOG.info('Created bucket %s', self.bucket)
def _retention(self, section):
kwargs = {}
for i in INTERVALS:
option = '%s_copies' % i
try:
kwargs[i] = self.__cfg.getint(section, option)
except (NoOptionError, NoSectionError):
LOG.warning(
'Option %s is not defined in section %s',
option,
section
)
return RetentionPolicy(**kwargs)
def bucket_name():
travis_job_number = os.environ.get('TRAVIS_JOB_NUMBER')
LOG.debug('TRAVIS_JOB_NUMBER=%s' % travis_job_number)
number = random.randint(0, 1000000)
LOG.debug('Default job number %d' % number)
if travis_job_number:
bucket = 'twindb-backup-test-travis-%s' % travis_job_number
else:
bucket = 'twindb-backup-test-travis-%d' % number
return '%s-%s' % (bucket, time.time())
def _get_name(self, filename_prefix):
LOG.debug('Suffix = %s', self.suffix)
self._name = osp.join(
self.get_prefix(),
self._media_type,
"{file}-{time}.{suffix}".format(
file=filename_prefix,
time=self._created_at,
suffix=self._suffix
)
)
return self._name
def enable_wsrep_desync(self):
"""
Try to enable wsrep_desync
:return: True if wsrep_desync was enabled. False if not supported
"""
try:
with self.get_connection() as connection:
with connection.cursor() as cursor:
cursor.execute('SET GLOBAL wsrep_desync=ON')
return True
except pymysql.Error as err:
LOG.debug(err)
return False
def candidate_parent(self, run_type):
"""
Find a backup copy that can be a parent
:param run_type: See :func:`~get_backup_type`.
:return: Backup copy or None
:rtype: MySQLCopy
"""
full_backup_index = INTERVALS.index(run_type)
LOG.debug('Looking a parent candidate for %s run', run_type)
for i in xrange(full_backup_index, len(INTERVALS)):
period_copies = getattr(self, INTERVALS[i])
LOG.debug(
'Checking %d %s copies',
len(period_copies),
INTERVALS[i]
)
for _, value in period_copies.iteritems():
try:
if value.type == 'full':
LOG.debug('Found parent %r', value)
return value
except KeyError:
return None
LOG.debug('No eligible parents')
return None
def s3_client(bucket_name):
LOG.debug('Bucket: %s' % bucket_name)
client = S3(
bucket=bucket_name,
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']
)
try:
assert client.create_bucket()
yield client
finally:
client.delete_bucket(force=True)
def delete_bucket(self, force=False):
"""Delete the bucket in gcs that was storing the backups.
:param force: If the bucket is non-empty then delete the objects
before deleting the bucket.
:type force: bool
:raise GCSDestinationError: if failed to delete the bucket.
"""
try:
self._bucket_obj.delete(force=force)
except (GoogleAPIError, GoogleAuthError) as err:
raise GCSDestinationError(err)
LOG.info('Deleted bucket %s', self.bucket)
def get_stream(self):
"""
Compress the input stream and return it as the output stream
:return: output stream handle
"""
with self._input as input_stream:
LOG.debug('Running %s', ' '.join(self._modifier_cmd))
proc = Popen(
self._modifier_cmd,
stdin=input_stream,
stdout=PIPE,
stderr=PIPE
)
yield proc.stdout
proc.communicate()
def _backup_stream(config, src, dst, callbacks=None):
"""
:param config: Tool config
:type config: TwinDBBackupConfig
:param src:
:param dst:
:param callbacks:
:return:
"""
stream = src.get_stream()
# Compression modifier
cmp_modifier = config.compression.get_modifier(stream)
stream = cmp_modifier.get_stream()
src.suffix += cmp_modifier.suffix
# KeepLocal modifier
if config.keep_local_path:
keep_local_path = config.keep_local_path
kl_modifier = KeepLocal(
stream,
osp.join(
keep_local_path,
src.get_name()
)
)
stream = kl_modifier.get_stream()
if callbacks is not None:
callbacks.append((kl_modifier, {
'keep_local_path': keep_local_path,
'dst': dst
}))
else:
LOG.debug('keep_local_path is not present in the config file')
# GPG modifier
if config.gpg:
gpg_modifier = Gpg(
stream,
config.gpg.recipient,
config.gpg.keyring
)
stream = gpg_modifier.get_stream()
src.suffix += '.gpg'
dst.save(stream, src.get_name())
def _load(self, status_as_json):
status = []
try:
status_as_obj = json.loads(status_as_json)
except ValueError:
raise CorruptedStatus(
'Could not load status from a bad JSON string %s'
% (status_as_json, )
)
for run_type in INTERVALS:
for key, value in status_as_obj[run_type].iteritems():
try:
host = key.split('/')[0]
file_name = key.split('/')[3]
kwargs = {
'type': value['type'],
'config': self.__serialize_config(value)
}
keys = [
'backup_started',
'backup_finished',
'binlog',
'parent',
'lsn',
'position',
'wsrep_provider_version',
]
for copy_key in keys:
if copy_key in value:
kwargs[copy_key] = value[copy_key]
copy = MySQLCopy(
host,
run_type,
file_name,
**kwargs
)
status.append(copy)
except IndexError as err:
LOG.error(err)
raise CorruptedStatus('Unexpected key %s' % key)
return status
def execute(self, cmd, quiet=False):
"""Execute a command on a remote SSH server.
:param cmd: Command for execution.
:type cmd: str
:param quiet: if quiet is True don't print error messages
:return: Handlers of stdin, stdout and stderr
:rtype: tuple
:raise SshDestinationError: if any error
"""
try:
with self._shell() as shell:
stdin_, stdout_, stderr_ = shell.exec_command(cmd)
exit_code = stdout_.channel.recv_exit_status()
if exit_code != 0:
if not quiet:
LOG.error("Failed while execute command %s", cmd)
LOG.error(stderr_.read())
raise SshClientException('%s exited with code %d' %
(cmd, exit_code))
return stdin_, stdout_, stderr_
except SSHException as err:
if not quiet:
LOG.error('Failed to execute %s', cmd)
raise SshClientException(err)
def ensure_tcp_port_listening(self, port, wait_timeout=10):
"""
Check that tcp port is open and ready to accept connections.
Keep checking up to wait_timeout seconds.
:param port: TCP port that is supposed to be listening.
:type port: int
:param wait_timeout: wait this many seconds until the port is ready.
:type wait_timeout: int
:return: True if the TCP port is listening.
:rtype: bool
"""
stop_waiting_at = time.time() + wait_timeout
while time.time() < stop_waiting_at:
try:
cmd = 'netstat -an | grep -w ^tcp | grep -w LISTEN ' \
'| grep -w 0.0.0.0:%d' % port
_, cout, cerr = self.execute_command(cmd)
LOG.debug('stdout: %s', cout.read())
LOG.debug('stderr: %s', cerr.read())
return True
except SshClientException as err:
LOG.debug(err)
time.sleep(1)
return False
def clone(self, dest_host, port, compress=False):
"""
Send backup to destination host
:param dest_host: Destination host
:type dest_host: str
:param port: Port to sending backup
:type port: int
:param compress: If True compress stream
:type compress: bool
:raise RemoteMySQLSourceError: if any error
"""
retry = 1
retry_time = 2
error_log = "/tmp/{src}_{src_port}-{dst}_{dst_port}.log".format(
src=self._ssh_client.host,
src_port=self._ssh_client.port,
dst=dest_host,
dst_port=port)
if compress:
compress_cmd = "| gzip -c - "
else:
compress_cmd = ""
cmd = "bash -c \"sudo %s " \
"--stream=xbstream " \
"--host=127.0.0.1 " \
"--backup " \
"--target-dir ./ 2> %s" \
" %s | ncat %s %d --send-only\"" \
% (self._xtrabackup, error_log, compress_cmd, dest_host, port)
while retry < 3:
try:
return self._ssh_client.execute(cmd)
except SshClientException as err:
LOG.warning(err)
LOG.info('Will try again in after %d seconds', retry_time)
time.sleep(retry_time)
retry_time *= 2
retry += 1
def verify_mysql_backup(twindb_config, dst_path, backup_file, hostname=None):
"""
Restore mysql backup and measure time
:param hostname:
:param backup_file:
:param dst_path:
:param twindb_config: tool configuration
:type twindb_config: TwinDBBackupConfig
"""
dst = twindb_config.destination(backup_source=hostname)
status = MySQLStatus(dst=dst)
copy = None
if backup_file == "latest":
copy = status.latest_backup
else:
for copy in status:
if backup_file.endswith(copy.key):
break
if copy is None:
return json.dumps(
{
'backup_copy': backup_file,
'restore_time': 0,
'success': False
},
indent=4,
sort_keys=True)
start_restore_time = time.time()
success = True
tmp_dir = tempfile.mkdtemp()
try:
LOG.debug('Verifying backup copy in %s', tmp_dir)
restore_from_mysql(twindb_config, copy, dst_path, tmp_dir)
edit_backup_my_cnf(dst_path)
except (TwinDBBackupError, OSError, IOError) as err:
LOG.error(err)
LOG.debug(traceback.format_exc())
success = False
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
end_restore_time = time.time()
restore_time = end_restore_time - start_restore_time
return json.dumps(
{
'backup_copy': copy.key,
'restore_time': restore_time,
'success': success
},
indent=4,
sort_keys=True)
def _load(self, status_as_json):
status = []
try:
status_as_obj = json.loads(status_as_json)
except ValueError:
raise CorruptedStatus(
"Could not load status from a bad JSON string %s"
% (status_as_json,)
)
for run_type in INTERVALS:
for key, value in status_as_obj[run_type].items():
try:
host = key.split("/")[0]
file_name = key.split("/")[3]
kwargs = {
"type": value["type"],
"config": self.__serialize_config(value),
}
keys = [
"backup_started",
"backup_finished",
"binlog",
"parent",
"lsn",
"position",
"wsrep_provider_version",
]
for copy_key in keys:
if copy_key in value:
kwargs[copy_key] = value[copy_key]
copy = MySQLCopy(host, run_type, file_name, **kwargs)
status.append(copy)
except IndexError as err:
LOG.error(err)
raise CorruptedStatus("Unexpected key %s" % key)
return status
def backup_files(run_type, config):
"""Backup local directories
:param run_type: Run type
:type run_type: str
:param config: Configuration
:type config: TwinDBBackupConfig
"""
backup_start = time.time()
try:
for directory in config.backup_dirs:
LOG.debug('copying %s', directory)
src = FileSource(directory, run_type)
dst = config.destination()
_backup_stream(config, src, dst)
src.apply_retention_policy(dst, config, run_type)
except (DestinationError, SourceError, SshClientException) as err:
raise OperationError(err)
export_info(config,
data=time.time() - backup_start,
category=ExportCategory.files,
measure_type=ExportMeasureType.backup)
def test_init_creates_instance_from_new(status_raw_content):
status = MySQLStatus(status_raw_content)
assert status.version == STATUS_FORMAT_VERSION
key = 'master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz'
copy = MySQLCopy(
'master1',
'hourly',
'mysql-2018-03-28_04_11_16.xbstream.gz',
backup_started=1522210276,
backup_finished=1522210295,
binlog='mysql-bin.000001',
parent='master1/daily/mysql/mysql-2018-03-28_04_09_53.xbstream.gz',
lsn=19903207,
config={
'/etc/my.cnf':
"""[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
user=mysql
# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0
server_id=100
gtid_mode=ON
log-bin=mysql-bin
log-slave-updates
enforce-gtid-consistency
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
"""
},
position=46855,
type='incremental')
assert key in status.hourly
LOG.debug("Copy %s: %r", copy.key, copy)
LOG.debug("Copy from status %s: %r", key, status[key])
assert status[key] == copy
def create_bucket(self):
"""Creates the bucket in s3 that will store the backups.
:raise S3DestinationError: if failed to create the bucket.
"""
bucket_exists = True
try:
self.s3_client.head_bucket(Bucket=self.bucket)
except ClientError as err:
# We come here meaning we did not find the bucket
if err.response['ResponseMetadata']['HTTPStatusCode'] == 404:
bucket_exists = False
else:
raise
if not bucket_exists:
LOG.info('Created bucket %s', self.bucket)
response = self.s3_client.create_bucket(Bucket=self.bucket)
self.validate_client_response(response)
return True
def docker_execute(client, container_id, cmd, tty=False):
"""Execute command in container
:param client: Docker client class instance
:type client: APIClient
:param container_id: Container Id from a dictionary that get_container
returns.
:type container_id: str
:param cmd: Command to execute
:type cmd: str or list
:return: A tuple with exit code and output.
:param tty: Using pseudo-TTY
:type tty: bool
:rtype: tuple(int, str)
"""
LOG.debug("Running %s", " ".join(cmd))
api = client.api
executor = api.exec_create(container_id, cmd, tty=tty)
exec_id = executor["Id"]
cout = api.exec_start(exec_id)
ret = api.exec_inspect(exec_id)["ExitCode"]
return ret, cout.decode("utf-8")
def _load(self, status_as_json):
status = []
try:
status_as_obj = json.loads(status_as_json)
except ValueError:
raise CorruptedStatus(
'Could not load status from a bad JSON string %s' %
(status_as_json, ))
for run_type in INTERVALS:
for key, value in status_as_obj[run_type].iteritems():
try:
host = key.split('/')[0]
file_name = key.split('/')[3]
kwargs = {
'type': value['type'],
'config': self.__serialize_config(value)
}
keys = [
'backup_started',
'backup_finished',
'binlog',
'parent',
'lsn',
'position',
'wsrep_provider_version',
]
for copy_key in keys:
if copy_key in value:
kwargs[copy_key] = value[copy_key]
copy = MySQLCopy(host, run_type, file_name, **kwargs)
status.append(copy)
except IndexError as err:
LOG.error(err)
raise CorruptedStatus('Unexpected key %s' % key)
return status
def test__take_mysql_backup(master1, docker_client, s3_client,
config_content_mysql_only, client_my_cnf):
twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"])
twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir
twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg"
my_cnf_path = "%s/my.cnf" % twindb_config_dir
with open(my_cnf_path, "w") as my_cnf:
my_cnf.write(client_my_cnf)
with open(twindb_config_host, "w") as fp:
content = config_content_mysql_only.format(
AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"],
AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"],
BUCKET=s3_client.bucket,
daily_copies=1,
hourly_copies=2,
MY_CNF="/etc/twindb/my.cnf",
)
fp.write(content)
cmd = [
"twindb-backup",
"--debug",
"--config",
twindb_config_guest,
"backup",
"hourly",
]
ret, cout = docker_execute(docker_client, master1["Id"], cmd)
assert_and_pause((ret == 0, ), cout)
cmd = ["twindb-backup", "--config", twindb_config_guest, "status"]
ret, cout = docker_execute(docker_client, master1["Id"], cmd)
LOG.debug("STDOUT: %s", cout)
key = list(json.loads(cout)["hourly"].keys())[0]
assert_and_pause((key.endswith(".xbstream.gz"), ), key)
def slave(docker_client, container_network, tmpdir_factory):
try:
platform = os.environ['PLATFORM']
except KeyError:
raise EnvironmentError("""The environment variable PLATFORM
must be defined. Allowed values are:
* centos
* debian
* ubuntu
""")
bootstrap_script = '/twindb-backup/support/bootstrap/master/' \
'%s/slave.sh' % platform
separator_pos = NODE_IMAGE.find(':')
image_name = NODE_IMAGE[:separator_pos +
1] + 'slave_' + NODE_IMAGE[separator_pos + 1:]
datadir = tmpdir_factory.mktemp('mysql')
twindb_config_dir = tmpdir_factory.mktemp('twindb')
container = get_container('slave',
docker_client,
container_network,
str(datadir),
twindb_config_dir=str(twindb_config_dir),
last_n=2,
image=image_name)
try:
timeout = time.time() + 30 * 60
LOG.info('Waiting until port TCP/22 becomes available')
while time.time() < timeout:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if sock.connect_ex((container['ip'], 22)) == 0:
break
time.sleep(1)
LOG.info('Still waiting')
LOG.info('Port TCP/22 is ready')
ret, _ = docker_execute(docker_client, container['Id'], ['ls'])
assert ret == 0
ret, cout = docker_execute(docker_client, container['Id'],
['bash', bootstrap_script])
print(cout)
assert ret == 0
yield container
finally:
LOG.info('Removing container %s', container['Id'])
docker_client.api.remove_container(container=container['Id'],
force=True)
def test_take_mysql_backup_aenc_restores_full(s3_client,
config_content_mysql_aenc,
tmpdir):
config = tmpdir.join('twindb-backup.cfg')
content = config_content_mysql_aenc.format(
AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'],
AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'],
BUCKET=s3_client.bucket,
daily_copies=1,
hourly_copies=2)
config.write(content)
cmd = ['twindb-backup', '--config', str(config), 'backup', 'daily']
assert call(cmd) == 0
cmd = ['twindb-backup', '--config', str(config), 'status']
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
cout, cerr = proc.communicate()
LOG.debug('STDOUT: %s', cout)
LOG.debug('STDERR: %s', cerr)
key = json.loads(cout)['daily'].keys()[0]
backup_copy = 's3://' + s3_client.bucket + '/' + key
dst_dir = str(tmpdir.mkdir('dst'))
cmd = [
'twindb-backup', '--debug', '--config',
str(config), 'restore', 'mysql', backup_copy, '--dst', dst_dir
]
assert call(cmd) == 0
call(['find', dst_dir])
assert os.path.exists(dst_dir + '/ibdata1')
assert os.path.exists(dst_dir + '/ib_logfile0')
assert os.path.exists(dst_dir + '/ib_logfile1')
assert os.path.exists(dst_dir + '/mysql/user.MYD')
assert os.path.exists(dst_dir + '/backup-my.cnf')
assert os.path.exists(dst_dir + '/xtrabackup_logfile')
assert os.path.exists(dst_dir + '/_config/etc/my.cnf') or \
os.path.exists(dst_dir + '/_config/etc/mysql/my.cnf')
def pause_test(msg):
"""Pause """
try:
if os.environ["PAUSE_TEST"]:
LOG.debug("Test paused")
LOG.debug(msg)
import time
time.sleep(36000)
except KeyError:
LOG.debug(
"Define the PAUSE_TEST environment variable if you'd like to pause the test"
)
LOG.debug("export PAUSE_TEST=1")
pass
def runner(docker_client, container_network, tmpdir_factory):
try:
platform = os.environ['PLATFORM']
except KeyError:
raise EnvironmentError("""The environment variable PLATFORM
must be defined. Allowed values are:
* centos
* debian
* ubuntu
""")
bootstrap_script = '/twindb-backup/support/bootstrap/master/' \
'%s/master1.sh' % platform
datadir = tmpdir_factory.mktemp('mysql')
twindb_config_dir = tmpdir_factory.mktemp('twindb')
container = get_container(name="runner",
client=docker_client,
network=container_network,
bootstrap_script=bootstrap_script,
last_n=3,
twindb_config_dir=str(twindb_config_dir),
datadir=datadir)
try:
ret, _ = docker_execute(docker_client, container['Id'], ['ls'])
assert ret == 0
ret, cout = docker_execute(docker_client, container['Id'],
['bash', bootstrap_script])
print(cout)
assert ret == 0
yield container
finally:
LOG.info('Removing container %s', container['Id'])
docker_client.api.remove_container(container=container['Id'],
force=True)
def add(self, path, key=None):
# pylint: disable=line-too-long
"""Add directory to cache.
The directory may be a full or relative path with backup copy.
The directory name must match with a file name of the backup copy.
If backup copy is
``/path/to/backups/master1/daily/mysql/mysql-2017-05-13_22_04_06.xbstream.gz``.
then the directory can be something like
``/var/tmp/mysql-2017-05-13_22_04_06.xbstream.gz/``.
Let's say we want to add
``/var/tmp/mysql-2017-05-13_22_04_06.xbstream.gz/`` to the cache in
``/var/tmp/cache``.
Then this method will create directory
``/var/tmp/cache/mysql-2017-05-13_22_04_06.xbstream.gz/``.
If you want to save directory ``/var/tmp/foo`` in cache under
a key name ``mysql-2017-05-13_22_04_06.xbstream.gz``
you need to specify the key e.g.
``add('/var/tmp/cache', 'mysql-2017-05-13_22_04_06.xbstream.gz')``
:param path: full or relative path
:type path: str
:param key: if specified the directory will be added as this key name
in the cache
:raise: CacheException if errors
"""
if key:
LOG.debug('Cache key %s', key)
dst = os.path.join(self.path, key)
else:
dst = os.path.join(self.path, os.path.basename(path))
LOG.debug('Saving content of %s in %s', path, dst)
try:
shutil.copytree(path, dst)
except OSError as err:
raise CacheException(err)
def master1(docker_client, container_network):
container = _get_master(1, docker_client, container_network)
timeout = time.time() + 30 * 60
while time.time() < timeout:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if sock.connect_ex((container['ip'], 3306)) == 0:
break
time.sleep(1)
raw_container = docker_client.containers.get('master1')
privileges_file = "/twindb-backup/vagrant/environment/puppet/" \
"modules/profile/files/mysql_grants.sql"
raw_container.exec_run('bash -c "mysql mysql < %s"'
% privileges_file)
yield container
if container:
LOG.info('Removing container %s', container['Id'])
docker_client.api.remove_container(container=container['Id'],
force=True)
def test__take_mysql_backup_aenc_suffix_gpg(s3_client,
config_content_mysql_aenc, tmpdir):
config = tmpdir.join('twindb-backup.cfg')
content = config_content_mysql_aenc.format(
AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'],
AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'],
BUCKET=s3_client.bucket,
daily_copies=1,
hourly_copies=2)
config.write(content)
cmd = ['twindb-backup', '--config', str(config), 'backup', 'daily']
assert call(cmd) == 0
cmd = ['twindb-backup', '--config', str(config), 'status']
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
cout, cerr = proc.communicate()
LOG.debug('STDOUT: %s', cout)
LOG.debug('STDERR: %s', cerr)
key = json.loads(cout)['daily'].keys()[0]
assert key.endswith('xbstream.gz.gpg')
def is_galera(self):
"""Check if local MySQL instance is a Galera cluster
:return: True if it's a Galera.
:rtype: bool
"""
try:
with self._cursor() as cursor:
cursor.execute("SELECT @@wsrep_on as wsrep_on")
row = cursor.fetchone()
return (str(row['wsrep_on']).lower() == "1" or
str(row['wsrep_on']).lower() == 'on')
except pymysql.InternalError as err:
error_code = err.args[0]
error_message = err.args[1]
if error_code == 1193:
LOG.debug('Galera is not supported or not enabled')
return False
else:
LOG.error(error_message)
raise
def _get_master(n, client, network):
"""
:param n: 1 or 2
:return: Container
"""
api = client.api
api.pull(NODE_IMAGE)
cwd = os.getcwd()
host_config = api.create_host_config(
binds={
cwd: {
'bind': '/twindb-backup',
'mode': 'rw',
}
},
dns=['8.8.8.8']
)
ip = '172.%d.3.%d' % (network['second_octet'], n)
networking_config = api.create_networking_config({
network['NAME']: api.create_endpoint_config(ipv4_address=ip)
})
LOG.debug(networking_config)
container = api.create_container(
image=NODE_IMAGE,
name='master%d' % n,
ports=[22, 3306],
host_config=host_config,
networking_config=networking_config,
volumes=['/twindb-backup'],
command='bash /twindb-backup/support/clone/master%d.sh' % n
# command='/bin/sleep 36000'
)
container['ip'] = ip
LOG.info('Created container %r', container)
try:
api.start(container['Id'])
LOG.info('Started %r', container)
return container
except APIError as err:
LOG.error(err)
client.api.remove_container(container=container['Id'], force=True)
def _backup_stream(config, src, dst, callbacks=None):
"""
:param config: Tool config
:type config: TwinDBBackupConfig
:param src:
:param dst:
:param callbacks:
:return:
"""
stream = src.get_stream()
# Compression modifier
cmp_modifier = config.compression.get_modifier(stream)
stream = cmp_modifier.get_stream()
src.suffix += cmp_modifier.suffix
# KeepLocal modifier
if config.keep_local_path:
keep_local_path = config.keep_local_path
kl_modifier = KeepLocal(stream,
osp.join(keep_local_path, src.get_name()))
stream = kl_modifier.get_stream()
if callbacks is not None:
callbacks.append((kl_modifier, {
"keep_local_path": keep_local_path,
"dst": dst
}))
else:
LOG.debug("keep_local_path is not present in the config file")
# GPG modifier
if config.gpg:
gpg_modifier = Gpg(stream, config.gpg.recipient, config.gpg.keyring)
stream = gpg_modifier.get_stream()
src.suffix += ".gpg"
dst.save(stream, src.get_name())
def get_connection(self):
"""
Connect to MySQL host and yield a connection.
:return: MySQL connection
:raise MySQLSourceError: if can't connect to server
"""
connection = None
try:
connection = pymysql.connect(
host=self.hostname,
read_default_file=self.defaults_file,
connect_timeout=self.connect_timeout,
cursorclass=pymysql.cursors.DictCursor,
)
yield connection
except OperationalError:
LOG.error("Can't connect to MySQL server on %s", self.hostname)
raise MySQLSourceError("Can't connect to MySQL server on %s" %
self.hostname)
finally:
if connection:
connection.close()
def run_backup_job(twindb_config,
run_type,
lock_file=LOCK_FILE,
binlogs_only=False):
"""
Grab a lock waiting up to allowed timeout and start backup jobs
:param twindb_config: Tool configuration
:type twindb_config: TwinDBBackupConfig
:param run_type: Run type
:type run_type: str
:param lock_file: File used as a lock
:type lock_file: str
:param binlogs_only: If True copy only binlogs.
:type binlogs_only: bool
"""
with timeout(get_timeout(run_type)):
try:
file_desriptor = open(lock_file, "w")
fcntl.flock(file_desriptor, fcntl.LOCK_EX)
LOG.debug(run_type)
if getattr(twindb_config.run_intervals, run_type):
backup_everything(run_type,
twindb_config,
binlogs_only=binlogs_only)
else:
LOG.debug("Not running because run_%s is no", run_type)
except IOError as err:
if err.errno != errno.EINTR:
LOG.debug(traceback.format_exc())
raise LockWaitTimeoutError(err)
msg = "Another instance of twindb-backup is running?"
if run_type == "hourly":
LOG.debug(msg)
else:
LOG.error(msg)
def backup_server(docker_client, container_network):
bootstrap_script = '/twindb-backup/support/bootstrap/backup_server.sh'
container = get_container(
'backup_server',
bootstrap_script,
docker_client,
container_network
)
timeout = time.time() + 30 * 60
while time.time() < timeout:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if sock.connect_ex((container['ip'], 22)) == 0:
break
time.sleep(1)
yield container
if container:
LOG.info('Removing container %s', container['Id'])
docker_client.api.remove_container(container=container['Id'],
force=True)
def get_full_copy_name(self, file_path):
"""
For a given backup copy find a parent. If it's a full copy
then return itself
:param file_path:
:return:
"""
try:
for run_type in INTERVALS:
for key in self.status()[run_type].keys():
if file_path.endswith(key):
if self.status()[run_type][key]['type'] == "full":
return file_path
else:
remote_part = file_path.replace(key, '')
parent = self.status()[run_type][key]['parent']
result = "%s%s" % (remote_part, parent)
return result
except (TypeError, KeyError) as err:
LOG.error('Failed to find parent of %s', file_path)
raise DestinationError(err)
raise DestinationError('Failed to find parent of %s' % file_path)
def test_get_stream(gs):
status = MySQLStatus(dst=gs)
copy = status['master1/daily/mysql/mysql-2019-04-04_05_29_05.xbstream.gz']
with gs.get_stream(copy) as stream:
LOG.debug('starting reading from pipe')
content = stream.read()
LOG.debug('finished reading from pipe')
assert len(content), 'Failed to read from GS'
LOG.info('Read %d bytes', len(content))
def get_destination(config, hostname=socket.gethostname()):
"""
Read config and return instance of Destination class.
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param hostname: Local hostname.
:type hostname: str
:return: Instance of destination class.
:rtype: BaseDestination
"""
destination = None
try:
destination = config.get('destination', 'backup_destination')
LOG.debug('Destination in the config %s', destination)
destination = destination.strip('"\'')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
LOG.critical("Backup destination must be specified "
"in the config file")
exit(-1)
if destination == "ssh":
host = config.get('ssh', 'backup_host')
try:
port = config.get('ssh', 'port')
except ConfigParser.NoOptionError:
port = 22
try:
ssh_key = config.get('ssh', 'ssh_key')
except ConfigParser.NoOptionError:
ssh_key = '/root/.ssh/id_rsa'
user = config.get('ssh', 'ssh_user')
remote_path = config.get('ssh', 'backup_dir')
return Ssh(SshConnectInfo(host=host, port=port, user=user,
key=ssh_key),
remote_path=remote_path,
hostname=hostname)
elif destination == "s3":
bucket = config.get('s3', 'BUCKET').strip('"\'')
access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'')
secret_access_key = config.get('s3',
'AWS_SECRET_ACCESS_KEY').strip('"\'')
default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'')
return S3(bucket,
AWSAuthOptions(access_key_id,
secret_access_key,
default_region=default_region),
hostname=hostname)
else:
LOG.critical('Destination %s is not supported', destination)
exit(-1)
def _get_status_key(status, key, variable):
LOG.debug('status = %s', json.dumps(status, indent=4, sort_keys=True))
LOG.debug('key = %s', key)
try:
for run_type in INTERVALS:
if key in status[run_type]:
return status[run_type][key][variable]
except KeyError:
pass
LOG.warning('key %s is not found', key)
return None
def _match_files(files, pattern=None):
LOG.debug('Pattern: %s', pattern)
LOG.debug('Unfiltered files: %r', files)
result = []
for fil in files:
if pattern:
if re.search(pattern, fil):
result.append(fil)
else:
result.append(fil)
LOG.debug('Filtered files: %r', result)
return result
def restore_file(cfg, dst, backup_copy):
"""Restore from file backup"""
LOG.debug('file: %r', cfg)
if not backup_copy:
LOG.info('No backup copy specified. Choose one from below:')
list_available_backups(cfg)
exit(1)
try:
ensure_empty(dst)
restore_from_file(cfg, backup_copy, dst)
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
except KeyboardInterrupt:
LOG.info('Exiting...')
kill_children()
exit(1)
