def test_save_exception_return_false(mock_mkdirname_r, mock_execute):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
dst = Ssh(remote_path='/path/to/backups')
mock_execute.side_effect = SshClientException
assert not dst.save(mock.MagicMock(), 'aaa/bbb/ccc/bar')
mock_mkdirname_r.assert_called_once_with(
'/path/to/backups/aaa/bbb/ccc/bar')
def test_save_exception_not_handled(mock_mkdir_r, mock_execute):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
dst = Ssh(remote_path='/path/to/backups')
mock_execute.side_effect = SshClientException
with pytest.raises(SshClientException):
dst.save(mock.MagicMock(), 'aaa/bbb/ccc/bar')
mock_mkdir_r.assert_called_once_with('/path/to/backups/aaa/bbb/ccc')
def test_ssh_exec_command():
connect_info = SshConnectInfo(
host='192.168.36.250',
key='/Users/aleks/src/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key',
user='******'
)
ssh = Ssh(ssh_connect_info=connect_info, remote_path='/tmp/aaa')
_, stdout, stderr = ssh._execute_command(['/bin/ls', '/'])
print(stdout.readlines())
def test_ssh_exec_command():
connect_info = SshConnectInfo(
host='192.168.36.250',
key=
'/Users/aleks/src/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key',
user='******')
ssh = Ssh(ssh_connect_info=connect_info, remote_path='/tmp/aaa')
_, stdout, stderr = ssh._execute_command(['/bin/ls', '/'])
print(stdout.readlines())
def test_write_status(mock_execute):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
mock_execute.return_value.__enter__.return_value = iter(
(mock_cin, None, None))
dst = Ssh(remote_path='/path/to/backups')
dst._write_status("{}")
mock_cin.write().asset_called_once_with({})
mock_execute.asset_called_once_with("cat - > %s" % dst.status_path)
def test_get_status_empty(mock_status_exists):
mock_status_exists.return_value = False
dst = Ssh(remote_path='/foo/bar')
status = dst.status()
assert status.hourly == {}
assert status.daily == {}
assert status.weekly == {}
assert status.monthly == {}
assert status.yearly == {}
def test_list_files():
dst = Ssh(
'/var/backups'
)
mock_client = mock.Mock()
mock_client.list_files.return_value = [
'foo',
'bar'
]
dst._ssh_client = mock_client
assert dst.list_files('xxx', pattern='foo') == ['foo']
def test_get_status_empty(mock_status_exists):
mock_status_exists.return_value = False
dst = Ssh(remote_path='/foo/bar')
assert dst.status() == {
'hourly': {},
'daily': {},
'weekly': {},
'monthly': {},
'yearly': {}
}
def test_mkdir_r(mock_execute):
mock_stdout = mock.Mock()
mock_stdout.channel.recv_exit_status.return_value = 0
mock_execute.return_value = mock_stdout, mock.Mock()
dst = Ssh(remote_path='some_dir')
# noinspection PyProtectedMember
dst._mkdir_r('/foo/bar')
mock_execute.assert_called_once_with('mkdir -p "/foo/bar"')
def test__status_exists(mock_client, out, result):
mock_stdout = mock.Mock()
mock_stdout.read.return_value = out
mock_client.return_value = iter(
(
mock.Mock(),
mock_stdout,
mock.Mock()
)
)
dst = Ssh(remote_path='/foo/bar')
# noinspection PyProtectedMember
assert dst._status_exists() == result
def get_destination(config, hostname=socket.gethostname()):
"""
Read config and return instance of Destination class.
:param config: Tool configuration.
:type config: ConfigParser.ConfigParser
:param hostname: Local hostname.
:type hostname: str
:return: Instance of destination class.
:rtype: BaseDestination
"""
destination = None
try:
destination = config.get('destination', 'backup_destination')
LOG.debug('Destination in the config %s', destination)
destination = destination.strip('"\'')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
LOG.critical("Backup destination must be specified "
"in the config file")
exit(-1)
if destination == "ssh":
host = config.get('ssh', 'backup_host')
try:
port = int(config.get('ssh', 'port'))
except ConfigParser.NoOptionError:
port = 22
try:
ssh_key = config.get('ssh', 'ssh_key')
except ConfigParser.NoOptionError:
ssh_key = '/root/.ssh/id_rsa'
LOG.debug('ssh_key is not defined in config. '
'Will use default %s', ssh_key)
user = config.get('ssh', 'ssh_user')
remote_path = config.get('ssh', 'backup_dir')
return Ssh(
remote_path,
SshConnectInfo(
host=host,
port=port,
user=user,
key=ssh_key),
hostname=hostname)
elif destination == "s3":
bucket = config.get('s3', 'BUCKET').strip('"\'')
access_key_id = config.get('s3', 'AWS_ACCESS_KEY_ID').strip('"\'')
secret_access_key = config.get('s3',
'AWS_SECRET_ACCESS_KEY').strip('"\'')
default_region = config.get('s3', 'AWS_DEFAULT_REGION').strip('"\'')
return S3(bucket, AWSAuthOptions(access_key_id,
secret_access_key,
default_region=default_region),
hostname=hostname)
else:
LOG.critical('Destination %s is not supported', destination)
exit(-1)
def test_save(mock_mkdirname_r, mock_execute):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
mock_execute.return_value.__enter__.return_value = iter(
(mock_cin, None, None))
dst = Ssh(remote_path='/path/to/backups')
mock_handler = mock.MagicMock()
mock_cin.read.return_value = 'foo'
assert dst.save(mock_handler, 'aaa/bbb/ccc/bar')
mock_execute.assert_called_once_with(
'cat - > /path/to/backups/aaa/bbb/ccc/bar')
mock_cin.write.assert_called_once()
mock_handler.read_assert_called_once()
mock_mkdirname_r.assert_called_once_with(
'/path/to/backups/aaa/bbb/ccc/bar')
def test_save_creates_remote_dirname(mock_mkdir_r, mock_execute, remote_path,
name, remote_dirname):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
mock_execute.return_value.__enter__.return_value = iter(
(mock_cin, None, None))
dst = Ssh(remote_path=remote_path)
mock_handler = mock.MagicMock()
mock_file_obj = mock.MagicMock()
mock_file_obj.read.return_value = None
mock_handler.__enter__.return_value = mock_file_obj
mock_cin.read.return_value = 'foo'
dst.save(mock_handler, 'aaa/bbb/ccc/bar')
mock_mkdir_r.assert_called_once_with('/path/to/backups/aaa/bbb/ccc')
def test__status_exists_raises_error(mock_run):
mock_stdout = mock.Mock()
mock_stdout.channel.recv_exit_status.return_value = 0
mock_stdout.read.return_value = 'foo'
mock_run.return_value = iter(
(
mock.Mock(),
mock_stdout,
mock.Mock()
)
)
dst = Ssh(remote_path='/foo/bar')
with pytest.raises(SshDestinationError):
# noinspection PyProtectedMember
dst._status_exists()
def test_save_creates_remote_dirname(mock_mkdir_r, mock_execute, remote_path, name, remote_dirname):
mock_cin = mock.Mock()
mock_cin.channel.recv_exit_status.return_value = 0
mock_execute.return_value.__enter__.return_value = iter(
(
mock_cin,
None,
None
)
)
dst = Ssh(remote_path=remote_path)
mock_handler = mock.MagicMock()
mock_file_obj = mock.MagicMock()
mock_file_obj.read.return_value = None
mock_handler.__enter__.return_value = mock_file_obj
mock_cin.read.return_value = 'foo'
dst.save(mock_handler, 'aaa/bbb/ccc/bar')
mock_mkdir_r.assert_called_once_with('/path/to/backups/aaa/bbb/ccc')
def test__clone_config(mock_get_root, mock_save):
mock_get_root.return_value = "/etc/my.cnf"
dst = Ssh()
rmt_sql = RemoteMySQLSource({
"run_type": INTERVALS[0],
"full_backup": INTERVALS[0],
"mysql_connect_info": MySQLConnectInfo("/"),
"ssh_connection_info": None
})
rmt_sql.clone_config(dst)
mock_get_root.assert_called_with()
mock_save.assert_called_with(dst, "/etc/my.cnf")
def destination(self, backup_source=socket.gethostname()):
"""
:param backup_source: Hostname of the host where backup is taken from.
:type backup_source: str
:return: Backup destination instance
:rtype: BaseDestination
"""
try:
backup_destination = self.__cfg.get("destination",
"backup_destination")
if backup_destination == "ssh":
return Ssh(
self.ssh.path,
hostname=backup_source,
ssh_host=self.ssh.host,
ssh_port=self.ssh.port,
ssh_user=self.ssh.user,
ssh_key=self.ssh.key,
)
elif backup_destination == "s3":
return S3(
bucket=self.s3.bucket,
aws_access_key_id=self.s3.aws_access_key_id,
aws_secret_access_key=self.s3.aws_secret_access_key,
aws_default_region=self.s3.aws_default_region,
hostname=backup_source,
)
elif backup_destination == "gcs":
return GCS(
bucket=self.gcs.bucket,
gc_credentials_file=self.gcs.gc_credentials_file,
gc_encryption_key=self.gcs.gc_encryption_key,
hostname=backup_source,
)
else:
raise ConfigurationError("Unsupported destination '%s'" %
backup_destination)
except NoSectionError as err:
raise ConfigurationError(
"%s is missing required section 'destination'" %
self._config_file) from err
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990,
compress=False,
):
"""Clone mysql backup of remote machine and stream it to slave
:param cfg: TwinDB Backup tool config
:type cfg: TwinDBBackupConfig
"""
LOG.debug("Remote MySQL Source: %s", split_host_port(source)[0])
LOG.debug("MySQL defaults: %s", cfg.mysql.defaults_file)
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
src = RemoteMySQLSource({
"ssh_host":
split_host_port(source)[0],
"ssh_user":
cfg.ssh.user,
"ssh_key":
cfg.ssh.key,
"mysql_connect_info":
MySQLConnectInfo(cfg.mysql.defaults_file,
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"backup_type":
"full",
})
xbstream_binary = cfg.mysql.xbstream_binary
LOG.debug("SSH destination: %s", split_host_port(destination)[0])
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
dst = Ssh(
"/tmp",
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.ssh.user,
ssh_key=cfg.ssh.key,
)
datadir = src.datadir
LOG.debug("datadir: %s", datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(compress, datadir, destination, dst, netcat_port, src,
xbstream_binary)
LOG.debug("Copying MySQL config to the destination")
src.clone_config(dst)
LOG.debug("Remote MySQL destination: %s", split_host_port(destination)[0])
LOG.debug("MySQL defaults: %s", cfg.mysql.defaults_file)
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
dst_mysql = RemoteMySQLSource({
"ssh_host":
split_host_port(destination)[0],
"ssh_user":
cfg.ssh.user,
"ssh_key":
cfg.ssh.key,
"mysql_connect_info":
MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(destination)[0],
),
"run_type":
INTERVALS[0],
"backup_type":
"full",
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug("Binlog coordinates: (%s, %d)", binlog, position)
LOG.debug("Starting MySQL on the destination")
_mysql_service(dst, action="start")
LOG.debug("MySQL started")
LOG.debug("Setting up replication.")
LOG.debug("Master host: %s", source)
LOG.debug("Replication user: %s", replication_user)
LOG.debug("Replication password: %s", replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(
host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position,
))
def test_mkdirname_r(mock_mkdir_r):
dst = Ssh(remote_path='')
# noinspection PyProtectedMember
dst._mkdirname_r('/foo/bar/xyz')
mock_mkdir_r.assert_called_once_with('/foo/bar')
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990,
compress=False):
"""Clone mysql backup of remote machine and stream it to slave"""
try:
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
src = RemoteMySQLSource({
"ssh_host":
split_host_port(source)[0],
"ssh_user":
cfg.get('ssh', 'ssh_user'),
"ssh_key":
cfg.get('ssh', 'ssh_key'),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"backup_type":
'full'
})
xbstream_binary = cfg.get('mysql', 'xbstream_binary')
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst = Ssh('/tmp',
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.get('ssh', 'ssh_user'),
ssh_key=cfg.get('ssh', 'ssh_key'))
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(compress, datadir, destination, dst, netcat_port,
src, xbstream_binary)
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst_mysql = RemoteMySQLSource({
"ssh_host":
split_host_port(destination)[0],
"ssh_user":
cfg.get('ssh', 'ssh_user'),
"ssh_key":
cfg.get('ssh', 'ssh_key'),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(destination)[0]),
"run_type":
INTERVALS[0],
"backup_type":
'full'
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
try:
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
LOG.debug('MySQL started')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position))
except (ConfigParser.NoOptionError, OperationalError) as err:
LOG.error(err)
exit(1)
def clone_mysql(cfg, source, destination, # pylint: disable=too-many-arguments
replication_user, replication_password,
netcat_port=9990,
compress=False):
"""Clone mysql backup of remote machine and stream it to slave
:param cfg: TwinDB Backup tool config
:type cfg: TwinDBBackupConfig
"""
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug(
'MySQL defaults: %s',
cfg.mysql.defaults_file
)
LOG.debug(
'SSH username: %s',
cfg.ssh.user
)
LOG.debug(
'SSH key: %s',
cfg.ssh.key
)
src = RemoteMySQLSource(
{
"ssh_host": split_host_port(source)[0],
"ssh_user": cfg.ssh.user,
"ssh_key": cfg.ssh.key,
"mysql_connect_info": MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(source)[0]),
"run_type": INTERVALS[0],
"backup_type": 'full'
}
)
xbstream_binary = cfg.mysql.xbstream_binary
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.ssh.user)
LOG.debug('SSH key: %s', cfg.ssh.key)
dst = Ssh(
'/tmp',
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.ssh.user,
ssh_key=cfg.ssh.key
)
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(
compress,
datadir,
destination,
dst,
netcat_port,
src,
xbstream_binary
)
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug(
'MySQL defaults: %s',
cfg.mysql.defaults_file
)
LOG.debug('SSH username: %s', cfg.ssh.user)
LOG.debug('SSH key: %s', cfg.ssh.key)
dst_mysql = RemoteMySQLSource({
"ssh_host": split_host_port(destination)[0],
"ssh_user": cfg.ssh.user,
"ssh_key": cfg.ssh.key,
"mysql_connect_info": MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(destination)[0]
),
"run_type": INTERVALS[0],
"backup_type": 'full'
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
LOG.debug('MySQL started')
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(
host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position
)
)
def test_list_files():
dst = Ssh('/var/backups')
mock_client = mock.Mock()
mock_client.list_files.return_value = ['foo', 'bar']
dst._ssh_client = mock_client
assert dst.list_files('xxx', pattern='foo') == ['foo']
def test_basename():
dst = Ssh(remote_path='/foo/bar')
assert dst.basename('/foo/bar/some_dir/some_file.txt') \
== 'some_dir/some_file.txt'
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990):
"""Clone mysql backup of remote machine and stream it to slave"""
try:
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
src = RemoteMySQLSource({
"ssh_connection_info":
SshConnectInfo(host=split_host_port(source)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"full_backup":
INTERVALS[0],
})
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst = Ssh(ssh_connect_info=SshConnectInfo(
host=split_host_port(destination)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')), )
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
try:
LOG.debug('Stopping MySQL on the destination')
_mysql_service(dst, action='stop')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
proc_netcat = Process(
target=dst.netcat,
args=("gunzip -c - | xbstream -x -C {datadir}".format(
datadir=datadir), ),
kwargs={'port': netcat_port})
proc_netcat.start()
LOG.debug('Starting netcat on the destination')
src.clone(dest_host=split_host_port(destination)[0], port=netcat_port)
proc_netcat.join()
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst_mysql = RemoteMySQLSource({
"ssh_connection_info":
SshConnectInfo(host=split_host_port(destination)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(destination)[0]),
"run_type":
INTERVALS[0],
"full_backup":
INTERVALS[0],
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
try:
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(source, replication_user, replication_password,
binlog, position)
except (ConfigParser.NoOptionError, OperationalError) as err:
LOG.error(err)
exit(1)
