def _run_remote_netcat(
compress,
datadir, # pylint: disable=too-many-arguments
destination,
dst,
netcat_port,
src):
netcat_cmd = "xbstream -x -C {datadir}".format(datadir=datadir)
if compress:
netcat_cmd = "gunzip -c - | %s" % netcat_cmd
# find unused port
while netcat_port < 64000:
if dst.ensure_tcp_port_listening(netcat_port, wait_timeout=1):
netcat_port += 1
else:
LOG.debug('Will use port %d for streaming', netcat_port)
break
proc_netcat = Process(target=dst.netcat,
args=(netcat_cmd, ),
kwargs={'port': netcat_port})
LOG.debug('Starting netcat on the destination')
proc_netcat.start()
nc_wait_timeout = 10
if not dst.ensure_tcp_port_listening(netcat_port,
wait_timeout=nc_wait_timeout):
LOG.error('netcat on the destination '
'is not ready after %d seconds', nc_wait_timeout)
proc_netcat.terminate()
exit(1)
src.clone(dest_host=split_host_port(destination)[0],
port=netcat_port,
compress=compress)
proc_netcat.join()
def _run_remote_netcat(compress, datadir, # pylint: disable=too-many-arguments
destination, dst, netcat_port, src, xbstream_path):
netcat_cmd = "{xbstream_binary} -x -C {datadir}".format(
xbstream_binary=xbstream_path,
datadir=datadir
)
if compress:
netcat_cmd = "gunzip -c - | %s" % netcat_cmd
# find unused port
while netcat_port < 64000:
if dst.ensure_tcp_port_listening(netcat_port, wait_timeout=1):
netcat_port += 1
else:
LOG.debug('Will use port %d for streaming', netcat_port)
break
proc_netcat = Process(
target=dst.netcat,
args=(netcat_cmd,),
kwargs={
'port': netcat_port
}
)
LOG.debug('Starting netcat on the destination')
proc_netcat.start()
nc_wait_timeout = 10
if not dst.ensure_tcp_port_listening(netcat_port,
wait_timeout=nc_wait_timeout):
LOG.error('netcat on the destination '
'is not ready after %d seconds', nc_wait_timeout)
proc_netcat.terminate()
exit(1)
src.clone(
dest_host=split_host_port(destination)[0],
port=netcat_port,
compress=compress
)
proc_netcat.join()
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990,
compress=False):
"""Clone mysql backup of remote machine and stream it to slave"""
try:
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
src = RemoteMySQLSource({
"ssh_host":
split_host_port(source)[0],
"ssh_user":
cfg.get('ssh', 'ssh_user'),
"ssh_key":
cfg.get('ssh', 'ssh_key'),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"backup_type":
'full'
})
xbstream_binary = cfg.get('mysql', 'xbstream_binary')
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst = Ssh('/tmp',
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.get('ssh', 'ssh_user'),
ssh_key=cfg.get('ssh', 'ssh_key'))
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(compress, datadir, destination, dst, netcat_port,
src, xbstream_binary)
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst_mysql = RemoteMySQLSource({
"ssh_host":
split_host_port(destination)[0],
"ssh_user":
cfg.get('ssh', 'ssh_user'),
"ssh_key":
cfg.get('ssh', 'ssh_key'),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(destination)[0]),
"run_type":
INTERVALS[0],
"backup_type":
'full'
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
try:
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
LOG.debug('MySQL started')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position))
except (ConfigParser.NoOptionError, OperationalError) as err:
LOG.error(err)
exit(1)
def clone_mysql(cfg, source, destination, # pylint: disable=too-many-arguments
replication_user, replication_password,
netcat_port=9990,
compress=False):
"""Clone mysql backup of remote machine and stream it to slave
:param cfg: TwinDB Backup tool config
:type cfg: TwinDBBackupConfig
"""
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug(
'MySQL defaults: %s',
cfg.mysql.defaults_file
)
LOG.debug(
'SSH username: %s',
cfg.ssh.user
)
LOG.debug(
'SSH key: %s',
cfg.ssh.key
)
src = RemoteMySQLSource(
{
"ssh_host": split_host_port(source)[0],
"ssh_user": cfg.ssh.user,
"ssh_key": cfg.ssh.key,
"mysql_connect_info": MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(source)[0]),
"run_type": INTERVALS[0],
"backup_type": 'full'
}
)
xbstream_binary = cfg.mysql.xbstream_binary
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.ssh.user)
LOG.debug('SSH key: %s', cfg.ssh.key)
dst = Ssh(
'/tmp',
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.ssh.user,
ssh_key=cfg.ssh.key
)
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(
compress,
datadir,
destination,
dst,
netcat_port,
src,
xbstream_binary
)
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug(
'MySQL defaults: %s',
cfg.mysql.defaults_file
)
LOG.debug('SSH username: %s', cfg.ssh.user)
LOG.debug('SSH key: %s', cfg.ssh.key)
dst_mysql = RemoteMySQLSource({
"ssh_host": split_host_port(destination)[0],
"ssh_user": cfg.ssh.user,
"ssh_key": cfg.ssh.key,
"mysql_connect_info": MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(destination)[0]
),
"run_type": INTERVALS[0],
"backup_type": 'full'
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
LOG.debug('MySQL started')
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(
host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position
)
)
def test_split_host_port(pair, host, port):
assert split_host_port(pair) == (host, port)
def test_split_host_port(pair, host, port):
assert split_host_port(pair) == (host, port)
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990):
"""Clone mysql backup of remote machine and stream it to slave"""
try:
LOG.debug('Remote MySQL Source: %s', split_host_port(source)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
src = RemoteMySQLSource({
"ssh_connection_info":
SshConnectInfo(host=split_host_port(source)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"full_backup":
INTERVALS[0],
})
LOG.debug('SSH destination: %s', split_host_port(destination)[0])
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst = Ssh(ssh_connect_info=SshConnectInfo(
host=split_host_port(destination)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')), )
datadir = src.datadir
LOG.debug('datadir: %s', datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
try:
LOG.debug('Stopping MySQL on the destination')
_mysql_service(dst, action='stop')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
proc_netcat = Process(
target=dst.netcat,
args=("gunzip -c - | xbstream -x -C {datadir}".format(
datadir=datadir), ),
kwargs={'port': netcat_port})
proc_netcat.start()
LOG.debug('Starting netcat on the destination')
src.clone(dest_host=split_host_port(destination)[0], port=netcat_port)
proc_netcat.join()
LOG.debug('Copying MySQL config to the destination')
src.clone_config(dst)
LOG.debug('Remote MySQL destination: %s',
split_host_port(destination)[0])
LOG.debug('MySQL defaults: %s', cfg.get('mysql',
'mysql_defaults_file'))
LOG.debug('SSH username: %s', cfg.get('ssh', 'ssh_user'))
LOG.debug('SSH key: %s', cfg.get('ssh', 'ssh_key'))
dst_mysql = RemoteMySQLSource({
"ssh_connection_info":
SshConnectInfo(host=split_host_port(destination)[0],
user=cfg.get('ssh', 'ssh_user'),
key=cfg.get('ssh', 'ssh_key')),
"mysql_connect_info":
MySQLConnectInfo(cfg.get('mysql', 'mysql_defaults_file'),
hostname=split_host_port(destination)[0]),
"run_type":
INTERVALS[0],
"full_backup":
INTERVALS[0],
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug('Binlog coordinates: (%s, %d)', binlog, position)
try:
LOG.debug('Starting MySQL on the destination')
_mysql_service(dst, action='start')
except TwinDBBackupError as err:
LOG.error(err)
exit(1)
LOG.debug('Setting up replication.')
LOG.debug('Master host: %s', source)
LOG.debug('Replication user: %s', replication_user)
LOG.debug('Replication password: %s', replication_password)
dst_mysql.setup_slave(source, replication_user, replication_password,
binlog, position)
except (ConfigParser.NoOptionError, OperationalError) as err:
LOG.error(err)
exit(1)
def clone_mysql(
cfg,
source,
destination, # pylint: disable=too-many-arguments
replication_user,
replication_password,
netcat_port=9990,
compress=False,
):
"""Clone mysql backup of remote machine and stream it to slave
:param cfg: TwinDB Backup tool config
:type cfg: TwinDBBackupConfig
"""
LOG.debug("Remote MySQL Source: %s", split_host_port(source)[0])
LOG.debug("MySQL defaults: %s", cfg.mysql.defaults_file)
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
src = RemoteMySQLSource({
"ssh_host":
split_host_port(source)[0],
"ssh_user":
cfg.ssh.user,
"ssh_key":
cfg.ssh.key,
"mysql_connect_info":
MySQLConnectInfo(cfg.mysql.defaults_file,
hostname=split_host_port(source)[0]),
"run_type":
INTERVALS[0],
"backup_type":
"full",
})
xbstream_binary = cfg.mysql.xbstream_binary
LOG.debug("SSH destination: %s", split_host_port(destination)[0])
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
dst = Ssh(
"/tmp",
ssh_host=split_host_port(destination)[0],
ssh_user=cfg.ssh.user,
ssh_key=cfg.ssh.key,
)
datadir = src.datadir
LOG.debug("datadir: %s", datadir)
if dst.list_files(datadir):
LOG.error("Destination datadir is not empty: %s", datadir)
exit(1)
_run_remote_netcat(compress, datadir, destination, dst, netcat_port, src,
xbstream_binary)
LOG.debug("Copying MySQL config to the destination")
src.clone_config(dst)
LOG.debug("Remote MySQL destination: %s", split_host_port(destination)[0])
LOG.debug("MySQL defaults: %s", cfg.mysql.defaults_file)
LOG.debug("SSH username: %s", cfg.ssh.user)
LOG.debug("SSH key: %s", cfg.ssh.key)
dst_mysql = RemoteMySQLSource({
"ssh_host":
split_host_port(destination)[0],
"ssh_user":
cfg.ssh.user,
"ssh_key":
cfg.ssh.key,
"mysql_connect_info":
MySQLConnectInfo(
cfg.mysql.defaults_file,
hostname=split_host_port(destination)[0],
),
"run_type":
INTERVALS[0],
"backup_type":
"full",
})
binlog, position = dst_mysql.apply_backup(datadir)
LOG.debug("Binlog coordinates: (%s, %d)", binlog, position)
LOG.debug("Starting MySQL on the destination")
_mysql_service(dst, action="start")
LOG.debug("MySQL started")
LOG.debug("Setting up replication.")
LOG.debug("Master host: %s", source)
LOG.debug("Replication user: %s", replication_user)
LOG.debug("Replication password: %s", replication_password)
dst_mysql.setup_slave(
MySQLMasterInfo(
host=split_host_port(source)[0],
port=split_host_port(source)[1],
user=replication_user,
password=replication_password,
binlog=binlog,
binlog_pos=position,
))
