def prepare_target_database_dump():
"""
Preparing the target database dump by the unpacked .tar.gz file
:return:
"""
output.message(output.Subject.TARGET, 'Extracting database dump', True)
mode.run_command(
helper.get_command('target', 'tar') + ' xzf ' + helper.get_dump_dir(mode.Client.TARGET) +
database_utility.database_dump_file_name + '.tar.gz -C ' +
helper.get_dump_dir(mode.Client.TARGET) + ' > /dev/null',
mode.Client.TARGET,
skip_dry_run=True
)
def create_origin_database_dump():
"""
Creating the origin database dump file
:return:
"""
if not mode.is_import():
parser.get_database_configuration(mode.Client.ORIGIN)
database_utility.generate_database_dump_filename()
helper.check_and_create_dump_dir(mode.Client.ORIGIN,
helper.get_dump_dir(mode.Client.ORIGIN))
_dump_file_path = helper.get_dump_dir(
mode.Client.ORIGIN) + database_utility.database_dump_file_name
_database_version = database_utility.get_database_version(mode.Client.ORIGIN)
output.message(
output.Subject.ORIGIN,
f'Creating database dump {output.CliFormat.BLACK}{_dump_file_path}{output.CliFormat.ENDC}',
True
)
_mysqldump_options = '--no-tablespaces '
# Remove --no-tablespaces option for mysql < 5.6
# @ToDo: Better option handling
if not _database_version is None:
if _database_version[0] == database_utility.DatabaseSystem.MYSQL and \
semantic_version.Version(_database_version[1]) < semantic_version.Version('5.6.0'):
_mysqldump_options = ''
# Run mysql dump command, e.g.
# mysqldump --no-tablespaces -u'db' -p'db' -h'db1' -P'3306' 'db' > /tmp/_db_08-10-2021_07-00.sql
mode.run_command(
helper.get_command(mode.Client.ORIGIN, 'mysqldump') + ' ' + _mysqldump_options +
database_utility.generate_mysql_credentials(mode.Client.ORIGIN) + ' \'' +
system.config[mode.Client.ORIGIN]['db']['name'] + '\' ' +
database_utility.generate_ignore_database_tables() +
database_utility.get_database_tables() +
' > ' + _dump_file_path,
mode.Client.ORIGIN,
skip_dry_run=True
)
database_utility.check_database_dump(mode.Client.ORIGIN, _dump_file_path)
database_utility.count_tables(mode.Client.ORIGIN, _dump_file_path)
prepare_origin_database_dump()
def prepare_origin_database_dump():
"""
Preparing the origin database dump file by compressing them as .tar.gz
:return:
"""
output.message(
output.Subject.ORIGIN,
'Compressing database dump',
True
)
mode.run_command(
helper.get_command(mode.Client.ORIGIN, 'tar') + ' cfvz ' + helper.get_dump_dir(
mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz -C ' +
helper.get_dump_dir(mode.Client.ORIGIN) + ' ' +
database_utility.database_dump_file_name + ' > /dev/null',
mode.Client.ORIGIN,
skip_dry_run=True
)
def put_origin_database_dump(origin_path):
"""
Uploading the origin database dump file
:param origin_path: String
:return:
"""
if mode.get_sync_mode() == mode.SyncMode.PROXY:
_subject = output.Subject.LOCAL
else:
_subject = output.Subject.ORIGIN
output.message(
_subject,
'Uploading database dump',
True
)
helper.check_and_create_dump_dir(mode.Client.TARGET, helper.get_dump_dir(mode.Client.TARGET))
if not system.config['dry_run']:
_localpath = origin_path + database_utility.database_dump_file_name + '.tar.gz'
_remotepath = helper.get_dump_dir(mode.Client.TARGET) + '/'
if system.config['use_rsync']:
rsync.run_rsync_command(
remote_client=mode.Client.TARGET,
origin_path=_localpath,
target_path=_remotepath,
target_ssh=system.config[mode.Client.TARGET]['user'] + '@' + system.config[mode.Client.TARGET]['host']
)
else:
#
# Download speed problems
# https://github.com/paramiko/paramiko/issues/60
#
sftp = get_sftp_client(client.ssh_client_target)
sftp.put(origin_path + database_utility.database_dump_file_name + '.tar.gz',
helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name + '.tar.gz',
upload_status)
sftp.close()
if not system.config['mute']:
print('')
def remove_origin_database_dump(keep_compressed_file=False):
"""
Removing the origin database dump files
:param keep_compressed_file: Boolean
:return:
"""
output.message(
output.Subject.ORIGIN,
'Cleaning up',
True
)
if system.config['dry_run']:
return
_file_path = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name
if mode.is_origin_remote():
sftp = remote_client.ssh_client_origin.open_sftp()
sftp.remove(_file_path)
if not keep_compressed_file:
sftp.remove(f'{_file_path}.tar.gz')
sftp.close()
else:
os.remove(_file_path)
if not keep_compressed_file:
os.remove(f'{_file_path}.tar.gz')
if keep_compressed_file:
if 'keep_dumps' in system.config[mode.Client.ORIGIN]:
helper.clean_up_dump_dir(mode.Client.ORIGIN,
helper.get_dump_dir(mode.Client.ORIGIN) + '*',
system.config[mode.Client.ORIGIN]['keep_dumps'])
output.message(
output.Subject.INFO,
f'Database dump file is saved to: {_file_path}.tar.gz',
True,
True
)
def get_origin_database_dump(target_path):
"""
Downloading the origin database dump files
:param target_path: String
:return:
"""
output.message(
output.Subject.ORIGIN,
'Downloading database dump',
True
)
if mode.get_sync_mode() != mode.SyncMode.PROXY:
helper.check_and_create_dump_dir(mode.Client.TARGET, target_path)
if not system.config['dry_run']:
_remotepath = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz'
_localpath = target_path
if system.config['use_rsync']:
rsync.run_rsync_command(
remote_client=mode.Client.ORIGIN,
origin_path=_remotepath,
target_path=_localpath,
origin_ssh=system.config[mode.Client.ORIGIN]['user'] + '@' + system.config[mode.Client.ORIGIN]['host']
)
else:
#
# Download speed problems
# https://github.com/paramiko/paramiko/issues/60
#
sftp = get_sftp_client(client.ssh_client_origin)
sftp.get(helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz',
target_path + database_utility.database_dump_file_name + '.tar.gz', download_status)
sftp.close()
if not system.config['mute']:
print('')
utility.remove_origin_database_dump()
def transfer_origin_database_dump():
"""
Transfer the origin database dump files
:return:
"""
if not mode.is_import():
if mode.get_sync_mode() == mode.SyncMode.RECEIVER:
get_origin_database_dump(helper.get_dump_dir(mode.Client.TARGET))
system.check_target_configuration()
elif mode.get_sync_mode() == mode.SyncMode.SENDER:
system.check_target_configuration()
put_origin_database_dump(helper.get_dump_dir(mode.Client.ORIGIN))
utility.remove_origin_database_dump()
elif mode.get_sync_mode() == mode.SyncMode.PROXY:
helper.create_local_temporary_data_dir()
get_origin_database_dump(system.default_local_sync_path)
system.check_target_configuration()
put_origin_database_dump(system.default_local_sync_path)
elif mode.get_sync_mode() == mode.SyncMode.SYNC_REMOTE or mode.get_sync_mode() == mode.SyncMode.SYNC_LOCAL:
system.check_target_configuration()
elif system.config['is_same_client']:
utility.remove_origin_database_dump(True)
else:
system.check_target_configuration()
def remove_target_database_dump():
"""
Removing the target database dump files
:return:
"""
_file_path = helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name
#
# Move dump to specified directory
#
if system.config['keep_dump']:
helper.create_local_temporary_data_dir()
_keep_dump_path = system.default_local_sync_path + database_utility.database_dump_file_name
mode.run_command(
helper.get_command('target',
'cp') + ' ' + _file_path + ' ' + _keep_dump_path,
mode.Client.TARGET
)
output.message(
output.Subject.INFO,
f'Database dump file is saved to: {_keep_dump_path}',
True,
True
)
#
# Clean up
#
if not mode.is_dump() and not mode.is_import():
output.message(
output.Subject.TARGET,
'Cleaning up',
True
)
if system.config['dry_run']:
return
if mode.is_target_remote():
sftp = remote_client.ssh_client_target.open_sftp()
sftp.remove(_file_path)
sftp.remove(f'{_file_path}.tar.gz')
sftp.close()
else:
if os.path.isfile(_file_path):
os.remove(_file_path)
if os.path.isfile(f'{_file_path}.tar.gz'):
os.remove(f'{_file_path}.tar.gz')
def import_database_dump():
"""
Importing the selected database dump file
:return:
"""
if not system.config['is_same_client'] and not mode.is_import():
prepare_target_database_dump()
if system.config['clear_database']:
output.message(
output.Subject.TARGET,
'Clearing database before import',
True
)
clear_database(mode.Client.TARGET)
database_utility.truncate_tables()
if not system.config['keep_dump'] and not mode.is_dump():
database_utility.get_database_version(mode.Client.TARGET)
output.message(
output.Subject.TARGET,
'Importing database dump',
True
)
if not mode.is_import():
_dump_path = helper.get_dump_dir(
mode.Client.TARGET) + database_utility.database_dump_file_name
else:
_dump_path = system.config['import']
if not system.config['yes']:
_host_name = helper.get_ssh_host_name(mode.Client.TARGET, True) if mode.is_remote(
mode.Client.TARGET) else 'local'
helper.confirm(
output.message(
output.Subject.TARGET,
f'Are you sure, you want to import the dump file into {_host_name} database?',
False
),
True
)
database_utility.check_database_dump(mode.Client.TARGET, _dump_path)
import_database_dump_file(mode.Client.TARGET, _dump_path)
if 'after_dump' in system.config['target']:
_after_dump = system.config['target']['after_dump']
output.message(
output.Subject.TARGET,
f'Importing after_dump file {output.CliFormat.BLACK}{_after_dump}{output.CliFormat.ENDC}',
True
)
import_database_dump_file(mode.Client.TARGET, _after_dump)
if 'post_sql' in system.config['target']:
output.message(
output.Subject.TARGET,
f'Running addition post sql commands',
True
)
for _sql_command in system.config['target']['post_sql']:
database_utility.run_database_command(mode.Client.TARGET, _sql_command, True)