def clear_database(client):
"""
Clearing the database by dropping all tables
https://www.techawaken.com/drop-tables-mysql-database/
{ mysql -hHOSTNAME -uUSERNAME -pPASSWORD -Nse 'show tables' DB_NAME; } |
( while read table; do if [ -z ${i+x} ]; then echo 'SET FOREIGN_KEY_CHECKS = 0;'; fi; i=1;
echo "drop table \`$table\`;"; done;
echo 'SET FOREIGN_KEY_CHECKS = 1;' ) |
awk '{print}' ORS=' ' | mysql -hHOSTNAME -uUSERNAME -pPASSWORD DB_NAME;
:param client: String
:return:
"""
mode.run_command(
'{ ' + helper.get_command(client, 'mysql') + ' ' +
database_utility.generate_mysql_credentials(client) +
' -Nse \'show tables\' \'' +
system.config[client]['db']['name'] + '\'; }' +
' | ( while read table; do if [ -z ${i+x} ]; then echo \'SET FOREIGN_KEY_CHECKS = 0;\'; fi; i=1; ' +
'echo "drop table \\`$table\\`;"; done; echo \'SET FOREIGN_KEY_CHECKS = 1;\' ) | awk \'{print}\' ORS=' ' | ' +
helper.get_command(client, 'mysql') + ' ' +
database_utility.generate_mysql_credentials(client) + ' ' +
system.config[client]['db']['name'],
client,
skip_dry_run=True
)
def clean_up_dump_dir(client, path, num=5):
"""
Clean up the dump directory from old dump files (only affect .sql and .tar.gz files)
:param client:
:param path:
:param num:
:return:
"""
# Distinguish stat command on os system (Darwin|Linux)
if check_os(client).strip() == 'Darwin':
_command = get_command(client, 'stat') + ' -f "%Sm %N" ' + path + ' | ' + get_command(
client,
'sort') + ' -rn | ' + get_command(
client, 'grep') + ' -E ".tar.gz|.sql"'
else:
_command = get_command(client, 'stat') + ' -c "%y %n" ' + path + ' | ' + \
get_command(client,'sort') + ' -rn | ' + get_command(client, 'grep') + \
' -E ".tar.gz|.sql"'
# List files in directory sorted by change date
_files = mode.run_command(
_command,
client,
True
).splitlines()
for i in range(len(_files)):
_filename = _files[i].rsplit(' ', 1)[-1]
# Remove oldest files chosen by keep_dumps count
if not i < num:
mode.run_command(
'rm ' + _filename,
client
)
def run_script(client=None, script='before'):
"""
Executing script command
:param client: String
:param script: String
:return:
"""
if client is None:
_config = system.config
_subject = output.Subject.LOCAL
client = mode.Client.LOCAL
else:
_config = system.config[client]
_subject = output.host_to_subject(client)
if not 'scripts' in _config:
return
if f'{script}' in _config['scripts']:
output.message(
_subject,
f'Running script {client}',
True
)
mode.run_command(
_config['scripts'][script],
client
)
def check_configuration(client):
"""
Checking Drupal database configuration with Drush
:param client: String
:return:
"""
_path = system.config[client]['path']
# Check Drush version
_raw_version = mode.run_command(
f'{helper.get_command(client, "drush")} status --fields=drush-version --format=string '
f'-r {_path}',
client,
True
)
output.message(
output.host_to_subject(client),
f'Drush version: {_raw_version}',
True
)
stdout = mode.run_command(
f'{helper.get_command(client, "drush")} core-status --pipe '
f'--fields=db-hostname,db-username,db-password,db-name,db-port '
f'-r {_path}',
client,
True
)
_db_config = parse_database_credentials(json.loads(stdout))
system.config[client]['db'] = _db_config
def check_and_create_dump_dir(client, path):
"""
Check if a path exists on the client system and creates the given path if necessary
:param client:
:param path:
:return:
"""
mode.run_command(
'[ ! -d "' + path + '" ] && mkdir -p "' + path + '"',
client
)
def prepare_target_database_dump():
"""
Preparing the target database dump by the unpacked .tar.gz file
:return:
"""
output.message(output.Subject.TARGET, 'Extracting database dump', True)
mode.run_command(
helper.get_command('target', 'tar') + ' xzf ' + helper.get_dump_dir(mode.Client.TARGET) +
database_utility.database_dump_file_name + '.tar.gz -C ' +
helper.get_dump_dir(mode.Client.TARGET) + ' > /dev/null',
mode.Client.TARGET,
skip_dry_run=True
)
def import_database_dump_file(client, filepath):
"""
Import a database dump file
:param client: String
:param filepath: String
:return:
"""
if helper.check_file_exists(client, filepath):
mode.run_command(
helper.get_command(client, 'mysql') + ' ' +
database_utility.generate_mysql_credentials(client) + ' \'' +
system.config[client]['db']['name'] + '\' < ' + filepath,
client,
skip_dry_run=True
)
def remove_target_database_dump():
"""
Removing the target database dump files
:return:
"""
_file_path = helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name
#
# Move dump to specified directory
#
if system.config['keep_dump']:
helper.create_local_temporary_data_dir()
_keep_dump_path = system.default_local_sync_path + database_utility.database_dump_file_name
mode.run_command(
helper.get_command('target',
'cp') + ' ' + _file_path + ' ' + _keep_dump_path,
mode.Client.TARGET
)
output.message(
output.Subject.INFO,
f'Database dump file is saved to: {_keep_dump_path}',
True,
True
)
#
# Clean up
#
if not mode.is_dump() and not mode.is_import():
output.message(
output.Subject.TARGET,
'Cleaning up',
True
)
if system.config['dry_run']:
return
if mode.is_target_remote():
sftp = remote_client.ssh_client_target.open_sftp()
sftp.remove(_file_path)
sftp.remove(f'{_file_path}.tar.gz')
sftp.close()
else:
if os.path.isfile(_file_path):
os.remove(_file_path)
if os.path.isfile(f'{_file_path}.tar.gz'):
os.remove(f'{_file_path}.tar.gz')
def check_configuration(client):
"""
Checking remote TYPO3 database configuration
:param client: String
:return:
"""
_path = system.config[client]['path']
if 'LocalConfiguration' in _path:
stdout = mode.run_command(
helper.get_command(client, 'php') + ' -r "echo json_encode(include \'' +
system.config[client][
'path'] + '\');"',
client,
True
)
_db_config = parse_database_credentials(json.loads(stdout)['DB'])
else:
# Try to parse settings from AdditionalConfiguration.php file
_db_config = {
'name': get_database_setting(client, 'dbname', system.config[client]['path']),
'host': get_database_setting(client, 'host', system.config[client]['path']),
'password': get_database_setting(client, 'password', system.config[client]['path']),
'port': get_database_setting(client, 'port', system.config[client]['path'])
if get_database_setting(client, 'port',
system.config[client]['path']) != '' else 3306,
'user': get_database_setting(client, 'user', system.config[client]['path']),
}
system.config[client]['db'] = _db_config
def check_configuration(client):
"""
Checking remote Symfony database configuration
:param client: String
:return:
"""
_path = system.config[client]['path']
# Check for symfony 2.8
if 'parameters.yml' in _path:
_db_config = {
'name': get_database_parameter(client, 'database_name', _path),
'host': get_database_parameter(client, 'database_host', _path),
'password': get_database_parameter(client, 'database_password', _path),
'port': get_database_parameter(client, 'database_port', _path),
'user': get_database_parameter(client, 'database_user', _path),
}
# Using for symfony >=3.4
else:
stdout = mode.run_command(
helper.get_command(client, 'grep') + ' -v "^#" ' + system.config[client][
'path'] + ' | ' + helper.get_command(client, 'grep') + ' DATABASE_URL',
client,
True
)
_db_config = parse_database_credentials(stdout)
system.config[client]['db'] = _db_config
def check_database_dump(client, filepath):
"""
Checking the last line of the dump file if it contains "-- Dump completed on"
:param client: String
:param filepath: String
:return:
"""
if system.config['check_dump']:
_line = mode.run_command(
helper.get_command(client, 'tail') + ' -n 1 ' + filepath,
client,
True,
skip_dry_run=True
)
if not _line:
return
if "-- Dump completed on" not in _line:
sys.exit(
output.message(
output.Subject.ERROR,
'Dump file is corrupted',
do_print=False
)
)
else:
output.message(
output.host_to_subject(client),
'Dump file is valid',
verbose_only=True
)
def check_file_exists(client, path):
"""
Check if a file exists
:param client: String
:param path: String file path
:return: Boolean
"""
return mode.run_command(f'[ -f {path} ] && echo "1"', client, True) == '1'
def create_origin_database_dump():
"""
Creating the origin database dump file
:return:
"""
if not mode.is_import():
parser.get_database_configuration(mode.Client.ORIGIN)
database_utility.generate_database_dump_filename()
helper.check_and_create_dump_dir(mode.Client.ORIGIN,
helper.get_dump_dir(mode.Client.ORIGIN))
_dump_file_path = helper.get_dump_dir(
mode.Client.ORIGIN) + database_utility.database_dump_file_name
_database_version = database_utility.get_database_version(mode.Client.ORIGIN)
output.message(
output.Subject.ORIGIN,
f'Creating database dump {output.CliFormat.BLACK}{_dump_file_path}{output.CliFormat.ENDC}',
True
)
_mysqldump_options = '--no-tablespaces '
# Remove --no-tablespaces option for mysql < 5.6
# @ToDo: Better option handling
if not _database_version is None:
if _database_version[0] == database_utility.DatabaseSystem.MYSQL and \
semantic_version.Version(_database_version[1]) < semantic_version.Version('5.6.0'):
_mysqldump_options = ''
# Run mysql dump command, e.g.
# mysqldump --no-tablespaces -u'db' -p'db' -h'db1' -P'3306' 'db' > /tmp/_db_08-10-2021_07-00.sql
mode.run_command(
helper.get_command(mode.Client.ORIGIN, 'mysqldump') + ' ' + _mysqldump_options +
database_utility.generate_mysql_credentials(mode.Client.ORIGIN) + ' \'' +
system.config[mode.Client.ORIGIN]['db']['name'] + '\' ' +
database_utility.generate_ignore_database_tables() +
database_utility.get_database_tables() +
' > ' + _dump_file_path,
mode.Client.ORIGIN,
skip_dry_run=True
)
database_utility.check_database_dump(mode.Client.ORIGIN, _dump_file_path)
database_utility.count_tables(mode.Client.ORIGIN, _dump_file_path)
prepare_origin_database_dump()
def prepare_origin_database_dump():
"""
Preparing the origin database dump file by compressing them as .tar.gz
:return:
"""
output.message(
output.Subject.ORIGIN,
'Compressing database dump',
True
)
mode.run_command(
helper.get_command(mode.Client.ORIGIN, 'tar') + ' cfvz ' + helper.get_dump_dir(
mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz -C ' +
helper.get_dump_dir(mode.Client.ORIGIN) + ' ' +
database_utility.database_dump_file_name + ' > /dev/null',
mode.Client.ORIGIN,
skip_dry_run=True
)
def synchronize(origin_path, target_path, exclude, client=mode.Client.LOCAL, pseudo_client=None, force_remote=False):
"""
Using rsync command to synchronize files between systems
:param origin_path: String
:param target_path: String
:param exclude: List
:param client: String
:param pseudo_client: String Client, which will be forced as remote client. Necessary for proxy transfer.
:param force_remote: Boolean
:return:
"""
_remote_client = None
if force_remote:
remote_client.load_ssh_client_origin()
_origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} '
_target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} '
elif mode.is_remote(mode.Client.ORIGIN) and pseudo_client != mode.Client.TARGET:
_remote_client = mode.Client.ORIGIN
_origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} '
_target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} '
elif mode.is_remote(mode.Client.TARGET) and pseudo_client != mode.Client.ORIGIN:
_remote_client = mode.Client.TARGET
_origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} '
_target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} '
elif not mode.is_remote(mode.Client.TARGET) and not mode.is_remote(mode.Client.ORIGIN):
_origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} '
_target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} '
_origin_name = helper.get_ssh_host_name(mode.Client.ORIGIN, True) if _remote_client == mode.Client.ORIGIN else ''
_target_name = helper.get_ssh_host_name(mode.Client.TARGET, True) if _remote_client == mode.Client.TARGET else ''
if not system.config['mute']:
print(
f'{_origin_subject}'
f'{_origin_name}'
f'{output.CliFormat.BLACK}{origin_path}{output.CliFormat.ENDC}'
)
print(
f'{_target_subject}'
f'{_target_name}'
f'{output.CliFormat.BLACK}{target_path}{output.CliFormat.ENDC}'
)
_origin_user_host = utility.get_host(mode.Client.ORIGIN) if _remote_client == mode.Client.ORIGIN else ''
_target_user_host = utility.get_host(mode.Client.TARGET) if _remote_client == mode.Client.TARGET else ''
_output = mode.run_command(
f'{utility.get_password_environment(_remote_client)}rsync {utility.get_options()} '
f'{utility.get_authorization(_remote_client)} {utility.get_excludes(exclude)}'
f'{_origin_user_host}{origin_path} {_target_user_host}{target_path}',
client,
True
)
utility.read_stats(_output)
def check_os(client):
"""
Check which system is running (Linux|Darwin)
:param client:
:return:
"""
return mode.run_command(
get_command(client, 'uname') + ' -s',
client,
True
)
def get_database_parameter(client, name, file):
"""
Parsing a single database variable from the parameters.yml file
hhttps://unix.stackexchange.com/questions/84922/extract-a-part-of-one-line-from-a-file-with-sed
:param client: String
:param name: String
:param file: String
:return:
"""
return mode.run_command(
helper.get_command(client, 'sed') + f' -n -e \'/{name}/ s/.*\\: *//p\' {file}',
client,
True
).replace('\n', '')
def get_database_parameter(client, name, file):
"""
Parsing a single database variable from the .env file
https://gist.github.com/judy2k/7656bfe3b322d669ef75364a46327836
:param client: String
:param name: String
:param file: String
:return:
"""
return mode.run_command(
helper.get_command(client, 'grep') + f' {name} {file} | cut -d \'=\' -f2',
client,
True
).replace('\n', '')
def run_database_command(client, command, force_database_name=False):
"""
Run a database command using the "mysql -e" command
:param client: String
:param command: String database command
:param force_database_name: Bool forces the database name
:return:
"""
_database_name = ' ' + system.config[client]['db']['name'] if force_database_name else ''
return mode.run_command(
helper.get_command(client, 'mysql') + ' ' + generate_mysql_credentials(
client) + _database_name + ' -e "' + command + '"',
client, True)
def get_database_setting(client, name, file):
"""
Parsing a single database variable from the wp-config.php file
https://stackoverflow.com/questions/63493645/extract-database-name-from-a-wp-config-php-file
:param client: String
:param name: String
:param file: String
:return:
"""
return mode.run_command(
helper.get_command(client, 'sed') +
f' -n "s/define( *\'{name}\', *\'\([^\']*\)\'.*/\\1/p" {file}',
client,
True
).replace('\n', '')
def check_rsync_version():
"""
Check rsync version
:return:
"""
_raw_version = mode.run_command(
'rsync --version',
mode.Client.LOCAL,
True
)
_version = parse_version(_raw_version)
output.message(
output.Subject.LOCAL,
f'rsync version {_version}'
)
def get_database_setting(client, name, file):
"""
Get database setting try to regex from AdditionalConfiguration
sed -nE "s/'dbname'.*=>.*'(.*)'.*$/\1/p" /var/www/html/tests/files/www1/AdditionalConfiguration.php
:param client: String
:param name: String
:param file: String
:return:
"""
return mode.run_command(
helper.get_command(client, 'sed') +
f' -nE "s/\'{name}\'.*=>.*\'(.*)\'.*$/\\1/p" {file}',
client,
True
).replace('\n', '').strip()
def check_sshpass_version():
"""
Check sshpass version
:return:
"""
_raw_version = mode.run_command(
'sshpass -V',
mode.Client.LOCAL,
force_output=True,
allow_fail=True
)
_version = parse_version(_raw_version)
if _version:
output.message(
output.Subject.LOCAL,
f'sshpass version {_version}'
)
system.config['use_sshpass'] = True
return True
def count_tables(client, filepath):
"""
Count the reference string in the database dump file to get the count of all exported tables
:param client: String
:param filepath: String
:return:
"""
_reference = 'CREATE TABLE'
_count = mode.run_command(
f'{helper.get_command(client, "grep")} -ao "{_reference}" {filepath} | wc -l | xargs',
client,
True,
skip_dry_run=True
)
if _count:
output.message(
output.host_to_subject(client),
f'{int(_count)} table(s) exported'
)
def run_rsync_command(remote_client, origin_path, target_path, origin_ssh = '', target_ssh = ''):
"""
:param localpath:
:param remotepath:
:return:
"""
if origin_ssh != '':
origin_ssh += ':'
if target_ssh != '':
target_ssh += ':'
_output = mode.run_command(
f'{get_password_environment(remote_client)}rsync {get_options()} '
f'{get_authorization(remote_client)} '
f'{origin_ssh}{origin_path} {target_ssh}{target_path}',
mode.Client.LOCAL,
True
)
read_stats(_output)
