def check_database_dump(client, filepath):
"""
Checking the last line of the dump file if it contains "-- Dump completed on"
:param client: String
:param filepath: String
:return:
"""
if system.config['check_dump']:
_line = mode.run_command(
helper.get_command(client, 'tail') + ' -n 1 ' + filepath,
client,
True,
skip_dry_run=True
)
if not _line:
return
if "-- Dump completed on" not in _line:
sys.exit(
output.message(
output.Subject.ERROR,
'Dump file is corrupted',
do_print=False
)
)
else:
output.message(
output.host_to_subject(client),
'Dump file is valid',
verbose_only=True
)
def check_configuration(client):
"""
Checking Drupal database configuration with Drush
:param client: String
:return:
"""
_path = system.config[client]['path']
# Check Drush version
_raw_version = mode.run_command(
f'{helper.get_command(client, "drush")} status --fields=drush-version --format=string '
f'-r {_path}',
client,
True
)
output.message(
output.host_to_subject(client),
f'Drush version: {_raw_version}',
True
)
stdout = mode.run_command(
f'{helper.get_command(client, "drush")} core-status --pipe '
f'--fields=db-hostname,db-username,db-password,db-name,db-port '
f'-r {_path}',
client,
True
)
_db_config = parse_database_credentials(json.loads(stdout))
system.config[client]['db'] = _db_config
def run_script(client=None, script='before'):
"""
Executing script command
:param client: String
:param script: String
:return:
"""
if client is None:
_config = system.config
_subject = output.Subject.LOCAL
client = mode.Client.LOCAL
else:
_config = system.config[client]
_subject = output.host_to_subject(client)
if not 'scripts' in _config:
return
if f'{script}' in _config['scripts']:
output.message(
_subject,
f'Running script {client}',
True
)
mode.run_command(
_config['scripts'][script],
client
)
def load_parser(client, parser):
"""
Loading parser and checking database configuration
:param client:
:param parser:
:return:
"""
_path = system.config[client]['path']
output.message(
output.host_to_subject(client),
f'Checking database configuration {output.CliFormat.BLACK}{_path}{output.CliFormat.ENDC}',
True
)
if client == mode.Client.ORIGIN:
if mode.is_origin_remote():
remote_client.load_ssh_client_origin()
else:
helper.run_script(client, 'before')
else:
if mode.is_target_remote():
remote_client.load_ssh_client_target()
else:
helper.run_script(client, 'before')
# Check only if database configuration is a file
if not helper.check_file_exists(client, _path) and _path[-1] != '/':
sys.exit(
output.message(
output.Subject.ERROR,
f'Database configuration for {client} not found: {_path}',
False
)
)
parser.check_configuration(client)
def validate_database_credentials(client):
"""
Validate the parsed database credentials
:param client: String
:return:
"""
output.message(
output.host_to_subject(client),
'Validating database credentials',
True
)
_db_credential_keys = ['name', 'host', 'password', 'user']
for _key in _db_credential_keys:
if _key not in system.config[client]['db']:
sys.exit(
output.message(
output.Subject.ERROR,
f'Missing database credential "{_key}" for {client} client',
False
)
)
if system.config[client]['db'][_key] is None or system.config[client]['db'][_key] == '':
sys.exit(
output.message(
output.Subject.ERROR,
f'Missing database credential "{_key}" for {client} client',
False
)
)
else:
output.message(
output.host_to_subject(client),
f'Database credential "{_key}" valid',
verbose_only=True
)
def count_tables(client, filepath):
"""
Count the reference string in the database dump file to get the count of all exported tables
:param client: String
:param filepath: String
:return:
"""
_reference = 'CREATE TABLE'
_count = mode.run_command(
f'{helper.get_command(client, "grep")} -ao "{_reference}" {filepath} | wc -l | xargs',
client,
True,
skip_dry_run=True
)
if _count:
output.message(
output.host_to_subject(client),
f'{int(_count)} table(s) exported'
)
def get_database_version(client):
"""
Check the database version and distinguish between mysql and mariadb
:param client:
:return: Tuple<String,String>
"""
_database_system = None
_version_number = None
try:
_database_version = run_database_command(client, 'SELECT VERSION();').splitlines()[1]
_database_system = DatabaseSystem.MYSQL
_version_number = re.search('(\d+\.)?(\d+\.)?(\*|\d+)', _database_version).group()
if DatabaseSystem.MARIADB.lower() in _database_version.lower():
_database_system = DatabaseSystem.MARIADB
output.message(
output.host_to_subject(client),
f'Database version: {_database_system} v{_version_number}',
True
)
finally:
return _database_system, _version_number
def run_command(command, client, force_output=False, allow_fail=False, skip_dry_run=False):
"""
Run command depending on the given client
:param command: String
:param client: String
:param force_output: Boolean
:param allow_fail: Boolean
:param skip_dry_run: Boolean
:return:
"""
if system.config['verbose']:
output.message(
output.host_to_subject(client),
output.CliFormat.BLACK + command + output.CliFormat.ENDC,
debug=True
)
if system.config['dry_run'] and skip_dry_run:
return
if is_remote(client):
if force_output:
return ''.join(remote_system.run_ssh_command_by_client(client, command).readlines()).strip()
else:
return remote_system.run_ssh_command_by_client(client, command)
else:
res = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
# Wait for the process end and print error in case of failure
out, err = res.communicate()
if res.wait() != 0 and err.decode() != '' and not allow_fail:
helper.run_script(script='error')
sys.exit(output.message(output.Subject.ERROR, err.decode(), False))
if force_output:
return out.decode().strip()
def load_ssh_client(ssh):
"""
Initializing the given ssh client
:param ssh: String
:return:
"""
_host_name = helper.get_ssh_host_name(ssh, True)
_ssh_client = paramiko.SSHClient()
_ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
_ssh_port = system.config[ssh]['port'] if 'port' in system.config[ssh] else 22
_ssh_key = None
_ssh_password = None
# Check authentication
if 'ssh_key' in system.config[ssh]:
_authentication_method = f'{output.CliFormat.BLACK} - ' \
f'(authentication: key){output.CliFormat.ENDC}'
_ssh_key = system.config[ssh]['ssh_key']
elif 'password' in system.config[ssh]:
_authentication_method = f'{output.CliFormat.BLACK} - ' \
f'authentication: password){output.CliFormat.ENDC}'
_ssh_password = system.config[ssh]['password']
elif 'ssh_agent' in system.config:
_authentication_method = f'{output.CliFormat.BLACK} - ' \
f'(authentication: key){output.CliFormat.ENDC}'
else:
sys.exit(
output.message(
output.Subject.ERROR,
'Missing SSH authentication. Neither ssh key nor ssh password given.',
False
)
)
# Try to connect to remote client via paramiko
try:
_ssh_client.connect(hostname=system.config[ssh]['host'],
username=system.config[ssh]['user'],
key_filename=_ssh_key,
password=_ssh_password,
port=_ssh_port,
compress=True,
timeout=default_timeout,
sock=get_jump_host_channel(ssh))
#
# Workaround for long-lasting requests
# https://stackoverflow.com/questions/50009688/python-paramiko-ssh-session-not-active-after-being-idle-for-many-hours
#
_ssh_client.get_transport().set_keepalive(60)
except paramiko.ssh_exception.AuthenticationException:
sys.exit(
output.message(
output.Subject.ERROR,
f'SSH authentication for {_host_name} failed',
False
)
)
output.message(
output.host_to_subject(ssh),
f'Initialize remote SSH connection {_host_name}{_authentication_method}',
True
)
return _ssh_client
def get_jump_host_channel(client):
"""
Provide an optional transport channel for a SSH jump host client
https://gist.github.com/tintoy/443c42ea3865680cd624039c4bb46219
:param client:
:return:
"""
_jump_host_channel = None
if 'jump_host' in system.config[client]:
# prepare jump host config
_jump_host_client = paramiko.SSHClient()
_jump_host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
_jump_host_host = system.config[client]['jump_host']['host']
_jump_host_user = system.config[client]['jump_host']['user'] if 'user' in system.config[client]['jump_host'] else system.config[client]['user']
if 'ssh_key' in system.config[client]['jump_host']:
_jump_host_ssh_key = system.config[client]['jump_host']['ssh_key']
elif 'ssh_key' in system.config[client]:
_jump_host_ssh_key = system.config[client]['ssh_key']
else:
_jump_host_ssh_key = None
if 'port' in system.config[client]['jump_host']:
_jump_host_port = system.config[client]['jump_host']['port']
elif 'port' in system.config[client]:
_jump_host_port = system.config[client]['port']
else:
_jump_host_port = 22
# connect to the jump host
_jump_host_client.connect(
hostname=_jump_host_host,
username=_jump_host_user,
key_filename=_jump_host_ssh_key,
password=system.config[client]['jump_host']['password'] if 'password' in system.config[client]['jump_host'] else None,
port=_jump_host_port,
compress=True,
timeout=default_timeout
)
global additional_ssh_clients
additional_ssh_clients.append(_jump_host_client)
# open the necessary channel
_jump_host_transport = _jump_host_client.get_transport()
_jump_host_channel = _jump_host_transport.open_channel(
'direct-tcpip',
dest_addr=(system.config[client]['host'], 22),
src_addr=(system.config[client]['jump_host']['private'] if 'private' in system.config[client]['jump_host'] else system.config[client]['jump_host']['host'], 22)
)
# print information
_destination_client = helper.get_ssh_host_name(client, minimal=True)
_jump_host_name = system.config[client]['jump_host']['name'] if 'name' in system.config[client]['jump_host'] else _jump_host_host
output.message(
output.host_to_subject(client),
f'Initialize remote SSH jump host {output.CliFormat.BLACK}local ➔ {output.CliFormat.BOLD}{_jump_host_name}{output.CliFormat.ENDC}{output.CliFormat.BLACK} ➔ {_destination_client}{output.CliFormat.ENDC}',
True
)
return _jump_host_channel