def check_options():
# check if ssh key authorization should be used
if 'ssh_key' in config['host']['origin']:
if os.path.isfile(config['host']['origin']['ssh_key']):
option['use_origin_ssh_key'] = True
else:
sys.exit(
output.message(output.get_subject().ERROR,
'SSH origin private key not found', False))
if 'ssh_key' in config['host']['target']:
if os.path.isfile(config['host']['origin']['ssh_key']):
option['use_target_ssh_key'] = True
else:
sys.exit(
output.message(output.get_subject().ERROR,
'SSH target private key not found', False))
if 'dump_dir' in config['host']['origin']:
option['default_origin_dump_dir'] = False
if 'dump_dir' in config['host']['target']:
option['default_target_dump_dir'] = False
if 'check_dump' in config['host']:
option['check_dump'] = config['host']['check_dump']
mode.check_sync_mode()
def remove_origin_database_dump(keep_compressed_file = False):
output.message(
output.get_subject().ORIGIN,
'Cleaning up',
True
)
_file_path = helper.get_origin_dump_dir() + database.origin_database_dump_file_name
if mode.is_origin_remote():
sftp = ssh_client_origin.open_sftp()
sftp.remove(_file_path)
if not keep_compressed_file:
sftp.remove(_file_path + '.tar.gz')
sftp.close()
else:
os.remove(_file_path)
if not keep_compressed_file:
os.remove(_file_path + '.tar.gz')
if keep_compressed_file:
output.message(
output.get_subject().INFO,
'Database dump file is saved to: ' + _file_path+ '.tar.gz',
True
)
def load_pip_modules():
import importlib
import subprocess
try:
import pip
except ImportError:
sys.exit(
output.message(output.get_subject().ERROR, 'Pip is not installed',
False))
output.message(output.get_subject().LOCAL, 'Checking pip modules', True)
package = 'paramiko'
try:
globals()[package] = importlib.import_module(package)
except ImportError:
subprocess.check_call(
[sys.executable, "-m", "pip", "install", package])
sys.exit(
output.message(
output.get_subject().INFO,
'First install of additional pip modules completed. Please re-run the script.',
False))
def print_footer():
if not system.option['keep_dump'] and not system.option['is_same_client']:
output.message(output.get_subject().INFO,
'Successfully synchronized databases', True)
else:
output.message(output.get_subject().INFO,
'Successfully created database dump', True)
def get_database_configuration(client):
system.config['db'] = {}
# check framework type
_base = ''
if 'type' in system.config['host']:
if system.config['host']['type'] == 'TYPO3':
_base = framework.TYPO3
elif system.config['host']['type'] == 'Symfony':
_base = framework.SYMFONY
else:
sys.exit(
output.message(output.get_subject().ERROR,
'Framework type not supported', False))
else:
_base = framework.TYPO3
if _base == framework.TYPO3:
sys.path.append('./extension')
from extension import typo3
_parser = typo3
elif _base == framework.SYMFONY:
sys.path.append('./extension')
from extension import symfony
_parser = symfony
if client == mode.get_clients().ORIGIN:
output.message(output.get_subject().INFO, 'Sync base: ' + _base, True)
load_parser_origin(_parser)
else:
load_parser_target(_parser)
def check_local_configuration(client):
if os.path.isfile(system.config['host'][client]['path']) == False:
sys.exit(
output.message(output.get_subject().ERROR,
'Local database configuration not found', False))
system.config['db'] = {}
_db_credentials = check_output(
helper.get_command(client, 'grep') + ' -v "^#" ' +
system.config['host'][client]['path'] + ' | ' +
helper.get_command(client, 'grep') + ' DATABASE_URL',
stderr=subprocess.STDOUT,
shell=True)
_db_config = parse_database_credentials(_db_credentials)
if system.option['verbose']:
if client == mode.get_clients().TARGET:
_subject = output.get_subject().TARGET
else:
_subject = output.get_subject().ORIGIN
output.message(
_subject,
output.get_bcolors().BLACK + helper.get_command(client, 'grep') +
' -v "^#" ' + system.config['host'][client]['path'] + ' | ' +
helper.get_command(client, 'grep') + ' DATABASE_URL' +
output.get_bcolors().ENDC, True)
system.config['db'][client] = _db_config
def load_ssh_client(ssh):
_ssh_client = system.paramiko.SSHClient()
_ssh_client.set_missing_host_key_policy(system.paramiko.AutoAddPolicy())
if 'port' in system.config['host'][ssh]:
_ssh_port = system.config['host'][ssh]['port']
else:
_ssh_port = 22
if 'ssh_key' in system.config['host'][ssh]:
try:
_ssh_client.connect(hostname=system.config['host'][ssh]['host'],
username=system.config['host'][ssh]['user'],
key_filename=system.config['host'][ssh]['ssh_key'],
port=_ssh_port,
compress=True)
except system.paramiko.ssh_exception.AuthenticationException:
sys.exit(
output.message(
output.get_subject().ERROR,
'SSH authentification for ' + system.config['host'][ssh]['host'] + ' failed',
False
)
)
_authentication_method = 'using key'
else:
try:
_ssh_client.connect(hostname=system.config['host'][ssh]['host'],
username=system.config['host'][ssh]['user'],
port=_ssh_port,
password=system.option['ssh_password'][ssh],
compress=True)
except system.paramiko.ssh_exception.AuthenticationException:
sys.exit(
output.message(
output.get_subject().ERROR,
'SSH authentification for ' + system.config['host'][ssh]['host'] + ' failed',
False
)
)
_authentication_method = 'using password'
output.message(
output.client_to_subject(ssh),
'Successfully connect to ' + system.config['host'][ssh]['user'] + '@' + system.config['host'][ssh][
'host'] + ' ' + _authentication_method,
True
)
return _ssh_client
def run_ssh_command(command, ssh_client=ssh_client_origin):
stdin, stdout, stderr = ssh_client.exec_command(command)
exit_status = stdout.channel.recv_exit_status()
err = stderr.read().decode()
if err and 0 != exit_status:
sys.exit(output.message(output.get_subject().ERROR, err, False))
elif err:
output.message(output.get_subject().WARNING, err, True)
return stdout
def get_host_configuration():
if os.path.isfile(default_local_host_file_path):
with open(default_local_host_file_path, 'r') as read_file:
config['host'] = json.load(read_file)
output.message(output.get_subject().LOCAL,
'Loading host configuration', True)
check_options()
else:
sys.exit(
output.message(output.get_subject().ERROR,
'Local host configuration not found', False))
def upload_status(sent, size):
sent_mb = round(float(sent) / 1024 / 1024, 1)
size = round(float(size) / 1024 / 1024, 1)
if (mode.get_sync_mode() == mode.get_sync_modes().PROXY):
_subject = output.get_subject().LOCAL
else:
_subject = output.get_subject().ORIGIN + output.get_bcolors().BLACK + '[LOCAL]' + output.get_bcolors().ENDC
sys.stdout.write(
_subject + " Status: {0} MB of {1} MB uploaded".
format(sent_mb, size, ))
sys.stdout.write('\r')
def check_sync_mode():
global sync_mode
if 'host' in system.config['host']['origin']:
sync_mode = sync_modes.RECEIVER
_description = output.get_bcolors(
).BLACK + '(REMOTE --> LOCAL)' + output.get_bcolors().ENDC
if 'host' in system.config['host']['target']:
sync_mode = sync_modes.SENDER
_description = output.get_bcolors(
).BLACK + '(LOCAL --> REMOTE)' + output.get_bcolors().ENDC
if 'host' in system.config['host']['origin'] and 'host' in system.config[
'host']['target']:
sync_mode = sync_modes.PROXY
_description = output.get_bcolors(
).BLACK + '(REMOTE --> LOCAL --> REMOTE)' + output.get_bcolors().ENDC
if not 'host' in system.config['host'][
'origin'] and not 'host' in system.config['host']['target']:
sync_mode = sync_modes.DUMP_LOCAL
_description = output.get_bcolors(
).BLACK + '(LOCAL, NO TRANSFER/IMPORT)' + output.get_bcolors().ENDC
system.option['is_same_client'] = True
if 'host' in system.config['host']['origin'] and 'host' in system.config[
'host']['target'] and system.config['host']['origin'][
'host'] == system.config['host']['target']['host']:
sync_mode = sync_modes.DUMP_REMOTE
_description = output.get_bcolors(
).BLACK + '(REMOTE, NO TRANSFER/IMPORT)' + output.get_bcolors().ENDC
system.option['is_same_client'] = True
output.message(output.get_subject().INFO,
'Sync mode: ' + sync_mode + ' ' + _description, True)
def prepare_target_database_dump():
output.message(output.get_subject().TARGET, 'Extracting database dump', True)
mode.run_command(
helper.get_command('target',
'tar') + ' xzf ' + helper.get_target_dump_dir() + origin_database_dump_file_name + '.tar.gz -C ' + helper.get_target_dump_dir(),
mode.get_clients().TARGET
)
def download_status(sent, size):
sent_mb = round(float(sent) / 1024 / 1024, 1)
size = round(float(size) / 1024 / 1024, 1)
sys.stdout.write(
output.get_subject().ORIGIN + output.get_bcolors().BLACK + '[REMOTE]' + output.get_bcolors().ENDC + " Status: {0} MB of {1} MB downloaded".
format(sent_mb, size, ))
sys.stdout.write('\r')
def load_parser_origin(parser):
# check origin
output.message(output.get_subject().ORIGIN,
'Checking database configuration', True)
if mode.is_origin_remote():
connect.load_ssh_client_origin()
parser.check_remote_configuration(mode.get_clients().ORIGIN)
else:
parser.check_local_configuration(mode.get_clients().ORIGIN)
def load_parser_target(parser):
# check target
output.message(output.get_subject().TARGET,
'Checking database configuration', True)
if mode.is_target_remote():
connect.load_ssh_client_target()
parser.check_remote_configuration(mode.get_clients().TARGET)
else:
parser.check_local_configuration(mode.get_clients().TARGET)
def get_password(client):
_password = getpass.getpass(
output.message(
output.get_subject().INFO,
'SSH password ' + config['host'][client]['user'] + '@' +
config['host'][client]['host'] + ': ', False))
while _password.strip() is '':
output.message(output.get_subject().WARNING,
'Password is empty. Please enter a valid password.',
True)
_password = getpass.getpass(
output.message(
output.get_subject().INFO,
'SSH password ' + config['host'][client]['user'] + '@' +
config['host'][client]['host'] + ': ', False))
return _password
def get_composer_information():
if os.path.isfile(
os.path.dirname(os.path.realpath(__file__)) + '/../composer.json'):
with open(
os.path.dirname(os.path.realpath(__file__)) +
'/../composer.json', 'r') as read_file:
return json.load(read_file)
else:
sys.exit(
output.message(output.get_subject().ERROR,
'Local composer information not found', False))
def prepare_origin_database_dump():
output.message(
output.get_subject().ORIGIN,
'Compressing database dump',
True
)
mode.run_command(
helper.get_command('origin',
'tar') + ' cfvz ' + helper.get_origin_dump_dir() + origin_database_dump_file_name + '.tar.gz -C ' + helper.get_origin_dump_dir() + ' ' + origin_database_dump_file_name,
mode.get_clients().ORIGIN
)
def run_command(command, client):
if client == clients.ORIGIN:
if system.option['verbose']:
output.message(
output.get_subject().ORIGIN,
output.get_bcolors().BLACK + command +
output.get_bcolors().ENDC, True)
if is_origin_remote():
return connect.run_ssh_command_origin(command)
else:
return os.system(command)
elif client == clients.TARGET:
if system.option['verbose']:
output.message(
output.get_subject().TARGET,
output.get_bcolors().BLACK + command +
output.get_bcolors().ENDC, True)
if is_target_remote():
return connect.run_ssh_command_target(command)
else:
return os.system(command)
def remove_target_database_dump():
_file_path = helper.get_target_dump_dir() + database.origin_database_dump_file_name
#
# Move dump to specified directory
#
if system.option['keep_dump']:
helper.create_local_temporary_data_dir()
_keep_dump_path = system.default_local_sync_path + database.origin_database_dump_file_name
mode.run_command(
helper.get_command('target',
'cp') + ' ' + _file_path + ' ' + _keep_dump_path,
mode.get_clients().TARGET
)
output.message(
output.get_subject().INFO,
'Database dump file is saved to: ' + _keep_dump_path,
True
)
#
# Clean up
#
if not system.option['is_same_client']:
output.message(
output.get_subject().TARGET,
'Cleaning up',
True
)
if mode.is_target_remote():
sftp = ssh_client_target.open_sftp()
sftp.remove(_file_path)
sftp.remove(_file_path + '.tar.gz')
sftp.close()
else:
if os.path.isfile(_file_path):
os.remove(_file_path)
if os.path.isfile(_file_path + '.tar.gz'):
os.remove(_file_path + '.tar.gz')
def put_origin_database_dump(origin_path):
sftp = ssh_client_target.open_sftp()
if (mode.get_sync_mode() == mode.get_sync_modes().PROXY):
_subject = output.get_subject().LOCAL
else:
_subject = output.get_subject().ORIGIN
output.message(
_subject,
'Uploading database dump',
True
)
#
# ToDo: Download speed problems
# https://github.com/paramiko/paramiko/issues/60
#
sftp.put(origin_path + database.origin_database_dump_file_name + '.tar.gz',
helper.get_target_dump_dir() + database.origin_database_dump_file_name + '.tar.gz', upload_status)
sftp.close()
print('')
def create_origin_database_dump():
generate_database_dump_filename()
output.message(
output.get_subject().ORIGIN,
'Creating database dump',
True
)
mode.run_command(
helper.get_command('origin', 'mysqldump') + ' ' + generate_mysql_credentials('origin') + ' ' +
system.config['db']['origin'][
'dbname'] + ' ' + generate_ignore_database_tables() + ' > ' + helper.get_origin_dump_dir() + origin_database_dump_file_name,
mode.get_clients().ORIGIN
)
prepare_origin_database_dump()
def get_origin_database_dump(target_path):
sftp = ssh_client_origin.open_sftp()
output.message(
output.get_subject().ORIGIN,
'Downloading database dump',
True
)
#
# ToDo: Download speed problems
# https://github.com/paramiko/paramiko/issues/60
#
sftp.get(helper.get_origin_dump_dir() + database.origin_database_dump_file_name + '.tar.gz',
target_path + database.origin_database_dump_file_name + '.tar.gz', download_status)
sftp.close()
print('')
remove_origin_database_dump()
def check_args_options(args):
global option
global default_local_host_file_path
global default_local_sync_path
if not args.file is None:
default_local_host_file_path = args.file
if not args.verbose is None:
option['verbose'] = True
if not args.keepdump is None:
default_local_sync_path = args.keepdump
if default_local_sync_path[-1] != '/':
default_local_sync_path += '/'
option['keep_dump'] = True
output.message(output.get_subject().INFO, '"Keep dump" option chosen',
True)
def parse_database_credentials(_db_credentials):
_db_credentials = str(_db_credentials).replace('\\n\'', '')
# DATABASE_URL=mysql://db-user:1234@db-host:3306/db-name
_db_credentials = re.findall(r"\/{2}(.+):(.+)@(.+):(\d+)\/(.+)",
_db_credentials)[0]
if len(_db_credentials) != 5:
sys.exit(
output.message(output.get_subject().ERROR,
'Mismatch of expected database credentials', False))
_db_config = {
'dbname': _db_credentials[4],
'host': _db_credentials[2],
'password': _db_credentials[1],
'port': _db_credentials[3],
'user': _db_credentials[0],
}
return _db_config
def import_database_dump():
if (not system.option['is_same_client']):
prepare_target_database_dump()
# @ToDo: Enable check_dump feature again
# if system.option['check_dump']:
# check_target_database_dump()
if not system.option['keep_dump'] and not system.option['is_same_client']:
output.message(
output.get_subject().TARGET,
'Importing database dump',
True
)
mode.run_command(
helper.get_command('target', 'mysql') + ' ' + generate_mysql_credentials('target') + ' ' +
system.config['db']['target'][
'dbname'] + ' < ' + helper.get_target_dump_dir() + origin_database_dump_file_name,
mode.get_clients().TARGET
)
def check_target_database_dump():
with open(system.default_local_sync_path + origin_database_dump_file_name) as f:
lines = f.readlines()
if "-- Dump completed on" not in lines[-1]:
sys.exit(output.message(output.get_subject().ERROR, 'Dump was not fully transferred', False))
def remove_temporary_data_dir():
if os.path.exists(system.default_local_sync_path):
shutil.rmtree(system.default_local_sync_path)
output.message(output.get_subject().LOCAL, 'Cleaning up', True)
