def check_options(): # check if ssh key authorization should be used if 'ssh_key' in config['host']['origin']: if os.path.isfile(config['host']['origin']['ssh_key']): option['use_origin_ssh_key'] = True else: sys.exit( output.message(output.get_subject().ERROR, 'SSH origin private key not found', False)) if 'ssh_key' in config['host']['target']: if os.path.isfile(config['host']['origin']['ssh_key']): option['use_target_ssh_key'] = True else: sys.exit( output.message(output.get_subject().ERROR, 'SSH target private key not found', False)) if 'dump_dir' in config['host']['origin']: option['default_origin_dump_dir'] = False if 'dump_dir' in config['host']['target']: option['default_target_dump_dir'] = False if 'check_dump' in config['host']: option['check_dump'] = config['host']['check_dump'] mode.check_sync_mode()
def remove_origin_database_dump(keep_compressed_file = False): output.message( output.get_subject().ORIGIN, 'Cleaning up', True ) _file_path = helper.get_origin_dump_dir() + database.origin_database_dump_file_name if mode.is_origin_remote(): sftp = ssh_client_origin.open_sftp() sftp.remove(_file_path) if not keep_compressed_file: sftp.remove(_file_path + '.tar.gz') sftp.close() else: os.remove(_file_path) if not keep_compressed_file: os.remove(_file_path + '.tar.gz') if keep_compressed_file: output.message( output.get_subject().INFO, 'Database dump file is saved to: ' + _file_path+ '.tar.gz', True )
def load_pip_modules(): import importlib import subprocess try: import pip except ImportError: sys.exit( output.message(output.get_subject().ERROR, 'Pip is not installed', False)) output.message(output.get_subject().LOCAL, 'Checking pip modules', True) package = 'paramiko' try: globals()[package] = importlib.import_module(package) except ImportError: subprocess.check_call( [sys.executable, "-m", "pip", "install", package]) sys.exit( output.message( output.get_subject().INFO, 'First install of additional pip modules completed. Please re-run the script.', False))
def print_footer(): if not system.option['keep_dump'] and not system.option['is_same_client']: output.message(output.get_subject().INFO, 'Successfully synchronized databases', True) else: output.message(output.get_subject().INFO, 'Successfully created database dump', True)
def get_database_configuration(client): system.config['db'] = {} # check framework type _base = '' if 'type' in system.config['host']: if system.config['host']['type'] == 'TYPO3': _base = framework.TYPO3 elif system.config['host']['type'] == 'Symfony': _base = framework.SYMFONY else: sys.exit( output.message(output.get_subject().ERROR, 'Framework type not supported', False)) else: _base = framework.TYPO3 if _base == framework.TYPO3: sys.path.append('./extension') from extension import typo3 _parser = typo3 elif _base == framework.SYMFONY: sys.path.append('./extension') from extension import symfony _parser = symfony if client == mode.get_clients().ORIGIN: output.message(output.get_subject().INFO, 'Sync base: ' + _base, True) load_parser_origin(_parser) else: load_parser_target(_parser)
def check_local_configuration(client): if os.path.isfile(system.config['host'][client]['path']) == False: sys.exit( output.message(output.get_subject().ERROR, 'Local database configuration not found', False)) system.config['db'] = {} _db_credentials = check_output( helper.get_command(client, 'grep') + ' -v "^#" ' + system.config['host'][client]['path'] + ' | ' + helper.get_command(client, 'grep') + ' DATABASE_URL', stderr=subprocess.STDOUT, shell=True) _db_config = parse_database_credentials(_db_credentials) if system.option['verbose']: if client == mode.get_clients().TARGET: _subject = output.get_subject().TARGET else: _subject = output.get_subject().ORIGIN output.message( _subject, output.get_bcolors().BLACK + helper.get_command(client, 'grep') + ' -v "^#" ' + system.config['host'][client]['path'] + ' | ' + helper.get_command(client, 'grep') + ' DATABASE_URL' + output.get_bcolors().ENDC, True) system.config['db'][client] = _db_config
def load_ssh_client(ssh): _ssh_client = system.paramiko.SSHClient() _ssh_client.set_missing_host_key_policy(system.paramiko.AutoAddPolicy()) if 'port' in system.config['host'][ssh]: _ssh_port = system.config['host'][ssh]['port'] else: _ssh_port = 22 if 'ssh_key' in system.config['host'][ssh]: try: _ssh_client.connect(hostname=system.config['host'][ssh]['host'], username=system.config['host'][ssh]['user'], key_filename=system.config['host'][ssh]['ssh_key'], port=_ssh_port, compress=True) except system.paramiko.ssh_exception.AuthenticationException: sys.exit( output.message( output.get_subject().ERROR, 'SSH authentification for ' + system.config['host'][ssh]['host'] + ' failed', False ) ) _authentication_method = 'using key' else: try: _ssh_client.connect(hostname=system.config['host'][ssh]['host'], username=system.config['host'][ssh]['user'], port=_ssh_port, password=system.option['ssh_password'][ssh], compress=True) except system.paramiko.ssh_exception.AuthenticationException: sys.exit( output.message( output.get_subject().ERROR, 'SSH authentification for ' + system.config['host'][ssh]['host'] + ' failed', False ) ) _authentication_method = 'using password' output.message( output.client_to_subject(ssh), 'Successfully connect to ' + system.config['host'][ssh]['user'] + '@' + system.config['host'][ssh][ 'host'] + ' ' + _authentication_method, True ) return _ssh_client
def run_ssh_command(command, ssh_client=ssh_client_origin): stdin, stdout, stderr = ssh_client.exec_command(command) exit_status = stdout.channel.recv_exit_status() err = stderr.read().decode() if err and 0 != exit_status: sys.exit(output.message(output.get_subject().ERROR, err, False)) elif err: output.message(output.get_subject().WARNING, err, True) return stdout
def get_host_configuration(): if os.path.isfile(default_local_host_file_path): with open(default_local_host_file_path, 'r') as read_file: config['host'] = json.load(read_file) output.message(output.get_subject().LOCAL, 'Loading host configuration', True) check_options() else: sys.exit( output.message(output.get_subject().ERROR, 'Local host configuration not found', False))
def upload_status(sent, size): sent_mb = round(float(sent) / 1024 / 1024, 1) size = round(float(size) / 1024 / 1024, 1) if (mode.get_sync_mode() == mode.get_sync_modes().PROXY): _subject = output.get_subject().LOCAL else: _subject = output.get_subject().ORIGIN + output.get_bcolors().BLACK + '[LOCAL]' + output.get_bcolors().ENDC sys.stdout.write( _subject + " Status: {0} MB of {1} MB uploaded". format(sent_mb, size, )) sys.stdout.write('\r')
def check_sync_mode(): global sync_mode if 'host' in system.config['host']['origin']: sync_mode = sync_modes.RECEIVER _description = output.get_bcolors( ).BLACK + '(REMOTE --> LOCAL)' + output.get_bcolors().ENDC if 'host' in system.config['host']['target']: sync_mode = sync_modes.SENDER _description = output.get_bcolors( ).BLACK + '(LOCAL --> REMOTE)' + output.get_bcolors().ENDC if 'host' in system.config['host']['origin'] and 'host' in system.config[ 'host']['target']: sync_mode = sync_modes.PROXY _description = output.get_bcolors( ).BLACK + '(REMOTE --> LOCAL --> REMOTE)' + output.get_bcolors().ENDC if not 'host' in system.config['host'][ 'origin'] and not 'host' in system.config['host']['target']: sync_mode = sync_modes.DUMP_LOCAL _description = output.get_bcolors( ).BLACK + '(LOCAL, NO TRANSFER/IMPORT)' + output.get_bcolors().ENDC system.option['is_same_client'] = True if 'host' in system.config['host']['origin'] and 'host' in system.config[ 'host']['target'] and system.config['host']['origin'][ 'host'] == system.config['host']['target']['host']: sync_mode = sync_modes.DUMP_REMOTE _description = output.get_bcolors( ).BLACK + '(REMOTE, NO TRANSFER/IMPORT)' + output.get_bcolors().ENDC system.option['is_same_client'] = True output.message(output.get_subject().INFO, 'Sync mode: ' + sync_mode + ' ' + _description, True)
def prepare_target_database_dump(): output.message(output.get_subject().TARGET, 'Extracting database dump', True) mode.run_command( helper.get_command('target', 'tar') + ' xzf ' + helper.get_target_dump_dir() + origin_database_dump_file_name + '.tar.gz -C ' + helper.get_target_dump_dir(), mode.get_clients().TARGET )
def download_status(sent, size): sent_mb = round(float(sent) / 1024 / 1024, 1) size = round(float(size) / 1024 / 1024, 1) sys.stdout.write( output.get_subject().ORIGIN + output.get_bcolors().BLACK + '[REMOTE]' + output.get_bcolors().ENDC + " Status: {0} MB of {1} MB downloaded". format(sent_mb, size, )) sys.stdout.write('\r')
def load_parser_origin(parser): # check origin output.message(output.get_subject().ORIGIN, 'Checking database configuration', True) if mode.is_origin_remote(): connect.load_ssh_client_origin() parser.check_remote_configuration(mode.get_clients().ORIGIN) else: parser.check_local_configuration(mode.get_clients().ORIGIN)
def load_parser_target(parser): # check target output.message(output.get_subject().TARGET, 'Checking database configuration', True) if mode.is_target_remote(): connect.load_ssh_client_target() parser.check_remote_configuration(mode.get_clients().TARGET) else: parser.check_local_configuration(mode.get_clients().TARGET)
def get_password(client): _password = getpass.getpass( output.message( output.get_subject().INFO, 'SSH password ' + config['host'][client]['user'] + '@' + config['host'][client]['host'] + ': ', False)) while _password.strip() is '': output.message(output.get_subject().WARNING, 'Password is empty. Please enter a valid password.', True) _password = getpass.getpass( output.message( output.get_subject().INFO, 'SSH password ' + config['host'][client]['user'] + '@' + config['host'][client]['host'] + ': ', False)) return _password
def get_composer_information(): if os.path.isfile( os.path.dirname(os.path.realpath(__file__)) + '/../composer.json'): with open( os.path.dirname(os.path.realpath(__file__)) + '/../composer.json', 'r') as read_file: return json.load(read_file) else: sys.exit( output.message(output.get_subject().ERROR, 'Local composer information not found', False))
def prepare_origin_database_dump(): output.message( output.get_subject().ORIGIN, 'Compressing database dump', True ) mode.run_command( helper.get_command('origin', 'tar') + ' cfvz ' + helper.get_origin_dump_dir() + origin_database_dump_file_name + '.tar.gz -C ' + helper.get_origin_dump_dir() + ' ' + origin_database_dump_file_name, mode.get_clients().ORIGIN )
def run_command(command, client): if client == clients.ORIGIN: if system.option['verbose']: output.message( output.get_subject().ORIGIN, output.get_bcolors().BLACK + command + output.get_bcolors().ENDC, True) if is_origin_remote(): return connect.run_ssh_command_origin(command) else: return os.system(command) elif client == clients.TARGET: if system.option['verbose']: output.message( output.get_subject().TARGET, output.get_bcolors().BLACK + command + output.get_bcolors().ENDC, True) if is_target_remote(): return connect.run_ssh_command_target(command) else: return os.system(command)
def remove_target_database_dump(): _file_path = helper.get_target_dump_dir() + database.origin_database_dump_file_name # # Move dump to specified directory # if system.option['keep_dump']: helper.create_local_temporary_data_dir() _keep_dump_path = system.default_local_sync_path + database.origin_database_dump_file_name mode.run_command( helper.get_command('target', 'cp') + ' ' + _file_path + ' ' + _keep_dump_path, mode.get_clients().TARGET ) output.message( output.get_subject().INFO, 'Database dump file is saved to: ' + _keep_dump_path, True ) # # Clean up # if not system.option['is_same_client']: output.message( output.get_subject().TARGET, 'Cleaning up', True ) if mode.is_target_remote(): sftp = ssh_client_target.open_sftp() sftp.remove(_file_path) sftp.remove(_file_path + '.tar.gz') sftp.close() else: if os.path.isfile(_file_path): os.remove(_file_path) if os.path.isfile(_file_path + '.tar.gz'): os.remove(_file_path + '.tar.gz')
def put_origin_database_dump(origin_path): sftp = ssh_client_target.open_sftp() if (mode.get_sync_mode() == mode.get_sync_modes().PROXY): _subject = output.get_subject().LOCAL else: _subject = output.get_subject().ORIGIN output.message( _subject, 'Uploading database dump', True ) # # ToDo: Download speed problems # https://github.com/paramiko/paramiko/issues/60 # sftp.put(origin_path + database.origin_database_dump_file_name + '.tar.gz', helper.get_target_dump_dir() + database.origin_database_dump_file_name + '.tar.gz', upload_status) sftp.close() print('')
def create_origin_database_dump(): generate_database_dump_filename() output.message( output.get_subject().ORIGIN, 'Creating database dump', True ) mode.run_command( helper.get_command('origin', 'mysqldump') + ' ' + generate_mysql_credentials('origin') + ' ' + system.config['db']['origin'][ 'dbname'] + ' ' + generate_ignore_database_tables() + ' > ' + helper.get_origin_dump_dir() + origin_database_dump_file_name, mode.get_clients().ORIGIN ) prepare_origin_database_dump()
def get_origin_database_dump(target_path): sftp = ssh_client_origin.open_sftp() output.message( output.get_subject().ORIGIN, 'Downloading database dump', True ) # # ToDo: Download speed problems # https://github.com/paramiko/paramiko/issues/60 # sftp.get(helper.get_origin_dump_dir() + database.origin_database_dump_file_name + '.tar.gz', target_path + database.origin_database_dump_file_name + '.tar.gz', download_status) sftp.close() print('') remove_origin_database_dump()
def check_args_options(args): global option global default_local_host_file_path global default_local_sync_path if not args.file is None: default_local_host_file_path = args.file if not args.verbose is None: option['verbose'] = True if not args.keepdump is None: default_local_sync_path = args.keepdump if default_local_sync_path[-1] != '/': default_local_sync_path += '/' option['keep_dump'] = True output.message(output.get_subject().INFO, '"Keep dump" option chosen', True)
def parse_database_credentials(_db_credentials): _db_credentials = str(_db_credentials).replace('\\n\'', '') # DATABASE_URL=mysql://db-user:1234@db-host:3306/db-name _db_credentials = re.findall(r"\/{2}(.+):(.+)@(.+):(\d+)\/(.+)", _db_credentials)[0] if len(_db_credentials) != 5: sys.exit( output.message(output.get_subject().ERROR, 'Mismatch of expected database credentials', False)) _db_config = { 'dbname': _db_credentials[4], 'host': _db_credentials[2], 'password': _db_credentials[1], 'port': _db_credentials[3], 'user': _db_credentials[0], } return _db_config
def import_database_dump(): if (not system.option['is_same_client']): prepare_target_database_dump() # @ToDo: Enable check_dump feature again # if system.option['check_dump']: # check_target_database_dump() if not system.option['keep_dump'] and not system.option['is_same_client']: output.message( output.get_subject().TARGET, 'Importing database dump', True ) mode.run_command( helper.get_command('target', 'mysql') + ' ' + generate_mysql_credentials('target') + ' ' + system.config['db']['target'][ 'dbname'] + ' < ' + helper.get_target_dump_dir() + origin_database_dump_file_name, mode.get_clients().TARGET )
def check_target_database_dump(): with open(system.default_local_sync_path + origin_database_dump_file_name) as f: lines = f.readlines() if "-- Dump completed on" not in lines[-1]: sys.exit(output.message(output.get_subject().ERROR, 'Dump was not fully transferred', False))
def remove_temporary_data_dir(): if os.path.exists(system.default_local_sync_path): shutil.rmtree(system.default_local_sync_path) output.message(output.get_subject().LOCAL, 'Cleaning up', True)