def clear_database(client): """ Clearing the database by dropping all tables https://www.techawaken.com/drop-tables-mysql-database/ { mysql -hHOSTNAME -uUSERNAME -pPASSWORD -Nse 'show tables' DB_NAME; } | ( while read table; do if [ -z ${i+x} ]; then echo 'SET FOREIGN_KEY_CHECKS = 0;'; fi; i=1; echo "drop table \`$table\`;"; done; echo 'SET FOREIGN_KEY_CHECKS = 1;' ) | awk '{print}' ORS=' ' | mysql -hHOSTNAME -uUSERNAME -pPASSWORD DB_NAME; :param client: String :return: """ mode.run_command( '{ ' + helper.get_command(client, 'mysql') + ' ' + database_utility.generate_mysql_credentials(client) + ' -Nse \'show tables\' \'' + system.config[client]['db']['name'] + '\'; }' + ' | ( while read table; do if [ -z ${i+x} ]; then echo \'SET FOREIGN_KEY_CHECKS = 0;\'; fi; i=1; ' + 'echo "drop table \\`$table\\`;"; done; echo \'SET FOREIGN_KEY_CHECKS = 1;\' ) | awk \'{print}\' ORS=' ' | ' + helper.get_command(client, 'mysql') + ' ' + database_utility.generate_mysql_credentials(client) + ' ' + system.config[client]['db']['name'], client, skip_dry_run=True )
def clean_up_dump_dir(client, path, num=5): """ Clean up the dump directory from old dump files (only affect .sql and .tar.gz files) :param client: :param path: :param num: :return: """ # Distinguish stat command on os system (Darwin|Linux) if check_os(client).strip() == 'Darwin': _command = get_command(client, 'stat') + ' -f "%Sm %N" ' + path + ' | ' + get_command( client, 'sort') + ' -rn | ' + get_command( client, 'grep') + ' -E ".tar.gz|.sql"' else: _command = get_command(client, 'stat') + ' -c "%y %n" ' + path + ' | ' + \ get_command(client,'sort') + ' -rn | ' + get_command(client, 'grep') + \ ' -E ".tar.gz|.sql"' # List files in directory sorted by change date _files = mode.run_command( _command, client, True ).splitlines() for i in range(len(_files)): _filename = _files[i].rsplit(' ', 1)[-1] # Remove oldest files chosen by keep_dumps count if not i < num: mode.run_command( 'rm ' + _filename, client )
def run_script(client=None, script='before'): """ Executing script command :param client: String :param script: String :return: """ if client is None: _config = system.config _subject = output.Subject.LOCAL client = mode.Client.LOCAL else: _config = system.config[client] _subject = output.host_to_subject(client) if not 'scripts' in _config: return if f'{script}' in _config['scripts']: output.message( _subject, f'Running script {client}', True ) mode.run_command( _config['scripts'][script], client )
def check_configuration(client): """ Checking Drupal database configuration with Drush :param client: String :return: """ _path = system.config[client]['path'] # Check Drush version _raw_version = mode.run_command( f'{helper.get_command(client, "drush")} status --fields=drush-version --format=string ' f'-r {_path}', client, True ) output.message( output.host_to_subject(client), f'Drush version: {_raw_version}', True ) stdout = mode.run_command( f'{helper.get_command(client, "drush")} core-status --pipe ' f'--fields=db-hostname,db-username,db-password,db-name,db-port ' f'-r {_path}', client, True ) _db_config = parse_database_credentials(json.loads(stdout)) system.config[client]['db'] = _db_config
def check_and_create_dump_dir(client, path): """ Check if a path exists on the client system and creates the given path if necessary :param client: :param path: :return: """ mode.run_command( '[ ! -d "' + path + '" ] && mkdir -p "' + path + '"', client )
def prepare_target_database_dump(): """ Preparing the target database dump by the unpacked .tar.gz file :return: """ output.message(output.Subject.TARGET, 'Extracting database dump', True) mode.run_command( helper.get_command('target', 'tar') + ' xzf ' + helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name + '.tar.gz -C ' + helper.get_dump_dir(mode.Client.TARGET) + ' > /dev/null', mode.Client.TARGET, skip_dry_run=True )
def import_database_dump_file(client, filepath): """ Import a database dump file :param client: String :param filepath: String :return: """ if helper.check_file_exists(client, filepath): mode.run_command( helper.get_command(client, 'mysql') + ' ' + database_utility.generate_mysql_credentials(client) + ' \'' + system.config[client]['db']['name'] + '\' < ' + filepath, client, skip_dry_run=True )
def remove_target_database_dump(): """ Removing the target database dump files :return: """ _file_path = helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name # # Move dump to specified directory # if system.config['keep_dump']: helper.create_local_temporary_data_dir() _keep_dump_path = system.default_local_sync_path + database_utility.database_dump_file_name mode.run_command( helper.get_command('target', 'cp') + ' ' + _file_path + ' ' + _keep_dump_path, mode.Client.TARGET ) output.message( output.Subject.INFO, f'Database dump file is saved to: {_keep_dump_path}', True, True ) # # Clean up # if not mode.is_dump() and not mode.is_import(): output.message( output.Subject.TARGET, 'Cleaning up', True ) if system.config['dry_run']: return if mode.is_target_remote(): sftp = remote_client.ssh_client_target.open_sftp() sftp.remove(_file_path) sftp.remove(f'{_file_path}.tar.gz') sftp.close() else: if os.path.isfile(_file_path): os.remove(_file_path) if os.path.isfile(f'{_file_path}.tar.gz'): os.remove(f'{_file_path}.tar.gz')
def check_configuration(client): """ Checking remote TYPO3 database configuration :param client: String :return: """ _path = system.config[client]['path'] if 'LocalConfiguration' in _path: stdout = mode.run_command( helper.get_command(client, 'php') + ' -r "echo json_encode(include \'' + system.config[client][ 'path'] + '\');"', client, True ) _db_config = parse_database_credentials(json.loads(stdout)['DB']) else: # Try to parse settings from AdditionalConfiguration.php file _db_config = { 'name': get_database_setting(client, 'dbname', system.config[client]['path']), 'host': get_database_setting(client, 'host', system.config[client]['path']), 'password': get_database_setting(client, 'password', system.config[client]['path']), 'port': get_database_setting(client, 'port', system.config[client]['path']) if get_database_setting(client, 'port', system.config[client]['path']) != '' else 3306, 'user': get_database_setting(client, 'user', system.config[client]['path']), } system.config[client]['db'] = _db_config
def check_configuration(client): """ Checking remote Symfony database configuration :param client: String :return: """ _path = system.config[client]['path'] # Check for symfony 2.8 if 'parameters.yml' in _path: _db_config = { 'name': get_database_parameter(client, 'database_name', _path), 'host': get_database_parameter(client, 'database_host', _path), 'password': get_database_parameter(client, 'database_password', _path), 'port': get_database_parameter(client, 'database_port', _path), 'user': get_database_parameter(client, 'database_user', _path), } # Using for symfony >=3.4 else: stdout = mode.run_command( helper.get_command(client, 'grep') + ' -v "^#" ' + system.config[client][ 'path'] + ' | ' + helper.get_command(client, 'grep') + ' DATABASE_URL', client, True ) _db_config = parse_database_credentials(stdout) system.config[client]['db'] = _db_config
def check_database_dump(client, filepath): """ Checking the last line of the dump file if it contains "-- Dump completed on" :param client: String :param filepath: String :return: """ if system.config['check_dump']: _line = mode.run_command( helper.get_command(client, 'tail') + ' -n 1 ' + filepath, client, True, skip_dry_run=True ) if not _line: return if "-- Dump completed on" not in _line: sys.exit( output.message( output.Subject.ERROR, 'Dump file is corrupted', do_print=False ) ) else: output.message( output.host_to_subject(client), 'Dump file is valid', verbose_only=True )
def check_file_exists(client, path): """ Check if a file exists :param client: String :param path: String file path :return: Boolean """ return mode.run_command(f'[ -f {path} ] && echo "1"', client, True) == '1'
def create_origin_database_dump(): """ Creating the origin database dump file :return: """ if not mode.is_import(): parser.get_database_configuration(mode.Client.ORIGIN) database_utility.generate_database_dump_filename() helper.check_and_create_dump_dir(mode.Client.ORIGIN, helper.get_dump_dir(mode.Client.ORIGIN)) _dump_file_path = helper.get_dump_dir( mode.Client.ORIGIN) + database_utility.database_dump_file_name _database_version = database_utility.get_database_version(mode.Client.ORIGIN) output.message( output.Subject.ORIGIN, f'Creating database dump {output.CliFormat.BLACK}{_dump_file_path}{output.CliFormat.ENDC}', True ) _mysqldump_options = '--no-tablespaces ' # Remove --no-tablespaces option for mysql < 5.6 # @ToDo: Better option handling if not _database_version is None: if _database_version[0] == database_utility.DatabaseSystem.MYSQL and \ semantic_version.Version(_database_version[1]) < semantic_version.Version('5.6.0'): _mysqldump_options = '' # Run mysql dump command, e.g. # mysqldump --no-tablespaces -u'db' -p'db' -h'db1' -P'3306' 'db' > /tmp/_db_08-10-2021_07-00.sql mode.run_command( helper.get_command(mode.Client.ORIGIN, 'mysqldump') + ' ' + _mysqldump_options + database_utility.generate_mysql_credentials(mode.Client.ORIGIN) + ' \'' + system.config[mode.Client.ORIGIN]['db']['name'] + '\' ' + database_utility.generate_ignore_database_tables() + database_utility.get_database_tables() + ' > ' + _dump_file_path, mode.Client.ORIGIN, skip_dry_run=True ) database_utility.check_database_dump(mode.Client.ORIGIN, _dump_file_path) database_utility.count_tables(mode.Client.ORIGIN, _dump_file_path) prepare_origin_database_dump()
def prepare_origin_database_dump(): """ Preparing the origin database dump file by compressing them as .tar.gz :return: """ output.message( output.Subject.ORIGIN, 'Compressing database dump', True ) mode.run_command( helper.get_command(mode.Client.ORIGIN, 'tar') + ' cfvz ' + helper.get_dump_dir( mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz -C ' + helper.get_dump_dir(mode.Client.ORIGIN) + ' ' + database_utility.database_dump_file_name + ' > /dev/null', mode.Client.ORIGIN, skip_dry_run=True )
def synchronize(origin_path, target_path, exclude, client=mode.Client.LOCAL, pseudo_client=None, force_remote=False): """ Using rsync command to synchronize files between systems :param origin_path: String :param target_path: String :param exclude: List :param client: String :param pseudo_client: String Client, which will be forced as remote client. Necessary for proxy transfer. :param force_remote: Boolean :return: """ _remote_client = None if force_remote: remote_client.load_ssh_client_origin() _origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} ' _target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} ' elif mode.is_remote(mode.Client.ORIGIN) and pseudo_client != mode.Client.TARGET: _remote_client = mode.Client.ORIGIN _origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} ' _target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} ' elif mode.is_remote(mode.Client.TARGET) and pseudo_client != mode.Client.ORIGIN: _remote_client = mode.Client.TARGET _origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} ' _target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[REMOTE]{output.CliFormat.ENDC} ' elif not mode.is_remote(mode.Client.TARGET) and not mode.is_remote(mode.Client.ORIGIN): _origin_subject = f'{output.Subject.ORIGIN}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} ' _target_subject = f'{output.Subject.TARGET}{output.CliFormat.BLACK}[LOCAL]{output.CliFormat.ENDC} ' _origin_name = helper.get_ssh_host_name(mode.Client.ORIGIN, True) if _remote_client == mode.Client.ORIGIN else '' _target_name = helper.get_ssh_host_name(mode.Client.TARGET, True) if _remote_client == mode.Client.TARGET else '' if not system.config['mute']: print( f'{_origin_subject}' f'{_origin_name}' f'{output.CliFormat.BLACK}{origin_path}{output.CliFormat.ENDC}' ) print( f'{_target_subject}' f'{_target_name}' f'{output.CliFormat.BLACK}{target_path}{output.CliFormat.ENDC}' ) _origin_user_host = utility.get_host(mode.Client.ORIGIN) if _remote_client == mode.Client.ORIGIN else '' _target_user_host = utility.get_host(mode.Client.TARGET) if _remote_client == mode.Client.TARGET else '' _output = mode.run_command( f'{utility.get_password_environment(_remote_client)}rsync {utility.get_options()} ' f'{utility.get_authorization(_remote_client)} {utility.get_excludes(exclude)}' f'{_origin_user_host}{origin_path} {_target_user_host}{target_path}', client, True ) utility.read_stats(_output)
def check_os(client): """ Check which system is running (Linux|Darwin) :param client: :return: """ return mode.run_command( get_command(client, 'uname') + ' -s', client, True )
def get_database_parameter(client, name, file): """ Parsing a single database variable from the parameters.yml file hhttps://unix.stackexchange.com/questions/84922/extract-a-part-of-one-line-from-a-file-with-sed :param client: String :param name: String :param file: String :return: """ return mode.run_command( helper.get_command(client, 'sed') + f' -n -e \'/{name}/ s/.*\\: *//p\' {file}', client, True ).replace('\n', '')
def get_database_parameter(client, name, file): """ Parsing a single database variable from the .env file https://gist.github.com/judy2k/7656bfe3b322d669ef75364a46327836 :param client: String :param name: String :param file: String :return: """ return mode.run_command( helper.get_command(client, 'grep') + f' {name} {file} | cut -d \'=\' -f2', client, True ).replace('\n', '')
def run_database_command(client, command, force_database_name=False): """ Run a database command using the "mysql -e" command :param client: String :param command: String database command :param force_database_name: Bool forces the database name :return: """ _database_name = ' ' + system.config[client]['db']['name'] if force_database_name else '' return mode.run_command( helper.get_command(client, 'mysql') + ' ' + generate_mysql_credentials( client) + _database_name + ' -e "' + command + '"', client, True)
def get_database_setting(client, name, file): """ Parsing a single database variable from the wp-config.php file https://stackoverflow.com/questions/63493645/extract-database-name-from-a-wp-config-php-file :param client: String :param name: String :param file: String :return: """ return mode.run_command( helper.get_command(client, 'sed') + f' -n "s/define( *\'{name}\', *\'\([^\']*\)\'.*/\\1/p" {file}', client, True ).replace('\n', '')
def check_rsync_version(): """ Check rsync version :return: """ _raw_version = mode.run_command( 'rsync --version', mode.Client.LOCAL, True ) _version = parse_version(_raw_version) output.message( output.Subject.LOCAL, f'rsync version {_version}' )
def get_database_setting(client, name, file): """ Get database setting try to regex from AdditionalConfiguration sed -nE "s/'dbname'.*=>.*'(.*)'.*$/\1/p" /var/www/html/tests/files/www1/AdditionalConfiguration.php :param client: String :param name: String :param file: String :return: """ return mode.run_command( helper.get_command(client, 'sed') + f' -nE "s/\'{name}\'.*=>.*\'(.*)\'.*$/\\1/p" {file}', client, True ).replace('\n', '').strip()
def check_sshpass_version(): """ Check sshpass version :return: """ _raw_version = mode.run_command( 'sshpass -V', mode.Client.LOCAL, force_output=True, allow_fail=True ) _version = parse_version(_raw_version) if _version: output.message( output.Subject.LOCAL, f'sshpass version {_version}' ) system.config['use_sshpass'] = True return True
def count_tables(client, filepath): """ Count the reference string in the database dump file to get the count of all exported tables :param client: String :param filepath: String :return: """ _reference = 'CREATE TABLE' _count = mode.run_command( f'{helper.get_command(client, "grep")} -ao "{_reference}" {filepath} | wc -l | xargs', client, True, skip_dry_run=True ) if _count: output.message( output.host_to_subject(client), f'{int(_count)} table(s) exported' )
def run_rsync_command(remote_client, origin_path, target_path, origin_ssh = '', target_ssh = ''): """ :param localpath: :param remotepath: :return: """ if origin_ssh != '': origin_ssh += ':' if target_ssh != '': target_ssh += ':' _output = mode.run_command( f'{get_password_environment(remote_client)}rsync {get_options()} ' f'{get_authorization(remote_client)} ' f'{origin_ssh}{origin_path} {target_ssh}{target_path}', mode.Client.LOCAL, True ) read_stats(_output)