def afterBackup(self): paramf_stat = 1 param = ConfigParser(allow_no_value=True) param.set(None, self.cmd[0], self.cmd[1]) opts = self.formatOption(self.backup_param) if opts: for i in opts: param.set(None, i[0], i[2]) tmppath = path.join(self.getLocalTmpDir(), self.backup_param_filename) with open(tmppath, 'w', encoding='utf-8') as wf: param.write(wf) wf.flush() if self._sshobject.transferFileToRemote( tmppath, path_join(self.target_dir[1], self.backup_param_filename)): paramf_stat = 0 if self.is_save_to_local: log.info('compress backup files') s = self.target_dir[1].rpartition('/') cmd = 'cd %s;tar -czpvf %s %s' % (s[0], s[2] + '.tar.gz', s[2]) stat, _ = self._sshobject.execute_cmd(cmd) log.info('compress backup files complete ') if stat == ConnectionBase.SHELL_SUCCESS: log.info( 'transfer compress backup files from remote to local ') log.info( 'transfer compress backup files from remote to local ,this may be a long time ,please wait in patience ...... ' ) self._sshobject.transferFileFromRemote( self.target_dir[1] + '.tar.gz', path_join(self.saved_local_path, s[2] + '.tar.gz')) log.info('transfer compress backup files complete .') return paramf_stat
def start_es(config): with ParamikoConnection(config.ssh_host, config.ssh_user, config.ssh_passwd, config.ssh_port) as pk: cmd = 'su -l elasticsearch -c "sh %s"' % path_join( config.es_base_path, ('scripts', 'start.sh')) pk.execute_backupground(cmd, logfile=path_join( get_yml_value(config.yml_config, *('path', 'logs')), get_yml_value(config.yml_config, *('cluster', 'name'))))
def updateRestoreConfig(self): super(MysqlHotBackup, self).updateRestoreConfig() self.restore_target_dir = self._config.restore_target_dir if not self._sshobject.fileExists(self.restore_target_dir): self._sshobject.mkdir(self.restore_target_dir) if self._sshobject.listdir(self.restore_target_dir): raise EnvironmentCheckException( 'Restore target directory %s must be empty' % self.restore_target_dir) paramf = path_join(self.full_dir[1], self.backup_param_filename) self.getBackupParam(paramf, self.full_backup_param_config) self.mysql_version[1] = self.full_backup_param_config.get( configparser.DEFAULTSECT, self.mysql_version[0], fallback=None) tmpmysqlpath = self.full_backup_param_config.get( configparser.DEFAULTSECT, self.mysql_base[0].replace('-', ''), fallback=None) self.mysql_base[1] = self.getRestoredMysqlBase( (self._config.mysql_software_path, tmpmysqlpath)) if not none_null_stringNone(self._config.server_id): self.server_id[1] = self._config.server_id if self.full_backup_param_config.get(configparser.DEFAULTSECT, self.compress[0].replace('-', ''), fallback=None): self.full_backup_decompress = True if self.incremental_dir[1]: cmd = 'cp -a %s %s' % (self.incremental_dir[1], self.tmp_dir) self._sshobject.execute_cmd(cmd) self.incremental_dir[1] = path_join( self.tmp_dir, self.incremental_dir[1].rpartition('/')[2]) paramf = path_join(self.incremental_dir[1], self.backup_param_filename) if self.incremental_backup_param_config == None: self.incremental_backup_param_config = ConfigParser( allow_no_value=True) self.getBackupParam(paramf, self.incremental_backup_param_config) if self.incremental_backup_param_config.get( configparser.DEFAULTSECT, self.compress[0].replace('-', ''), fallback=None): self.incremental_backup_decompress = True self._sshobject.execute_cmd('cd {};mkdir -p data var log'.format( self.restore_target_dir)) self.datadir[1] = path_join(self.restore_target_dir, 'data') self.log_err_dir = path_join(self.restore_target_dir, 'log') self.setRestoreCnfFile()
def afterBackup(self): paramf_stat = super().afterBackup() cmd = 'cp %s %s ' % (self.defaults_file[1], path_join(self.target_dir[1], self.backup_mysql_cnfname)) cnf_stat, _ = self._sshobject.execute_cmd(cmd) return paramf_stat | cnf_stat
def prepareRestoreEnv(self): super(Xtrabackup, self).prepareRestoreEnv() if self.incremental_basedir[1]: cmd = 'cp -a %s %s' % (self.incremental_basedir[1], self.tmp_dir) self._sshobject.execute_cmd(cmd) self.incremental_basedir[1] = path_join( self.tmp_dir, self.incremental_basedir[1].rpartition('/'))
def makeCnf(self): cnf = ConfigParser(allow_no_value=True) for sec in MYSQL57_CNF_VAR_PREFERENCE.keys(): cnf.add_section(sec) for option in MYSQL57_CNF_VAR_PREFERENCE[sec]: if isinstance(option, (list, tuple)): if option[1]: cnf.set( sec, option[0], str( getVariable(option[1].replace('-', '_'), self._conn))) else: cnf.set( sec, option[0], str( getVariable(option[0].replace('-', '_'), self._conn))) else: cnf.set( sec, option, str(getVariable(option.replace('-', '_'), self._conn))) tmplocalcnfpath = path.join(self.getLocalTmpDir(), self.backup_mysql_cnfname) with open(tmplocalcnfpath, 'w', encoding='utf-8') as wf: cnf.write(wf) tmpcnfpath = path_join(self.tmp_dir, self.backup_mysql_cnfname) self._sshobject.transferFileToRemote(tmplocalcnfpath, tmpcnfpath) return tmpcnfpath
def getRestoredMysqlBase(self, mysqldpath): if not isinstance(mysqldpath, (list, tuple)): mysqldpath = (mysqldpath, ) for mpath in mysqldpath: if self._sshobject.isFile(mpath): mpath = mpath.rpartition('/')[0] cmd = 'cd %s;find . -name mysqld ' % mpath stat, mysqlds = self._sshobject.execute_cmd(cmd, False) mysqlds = to_text(mysqlds) for p in mysqlds.splitlines(): sv = self.findMysqldVersion(path_join(mpath, p)) if sv == self.mysql_version[1]: return path_join(mpath, p).rpartition('/')[0].rpartition('/')[0] raise MysqlVersionNotMatchException( 'Can not find a mysql which version match with backup files')
def afterRestore(self): super(MysqlHotBackup, self).afterRestore() if self._config.ssh_user != 'mysql': cmd = 'chown -R mysql:mysql %s' % (self.restore_target_dir + '/*') self._sshobject.execute_cmd(cmd) cmd = self.compactItem( False, path_join(self.mysql_base[1], 'bin/mysqld_safe'), self.defaults_file, '&') self._sshobject.execute_backupground(cmd) if self.getMysqldCommand(): if self._config.create_slave: a = MasterSlaveConfig() a.master_host = self._config.master_ip a.master_port = self._config.master_port a.master_conn_user = self._config.master_user a.master_conn_password = self._config.master_password a.slave_conn_user = self._config.master_user a.slave_conn_password = self._config.master_password a.slave_host = self._config.host a.slave_port = self._config.port a.repl_user = self._config.replica_user a.repl_password = self._config.replica_password s = self.getReplicaPos() if s[0]: a.gtid_enable = True else: a.gtid_enable = False a.binlog_file = s[1] a.binlog_pos = s[2] res = makeReplicate(a) if res: return ConnectionBase.SHELL_SUCCESS
def do_restore(self): super(Xtrabackup, self).do_restore() cmd = 'cp %s %s' % (self.defaults_file[1], path_join(self.restore_target_dir, self.restored_mysql_cnfname)) self._sshobject.execute_cmd(cmd) self.defaults_file[1] = path_join(self.restore_target_dir, self.restored_mysql_cnfname) self.target_dir[1] = self.full_dir[1] cmd = self.compactItem(False, self.backupsoftwarepath, self.defaults_file, self.target_dir, self.copy_back, '2>&1') stat, _ = self._sshobject.execute_cmd(cmd) checkStatAndRaise( stat, RestoreFailedException, ' when copy backup files to restore target directory.') return stat
def _checkBackupOk(self, logdir): logpath = path_join(logdir, self.backup_log_filename) if self._sshobject.fileExists(logpath): cmd = 'tail -1 %s ' % logpath stat, data = self._sshobject.execute_cmd(cmd, False) if data: data = to_text(data) if data.find(self.backup_success_flag) > -1: return True return False
def getReplicaPos(self): base = self.incremental_dir[1] if self.incremental_dir[ 1] else self.full_dir[1] b = self.getRemoteFileContent(path_join(base, 'xtrabackup_binlog_info')) b = b.split() if len(b) == 3: if b[2]: return (True, b[2]) return (False, b[0], b[1])
def do_backup(self): log.debug('doing backup') self.cmd[1] = self.getBackupCmd() tmpbackuplog = path.join(self.getLocalTmpDir(), self.backup_log_filename) stat, data = self._sshobject.execute_cmd(self.cmd[1], logfile=tmpbackuplog) self._sshobject.transferFileToRemote( tmpbackuplog, path_join(self.target_dir[1], self.backup_log_filename)) if stat == ConnectionBase.SHELL_SUCCESS: self.result_stat = self.RESULT_PARTIAL_SUCCESS return stat
def updateRestoreConfig(self): super(MysqlBackup, self).updateRestoreConfig() if not self.checkBackupOk(): raise CheckMysqlBackupFileException( 'check backup file failed , is the backup file exists ? the other reason may be the last backup had not completed success!' ) cmd = 'cp -a %s %s' % (self.full_dir[1], self.tmp_dir) stat, _ = self._sshobject.execute_cmd(cmd) if stat != ConnectionBase.SHELL_SUCCESS: raise FileCopyException( 'copy backup file from %s to temp directory %s failed!' % (self.full_dir[1], self.tmp_dir)) self.full_dir[1] = path_join(self.tmp_dir, self.full_dir[1].rpartition('/')[2])
def install(config): with ParamikoConnection(config.ssh_host, config.ssh_user, config.ssh_passwd, config.ssh_port) as pk: execute_cmd = functools.partial(ssh_connect.exec, pk.execute_cmd) mkdir = functools.partial(ssh_connect.exec, pk.mkdir) mkdir((config.es_base_path, )) mkdir((get_yml_value(config.yml_config, *('path', 'data')), )) mkdir((get_yml_value(config.yml_config, *('path', 'logs')), )) mkdir((path_join(config.es_base_path, 'config'), )) mkdir((path_join(config.es_base_path, 'scripts'), )) mkdir((path_join(config.es_base_path, 'var'), )) mkdir((path_join(config.es_base_path, 'software'), )) cmd = 'cp %s %s' % (config.es_tgz_path, path_join(config.es_base_path, 'software')) execute_cmd((cmd, )) cmd = 'cd %s ; tar -xzpvf %s ' % (path_join( config.es_base_path, 'software'), os.path.split( config.es_tgz_path)[1]) execute_cmd((cmd, )) software_name = os.path.split(config.es_tgz_path)[1][:-7] cmd = 'cp %s %s'%(path_join(config.es_base_path,('software',software_name,'config','log4j2.properties')), \ path_join(config.es_base_path,'config')) execute_cmd((cmd, )) cmd = 'cp %s %s'%(path_join(config.es_base_path,('software',software_name,'config','jvm.options')), \ path_join(config.es_base_path,'config')) execute_cmd((cmd, )) transferFileToRemote = functools.partial(ssh_connect.exec, pk.transferFileToRemote) result = transferFileToRemote( (os.path.join(os.path.dirname(os.path.abspath(__file__)), 'root_execute.sh'), path_join(config.es_base_path, ('scripts', 'root_execute.sh')))) if config.ssh_user == 'root': cmd = 'chmod u+x %s' % path_join(config.es_base_path, ('scripts', 'root_execute.sh')) execute_cmd((cmd, )) cmd = 'sh %s' % path_join(config.es_base_path, ('scripts', 'root_execute.sh')) execute_cmd(cmd) tmp_file=os.path.join(os.path.dirname(os.path.abspath(__file__)),'elasticsearch.yml'+formatDateTime()+get_yml_value(config.yml_config,*('http','host')) + \ get_yml_value(config.yml_config,*('http','port'))) with open(tmp_file, 'w', encoding='utf-8') as f: yaml.dump(config.yml_config, f) result = transferFileToRemote( (tmp_file, path_join(config.es_base_path, ('config', 'elasticsearch.yml')))) s = 'sed -r -i -e \"s=\"\'^\\s*-Xms[0-9]+g\\s*$\'\"=-Xms%s=\" -e \"s=\"\'^\\s*-Xmx[0-9]+g\\s*$\'\"=-Xmx%s=\" %s' for jvm_path in (path_join(config.es_base_path, ('config', 'jvm.options')), path_join(config.es_base_path, ('software', software_name, 'config', 'jvm.options'))): cmd = s % (config.jvm_heap, config.jvm_heap, jvm_path) execute_cmd((cmd, )) transferFileToRemote( (os.path.join(os.path.dirname(os.path.abspath(__file__)), 'start.sh'), path_join(config.es_base_path, ('scripts', 'start.sh')))) transferFileToRemote( (os.path.join(os.path.dirname(os.path.abspath(__file__)), 'stop.sh'), path_join(config.es_base_path, ('scripts', 'stop.sh')))) user_es = 'elasticsearch' cmd = 'id -u %s' % user_es _, data = execute_cmd((cmd, False)) data = to_text(data) if not re.match(r'^[0-9]+', data): cmd = 'useradd -m -s /sbin/nologin -U %s' % user_es execute_cmd((cmd, )) cmd = 'chown -R %s:%s %s %s %s' % ( user_es, user_es, config.es_base_path, get_yml_value(config.yml_config, *('path', 'data')), get_yml_value(config.yml_config, *('path', 'logs'))) execute_cmd((cmd, ))
def updateRestoreConfig(self): super(MysqlLogicBackup, self).updateRestoreConfig() self.restore_log_file = path_join(self.full_dir[1], self.restore_log_name) self.sql_file = path_join(self.full_dir[1], self.sql_file_name)
def updateBackupConfig(self): super(MysqlLogicBackup, self).updateBackupConfig() self.backup_log_file = path_join(self.target_dir[1], self.backup_log_filename) self.sql_file = path_join(self.target_dir[1], self.sql_file_name) self.databases[1] = self._config.databases
def setRestoreCnfFile(self): cnf = path_join(self.full_dir[1], self.backup_mysql_cnfname) if self._sshobject.fileExists(cnf): self.defaults_file[1] = cnf else: raise MysqlCnfFileNotExistsException( 'can not find mysql config file in backup directory : {}'. format(self.full_dir[1])) cmd = 'cat %s' % cnf stat, cnfContent = self._sshobject.execute_cmd(cmd, False) checkStatAndRaise(stat, ReadBackupConfigFileException, cnf) self.mysql_cnf_config.read_string(to_text(cnfContent)) self.old_mysql_port = self.mysql_cnf_config.get('mysqld', 'port', fallback=None) new_softwarebase = self.mysql_base[1] old_softwarebase = self.getOldBackupConfig(self.mysql_base[0].replace( '-', '')) new_data_base = rpartition(self.datadir[1], '/')[0] old_data_base = rpartition( self.getOldBackupConfig(self.datadir[0].replace('-', '')), '/')[0] def replace_cnfconfig(type, sec, option): if type == constant.SOFTWARE_PATH: o = old_softwarebase n = new_softwarebase else: o = old_data_base n = new_data_base ov = self.mysql_cnf_config.get(sec, option, fallback=None) if ov: self.mysql_cnf_config.set(sec, option, ov.replace(o, n)) for sec in MYSQL57_CNF_VAR_PREFERENCE.keys(): for item in MYSQL57_CNF_VAR_PREFERENCE[sec]: if isinstance(item, (list, tuple)): if item[2]: replace_cnfconfig(item[2], sec, item[0]) self.mysql_cnf_config.set(sec, 'port', str(self._config.port)) if self.server_id[1]: self.mysql_cnf_config.set('mysqld', 'server-id', self.server_id[1]) for op in ('pid_file', 'socket'): o_v = self.mysql_cnf_config.get('mysqld', op, fallback=None) self.mysql_cnf_config.set( 'mysqld', op, o_v.replace(str(self.old_mysql_port), str(self._config.port))) tmplogpath = self.mysql_cnf_config.get('mysqld', 'log-error', fallback=None) if not tmplogpath: tmplogpath = path_join(self.log_err_dir, 'log.err') self.mysql_cnf_config.set('mysqld', 'log-error', tmplogpath) if not self._sshobject.fileExists(tmplogpath): cmd = 'touch %s' % tmplogpath self._sshobject.execute_cmd(cmd) tmplocalpath = path.join(self.getLocalTmpDir(), self.backup_mysql_cnfname) with open(tmplocalpath, 'w') as wf: self.mysql_cnf_config.write(wf) self._sshobject.transferFileToRemote(tmplocalpath, self.defaults_file[1])
def backupInit(self): self._config.backup_dir = path_join(self._config.backup_base_dir, formatDate())
def preInit(self): cmd = 'cd ~;pwd' _, tmp_dir = self._sshobject.execute_cmd(cmd, False) self.tmp_dir = path_join(to_text(tmp_dir), formatDateTime()) self._sshobject.mkdir(self.tmp_dir)