def afterBackup(self):
paramf_stat = 1
param = ConfigParser(allow_no_value=True)
param.set(None, self.cmd[0], self.cmd[1])
opts = self.formatOption(self.backup_param)
if opts:
for i in opts:
param.set(None, i[0], i[2])
tmppath = path.join(self.getLocalTmpDir(), self.backup_param_filename)
with open(tmppath, 'w', encoding='utf-8') as wf:
param.write(wf)
wf.flush()
if self._sshobject.transferFileToRemote(
tmppath,
path_join(self.target_dir[1], self.backup_param_filename)):
paramf_stat = 0
if self.is_save_to_local:
log.info('compress backup files')
s = self.target_dir[1].rpartition('/')
cmd = 'cd %s;tar -czpvf %s %s' % (s[0], s[2] + '.tar.gz', s[2])
stat, _ = self._sshobject.execute_cmd(cmd)
log.info('compress backup files complete ')
if stat == ConnectionBase.SHELL_SUCCESS:
log.info(
'transfer compress backup files from remote to local ')
log.info(
'transfer compress backup files from remote to local ,this may be a long time ,please wait in patience ...... '
)
self._sshobject.transferFileFromRemote(
self.target_dir[1] + '.tar.gz',
path_join(self.saved_local_path, s[2] + '.tar.gz'))
log.info('transfer compress backup files complete .')
return paramf_stat
def start_es(config):
with ParamikoConnection(config.ssh_host, config.ssh_user,
config.ssh_passwd, config.ssh_port) as pk:
cmd = 'su -l elasticsearch -c "sh %s"' % path_join(
config.es_base_path, ('scripts', 'start.sh'))
pk.execute_backupground(cmd,
logfile=path_join(
get_yml_value(config.yml_config,
*('path', 'logs')),
get_yml_value(config.yml_config,
*('cluster', 'name'))))
def updateRestoreConfig(self):
super(MysqlHotBackup, self).updateRestoreConfig()
self.restore_target_dir = self._config.restore_target_dir
if not self._sshobject.fileExists(self.restore_target_dir):
self._sshobject.mkdir(self.restore_target_dir)
if self._sshobject.listdir(self.restore_target_dir):
raise EnvironmentCheckException(
'Restore target directory %s must be empty' %
self.restore_target_dir)
paramf = path_join(self.full_dir[1], self.backup_param_filename)
self.getBackupParam(paramf, self.full_backup_param_config)
self.mysql_version[1] = self.full_backup_param_config.get(
configparser.DEFAULTSECT, self.mysql_version[0], fallback=None)
tmpmysqlpath = self.full_backup_param_config.get(
configparser.DEFAULTSECT,
self.mysql_base[0].replace('-', ''),
fallback=None)
self.mysql_base[1] = self.getRestoredMysqlBase(
(self._config.mysql_software_path, tmpmysqlpath))
if not none_null_stringNone(self._config.server_id):
self.server_id[1] = self._config.server_id
if self.full_backup_param_config.get(configparser.DEFAULTSECT,
self.compress[0].replace('-', ''),
fallback=None):
self.full_backup_decompress = True
if self.incremental_dir[1]:
cmd = 'cp -a %s %s' % (self.incremental_dir[1], self.tmp_dir)
self._sshobject.execute_cmd(cmd)
self.incremental_dir[1] = path_join(
self.tmp_dir, self.incremental_dir[1].rpartition('/')[2])
paramf = path_join(self.incremental_dir[1],
self.backup_param_filename)
if self.incremental_backup_param_config == None:
self.incremental_backup_param_config = ConfigParser(
allow_no_value=True)
self.getBackupParam(paramf, self.incremental_backup_param_config)
if self.incremental_backup_param_config.get(
configparser.DEFAULTSECT,
self.compress[0].replace('-', ''),
fallback=None):
self.incremental_backup_decompress = True
self._sshobject.execute_cmd('cd {};mkdir -p data var log'.format(
self.restore_target_dir))
self.datadir[1] = path_join(self.restore_target_dir, 'data')
self.log_err_dir = path_join(self.restore_target_dir, 'log')
self.setRestoreCnfFile()
def afterBackup(self):
paramf_stat = super().afterBackup()
cmd = 'cp %s %s ' % (self.defaults_file[1],
path_join(self.target_dir[1],
self.backup_mysql_cnfname))
cnf_stat, _ = self._sshobject.execute_cmd(cmd)
return paramf_stat | cnf_stat
def prepareRestoreEnv(self):
super(Xtrabackup, self).prepareRestoreEnv()
if self.incremental_basedir[1]:
cmd = 'cp -a %s %s' % (self.incremental_basedir[1], self.tmp_dir)
self._sshobject.execute_cmd(cmd)
self.incremental_basedir[1] = path_join(
self.tmp_dir, self.incremental_basedir[1].rpartition('/'))
def makeCnf(self):
cnf = ConfigParser(allow_no_value=True)
for sec in MYSQL57_CNF_VAR_PREFERENCE.keys():
cnf.add_section(sec)
for option in MYSQL57_CNF_VAR_PREFERENCE[sec]:
if isinstance(option, (list, tuple)):
if option[1]:
cnf.set(
sec, option[0],
str(
getVariable(option[1].replace('-', '_'),
self._conn)))
else:
cnf.set(
sec, option[0],
str(
getVariable(option[0].replace('-', '_'),
self._conn)))
else:
cnf.set(
sec, option,
str(getVariable(option.replace('-', '_'), self._conn)))
tmplocalcnfpath = path.join(self.getLocalTmpDir(),
self.backup_mysql_cnfname)
with open(tmplocalcnfpath, 'w', encoding='utf-8') as wf:
cnf.write(wf)
tmpcnfpath = path_join(self.tmp_dir, self.backup_mysql_cnfname)
self._sshobject.transferFileToRemote(tmplocalcnfpath, tmpcnfpath)
return tmpcnfpath
def getRestoredMysqlBase(self, mysqldpath):
if not isinstance(mysqldpath, (list, tuple)):
mysqldpath = (mysqldpath, )
for mpath in mysqldpath:
if self._sshobject.isFile(mpath):
mpath = mpath.rpartition('/')[0]
cmd = 'cd %s;find . -name mysqld ' % mpath
stat, mysqlds = self._sshobject.execute_cmd(cmd, False)
mysqlds = to_text(mysqlds)
for p in mysqlds.splitlines():
sv = self.findMysqldVersion(path_join(mpath, p))
if sv == self.mysql_version[1]:
return path_join(mpath,
p).rpartition('/')[0].rpartition('/')[0]
raise MysqlVersionNotMatchException(
'Can not find a mysql which version match with backup files')
def afterRestore(self):
super(MysqlHotBackup, self).afterRestore()
if self._config.ssh_user != 'mysql':
cmd = 'chown -R mysql:mysql %s' % (self.restore_target_dir + '/*')
self._sshobject.execute_cmd(cmd)
cmd = self.compactItem(
False, path_join(self.mysql_base[1], 'bin/mysqld_safe'),
self.defaults_file, '&')
self._sshobject.execute_backupground(cmd)
if self.getMysqldCommand():
if self._config.create_slave:
a = MasterSlaveConfig()
a.master_host = self._config.master_ip
a.master_port = self._config.master_port
a.master_conn_user = self._config.master_user
a.master_conn_password = self._config.master_password
a.slave_conn_user = self._config.master_user
a.slave_conn_password = self._config.master_password
a.slave_host = self._config.host
a.slave_port = self._config.port
a.repl_user = self._config.replica_user
a.repl_password = self._config.replica_password
s = self.getReplicaPos()
if s[0]:
a.gtid_enable = True
else:
a.gtid_enable = False
a.binlog_file = s[1]
a.binlog_pos = s[2]
res = makeReplicate(a)
if res:
return ConnectionBase.SHELL_SUCCESS
def do_restore(self):
super(Xtrabackup, self).do_restore()
cmd = 'cp %s %s' % (self.defaults_file[1],
path_join(self.restore_target_dir,
self.restored_mysql_cnfname))
self._sshobject.execute_cmd(cmd)
self.defaults_file[1] = path_join(self.restore_target_dir,
self.restored_mysql_cnfname)
self.target_dir[1] = self.full_dir[1]
cmd = self.compactItem(False, self.backupsoftwarepath,
self.defaults_file, self.target_dir,
self.copy_back, '2>&1')
stat, _ = self._sshobject.execute_cmd(cmd)
checkStatAndRaise(
stat, RestoreFailedException,
' when copy backup files to restore target directory.')
return stat
def _checkBackupOk(self, logdir):
logpath = path_join(logdir, self.backup_log_filename)
if self._sshobject.fileExists(logpath):
cmd = 'tail -1 %s ' % logpath
stat, data = self._sshobject.execute_cmd(cmd, False)
if data:
data = to_text(data)
if data.find(self.backup_success_flag) > -1:
return True
return False
def getReplicaPos(self):
base = self.incremental_dir[1] if self.incremental_dir[
1] else self.full_dir[1]
b = self.getRemoteFileContent(path_join(base,
'xtrabackup_binlog_info'))
b = b.split()
if len(b) == 3:
if b[2]:
return (True, b[2])
return (False, b[0], b[1])
def do_backup(self):
log.debug('doing backup')
self.cmd[1] = self.getBackupCmd()
tmpbackuplog = path.join(self.getLocalTmpDir(),
self.backup_log_filename)
stat, data = self._sshobject.execute_cmd(self.cmd[1],
logfile=tmpbackuplog)
self._sshobject.transferFileToRemote(
tmpbackuplog,
path_join(self.target_dir[1], self.backup_log_filename))
if stat == ConnectionBase.SHELL_SUCCESS:
self.result_stat = self.RESULT_PARTIAL_SUCCESS
return stat
def updateRestoreConfig(self):
super(MysqlBackup, self).updateRestoreConfig()
if not self.checkBackupOk():
raise CheckMysqlBackupFileException(
'check backup file failed , is the backup file exists ? the other reason may be the last backup had not completed success!'
)
cmd = 'cp -a %s %s' % (self.full_dir[1], self.tmp_dir)
stat, _ = self._sshobject.execute_cmd(cmd)
if stat != ConnectionBase.SHELL_SUCCESS:
raise FileCopyException(
'copy backup file from %s to temp directory %s failed!' %
(self.full_dir[1], self.tmp_dir))
self.full_dir[1] = path_join(self.tmp_dir,
self.full_dir[1].rpartition('/')[2])
def install(config):
with ParamikoConnection(config.ssh_host, config.ssh_user,
config.ssh_passwd, config.ssh_port) as pk:
execute_cmd = functools.partial(ssh_connect.exec, pk.execute_cmd)
mkdir = functools.partial(ssh_connect.exec, pk.mkdir)
mkdir((config.es_base_path, ))
mkdir((get_yml_value(config.yml_config, *('path', 'data')), ))
mkdir((get_yml_value(config.yml_config, *('path', 'logs')), ))
mkdir((path_join(config.es_base_path, 'config'), ))
mkdir((path_join(config.es_base_path, 'scripts'), ))
mkdir((path_join(config.es_base_path, 'var'), ))
mkdir((path_join(config.es_base_path, 'software'), ))
cmd = 'cp %s %s' % (config.es_tgz_path,
path_join(config.es_base_path, 'software'))
execute_cmd((cmd, ))
cmd = 'cd %s ; tar -xzpvf %s ' % (path_join(
config.es_base_path, 'software'), os.path.split(
config.es_tgz_path)[1])
execute_cmd((cmd, ))
software_name = os.path.split(config.es_tgz_path)[1][:-7]
cmd = 'cp %s %s'%(path_join(config.es_base_path,('software',software_name,'config','log4j2.properties')), \
path_join(config.es_base_path,'config'))
execute_cmd((cmd, ))
cmd = 'cp %s %s'%(path_join(config.es_base_path,('software',software_name,'config','jvm.options')), \
path_join(config.es_base_path,'config'))
execute_cmd((cmd, ))
transferFileToRemote = functools.partial(ssh_connect.exec,
pk.transferFileToRemote)
result = transferFileToRemote(
(os.path.join(os.path.dirname(os.path.abspath(__file__)),
'root_execute.sh'),
path_join(config.es_base_path, ('scripts', 'root_execute.sh'))))
if config.ssh_user == 'root':
cmd = 'chmod u+x %s' % path_join(config.es_base_path,
('scripts', 'root_execute.sh'))
execute_cmd((cmd, ))
cmd = 'sh %s' % path_join(config.es_base_path,
('scripts', 'root_execute.sh'))
execute_cmd(cmd)
tmp_file=os.path.join(os.path.dirname(os.path.abspath(__file__)),'elasticsearch.yml'+formatDateTime()+get_yml_value(config.yml_config,*('http','host')) + \
get_yml_value(config.yml_config,*('http','port')))
with open(tmp_file, 'w', encoding='utf-8') as f:
yaml.dump(config.yml_config, f)
result = transferFileToRemote(
(tmp_file,
path_join(config.es_base_path, ('config', 'elasticsearch.yml'))))
s = 'sed -r -i -e \"s=\"\'^\\s*-Xms[0-9]+g\\s*$\'\"=-Xms%s=\" -e \"s=\"\'^\\s*-Xmx[0-9]+g\\s*$\'\"=-Xmx%s=\" %s'
for jvm_path in (path_join(config.es_base_path,
('config', 'jvm.options')),
path_join(config.es_base_path,
('software', software_name, 'config',
'jvm.options'))):
cmd = s % (config.jvm_heap, config.jvm_heap, jvm_path)
execute_cmd((cmd, ))
transferFileToRemote(
(os.path.join(os.path.dirname(os.path.abspath(__file__)),
'start.sh'),
path_join(config.es_base_path, ('scripts', 'start.sh'))))
transferFileToRemote(
(os.path.join(os.path.dirname(os.path.abspath(__file__)),
'stop.sh'),
path_join(config.es_base_path, ('scripts', 'stop.sh'))))
user_es = 'elasticsearch'
cmd = 'id -u %s' % user_es
_, data = execute_cmd((cmd, False))
data = to_text(data)
if not re.match(r'^[0-9]+', data):
cmd = 'useradd -m -s /sbin/nologin -U %s' % user_es
execute_cmd((cmd, ))
cmd = 'chown -R %s:%s %s %s %s' % (
user_es, user_es, config.es_base_path,
get_yml_value(config.yml_config, *('path', 'data')),
get_yml_value(config.yml_config, *('path', 'logs')))
execute_cmd((cmd, ))
def updateRestoreConfig(self):
super(MysqlLogicBackup, self).updateRestoreConfig()
self.restore_log_file = path_join(self.full_dir[1],
self.restore_log_name)
self.sql_file = path_join(self.full_dir[1], self.sql_file_name)
def updateBackupConfig(self):
super(MysqlLogicBackup, self).updateBackupConfig()
self.backup_log_file = path_join(self.target_dir[1],
self.backup_log_filename)
self.sql_file = path_join(self.target_dir[1], self.sql_file_name)
self.databases[1] = self._config.databases
def setRestoreCnfFile(self):
cnf = path_join(self.full_dir[1], self.backup_mysql_cnfname)
if self._sshobject.fileExists(cnf):
self.defaults_file[1] = cnf
else:
raise MysqlCnfFileNotExistsException(
'can not find mysql config file in backup directory : {}'.
format(self.full_dir[1]))
cmd = 'cat %s' % cnf
stat, cnfContent = self._sshobject.execute_cmd(cmd, False)
checkStatAndRaise(stat, ReadBackupConfigFileException, cnf)
self.mysql_cnf_config.read_string(to_text(cnfContent))
self.old_mysql_port = self.mysql_cnf_config.get('mysqld',
'port',
fallback=None)
new_softwarebase = self.mysql_base[1]
old_softwarebase = self.getOldBackupConfig(self.mysql_base[0].replace(
'-', ''))
new_data_base = rpartition(self.datadir[1], '/')[0]
old_data_base = rpartition(
self.getOldBackupConfig(self.datadir[0].replace('-', '')), '/')[0]
def replace_cnfconfig(type, sec, option):
if type == constant.SOFTWARE_PATH:
o = old_softwarebase
n = new_softwarebase
else:
o = old_data_base
n = new_data_base
ov = self.mysql_cnf_config.get(sec, option, fallback=None)
if ov:
self.mysql_cnf_config.set(sec, option, ov.replace(o, n))
for sec in MYSQL57_CNF_VAR_PREFERENCE.keys():
for item in MYSQL57_CNF_VAR_PREFERENCE[sec]:
if isinstance(item, (list, tuple)):
if item[2]:
replace_cnfconfig(item[2], sec, item[0])
self.mysql_cnf_config.set(sec, 'port', str(self._config.port))
if self.server_id[1]:
self.mysql_cnf_config.set('mysqld', 'server-id', self.server_id[1])
for op in ('pid_file', 'socket'):
o_v = self.mysql_cnf_config.get('mysqld', op, fallback=None)
self.mysql_cnf_config.set(
'mysqld', op,
o_v.replace(str(self.old_mysql_port), str(self._config.port)))
tmplogpath = self.mysql_cnf_config.get('mysqld',
'log-error',
fallback=None)
if not tmplogpath:
tmplogpath = path_join(self.log_err_dir, 'log.err')
self.mysql_cnf_config.set('mysqld', 'log-error', tmplogpath)
if not self._sshobject.fileExists(tmplogpath):
cmd = 'touch %s' % tmplogpath
self._sshobject.execute_cmd(cmd)
tmplocalpath = path.join(self.getLocalTmpDir(),
self.backup_mysql_cnfname)
with open(tmplocalpath, 'w') as wf:
self.mysql_cnf_config.write(wf)
self._sshobject.transferFileToRemote(tmplocalpath,
self.defaults_file[1])
def backupInit(self):
self._config.backup_dir = path_join(self._config.backup_base_dir,
formatDate())
def preInit(self):
cmd = 'cd ~;pwd'
_, tmp_dir = self._sshobject.execute_cmd(cmd, False)
self.tmp_dir = path_join(to_text(tmp_dir), formatDateTime())
self._sshobject.mkdir(self.tmp_dir)
