def step_impl(context, fildername,hostname):
cmd = "mkdir {0}".format(fildername)
context.logger.info("sed cmd is :{0}".format(cmd))
if hostname.startswith('dble'):
ssh = get_ssh(hostname)
else:
ssh = get_ssh(hostname)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(len(stderr) == 0, "create filder content with:{1}, got err:{0}".format(stderr, cmd))
def step_impl(context,result,hostname):
cmd = context.text.strip()
if hostname.startswith("dble"):
ssh = get_ssh(hostname)
else:
ssh = get_ssh(hostname)
rc, stdout, stderr = ssh.exec_command(cmd)
context.logger.info("execute cmd:{0}".format(cmd))
stderr = stderr.lower()
assert stderr.find("error") == -1, "execute cmd:{0} error:{1}".format(cmd, stderr)
setattr(context,result,stdout)
def impl_step(context,host,pattern,resultName):
if host.startswith('dble'):
ssh = get_ssh(host)
else:
ssh = get_ssh(host)
oscmd = context.text.strip()
rc, stdout, stderr = ssh.exec_command(oscmd)
assert_that(len(stderr) == 0, 'expect no err ,but: {0}'.format(stderr))
results = list(set(re.findall(pattern,stdout)))
assert_that(len(results)),"regular matching result is empty"
context.logger.info("regular matching result:{0}".format(results))
setattr(context,resultName,results)
def step_impl(context, btraceScript, host):
sshClient = get_ssh(host)
isBtraceRunning = check_btrace_running(sshClient, btraceScript)
context.logger.info("isBtraceRunning:{0} before try to stop {1}".format(isBtraceRunning, btraceScript))
if isBtraceRunning:
stop_btrace(sshClient, btraceScript)
def step_impl(context,host1,role,host2,oscmd="cd /root/sandboxes/msb_5_7_25/data"):
user = ''
password = ''
port = ''
if host1.startswith('dble'):
node = get_node(host1)
if role == "admin":
user = node.manager_user
password = node.manager_password
port = node.manager_port
else:
user = node.client_user
password = node.client_password
port = node.client_port
else:
node = get_node(host1)
user = node.mysql_user
password = node.mysql_password
port = node.mysql_port
ip = node.ip
ssh = get_ssh(host2)
if host2.startswith('mysql8'):
oscmd = "cd /root/sandboxes/msb_8_0_18/data"
sql_cmd_str = context.text.strip()
sql_cmd_list = sql_cmd_str.splitlines()
context.logger.info("sql list: {0}".format(sql_cmd_list))
for sql_cmd in sql_cmd_list:
cmd = '{5} && mysql -h{0} -u{1} -p{2} -P{3} -c -e"{4}"'.format(ip, user, password, port,sql_cmd,oscmd)
stdin, stdout, stderr = ssh.exec_command(cmd)
context.logger.info("execute cmd:{0}".format(cmd))
stderr = stderr.lower()
assert stderr.find("error") == -1, "execute cmd: {0} err:{1}".format(cmd,stderr)
time.sleep(3)
def get_zk_meta_on_zkCli_sh(context, path, info):
# zk_method : zk cmd means : ls ,get, rmr, set, config ... etc...you can run help on zkCli.sh for more info
# /dble/cluster-1/conf/sharding
# info is function name such as:enum_func
# hostname is dble-1 ,dble-2,dble-3
cmd = "cd {0}/bin && ./zkCli.sh get {1}|grep '{2}'".format(
context.cfg_zookeeper['home'], path, info)
# node = str(hostname)
cmd_ssh = get_ssh("dble-1")
rc, sto, ste = cmd_ssh.exec_command(cmd)
func_name = []
assert_that(sto, not_(empty()),
"sto is not empty and it is : {0}".format(sto))
LOGGER.debug("the sto is empty : {0}".format(sto))
outcome_dict = json.loads(sto)
LOGGER.debug(
"add debug to check the result of executing {0} is :sto:{1}".format(
cmd, outcome_dict))
assert_that(
outcome_dict, has_key("function"),
"we have a key named function and they are:{0}".format(outcome_dict))
LOGGER.debug("{0} function is : {1} ".format(path,
outcome_dict.get('function')))
func_list = outcome_dict['function']
for func_obj in func_list:
if func_obj.get('name'):
LOGGER.debug("{0} function is : {1} ".format(
path, func_obj.get('name')))
func_name.append(func_obj['name'])
else:
LOGGER.debug("there are no keys named name in {0} ".format(path))
LOGGER.debug("we find function name are : {0}".format(func_name))
return func_name
def step_impl(context, filename, hostname, line_num_rs=None):
ssh = get_ssh(hostname)
for row in context.table:
key = row["key"]
interval = row["interval_times"]
if line_num_rs:
line_num = getattr(context, line_num_rs, 0)
logger.debug("line_num {0}".format(line_num))
cmd = "tail -n +{2} {1} | grep -n \'{0}\'".format(key, filename, line_num)
else:
cmd = "grep \'{0}\' {1}".format(key, filename)
rc, stdout, stderr = ssh.exec_command(cmd)
str_list = stdout.splitlines()
pre_datetime = ""
for info in str_list:
line_data = re.search(r"(\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2})", info)
logger.debug("log datetime is {0}".format(line_data.groups()))
if line_data:
line_datetime = line_data.groups()[0]
datetime1 = datetime.datetime.strptime(line_datetime, '%Y-%m-%d %H:%M:%S')
if pre_datetime:
datetime2 = datetime.datetime.strptime(pre_datetime, '%Y-%m-%d %H:%M:%S')
datetime_interval = (datetime1-datetime2).seconds
logger.debug("datetime interval is {0}".format(int(datetime_interval) == int(interval)))
assert_that(int(datetime_interval) == int(interval),
"last datetime is {0}, current datetime is {1}, "
"expect the time interval of \"{2}\" is {3} in {4}, "
"but the actual value is {5}".format(pre_datetime, line_datetime, key, interval, filename, datetime_interval))
pre_datetime = line_datetime
else:
pre_datetime = line_datetime
continue
else:
continue
def check_text_at_least_one(context,filename,hostname,checkFromLine=0):
strs = context.text.strip()
strs_list = strs.splitlines()
ssh = get_ssh(hostname)
cmd = "tail -n +{3} {2} | grep -n -e \"{0}\" -e \'{1}\'".format(strs_list[0], strs_list[1], filename, checkFromLine)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(len(stdout) > 0, "expect \"{0}\" exist in file {1} at least one,but has not both ".format(strs_list, filename))
def check_text_times(context, filename, hostname, num, checkFromLine=0):
strs = context.text.strip()
strs_list = strs.splitlines()
ssh = get_ssh(hostname)
for str in strs_list:
cmd = "tail -n +{2} {1} | grep -n \"{0}\" | wc -l".format(str, filename, checkFromLine)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(stdout == num, "expect \"{0}\" exists in file {1} with {2} times, but was {3} times".format(str, filename, num, stdout))
def step_impl(context, filename, hostname):
ssh = get_ssh(hostname)
for row in context.table:
str = row["key"]
num = row["occur_times"]
cmd = "grep \'{0}\' {1}|wc -l".format(str, filename)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(stdout == num,
"expect the occur times of \"{0}\" is {1} in {2},but the actual value is {3}".format(str, num, filename,
stdout))
def before_all(context):
setup_logging('./conf/logging.yaml')
steps_logger = logging.getLogger('root')
context.logger = steps_logger
logger.info('*' * 30)
logger.info('* DBLE TEST START *')
logger.info('*' * 30)
logger.info('Enter hook before_all')
test_config = context.config.userdata["test_config"].lower(
) #"./conf/auto_dble_test.yaml"
#convert auto_dble_test.yaml attr to context attr
parsed = load_yaml_config("./conf/" + test_config)
for name, values in parsed.iteritems():
setattr(context, name, values)
context.userDebug = context.config.userdata["user_debug"].lower() == "true"
context.is_cluster = context.config.userdata["is_cluster"].lower(
) == "true"
if context.is_cluster:
init_meta(context, "dble_cluster")
else:
init_meta(context, "dble")
for node in DbleMeta.dbles:
disable_cluster_config_in_node(context, node)
init_meta(context, "mysqls")
context.ssh_client = get_ssh(context.cfg_dble['dble']['hostname'])
context.ssh_sftp = get_sftp(context.cfg_dble['dble']['hostname'])
try:
para_dble_conf = context.config.userdata.pop('dble_conf')
except KeyError:
raise KeyError(
'Not define userdata dble_conf, usage: behave -D dble_conf=XXX ...'
)
init_dble_conf(context, para_dble_conf)
reinstall = context.config.userdata["reinstall"].lower() == "true"
reset = context.config.userdata["reset"].lower() == "true"
logger.info("run test with environment reinstall: {0}, reset: {1}".format(
reinstall, reset))
context.need_download = context.config.userdata[
"install_from_local"].lower() != "true"
if reinstall:
install_dble_in_all_nodes(context)
if reset:
reset_dble(context)
else:
logger.info('give new install')
logger.info('Exit hook <{0}>'.format('before_all'))
def check_text(context,flag,filename,hostname,checkFromLine=0):
strs = context.text.strip()
strs_list = strs.splitlines()
ssh = get_ssh(hostname)
for str in strs_list:
cmd = "tail -n +{2} {1} | grep -n \"{0}\"".format(str,filename,checkFromLine)
rc, stdout, stderr = ssh.exec_command(cmd)
if flag =="N":
assert_that(len(stdout) == 0,"expect \"{0}\" not exist in file {1},but exist".format(str,filename))
else:#default take flag as Y
assert_that(len(stdout) > 0, "expect \"{0}\" exist in file {1},but not".format(str, filename))
def step_impl(context,flag,dirname,hostname):
strs = context.text.strip()
strs_list = strs.splitlines()
ssh = get_ssh(hostname)
for str in strs_list:
cmd = "find {0} -name {1}".format(dirname, str)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(len(stderr) == 0, "exec command: {0} failed for: {1}".format(cmd, stderr[0:200]))
if flag == "not":
assert_that(len(stdout) == 0, "expect \"{0}\" not exist in dir {1},but exist".format(str, dirname))
else:
assert_that(len(stdout) > 0, "expect \"{0}\" exist in dir {1},but not".format(str, dirname))
def step_impl(context, filename):
# remove file in behave resides server
if os.path.exists(filename):
os.remove(filename)
# remove file in dble
cmd = "rm -rf {0}".format(filename)
rc, stdout, stderr = context.ssh_client.exec_command(cmd)
assert len(stderr)==0, "rm file in dble fail for {0}".format(stderr)
# remove file in compare mysql
dble_node_ssh = get_ssh(context.cfg_mysql['compare_mysql']['master1']['hostname'])
rc, stdout, stderr = dble_node_ssh.exec_command(cmd)
assert len(stderr)==0, "rm file in compare mysql fail for {0}".format(stderr)
def step_impl(context,file,host):
sshClient = get_ssh(host)
retry = 0
isFound = False
while retry < 5:
time.sleep(2) # a interval wait for query run into
cmd = "cat {0} | grep '{1}' -c".format(file, context.text.strip())
rc, sto, ste = sshClient.exec_command(cmd)
assert len(ste) == 0, "check err:{0}".format(ste)
isFound = int(sto) == 1
if isFound:
context.logger.debug("expect text is found in {0}s".format((retry + 1) * 2))
break
retry = retry + 1
assert isFound, "can not find expect text '{0}' in {1}".format(context.text, file)
def step_impl(context,hostname,num=None,rs_name=None):
cmd = context.text.strip()
ssh = get_ssh(hostname)
if rs_name is not None:
param_value = getattr(context, rs_name)
assert param_value, "expect parameter not found in {0}".format(rs_name)
cmd = cmd.format(param_value)
rc, stdout, stderr = ssh.exec_command(cmd)
stderr = stderr.lower()
assert stderr.find("error") == -1, "execute cmd: {0} err:{1}".format(cmd,stderr)
if num is not None:
assert int(stdout) >= int(num), "expect {0} less than result {1} ,but not ".format(num, int(stdout))
def check_cluster_successd(context, expectNodes):
if not hasattr(context, "retry_check_zk_nodes"):
context.retry_check_zk_nodes = 0
context.check_zk_nodes_success = False
realNodes = []
cmd = "cd {0}/bin && ./zkCli.sh ls /dble/cluster-1/online|grep -v ':'|grep -v ^$ ".format(
context.cfg_zookeeper['home'])
cmd_ssh = get_ssh("dble-1")
rc, sto, ste = cmd_ssh.exec_command(cmd)
LOGGER.debug(
"add debug to check the result of executing {0} is :sto:{1}".format(
cmd, sto))
sub_sto = re.findall(r'[\[](.*)[\]]', sto)
LOGGER.debug(
"add debug to check the result of sub_sto is :{0}".format(sub_sto))
nodes = sub_sto[0].replace(",", " ").split()
LOGGER.debug("add debug to check the result of nodes is:{0}".format(nodes))
for id in nodes:
LOGGER.info("id:{0}".format(id))
hostname = "dble-{0}".format(id)
realNodes.append(hostname)
if (expectNodes == realNodes):
context.check_zk_nodes_success = True
if not context.check_zk_nodes_success:
if context.retry_check_zk_nodes < 10:
context.retry_check_zk_nodes = context.retry_check_zk_nodes + 1
time.sleep(10)
check_cluster_successd(context, expectNodes)
else:
LOGGER.info(
"The online dbles detected by zk do not meet expectations after 10 times try,expectNodes:{0},realNodes:{1}"
.format(expectNodes, realNodes))
delattr(context, "retry_check_zk_nodes")
else:
delattr(context, "retry_check_zk_nodes")
def restore(self):
# if "restore_sys_time" in self._scenario.tags:
# utils.restore_sys_time()
#
# if "aft_reset_replication" in self._scenario.tags:
# utils.reset_repl()
#
if "restore_network" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_network'")
logger.debug("params_dic is: {0}".format(params_dic))
if params_dic:
paras = params_dic["restore_network"].split(",")
# paras = paras.split(",")
else:
paras = ""
logger.debug("try to restore_network: {0}".format(paras))
for host_name in paras:
logger.debug("the value of host_name is: {0}".format(host_name))
cmd = "iptables -F"
ssh = get_ssh(host_name)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(len(stderr) == 0, "restore network with command:{1}, got err:{0}".format(stderr, cmd))
if "restore_view" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_view'")
if params_dic:
paras = params_dic["restore_view"]
else:
paras = {}
for host_name, mysql_vars in paras.items():
if host_name.find('dble') != -1:
mode = "user"
else:
mode = "mysql"
for k, v in mysql_vars.items():
list_value = filter(lambda x: x, v.split(","))
view_value = ""
for value in list_value:
view_value = view_value + "{0}.{1},".format(k, value)
query = "drop view if exists " + view_value
sql = query[:-1] # delete the last ','
# logger.debug("the sql is: {0}".format(sql))
execute_sql_in_host(host_name, {"sql": sql}, mode)
if "restore_mysql_service" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_mysql_service'")
if params_dic:
paras = params_dic["restore_mysql_service"]
else:
paras = {}
logger.debug("try to restore_mysql_service: {0}".format(paras))
for host_name, mysql_vars in paras.items():
logger.debug("the value of host_name is: {0}".format(host_name))
for k, v in mysql_vars.items():
if v:
mysql = ObjectFactory.create_mysql_object(host_name)
mysql.start()
if "restore_global_setting" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_global_setting'")
if params_dic:
paras = params_dic["restore_global_setting"]
else:
paras = {}
logger.debug("try to restore_global_setting of mysqls: {0}".format(paras))
for mysql, mysql_vars in paras.items():
query = "set global "
for k, v in mysql_vars.items():
query = query + "{0}={1},".format(k, v)
sql = query[:-1] # delete the last ','
execute_sql_in_host(mysql, {"sql": sql})
if "restore_mysql_config" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_mysql_config'")
if params_dic:
paras = params_dic["restore_mysql_config"]
else:
paras = {}
logger.debug("try to restore_mysql_config of mysqls: {0}".format(paras))
for host_name, mysql_vars in paras.items():
sed_str = ""
for k, v in mysql_vars.items():
m = ['log-bin', 'binlog_format', 'relay-log']
if k in m:
sed_str += "/{0}/d\n".format(k)
else:
sed_str += "/{0}/d\n/server-id/a {0}={1}\n".format(k, v)
mysql = ObjectFactory.create_mysql_object(host_name)
mysql.restart(sed_str)
if "restore_xa_recover" in self._scenario.tags:
params_dic = self.get_tag_params("{'restore_xa_recover'")
if params_dic:
paras = params_dic["restore_xa_recover"]
else:
paras = {}
logger.debug("try to restore_xa_recover of mysqls: {0}".format(paras))
for host_name in paras:
if host_name.find('dble') != -1:
mode = "user"
else:
mode = "mysql"
logger.debug("the value of host_name is: {0}, mode: {1}".format(host_name, mode))
sql = "xa recover"
res, err = execute_sql_in_host(host_name, {"sql": sql}, mode)
if res:
for data in res:
sql = "xa rollback '{0}'".format(data[3])
execute_sql_in_host(host_name, {"sql": sql}, mode)
if "delete_mysql_tables" in self._scenario.tags:
params_dic = self.get_tag_params("{'delete_mysql_tables'")
if params_dic:
paras = params_dic["delete_mysql_tables"]
else:
paras = {}
for host_name in paras.keys():
logger.debug("try to delete_mysql_tables of mysqls: {0}".format(host_name))
ssh = get_ssh(host_name)
if host_name.find("dble") != -1:
mode = "user"
else:
mode = "mysql"
logger.debug("the value of host_name is: {0}, mode: {1}".format(host_name, mode))
# in case of program exits unexpected, tables.txt Residue in /tmp
rc, stdout, stderr = ssh.exec_command("find /tmp -name tables.txt")
if len(stdout) > 0:
ssh.exec_command("rm -rf /tmp/tables.txt")
for database in paras[host_name]:
generate_drop_tables_sql = "select concat('drop table if exists ',table_name,';') from information_schema.TABLES where table_schema='{0}' into outfile '/tmp/tables.txt'".format(database)
execute_sql_in_host(host_name, {"sql": generate_drop_tables_sql}, mode)
# MySQLDB not support source grammar,replace with ssh cmd
rc, stdout, stderr = ssh.exec_command("mysql -uroot -p111111 -D{0} -e 'source /tmp/tables.txt'".format(database))
stderr = stderr.lower()
assert stderr.find("error") == -1, "deletet mysql in {0}:{1} failed, err: {2}".format(host_name, database, stderr)
ssh.exec_command("rm -rf /tmp/tables.txt")
logger.debug("{0} tables has been delete success".format(host_name))
logger.info("all required tables has been delete success")
def step_impl(context, path, hostname):
ssh = get_ssh(hostname)
cmd = "rm -rf " + path
rc, stdout, stderr = ssh.exec_command(cmd)
logging.debug("rc:{0}, stdout:{1}, stderr:{2}\n".format(rc, stdout, stderr))
def step_impl(context, path, hostname, exist_or_not):
ssh = get_ssh(hostname)
cmd = "find " + path
rc, stdout, stderr = ssh.exec_command(cmd)
logger.debug("rc:{0}, stdout:{1}, stderr:{2}\n".format(rc, stdout,stderr))
assert (exist_or_not == "not exist" and stderr != "") or (exist_or_not == "exist" and stderr == "")
def step_impl(context, host):
sshClient = get_ssh(host)
kill_query(sshClient, context.text, context)
def step_impl(context,filename,hostname):
cmd = "rm -rf {0}".format(filename)
ssh = get_ssh(hostname)
rc, stdout, stderr = ssh.exec_command(cmd)
assert_that(len(stderr)==0 ,"get err {0} with deleting {1}".format(stderr,filename))
