def step_impl(context):
mysql_node = get_node("mysql-master1")
# copy dble's dbseq.sql to local
dble_node = get_node("dble-1")
source_remote_file = "{0}/dble/conf/dbseq.sql".format(
dble_node.install_dir)
target_remote_file = "{0}/data/dbseq.sql".format(mysql_node.install_path)
local_file = "{0}/dbseq.sql".format(os.getcwd())
ssh_client = mysql_node.ssh_conn
cmd = "rm -rf {0}".format(local_file)
ssh_client.exec_command(cmd)
context.ssh_sftp.sftp_get(source_remote_file, local_file)
mysql_node.sftp_conn.sftp_put(local_file, target_remote_file)
cmd = "mysql -utest -p111111 db1 < {0}".format(target_remote_file)
ssh_client.exec_command(cmd)
#execute dbseq.sql at the node configed in sequence file
execute_sql_in_host(
"mysql-master1",
info_dic={
"sql":
"insert into DBLE_SEQUENCE values ('`schema1`.`test_auto`', 3, 1)",
"db": "db1"
})
def step_impl(context,host1,role,host2,oscmd="cd /root/sandboxes/msb_5_7_25/data"):
user = ''
password = ''
port = ''
if host1.startswith('dble'):
node = get_node(host1)
if role == "admin":
user = node.manager_user
password = node.manager_password
port = node.manager_port
else:
user = node.client_user
password = node.client_password
port = node.client_port
else:
node = get_node(host1)
user = node.mysql_user
password = node.mysql_password
port = node.mysql_port
ip = node.ip
ssh = get_ssh(host2)
if host2.startswith('mysql8'):
oscmd = "cd /root/sandboxes/msb_8_0_18/data"
sql_cmd_str = context.text.strip()
sql_cmd_list = sql_cmd_str.splitlines()
context.logger.info("sql list: {0}".format(sql_cmd_list))
for sql_cmd in sql_cmd_list:
cmd = '{5} && mysql -h{0} -u{1} -p{2} -P{3} -c -e"{4}"'.format(ip, user, password, port,sql_cmd,oscmd)
stdin, stdout, stderr = ssh.exec_command(cmd)
context.logger.info("execute cmd:{0}".format(cmd))
stderr = stderr.lower()
assert stderr.find("error") == -1, "execute cmd: {0} err:{1}".format(cmd,stderr)
time.sleep(3)
def step_impl(context, filename, num=None):
logger.debug("*** debug context.text:{0}".format(context.text))
text = context.text
# remove old file in behave resides server
if os.path.exists(filename):
os.remove(filename)
if num is None:
if operator.eq(text, '10000+lines'):
with open(filename, 'w') as fp:
col1 = 1
col2 = col1 + 1
col3 = "abcd"
col4 = "1234"
for i in range(15000):
data = str(col1)+','+str(col2)+','+str(col3)+','+str(col4)
fp.write(data + '\n')
col1 = col1+1
col2 = col2+1
elif text.find("68888") == 1:
s = "a"
with open(filename, 'w') as fp:
fp.writelines(s + ",")
for i in range(68888):
fp.writelines(s)
else:
with open(filename, 'w') as fp:
fp.write(context.text)
else:
num_int = int(num)
with open(filename, 'w') as fp:
col1 = 1
col2 = col1 + 1
col3 = "abcd"
col4 = "1234"
for i in range(num_int):
data = str(col1)+','+str(col2)+','+str(col3)+','+str(col4)
fp.write(data + '\n')
col1= col1+1
col2 = col2+1
# cp file to dble
dble_node = get_node("dble-1")
remote_file = "{0}/dble/{1}".format(dble_node.install_dir,filename)
context.ssh_sftp.sftp_put(filename, remote_file)
# create file in compare mysql
compare_mysql_node = get_node("mysql")
compare_mysql_sftp = compare_mysql_node.sftp_conn
remote_file = "{0}/data/{1}".format(compare_mysql_node.install_path, filename)
compare_mysql_sftp.sftp_put(filename, remote_file)
def step_impl(context, filename, hostname):
node = get_node(hostname)
local_file = filename
only_filename=local_file.split("/")[-1]
logger.debug("filename is ".format(only_filename))
remote_file = "{0}/{1}".format(node.install_dir,only_filename)
node.sftp_conn.sftp_put(local_file, remote_file)
def step_impl(context, host_name, result_var=None):
linux_cmd = context.text
assert linux_cmd, "expect linux command not null,but it is"
node = get_node(host_name)
# replace all vars in linux_cmd with corresponding node attribute value, node attr var in {} mode
node_vars = re.findall(r'\{node:(.*?)\}', linux_cmd, re.I)
logger.debug("debug node attr vars: {}".format(node_vars))
for var in node_vars:
linux_cmd = linux_cmd.replace("{node:" + var + "}",
str(getattr(node, var)))
# replace all vars in linux_cmd with corresponding context attribute value, context attr var in %% mode
context_vars = re.findall(r'\{context:(.*?)\}', linux_cmd, re.I)
logger.debug("debug context attr vars: {}".format(context_vars))
for var in context_vars:
linux_cmd = linux_cmd.replace("{context:" + var + "}",
getattr(context, var))
rc, sto, ste = node.ssh_conn.exec_command(linux_cmd)
assert len(ste) == 0, "execute linux cmd {} failed for {}".format(
linux_cmd, ste)
if result_var:
sto_list = sto.split("\n")
setattr(context, result_var, sto_list)
def step_impl(context, btraceScript, host):
node = get_node(host)
sshClient = node.ssh_conn
isBtraceRunning = check_btrace_running(sshClient, btraceScript)
context.logger.info("isBtraceRunning:{0} before try to run {1}".format(isBtraceRunning, btraceScript))
if not isBtraceRunning:
sftpClient = get_sftp(host)
localFile = "assets/{0}".format(btraceScript)
remoteFile = "{0}/dble/{1}".format(node.install_dir, btraceScript)
sftpClient.sftp_put(localFile, remoteFile)
global btrace_threads
thd = Thread(target=run_btrace_script, args=(sshClient, remoteFile), name=btraceScript)
btrace_threads.append(thd)
thd.setDaemon(True)
thd.start()
# make sure the btrace is working
check_btrace_status_count = 0
while check_btrace_status_count < 5:
try:
btraceRunningSuccess = check_btrace_running(sshClient, btraceScript)
if btraceRunningSuccess:
context.logger.debug("isBtraceRunning: {0} after try to run {1}, total sleep {2} seconds waiting for the btrace started".format(btraceRunningSuccess, btraceScript, check_btrace_status_count * 2))
break
else:
time.sleep(2)
check_btrace_status_count = check_btrace_status_count + 1
except Exception as e:
raise
def step_impl(context, host_name, result_var=None, exception_var=None):
linux_cmd = context.text
assert linux_cmd, "expect linux command not null,but it is"
if host_name == "behave":
node = None
else:
node = get_node(host_name)
# replace all vars in linux_cmd with corresponding node attribute value, node attr var in {} mode
node_vars = re.findall(r'\{node:(.*?)\}', linux_cmd, re.I)
logger.debug("debug node attr vars: {}".format(node_vars))
for var in node_vars:
linux_cmd = linux_cmd.replace("{node:" + var + "}", str(getattr(node, var)))
# replace all vars in linux_cmd with corresponding context attribute value, context attr var in %% mode
context_vars = re.findall(r'\{context:(.*?)\}', linux_cmd, re.I)
logger.debug("debug context attr vars: {}".format(context_vars))
for var in context_vars:
linux_cmd = linux_cmd.replace("{context:" + var + "}", getattr(context, var))
if node:
rc, sto, ste = node.ssh_conn.exec_command(linux_cmd)
if exception_var:
assert_that(str(ste), contains_string(str(exception_var)), "expect execute linux cmd {} failed for {}, real err: {}".format(linux_cmd, exception_var, ste))
else:
assert len(ste) == 0, "execute linux cmd {} failed for {}".format(linux_cmd, ste)
else:
status = os.system(linux_cmd)
assert status == 0, "cmd {} failed".format(linux_cmd)
if result_var:
sto_list = sto.splitlines()
setattr(context, result_var, sto_list)
def dble_mng_connect_test(context, ip):
node = get_node("dble-1")
user = node.manager_user
passwd = str(node.manager_password)
port = node.manager_port
conn = None
isSuccess = False
max_try = 5
while conn is None:
try:
conn = DBUtil(ip, user, passwd, '', port, context)
except MySQLdb.Error as e:
context.logger.info("connect to {0} err:{1}".format(ip, e))
conn = None
finally:
max_try -= 1
if max_try < 0 and conn is None: break
if conn is not None:
isSuccess = True
conn.close()
break
sleep(5)
assert_that(
isSuccess, "connect test to {0}:9066 failed after {1} seconds".format(
ip, 5 * max_try))
context.logger.info("create connection to dble 9066 success")
def step_impl(context, hostname, logfile):
rs = context.table
node = get_node(hostname)
logpath = "{0}/dble/logs/{1}".format(node.install_dir, logfile)
cmd = "cat {0}".format(logpath)
ssh_client = node.ssh_conn
rc, sto, ste = ssh_client.exec_command(cmd)
assert len(ste) == 0, "cat dble.log failed for: {0}".format(ste[0:200])
context.logger.debug("show warn content")
for row in rs:
level = row[1] # 1 is level
detail = row[2] # 2 is detail
if level == "WARNING":
# if (detail.rfind('$') != -1):
# dble_version = context.cfg_dble['ftp_path'].split('/')[-2]
# detail = detail.replace("${version}", dble_version)
detail = detail.replace("[", "\[")
detail = detail.replace("]", "\]")
str_to_find = "WARN \[.*?\].*{0}".format(detail)
found_in_log = re.search(str_to_find, sto,
flags=re.IGNORECASE) is not None
assert found_in_log, "warning {0} not found in log".format(row)
context.logger.debug("warning is found in log:{0}".format(row))
else:
context.logger.debug(
"row:{0}, is not warning, not search".format(row))
def add_file(context, mapFile):
node = get_node("dble-1")
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = str(context.text)
cmd = "echo '{0}' > {1}".format(text, targetFile)
rc, sto, err = node.ssh_conn.exec_command(cmd)
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def step_impl(context,mapFile,hostname):
node = get_node(hostname)
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = str(context.text)
cmd = "echo '{0}' > {1}".format(text, targetFile)
ssh = node.ssh_conn
rc, sto, err = ssh.exec_command(cmd)
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def get_encrypt(context, string):
node = get_node("dble-1")
cmd = "source /etc/profile && sh {0}/dble/bin/encrypt.sh {1}".format(
node.install_dir, string)
rc, sto, ste = node.ssh_conn.exec_command(cmd)
return sto.split('\n')[1]
def step_impl(context, hostname, file):
node = get_node(hostname)
path = "{0}".format(file)
print("{0}".format(file))
cmd = "cat {0}".format(path)
ssh_client = node.ssh_conn
rc, sto, ste = ssh_client.exec_command(cmd)
print("rc:{0}; sto:{1}; ste:{2}\n".format(rc, sto, ste))
assert len(sto) == 0, "cat file is not empty!"
assert len(ste) == 0, "cat file failed for: {0}".format(ste[0:200])
def step_impl(context, host):
node = get_node(host)
context.logger.debug("btrace is running, start query!!!")
time.sleep(5)
for row in context.table:
query_meta = QueryMeta(row.as_dict(), "admin", node)
cmd = u"nohup mysql -u{} -p{} -P{} -c -e\"{}\" >/tmp/dble_query.log 2>&1 &".format(query_meta.user, query_meta.passwd, query_meta.port, query_meta.sql)
rc, sto, ste = node.ssh_conn.exec_command(cmd)
assert len(ste) == 0, "impossible err occur"
def step_impl(context,time_param,curTime,mapFile,hostname):
node = get_node(hostname)
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = "{0}={1}".format(time_param,getattr(context,curTime)[0][0])
context.logger.info("{0} = {1}".format(time_param,getattr(context,curTime)[0][0]))
sed_cmd_str = "sed -i '$a {0}' {1}".format(text,targetFile)
ssh = node.ssh_conn
rc, sto, err = ssh.exec_command(sed_cmd_str)
logger.debug("execute cmd: {0}\n".format(sed_cmd_str))
context.logger.info("execute cmd: {0}".format(sed_cmd_str))
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def get_compare_conn(context, default_db="schema1"):
node = get_node("mysql")
m_ip = node.ip
m_port = node.mysql_port
m_user = node.mysql_user
m_passwd = node.mysql_password
conn_mysql = DBUtil(m_ip, m_user, m_passwd, default_db, m_port, context)
conn_dble = get_dble_conn(context, default_db)
return conn_mysql, conn_dble
def reset_zk_nodes(context):
if not hasattr(context, "reset_zk_time"):
context.reset_zk_time = 0
node = get_node("dble-1")
ssh_client = node.ssh_conn
resetCmd = "cd {0}/zookeeper/bin && sh zkCli.sh deleteall /dble".format(
node.install_dir)
rc, sto, ste = ssh_client.exec_command(resetCmd)
if context.reset_zk_time < 3:
context.reset_zk_time = context.reset_zk_time + 1
reset_zk_nodes(context)
def step_impl(context, cmd, result, host):
node = get_node(host)
ip = node._ip
port = node._mysql_port
user = "******"
passwd = "111111"
db = ""
conn = DBUtil(ip, user, passwd, db, port, context)
if hasattr(context, result):
for r in getattr(context, result):
adminsql = "{0} {1}".format(cmd, r[3])
conn.query(adminsql)
def step_impl(context, btraceScript, host, num_expr="==1"):
node = get_node(host)
sshClient = node.ssh_conn
remoteFile = "{0}/dble/{1}".format(node.install_dir, btraceScript)
try:
int(num_expr)
num_expr = "=={}".format(num_expr)
except Exception as e:
context.logger.debug("num already in expr")
check_btrace_output(sshClient, remoteFile, context.text.strip(), context, num_expr)
def step_impl(context, mysqlID, host):
id = getattr(context, mysqlID)
node = get_node(host)
ip = node._ip
user = node.mysql_user
passwd = node.mysql_password
port = node.mysql_port
db = ''
sql = "kill {0}".format(id)
conn = DBUtil(ip, user, passwd, db, port, context)
res, err = conn.query(sql)
assert err is None, "kill connection is failed: {0}".format(err)
def get_admin_conn(context, user="", passwd=""):
node = get_node("dble-1")
if len(user.strip()) == 0:
user = node.manager_user
if len(passwd.strip()) == 0:
passwd = str(node.manager_password)
conn = None
try:
conn = DBUtil(node.ip, user, passwd, "", node.manager_port, context)
except MySQLdb.Error as e:
assert False, "create manager conn meets error:{0}".format(e.args)
return conn
def step_impl(context, host, result, mysqlID):
session_list = getattr(context, result)
list_session = filter(lambda x: x, session_list[0][2].split("]"))
# context.logger.info("list_session is {0}".format(list_session))
ip = get_node(host)._ip
pattern = re.compile(ip)
for i in list_session:
matchObj1 = re.search(pattern, i)
if matchObj1:
matchObj2 = re.search("mysqlId = ([0-9]*)", i)
break
assert matchObj2, "not found mysqlID in {0}".format(host)
setattr(context, mysqlID, matchObj2.group(1))
context.logger.info("mysqlID is {0}".format(matchObj2.group(1)))
def step_impl(context, var_version):
node = get_node("dble-1")
cmd_server_version = "grep '<dble:server' {0}/dble/conf/server_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
cmd_schema_version = "grep '<dble:schema' {0}/dble/conf/schema_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
cmd_rule_version = "grep '<dble:rule' {0}/dble/conf/rule_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
rc1, sto1, ste1 = context.ssh_client.exec_command(cmd_server_version)
rc2, sto2, ste2 = context.ssh_client.exec_command(cmd_schema_version)
rc3, sto3, ste3 = context.ssh_client.exec_command(cmd_rule_version)
assert sto1 == sto2 == sto3, "versions in server_template.xml schema_template.xml rule_template.xml are not the same"
setattr(context, var_version, sto1)
def exec_admin_cmd(context, adminsql, user="", passwd="", result=""):
node = get_node("dble-1")
if len(user.strip()) == 0:
user = node.manager_user
if len(passwd.strip()) == 0:
passwd = str(node.manager_password)
if len(result.strip()) != 0:
adminsql = "{0} {1}".format(adminsql, getattr(context, result)[0][0])
if context.text: expect = context.text
else: expect = "success"
context.execute_steps(u"""
Then execute sql in "dble-1" in "admin" mode
| user | passwd | sql | expect |
| {0} | {1} | {2} | {3} |
""".format(user, passwd, adminsql, expect))
def check_restart_dble_failed(context, hostname):
node = get_node(hostname)
start_dble_in_node(context, node, False)
def unistall_dble_by_hostname(context, hostname):
node = get_node(hostname)
uninstall_dble_in_node(context, node)
def step_impl(context, hostname, manager_port=None):
node = get_node(hostname)
if manager_port:
node.manager_port = int(manager_port)
restart_dble(context, node)
def step_impl(context, nodeName):
node = get_node(nodeName)
replace_config_in_node(context, node)
def create_mysql_object(self, id):
mysql_meta = get_node(id)
return MySQLObject(mysql_meta)
def create_dble_object(self, id):
dble_meta = get_node(id)
return DbleObject(dble_meta)
