def step_impl(context, filename, hostname):
node = get_node(hostname)
local_file = filename
only_filename=local_file.split("/")[-1]
logger.debug("filename is ".format(only_filename))
remote_file = "{0}/{1}".format(node.install_dir,only_filename)
node.sftp_conn.sftp_put(local_file, remote_file)
def test_schema_permission(context):
dble_conn = get_dble_conn(context)
node = get_node("dble-1")
for row in context.table:
# prepare table
test_conn = DBUtil(node.ip, row['user'], row['password'], row['schema'],
node.client_port, context)
sql = "drop table if exists {0}".format(row['table'])
test_conn.query(sql)
sql = "create table {0}(id int, data varchar(10))".format(row['table'])
test_conn.query(sql)
sql = "insert into {0} values (1,'aaa'),(2,'bbb'),(3,'ccc'),(4,'ddd')".format(row['table'])
dble_conn.query(sql)
# prepare sql
value = row['dml']
LOGGER.info("dml:{0}".format(value))
del_sql = "delete from {0} where id = 1".format(row['table'])
sel_sql = "select * from {0}".format(row['table'])
upd_sql = "update {0} set data='AAA' where id = 1".format(row['table'])
ins_sql = "insert into {0} values (5, 'eee')".format(row['table'])
lack_priv_msg = "The statement DML privilege check is not passed"
sqls_in_priv_order = [ins_sql,upd_sql,sel_sql,del_sql]
# do test
for priv, sql in zip(value, sqls_in_priv_order):
res, errs = test_conn.query(sql)
if priv == "1":
assert_that(errs, is_(None))
else:
assert_that(errs[1], contains_string(lack_priv_msg))
test_conn.close()
dble_conn.close()
def step_impl(context,hostname, logfile):
rs = context.table
node = get_node(hostname)
logpath = "{0}/dble/logs/{1}".format(node.install_dir, logfile)
cmd = "cat {0}".format(logpath)
ssh_client = node.ssh_conn
rc, sto, ste = ssh_client.exec_command(cmd)
assert len(ste)==0, "cat dble.log failed for: {0}".format(ste[0:200])
context.logger.debug("show warn content")
for row in rs:
level = row[1] # 1 is level
detail = row[2] # 2 is detail
if level == "WARNING":
# if (detail.rfind('$') != -1):
# dble_version = context.cfg_dble['ftp_path'].split('/')[-2]
# detail = detail.replace("${version}", dble_version)
detail = detail.replace("[", "\[")
detail = detail.replace("]", "\]")
str_to_find = "WARN \[.*?\].*{0}".format(detail)
found_in_log = re.search(str_to_find, sto, flags=re.IGNORECASE) is not None
assert found_in_log, "warning {0} not found in log".format(row)
context.logger.debug("warning is found in log:{0}".format(row))
else:
context.logger.debug("row:{0}, is not warning, not search".format(row))
def step_impl(context, mapFile, hostname):
node = get_node(hostname)
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = str(context.text)
cmd = "echo '{0}' > {1}".format(text, targetFile)
ssh = node.ssh_conn
rc, sto, err = ssh.exec_command(cmd)
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def add_file(context, mapFile):
node = get_node("dble-1")
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = str(context.text)
cmd = "echo '{0}' > {1}".format(text, targetFile)
rc, sto, err = node.ssh_conn.exec_command(cmd)
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def get_encrypt(context, string):
node = get_node("dble-1")
cmd = "source /etc/profile && sh {0}/dble/bin/encrypt.sh {1}".format(
node.install_dir, string)
rc, sto, ste = node.ssh_conn.exec_command(cmd)
return sto.split('\n')[1]
def test_readonly_schema(context):
text = json.loads(context.text)
dble_conn = get_dble_conn(context)
node = get_node("dble-1")
for item in text:
test_conn = DBUtil(node.ip, item['user'], item['password'],
item['schema'], node.client_port, context)
sql = "drop table if exists {0}".format(item['table'])
dble_conn.query(sql)
sql = "create table {0}(id int, data varchar(10))".format(
item['table'])
dble_conn.query(sql)
sql = "insert into {0} values (1,'aaa'),(2,'bbb'),(3,'ccc'),(4,'ddd')".format(
item['table'])
dble_conn.query(sql)
permission = []
value = int(item['dml'])
LOGGER.info("dml: {0}".format(value / 1))
for i in range(4):
if int(value % 10) == 1:
if i == 0:
permission.append("DELETE")
if i == 1:
permission.append("SELECT")
if i == 2:
permission.append("UPDATE")
if i == 3:
permission.append("INSERT")
value = value / 10
LOGGER.info("permission: {0}".format(permission))
del_sql = "delete from {0} where id = 1".format(item['table'])
sel_sql = "select * from {0}".format(item['table'])
upd_sql = "update {0} set data='AAA' where id = 1".format(
item['table'])
ins_sql = "insert into {0} values (5, 'eee')".format(item['table'])
if "DELETE" in permission:
res, errs = test_conn.query(del_sql)
assert_that(errs[1], contains_string('User READ ONLY'))
if "SELECT" in permission:
res, errs = test_conn.query(sel_sql)
assert_that(errs, is_(None))
else:
res, errs = test_conn.query(sel_sql)
assert_that(
errs[1],
contains_string(
'The statement DML privilege check is not passed'))
if "UPDATE" in permission:
res, errs = test_conn.query(upd_sql)
assert_that(errs[1], contains_string('User READ ONLY'))
if "INSERT" in permission:
res, errs = test_conn.query(ins_sql)
assert_that(errs[1], contains_string('User READ ONLY'))
test_conn.close()
dble_conn.close()
def step_impl(context,hostname, file):
node = get_node(hostname)
path = "{0}".format(file)
print ("{0}".format(file))
cmd = "cat {0}".format(path)
ssh_client = node.ssh_conn
rc, sto, ste = ssh_client.exec_command(cmd)
print ("rc:{0}; sto:{1}; ste:{2}\n".format(rc, sto, ste))
assert len(sto)==0, "cat file is not empty!"
assert len(ste)==0, "cat file failed for: {0}".format(ste[0:200])
def step_impl(context,curTime,mapFile,hostname):
node = get_node(hostname)
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = "START_TIME={0}".format(getattr(context,curTime)[0][0])
context.logger.info("START_TIME = {0}".format(getattr(context,curTime)[0][0]))
sed_cmd_str = "sed -i '/START_TIME/c {0}' {1}".format(text,targetFile)
ssh = node.ssh_conn
rc, sto, err = ssh.exec_command(sed_cmd_str)
context.logger.info("execute cmd: {0}".format(sed_cmd_str))
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def step_impl(context, filename):
logger.debug("*** debug context.text:{0}".format(context.text))
text = context.text
# remove old file in behave resides server
if os.path.exists(filename):
os.remove(filename)
if cmp(text, '10000+lines') == 0:
with open(filename, 'w') as fp:
col1 = 1
col2 = col1 + 1
col3 = "abcd"
col4 = "1234"
for i in xrange(15000):
data = str(col1) + ',' + str(col2) + ',' + str(
col3) + ',' + str(col4)
fp.write(data + '\n')
col1 = col1 + 1
col2 = col2 + 1
elif text.find("68888") == 1:
s = "a"
with open(filename, 'w') as fp:
fp.writelines(s + ",")
for i in xrange(68888):
fp.writelines(s)
else:
with open(filename, 'w') as fp:
fp.write(context.text)
# cp file to dble
dble_node = get_node("dble-1")
remote_file = "{0}/dble/{1}".format(dble_node.install_dir, filename)
context.ssh_sftp.sftp_put(filename, remote_file)
# create file in compare mysql
compare_mysql_node = get_node("mysql")
compare_mysql_sftp = compare_mysql_node.sftp_conn
remote_file = "{0}/data/{1}".format(compare_mysql_node.install_path,
filename)
compare_mysql_sftp.sftp_put(filename, remote_file)
def step_impl(context, host):
node = get_node(host)
sshClient = node.ssh_conn
context.logger.debug("btrace is running, start query!!!")
time.sleep(5)
for row in context.table:
query_meta = QueryMeta(row.as_dict(), "user", node)
cmd = u"nohup mysql -u{} -p{} -P{} -c -D{} -e'{}' >/tmp/dble_user_query.log 2>&1 &".format(query_meta.user,query_meta.passwd,query_meta.port,query_meta.db,query_meta.sql)
rc, sto, ste = sshClient.exec_command(cmd)
assert len(ste)==0, "impossible err occur"
def reset_zk_nodes(context):
if not hasattr(context, "reset_zk_time"):
context.reset_zk_time = 0
node = get_node("dble-1")
ssh_client = node.ssh_conn
resetCmd = "cd {0}/zookeeper/bin && sh zkCli.sh deleteall /dble".format(
node.install_dir)
rc, sto, ste = ssh_client.exec_command(resetCmd)
if context.reset_zk_time < 3:
context.reset_zk_time = context.reset_zk_time + 1
reset_zk_nodes(context)
def step_impl(context,time_param,curTime,mapFile,hostname):
node = get_node(hostname)
targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile)
text = "{0}={1}".format(time_param,getattr(context,curTime)[0][0])
context.logger.info("{0} = {1}".format(time_param,getattr(context,curTime)[0][0]))
sed_cmd_str = "sed -i '/{0}/c {1}' {2}".format(time_param,text,targetFile)
ssh = node.ssh_conn
rc, sto, err = ssh.exec_command(sed_cmd_str)
print (("execute cmd: {0}\n\n\n".format(sed_cmd_str)))
logger.debug("execute cmd: {0}\n".format(sed_cmd_str))
context.logger.info("execute cmd: {0}".format(sed_cmd_str))
assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def step_impl(context, btraceScript, host, num_expr="==1"):
node = get_node(host)
sshClient = node.ssh_conn
remoteFile = "{0}/dble/{1}".format(node.install_dir, btraceScript)
try:
int(num_expr)
num_expr = "=={}".format(num_expr)
except Exception as e:
context.logger.debug("num already in expr")
check_btrace_output(sshClient, remoteFile, context.text.strip(), context, num_expr)
def get_admin_conn(context, user="", passwd=""):
node = get_node("dble-1")
if len(user.strip()) == 0:
user = node.manager_user
if len(passwd.strip()) == 0:
passwd = str(node.manager_password)
conn = None
try:
conn = DBUtil(node.ip, user, passwd, "", node.manager_port, context)
except MySQLdb.Error, e:
assert False, "create manager conn meets error:{0}".format(e.args)
def step_impl(context, mysqlID, host):
id = getattr(context, mysqlID)
node = get_node(host)
ip = node._ip
user = node.mysql_user
passwd = node.mysql_password
port = node.mysql_port
db = ''
sql = "kill {0}".format(id)
conn = DBUtil(ip, user, passwd, db, port, context)
res, err = conn.query(sql)
assert err is None, "kill connection is failed: {0}".format(err)
def step_impl(context, cmd, result, host):
node = get_node(host)
ip = node._ip
port = node._mysql_port
user = "******"
passwd = "111111"
db = ""
conn = DBUtil(ip, user, passwd, db, port, context)
if hasattr(context, result):
for r in getattr(context, result):
adminsql = "{0} {1}".format(cmd, r[3])
conn.query(adminsql)
def step_impl(context, host1, role, host2, oscmd='cd /usr/local/mysql/data'):
user = ''
password = ''
port = ''
if host1.startswith('dble'):
node = get_node(host1)
if role == "admin":
user = node.manager_user
password = node.manager_password
port = node.manager_port
else:
user = node.client_user
password = node.client_password
port = node.client_port
else:
node = get_node(host1)
user = node.mysql_user
password = node.mysql_password
port = node.mysql_port
ip = node.ip
if host2.startswith('dble'):
ssh = get_ssh(host2)
else:
ssh = get_ssh(host2)
sql_cmd_str = context.text.strip()
sql_cmd_list = sql_cmd_str.splitlines()
context.logger.info("sql list: {0}".format(sql_cmd_list))
for sql_cmd in sql_cmd_list:
cmd = '{5} && mysql -h{0} -u{1} -p{2} -P{3} -c -e"{4}"'.format(
ip, user, password, port, sql_cmd, oscmd)
stdin, stdout, stderr = ssh.exec_command(cmd)
context.logger.info("execute cmd:{0}".format(cmd))
stderr = stderr.lower()
assert stderr.find("error") == -1, "execute cmd: {0} err:{1}".format(
cmd, stderr)
time.sleep(3)
def step_impl(context, host_name, target_file):
"""
dble config files is change, but config file in behave is not, then scp the changed files out first, then apply user's config change, last, put it back
:param context:
:param host_name:
:param target_file: fullpath file
:return:
"""
node = get_node("dble-1")
local_file = get_abs_path(context, target_file)
source_remote_file = "{0}/dble/conf/{1}".format(node.install_dir,
target_file)
node.sftp_conn.sftp_get(source_remote_file, local_file)
def step_impl(context, host, result, mysqlID):
session_list = getattr(context, result)
list_session = filter(lambda x: x,
session_list[0][2].split("MySQLConnection"))
ip = get_node(host)._ip
pattern = re.compile(ip)
for i in list_session:
matchObj1 = re.search(pattern, i)
if matchObj1:
matchObj2 = re.search("mysqlId=([0-9]*)", i)
break
assert matchObj2, "not found mysqlID in {0}".format(host)
setattr(context, mysqlID, matchObj2.group(1))
context.logger.info("mysqlID is {0}".format(matchObj2.group(1)))
def step_impl(context, var_version):
node = get_node("dble-1")
cmd_server_version = "grep '<dble:server' {0}/dble/conf/server_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
cmd_schema_version = "grep '<dble:schema' {0}/dble/conf/schema_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
cmd_rule_version = "grep '<dble:rule' {0}/dble/conf/rule_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format(
node.install_dir)
rc1, sto1, ste1 = context.ssh_client.exec_command(cmd_server_version)
rc2, sto2, ste2 = context.ssh_client.exec_command(cmd_schema_version)
rc3, sto3, ste3 = context.ssh_client.exec_command(cmd_rule_version)
assert sto1 == sto2 == sto3, "versions in server_template.xml schema_template.xml rule_template.xml are not the same"
setattr(context, var_version, sto1)
def step_impl(context, btraceScript, host):
node = get_node(host)
sshClient = node.ssh_conn
isBtraceRunning = check_btrace_running(sshClient,btraceScript)
context.logger.info("isBtraceRunning:{0} before try to run {1}".format(isBtraceRunning, btraceScript))
if not isBtraceRunning:
sftpClient = get_sftp(host)
localFile = "assets/{0}".format(btraceScript)
remoteFile = "{0}/dble/{1}".format(node.install_dir,btraceScript)
sftpClient.sftp_put(localFile, remoteFile)
global btrace_threads
thd = Thread(target=run_btrace_script, args=(sshClient, remoteFile),name=btraceScript)
btrace_threads.append(thd)
thd.setDaemon(True)
thd.start()
# for btrace into work
time.sleep(2)
def check_restart_dble_failed(context, hostname):
node = get_node(hostname)
start_dble_in_node(context, node, False)
def get_dble_connect(context, host_name, default_db="schema1"):
node = get_node(host_name)
conn = DBUtil(node.ip, node.client_user, node.client_password, default_db,
node.client_port, context)
return conn
def step_impl(context, hostname):
node = get_node(hostname)
restart_dble(context, node)
def upload_and_replace_conf(context, filename, host='dble-1'):
node = get_node(host)
local_file = get_abs_path(context, filename)
remote_file = "{0}/dble/conf/{1}".format(node.install_dir, filename)
node.sftp_conn.sftp_put(local_file, remote_file)
def get_dble_conn(context, default_db="schema1", node=None):
if node is None:
node = get_node("dble-1")
conn = DBUtil(node.ip, node.client_user, node.client_password, default_db,
node.client_port, context)
return conn
def unistall_dble_by_hostname(context, hostname):
node = get_node(hostname)
uninstall_dble_in_node(context, node)
def step_impl(context, nodeName):
node = get_node(nodeName)
replace_config_in_node(context, node)
def install_dble_in_host(context, hostname):
node = get_node(hostname)
install_dble_in_node(context, node)
