def step_impl(context, filename, hostname): node = get_node(hostname) local_file = filename only_filename=local_file.split("/")[-1] logger.debug("filename is ".format(only_filename)) remote_file = "{0}/{1}".format(node.install_dir,only_filename) node.sftp_conn.sftp_put(local_file, remote_file)
def test_schema_permission(context): dble_conn = get_dble_conn(context) node = get_node("dble-1") for row in context.table: # prepare table test_conn = DBUtil(node.ip, row['user'], row['password'], row['schema'], node.client_port, context) sql = "drop table if exists {0}".format(row['table']) test_conn.query(sql) sql = "create table {0}(id int, data varchar(10))".format(row['table']) test_conn.query(sql) sql = "insert into {0} values (1,'aaa'),(2,'bbb'),(3,'ccc'),(4,'ddd')".format(row['table']) dble_conn.query(sql) # prepare sql value = row['dml'] LOGGER.info("dml:{0}".format(value)) del_sql = "delete from {0} where id = 1".format(row['table']) sel_sql = "select * from {0}".format(row['table']) upd_sql = "update {0} set data='AAA' where id = 1".format(row['table']) ins_sql = "insert into {0} values (5, 'eee')".format(row['table']) lack_priv_msg = "The statement DML privilege check is not passed" sqls_in_priv_order = [ins_sql,upd_sql,sel_sql,del_sql] # do test for priv, sql in zip(value, sqls_in_priv_order): res, errs = test_conn.query(sql) if priv == "1": assert_that(errs, is_(None)) else: assert_that(errs[1], contains_string(lack_priv_msg)) test_conn.close() dble_conn.close()
def step_impl(context,hostname, logfile): rs = context.table node = get_node(hostname) logpath = "{0}/dble/logs/{1}".format(node.install_dir, logfile) cmd = "cat {0}".format(logpath) ssh_client = node.ssh_conn rc, sto, ste = ssh_client.exec_command(cmd) assert len(ste)==0, "cat dble.log failed for: {0}".format(ste[0:200]) context.logger.debug("show warn content") for row in rs: level = row[1] # 1 is level detail = row[2] # 2 is detail if level == "WARNING": # if (detail.rfind('$') != -1): # dble_version = context.cfg_dble['ftp_path'].split('/')[-2] # detail = detail.replace("${version}", dble_version) detail = detail.replace("[", "\[") detail = detail.replace("]", "\]") str_to_find = "WARN \[.*?\].*{0}".format(detail) found_in_log = re.search(str_to_find, sto, flags=re.IGNORECASE) is not None assert found_in_log, "warning {0} not found in log".format(row) context.logger.debug("warning is found in log:{0}".format(row)) else: context.logger.debug("row:{0}, is not warning, not search".format(row))
def step_impl(context, mapFile, hostname): node = get_node(hostname) targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile) text = str(context.text) cmd = "echo '{0}' > {1}".format(text, targetFile) ssh = node.ssh_conn rc, sto, err = ssh.exec_command(cmd) assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def add_file(context, mapFile): node = get_node("dble-1") targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile) text = str(context.text) cmd = "echo '{0}' > {1}".format(text, targetFile) rc, sto, err = node.ssh_conn.exec_command(cmd) assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def get_encrypt(context, string): node = get_node("dble-1") cmd = "source /etc/profile && sh {0}/dble/bin/encrypt.sh {1}".format( node.install_dir, string) rc, sto, ste = node.ssh_conn.exec_command(cmd) return sto.split('\n')[1]
def test_readonly_schema(context): text = json.loads(context.text) dble_conn = get_dble_conn(context) node = get_node("dble-1") for item in text: test_conn = DBUtil(node.ip, item['user'], item['password'], item['schema'], node.client_port, context) sql = "drop table if exists {0}".format(item['table']) dble_conn.query(sql) sql = "create table {0}(id int, data varchar(10))".format( item['table']) dble_conn.query(sql) sql = "insert into {0} values (1,'aaa'),(2,'bbb'),(3,'ccc'),(4,'ddd')".format( item['table']) dble_conn.query(sql) permission = [] value = int(item['dml']) LOGGER.info("dml: {0}".format(value / 1)) for i in range(4): if int(value % 10) == 1: if i == 0: permission.append("DELETE") if i == 1: permission.append("SELECT") if i == 2: permission.append("UPDATE") if i == 3: permission.append("INSERT") value = value / 10 LOGGER.info("permission: {0}".format(permission)) del_sql = "delete from {0} where id = 1".format(item['table']) sel_sql = "select * from {0}".format(item['table']) upd_sql = "update {0} set data='AAA' where id = 1".format( item['table']) ins_sql = "insert into {0} values (5, 'eee')".format(item['table']) if "DELETE" in permission: res, errs = test_conn.query(del_sql) assert_that(errs[1], contains_string('User READ ONLY')) if "SELECT" in permission: res, errs = test_conn.query(sel_sql) assert_that(errs, is_(None)) else: res, errs = test_conn.query(sel_sql) assert_that( errs[1], contains_string( 'The statement DML privilege check is not passed')) if "UPDATE" in permission: res, errs = test_conn.query(upd_sql) assert_that(errs[1], contains_string('User READ ONLY')) if "INSERT" in permission: res, errs = test_conn.query(ins_sql) assert_that(errs[1], contains_string('User READ ONLY')) test_conn.close() dble_conn.close()
def step_impl(context,hostname, file): node = get_node(hostname) path = "{0}".format(file) print ("{0}".format(file)) cmd = "cat {0}".format(path) ssh_client = node.ssh_conn rc, sto, ste = ssh_client.exec_command(cmd) print ("rc:{0}; sto:{1}; ste:{2}\n".format(rc, sto, ste)) assert len(sto)==0, "cat file is not empty!" assert len(ste)==0, "cat file failed for: {0}".format(ste[0:200])
def step_impl(context,curTime,mapFile,hostname): node = get_node(hostname) targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile) text = "START_TIME={0}".format(getattr(context,curTime)[0][0]) context.logger.info("START_TIME = {0}".format(getattr(context,curTime)[0][0])) sed_cmd_str = "sed -i '/START_TIME/c {0}' {1}".format(text,targetFile) ssh = node.ssh_conn rc, sto, err = ssh.exec_command(sed_cmd_str) context.logger.info("execute cmd: {0}".format(sed_cmd_str)) assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def step_impl(context, filename): logger.debug("*** debug context.text:{0}".format(context.text)) text = context.text # remove old file in behave resides server if os.path.exists(filename): os.remove(filename) if cmp(text, '10000+lines') == 0: with open(filename, 'w') as fp: col1 = 1 col2 = col1 + 1 col3 = "abcd" col4 = "1234" for i in xrange(15000): data = str(col1) + ',' + str(col2) + ',' + str( col3) + ',' + str(col4) fp.write(data + '\n') col1 = col1 + 1 col2 = col2 + 1 elif text.find("68888") == 1: s = "a" with open(filename, 'w') as fp: fp.writelines(s + ",") for i in xrange(68888): fp.writelines(s) else: with open(filename, 'w') as fp: fp.write(context.text) # cp file to dble dble_node = get_node("dble-1") remote_file = "{0}/dble/{1}".format(dble_node.install_dir, filename) context.ssh_sftp.sftp_put(filename, remote_file) # create file in compare mysql compare_mysql_node = get_node("mysql") compare_mysql_sftp = compare_mysql_node.sftp_conn remote_file = "{0}/data/{1}".format(compare_mysql_node.install_path, filename) compare_mysql_sftp.sftp_put(filename, remote_file)
def step_impl(context, host): node = get_node(host) sshClient = node.ssh_conn context.logger.debug("btrace is running, start query!!!") time.sleep(5) for row in context.table: query_meta = QueryMeta(row.as_dict(), "user", node) cmd = u"nohup mysql -u{} -p{} -P{} -c -D{} -e'{}' >/tmp/dble_user_query.log 2>&1 &".format(query_meta.user,query_meta.passwd,query_meta.port,query_meta.db,query_meta.sql) rc, sto, ste = sshClient.exec_command(cmd) assert len(ste)==0, "impossible err occur"
def reset_zk_nodes(context): if not hasattr(context, "reset_zk_time"): context.reset_zk_time = 0 node = get_node("dble-1") ssh_client = node.ssh_conn resetCmd = "cd {0}/zookeeper/bin && sh zkCli.sh deleteall /dble".format( node.install_dir) rc, sto, ste = ssh_client.exec_command(resetCmd) if context.reset_zk_time < 3: context.reset_zk_time = context.reset_zk_time + 1 reset_zk_nodes(context)
def step_impl(context,time_param,curTime,mapFile,hostname): node = get_node(hostname) targetFile = "{0}/dble/conf/{1}".format(node.install_dir, mapFile) text = "{0}={1}".format(time_param,getattr(context,curTime)[0][0]) context.logger.info("{0} = {1}".format(time_param,getattr(context,curTime)[0][0])) sed_cmd_str = "sed -i '/{0}/c {1}' {2}".format(time_param,text,targetFile) ssh = node.ssh_conn rc, sto, err = ssh.exec_command(sed_cmd_str) print (("execute cmd: {0}\n\n\n".format(sed_cmd_str))) logger.debug("execute cmd: {0}\n".format(sed_cmd_str)) context.logger.info("execute cmd: {0}".format(sed_cmd_str)) assert_that(err, is_(''), "expect no err, but err is: {0}".format(err))
def step_impl(context, btraceScript, host, num_expr="==1"): node = get_node(host) sshClient = node.ssh_conn remoteFile = "{0}/dble/{1}".format(node.install_dir, btraceScript) try: int(num_expr) num_expr = "=={}".format(num_expr) except Exception as e: context.logger.debug("num already in expr") check_btrace_output(sshClient, remoteFile, context.text.strip(), context, num_expr)
def get_admin_conn(context, user="", passwd=""): node = get_node("dble-1") if len(user.strip()) == 0: user = node.manager_user if len(passwd.strip()) == 0: passwd = str(node.manager_password) conn = None try: conn = DBUtil(node.ip, user, passwd, "", node.manager_port, context) except MySQLdb.Error, e: assert False, "create manager conn meets error:{0}".format(e.args)
def step_impl(context, mysqlID, host): id = getattr(context, mysqlID) node = get_node(host) ip = node._ip user = node.mysql_user passwd = node.mysql_password port = node.mysql_port db = '' sql = "kill {0}".format(id) conn = DBUtil(ip, user, passwd, db, port, context) res, err = conn.query(sql) assert err is None, "kill connection is failed: {0}".format(err)
def step_impl(context, cmd, result, host): node = get_node(host) ip = node._ip port = node._mysql_port user = "******" passwd = "111111" db = "" conn = DBUtil(ip, user, passwd, db, port, context) if hasattr(context, result): for r in getattr(context, result): adminsql = "{0} {1}".format(cmd, r[3]) conn.query(adminsql)
def step_impl(context, host1, role, host2, oscmd='cd /usr/local/mysql/data'): user = '' password = '' port = '' if host1.startswith('dble'): node = get_node(host1) if role == "admin": user = node.manager_user password = node.manager_password port = node.manager_port else: user = node.client_user password = node.client_password port = node.client_port else: node = get_node(host1) user = node.mysql_user password = node.mysql_password port = node.mysql_port ip = node.ip if host2.startswith('dble'): ssh = get_ssh(host2) else: ssh = get_ssh(host2) sql_cmd_str = context.text.strip() sql_cmd_list = sql_cmd_str.splitlines() context.logger.info("sql list: {0}".format(sql_cmd_list)) for sql_cmd in sql_cmd_list: cmd = '{5} && mysql -h{0} -u{1} -p{2} -P{3} -c -e"{4}"'.format( ip, user, password, port, sql_cmd, oscmd) stdin, stdout, stderr = ssh.exec_command(cmd) context.logger.info("execute cmd:{0}".format(cmd)) stderr = stderr.lower() assert stderr.find("error") == -1, "execute cmd: {0} err:{1}".format( cmd, stderr) time.sleep(3)
def step_impl(context, host_name, target_file): """ dble config files is change, but config file in behave is not, then scp the changed files out first, then apply user's config change, last, put it back :param context: :param host_name: :param target_file: fullpath file :return: """ node = get_node("dble-1") local_file = get_abs_path(context, target_file) source_remote_file = "{0}/dble/conf/{1}".format(node.install_dir, target_file) node.sftp_conn.sftp_get(source_remote_file, local_file)
def step_impl(context, host, result, mysqlID): session_list = getattr(context, result) list_session = filter(lambda x: x, session_list[0][2].split("MySQLConnection")) ip = get_node(host)._ip pattern = re.compile(ip) for i in list_session: matchObj1 = re.search(pattern, i) if matchObj1: matchObj2 = re.search("mysqlId=([0-9]*)", i) break assert matchObj2, "not found mysqlID in {0}".format(host) setattr(context, mysqlID, matchObj2.group(1)) context.logger.info("mysqlID is {0}".format(matchObj2.group(1)))
def step_impl(context, var_version): node = get_node("dble-1") cmd_server_version = "grep '<dble:server' {0}/dble/conf/server_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format( node.install_dir) cmd_schema_version = "grep '<dble:schema' {0}/dble/conf/schema_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format( node.install_dir) cmd_rule_version = "grep '<dble:rule' {0}/dble/conf/rule_template.xml| grep -o 'version=\".*\"' | grep -o '[0-9]*\.[0-9]*'".format( node.install_dir) rc1, sto1, ste1 = context.ssh_client.exec_command(cmd_server_version) rc2, sto2, ste2 = context.ssh_client.exec_command(cmd_schema_version) rc3, sto3, ste3 = context.ssh_client.exec_command(cmd_rule_version) assert sto1 == sto2 == sto3, "versions in server_template.xml schema_template.xml rule_template.xml are not the same" setattr(context, var_version, sto1)
def step_impl(context, btraceScript, host): node = get_node(host) sshClient = node.ssh_conn isBtraceRunning = check_btrace_running(sshClient,btraceScript) context.logger.info("isBtraceRunning:{0} before try to run {1}".format(isBtraceRunning, btraceScript)) if not isBtraceRunning: sftpClient = get_sftp(host) localFile = "assets/{0}".format(btraceScript) remoteFile = "{0}/dble/{1}".format(node.install_dir,btraceScript) sftpClient.sftp_put(localFile, remoteFile) global btrace_threads thd = Thread(target=run_btrace_script, args=(sshClient, remoteFile),name=btraceScript) btrace_threads.append(thd) thd.setDaemon(True) thd.start() # for btrace into work time.sleep(2)
def check_restart_dble_failed(context, hostname): node = get_node(hostname) start_dble_in_node(context, node, False)
def get_dble_connect(context, host_name, default_db="schema1"): node = get_node(host_name) conn = DBUtil(node.ip, node.client_user, node.client_password, default_db, node.client_port, context) return conn
def step_impl(context, hostname): node = get_node(hostname) restart_dble(context, node)
def upload_and_replace_conf(context, filename, host='dble-1'): node = get_node(host) local_file = get_abs_path(context, filename) remote_file = "{0}/dble/conf/{1}".format(node.install_dir, filename) node.sftp_conn.sftp_put(local_file, remote_file)
def get_dble_conn(context, default_db="schema1", node=None): if node is None: node = get_node("dble-1") conn = DBUtil(node.ip, node.client_user, node.client_password, default_db, node.client_port, context) return conn
def unistall_dble_by_hostname(context, hostname): node = get_node(hostname) uninstall_dble_in_node(context, node)
def step_impl(context, nodeName): node = get_node(nodeName) replace_config_in_node(context, node)
def install_dble_in_host(context, hostname): node = get_node(hostname) install_dble_in_node(context, node)