def sync_ol_conf_to_local(conf_path): if os.path.exists(conf_path) == False: # print "tmp_ol_conf path not exists, mkdir -p" update_errorlog("[%s] %s\n" % (get_now_time(), "tmp_ol_conf path not exists, mkdir -p")) os.popen("mkdir -p " + conf_path) update_errorlog("[%s] start rsync ol_conf to local\n" % get_now_time()) rsync_path = online_path if (rsync_path[0:1] == "/"): rsync_path = rsync_path[1:] if (rsync_path[len(rsync_path) - 1:] != "/"): rsync_path = rsync_path + "/" rsync_path = rsync_path + "conf/" arg2 = conf_path if (conf_path[len(conf_path) - 1:] != "/"): arg2 = conf_path + "/" arg = "%s::%s" % (online_host, rsync_path) stdlog = "" errlog = "" asycmd = asycommands.TrAsyCommands(timeout=30 * 60) asycmd_list.append(asycmd) for iotype, line in asycmd.execute_with_data(['rsync', '-ravl', arg, arg2], shell=False): if (iotype is 1): stdlog += line + '\n' elif (iotype is 2): errlog += line + '\n' if (asycmd.return_code() != 0): update_errorlog("[%s] rsync ol_conf to local Error\n" % get_now_time()) update_errorlog(errlog) return 1 update_errorlog("[%s] rsync ol_conf to local success\n" % get_now_time()) return 0
def sggp_lanch(file_path, start_script, log): # rules: start_script must put pid in `PID` file: echo $! > PID # return a tuple(retcode, pid) # lanch(sggp_path, "start_qw_group.sh", -1, log) pid = list() asycmd = asycommands.TrAsyCommands(timeout=30) asycmd_list.append(asycmd) child = subprocess.Popen(['/bin/sh', start_script], shell=False, cwd=file_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) child.wait() if (child.returncode != 0): log.append(child.stderr.read()) return (-1, pid) for iotype, line in asycmd.execute_with_data( ['/bin/cat', file_path + "/PID"], shell=False): if (iotype == 1 and line != ""): try: pid.append(int(line)) except: continue if len(pid) == 0: return (-2, pid) proc = None for subpid in pid: try: proc = psutil.Process(subpid) except: log.append("process %d is not alive" % pid) return (-3, pid) return (0, pid)
def lanch(file_path, start_script, port, log): # rules: start_script must put pid in `PID` file: echo $! > PID # return a tuple(retcode, pid) # lanch(sggp_path, "start_qw_group.sh", -1, log) pid = -1 asycmd = asycommands.TrAsyCommands(timeout=30) asycmd_list.append(asycmd) child = subprocess.Popen(['/bin/sh', start_script], shell=False, cwd=file_path, stderr=subprocess.PIPE) child.wait() if (child.returncode != 0): log.append(child.stderr.read()) return (-1, pid) for iotype, line in asycmd.execute_with_data( ['/bin/cat', file_path + "/PID"], shell=False): if (iotype == 1 and line != ""): try: if (pid == -1): pid = int(line) else: tmp = int(line) pid = max(pid, tmp) except: continue if (pid == -1): return (-2, pid) proc = None try: proc = psutil.Process(pid) except: log.append("process %d is not alive" % pid) return (-3, pid) if (port is -1): return (0, pid) is_alive = True start_time = 0 proc_list.append(pid) while is_alive: try: conn_list = proc.connections() except: is_alive = False break listened = False for conn in conn_list: if (conn.status == "LISTEN" or conn.status == "NONE") and conn.laddr[1] == port: listened = True break if listened: break time.sleep(1) start_time += 1 if not is_alive: log.append("process start failed") proc_list.remove(pid) return (-3, pid) return (start_time, pid)
def configure_sggp_test(sggp_path, qps, time, press_expid, press_rate): if qps == '': qps = 1000 if time == '' or time > 30: time = 30 thread_size = int(qps / 4) cfg_expall = confhelper.ConfReader(sggp_path + '/web_qw_expall.ini') cfg_expall.setValue('web_qw_exp', 'press_qps', qps) cfg_expall.setValue('web_qw_exp', 'thread_size', thread_size) cfg_expall.setValue('web_qw_exp', 'press_time', time) cfg_online = confhelper.ConfReader(sggp_path + '/web_qw_online.ini') cfg_online.setValue('web_qw', 'press_qps', qps) cfg_online.setValue('web_qw', 'thread_size', thread_size) cfg_online.setValue('web_qw', 'press_time', time) if (os.path.exists(sggp_path + '/start_qw_test.sh')): update_errorlog("[%s] start_qw_test is exist,del it\n" % get_now_time()) os.popen('rm -rf %s' % (sggp_path + '/start_qw_test.sh')) if press_expid != 0 and press_rate > 0: expid = hex(press_expid)[2:] + '^0^0^0^0^0^0^0^0' commandline = 'echo ' + expid + ' | /search/odin/daemon/webqw/tools/sggp/data/Encode -f utf8 -t utf16' asycmd = asycommands.TrAsyCommands(timeout=240) asycmd_list.append(asycmd) for iotype, line in asycmd.execute_with_data([commandline], shell=True): exp_id = "exp_id=" + line base_query = sggp_query_path + 'query_qw_base' command = '''awk '{print "''' + exp_id + """"$0}' """ + base_query + ">" + sggp_query_path + "query_qw_expid" try: child = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) child.communicate(input=None) child.poll() except Exception as e: update_errorlog("[%s] create expid query wrong ,except:%s\n" % (get_now_time(), e)) if press_rate < 100: qw_expid_qps = qps * press_rate / 100 qw_qps = 1000 - qw_expid_qps cfg = confhelper.ConfReader(sggp_path + '/web_qw_group.ini') cfg.setValue('web_qw_exp', 'press_qps', int(qw_expid_qps)) cfg.setValue('web_qw_exp', 'thread_size', int(qw_expid_qps)) cfg.setValue('web_qw', 'press_qps', int(qw_qps)) cfg.setValue('web_qw', 'thread_size', int(qw_qps / 5)) cfg.setValue('web_qw_exp', 'press_time', time) cfg.setValue('web_qw', 'press_time', time) os.symlink(sggp_path + '/start_qw_group.sh', sggp_path + '/start_qw_test.sh') elif press_rate == 100: os.symlink(sggp_path + '/start_qw_expall.sh', sggp_path + '/start_qw_test.sh') else: os.symlink(sggp_path + '/start_qw_online.sh', sggp_path + '/start_qw_test.sh') return 0
def get_performance(log_file, performance, cost_type): update_errorlog("[%s] start to get performance result %s \n" % (get_now_time(), log_file)) bakfile = runlogbak + cost_type + '_err_' + str(mission_id) os.popen("cp %s %s" % (log_file, bakfile)) if (os.path.exists(log_file) is False): performance.append(log_file + " is not exists") return -1 asycmd = asycommands.TrAsyCommands(timeout=240) asycmd_list.append(asycmd) for iotype, line in asycmd.execute_with_data(['python2', cost_tool, log_file], shell=False): performance.append(line) if (asycmd.return_code() != 0): return asycmd.return_code() return 0
def sync_olbl_data_to_local(data_path): arg = "%s::%s" % (online_host, data_path[1:]) stdlog = "" errlog = "" asycmd = asycommands.TrAsyCommands(timeout=30 * 30) for iotype, line in asycmd.execute_with_data(['rsync', '-ravl', arg, data_path], shell=False): if (iotype is 1): stdlog += line.encode('utf-8') + '\n' elif (iotype is 2): errlog += line + '\n' if (asycmd.return_code() != 0): update_errorlog("[%s] rsync olbl_data to local Error\n" % get_now_time()) update_errorlog(errlog) return 1 update_errorlog("[%s] rsync olbl_data to local success\n" % get_now_time()) return 0
def make_env(file_path): asycmd = asycommands.TrAsyCommands(timeout=1200) asycmd_list.append(asycmd) make_log = "" for iotype, line in asycmd.execute_with_data(['make', '-j8'], shell=False, cwd=file_path): if iotype is 2: make_log += line + "\n" if (asycmd.return_code() != 0): # timeout or error, then try again make_log = "" for iotype, line in asycmd.execute_with_data(['make' '-j8'], shell=False, cwd=file_path): if iotype is 2: make_log += line if (asycmd.return_code() != 0): update_errorlog(make_log) return 2 update_errorlog("[%s] make success\n" % get_now_time()) return 0
def performance_once(file_path, performance_result, cost_type): asycmd = asycommands.TrAsyCommands(timeout=120) asycmd_list.append(asycmd) # kill lt-queryoptimiz for iotype, line in asycmd.execute_with_data( ['ps -ef|grep lt-queryoptimiz|grep -v grep'], shell=True): if (line.find('lt-queryoptimiz') != -1): pid = int(line.split()[1]) stop_proc(pid) # clean Mem sync_cmd = subprocess.Popen(['sync'], shell=False, cwd=file_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sync_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s sync success \n" % (get_now_time(), cost_type)) else: update_errorlog("[%s] %s sync error \n" % (get_now_time(), cost_type)) echo_three_cmd = subprocess.Popen(['echo 3 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) echo_three_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s free mem success \n" % (get_now_time(), cost_type)) else: update_errorlog( "[%s] %s free pagecache, dentries and inodes error \n" % (get_now_time(), cost_type)) echo_one_cmd = subprocess.Popen(['echo 0 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) echo_one_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s reset success \n" % (get_now_time(), cost_type)) else: update_errorlog("[%s] %s reset free error \n" % (get_now_time(), cost_type)) log = [] # start lt-queryoptimiz update_errorlog("[%s] Begin Start %s webqw\n" % (get_now_time(), cost_type)) (ret, service_pid) = lanch(file_path + "/QueryOptimizer", "start.sh", 8019, log) if (ret < 0): bakfile = runlogbak + cost_type + '_starterr_' + str(mission_id) os.popen("cp %s %s" % (file_path + '/QueryOptimizer/err.log', bakfile)) update_errorlog("[%s] %s webqw Start error, errlog path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak)) for fname in os.listdir(file_path + '/QueryOptimizer'): if 'core' in fname: corefile = runlogbak + cost_type + '_startcore_' + str( mission_id) os.popen("cp %s %s" % (file_path + '/QueryOptimizer/core.*', corefile)) update_errorlog( "[%s] %s webqw Start core, core file path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak)) time.sleep(0.5) up_log = "" for line in log: up_log += "[%s] %s" % (get_now_time(), line + '\n') update_errorlog("%s\n" % (up_log)) for iotype, line in asycmd.execute_with_data( ['/bin/tail', '-50', file_path + "/QueryOptimizer/err.log"], shell=False): up_log += line + '\n' update_errorlog( up_log.decode('gbk').encode('utf-8').replace("'", "\\'")) return -1 update_errorlog("[%s] %s webqw Start OK, cost %d s, PID %s \n" % (get_now_time(), cost_type, ret, str(service_pid))) # Start PressTool log = [] update_errorlog("[%s] Begin start PressTool\n" % get_now_time()) if cost_type == 'cost_test': (ret, tools_pid) = sggp_lanch(sggp_path, "start_qw_test.sh", log) else: (ret, tools_pid) = sggp_lanch(sggp_path, "start_qw_base.sh", log) print ret, tools_pid if (ret < 0): time.sleep(0.5) up_log = "" for line in log: up_log += "[%s] %s" % (get_now_time(), line + '\n') update_errorlog("%s\n" % (up_log)) up_log = "" for iotype, line in asycmd.execute_with_data( ['/bin/tail', '-50', sggp_path + "/err"], shell=False): up_log += line + '\n' update_errorlog( up_log.decode('gbk').encode('utf-8').replace("'", "\\'")) return -1 update_errorlog("[%s] PressTool Start OK, PIDs %s\n" % (get_now_time(), str(tools_pid))) update_errorlog("[%s] Wait PressTool...\n" % get_now_time()) # Wait PressTool Stop for subpid in tools_pid: wait_to_die(subpid, 5 * 30, file_path, cost_type) update_errorlog("[%s] PressTool stoped\n" % get_now_time()) # Stop webqw stop_proc(service_pid) update_errorlog("[%s] %s webqw stoped\n" % (get_now_time(), cost_type)) return get_performance(file_path + '/QueryOptimizer/err.log', performance_result, cost_type)
def scpnewdata(file_path, host_ip, username, password, newdata_path): new_data = newdata_path.split('\n') new_data_lst = list() for nd in new_data: if nd != "": new_data_lst.append(nd) # scp whole data to test if len(new_data_lst) == 1 and new_data_lst[0].split(';')[1] == 'data': rd_path = new_data_lst[0].split(';')[0] if os.path.exists(file_path + '/data'): os.popen("rm -rf %s" % (file_path + '/data')) os.popen("mkdir -p %s" % (file_path + '/data')) arg = "%s::%s" % (host_ip, rd_path[1:] + '/') arg2 = file_path + '/data/' stdlog = "" errlog = "" asycmd = asycommands.TrAsyCommands(timeout=30 * 300) for iotype, line in asycmd.execute_with_data( ['rsync', '-ravl', arg, arg2], shell=False): if (iotype is 1): stdlog += line.encode('utf-8') + '\n' elif (iotype is 2): errlog += line + '\n' if (asycmd.return_code() != 0): update_errorlog("[%s] rsync ol_data to local Error\n" % get_now_time()) update_errorlog(errlog) return 1 update_errorlog("[%s] rsync ol_data to local success\n" % get_now_time()) else: new_data_dir = dict() for new_data in new_data_lst: new_data_dir[new_data.split(';')[0]] = new_data.split(';')[1] for item in new_data_dir: files = os.path.split(item)[1] if "{" in files: filename_lst = files[1:-1].split(',') for filename in filename_lst: if os.path.exists(file_path + '/' + new_data_dir[item] + '/' + filename): os.popen("rm -rf %s" % (file_path + '/' + new_data_dir[item] + '/' + filename)) else: if os.path.exists(file_path + '/' + new_data_dir[item] + '/' + files): os.popen( "rm -rf %s" % (file_path + '/' + new_data_dir[item] + '/' + files)) update_errorlog("[%s] try scp rd data %s to test enviroment\n" % (get_now_time(), new_data_dir[item])) passwd_key = '.*assword.*' cmdline = 'scp -r %s@%s:%s %s/' % ( username, host_ip, item, file_path + '/' + new_data_dir[item]) try: child = pexpect.spawn(cmdline) expect_result = child.expect([r'assword:', r'yes/no'], timeout=30) if expect_result == 0: child.sendline(password) elif expect_result == 1: child.sendline('yes') child.expect(passwd_key, timeout=30) child.sendline(password) child.timeout = 120 child.expect(pexpect.EOF) except Exception as e: update_errorlog("[%s] %s, scp rd data \n" % (get_now_time(), e)) update_errorlog( "[%s] try scp rd data %s to test enviroment ok\n" % (get_now_time(), new_data_dir[item])) return 0
def run_diff(file_path, cost_type, mission_id): asycmd = asycommands.TrAsyCommands(timeout=120) asycmd_list.append(asycmd) # kill lt-queryoptimiz for iotype, line in asycmd.execute_with_data(['ps -ef|grep lt-queryoptimiz|grep -v grep'], shell=True): if (line.find('lt-queryoptimiz') != -1): pid = int(line.split()[1]) stop_proc(pid) # clean Mem sync_cmd = subprocess.Popen(['sync'], shell=False, cwd=file_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sync_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s sync success \n" % (get_now_time(), cost_type)) else: update_errorlog("[%s] %s sync error \n" % (get_now_time(), cost_type)) echo_three_cmd = subprocess.Popen(['echo 3 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE) echo_three_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s free mem success \n" % (get_now_time(), cost_type)) else: update_errorlog("[%s] %s free pagecache, dentries and inodes error \n" % (get_now_time(), cost_type)) echo_one_cmd = subprocess.Popen(['echo 0 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE) echo_one_cmd.wait() if (sync_cmd.returncode == 0): update_errorlog("[%s] %s reset success \n" % (get_now_time(), cost_type)) else: update_errorlog("[%s] %s reset free error \n" % (get_now_time(), cost_type)) log = [] # start lt-queryoptimiz update_errorlog("[%s] Begin Start %s webqw\n" % (get_now_time(), cost_type)) (ret, service_pid) = lanch(file_path + "/QueryOptimizer", "start.sh", 8019, log) if (ret < 0): bakfile = runlogbak + cost_type + '_starterr_' + str(mission_id) os.popen("cp %s %s" % (file_path + '/QueryOptimizer/err.log', bakfile)) update_errorlog("[%s] %s webqw Start error, errlog path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak)) for fname in os.listdir(file_path + '/QueryOptimizer'): if 'core' in fname: corefile = runlogbak + cost_type + '_startcore_' + str(mission_id) os.popen("cp %s %s" % (file_path + '/QueryOptimizer/core.*', corefile)) update_errorlog("[%s] %s webqw Start core, core file path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak)) time.sleep(0.5) up_log = "" for line in log: up_log += "[%s] %s" % (get_now_time(), line + '\n') update_errorlog("%s\n" % (up_log)) for iotype, line in asycmd.execute_with_data(['/bin/tail', '-50', file_path + "/QueryOptimizer/err.log"],shell=False): up_log += line + '\n' update_errorlog(up_log.decode('gbk').encode('utf-8').replace("'", "\\'")) return -1 update_errorlog("[%s] %s webqw Start OK, cost %d s, PID %s \n" % (get_now_time(), cost_type, ret, str(service_pid))) # 2000 diff longDiffResult = longDiff.diff_query("http://webqw01.web.djt.ted:8019/request","http://10.134.100.44:8019/request",mission_id) if longDiffResult == 0: update_errorlog("2000 diff Result insert Datebase success!") else: update_errorlog("2000 diff Result insert Datebase failed!") # Stop webqw stop_proc(service_pid) update_errorlog("[%s] %s webqw stoped\n" % (get_now_time(), cost_type)) return 0