def sync_ol_conf_to_local(conf_path):
if os.path.exists(conf_path) == False:
# print "tmp_ol_conf path not exists, mkdir -p"
update_errorlog("[%s] %s\n" % (get_now_time(), "tmp_ol_conf path not exists, mkdir -p"))
os.popen("mkdir -p " + conf_path)
update_errorlog("[%s] start rsync ol_conf to local\n" % get_now_time())
rsync_path = online_path
if (rsync_path[0:1] == "/"):
rsync_path = rsync_path[1:]
if (rsync_path[len(rsync_path) - 1:] != "/"):
rsync_path = rsync_path + "/"
rsync_path = rsync_path + "conf/"
arg2 = conf_path
if (conf_path[len(conf_path) - 1:] != "/"):
arg2 = conf_path + "/"
arg = "%s::%s" % (online_host, rsync_path)
stdlog = ""
errlog = ""
asycmd = asycommands.TrAsyCommands(timeout=30 * 60)
asycmd_list.append(asycmd)
for iotype, line in asycmd.execute_with_data(['rsync', '-ravl', arg, arg2], shell=False):
if (iotype is 1):
stdlog += line + '\n'
elif (iotype is 2):
errlog += line + '\n'
if (asycmd.return_code() != 0):
update_errorlog("[%s] rsync ol_conf to local Error\n" % get_now_time())
update_errorlog(errlog)
return 1
update_errorlog("[%s] rsync ol_conf to local success\n" % get_now_time())
return 0
def sggp_lanch(file_path, start_script, log):
# rules: start_script must put pid in `PID` file: echo $! > PID
# return a tuple(retcode, pid)
# lanch(sggp_path, "start_qw_group.sh", -1, log)
pid = list()
asycmd = asycommands.TrAsyCommands(timeout=30)
asycmd_list.append(asycmd)
child = subprocess.Popen(['/bin/sh', start_script],
shell=False,
cwd=file_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
child.wait()
if (child.returncode != 0):
log.append(child.stderr.read())
return (-1, pid)
for iotype, line in asycmd.execute_with_data(
['/bin/cat', file_path + "/PID"], shell=False):
if (iotype == 1 and line != ""):
try:
pid.append(int(line))
except:
continue
if len(pid) == 0:
return (-2, pid)
proc = None
for subpid in pid:
try:
proc = psutil.Process(subpid)
except:
log.append("process %d is not alive" % pid)
return (-3, pid)
return (0, pid)
def lanch(file_path, start_script, port, log):
# rules: start_script must put pid in `PID` file: echo $! > PID
# return a tuple(retcode, pid)
# lanch(sggp_path, "start_qw_group.sh", -1, log)
pid = -1
asycmd = asycommands.TrAsyCommands(timeout=30)
asycmd_list.append(asycmd)
child = subprocess.Popen(['/bin/sh', start_script],
shell=False,
cwd=file_path,
stderr=subprocess.PIPE)
child.wait()
if (child.returncode != 0):
log.append(child.stderr.read())
return (-1, pid)
for iotype, line in asycmd.execute_with_data(
['/bin/cat', file_path + "/PID"], shell=False):
if (iotype == 1 and line != ""):
try:
if (pid == -1):
pid = int(line)
else:
tmp = int(line)
pid = max(pid, tmp)
except:
continue
if (pid == -1):
return (-2, pid)
proc = None
try:
proc = psutil.Process(pid)
except:
log.append("process %d is not alive" % pid)
return (-3, pid)
if (port is -1):
return (0, pid)
is_alive = True
start_time = 0
proc_list.append(pid)
while is_alive:
try:
conn_list = proc.connections()
except:
is_alive = False
break
listened = False
for conn in conn_list:
if (conn.status == "LISTEN"
or conn.status == "NONE") and conn.laddr[1] == port:
listened = True
break
if listened:
break
time.sleep(1)
start_time += 1
if not is_alive:
log.append("process start failed")
proc_list.remove(pid)
return (-3, pid)
return (start_time, pid)
def configure_sggp_test(sggp_path, qps, time, press_expid, press_rate):
if qps == '':
qps = 1000
if time == '' or time > 30:
time = 30
thread_size = int(qps / 4)
cfg_expall = confhelper.ConfReader(sggp_path + '/web_qw_expall.ini')
cfg_expall.setValue('web_qw_exp', 'press_qps', qps)
cfg_expall.setValue('web_qw_exp', 'thread_size', thread_size)
cfg_expall.setValue('web_qw_exp', 'press_time', time)
cfg_online = confhelper.ConfReader(sggp_path + '/web_qw_online.ini')
cfg_online.setValue('web_qw', 'press_qps', qps)
cfg_online.setValue('web_qw', 'thread_size', thread_size)
cfg_online.setValue('web_qw', 'press_time', time)
if (os.path.exists(sggp_path + '/start_qw_test.sh')):
update_errorlog("[%s] start_qw_test is exist,del it\n" %
get_now_time())
os.popen('rm -rf %s' % (sggp_path + '/start_qw_test.sh'))
if press_expid != 0 and press_rate > 0:
expid = hex(press_expid)[2:] + '^0^0^0^0^0^0^0^0'
commandline = 'echo ' + expid + ' | /search/odin/daemon/webqw/tools/sggp/data/Encode -f utf8 -t utf16'
asycmd = asycommands.TrAsyCommands(timeout=240)
asycmd_list.append(asycmd)
for iotype, line in asycmd.execute_with_data([commandline],
shell=True):
exp_id = "exp_id=" + line
base_query = sggp_query_path + 'query_qw_base'
command = '''awk '{print "''' + exp_id + """"$0}' """ + base_query + ">" + sggp_query_path + "query_qw_expid"
try:
child = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
child.communicate(input=None)
child.poll()
except Exception as e:
update_errorlog("[%s] create expid query wrong ,except:%s\n" %
(get_now_time(), e))
if press_rate < 100:
qw_expid_qps = qps * press_rate / 100
qw_qps = 1000 - qw_expid_qps
cfg = confhelper.ConfReader(sggp_path + '/web_qw_group.ini')
cfg.setValue('web_qw_exp', 'press_qps', int(qw_expid_qps))
cfg.setValue('web_qw_exp', 'thread_size', int(qw_expid_qps))
cfg.setValue('web_qw', 'press_qps', int(qw_qps))
cfg.setValue('web_qw', 'thread_size', int(qw_qps / 5))
cfg.setValue('web_qw_exp', 'press_time', time)
cfg.setValue('web_qw', 'press_time', time)
os.symlink(sggp_path + '/start_qw_group.sh',
sggp_path + '/start_qw_test.sh')
elif press_rate == 100:
os.symlink(sggp_path + '/start_qw_expall.sh',
sggp_path + '/start_qw_test.sh')
else:
os.symlink(sggp_path + '/start_qw_online.sh',
sggp_path + '/start_qw_test.sh')
return 0
def get_performance(log_file, performance, cost_type):
update_errorlog("[%s] start to get performance result %s \n" % (get_now_time(), log_file))
bakfile = runlogbak + cost_type + '_err_' + str(mission_id)
os.popen("cp %s %s" % (log_file, bakfile))
if (os.path.exists(log_file) is False):
performance.append(log_file + " is not exists")
return -1
asycmd = asycommands.TrAsyCommands(timeout=240)
asycmd_list.append(asycmd)
for iotype, line in asycmd.execute_with_data(['python2', cost_tool, log_file], shell=False):
performance.append(line)
if (asycmd.return_code() != 0):
return asycmd.return_code()
return 0
def sync_olbl_data_to_local(data_path):
arg = "%s::%s" % (online_host, data_path[1:])
stdlog = ""
errlog = ""
asycmd = asycommands.TrAsyCommands(timeout=30 * 30)
for iotype, line in asycmd.execute_with_data(['rsync', '-ravl', arg, data_path], shell=False):
if (iotype is 1):
stdlog += line.encode('utf-8') + '\n'
elif (iotype is 2):
errlog += line + '\n'
if (asycmd.return_code() != 0):
update_errorlog("[%s] rsync olbl_data to local Error\n" % get_now_time())
update_errorlog(errlog)
return 1
update_errorlog("[%s] rsync olbl_data to local success\n" % get_now_time())
return 0
def make_env(file_path):
asycmd = asycommands.TrAsyCommands(timeout=1200)
asycmd_list.append(asycmd)
make_log = ""
for iotype, line in asycmd.execute_with_data(['make', '-j8'], shell=False, cwd=file_path):
if iotype is 2:
make_log += line + "\n"
if (asycmd.return_code() != 0): # timeout or error, then try again
make_log = ""
for iotype, line in asycmd.execute_with_data(['make' '-j8'], shell=False, cwd=file_path):
if iotype is 2:
make_log += line
if (asycmd.return_code() != 0):
update_errorlog(make_log)
return 2
update_errorlog("[%s] make success\n" % get_now_time())
return 0
def performance_once(file_path, performance_result, cost_type):
asycmd = asycommands.TrAsyCommands(timeout=120)
asycmd_list.append(asycmd)
# kill lt-queryoptimiz
for iotype, line in asycmd.execute_with_data(
['ps -ef|grep lt-queryoptimiz|grep -v grep'], shell=True):
if (line.find('lt-queryoptimiz') != -1):
pid = int(line.split()[1])
stop_proc(pid)
# clean Mem
sync_cmd = subprocess.Popen(['sync'],
shell=False,
cwd=file_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sync_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s sync success \n" %
(get_now_time(), cost_type))
else:
update_errorlog("[%s] %s sync error \n" % (get_now_time(), cost_type))
echo_three_cmd = subprocess.Popen(['echo 3 > /proc/sys/vm/drop_caches'],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
echo_three_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s free mem success \n" %
(get_now_time(), cost_type))
else:
update_errorlog(
"[%s] %s free pagecache, dentries and inodes error \n" %
(get_now_time(), cost_type))
echo_one_cmd = subprocess.Popen(['echo 0 > /proc/sys/vm/drop_caches'],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
echo_one_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s reset success \n" %
(get_now_time(), cost_type))
else:
update_errorlog("[%s] %s reset free error \n" %
(get_now_time(), cost_type))
log = []
# start lt-queryoptimiz
update_errorlog("[%s] Begin Start %s webqw\n" %
(get_now_time(), cost_type))
(ret, service_pid) = lanch(file_path + "/QueryOptimizer", "start.sh", 8019,
log)
if (ret < 0):
bakfile = runlogbak + cost_type + '_starterr_' + str(mission_id)
os.popen("cp %s %s" % (file_path + '/QueryOptimizer/err.log', bakfile))
update_errorlog("[%s] %s webqw Start error, errlog path %s s\n" %
(get_now_time(), cost_type, local_ip + runlogbak))
for fname in os.listdir(file_path + '/QueryOptimizer'):
if 'core' in fname:
corefile = runlogbak + cost_type + '_startcore_' + str(
mission_id)
os.popen("cp %s %s" %
(file_path + '/QueryOptimizer/core.*', corefile))
update_errorlog(
"[%s] %s webqw Start core, core file path %s s\n" %
(get_now_time(), cost_type, local_ip + runlogbak))
time.sleep(0.5)
up_log = ""
for line in log:
up_log += "[%s] %s" % (get_now_time(), line + '\n')
update_errorlog("%s\n" % (up_log))
for iotype, line in asycmd.execute_with_data(
['/bin/tail', '-50', file_path + "/QueryOptimizer/err.log"],
shell=False):
up_log += line + '\n'
update_errorlog(
up_log.decode('gbk').encode('utf-8').replace("'", "\\'"))
return -1
update_errorlog("[%s] %s webqw Start OK, cost %d s, PID %s \n" %
(get_now_time(), cost_type, ret, str(service_pid)))
# Start PressTool
log = []
update_errorlog("[%s] Begin start PressTool\n" % get_now_time())
if cost_type == 'cost_test':
(ret, tools_pid) = sggp_lanch(sggp_path, "start_qw_test.sh", log)
else:
(ret, tools_pid) = sggp_lanch(sggp_path, "start_qw_base.sh", log)
print
ret, tools_pid
if (ret < 0):
time.sleep(0.5)
up_log = ""
for line in log:
up_log += "[%s] %s" % (get_now_time(), line + '\n')
update_errorlog("%s\n" % (up_log))
up_log = ""
for iotype, line in asycmd.execute_with_data(
['/bin/tail', '-50', sggp_path + "/err"], shell=False):
up_log += line + '\n'
update_errorlog(
up_log.decode('gbk').encode('utf-8').replace("'", "\\'"))
return -1
update_errorlog("[%s] PressTool Start OK, PIDs %s\n" %
(get_now_time(), str(tools_pid)))
update_errorlog("[%s] Wait PressTool...\n" % get_now_time())
# Wait PressTool Stop
for subpid in tools_pid:
wait_to_die(subpid, 5 * 30, file_path, cost_type)
update_errorlog("[%s] PressTool stoped\n" % get_now_time())
# Stop webqw
stop_proc(service_pid)
update_errorlog("[%s] %s webqw stoped\n" % (get_now_time(), cost_type))
return get_performance(file_path + '/QueryOptimizer/err.log',
performance_result, cost_type)
def scpnewdata(file_path, host_ip, username, password, newdata_path):
new_data = newdata_path.split('\n')
new_data_lst = list()
for nd in new_data:
if nd != "":
new_data_lst.append(nd)
# scp whole data to test
if len(new_data_lst) == 1 and new_data_lst[0].split(';')[1] == 'data':
rd_path = new_data_lst[0].split(';')[0]
if os.path.exists(file_path + '/data'):
os.popen("rm -rf %s" % (file_path + '/data'))
os.popen("mkdir -p %s" % (file_path + '/data'))
arg = "%s::%s" % (host_ip, rd_path[1:] + '/')
arg2 = file_path + '/data/'
stdlog = ""
errlog = ""
asycmd = asycommands.TrAsyCommands(timeout=30 * 300)
for iotype, line in asycmd.execute_with_data(
['rsync', '-ravl', arg, arg2], shell=False):
if (iotype is 1):
stdlog += line.encode('utf-8') + '\n'
elif (iotype is 2):
errlog += line + '\n'
if (asycmd.return_code() != 0):
update_errorlog("[%s] rsync ol_data to local Error\n" %
get_now_time())
update_errorlog(errlog)
return 1
update_errorlog("[%s] rsync ol_data to local success\n" %
get_now_time())
else:
new_data_dir = dict()
for new_data in new_data_lst:
new_data_dir[new_data.split(';')[0]] = new_data.split(';')[1]
for item in new_data_dir:
files = os.path.split(item)[1]
if "{" in files:
filename_lst = files[1:-1].split(',')
for filename in filename_lst:
if os.path.exists(file_path + '/' + new_data_dir[item] +
'/' + filename):
os.popen("rm -rf %s" %
(file_path + '/' + new_data_dir[item] + '/' +
filename))
else:
if os.path.exists(file_path + '/' + new_data_dir[item] + '/' +
files):
os.popen(
"rm -rf %s" %
(file_path + '/' + new_data_dir[item] + '/' + files))
update_errorlog("[%s] try scp rd data %s to test enviroment\n" %
(get_now_time(), new_data_dir[item]))
passwd_key = '.*assword.*'
cmdline = 'scp -r %s@%s:%s %s/' % (
username, host_ip, item, file_path + '/' + new_data_dir[item])
try:
child = pexpect.spawn(cmdline)
expect_result = child.expect([r'assword:', r'yes/no'],
timeout=30)
if expect_result == 0:
child.sendline(password)
elif expect_result == 1:
child.sendline('yes')
child.expect(passwd_key, timeout=30)
child.sendline(password)
child.timeout = 120
child.expect(pexpect.EOF)
except Exception as e:
update_errorlog("[%s] %s, scp rd data \n" %
(get_now_time(), e))
update_errorlog(
"[%s] try scp rd data %s to test enviroment ok\n" %
(get_now_time(), new_data_dir[item]))
return 0
def run_diff(file_path, cost_type, mission_id):
asycmd = asycommands.TrAsyCommands(timeout=120)
asycmd_list.append(asycmd)
# kill lt-queryoptimiz
for iotype, line in asycmd.execute_with_data(['ps -ef|grep lt-queryoptimiz|grep -v grep'], shell=True):
if (line.find('lt-queryoptimiz') != -1):
pid = int(line.split()[1])
stop_proc(pid)
# clean Mem
sync_cmd = subprocess.Popen(['sync'], shell=False, cwd=file_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sync_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s sync success \n" % (get_now_time(), cost_type))
else:
update_errorlog("[%s] %s sync error \n" % (get_now_time(), cost_type))
echo_three_cmd = subprocess.Popen(['echo 3 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
echo_three_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s free mem success \n" % (get_now_time(), cost_type))
else:
update_errorlog("[%s] %s free pagecache, dentries and inodes error \n" % (get_now_time(), cost_type))
echo_one_cmd = subprocess.Popen(['echo 0 > /proc/sys/vm/drop_caches'], shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
echo_one_cmd.wait()
if (sync_cmd.returncode == 0):
update_errorlog("[%s] %s reset success \n" % (get_now_time(), cost_type))
else:
update_errorlog("[%s] %s reset free error \n" % (get_now_time(), cost_type))
log = []
# start lt-queryoptimiz
update_errorlog("[%s] Begin Start %s webqw\n" % (get_now_time(), cost_type))
(ret, service_pid) = lanch(file_path + "/QueryOptimizer", "start.sh", 8019, log)
if (ret < 0):
bakfile = runlogbak + cost_type + '_starterr_' + str(mission_id)
os.popen("cp %s %s" % (file_path + '/QueryOptimizer/err.log', bakfile))
update_errorlog("[%s] %s webqw Start error, errlog path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak))
for fname in os.listdir(file_path + '/QueryOptimizer'):
if 'core' in fname:
corefile = runlogbak + cost_type + '_startcore_' + str(mission_id)
os.popen("cp %s %s" % (file_path + '/QueryOptimizer/core.*', corefile))
update_errorlog("[%s] %s webqw Start core, core file path %s s\n" % (get_now_time(), cost_type, local_ip + runlogbak))
time.sleep(0.5)
up_log = ""
for line in log:
up_log += "[%s] %s" % (get_now_time(), line + '\n')
update_errorlog("%s\n" % (up_log))
for iotype, line in asycmd.execute_with_data(['/bin/tail', '-50', file_path + "/QueryOptimizer/err.log"],shell=False):
up_log += line + '\n'
update_errorlog(up_log.decode('gbk').encode('utf-8').replace("'", "\\'"))
return -1
update_errorlog("[%s] %s webqw Start OK, cost %d s, PID %s \n" % (get_now_time(), cost_type, ret, str(service_pid)))
# 2000 diff
longDiffResult = longDiff.diff_query("http://webqw01.web.djt.ted:8019/request","http://10.134.100.44:8019/request",mission_id)
if longDiffResult == 0:
update_errorlog("2000 diff Result insert Datebase success!")
else:
update_errorlog("2000 diff Result insert Datebase failed!")
# Stop webqw
stop_proc(service_pid)
update_errorlog("[%s] %s webqw stoped\n" % (get_now_time(), cost_type))
return 0
