def invoke_server(userip, cmd):
#ssh do not use -tt here
cmd = cmd.replace(' ', delimit)
cmd = 'ssh ' + userip + ' python ' + host2dir(
userip) + '/invoke_server.py ' + cmd
subprocess.call(cmd, shell=True) # call has to block and wait here
print "coor cmd: " + cmd
def get_power(hoststr='', suffix='', LOG=0):
if not hoststr:
hoststr = get_ip_addr()
try:
if hoststr.startswith('tarekc'):
name = hoststr.split('.')[0]
elif hoststr[0].isalpha(): # t1
hip = host2ip[hoststr]
if hip in ip2tarekc.keys():
name = ip2tarekc[hip]
else:
name = get_namehostip('hname', hip) # help from namehostip
elif hoststr.startswith('1'): # 172.
if hoststr in ip2tarekc.keys():
name = ip2tarekc[hoststr] # is ip already in
else:
name = get_namehostip('hname', hoststr) # help from namehostip
pport = tarekc2powerport[name]
#print 'Power for '+hoststr+' '+pport
cmd = 'php ' + mypydir + '/avocent_measure.php ' + pport
proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=open('/dev/null', 'w'))
output = proc.communicate()[0].strip()
if LOG:
fid = open(host2dir(name) + '/power' + suffix + '.log', 'a')
fid.write(('%.4f' % time.time()) + ' ' + hoststr + ' ' + output +
'\n')
fid.close()
# SET LOG
# logging.basicConfig(level=logging.INFO)
# logger = logging.getLogger(__name__)
# handler = logging.FileHandler(host2dir(name)+'/power'+suffix+'.log')
# handler.setLevel(logging.INFO)
# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# handler.setFormatter(formatter)
# logger.addHandler(handler)
#logger.info(hoststr+' '+output)
except:
print hoststr + ' ' + name + ' has NO power measure!'
output = ''
return output
def th_sh(h,script): cmd="ssh -tt "+host2userip[h]+ " sh "+host2dir(h)+"/"+script print '\nrun: '+cmd subprocess.call(cmd,shell=True)
t.start() thqueue.append(t) # gen php file first: cmd='python ./gen-script_MAC.py '+et print cmd subprocess.call(cmd.split()) print 'copy...' cmd='python ./cp_to_MAC.py '+et print '\n\n\nrun: '+cmd subprocess.call(cmd.split()) time.sleep(1) for th in thqueue: th.join() userip=host2userip[coor] cmd="ssh -tt "+userip+ " 'python "+host2dir(coor)+"/webtest_coor.py "+et+"'" print '\n\n\nrun: '+cmd subprocess.call(cmd,shell=True) # has to shell, or no such file... time.sleep(1) cmd='python ./col_res_MAC.py '+et print '\n\n\nrun: '+cmd subprocess.call(cmd.split()) #t make sure mysqld, lighttpd, haproxy, php-fpm, memcached #e make sure mysql, lighttpd, haproxy, php5-fpm, memcached
def th_monitor_run(rewrite,sampleIntv,measureIntv=0.5):
global serverruntime,nametag,runflag,monitordict,monitorsuffix
global masterlist,slavelist,allhosts
if rewrite:
try:
fn=glob.glob(host2dir(tname)+'/monitor*')
for rmf in fn:
os.remove(rmf)
except:
print 'monitor.log already removed !'
print tname+" "+nametag+' thread running...\nmonitordict:',monitordict,sampleIntv
while serverruntime>0 and runflag:
fid=open(host2dir(tname)+'/monitor-'+monitorsuffix+'-'+nametag+'-'+tname+'.log','a')
sttime=time.time()
if monitordict['cpu']>0:
core=monitor.get_cpu_count() # cpu logical number
dic=monitor.get_cpu_usage(intv=measureIntv) # us, sy, % each core
msg='cpu'
for i in range(core):
msg=msg+' %.1f,%.1f'%(dic[i]['usr'],dic[i]['sys'])
fid.write(('%.4f'%time.time())+' '+msg+'\n')
if monitordict['mem']>0:
dic=monitor.get_mem_usage() # used %, total MB
msg='mem %.1f %d'%(dic['used'],dic['total'])
fid.write(('%.4f'%time.time())+' '+msg+'\n')
if monitordict['nettxrx']>0:
dic=monitor.get_net_txrx(rlevel=4) # tx,rx,dropin,dropout
msg='nettxrx %.4f %.4f %d %d %d %d %d %d'%(dic['tx'],dic['rx'],dic['dropin'],dic['dropout'],dic['errin'],dic['errout'],dic['txpac'],dic['rxpac'])
fid.write(('%.4f'%time.time())+' '+msg+'\n')
if nametag=='mem':
pname=host2servicedict(tname)['mem'][0] # memcached proc num
dic=monitor.get_procnum(pname)
fid.write(('%.4f'%time.time())+' '+pname+' '+`dic[pname]`+'\n')
if nametag=='web':
pname=host2servicedict(tname)['web'][0] # php-fpm proc num
dic=monitor.get_procnum(pname)
fid.write(('%.4f'%time.time())+' '+pname+' '+`dic[pname]`+'\n')
pname=host2servicedict(tname)['web'][1] # memcached proc num
dic=monitor.get_procnum(pname)
fid.write(('%.4f'%time.time())+' '+pname+' '+`dic[pname]`+'\n')
if monitordict['netconn']>0:
dic=monitor.get_net_conn() # {rip:{CONN_TIME_WAIT:1000, }, }
for rip in dic.keys():
if rip in ip2host.keys():
tn=ip2host[rip]
ripdic=dic[rip]
if tn in masterlist:
msg='to-master '
elif tn in slavelist:
msg='to-slave '
elif tn in coorlist:
msg='to-coor '
else:
msg='to-? '
msg=msg+tn
for key in ripdic:
msg=msg+' '+key+' '+str(ripdic[key])
fid.write(('%.4f'%time.time())+' '+msg+'\n')
fid.close()
endtime=time.time()-sttime
time.sleep(max(0,sampleIntv-endtime))
print tname+' mon thread exit.'+nametag
rpctimeout=int(readconf.get_conf(et,'servertimeout')) skipservicecheck=int(readconf.get_conf(et,'skipservicecheck')) skipwarmup=int(readconf.get_conf(et,'skipwarmup')) cleanlog=int(readconf.get_conf(et,'cleanlog')) cleanres=int(readconf.get_conf(et,'cleanres')) delaytime=1.0/rqps # START RPC SERVER tgen = threading.Thread(target=rpc_server_thread) tgen.setDaemon(True) # when main exit, this thread also terminate tgen.start() print tname+" RPC server thread running... "+nametag # cleaning if cleanlog>=1: cmd='ssh -tt '+host2userip[tname]+' sh '+host2dir(tname)+'/clean_log.sh' print '\nclean cmd: '+cmd subprocess.call(cmd.split()) try: fn=glob.glob(host2dir(tname)+'/*.log') for rmf in fn: os.remove(rmf) except: print '*.log already removed !' if cleanres>=1: cmd='ssh -tt '+host2userip[tname]+' sh '+host2dir(tname)+'/clean_res.sh' print '\nclean cmd: '+cmd subprocess.call(cmd.split()) # if nametag=='': # if tname in masterlist:
sys.exit(1)
print 'my role: ' + nametag
# START RPC SERVER
tgen = threading.Thread(target=rpc_server_thread)
tgen.setDaemon(True) # when main exit, this thread also terminate
tgen.start()
print tname + " RPC server thread running... " + nametag
# INVOKE REMOTE
thqueue = []
MANUALINVOKE = int(readconf.get_conf(et, 'manualinvoke'))
if MANUALINVOKE == 0:
print "coor invoking python servers..."
for n in workerlist: # nametag at last
cmd = 'python ' + host2dir(n) + '/webtest_worker.py ' + et
thqueue.append(use_thread_invoke_server(host2userip[n], cmd))
# wait worker to start RPC...
while len(thqueue) > 0:
t = thqueue.pop(0)
print 'coor: join th invoke_server %s' % t.getName()
t.join() # make sure every work has started RPC
time.sleep(1)
else:
print '\n\nYou need to invoke rpc on servers yourself! '
time.sleep(3)
# cleaning
if cleanlog >= 1:
cmd = 'ssh -tt ' + host2userip[tname] + ' sh ' + host2dir(
tname) + '/clean_log.sh'