def __init__(self,**kwargs):
self.reqlist = list()
self.threadlist = list()
self.timesofdeath = 4
self.timing = 0
self.number_req_check = 0
self.requests_offset = 0
self.total_requests_in_db = 0
self.activeThreadCount = 0
#以下三项 tenantid，request_quota,disruptive从json配置文件里读取
self.tenantid = kwargs.get('tenantid',3)
self.tenant_is_disruptive = True if kwargs.get('disruptive',0) == 1 else False
# 如果为 -1 则是休眠 随机时间,为0 则不休眠
self.tenant_sleep_for_excess = kwargs.get('sleep_for_excess',5) if self.tenant_is_disruptive else 0
self.tenant_quota = kwargs.get('tenant_quota',200)
self.tenant_threads_over_quota = kwargs.get('threads_over_quota',100)
self.tennat_log_dir = kwargs.get('log_dir','/home/ceph-log/')
self.request_rate = kwargs.get('request_rate',5000)
self.requests_per_make = kwargs.get('requests_per_make',1)
self.number_of_request = int(self.request_rate * 0.1)
self.number_of_make_request = int(self.number_of_request / self.requests_per_make)
self.threads_of_pool =kwargs.get('threads_of_pool',16)
self.requests_pool = threadpool.ThreadPool(self.threads_of_pool)
self.total_request_generated = 0
self.process_begined = True
self.last_request_thread_file_name = "request_thread_%s.log"%(str(time.time()))
self.last_responses_file_name = None
self.total_running_time = kwargs.get('total_generate_time',6)
self.mIATGenerator= kwargs.get('IATGenerator',None)
self.interval_get_number_of_active_thread = kwargs.get("interval_get_number_of_active_thread",1)
self.check_time = kwargs.get('check_time',5)
self.fs_module = kwargs.get('fs_module',None)
self.fs_class_name = kwargs.get('fs_class_name',None)
self.mysql_conn,self.mcur=mysql_utils.connMySQL(host='10.65.7.151')
self.mcur.execute("select count(*) from requests where tid = %d" % (self.tenantid))
self.total_requests_in_db = self.mcur.fetchone()[0]
print "total request:",self.total_requests_in_db
mysql_utils.closeConn(self.mysql_conn,self.mcur)
def main(config_path="cfbench.json"):
import os
isSuccess,config_objs=read_config(os.path.join(os.getcwd(),config_path))
if isSuccess==False:
print "read json %s except error" %(config_path)
return
db_conn,mcur = mysql_utils.connMySQL(host='10.65.7.151')
mcur.execute('select max(respid) from responses')
last_resp_id=mcur.fetchone()
if len(last_resp_id) ==0 or( len(last_resp_id) != 0 and last_resp_id[0]==None) :
last_resp_id = []
last_resp_id.append(0)
iat_module,iat_name = configuration(config_objs['request_of_iat']['name'],config_objs['request_of_iat']['conf_tag'])
mIATGenerator=getattr(iat_module,iat_name)(**(config_objs['request_of_iat']['kwargs']))
kwargs_context={}
FS_MODULE,FS_CLASS_NAME=configuration(config_objs['file_system']['name'],config_objs['file_system']['conf_tag'])
kwargs_context.update({'fs_module':FS_MODULE,'fs_class_name':FS_CLASS_NAME})
kwargs_context.update(config_objs['request_generator']['kwargs'])
kwargs_context.update({'IATGenerator':mIATGenerator})
kwargs_context.update({'interval_get_number_of_active_thread':config_objs['interval_get_number_of_active_thread']})
kwargs_context.update(config_objs['tenant'])
req_module,req_name = configuration(config_objs['request_generator']['name'],config_objs['request_generator']['conf_tag'])
mRequestGenerator = getattr(req_module,req_name)(**kwargs_context)
btime = time.time()
mRequestGenerator.runGenerator()
endtime = time.time()
mcur.execute('insert into analysis(tid,disruptive,sleep_for_excess,'\
'quota,threads_over_quota,total_generate_time,requests_per_make,threads_of_pool,run_time,begintime,endtime,last_resp_id) values' \
'(%d,"%s",%d,%d,%d,%d,%d,%d,"%s","%s","%s",%d)' %\
(config_objs['tenant']['tenantid'],str(config_objs['tenant']['disruptive']),config_objs['tenant']['sleep_for_excess'],\
config_objs['tenant']['tenant_quota'],config_objs['tenant']['threads_over_quota'],config_objs['request_generator']['kwargs']['total_generate_time'],\
config_objs['request_generator']['kwargs']['request_per_make'],config_objs['request_generator']['kwargs']['threads_of_pool'],str(endtime - btime),\
str(btime),str(endtime),last_resp_id[0]))
db_conn.commit()
mysql_utils.closeConn(db_conn,mcur)
return
def getRequest(self):
# get requests from database mysql
self.mysql_conn,self.mcur=mysql_utils.connMySQL(host='10.65.7.151')
reqitems = self.mcur.execute("select * from requests where tid = %d limit %d,%d" % \
(self.tenantid,self.requests_offset,self.number_of_request))
reqitems = self.mcur.fetchall()
reqitems2 = None
if len(reqitems)<self.number_of_request:
self.mcur.execute("select * from requests where tid = %d limit %d,%d" % \
(self.tenantid,0,self.number_of_request - len(reqitems)))
reqitems2 = self.mcur.fetchall()
print "req not enough",len(reqitems2)
mysql_utils.closeConn(self.mysql_conn,self.mcur)
self.requests_offset = self.requests_offset + len(reqitems) if \
self.requests_offset + len(reqitems) < self.total_requests_in_db - 1 else 0
# request dictoary iosize :(kB) iotype:w-write,r--read,d-delete,m--mkdir
# co--copy c--create ,r--read
for i in xrange(len(reqitems)):
self.reqlist.append( {'reqid':reqitems[i%len(reqitems)][1],'tid':reqitems[i%len(reqitems)][2],\
'reqtype':reqitems[i%len(reqitems)][3],'reqsize':reqitems[i%len(reqitems)][4],\
'reqdst':reqitems[i%len(reqitems)][5],'reqsrc':reqitems[i%len(reqitems)][6],\
'ftype':reqitems[i%len(reqitems)][7],'tenant_quota':self.tenant_quota})
if reqitems2 != None:
for i in xrange(len(reqitems2)):
self.reqlist.append( {'reqid':reqitems[i%len(reqitems)][1],'tid':reqitems[i%len(reqitems)][2],\
'reqtype':reqitems[i%len(reqitems)][3],'reqsize':reqitems[i%len(reqitems)][4],\
'reqdst':reqitems[i%len(reqitems)][5],'reqsrc':reqitems[i%len(reqitems)][6],\
'ftype':reqitems[i%len(reqitems)][7],'tenant_quota':self.tenant_quota})