def compute_write_main(total_time):
try:
eva_utils.compute_ts_stat()
eva_utils.compute_stat()
except ZeroDivisionError:
common.g_logger.error(traceback.print_exc())
mail = open(common.MAIL_PATH, 'a')
web = None
if conf.g_web_report_type != '' and conf.g_web_report:
web = open(common.WEB_PATH, 'a')
desp = 'data={datasize} key={ks} value={vs} lg={lg} cf={cf} run_time={t}'.format(
datasize=conf.g_datasize, ks=get_data_size(conf.g_test_conf[conf.KEY_SIZE]),
vs=get_data_size(conf.g_test_conf[conf.VALUE_SIZE]), lg=conf.g_test_conf[conf.LG_NUM],
cf=conf.g_test_conf[conf.CF_NUM], t=total_time, e=str(common.g_force_exit))
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
desp += 'write_speed={ws}/TS*M schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
desp += 'write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
rs=int(int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) * int(conf.g_test_conf[conf.TABLET_NUM]) /
int(conf.g_test_conf[conf.TS_NUMBER])),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
desp += 'write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
buffer=float(conf.g_test_conf[conf.SCAN_BUFFER])/common.MEGA)
eva_utils.write_email(mail, web, desp)
def compute_write_main(total_time):
try:
eva_utils.compute_ts_stat()
eva_utils.compute_stat()
except ZeroDivisionError:
common.g_logger.error(traceback.print_exc())
mail = open(common.MAIL_PATH, 'a')
web = None
if conf.g_web_report_type != '' and conf.g_web_report:
web = open(common.WEB_PATH, 'a')
desp = 'data={datasize} key={ks} value={vs} lg={lg} cf={cf} run_time={t}'.format(
datasize=conf.g_datasize,
ks=get_data_size(conf.g_test_conf[conf.KEY_SIZE]),
vs=get_data_size(conf.g_test_conf[conf.VALUE_SIZE]),
lg=conf.g_test_conf[conf.LG_NUM],
cf=conf.g_test_conf[conf.CF_NUM],
t=total_time,
e=str(common.g_force_exit))
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[
conf.MODE] == conf.MODE_RAND_WRITE:
desp += 'write_speed={ws}/TS*M schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
desp += 'write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
rs=int(
int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) *
int(conf.g_test_conf[conf.TABLET_NUM]) /
int(conf.g_test_conf[conf.TS_NUMBER])),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
desp += 'write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
buffer=float(conf.g_test_conf[conf.SCAN_BUFFER]) / common.MEGA)
eva_utils.write_email(mail, web, desp)
def run_test():
common.g_query_thread.start()
common.g_logger.info('running tera_mark with {n} tablets'.format(
n=conf.g_test_conf[conf.TABLET_NUM]))
wait_list = []
kill_list = []
start_time = time.time()
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or\
conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
wait_list, kill_list = run_write_test()
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
wait_list, kill_list = run_scan_test()
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
wait_list, kill_list = run_read_test()
count = 0
wait_num = conf.g_test_conf[conf.TABLET_NUM] * 2 / 3
while count < wait_num:
count = 0
for ret in wait_list:
if ret.poll() is not None:
count += 1
time.sleep(common.QUERY_INTERVAL)
for ret in wait_list:
try:
ret.kill()
except OSError:
pass
end_time = time.time()
for ret in kill_list:
ret.kill()
total_time = int(end_time - start_time)
hours = total_time / 3600
mins = (total_time % 3600) / 60
total_time = str(hours) + 'h' + str(mins) + 'm'
common.g_logger.info('done running test: ' + total_time)
common.g_exit = True
common.g_query_event.set()
common.g_query_thread.join()
try:
eva_utils.compute_ts_stat()
eva_utils.compute_stat()
except ZeroDivisionError:
common.g_logger.error(traceback.print_exc())
mail = open(common.MAIL_PATH, 'a')
desp = 'data_size={datasize}G key_size={ks}B value_size={vs}B lg_number={lg} cf_number={cf} running_time={t}'.format(
datasize=conf.g_test_conf[conf.CF_NUM] *
conf.g_test_conf[conf.TABLET_NUM] * conf.g_test_conf[conf.ENTRY_NUM] *
conf.g_test_conf[conf.ENTRY_SIZE] / 1000000000,
ks=conf.g_test_conf[conf.KEY_SIZE],
vs=conf.g_test_conf[conf.VALUE_SIZE],
lg=conf.g_test_conf[conf.LG_NUM],
cf=conf.g_test_conf[conf.CF_NUM],
t=total_time)
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[
conf.MODE] == conf.MODE_RAND_WRITE:
desp += ' write_speed={ws}/TS*M schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
desp += ' write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
rs=int(
int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) *
int(conf.g_test_conf[conf.TABLET_NUM]) /
int(conf.g_test_conf[conf.TS_NUMBER])),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
desp += ' write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
buffer=float(conf.g_test_conf[conf.SCAN_BUFFER]) / common.MEGA)
eva_utils.write_email(mail, desp)
mail.close()
def run_test():
common.g_query_thread.start()
common.g_logger.info('running tera_mark with {n} tablets'.format(n=conf.g_test_conf[conf.TABLET_NUM]))
wait_list = []
kill_list = []
start_time = time.time()
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or\
conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
wait_list, kill_list = run_write_test()
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
wait_list, kill_list = run_scan_test()
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
wait_list, kill_list = run_read_test()
count = 0
wait_num = conf.g_test_conf[conf.TABLET_NUM] * 2 / 3
while count < wait_num:
count = 0
for ret in wait_list:
if ret.poll() is not None:
count += 1
time.sleep(common.QUERY_INTERVAL)
for ret in wait_list:
try:
ret.kill()
except OSError:
pass
end_time = time.time()
for ret in kill_list:
ret.kill()
total_time = int(end_time - start_time)
hours = total_time / 3600
mins = (total_time % 3600) / 60
total_time = str(hours) + 'h' + str(mins) + 'm'
common.g_logger.info('done running test: ' + total_time)
common.g_exit = True
common.g_query_event.set()
common.g_query_thread.join()
try:
eva_utils.compute_ts_stat()
eva_utils.compute_stat()
except ZeroDivisionError:
common.g_logger.error(traceback.print_exc())
mail = open(common.MAIL_PATH, 'a')
desp = 'data_size={datasize}G key_size={ks}B value_size={vs}B lg_number={lg} cf_number={cf} running_time={t}'.format(
datasize=conf.g_test_conf[conf.CF_NUM] * conf.g_test_conf[conf.TABLET_NUM] *
conf.g_test_conf[conf.ENTRY_NUM] * conf.g_test_conf[conf.ENTRY_SIZE] / 1000000000,
ks=conf.g_test_conf[conf.KEY_SIZE], vs=conf.g_test_conf[conf.VALUE_SIZE], lg=conf.g_test_conf[conf.LG_NUM],
cf=conf.g_test_conf[conf.CF_NUM], t=total_time)
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
desp += ' write_speed={ws}/TS*M schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
desp += ' write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
rs=int(int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) * int(conf.g_test_conf[conf.TABLET_NUM]) /
int(conf.g_test_conf[conf.TS_NUMBER])),
s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
desp += ' write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
buffer=float(conf.g_test_conf[conf.SCAN_BUFFER])/common.MEGA)
eva_utils.write_email(mail, desp)
mail.close()