Пример #1
0
Файл: run.py Проект: Aleda/tera
def compute_write_main(total_time):
    try:
        eva_utils.compute_ts_stat()
        eva_utils.compute_stat()
    except ZeroDivisionError:
        common.g_logger.error(traceback.print_exc())

    mail = open(common.MAIL_PATH, 'a')
    web = None
    if conf.g_web_report_type != '' and conf.g_web_report:
        web = open(common.WEB_PATH, 'a')
    desp = 'data={datasize} key={ks} value={vs} lg={lg} cf={cf} run_time={t}'.format(
        datasize=conf.g_datasize, ks=get_data_size(conf.g_test_conf[conf.KEY_SIZE]),
        vs=get_data_size(conf.g_test_conf[conf.VALUE_SIZE]), lg=conf.g_test_conf[conf.LG_NUM],
        cf=conf.g_test_conf[conf.CF_NUM], t=total_time, e=str(common.g_force_exit))
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
        desp += 'write_speed={ws}/TS*M schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        desp += 'write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            rs=int(int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) * int(conf.g_test_conf[conf.TABLET_NUM]) /
                   int(conf.g_test_conf[conf.TS_NUMBER])),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        desp += 'write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            buffer=float(conf.g_test_conf[conf.SCAN_BUFFER])/common.MEGA)
    eva_utils.write_email(mail, web, desp)
Пример #2
0
Файл: run.py Проект: ktcer/tera
def compute_write_main(total_time):
    try:
        eva_utils.compute_ts_stat()
        eva_utils.compute_stat()
    except ZeroDivisionError:
        common.g_logger.error(traceback.print_exc())

    mail = open(common.MAIL_PATH, 'a')
    web = None
    if conf.g_web_report_type != '' and conf.g_web_report:
        web = open(common.WEB_PATH, 'a')
    desp = 'data={datasize} key={ks} value={vs} lg={lg} cf={cf} run_time={t}'.format(
        datasize=conf.g_datasize,
        ks=get_data_size(conf.g_test_conf[conf.KEY_SIZE]),
        vs=get_data_size(conf.g_test_conf[conf.VALUE_SIZE]),
        lg=conf.g_test_conf[conf.LG_NUM],
        cf=conf.g_test_conf[conf.CF_NUM],
        t=total_time,
        e=str(common.g_force_exit))
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[
            conf.MODE] == conf.MODE_RAND_WRITE:
        desp += 'write_speed={ws}/TS*M schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        desp += 'write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            rs=int(
                int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) *
                int(conf.g_test_conf[conf.TABLET_NUM]) /
                int(conf.g_test_conf[conf.TS_NUMBER])),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        desp += 'write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            buffer=float(conf.g_test_conf[conf.SCAN_BUFFER]) / common.MEGA)
    eva_utils.write_email(mail, web, desp)
Пример #3
0
def run_test():
    common.g_query_thread.start()
    common.g_logger.info('running tera_mark with {n} tablets'.format(
        n=conf.g_test_conf[conf.TABLET_NUM]))
    wait_list = []
    kill_list = []
    start_time = time.time()
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or\
       conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
        wait_list, kill_list = run_write_test()
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        wait_list, kill_list = run_scan_test()
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        wait_list, kill_list = run_read_test()

    count = 0
    wait_num = conf.g_test_conf[conf.TABLET_NUM] * 2 / 3
    while count < wait_num:
        count = 0
        for ret in wait_list:
            if ret.poll() is not None:
                count += 1
        time.sleep(common.QUERY_INTERVAL)

    for ret in wait_list:
        try:
            ret.kill()
        except OSError:
            pass
    end_time = time.time()

    for ret in kill_list:
        ret.kill()
    total_time = int(end_time - start_time)
    hours = total_time / 3600
    mins = (total_time % 3600) / 60
    total_time = str(hours) + 'h' + str(mins) + 'm'

    common.g_logger.info('done running test: ' + total_time)
    common.g_exit = True
    common.g_query_event.set()
    common.g_query_thread.join()

    try:
        eva_utils.compute_ts_stat()
        eva_utils.compute_stat()
    except ZeroDivisionError:
        common.g_logger.error(traceback.print_exc())

    mail = open(common.MAIL_PATH, 'a')
    desp = 'data_size={datasize}G key_size={ks}B value_size={vs}B lg_number={lg} cf_number={cf} running_time={t}'.format(
        datasize=conf.g_test_conf[conf.CF_NUM] *
        conf.g_test_conf[conf.TABLET_NUM] * conf.g_test_conf[conf.ENTRY_NUM] *
        conf.g_test_conf[conf.ENTRY_SIZE] / 1000000000,
        ks=conf.g_test_conf[conf.KEY_SIZE],
        vs=conf.g_test_conf[conf.VALUE_SIZE],
        lg=conf.g_test_conf[conf.LG_NUM],
        cf=conf.g_test_conf[conf.CF_NUM],
        t=total_time)
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[
            conf.MODE] == conf.MODE_RAND_WRITE:
        desp += ' write_speed={ws}/TS*M schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        desp += ' write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            rs=int(
                int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) *
                int(conf.g_test_conf[conf.TABLET_NUM]) /
                int(conf.g_test_conf[conf.TS_NUMBER])),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        desp += ' write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            buffer=float(conf.g_test_conf[conf.SCAN_BUFFER]) / common.MEGA)
    eva_utils.write_email(mail, desp)
    mail.close()
Пример #4
0
def run_test():
    common.g_query_thread.start()
    common.g_logger.info('running tera_mark with {n} tablets'.format(n=conf.g_test_conf[conf.TABLET_NUM]))
    wait_list = []
    kill_list = []
    start_time = time.time()
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or\
       conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
        wait_list, kill_list = run_write_test()
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        wait_list, kill_list = run_scan_test()
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        wait_list, kill_list = run_read_test()

    count = 0
    wait_num = conf.g_test_conf[conf.TABLET_NUM] * 2 / 3
    while count < wait_num:
        count = 0
        for ret in wait_list:
            if ret.poll() is not None:
                count += 1
        time.sleep(common.QUERY_INTERVAL)

    for ret in wait_list:
        try:
            ret.kill()
        except OSError:
            pass
    end_time = time.time()

    for ret in kill_list:
        ret.kill()
    total_time = int(end_time - start_time)
    hours = total_time / 3600
    mins = (total_time % 3600) / 60
    total_time = str(hours) + 'h' + str(mins) + 'm'

    common.g_logger.info('done running test: ' + total_time)
    common.g_exit = True
    common.g_query_event.set()
    common.g_query_thread.join()

    try:
        eva_utils.compute_ts_stat()
        eva_utils.compute_stat()
    except ZeroDivisionError:
        common.g_logger.error(traceback.print_exc())

    mail = open(common.MAIL_PATH, 'a')
    desp = 'data_size={datasize}G key_size={ks}B value_size={vs}B lg_number={lg} cf_number={cf} running_time={t}'.format(
        datasize=conf.g_test_conf[conf.CF_NUM] * conf.g_test_conf[conf.TABLET_NUM] *
                 conf.g_test_conf[conf.ENTRY_NUM] * conf.g_test_conf[conf.ENTRY_SIZE] / 1000000000,
        ks=conf.g_test_conf[conf.KEY_SIZE], vs=conf.g_test_conf[conf.VALUE_SIZE], lg=conf.g_test_conf[conf.LG_NUM],
        cf=conf.g_test_conf[conf.CF_NUM], t=total_time)
    if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
        desp += ' write_speed={ws}/TS*M schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_READ:
        desp += ' write_speed={ws}/TS*M read_speed={rs}/TS*Qps schema={s}'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            rs=int(int(conf.g_test_conf[conf.READ_SPEED_LIMIT]) * int(conf.g_test_conf[conf.TABLET_NUM]) /
                   int(conf.g_test_conf[conf.TS_NUMBER])),
            s=json.dumps(conf.g_test_conf[conf.SCHEMA], separators=(',', ':')))
    elif conf.g_test_conf[conf.MODE] == conf.MODE_SCAN:
        desp += ' write_speed={ws}/TS*M scan_buffer={buffer}/M'.format(
            ws=int(conf.g_speed_limit) / int(conf.g_test_conf[conf.TS_NUMBER]),
            buffer=float(conf.g_test_conf[conf.SCAN_BUFFER])/common.MEGA)
    eva_utils.write_email(mail, desp)
    mail.close()