def run_write_test():
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME], common.CREATE, conf.g_test_conf[conf.SCHEMA])
wait_list = []
for i in range(conf.g_test_conf[conf.TABLET_NUM]):
prefix = '%04d' % i
bench_cmd = conf.TERA_BENCH + " | awk -F '\t' '{print \"" + prefix + """\"$1"\t"$2"\t"$3"\t"$4}' """
if conf.g_test_conf[conf.KV] is True:
bench_cmd = conf.TERA_BENCH + " | awk -F '\t' '{print \"" + prefix + """\"$1"\t"$2}' """
cmd = '{bench} | ./tera_mark --mode=w --tablename={name} --type=async --verify=false --entry_limit={limit}'.\
format(bench=bench_cmd, name=conf.g_test_conf[conf.TABLE_NAME], limit=str(conf.g_test_conf[conf.WRITE_SPEED_LIMIT]))
print cmd
fout = open('../tmp/'+str(i)+'.w.out', 'w')
ferr = open('../tmp/'+str(i)+'.w.err', 'w')
ret = subprocess.Popen(cmd, stdout=fout, stderr=ferr, shell=True)
wait_list.append(ret)
return wait_list, []
def parse_input():
conf_file = sys.argv[1]
conf_dict = json.load(open(conf_file, 'r'))
bench_cmd_prefix = './tera_bench --compression_ratio=1 '
for pre, post in conf_dict.iteritems():
if pre == conf.TABLE_NAME:
conf.g_test_conf[conf.TABLE_NAME] = post
if pre == conf.KEY_SIZE:
conf.g_test_conf[conf.KEY_SIZE] = int(post)
bench_cmd_prefix += '--key_size={ks} '.format(ks=post)
if pre == conf.VALUE_SIZE:
conf.g_test_conf[conf.VALUE_SIZE] = int(post)
bench_cmd_prefix += '--value_size={vs} '.format(vs=post)
if pre == conf.MODE:
conf.g_test_conf[conf.MODE] = post
if post == conf.MODE_SEQ_WRITE:
bench_cmd_prefix += '--benchmarks=seq '
common.EMAIL_BLOCK_TITLE = 'Sequential Write'
elif post == conf.MODE_RAND_WRITE:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Random Write'
elif post == conf.MODE_SCAN:
common.EMAIL_BLOCK_TITLE = 'Scan'
elif post == conf.MODE_READ:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Read'
elif pre == conf.TABLET_NUM:
conf.g_test_conf[conf.TABLET_NUM] = int(post)
elif pre == conf.SPLIT_SIZE:
conf.g_test_conf[conf.SPLIT_SIZE] = int(post)
elif pre == conf.TS_NUMBER:
conf.g_test_conf[conf.TS_NUMBER] = int(post)
elif pre == conf.WRITE_SPEED_LIMIT:
conf.g_speed_limit = post
elif pre == conf.READ_SPEED_LIMIT:
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(post)
elif pre == 'random':
bench_cmd_prefix += '--random=' + post + ' '
elif pre == conf.ENTRY_NUM:
num = float(post) * 1000000
bench_cmd_prefix += '--num=' + str(int(num)) + ' '
conf.g_test_conf[conf.ENTRY_NUM] = int(num)
elif pre == conf.SCAN_BUFFER:
buffer = float(post)
buffer *= common.MEGA
conf.g_test_conf[conf.SCAN_BUFFER] = str(int(buffer))
elif pre == conf.SCHEMA:
conf.g_test_conf[conf.SCHEMA] = post
conf.g_test_conf[conf.LG_NUM] = len(post)
conf.g_test_conf[conf.ENTRY_SIZE] = conf.g_test_conf[conf.KEY_SIZE] + conf.g_test_conf[conf.VALUE_SIZE]
conf.g_test_conf[conf.WRITE_SPEED_LIMIT] = int(float(conf.g_speed_limit) / conf.g_test_conf[conf.TABLET_NUM] * 1024 * 1024 / conf.g_test_conf[conf.ENTRY_SIZE])
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(float(conf.g_test_conf[conf.READ_SPEED_LIMIT]) / conf.g_test_conf[conf.TABLET_NUM])
conf.g_test_conf[conf.CF_NUM], conf.g_test_conf[conf.CF] = \
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME], conf.CF, conf.g_test_conf[conf.SCHEMA])
if conf.g_test_conf[conf.CF] != '':
bench_cmd_prefix += '--cf=' + conf.g_test_conf[conf.CF]
conf.TERA_BENCH = bench_cmd_prefix
common.g_logger.info('running tera_mark: ' + str(conf.g_test_conf))
def run_write_test():
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME],
common.CREATE, conf.g_test_conf[conf.SCHEMA])
wait_list = []
for i in range(conf.g_test_conf[conf.TABLET_NUM]):
prefix = '%04d' % i
bench_cmd = conf.TERA_BENCH + " | awk -F '\t' '{print \"" + prefix + """\"$1"\t"$2"\t"$3"\t"$4}' """
if conf.g_test_conf[conf.KV] is True:
bench_cmd = conf.TERA_BENCH + " | awk -F '\t' '{print \"" + prefix + """\"$1"\t"$2}' """
cmd = '{bench} | ./tera_mark --mode=w --tablename={name} --type=async --verify=false --entry_limit={limit}'.\
format(bench=bench_cmd, name=conf.g_test_conf[conf.TABLE_NAME], limit=str(conf.g_test_conf[conf.WRITE_SPEED_LIMIT]))
print cmd
fout = open('../tmp/' + str(i) + '.w.out', 'w')
ferr = open('../tmp/' + str(i) + '.w.err', 'w')
ret = subprocess.Popen(cmd, stdout=fout, stderr=ferr, shell=True)
wait_list.append(ret)
return wait_list, []
def parse_input():
conf_file = sys.argv[1]
if len(sys.argv) == 3:
conf.g_web_report = False
conf_dict = json.load(open(conf_file, 'r'))
bench_cmd_prefix = './tera_bench --compression_ratio=1 '
for pre, post in conf_dict.iteritems():
if pre == conf.TABLE_NAME:
conf.g_test_conf[conf.TABLE_NAME] = post
if pre == conf.KEY_SIZE:
conf.g_test_conf[conf.KEY_SIZE] = int(post)
bench_cmd_prefix += '--key_size={ks} '.format(ks=post)
if pre == conf.VALUE_SIZE:
conf.g_test_conf[conf.VALUE_SIZE] = int(post)
bench_cmd_prefix += '--value_size={vs} '.format(vs=post)
if pre == conf.KEY_SEED:
conf.g_test_conf[conf.KEY_SEED] = int(post)
bench_cmd_prefix += '--key_seed={ks} '.format(ks=post)
if pre == conf.VALUE_SEED:
conf.g_test_conf[conf.VALUE_SEED] = int(post)
bench_cmd_prefix += '--value_seed={vs} '.format(vs=post)
if pre == conf.MODE:
conf.g_test_conf[conf.MODE] = post
if post == conf.MODE_SEQ_WRITE:
bench_cmd_prefix += '--benchmarks=seq '
common.EMAIL_BLOCK_TITLE = 'Sequential Write'
elif post == conf.MODE_RAND_WRITE:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Random Write'
elif post == conf.MODE_SCAN:
common.EMAIL_BLOCK_TITLE = 'Scan'
elif post == conf.MODE_READ:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Read'
elif pre == conf.TABLET_NUM:
conf.g_test_conf[conf.TABLET_NUM] = int(post)
elif pre == conf.SPLIT_SIZE:
conf.g_test_conf[conf.SPLIT_SIZE] = int(post)
elif pre == conf.TS_NUMBER:
conf.g_test_conf[conf.TS_NUMBER] = int(post)
elif pre == conf.WRITE_SPEED_LIMIT:
conf.g_speed_limit = post
elif pre == conf.READ_SPEED_LIMIT:
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(post)
elif pre == 'random':
bench_cmd_prefix += '--random=' + post + ' '
elif pre == conf.ENTRY_NUM:
num = float(post) * common.MEGA
bench_cmd_prefix += '--num=' + str(int(num)) + ' '
conf.g_test_conf[conf.ENTRY_NUM] = int(num)
elif pre == conf.SCAN_BUFFER:
buffer = float(post)
buffer *= common.MEGA
conf.g_test_conf[conf.SCAN_BUFFER] = str(int(buffer))
elif pre == conf.SCHEMA:
conf.g_test_conf[conf.SCHEMA] = post
conf.g_test_conf[conf.LG_NUM] = len(post)
elif pre == conf.STEP:
conf.g_test_conf[conf.STEP] = post
elif pre == 'web_report_type':
conf.g_web_report_type = post
conf.g_test_conf[conf.ENTRY_SIZE] = conf.g_test_conf[
conf.KEY_SIZE] + conf.g_test_conf[conf.VALUE_SIZE]
conf.g_test_conf[conf.WRITE_SPEED_LIMIT] = int(
round(
float(conf.g_speed_limit) / conf.g_test_conf[conf.TABLET_NUM] *
common.MEGA / conf.g_test_conf[conf.ENTRY_SIZE]))
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(
float(conf.g_test_conf[conf.READ_SPEED_LIMIT]) /
conf.g_test_conf[conf.TABLET_NUM])
conf.g_test_conf[conf.CF_NUM], conf.g_test_conf[conf.CF] = \
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME], conf.CF, conf.g_test_conf[conf.SCHEMA])
if conf.g_test_conf[conf.CF] != '':
bench_cmd_prefix += '--cf=' + conf.g_test_conf[conf.CF] + ' '
if conf.g_test_conf[conf.STEP] == 'True':
bench_cmd_prefix += '--key_step=' + str(
common.RANDOM_MAX / conf.g_test_conf[conf.ENTRY_NUM]) + ' '
conf.TERA_BENCH = bench_cmd_prefix
conf.g_datasize = (conf.g_test_conf[conf.CF_NUM] *
conf.g_test_conf[conf.TABLET_NUM] *
conf.g_test_conf[conf.ENTRY_NUM] *
conf.g_test_conf[conf.ENTRY_SIZE])
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[
conf.MODE] == conf.MODE_RAND_WRITE:
print '\t%-25s' % 'estimated running time:', get_time_form(
(conf.g_datasize >> 20) / float(conf.g_speed_limit))
else:
print '\t%-25s' % 'estimated running time:', get_time_form(
conf.g_test_conf[conf.ENTRY_NUM] /
conf.g_test_conf[conf.READ_SPEED_LIMIT])
conf.g_datasize = get_data_size(conf.g_datasize)
print '\t%-25s' % 'user data size:', conf.g_datasize
if common.g_logger is not None:
common.g_logger.info('running tera_mark: ' + str(conf.g_test_conf))
def parse_input():
conf_file = sys.argv[1]
conf_dict = json.load(open(conf_file, 'r'))
bench_cmd_prefix = './tera_bench --compression_ratio=1 '
for pre, post in conf_dict.iteritems():
if pre == conf.TABLE_NAME:
conf.g_test_conf[conf.TABLE_NAME] = post
if pre == conf.KEY_SIZE:
conf.g_test_conf[conf.KEY_SIZE] = int(post)
bench_cmd_prefix += '--key_size={ks} '.format(ks=post)
if pre == conf.VALUE_SIZE:
conf.g_test_conf[conf.VALUE_SIZE] = int(post)
bench_cmd_prefix += '--value_size={vs} '.format(vs=post)
if pre == conf.MODE:
conf.g_test_conf[conf.MODE] = post
if post == conf.MODE_SEQ_WRITE:
bench_cmd_prefix += '--benchmarks=seq '
common.EMAIL_BLOCK_TITLE = 'Sequential Write'
elif post == conf.MODE_RAND_WRITE:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Random Write'
elif post == conf.MODE_SCAN:
common.EMAIL_BLOCK_TITLE = 'Scan'
elif post == conf.MODE_READ:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Read'
elif pre == conf.TABLET_NUM:
conf.g_test_conf[conf.TABLET_NUM] = int(post)
elif pre == conf.SPLIT_SIZE:
conf.g_test_conf[conf.SPLIT_SIZE] = int(post)
elif pre == conf.TS_NUMBER:
conf.g_test_conf[conf.TS_NUMBER] = int(post)
elif pre == conf.WRITE_SPEED_LIMIT:
conf.g_speed_limit = post
elif pre == conf.READ_SPEED_LIMIT:
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(post)
elif pre == 'random':
bench_cmd_prefix += '--random=' + post + ' '
elif pre == conf.ENTRY_NUM:
num = float(post) * 1000000
bench_cmd_prefix += '--num=' + str(int(num)) + ' '
conf.g_test_conf[conf.ENTRY_NUM] = int(num)
elif pre == conf.SCAN_BUFFER:
buffer = float(post)
buffer *= common.MEGA
conf.g_test_conf[conf.SCAN_BUFFER] = str(int(buffer))
elif pre == conf.SCHEMA:
conf.g_test_conf[conf.SCHEMA] = post
conf.g_test_conf[conf.LG_NUM] = len(post)
conf.g_test_conf[conf.ENTRY_SIZE] = conf.g_test_conf[
conf.KEY_SIZE] + conf.g_test_conf[conf.VALUE_SIZE]
conf.g_test_conf[conf.WRITE_SPEED_LIMIT] = int(
float(conf.g_speed_limit) / conf.g_test_conf[conf.TABLET_NUM] * 1024 *
1024 / conf.g_test_conf[conf.ENTRY_SIZE])
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(
float(conf.g_test_conf[conf.READ_SPEED_LIMIT]) /
conf.g_test_conf[conf.TABLET_NUM])
conf.g_test_conf[conf.CF_NUM], conf.g_test_conf[conf.CF] = \
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME], conf.CF, conf.g_test_conf[conf.SCHEMA])
if conf.g_test_conf[conf.CF] != '':
bench_cmd_prefix += '--cf=' + conf.g_test_conf[conf.CF]
conf.TERA_BENCH = bench_cmd_prefix
common.g_logger.info('running tera_mark: ' + str(conf.g_test_conf))
def parse_input():
conf_file = sys.argv[1]
if len(sys.argv) == 3:
conf.g_web_report = False
conf_dict = json.load(open(conf_file, 'r'))
bench_cmd_prefix = './tera_bench --compression_ratio=1 '
for pre, post in conf_dict.iteritems():
if pre == conf.TABLE_NAME:
conf.g_test_conf[conf.TABLE_NAME] = post
if pre == conf.KEY_SIZE:
conf.g_test_conf[conf.KEY_SIZE] = int(post)
bench_cmd_prefix += '--key_size={ks} '.format(ks=post)
if pre == conf.VALUE_SIZE:
conf.g_test_conf[conf.VALUE_SIZE] = int(post)
bench_cmd_prefix += '--value_size={vs} '.format(vs=post)
if pre == conf.KEY_SEED:
conf.g_test_conf[conf.KEY_SEED] = int(post)
bench_cmd_prefix += '--key_seed={ks} '.format(ks=post)
if pre == conf.VALUE_SEED:
conf.g_test_conf[conf.VALUE_SEED] = int(post)
bench_cmd_prefix += '--value_seed={vs} '.format(vs=post)
if pre == conf.MODE:
conf.g_test_conf[conf.MODE] = post
if post == conf.MODE_SEQ_WRITE:
bench_cmd_prefix += '--benchmarks=seq '
common.EMAIL_BLOCK_TITLE = 'Sequential Write'
elif post == conf.MODE_RAND_WRITE:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Random Write'
elif post == conf.MODE_SCAN:
common.EMAIL_BLOCK_TITLE = 'Scan'
elif post == conf.MODE_READ:
bench_cmd_prefix += '--benchmarks=random '
common.EMAIL_BLOCK_TITLE = 'Read'
elif pre == conf.TABLET_NUM:
conf.g_test_conf[conf.TABLET_NUM] = int(post)
elif pre == conf.SPLIT_SIZE:
conf.g_test_conf[conf.SPLIT_SIZE] = int(post)
elif pre == conf.TS_NUMBER:
conf.g_test_conf[conf.TS_NUMBER] = int(post)
elif pre == conf.WRITE_SPEED_LIMIT:
conf.g_speed_limit = post
elif pre == conf.READ_SPEED_LIMIT:
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(post)
elif pre == 'random':
bench_cmd_prefix += '--random=' + post + ' '
elif pre == conf.ENTRY_NUM:
num = float(post) * common.MEGA
bench_cmd_prefix += '--num=' + str(int(num)) + ' '
conf.g_test_conf[conf.ENTRY_NUM] = int(num)
elif pre == conf.SCAN_BUFFER:
buffer = float(post)
buffer *= common.MEGA
conf.g_test_conf[conf.SCAN_BUFFER] = str(int(buffer))
elif pre == conf.SCHEMA:
conf.g_test_conf[conf.SCHEMA] = post
conf.g_test_conf[conf.LG_NUM] = len(post)
elif pre == conf.STEP:
conf.g_test_conf[conf.STEP] = post
elif pre == 'web_report_type':
conf.g_web_report_type = post
conf.g_test_conf[conf.ENTRY_SIZE] = conf.g_test_conf[conf.KEY_SIZE] + conf.g_test_conf[conf.VALUE_SIZE]
conf.g_test_conf[conf.WRITE_SPEED_LIMIT] = int(round(float(conf.g_speed_limit) / conf.g_test_conf[conf.TABLET_NUM] * common.MEGA / conf.g_test_conf[conf.ENTRY_SIZE]))
conf.g_test_conf[conf.READ_SPEED_LIMIT] = int(float(conf.g_test_conf[conf.READ_SPEED_LIMIT]) / conf.g_test_conf[conf.TABLET_NUM])
conf.g_test_conf[conf.CF_NUM], conf.g_test_conf[conf.CF] = \
eva_utils.table_manipulate(conf.g_test_conf[conf.TABLE_NAME], conf.CF, conf.g_test_conf[conf.SCHEMA])
if conf.g_test_conf[conf.CF] != '':
bench_cmd_prefix += '--cf=' + conf.g_test_conf[conf.CF] + ' '
if conf.g_test_conf[conf.STEP] == 'True':
bench_cmd_prefix += '--key_step=' + str(common.RANDOM_MAX / conf.g_test_conf[conf.ENTRY_NUM]) + ' '
conf.TERA_BENCH = bench_cmd_prefix
conf.g_datasize = (conf.g_test_conf[conf.CF_NUM] * conf.g_test_conf[conf.TABLET_NUM] *
conf.g_test_conf[conf.ENTRY_NUM] * conf.g_test_conf[conf.ENTRY_SIZE])
if conf.g_test_conf[conf.MODE] == conf.MODE_SEQ_WRITE or conf.g_test_conf[conf.MODE] == conf.MODE_RAND_WRITE:
print '\t%-25s' % 'estimated running time:', get_time_form((conf.g_datasize >> 20) / float(conf.g_speed_limit))
else:
print '\t%-25s' % 'estimated running time:', get_time_form(conf.g_test_conf[conf.ENTRY_NUM] /
conf.g_test_conf[conf.READ_SPEED_LIMIT])
conf.g_datasize = get_data_size(conf.g_datasize)
print '\t%-25s' % 'user data size:', conf.g_datasize
if common.g_logger is not None:
common.g_logger.info('running tera_mark: ' + str(conf.g_test_conf))