def one_round():
t_num = 64
for data_set_name in data_set_lst:
for our_algorithm in our_exec_name_lst:
for kt_type_name in ['12', '23']:
# for hierarchy in ['NO', 'YES']:
# for hierarchy in ['YES']:
for hierarchy in ['NO']:
statistics_dir = os.sep.join(
map(str,
[work_dir, data_set_name, t_num, hierarchy]))
output_dir = os.sep.join([statistics_dir, 'output'])
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'log']))
os.system('mkdir -p ' + output_dir)
dstat_file_path = statistics_dir + os.sep + our_algorithm + '-' + kt_type_name + '-dstat.log'
log_file_path = os.sep.join([
statistics_dir, 'log',
'-'.join([our_algorithm, kt_type_name, 'raw.log'])
])
logger.info('log file path: {}'.format(log_file_path))
# 1st: append headers
append_header(dstat_file_path)
append_header(log_file_path)
# 2nd: run exec cmd
algorithm_path = our_exec_path + os.sep + our_algorithm
os.chdir(output_dir)
params_lst = map(str, [
'cgexec -g memory:yche-exp' if with_c_group else
'', algorithm_path, data_set_path + os.sep +
data_set_name, kt_type_name, hierarchy
])
cmd = ' '.join(params_lst)
logger.info('exec-cmd: {}'.format(cmd))
time_out = 3600 * 10
my_env = os.environ.copy()
def execute_cmd(my_cmd):
logger.info('sub-process: {}'.format(my_cmd))
os.system(my_cmd)
# 3rd: spawn a new process to run the exec
dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path
p = Process(target=execute_cmd, args=(dstat_cmd, ))
p.start()
my_env['OMP_NUM_THREADS'] = str(t_num)
tle_flag, info, correct_info = time_out_util.run_with_timeout(
cmd, timeout_sec=time_out, env=my_env)
time_out_util.kill_term_recursive(p.pid)
modify_dstat_file(dstat_file_path)
# 4th: append outputs
if len(info) > 0:
with open(log_file_path, 'a+') as ofs:
ofs.write(info)
logger.info('finish: {}'.format(cmd))
def one_round():
for data_set_name in data_set_lst:
for reorder_method in filtered_reorder_lst:
for our_algorithm in our_exec_name_lst:
for t_num in thread_num_lst :
statistics_dir = os.sep.join(
map(str, ['.', exp_res_root_name, folder_name, data_set_name, reorder_method, t_num]))
os.system('mkdir -p ' + os.sep.join([statistics_dir, 'log']))
os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat']))
os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat_clean']))
statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log'
dstat_file_path = os.sep.join([statistics_dir, 'dstat', our_algorithm + '-dstat.log'])
log_file_path = os.sep.join([statistics_dir, 'log', our_algorithm + '-raw.log'])
logger.info('stat file path: {}'.format(statistics_file_path))
# 1st: write header
append_header(statistics_file_path)
append_header(dstat_file_path)
append_header(log_file_path)
# 2nd: run exec cmd
algorithm_path = our_exec_path + os.sep + our_algorithm
params_lst = map(str, ['cgexec -g memory:yche-exp' if with_c_group else '',
algorithm_path, data_set_path + os.sep + data_set_name, reorder_method,
statistics_file_path])
cmd = ' '.join(params_lst)
logger.info('exec-cmd: {}'.format(cmd))
time_out = 3600 * 5 if 'cuda' not in our_algorithm else 600
my_env = os.environ.copy()
def execute_cmd(my_cmd):
logger.info('sub-process: {}'.format(my_cmd))
os.system(my_cmd)
# 3rd: spawn a new process to run the exec
dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path
p = Process(target=execute_cmd, args=(dstat_cmd,))
p.start()
my_env['OMP_NUM_THREADS'] = str(t_num)
tle_flag, info, correct_info = time_out_util.run_with_timeout(cmd, timeout_sec=time_out,
env=my_env)
time_out_util.kill_term_recursive(p.pid)
modify_dstat_file(dstat_file_path)
# 4th: append outputs
write_split(statistics_file_path)
with open(statistics_file_path, 'a+') as ifs:
ifs.write(correct_info)
ifs.write('\nis_time_out:' + str(tle_flag))
ifs.write(my_splitter + time.ctime() + my_splitter)
ifs.write('\n\n\n\n')
if len(info) > 0:
with open(log_file_path, 'a+') as ofs:
ofs.write(info)
logger.info('finish: {}'.format(cmd))
def one_round(eps_lst, c_lst):
data_set_name = 'ca-GrQc'
for our_algorithm in our_exec_name_lst:
for eps in list(reversed(eps_lst)):
for c in c_lst:
statistics_dir = os.sep.join(
map(str, [
exp_res_root_name, folder_name, data_set_name, c,
eps
]))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'log']))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'dstat']))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'dstat_clean']))
statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log'
dstat_file_path = os.sep.join([
statistics_dir, 'dstat', our_algorithm + '-dstat.log'
])
log_file_path = os.sep.join(
[statistics_dir, 'log', our_algorithm + '-raw.log'])
logger.info(
'stat file path: {}'.format(statistics_file_path))
# 1st: write header
append_header(statistics_file_path)
append_header(dstat_file_path)
append_header(log_file_path)
# 2nd: run exec cmd
algorithm_path = our_exec_path + os.sep + our_algorithm
params_lst = map(str, [
'cgexec -g memory:yche-exp' if with_c_group else '',
algorithm_path, data_set_name, eps, c,
statistics_file_path
])
cmd = ' '.join(params_lst)
logger.info('exec-cmd: {}'.format(cmd))
time_out = 600
my_env = os.environ.copy()
my_env[
"OMP_NUM_THREADS"] = '1' if our_algorithm != 'probesim_ss_ap_bench' else '56'
def execute_cmd(my_cmd):
logger.info('sub-process: {}'.format(my_cmd))
os.system(my_cmd)
# 3rd: spawn a new process to run the exec
dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path
p = Process(target=execute_cmd, args=(dstat_cmd, ))
p.start()
tle_flag, info, correct_info = time_out_util.run_with_timeout(
cmd, timeout_sec=time_out, env=my_env)
time_out_util.kill_term_recursive(p.pid)
modify_dstat_file(dstat_file_path)
# 4th: append outputs
write_split(statistics_file_path)
with open(statistics_file_path, 'a+') as ifs:
ifs.write(correct_info)
ifs.write('\nis_time_out:' + str(tle_flag))
ifs.write(my_splitter + time.ctime() + my_splitter)
ifs.write('\n\n\n\n')
if len(info) > 0:
with open(log_file_path, 'a+') as ofs:
ofs.write(info)
logger.info('finish: {}'.format(cmd))
def one_round():
data_set_name = 'ca-HepTh'
for our_algorithm in our_exec_name_lst:
for update_tag in ['ins', 'del']:
for edge_num in [str(i) for i in range(5000, 30000, 5000)]:
statistics_dir = os.sep.join(
map(str, [
exp_res_root_name, folder_name, data_set_name,
update_tag, edge_num
]))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'log']))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'dstat']))
os.system('mkdir -p ' +
os.sep.join([statistics_dir, 'dstat_clean']))
statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log'
dstat_file_path = os.sep.join([
statistics_dir, 'dstat', our_algorithm + '-dstat.log'
])
log_file_path = os.sep.join(
[statistics_dir, 'log', our_algorithm + '-raw.log'])
logger.info(
'stat file path: {}'.format(statistics_file_path))
# 1st: write header
append_header(statistics_file_path)
append_header(dstat_file_path)
append_header(log_file_path)
# 2nd: run exec cmd
algorithm_path = our_exec_path + os.sep + our_algorithm
params_lst = map(str, [
'cgexec -g memory:yche-exp' if with_c_group else '',
algorithm_path, data_set_name, edge_num, update_tag,
statistics_file_path
])
cmd = ' '.join(params_lst)
logger.info('exec-cmd: {}'.format(cmd))
time_out = 800000
my_env = os.environ.copy()
def execute_cmd(my_cmd):
logger.info('sub-process: {}'.format(my_cmd))
os.system(my_cmd)
# 3rd: spawn a new process to run the exec
dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path
p = Process(target=execute_cmd, args=(dstat_cmd, ))
p.start()
tle_flag, info, correct_info = time_out_util.run_with_timeout(
cmd, timeout_sec=time_out, env=my_env)
time_out_util.kill_term_recursive(p.pid)
modify_dstat_file(dstat_file_path)
# 4th: append outputs
write_split(statistics_file_path)
with open(statistics_file_path, 'a+') as ifs:
ifs.write(correct_info)
ifs.write('\nis_time_out:' + str(tle_flag))
ifs.write(my_splitter + time.ctime() + my_splitter)
ifs.write('\n\n\n\n')
if len(info) > 0:
with open(log_file_path, 'a+') as ofs:
ofs.write(info)
logger.info('finish: {}'.format(cmd))
