def one_round(): t_num = 64 for data_set_name in data_set_lst: for our_algorithm in our_exec_name_lst: for kt_type_name in ['12', '23']: # for hierarchy in ['NO', 'YES']: # for hierarchy in ['YES']: for hierarchy in ['NO']: statistics_dir = os.sep.join( map(str, [work_dir, data_set_name, t_num, hierarchy])) output_dir = os.sep.join([statistics_dir, 'output']) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'log'])) os.system('mkdir -p ' + output_dir) dstat_file_path = statistics_dir + os.sep + our_algorithm + '-' + kt_type_name + '-dstat.log' log_file_path = os.sep.join([ statistics_dir, 'log', '-'.join([our_algorithm, kt_type_name, 'raw.log']) ]) logger.info('log file path: {}'.format(log_file_path)) # 1st: append headers append_header(dstat_file_path) append_header(log_file_path) # 2nd: run exec cmd algorithm_path = our_exec_path + os.sep + our_algorithm os.chdir(output_dir) params_lst = map(str, [ 'cgexec -g memory:yche-exp' if with_c_group else '', algorithm_path, data_set_path + os.sep + data_set_name, kt_type_name, hierarchy ]) cmd = ' '.join(params_lst) logger.info('exec-cmd: {}'.format(cmd)) time_out = 3600 * 10 my_env = os.environ.copy() def execute_cmd(my_cmd): logger.info('sub-process: {}'.format(my_cmd)) os.system(my_cmd) # 3rd: spawn a new process to run the exec dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path p = Process(target=execute_cmd, args=(dstat_cmd, )) p.start() my_env['OMP_NUM_THREADS'] = str(t_num) tle_flag, info, correct_info = time_out_util.run_with_timeout( cmd, timeout_sec=time_out, env=my_env) time_out_util.kill_term_recursive(p.pid) modify_dstat_file(dstat_file_path) # 4th: append outputs if len(info) > 0: with open(log_file_path, 'a+') as ofs: ofs.write(info) logger.info('finish: {}'.format(cmd))
def one_round(): for data_set_name in data_set_lst: for reorder_method in filtered_reorder_lst: for our_algorithm in our_exec_name_lst: for t_num in thread_num_lst : statistics_dir = os.sep.join( map(str, ['.', exp_res_root_name, folder_name, data_set_name, reorder_method, t_num])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'log'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat_clean'])) statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log' dstat_file_path = os.sep.join([statistics_dir, 'dstat', our_algorithm + '-dstat.log']) log_file_path = os.sep.join([statistics_dir, 'log', our_algorithm + '-raw.log']) logger.info('stat file path: {}'.format(statistics_file_path)) # 1st: write header append_header(statistics_file_path) append_header(dstat_file_path) append_header(log_file_path) # 2nd: run exec cmd algorithm_path = our_exec_path + os.sep + our_algorithm params_lst = map(str, ['cgexec -g memory:yche-exp' if with_c_group else '', algorithm_path, data_set_path + os.sep + data_set_name, reorder_method, statistics_file_path]) cmd = ' '.join(params_lst) logger.info('exec-cmd: {}'.format(cmd)) time_out = 3600 * 5 if 'cuda' not in our_algorithm else 600 my_env = os.environ.copy() def execute_cmd(my_cmd): logger.info('sub-process: {}'.format(my_cmd)) os.system(my_cmd) # 3rd: spawn a new process to run the exec dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path p = Process(target=execute_cmd, args=(dstat_cmd,)) p.start() my_env['OMP_NUM_THREADS'] = str(t_num) tle_flag, info, correct_info = time_out_util.run_with_timeout(cmd, timeout_sec=time_out, env=my_env) time_out_util.kill_term_recursive(p.pid) modify_dstat_file(dstat_file_path) # 4th: append outputs write_split(statistics_file_path) with open(statistics_file_path, 'a+') as ifs: ifs.write(correct_info) ifs.write('\nis_time_out:' + str(tle_flag)) ifs.write(my_splitter + time.ctime() + my_splitter) ifs.write('\n\n\n\n') if len(info) > 0: with open(log_file_path, 'a+') as ofs: ofs.write(info) logger.info('finish: {}'.format(cmd))
def one_round(eps_lst, c_lst): data_set_name = 'ca-GrQc' for our_algorithm in our_exec_name_lst: for eps in list(reversed(eps_lst)): for c in c_lst: statistics_dir = os.sep.join( map(str, [ exp_res_root_name, folder_name, data_set_name, c, eps ])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'log'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat_clean'])) statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log' dstat_file_path = os.sep.join([ statistics_dir, 'dstat', our_algorithm + '-dstat.log' ]) log_file_path = os.sep.join( [statistics_dir, 'log', our_algorithm + '-raw.log']) logger.info( 'stat file path: {}'.format(statistics_file_path)) # 1st: write header append_header(statistics_file_path) append_header(dstat_file_path) append_header(log_file_path) # 2nd: run exec cmd algorithm_path = our_exec_path + os.sep + our_algorithm params_lst = map(str, [ 'cgexec -g memory:yche-exp' if with_c_group else '', algorithm_path, data_set_name, eps, c, statistics_file_path ]) cmd = ' '.join(params_lst) logger.info('exec-cmd: {}'.format(cmd)) time_out = 600 my_env = os.environ.copy() my_env[ "OMP_NUM_THREADS"] = '1' if our_algorithm != 'probesim_ss_ap_bench' else '56' def execute_cmd(my_cmd): logger.info('sub-process: {}'.format(my_cmd)) os.system(my_cmd) # 3rd: spawn a new process to run the exec dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path p = Process(target=execute_cmd, args=(dstat_cmd, )) p.start() tle_flag, info, correct_info = time_out_util.run_with_timeout( cmd, timeout_sec=time_out, env=my_env) time_out_util.kill_term_recursive(p.pid) modify_dstat_file(dstat_file_path) # 4th: append outputs write_split(statistics_file_path) with open(statistics_file_path, 'a+') as ifs: ifs.write(correct_info) ifs.write('\nis_time_out:' + str(tle_flag)) ifs.write(my_splitter + time.ctime() + my_splitter) ifs.write('\n\n\n\n') if len(info) > 0: with open(log_file_path, 'a+') as ofs: ofs.write(info) logger.info('finish: {}'.format(cmd))
def one_round(): data_set_name = 'ca-HepTh' for our_algorithm in our_exec_name_lst: for update_tag in ['ins', 'del']: for edge_num in [str(i) for i in range(5000, 30000, 5000)]: statistics_dir = os.sep.join( map(str, [ exp_res_root_name, folder_name, data_set_name, update_tag, edge_num ])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'log'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat'])) os.system('mkdir -p ' + os.sep.join([statistics_dir, 'dstat_clean'])) statistics_file_path = statistics_dir + os.sep + our_algorithm + '.log' dstat_file_path = os.sep.join([ statistics_dir, 'dstat', our_algorithm + '-dstat.log' ]) log_file_path = os.sep.join( [statistics_dir, 'log', our_algorithm + '-raw.log']) logger.info( 'stat file path: {}'.format(statistics_file_path)) # 1st: write header append_header(statistics_file_path) append_header(dstat_file_path) append_header(log_file_path) # 2nd: run exec cmd algorithm_path = our_exec_path + os.sep + our_algorithm params_lst = map(str, [ 'cgexec -g memory:yche-exp' if with_c_group else '', algorithm_path, data_set_name, edge_num, update_tag, statistics_file_path ]) cmd = ' '.join(params_lst) logger.info('exec-cmd: {}'.format(cmd)) time_out = 800000 my_env = os.environ.copy() def execute_cmd(my_cmd): logger.info('sub-process: {}'.format(my_cmd)) os.system(my_cmd) # 3rd: spawn a new process to run the exec dstat_cmd = 'dstat -tcdrlmgyn --fs >> ' + dstat_file_path p = Process(target=execute_cmd, args=(dstat_cmd, )) p.start() tle_flag, info, correct_info = time_out_util.run_with_timeout( cmd, timeout_sec=time_out, env=my_env) time_out_util.kill_term_recursive(p.pid) modify_dstat_file(dstat_file_path) # 4th: append outputs write_split(statistics_file_path) with open(statistics_file_path, 'a+') as ifs: ifs.write(correct_info) ifs.write('\nis_time_out:' + str(tle_flag)) ifs.write(my_splitter + time.ctime() + my_splitter) ifs.write('\n\n\n\n') if len(info) > 0: with open(log_file_path, 'a+') as ofs: ofs.write(info) logger.info('finish: {}'.format(cmd))