if __name__ == "__main__": ''' #TODO: 100-iteration solver cp 100 iteration solver to directory to evaluate run 100-iterations solver (for various numbers of GPUs) parse results ''' #train_dir = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_r_64_64_incr_r_64_64_CEratio_0.125_freq_2' train_dir = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/' n_gpu = 32 gen_solver_prototxt(train_dir, n_gpu) training_cmd = './do_training.sh %s %d' %(train_dir, n_gpu) os.system(training_cmd) #TODO: parse results. latest_log = get_latest_log(train_dir, for_timing=True) time_stats = get_time_per_iter(latest_log) ''' log_fname = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/train_Mon_2015_12_14__16_07_30.log' time_stats = get_time_per_iter(log_fname) print time_stats '''
f.write(out_st)
f.close()
if __name__ == "__main__":
'''
#TODO: 100-iteration solver
cp 100 iteration solver to directory to evaluate
run 100-iterations solver (for various numbers of GPUs)
parse results
'''
#train_dir = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_r_64_64_incr_r_64_64_CEratio_0.125_freq_2'
train_dir = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/'
n_gpu = 32
gen_solver_prototxt(train_dir, n_gpu)
training_cmd = './do_training.sh %s %d' % (train_dir, n_gpu)
os.system(training_cmd)
#TODO: parse results.
latest_log = get_latest_log(train_dir, for_timing=True)
time_stats = get_time_per_iter(latest_log)
'''
log_fname = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/train_Mon_2015_12_14__16_07_30.log'
time_stats = get_time_per_iter(log_fname)
print time_stats
'''
#ASSUME: we have time_training_<datetime>.log in some of the net_dir subdirectories
if __name__ == "__main__":
log_fnames = []
#log_fnames.append('/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/train_Mon_2015_12_14__16_07_30.log')
#log_fnames.append('/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_r_64_64_incr_r_64_64_CEratio_1.000_freq_2/train_Sun_2015_12_20__19_35_20.log')
#log_fnames.append('/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/FireNet_8_fireLayers_base_r_64_64_incr_r_64_64_CEratio_0.125_freq_2/train_Thu_2015_12_17__21_30_39.log')
#log_fnames.append('/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/microbenchmark_1gpu/FireNet_8_fireLayers_base_64_64_64_incr_64_64_64_freq_2/train_Tue_2016_01_05__21_21_07.log')
#log_fnames.append('/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/microbenchmark_1gpu/FireNet_8_fireLayers_base_r_64_64_incr_r_64_64_CEratio_0.125_freq_2/train_Tue_2016_01_05__21_15_36.log')
net_dir = '/lustre/atlas/scratch/forresti/csc103/dnn_exploration/nets_nov2015_done/'
for d in sorted(os.listdir(net_dir)):
#TODO: possibly replace the following with 'get_latest_log'
train_dir = net_dir + '/' + d
for f in sorted(os.listdir(train_dir)):
if 'time_training' in f:
log_fnames.append(train_dir + '/' + f)
print log_fnames
for log_fname in log_fnames:
print log_fname
time_stats = get_time_per_iter(log_fname)
#print time_stats
print ' ', time_stats['mean'], 'sec per iter'
print '' #newline
