def set_dropout_on(self): DropoutLayer.SetDropoutOn()
def train_net(config): # UNPACK CONFIGS (flag_para_load, train_filenames, val_filenames, train_labels, val_labels, img_mean) = unpack_configs(config) # pycuda set up drv.init() dev = drv.Device(int(config['gpu'][-1])) ctx = dev.make_context() if flag_para_load: # zmq set up sock = zmq.Context().socket(zmq.PAIR) sock.connect('tcp://*****:*****@ iter = ', num_iter print 'training cost:', cost_ij if config['print_train_error']: print 'training error rate:', train_error() if flag_para_load and (count < len(minibatch_range)): load_send_queue.put('calc_finished') ############### Test on Validation Set ################## DropoutLayer.SetDropoutOff() this_validation_error, this_validation_loss = get_val_error_loss( rand_arr, shared_x, shared_y, val_filenames, val_labels, flag_para_load, img_mean, batch_size, validate_model, send_queue=load_send_queue, recv_queue=load_recv_queue) print('epoch %i: validation loss %f ' % (epoch, this_validation_loss)) print('epoch %i: validation error %f %%' % (epoch, this_validation_error * 100.)) val_record.append([this_validation_error, this_validation_loss]) np.save(config['weights_dir'] + 'val_record.npy', val_record) DropoutLayer.SetDropoutOn() ############################################ # Adapt Learning Rate step_idx = adjust_learning_rate(config, epoch, step_idx, val_record, learning_rate) # Save weights if epoch % config['snapshot_freq'] == 0: save_weights(layers, config['weights_dir'], epoch) np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy', learning_rate.get_value()) save_momentums(vels, config['weights_dir'], epoch) print('Optimization complete.')
def train_net(config, private_config): # UNPACK CONFIGS (train_videos_spatial_jhmdb,val_videos_spatial_jhmdb,train_videos_temporal_jhmdb,val_videos_temporal_jhmdb, train_targets,val_targets, train_labels_jhmdb,val_labels_jhmdb) = unpack_configs_jhmdb(config,gpu_id=private_config['gpu_id']) # print('val_len',len(val_videos_spatial_jhmdb),'train_len',len(train_videos_spatial_jhmdb)) if config['modal']=='rgb': train_videos = list(train_videos_spatial_jhmdb) test_videos = list(val_videos_spatial_jhmdb) else: train_videos = list(train_videos_temporal_jhmdb) test_videos = list(val_videos_temporal_jhmdb) print('jhmdb_len',len(train_videos),len(train_labels_jhmdb))#,len(tr_video_length_jhmdb)) flag_para_load =config['para_load'] gpu_send_queue = private_config['queue_gpu_send'] gpu_recv_queue = private_config['queue_gpu_recv'] # pycuda and zmq set up drv.init() dev = drv.Device(int(private_config['gpu'][-1])) ctx = dev.make_context() sock_gpu = zmq.Context().socket(zmq.PAIR) if private_config['flag_client']: sock_gpu.connect('tcp://*****:*****@ iter = ', num_iter print 'training cost:', cost_ij,'cost_nll:',cost_nll,'cost_attention:',cost_att if config['print_train_error']: error_ij = train_error() gpu_send_queue.put(error_ij) that_error = gpu_recv_queue.get() error_ij = (error_ij + that_error) / 2. if private_config['flag_verbose']: print 'training error rate:', error_ij if flag_para_load and (count < len(minibatch_range)): load_send_queue.put('calc_finished') if count%20 == 0: e = time.time() print "time per 20 iter:", (e - s) # ############### Test on Validation Set ################## DropoutLayer.SetDropoutOff() this_val_error, this_val_loss = get_test_error(config, shared_x, shared_mask, shared_y,shared_target,shared_use_noise, shared_conv,test_videos, val_labels_jhmdb, flag_para_load, batch_size,num_seq, validate_model_lstm,train_model, send_queue=load_send_queue, recv_queue=load_recv_queue) # report validation stats gpu_send_queue.put(this_val_error) that_val_error = gpu_recv_queue.get() this_val_error = (this_val_error + that_val_error) / 2. gpu_send_queue.put(this_val_loss) that_val_loss = gpu_recv_queue.get() this_val_loss = (this_val_loss + that_val_loss) / 2. if private_config['flag_verbose']: print('epoch %i: test loss of jhmdb %f ' % (epoch, this_val_loss)) print('epoch %i: test error of jhmdb %f %%' % (epoch, this_val_error * 100.)) val_record.append([this_val_error, this_val_loss]) if private_config['flag_save']: np.save(config['weights_dir'] + 'test_record_jhmdb.npy', val_record) DropoutLayer.SetDropoutOn() ########################################### # Adapt Learning Rate step_idx = adjust_learning_rate(config, epoch, step_idx, val_record, learning_rate) # Save Weights, only one of them will do if private_config['flag_save'] : if epoch % config['snapshot_freq'] == 0: save_weights(layers, config['weights_dir'], epoch) np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy', learning_rate.get_value()) save_momentums(vels, config['weights_dir'], epoch) print('Optimization complete.')
def train_net(config, private_config): # UNPACK CONFIGS (flag_para_load, flag_datalayer, train_filenames, val_filenames, train_labels, val_labels, img_mean) = \ unpack_configs(config, ext_data=private_config['ext_data'], ext_label=private_config['ext_label']) gpu_send_queue = private_config['queue_gpu_send'] gpu_recv_queue = private_config['queue_gpu_recv'] # pycuda and zmq set up drv.init() dev = drv.Device(int(private_config['gpu'][-1])) ctx = dev.make_context() sock_gpu = zmq.Context().socket(zmq.PAIR) if private_config['flag_client']: sock_gpu.connect('tcp://*****:*****@ iter = ', num_iter print 'training cost:', cost_ij if config['print_train_error']: error_ij = train_error() gpu_send_queue.put(error_ij) that_error = gpu_recv_queue.get() error_ij = (error_ij + that_error) / 2. if private_config['flag_verbose']: print 'training error rate:', error_ij if flag_para_load and (count < len(minibatch_range)): load_send_queue.put('calc_finished') ############### Test on Validation Set ################## DropoutLayer.SetDropoutOff() this_val_error, this_val_loss = get_val_error_loss( rand_arr, shared_x, shared_y, val_filenames, val_labels, flag_datalayer, flag_para_load, batch_size, validate_model, send_queue=load_send_queue, recv_queue=load_recv_queue) # report validation stats gpu_send_queue.put(this_val_error) that_val_error = gpu_recv_queue.get() this_val_error = (this_val_error + that_val_error) / 2. gpu_send_queue.put(this_val_loss) that_val_loss = gpu_recv_queue.get() this_val_loss = (this_val_loss + that_val_loss) / 2. if private_config['flag_verbose']: print('epoch %i: validation loss %f ' % (epoch, this_val_loss)) print('epoch %i: validation error %f %%' % (epoch, this_val_error * 100.)) val_record.append([this_val_error, this_val_loss]) if private_config['flag_save']: np.save(config['weights_dir'] + 'val_record.npy', val_record) DropoutLayer.SetDropoutOn() ############################################ # Adapt Learning Rate step_idx = adjust_learning_rate(config, epoch, step_idx, val_record, learning_rate) # Save Weights, only one of them will do if private_config['flag_save']: if epoch % config['snapshot_freq'] == 0: save_weights(layers, config['weights_dir'], epoch) np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy', learning_rate.get_value()) save_momentums(vels, config['weights_dir'], epoch) print('Optimization complete.')
def train_net(config): # UNPACK CONFIGS (flag_para_load, train_filenames, val_filenames, train_labels, val_labels, img_mean) = unpack_configs(config) if flag_para_load: # zmq set up sock = zmq.Context().socket(zmq.PAIR) sock.connect('tcp://*****:*****@iter " + str(count) if count == 1: s = time.time() if count == 20: e = time.time() print "time per 20 iter:", (e - s) logger.info("time per 20 iter: %f" % (e - s)) cost_ij = train_model_wrap(train_model, shared_x, shared_y, rand_arr, img_mean, count, minibatch_index, minibatch_range, batch_size, train_filenames, train_labels, flag_para_load, config['batch_crop_mirror'], send_queue=load_send_queue, recv_queue=load_recv_queue) if num_iter % config['print_freq'] == 0: #print 'training @ iter = ', num_iter #print 'training cost:', cost_ij logger.info("training @ iter = %i" % (num_iter)) logger.info("training cost: %lf" % (cost_ij)) if config['print_train_error']: logger.info('training error rate: %lf' % train_error()) #print 'training error rate:', train_error() if flag_para_load and (count < len(minibatch_range)): load_send_queue.put('calc_finished') ############### Test on Validation Set ################## #""" DropoutLayer.SetDropoutOff() # result_list = [ this_validation_error, this_validation_error_top5, this_validation_loss ] # or # result_list = [ this_validation_error, this_validation_loss ] result_list = get_val_error_loss( #this_validation_error, this_validation_loss = get_val_error_loss( rand_arr, shared_x, shared_y, val_filenames, val_labels, flag_para_load, img_mean, batch_size, validate_model, send_queue=load_send_queue, recv_queue=load_recv_queue, flag_top_5=flag_top5) logger.info(('epoch %i: validation loss %f ' % (epoch, result_list[-1]))) #print('epoch %i: validation loss %f ' % # (epoch, this_validation_loss)) if flag_top5: logger.info(('epoch %i: validation error (top 1) %f %%, (top5) %f %%' % (epoch, result_list[0] * 100., result_list[1] * 100.))) else: logger.info(('epoch %i: validation error %f %%' % (epoch, result_list[0] * 100.))) #print('epoch %i: validation error %f %%' % # (epoch, this_validation_error * 100.)) val_record.append(result_list) #val_record.append([this_validation_error, this_validation_loss]) np.save(config['weights_dir'] + 'val_record.npy', val_record) DropoutLayer.SetDropoutOn() ############################################ # Adapt Learning Rate step_idx = adjust_learning_rate(config, epoch, step_idx, val_record, learning_rate) # Save weights if epoch % config['snapshot_freq'] == 0: save_weights(layers, config['weights_dir'], epoch) np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy', learning_rate.get_value()) save_momentums(vels, config['weights_dir'], epoch) #""" print('Optimization complete.')
loss, error = validate_model() validation_losses.append(loss) validation_errors.append(error) return (np.mean(validation_losses), np.mean(validation_errors)) (val_loss, val_error) = validate(test_set) print('epoch %i: validation loss %f ' % (epoch, val_loss)) print('epoch %i: validation error %f %%' % (epoch, val_error * 100.)) val_record.append([val_error, val_loss]) np.save(config['weights_dir'] + 'val_record.npy', val_record) DropoutLayer.SetDropoutOn() ############################################ # Adapt Learning Rate step_idx = adjust_learning_rate(config, epoch, step_idx, val_record, learning_rate) # Save weights if epoch % config['snapshot_freq'] == 0: save_weights(layers, config['weights_dir'], epoch) np.save(config['weights_dir'] + 'lr_' + str(epoch) + '.npy', learning_rate.get_value()) save_momentums(vels, config['weights_dir'], epoch) print('Optimization complete.')