def continue_train_model(last_batch_idx, data_stream, energy_optimizer, generator_optimizer, model_config_dict, model_test_name): model_list = glob.glob(samples_dir +'/*.pkl') # load parameters model_param_dicts = unpickle(model_list[0]) generator_models = load_generator_model(min_num_gen_filters=model_config_dict['min_num_gen_filters'], model_params_dict=model_param_dicts) generator_function = generator_models[0] generator_params = generator_models[1] energy_models = load_energy_model(num_experts=model_config_dict['expert_size'], model_params_dict=model_param_dicts) feature_function = energy_models[0] # norm_function = energy_models[1] expert_function = energy_models[1] # prior_function = energy_models[3] energy_params = energy_models[2] # compile functions print 'COMPILING MODEL UPDATER' t=time() generator_updater, generator_optimizer_params = set_generator_update_function(energy_feature_function=feature_function, # energy_norm_function=norm_function, energy_expert_function=expert_function, # energy_prior_function=prior_function, generator_function=generator_function, generator_params=generator_params, generator_optimizer=generator_optimizer, init_param_dict=model_param_dicts) energy_updater, energy_optimizer_params = set_energy_update_function(energy_feature_function=feature_function, # energy_norm_function=norm_function, energy_expert_function=expert_function, # energy_prior_function=prior_function, generator_function=generator_function, energy_params=energy_params, energy_optimizer=energy_optimizer, init_param_dict=model_param_dicts) print '%.2f SEC '%(time()-t) print 'COMPILING SAMPLING FUNCTION' t=time() sampling_function = set_sampling_function(generator_function=generator_function) print '%.2f SEC '%(time()-t) # set fixed hidden data for sampling fixed_hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'], high=model_config_dict['hidden_distribution'], size=(model_config_dict['num_display'], model_config_dict['hidden_size']))) print 'START TRAINING' # for each epoch input_energy_list = [] sample_energy_list = [] batch_count = 0 for e in xrange(model_config_dict['epochs']): # train phase batch_iters = data_stream.get_epoch_iterator() # for each batch for b, batch_data in enumerate(batch_iters): # batch count up batch_count += 1 if batch_count<last_batch_idx: continue # set update function inputs input_data = transform(batch_data[0]) num_data = input_data.shape[0] hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'], high=model_config_dict['hidden_distribution'], size=(num_data, model_config_dict['hidden_size']))) noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape)) update_input = [hidden_data, noise_data] update_output = generator_updater(*update_input) entropy_weights = update_output[1].mean() entropy_cost = update_output[2].mean() noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape)) update_input = [input_data, hidden_data, noise_data] update_output = energy_updater(*update_input) input_energy = update_output[0].mean() sample_energy = update_output[1].mean() input_energy_list.append(input_energy) sample_energy_list.append(sample_energy) if batch_count%10==0: print '================================================================' print 'BATCH ITER #{}'.format(batch_count), model_test_name print '================================================================' print ' TRAIN RESULTS' print '================================================================' print ' input energy : ', input_energy_list[-1] print '----------------------------------------------------------------' print ' sample energy : ', sample_energy_list[-1] print '----------------------------------------------------------------' print ' entropy weight : ', entropy_weights print '----------------------------------------------------------------' print ' entropy cost : ', entropy_cost print '================================================================' if batch_count%100==0: # sample data sample_data = sampling_function(fixed_hidden_data)[0] sample_data = np.asarray(sample_data) save_as = samples_dir + '/' + model_test_name + '_SAMPLES{}.png'.format(batch_count) color_grid_vis(inverse_transform(sample_data).transpose([0,2,3,1]), (16, 16), save_as) np.save(file=samples_dir + '/' + model_test_name +'_input_energy', arr=np.asarray(input_energy_list)) np.save(file=samples_dir + '/' + model_test_name +'_sample_energy', arr=np.asarray(sample_energy_list)) save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl' save_model(tensor_params_list=generator_params[0] + generator_params[1] + energy_params + generator_optimizer_params + energy_optimizer_params, save_to=save_as)
def train_model(data_stream, model_optimizer, model_config_dict, model_test_name): encoder_model = set_encoder_model(model_config_dict["hidden_size"], model_config_dict["min_num_gen_filters"]) encoder_function = encoder_model[0] encoder_parameters = encoder_model[1] decoder_model = set_decoder_model(model_config_dict["hidden_size"], model_config_dict["min_num_eng_filters"]) decoder_function = decoder_model[0] decoder_parameters = decoder_model[1] # compile functions print "COMPILING UPDATER FUNCTION" t = time() updater_function = set_updater_function( encoder_function=encoder_function, decoder_function=decoder_function, encoder_params=encoder_parameters, decoder_params=decoder_parameters, optimizer=model_optimizer, ) print "%.2f SEC " % (time() - t) print "COMPILING SAMPLING FUNCTION" t = time() sampling_function = set_sampling_function(decoder_function=decoder_function) print "%.2f SEC " % (time() - t) # set fixed hidden data for sampling fixed_hidden_data = floatX(np_rng.normal(size=(model_config_dict["num_display"], model_config_dict["hidden_size"]))) print "START TRAINING" # for each epoch recon_cost_list = [] moment_match_cost_list = [] model_cost_list = [] batch_count = 0 for e in xrange(model_config_dict["epochs"]): # train phase batch_iters = data_stream.get_epoch_iterator() # for each batch for b, batch_data in enumerate(batch_iters): # set update function inputs positive_visible_data = transform(batch_data[0]) negative_hidden_data = floatX( np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict["hidden_size"])) ) moment_cost_weight = 1.0 updater_inputs = [positive_visible_data, negative_hidden_data, moment_cost_weight] updater_outputs = updater_function(*updater_inputs) recon_cost = updater_outputs[0].mean() moment_match_cost = updater_outputs[1].mean() model_cost = updater_outputs[2].mean() recon_cost_list.append(recon_cost) moment_match_cost_list.append(moment_match_cost) model_cost_list.append(model_cost) # batch count up batch_count += 1 if batch_count % 1 == 0: print "================================================================" print "BATCH ITER #{}".format(batch_count), model_test_name print "================================================================" print " TRAIN RESULTS" print "================================================================" print " recon cost : ", recon_cost_list[-1] print "----------------------------------------------------------------" print " moment cost : ", moment_match_cost_list[-1] print "----------------------------------------------------------------" print " model cost : ", model_cost_list[-1] print "================================================================" if batch_count % 100 == 0: # sample data sample_data = sampling_function(fixed_hidden_data)[0] save_as = samples_dir + "/" + model_test_name + "_SAMPLES(NEGATIVE){}.png".format(batch_count) color_grid_vis(inverse_transform(sample_data).transpose([0, 2, 3, 1]), (16, 16), save_as) # save costs np.save(file=samples_dir + "/" + model_test_name + "_recon_cost", arr=np.asarray(recon_cost_list)) np.save( file=samples_dir + "/" + model_test_name + "_moment_cost", arr=np.asarray(moment_match_cost_list) ) np.save(file=samples_dir + "/" + model_test_name + "_model_cost", arr=np.asarray(model_cost_list)) if batch_count % 1000 == 0: save_as = samples_dir + "/" + model_test_name + "_MODEL.pkl" save_model(tensor_params_list=decoder_parameters, save_to=save_as)
def train_model(data_stream, energy_optimizer, generator_optimizer, model_config_dict, model_test_name): generator_models = set_generator_model( num_hiddens=model_config_dict["hidden_size"], min_num_gen_filters=model_config_dict["min_num_gen_filters"] ) generator_function = generator_models[0] generator_params = generator_models[1] energy_models = set_energy_model( num_experts=model_config_dict["expert_size"], min_num_eng_filters=model_config_dict["min_num_eng_filters"] ) feature_function = energy_models[0] # norm_function = energy_models[1] expert_function = energy_models[1] # prior_function = energy_models[3] energy_params = energy_models[2] # compile functions print "COMPILING MODEL UPDATER" t = time() generator_updater = set_generator_update_function( energy_feature_function=feature_function, # energy_norm_function=norm_function, energy_expert_function=expert_function, # energy_prior_function=prior_function, generator_function=generator_function, generator_params=generator_params, generator_optimizer=generator_optimizer, ) energy_updater = set_energy_update_function( energy_feature_function=feature_function, # energy_norm_function=norm_function, energy_expert_function=expert_function, # energy_prior_function=prior_function, generator_function=generator_function, energy_params=energy_params, energy_optimizer=energy_optimizer, ) print "%.2f SEC " % (time() - t) print "COMPILING SAMPLING FUNCTION" t = time() sampling_function = set_sampling_function(generator_function=generator_function) print "%.2f SEC " % (time() - t) # set fixed hidden data for sampling fixed_hidden_data = floatX( np_rng.uniform( low=-model_config_dict["hidden_distribution"], high=model_config_dict["hidden_distribution"], size=(model_config_dict["num_display"], model_config_dict["hidden_size"]), ) ) print "START TRAINING" # for each epoch input_energy_list = [] sample_energy_list = [] batch_count = 0 for e in xrange(model_config_dict["epochs"]): # train phase batch_iters = data_stream.get_epoch_iterator() # for each batch for b, batch_data in enumerate(batch_iters): # set update function inputs input_data = transform(batch_data[0]) num_data = input_data.shape[0] hidden_data = floatX( np_rng.uniform( low=-model_config_dict["hidden_distribution"], high=model_config_dict["hidden_distribution"], size=(num_data, model_config_dict["hidden_size"]), ) ) noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape)) update_input = [hidden_data, noise_data] update_output = generator_updater(*update_input) entropy_weights = update_output[1].mean() entropy_cost = update_output[2].mean() noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape)) update_input = [input_data, hidden_data, noise_data] update_output = energy_updater(*update_input) input_energy = update_output[0].mean() sample_energy = update_output[1].mean() input_energy_list.append(input_energy) sample_energy_list.append(sample_energy) # batch count up batch_count += 1 if batch_count % 10 == 0: print "================================================================" print "BATCH ITER #{}".format(batch_count), model_test_name print "================================================================" print " TRAIN RESULTS" print "================================================================" print " input energy : ", input_energy_list[-1] print "----------------------------------------------------------------" print " sample energy : ", sample_energy_list[-1] print "----------------------------------------------------------------" print " entropy weight : ", entropy_weights print "----------------------------------------------------------------" print " entropy cost : ", entropy_cost print "================================================================" if batch_count % 100 == 0: # sample data sample_data = sampling_function(fixed_hidden_data)[0] sample_data = np.asarray(sample_data) save_as = samples_dir + "/" + model_test_name + "_SAMPLES(TRAIN){}.png".format(batch_count) color_grid_vis(inverse_transform(sample_data).transpose([0, 2, 3, 1]), (16, 16), save_as) np.save(file=samples_dir + "/" + model_test_name + "_input_energy", arr=np.asarray(input_energy_list)) np.save(file=samples_dir + "/" + model_test_name + "_sample_energy", arr=np.asarray(sample_energy_list)) save_as = samples_dir + "/" + model_test_name + "_MODEL.pkl" save_model( tensor_params_list=generator_params[0] + generator_params[1] + energy_params, save_to=save_as )
def train_model(data_stream, energy_optimizer, generator_optimizer, generator_bn_optimizer, model_config_dict, model_test_name): [generator_function, generator_params, generator_bn_params] = set_generator_model(model_config_dict['hidden_size'], model_config_dict['min_num_gen_filters']) [feature_function, energy_function, energy_params] = set_energy_model(model_config_dict['expert_size'], model_config_dict['min_num_eng_filters']) # compile functions print 'COMPILING ENERGY UPDATER' t=time() energy_updater = set_energy_update_function(feature_function=feature_function, energy_function=energy_function, generator_function=generator_function, energy_params=energy_params, energy_optimizer=energy_optimizer) print '%.2f SEC '%(time()-t) print 'COMPILING GENERATOR UPDATER' t=time() generator_updater = set_generator_update_function(feature_function=feature_function, energy_function=energy_function, generator_function=generator_function, generator_params=generator_params, generator_bn_params=generator_bn_params, generator_optimizer=generator_optimizer, generator_bn_optimizer=generator_bn_optimizer) print '%.2f SEC '%(time()-t) print 'COMPILING SAMPLING FUNCTION' t=time() sampling_function = set_sampling_function(generator_function=generator_function) print '%.2f SEC '%(time()-t) # set fixed hidden data for sampling fixed_hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'], high=model_config_dict['hidden_distribution'], size=(model_config_dict['num_display'], model_config_dict['hidden_size']))) print 'START TRAINING' # for each epoch input_energy_list = [] sample_energy_list = [] batch_count = 0 for e in xrange(model_config_dict['epochs']): # train phase batch_iters = data_stream.get_epoch_iterator() # for each batch for b, batch_data in enumerate(batch_iters): # set update function inputs input_data = transform(batch_data[0]) num_data = input_data.shape[0] hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'], high=model_config_dict['hidden_distribution'], size=(num_data, model_config_dict['hidden_size']))) noise_data = floatX(np_rng.normal(scale=0.01*(0.99**int(batch_count/100)), size=(num_data, num_channels, input_shape, input_shape))) updater_inputs = [input_data, hidden_data, noise_data, batch_count] updater_outputs = generator_updater(*updater_inputs) noise_data = floatX(np_rng.normal(scale=0.01*(0.99**int(batch_count/100)), size=(num_data, num_channels, input_shape, input_shape))) updater_inputs = [input_data, hidden_data, noise_data, batch_count] updater_outputs = energy_updater(*updater_inputs) # get output values input_energy = updater_outputs[0].mean() sample_energy = updater_outputs[1].mean() input_energy_list.append(input_energy) sample_energy_list.append(sample_energy) # batch count up batch_count += 1 if batch_count%1==0: print '================================================================' print 'BATCH ITER #{}'.format(batch_count), model_test_name print '================================================================' print ' TRAIN RESULTS' print '================================================================' print ' input energy : ', input_energy_list[-1] print '----------------------------------------------------------------' print ' sample energy : ', sample_energy_list[-1] print '================================================================' if batch_count%1000==0: # sample data [sample_data_t, sample_data_f] = sampling_function(fixed_hidden_data) sample_data_t = np.asarray(sample_data_t) save_as = samples_dir + '/' + model_test_name + '_SAMPLES(TRAIN){}.png'.format(batch_count) color_grid_vis(inverse_transform(sample_data_t).transpose([0,2,3,1]), (16, 16), save_as) sample_data_f = np.asarray(sample_data_f) save_as = samples_dir + '/' + model_test_name + '_SAMPLES(TEST){}.png'.format(batch_count) color_grid_vis(inverse_transform(sample_data_f).transpose([0,2,3,1]), (16, 16), save_as) np.save(file=samples_dir + '/' + model_test_name +'_input_energy', arr=np.asarray(input_energy_list)) np.save(file=samples_dir + '/' + model_test_name +'_sample_energy', arr=np.asarray(sample_energy_list)) save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl' save_model(tensor_params_list=generator_params + generator_bn_params + energy_params, save_to=save_as)
def train_model(data_stream, model_optimizer, model_config_dict, model_test_name): encoder_model = set_encoder_model(model_config_dict['hidden_size'], model_config_dict['min_num_gen_filters']) encoder_feature_function = encoder_model[0] encoder_mean_function = encoder_model[1] encoder_var_function = encoder_model[2] encoder_parameters = encoder_model[3] decoder_model = set_decoder_model(model_config_dict['hidden_size'], model_config_dict['min_num_eng_filters']) decoder_function = decoder_model[0] decoder_parameters = decoder_model[1] # compile functions print 'COMPILING UPDATER FUNCTION' t=time() updater_function = set_updater_function(encoder_feature_function=encoder_feature_function, encoder_mean_function=encoder_mean_function, encoder_var_function=encoder_var_function, decoder_function=decoder_function, encoder_params=encoder_parameters, decoder_params=decoder_parameters, optimizer=model_optimizer) print '%.2f SEC '%(time()-t) print 'COMPILING SAMPLING FUNCTION' t=time() sampling_function = set_sampling_function(decoder_function=decoder_function) print '%.2f SEC '%(time()-t) # set fixed hidden data for sampling fixed_hidden_data = floatX(np_rng.normal(size=(model_config_dict['num_display'], model_config_dict['hidden_size']))) print 'START TRAINING' # for each epoch vae_cost_list = [] recon_cost_list = [] kl_cost_list = [] moment_match_cost_list = [] batch_count = 0 for e in xrange(model_config_dict['epochs']): # train phase batch_iters = data_stream.get_epoch_iterator() # for each batch for b, batch_data in enumerate(batch_iters): # set update function inputs positive_visible_data = transform(batch_data[0]) positive_hidden_data = floatX(np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict['hidden_size']))) negative_hidden_data = floatX(np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict['hidden_size']))) moment_cost_weight = 1.0 updater_inputs = [positive_visible_data, positive_hidden_data, negative_hidden_data, moment_cost_weight] updater_outputs = updater_function(*updater_inputs) vae_cost = updater_outputs[0].mean() recon_cost = updater_outputs[1].mean() kl_cost = updater_outputs[2].mean() moment_match_cost = updater_outputs[3].mean() recon_samples = updater_outputs[4] vae_cost_list.append(vae_cost) recon_cost_list.append(recon_cost) kl_cost_list.append(kl_cost) moment_match_cost_list.append(moment_match_cost) # batch count up batch_count += 1 if batch_count%10==0: print '================================================================' print 'BATCH ITER #{}'.format(batch_count), model_test_name print '================================================================' print ' TRAIN RESULTS' print '================================================================' print ' vae cost : ', vae_cost_list[-1] print '----------------------------------------------------------------' print ' recon cost : ', recon_cost_list[-1] print '----------------------------------------------------------------' print ' kl cost : ', kl_cost_list[-1] print '----------------------------------------------------------------' print ' moment cost : ', moment_match_cost_list[-1] print '================================================================' if batch_count%500==0: # sample data sample_data = sampling_function(fixed_hidden_data)[0] save_as = samples_dir + '/' + model_test_name + '_SAMPLES(NEGATIVE){}.png'.format(batch_count) color_grid_vis(inverse_transform(sample_data).transpose([0,2,3,1]), (16, 16), save_as) # recon data save_as = samples_dir + '/' + model_test_name + '_ORIGINALS(POSITIVE){}.png'.format(batch_count) color_grid_vis(inverse_transform(positive_visible_data).transpose([0,2,3,1]), (12, 12), save_as) save_as = samples_dir + '/' + model_test_name + '_RECONSTRUCTIONS(POSITIVE){}.png'.format(batch_count) color_grid_vis(inverse_transform(recon_samples).transpose([0,2,3,1]), (12, 12), save_as) # save costs np.save(file=samples_dir + '/' + model_test_name +'_vae_cost', arr=np.asarray(vae_cost_list)) np.save(file=samples_dir + '/' + model_test_name +'_recon_cost', arr=np.asarray(recon_cost_list)) np.save(file=samples_dir + '/' + model_test_name +'_kl_cost', arr=np.asarray(kl_cost_list)) np.save(file=samples_dir + '/' + model_test_name +'_moment_cost', arr=np.asarray(moment_match_cost_list)) if batch_count%1000==0: save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl' save_model(tensor_params_list=decoder_parameters, save_to=save_as)