def main(hparams): # Set up some stuff accoring to hparams hparams.n_input = np.prod(hparams.image_shape) utils.set_num_measurements(hparams) utils.print_hparams(hparams) # get inputs data_dict = model_input(hparams) estimator = utils.get_estimator(hparams, hparams.model_types[0]) print(estimator) hparams.checkpoint_dir = utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) h_hats_dict = {model_type: {} for model_type in hparams.model_types} for key, x in data_dict.iteritems(): if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([ os.path.isfile(save_path) for save_path in save_paths.values() ]) if is_saved: continue # Get Rx data Rx = data_dict[key]['Rx_data'] Tx = data_dict[key]['Tx_data'] H = data_dict[key]['H_data'] Pilot_Rx = utils.get_pilot(Rx) print('Pilot_shape', Pilot_Rx.shape) Pilot_Rx = Pilot_Rx[0::2] + Pilot_Rx[1::2] * 1j Pilot_Tx = utils.get_pilot(Tx) Pilot_Tx = Pilot_Tx[0::2] + Pilot_Tx[1::2] * 1j Pilot_complex = Pilot_Rx / Pilot_Tx Pilot = np.empty((Pilot_complex.size * 2, ), dtype=Pilot_Rx.dtype) Pilot[0::2] = np.real(Pilot_complex) Pilot[1::2] = np.imag(Pilot_complex) Pilot = np.reshape(Pilot, [1, -1]) / 2.5 # Construct estimates using each estimator h_hat = estimator(Tx, Rx, Pilot, hparams) # Compute and store measurement and l2 loss # measurement_losses['dcgan'][key] = utils.get_measurement_loss(h_hat, Tx, Rx) # l2_losses['dcgan'][key] = utils.get_l2_loss(h_hat, H) print "Processed upto image {0} / {1}".format(key + 1, len(data_dict)) # Checkpointing if (hparams.save_images) and ((key + 1) % hparams.checkpoint_iter == 0): # utils.checkpoint(key,h_hat, measurement_losses, l2_losses, save_image, hparams) utils.save_channel_image(key + 1, h_hat, hparams) utils.save_channel_mat(key + 1, h_hat, hparams) print '\nProcessed and saved first ', key + 1, 'channels\n'
def main(hparams): # Set up some stuff accoring to hparams hparams.n_input = np.prod(hparams.image_shape) utils.set_num_measurements(hparams) utils.print_hparams(hparams) # get inputs data_dict = model_input(hparams) estimator = utils.get_estimator(hparams, 'vae') utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) h_hats_dict = {model_type: {} for model_type in hparams.model_types} for key, x in data_dict.iteritems(): if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([ os.path.isfile(save_path) for save_path in save_paths.values() ]) if is_saved: continue # Get Rx data Rx = data_dict[key]['Rx_data'] Tx = data_dict[key]['Rx_data'] H = data_dict[key]['H_data'] # Construct estimates using each estimator h_hat = estimator(Tx, Rx, hparams) # Save the estimate h_hats_dict['vae'][key] = h_hat # Compute and store measurement and l2 loss measurement_losses['vae'][key] = utils.get_measurement_loss( h_hat, Tx, Rx) l2_losses['vae'][key] = utils.get_l2_loss(h_hat, H) print 'Processed upto image {0} / {1}'.format(key + 1, len(data_dict)) # Checkpointing if (hparams.save_images) and ((key + 1) % hparams.checkpoint_iter == 0): utils.checkpoint(key, h_hat, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved first ', key + 1, 'channels\n'
def process_or_load_hparams(out_dir, default_hparams, hparams_path): hparams = default_hparams # if a Hparams path is given as argument, override the default_hparams. hparams = utils.maybe_parse_standard_hparams(hparams, hparams_path) # extend HParams to add some parameters necessary for the training. hparams = process_hparams(hparams) # check compatibility of HParams check_hparams(hparams) # Save HParams utils.save_hparams(out_dir, hparams) # Print HParams print("Print hyperparameters:") utils.print_hparams(hparams) return hparams
def create_or_load_hparams(out_dir, default_hparams, flags): """Create hparams or load hparams from out_dir.""" hparams = utils.load_hparams(out_dir) if not hparams: hparams = default_hparams hparams = utils.maybe_parse_standard_hparams(hparams, flags.hparams_path) hparams.add_hparam("x_dim", hparams.img_width * hparams.img_height) else: hparams = utils.ensure_compatible_hparams(hparams, default_hparams, flags) # Save HParams utils.save_hparams(out_dir, hparams) # Print HParams utils.print_hparams(hparams) return hparams
def create_or_load_hparams(out_dir, default_hparams, flags): # if the out_dir already contains hparams file, load these hparams. hparams = utils.load_hparams(out_dir) if not hparams: hparams = default_hparams hparams = utils.maybe_parse_standard_hparams(hparams, flags.hparams_path) hparams = extend_hparams(hparams) else: #ensure that the loaded hparams and the command line hparams are compatible. If not, the command line hparams are overwritten! hparams = utils.ensure_compatible_hparams(hparams, default_hparams, flags) # Save HParams utils.save_hparams(out_dir, hparams) # Print HParams print("Print hyperparameters:") utils.print_hparams(hparams) return hparams
def create_or_load_hparams( out_dir, default_hparams, hparams_path, save_hparams=True): """Create hparams or load hparams from out_dir.""" hparams = utils.load_hparams(out_dir) if not hparams: hparams = default_hparams hparams = utils.maybe_parse_standard_hparams( hparams, hparams_path) else: hparams = ensure_compatible_hparams(hparams, default_hparams, hparams_path) hparams = extend_hparams(hparams) # Save HParams if save_hparams: utils.save_hparams(out_dir, hparams) for metric in hparams.metrics: utils.save_hparams(getattr(hparams, "best_" + metric + "_dir"), hparams) # Print HParams utils.print_hparams(hparams) return hparams
def main(hparams): # Set up some stuff according to hparams hparams.n_input = np.prod(hparams.image_shape) maxiter = hparams.max_outer_iter utils.print_hparams(hparams) # get inputs xs_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) x_hats_dict = {'dcgan' : {}} x_batch_dict = {} for key, x in xs_dict.iteritems(): if hparams.lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([os.path.isfile(save_path) for save_path in save_paths.values()]) if is_saved: continue x_batch_dict[key] = x if len(x_batch_dict) < hparams.batch_size: continue # Reshape input x_batch_list = [x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems()] x_batch = np.concatenate(x_batch_list) # Construct measurements A_outer = utils.get_outer_A(hparams) y_batch_outer=np.matmul(x_batch, A_outer) x_main_batch = 0.0 * x_batch z_opt_batch = np.random.randn(hparams.batch_size, 100) for k in range(maxiter): x_est_batch=x_main_batch + hparams.outer_learning_rate*(np.matmul((y_batch_outer-np.matmul(x_main_batch,A_outer)),A_outer.T)) estimator = estimators['dcgan'] x_hat_batch,z_opt_batch = estimator(x_est_batch,z_opt_batch, hparams) x_main_batch=x_hat_batch for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y = y_batch_outer[i] x_hat = x_hat_batch[i] # Save the estimate x_hats_dict['dcgan'][key] = x_hat # Compute and store measurement and l2 loss measurement_losses['dcgan'][key] = utils.get_measurement_loss(x_hat, A_outer, y) l2_losses['dcgan'][key] = utils.get_l2_loss(x_hat, x) print 'Processed upto image {0} / {1}'.format(key+1, len(xs_dict)) # Checkpointing if (hparams.save_images) and ((key+1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) #x_hats_dict = {'dcgan' : {}} print '\nProcessed and saved first ', key+1, 'images\n' x_batch_dict = {} # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict)) if hparams.print_stats: for model_type in hparams.model_types: print model_type mean_m_loss = np.mean(measurement_losses[model_type].values()) mean_l2_loss = np.mean(l2_losses[model_type].values()) print 'mean measurement loss = {0}'.format(mean_m_loss) print 'mean l2 loss = {0}'.format(mean_l2_loss) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print '\nDid NOT process last {} images because they did not fill up the last batch.'.format(len(x_batch_dict)) print 'Consider rerunning lazily with a smaller batch size.'
def train(self, train_writer_path, log_f, restore_folder=None, restore_name=None): if not self.is_training: return sess = self.sess saver = self.saver if train_writer_path: train_writer = tf.summary.FileWriter(train_writer_path, sess.graph) time.clock() sess.run(self.init) if restore_folder: restore_path = os.path.join(restore_folder, restore_name) saver.restore(sess, restore_path) utils.print_out('load from %s' % restore_path, log_f) # with self.graph.as_default(): # gru_att_cov_variables = [] # gru_att_cov_variables.extend(get_collections_from_scope('tgt_embedding')) # gru_att_cov_variables.extend(get_collections_from_scope('gru')) # gru_att_cov_variables.extend(get_collections_from_scope('att')) # gru_att_cov_variables.extend(get_collections_from_scope('coverage')) # gru_att_cov_variables.extend(get_collections_from_scope('project')) # init_gru_att_cov_variables = tf.variables_initializer(var_list=gru_att_cov_variables) # sess.run(init_gru_att_cov_variables) # utils.print_out('random init gru_att_cov variables', log_f) global_step = 0 epoch = 0 utils.print_hparams(cfg, f=log_f) sess.run(self.init_iter_train) # 初始化训练输入 utils.print_out('init_iter_train', log_f) sess.run(self.init_iter_vaild) # 初始化验证输入 utils.print_out('init_iter_vaild', log_f) learning_rate = 0 loss = 0 try: utils.print_out("start at %s" % get_local_time(), log_f) while epoch < cfg.epochs: epoch += 1 i_steps = 0 while i_steps < cfg.each_steps: try: _, loss, global_step, learning_rate, summary = \ sess.run([self.train_op, self.loss_train, self.global_step, self.learning_rate, self.train_summary]) i_steps += 1 except tf.errors.OutOfRangeError: sess.run(self.init_iter_train) if global_step % cfg.print_frq == 0: utils.print_out( 'epoch %d, step %d, gloSp %d, lr %.4f, loss %.4f' % (epoch, i_steps, global_step, learning_rate, loss), log_f) if global_step % cfg.summary_frq == 0: summary = sess.run(self.train_summary) if train_writer_path: train_writer.add_summary(summary, global_step=global_step) if epoch % cfg.val_frq == 0: val_count = 0 val_loss = 0 val_acc = 0 val_edit_dist = 0 true_sample_words = [''] pred_sample_words = [''] i_loss = 0 i_acc = 0 while val_count < cfg.val_steps: try: i_loss, i_acc, true_sample_words, pred_sample_words, summary = \ sess.run([self.loss_vaild, self.accuracy_vaild, self.train_lookUpTgt_vaild, self.infer_lookUpTgt_vaild, self.vaild_summary]) val_count += 1 except tf.errors.OutOfRangeError: sess.run(self.init_iter_vaild) val_loss += i_loss val_acc += i_acc c_val_edit_dist = [] for t, p in zip(true_sample_words, pred_sample_words): edit_dist = utils.normal_leven(t, p) c_val_edit_dist.append(edit_dist) c_val_edit_dist = sum(c_val_edit_dist) / float( len(c_val_edit_dist)) val_edit_dist += c_val_edit_dist val_acc /= val_count val_loss /= val_count val_edit_dist /= val_count timeStamp = int(time.time()) timeArray = time.localtime(timeStamp) styleTime = time.strftime("%Y_%m_%d_%H_%M_%S", timeArray) utils.print_out( '%s ### val loss %.4f, acc %.4f, edit_dist %.4f' % (styleTime, val_loss, val_acc, val_edit_dist), log_f) if train_writer_path: train_writer.add_summary(summary, global_step=global_step) test_show_size = min(cfg.test_show_size, len(true_sample_words)) for i in range(test_show_size): str_tr = ''.join(true_sample_words[i]) str_pd = ''.join(pred_sample_words[i]) utils.print_out(" ## true: %s" % (str_tr), log_f) utils.print_out(" pred: %s" % (str_pd), log_f) if epoch % cfg.save_frq == 0 and train_writer_path: checkPoint_path = os.path.join(train_writer_path, "checkPoint.model") saver.save(sess, checkPoint_path, global_step=global_step) utils.print_out( " global step %d, check point save to %s-%d" % (global_step, checkPoint_path, global_step), log_f) except Exception as e: utils.print_out( "!!!! Interrupt ## end training, global step %d" % (global_step), log_f) if len(e.args) > 0: utils.print_out("An error occurred. {}".format(e.args[-1]), log_f) traceback.print_exc() finally: if train_writer_path: checkPoint_path = os.path.join(train_writer_path, "end_checkPoint.model") saver.save(sess, checkPoint_path, global_step=global_step) utils.print_out( " end training, global step %d, check point save to %s-%d" % (global_step, checkPoint_path, global_step), log_f) utils.print_out("end at %s" % get_local_time(), log_f) return epoch
def main(hparams): # Set up some stuff according to hparams utils.set_up_dir(hparams.ckpt_dir) utils.set_up_dir(hparams.sample_dir) utils.print_hparams(hparams) # encode x_ph = tf.placeholder(tf.float32, [None, hparams.n_input], name='x_ph') z_mean, z_log_sigma_sq = model_def.encoder(hparams, x_ph, 'enc', reuse=False) # sample eps = tf.random_normal((hparams.batch_size, hparams.n_z), 0, 1, dtype=tf.float32) z_sigma = tf.sqrt(tf.exp(z_log_sigma_sq)) z = z_mean + z_sigma * eps # reconstruct logits, x_reconstr_mean = model_def.generator(hparams, z, 'gen', reuse=False) # generator sampler z_ph = tf.placeholder(tf.float32, [None, hparams.n_z], name='x_ph') _, x_sample = model_def.generator(hparams, z_ph, 'gen', reuse=True) # define loss and update op total_loss = model_def.get_loss(x_ph, logits, z_mean, z_log_sigma_sq) opt = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) update_op = opt.minimize(total_loss) # Sanity checks for var in tf.global_variables(): print var.op.name print '' # Get a new session sess = tf.Session() # Model checkpointing setup model_saver = tf.train.Saver() init_op = tf.global_variables_initializer() sess.run(init_op) # Attempt to restore variables from checkpoint start_epoch = utils.try_restore(hparams, sess, model_saver) # Get data iterator iterator = data_input.omniglot_data_iterator() # Training for epoch in range(start_epoch + 1, hparams.training_epochs): avg_loss = 0.0 num_batches = hparams.num_samples // hparams.batch_size batch_num = 0 for (x_batch_val, _) in iterator(hparams, num_batches): batch_num += 1 feed_dict = {x_ph: x_batch_val} _, loss_val = sess.run([update_op, total_loss], feed_dict=feed_dict) avg_loss += loss_val / hparams.num_samples * hparams.batch_size if batch_num % 100 == 0: x_reconstr_mean_val = sess.run(x_reconstr_mean, feed_dict={x_ph: x_batch_val}) z_val = np.random.randn(hparams.batch_size, hparams.n_z) x_sample_val = sess.run(x_sample, feed_dict={z_ph: z_val}) utils.save_images( np.reshape(x_reconstr_mean_val, [-1, 28, 28]), [10, 10], '{}/reconstr_{:02d}_{:04d}.png'.format( hparams.sample_dir, epoch, batch_num)) utils.save_images( np.reshape(x_batch_val, [-1, 28, 28]), [10, 10], '{}/orig_{:02d}_{:04d}.png'.format(hparams.sample_dir, epoch, batch_num)) utils.save_images( np.reshape(x_sample_val, [-1, 28, 28]), [10, 10], '{}/sampled_{:02d}_{:04d}.png'.format( hparams.sample_dir, epoch, batch_num)) if epoch % hparams.summary_epoch == 0: print "Epoch:", '%04d' % (epoch), 'Avg loss = {:.9f}'.format( avg_loss) if epoch % hparams.ckpt_epoch == 0: save_path = os.path.join(hparams.ckpt_dir, 'omniglot_vae_model') model_saver.save(sess, save_path, global_step=epoch) save_path = os.path.join(hparams.ckpt_dir, 'omniglot_vae_model') model_saver.save(sess, save_path, global_step=hparams.training_epochs - 1)
def main(hparams): # set up perceptual loss device = 'cuda:0' percept = PerceptualLoss( model="net-lin", net="vgg", use_gpu=device.startswith("cuda") ) utils.print_hparams(hparams) # get inputs xs_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses, lpips_scores, z_hats = utils.load_checkpoints(hparams) x_hats_dict = {model_type : {} for model_type in hparams.model_types} x_batch_dict = {} A = utils.get_A(hparams) noise_batch = hparams.noise_std * np.random.standard_t(2, size=(hparams.batch_size, hparams.num_measurements)) for key, x in xs_dict.items(): if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([os.path.isfile(save_path) for save_path in save_paths.values()]) if is_saved: continue x_batch_dict[key] = x if len(x_batch_dict) < hparams.batch_size: continue # Reshape input x_batch_list = [x.reshape(1, hparams.n_input) for _, x in x_batch_dict.items()] x_batch = np.concatenate(x_batch_list) # Construct noise and measurements y_batch = utils.get_measurements(x_batch, A, noise_batch, hparams) # Construct estimates using each estimator for model_type in hparams.model_types: estimator = estimators[model_type] x_hat_batch, z_hat_batch, m_loss_batch = estimator(A, y_batch, hparams) for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y_train = y_batch[i] x_hat = x_hat_batch[i] # Save the estimate x_hats_dict[model_type][key] = x_hat # Compute and store measurement and l2 loss measurement_losses[model_type][key] = m_loss_batch[key] l2_losses[model_type][key] = utils.get_l2_loss(x_hat, x) lpips_scores[model_type][key] = utils.get_lpips_score(percept, x_hat, x, hparams.image_shape) z_hats[model_type][key] = z_hat_batch[i] print('Processed upto image {0} / {1}'.format(key+1, len(xs_dict))) # Checkpointing if (hparams.save_images) and ((key+1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, lpips_scores, z_hats, save_image, hparams) x_hats_dict = {model_type : {} for model_type in hparams.model_types} print('\nProcessed and saved first ', key+1, 'images\n') x_batch_dict = {} # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, lpips_scores, z_hats, save_image, hparams) print('\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict))) if hparams.print_stats: for model_type in hparams.model_types: print(model_type) measurement_loss_list = list(measurement_losses[model_type].values()) l2_loss_list = list(l2_losses[model_type].values()) mean_m_loss = np.mean(measurement_loss_list) mean_l2_loss = np.mean(l2_loss_list) print('mean measurement loss = {0}'.format(mean_m_loss)) print('mean l2 loss = {0}'.format(mean_l2_loss)) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print('\nDid NOT process last {} images because they did not fill up the last batch.'.format(len(x_batch_dict))) print('Consider rerunning lazily with a smaller batch size.')
'measurement-type', 'noise-std', 'num-measurements', 'model-types', 'zprior_weight', 'dloss1_weight', 'lmbd', 'max-update-iter', 'num-random-restarts', ] for hparam in HPARAM_NAMES_FOR_GRID_SEARCH: PARSER.add_argument('--' + hparam, metavar=hparam + '-val', type=str, nargs='+', default=['0'], help='Values of ' + hparam) PARSER.add_argument( '--scripts-base-dir', type=str, default='../scripts/', help='Base directory to save scripts: Absolute path or relative to src' ) HPARAMS = PARSER.parse_args() utils.print_hparams(HPARAMS) create_scripts(HPARAMS, HPARAM_NAMES_FOR_GRID_SEARCH)
if (model_name == "NFFM"): hparam = NFFM_params elif (model_name == "DeepFM"): hparam = DeepFM_params elif (model_name == "FFM"): hparam = FFM_params elif (model_name == "FM"): hparam = FM_params elif (model_name == "DCN"): hparam = DCN_params elif (model_name == "XDeepFM"): hparam = XDeepFM_params elif (model_name == "AFM"): hparam = AFM_params utils.print_hparams(hparam) # 打印模型参数 model = ctrNet.build_model(hparam) # 建立模型 print("Start " + model_name) model.train(train_data=(train_df[features], train_df['label']), dev_data=(dev_df[features], dev_df['label'])) preds = model.infer(dev_data=(test_df[features], test_df['label'])) fpr, tpr, thresholds = metrics.roc_curve(test_df['label'] + 1, preds, pos_label=2) auc = metrics.auc(fpr, tpr) print("last model auc {}".format(auc)) print(model_name + " Done")
def main(hparams): # os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # Set up some stuff according to hparams utils.set_up_dir(hparams.ckpt_dir) utils.set_up_dir(hparams.sample_dir) utils.print_hparams(hparams) # encode x_ph = tf.placeholder(tf.float32, [None, hparams.n_input], name='x_ph') _,x_reconstr_mean,_, loss_list = model_def.model(hparams,x_ph,['enc','gen'],[False,False]) total_loss = tf.add_n(loss_list, name='total_loss') # z_mean, z_log_sigma_sq = model_def.encoder(hparams, x_ph, 'enc', reuse=False) # # # sample # eps = tf.random_normal((hparams.batch_size, hparams.n_z), 0, 1, dtype=tf.float32) # z_sigma = tf.sqrt(tf.exp(z_log_sigma_sq)) # z = z_mean + z_sigma * eps # # # reconstruct # logits, x_reconstr_mean, _ = model_def.generator(hparams, z, 'gen', reuse=False) # # generator sampler z_ph = tf.placeholder(tf.float32, [None, hparams.grid[-1]], name='x_ph') x_sample = [] for i in range(len(hparams.grid)): _, x_sample_tmp, _ = model_def.generator_i(hparams, model_def.slicer_dec(hparams,i,None,z_ph), 'gen', True,i) x_sample.append(x_sample_tmp) # _, x_sample, _ = model_def.generator(hparams, z_ph, 'gen', reuse=True) # # define loss and update op # total_loss = model_def.get_loss(x_ph, logits, z_mean, z_log_sigma_sq) opt = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) update_op = opt.minimize(total_loss) # print([el.name for el in tf.trainable_variables()]) # Sanity checks for var in tf.global_variables(): print(var.op.name) print('') # print([o.name for o in tf.trainable_variables()]) # Get a new session sess = tf.Session() # Model checkpointing setup model_saver = tf.train.Saver() init_op = tf.global_variables_initializer() sess.run(init_op) # Attempt to restore variables from checkpoint start_epoch = utils.try_restore(hparams, sess, model_saver) # Get data iterator iterator = data_input.mnist_data_iteratior(dataset=hparams.dataset) # Training for epoch in range(start_epoch+1, hparams.training_epochs): avg_loss = 0.0 num_batches = hparams.num_samples // hparams.batch_size batch_num = 0 for (x_batch_val, _) in iterator(hparams, num_batches): batch_num += 1 feed_dict = {x_ph: x_batch_val} _, loss_val = sess.run([update_op, total_loss], feed_dict=feed_dict) avg_loss += loss_val / hparams.num_samples * hparams.batch_size if batch_num % 100 == 0: x_reconstr_mean_val = sess.run(x_reconstr_mean, feed_dict={x_ph: x_batch_val}) z_val = np.random.randn(hparams.batch_size, hparams.grid[-1]) x_sample_val = sess.run(x_sample, feed_dict={z_ph: z_val}) # print(sess.run(hparams.track[0], feed_dict={z_ph: z_val})) # print(sess.run(hparams.track[1], feed_dict={z_ph: z_val})) # s1 = sess.run(hparams.x_ph_ref[0], feed_dict={x_ph: x_batch_val}) # s2 = sess.run(hparams.logits_ref[0], feed_dict={x_ph: x_batch_val}) # s3 = sess.run(loss_list, feed_dict={x_ph: x_batch_val}) # print(s1.shape) # print(s2.shape) # print(s3) utils.save_images(np.reshape(x_batch_val, [-1, 28, 28]), \ [10, 10], \ '{}/orig_{:02d}_{:04d}.png'.format(hparams.sample_dir, epoch, batch_num)) for i in range(len(hparams.grid)): utils.save_images(np.reshape(x_reconstr_mean_val[i], [-1, 28, 28]), [10, 10], '{}/reconstr_{}_{:02d}_{:04d}.png'.format(hparams.sample_dir, hparams.grid[i], epoch, batch_num)) utils.save_images(np.reshape(x_sample_val[i], [-1, 28, 28]), [10, 10], '{}/sampled_{}_{:02d}_{:04d}.png'.format(hparams.sample_dir, hparams.grid[i], epoch, batch_num)) if epoch % hparams.summary_epoch == 0: print("Epoch:", '%04d' % (epoch), 'Avg loss = {:.9f}'.format(avg_loss)) if epoch % hparams.ckpt_epoch == 0: save_path = os.path.join(hparams.ckpt_dir, '{}_vae_model_flex_hid'.format(hparams.dataset)+str('_'.join(map(str,hparams.grid)))) model_saver.save(sess, save_path, global_step=epoch) save_path = os.path.join(hparams.ckpt_dir, '{}_vae_model_flex_hid'.format(hparams.dataset)+str('_'.join(map(str,hparams.grid)))) model_saver.save(sess, save_path, global_step=hparams.training_epochs-1)
log_path = os.path.join(log_dir, 'train.log') logging.basicConfig(level=logging.DEBUG, # 控制台打印的日志级别 format='%(asctime)s - %(levelname)s: %(message)s', # 日志格式 handlers=[ logging.FileHandler(log_path), logging.StreamHandler(sys.stdout)] ) training_data = get_data_loader() evaluation_data = get_data_loader() logging.info(str(model)) logging.info(str(print_hparams(hp))) logging.info('Data loaded!') logging.info('Data size: ' + str(len(training_data))) logging.info('Total Model parameters: ' + f'{sum(p.numel() for p in model.parameters() if p.requires_grad):,}') epoch = int(os.path.basename(hp.restore_path).replace('.pt', '')) if hp.restore_path else 1 if hp.mode == 'train': while epoch < hp.training_epochs + 1: epoch_start_time = time.time() train(training_data) scheduler.step() eval(evaluation_data) if epoch % hp.checkpoint_save_interval == 0:
def main(hparams): hparams.n_input = np.prod(hparams.image_shape) maxiter = hparams.max_outer_iter utils.print_hparams(hparams) xs_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) x_hats_dict = {'dcgan': {}} x_batch_dict = {} for key, x in xs_dict.iteritems(): x_batch_dict[key] = x if len(x_batch_dict) < hparams.batch_size: continue x_coll = [ x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems() ] x_batch = np.concatenate(x_coll) A_outer = utils.get_outer_A(hparams) # 1bitify y_batch_outer = np.sign(np.matmul(x_batch, A_outer)) x_main_batch = 0.0 * x_batch z_opt_batch = np.random.randn(hparams.batch_size, 100) for k in range(maxiter): x_est_batch = x_main_batch + hparams.outer_learning_rate * ( np.matmul( (y_batch_outer - np.sign(np.matmul(x_main_batch, A_outer))), A_outer.T)) estimator = estimators['dcgan'] x_hat_batch, z_opt_batch = estimator(x_est_batch, z_opt_batch, hparams) x_main_batch = x_hat_batch for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y = y_batch_outer[i] x_hat = x_hat_batch[i] x_hats_dict['dcgan'][key] = x_hat measurement_losses['dcgan'][key] = utils.get_measurement_loss( x_hat, A_outer, y) l2_losses['dcgan'][key] = utils.get_l2_loss(x_hat, x) print 'Processed upto image {0} / {1}'.format(key + 1, len(xs_dict)) if (hparams.save_images) and ((key + 1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved first ', key + 1, 'images\n' x_batch_dict = {} if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict)) if hparams.print_stats: for model_type in hparams.model_types: print model_type mean_m_loss = np.mean(measurement_losses[model_type].values()) mean_l2_loss = np.mean(l2_losses[model_type].values()) print 'mean measurement loss = {0}'.format(mean_m_loss) print 'mean l2 loss = {0}'.format(mean_l2_loss) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print '\nDid NOT process last {} images because they did not fill up the last batch.'.format( len(x_batch_dict)) print 'Consider rerunning lazily with a smaller batch size.'
def main(hparams): # if not hparams.use_gpu: # os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # Set up some stuff accoring to hparams hparams.n_input = np.prod(hparams.image_shape) #hparams.stdv = 10 #adjust to HPARAM in model_def.py #hparams.mean = 0 #adjust to HPARAM in model_def.py utils.set_num_measurements(hparams) utils.print_hparams(hparams) hparams.bol = False # hparams.dict_flag = False # get inputs if hparams.input_type == 'dict-input':# or hparams.dict_flag: hparams_load_key = copy.copy(hparams) hparams_load_key.input_type = 'full-input' hparams_load_key.measurement_type = 'project' hparams_load_key.zprior_weight = 0.0 hparams.key_field = np.load(utils.get_checkpoint_dir(hparams_load_key, hparams.model_types[0])+'candidates.npy').item() print(hparams.measurement_type) xs_dict, label_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) sh = utils.SaveHandler() sh.load_or_init_all(hparams.save_images,hparams.model_types,sh.get_pkl_filepaths(hparams,use_all=True)) if label_dict is None: print('No labels exist.') del sh.class_loss # measurement_losses, l2_losses, emd_losses, x_orig, x_rec, noise_batch = utils.load_checkpoints(hparams) if hparams.input_type == 'gen-span': np.save(utils.get_checkpoint_dir(hparams, hparams.model_types[0])+'z.npy',hparams.z_from_gen) np.save(utils.get_checkpoint_dir(hparams, hparams.model_types[0])+'images.npy',hparams.images_mat) x_hats_dict = {model_type : {} for model_type in hparams.model_types} x_batch_dict = {} x_batch=[] x_hat_batch=[] # l2_losses2=np.zeros((len(xs_dict),1)) # distances_arr=[] image_distance =np.zeros((len(xs_dict),1)) hparams.x = [] # TO REMOVE for key, x in xs_dict.iteritems(): #//each batch once (x_batch_dict emptied at end) if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([os.path.isfile(save_path) for save_path in save_paths.values()]) if is_saved: continue x_batch_dict[key] = x hparams.x.append(x)#To REMOVE if len(x_batch_dict) < hparams.batch_size: continue # Reshape input x_batch_list = [x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems()] x_batch = np.concatenate(x_batch_list) # x_batch, known_distortion, distances = get_random_distortion(x_batch) # distances_arr[(key-1)*hparams.batch_size:key*hparams.batch_size] = distances # xs_dict[(key-1)*hparams.batch_size:key*hparams.batch_size] =x_batch # Construct noise and measurements recovered, optim = utils.load_if_optimized(hparams) if recovered and np.linalg.norm(optim.x_orig-x_batch) < 1e-10: hparams.optim = optim hparams.recovered = True else: hparams.recovered=False optim.x_orig = x_batch hparams.optim = optim A, noise_batch, y_batch, c_val = utils.load_meas(hparams,sh,x_batch,xs_dict) hparams.optim.noise_batch = noise_batch if c_val: continue if hparams.measurement_type == 'sample_distribution': plot_distribution(hparams,x_batch) # for i in range(z.shape[1]):#range(1): # plt.hist(z[i,:], facecolor='blue', alpha=0.5) # directory_distr = # pl.savefig("abc.png") elif hparams.measurement_type == 'autoencoder': plot_reconstruction(hparams,x_batch) else: # Construct estimates using each estimator for model_type in hparams.model_types: estimator = estimators[model_type] start = time.time() tmp = estimator(A, y_batch, hparams) if isinstance(tmp,tuple): x_hat_batch = tmp[0] sh.z_rec = tmp[1] else: x_hat_batch = tmp del sh.z_rec end = time.time() duration = end-start print('The calculation needed {} time'.format(datetime.timedelta(seconds=duration))) np.save(utils.get_checkpoint_dir(hparams, model_type)+'elapsed_time',duration) # DEBUGGING = [] for i, key in enumerate(x_batch_dict.keys()): # x = xs_dict[key]+known_distortion[i] x = xs_dict[key] y = y_batch[i] x_hat = x_hat_batch[i] # plt.figure() # plt.imshow(np.reshape(x_hat, [64, 64, 3])*255)#, interpolation="nearest", cmap=plt.cm.gray) # plt.show() # Save the estimate x_hats_dict[model_type][key] = x_hat # Compute and store measurement and l2 loss sh.measurement_losses[model_type][key] = utils.get_measurement_loss(x_hat, A, y) # DEBUGGING.append(np.sum((x_hat.dot(A)-y)**2)/A.shape[1]) sh.l2_losses[model_type][key] = utils.get_l2_loss(x_hat, x) if hparams.class_bol and label_dict is not None: try: sh.class_losses[model_type][key] = utils.get_classifier_loss(hparams,x_hat,label_dict[key]) except: sh.class_losses[model_type][key] = NaN warnings.warn('Class loss unsuccessfull, most likely due to corrupted memory. Simply retry.') if hparams.emd_bol: try: _,sh.emd_losses[model_type][key] = utils.get_emd_loss(x_hat, x) if 'nonneg' not in hparams.tv_or_lasso_mode and 'pca' in model_type: warnings.warn('EMD requires nonnegative images, for safety insert nonneg into tv_or_lasso_mode') except ValueError: warnings.warn('EMD calculation unsuccesfull (most likely due to negative images)') pass # if l2_losses[model_type][key]-measurement_losses[model_type][key]!=0: # print('NO') # print(y) # print(x) # print(np.mean((x-y)**2)) image_distance[i] = np.linalg.norm(x_hat-x) # l2_losses2[key] = np.mean((x_hat-x)**2) # print('holla') # print(l2_losses2[key]) # print(np.linalg.norm(x_hat-x)**2/len(xs_dict[0])) # print(np.linalg.norm(x_hat-x)/len(xs_dict[0])) # print(np.linalg.norm(x_hat-x)) print('Processed upto image {0} / {1}'.format(key+1, len(xs_dict))) sh.x_orig = x_batch sh.x_rec = x_hat_batch sh.noise = noise_batch #ACTIVATE ON DEMAND #plot_bad_reconstruction(measurement_losses,x_batch) # Checkpointing if (hparams.save_images) and ((key+1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, save_image, sh, hparams) x_hats_dict = {model_type : {} for model_type in hparams.model_types} print('\nProcessed and saved first ', key+1, 'images\n') x_batch_dict = {} if 'wavelet' in hparams.model_types[0]: print np.abs(sh.x_rec) print('The average sparsity is {}'.format(np.sum(np.abs(sh.x_rec)>=0.0001)/float(hparams.batch_size))) # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, save_image, sh, hparams) print('\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict))) if hparams.dataset in ['mnist', 'fashion-mnist']: if np.array(x_batch).size: utilsM.save_images(np.reshape(x_batch, [-1, 28, 28]), [8, 8],utils.get_checkpoint_dir(hparams, hparams.model_types[0])+'original.png') if np.array(x_hat_batch).size: utilsM.save_images(np.reshape(x_hat_batch, [-1, 28, 28]), [8, 8],utils.get_checkpoint_dir(hparams, hparams.model_types[0])+'reconstruction.png') for model_type in hparams.model_types: # print(model_type) mean_m_loss = np.mean(sh.measurement_losses[model_type].values()) mean_l2_loss = np.mean(sh.l2_losses[model_type].values()) #\|XHUT-X\|**2/784/64 if hparams.emd_bol: mean_emd_loss = np.mean(sh.emd_losses[model_type].values()) if label_dict is not None: mean_class_loss = np.mean(sh.class_losses[model_type].values()) print('mean class loss = {0}'.format(mean_class_loss)) # print(image_distance) mean_norm_loss = np.mean(image_distance)#sum_i(\|xhut_i-x_i\|)/64 # mean_rep_error = np.mean(distances_arr) # mean_opt_meas_error_pixel = np.mean(np.array(l2_losses[model_type].values())-np.array(distances_arr)/xs_dict[0].shape) # mean_opt_meas_error = np.mean(image_distance-distances_arr) print('mean measurement loss = {0}'.format(mean_m_loss)) # print np.sum(np.asarray(DEBUGGING))/64 print('mean l2 loss = {0}'.format(mean_l2_loss)) if hparams.emd_bol: print('mean emd loss = {0}'.format(mean_emd_loss)) print('mean distance = {0}'.format(mean_norm_loss)) print('mean distance pixelwise = {0}'.format(mean_norm_loss/len(xs_dict[xs_dict.keys()[0]]))) # print('mean representation error = {0}'.format(mean_rep_error)) # print('mean optimization plus measurement error = {0}'.format(mean_opt_meas_error)) # print('mean optimization plus measurement error per pixel = {0}'.format(mean_opt_meas_error_pixel)) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print('\nDid NOT process last {} images because they did not fill up the last batch.'.format(len(x_batch_dict))) print('Consider rerunning lazily with a smaller batch size.')
def main(hparams): hparams.n_input = np.prod(hparams.image_shape) hparams.model_type = 'vae' maxiter = hparams.max_outer_iter utils.print_hparams(hparams) xs_dict = model_input(hparams) # returns the images estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) x_hats_dict = {'vae': {}} x_batch_dict = {} for key, x in xs_dict.iteritems(): print key x_batch_dict[key] = x #placing images in dictionary if len(x_batch_dict) < hparams.batch_size: continue x_coll = [ x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems() ] #Generates the columns of input x x_batch = np.concatenate(x_coll) # Generates entire X A_outer = utils.get_outer_A(hparams) # Created the random matric A noise_batch = hparams.noise_std * np.random.randn( hparams.batch_size, 100) y_batch_outer = np.sign( np.matmul(x_batch, A_outer) ) # Multiplication of A and X followed by quantization on 4 levels #y_batch_outer = np.matmul(x_batch, A_outer) x_main_batch = 0.0 * x_batch z_opt_batch = np.random.randn(hparams.batch_size, 20) #Input to the generator of the GAN for k in range(maxiter): x_est_batch = x_main_batch + hparams.outer_learning_rate * ( np.matmul( (y_batch_outer - np.sign(np.matmul(x_main_batch, A_outer))), A_outer.T)) #x_est_batch = x_main_batch + hparams.outer_learning_rate * (np.matmul((y_batch_outer - np.matmul(x_main_batch, A_outer)), A_outer.T)) # Gradient decent in x is done estimator = estimators['vae'] x_hat_batch, z_opt_batch = estimator( x_est_batch, z_opt_batch, hparams) # Projectin on the GAN x_main_batch = x_hat_batch dist = np.linalg.norm(x_batch - x_main_batch) / 784 print 'cool' print dist for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y = y_batch_outer[i] x_hat = x_hat_batch[i] # Save the estimate x_hats_dict['vae'][key] = x_hat # Compute and store measurement and l2 loss measurement_losses['vae'][key] = utils.get_measurement_loss( x_hat, A_outer, y) l2_losses['vae'][key] = utils.get_l2_loss(x_hat, x) print 'Processed upto image {0} / {1}'.format(key + 1, len(xs_dict)) # Checkpointing if (hparams.save_images) and ((key + 1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) #x_hats_dict = {'dcgan' : {}} print '\nProcessed and saved first ', key + 1, 'images\n' x_batch_dict = {} # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict)) if hparams.print_stats: for model_type in hparams.model_types: print model_type mean_m_loss = np.mean(measurement_losses[model_type].values()) mean_l2_loss = np.mean(l2_losses[model_type].values()) print 'mean measurement loss = {0}'.format(mean_m_loss) print 'mean l2 loss = {0}'.format(mean_l2_loss) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print '\nDid NOT process last {} images because they did not fill up the last batch.'.format( len(x_batch_dict)) print 'Consider rerunning lazily with a smaller batch size.'
cfg.iter_routing=iter_routing if not debug: t = str(int(time.time())) out_dir = os.path.join(cfg.out_dir, "log_%s" % t) if not os.path.exists(out_dir): os.makedirs(out_dir) log_file = os.path.join(out_dir, "log_%s" % t) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) else: log_f = None out_dir = None utils.print_hparams(cfg, f=log_f) train(do_k_fold, out_dir, log_f) utils.print_out('END grid %d' % i) else: if not debug: t = str(int(time.time())) out_dir = os.path.join(cfg.out_dir, "log_%s" % t) if not os.path.exists(out_dir): os.makedirs(out_dir) log_file = os.path.join(out_dir, "log_%s" % t) log_f = tf.gfile.GFile(log_file, mode="a") utils.print_out("# log_file=%s" % log_file, log_f) else: log_f = None
'ratio_click_of_consumptionAbility_in_aid', 'ratio_click_of_age_in_advertiserId', 'ratio_click_of_productType_in_uid', 'ratio_click_of_productType_in_consumptionAbility' ], mutil_features=[ 'interest1', 'interest2', 'interest3', 'interest4', 'interest5', 'kw1', 'kw2', 'kw3', 'topic1', 'topic2', 'topic3', 'appIdAction', 'appIdInstall', 'marriageStatus', 'ct', 'os' ], ) hparams = create_hparams() hparams.path = './model/' utils.print_hparams(hparams) hparams.aid = [ 'aid', 'advertiserId', 'campaignId', 'creativeId', 'creativeSize', 'adCategoryId', 'productId', 'productType', 'cvr_of_creativeId_and_onehot2', 'cvr_of_creativeId_and_onehot9', 'cvr_of_creativeId_and_onehot16', 'cvr_of_creativeId_and_onehot10', 'cvr_of_creativeId_and_onehot15', 'cvr_of_creativeId_and_onehot14', 'cvr_of_creativeId_and_onehot13', 'cvr_of_creativeId_and_onehot18', 'cvr_of_aid_and_age', 'cvr_of_creativeSize', 'cvr_of_uid_and_adCategoryId', 'cvr_of_uid_and_creativeSize', 'ratio_click_of_productType_in_uid' ] hparams.user = [ 'age', 'gender', 'education', 'consumptionAbility', 'LBS', 'carrier', 'house', 'interest1', 'interest2', 'interest3', 'interest4', 'interest5', 'kw1', 'kw2', 'kw3', 'topic1', 'topic2', 'topic3', 'appIdAction',
def main(hparams): # Set up some stuff accoring to hparams hparams.n_input = np.prod(hparams.image_shape) utils.set_num_measurements(hparams) utils.print_hparams(hparams) # get inputs xs_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) x_hats_dict = {model_type: {} for model_type in hparams.model_types} x_batch_dict = {} for key, x in xs_dict.iteritems(): if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([ os.path.isfile(save_path) for save_path in save_paths.values() ]) if is_saved: continue x_batch_dict[key] = x if len(x_batch_dict) < hparams.batch_size: continue # Reshape input x_batch_list = [ x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems() ] x_batch = np.concatenate(x_batch_list) # Construct noise and measurements A = utils.get_A(hparams) noise_batch = hparams.noise_std * np.random.randn( hparams.batch_size, hparams.num_measurements) if hparams.measurement_type == 'project': y_batch = x_batch + noise_batch else: y_batch = np.matmul(x_batch, A) + noise_batch # Construct estimates using each estimator for model_type in hparams.model_types: estimator = estimators[model_type] x_hat_batch = estimator(A, y_batch, hparams) for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y = y_batch[i] x_hat = x_hat_batch[i] # Save the estimate x_hats_dict[model_type][key] = x_hat # Compute and store measurement and l2 loss measurement_losses[model_type][ key] = utils.get_measurement_loss(x_hat, A, y) l2_losses[model_type][key] = utils.get_l2_loss(x_hat, x) print('Processed upto image {0} / {1}'.format(key + 1, len(xs_dict))) # Checkpointing if (hparams.save_images) and ((key + 1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) x_hats_dict = { model_type: {} for model_type in hparams.model_types } print('\nProcessed and saved first ', key + 1, 'images\n') x_batch_dict = {} # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print('\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict))) if hparams.print_stats: for model_type in hparams.model_types: print(model_type) mean_m_loss = np.mean(measurement_losses[model_type].values()) mean_l2_loss = np.mean(l2_losses[model_type].values()) print('mean measurement loss = {0}'.format(mean_m_loss)) print('mean l2 loss = {0}'.format(mean_l2_loss)) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print( '\nDid NOT process last {} images because they did not fill up the last batch.' .format(len(x_batch_dict))) print('Consider rerunning lazily with a smaller batch size.')
def main(hparams): # Set up some stuff according to hparams utils.set_up_dir(hparams.ckpt_dir) utils.set_up_dir(hparams.sample_dir) utils.print_hparams(hparams) # encode x_ph = tf.placeholder(tf.float32, [None, hparams.n_input], name='x_ph') z_mean, z_log_sigma_sq = model_def.encoder(hparams, x_ph, 'enc', reuse=False) # sample eps = tf.random_normal((hparams.batch_size, hparams.n_z), 0, 1, dtype=tf.float32) z_sigma = tf.sqrt(tf.exp(z_log_sigma_sq)) z = z_mean + z_sigma * eps # reconstruct logits, x_reconstr_mean = model_def.generator(hparams, z, 'gen', reuse=False) # generator sampler z_ph = tf.placeholder(tf.float32, [None, hparams.n_z], name='x_ph') _, x_sample = model_def.generator(hparams, z_ph, 'gen', reuse=True) # define loss and update op total_loss = model_def.get_loss(x_ph, logits, z_mean, z_log_sigma_sq) opt = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate) update_op = opt.minimize(total_loss) # Sanity checks for var in tf.global_variables(): print var.op.name print '' # Get a new session sess = tf.Session() # Model checkpointing setup model_saver = tf.train.Saver() init_op = tf.global_variables_initializer() sess.run(init_op) # Attempt to restore variables from checkpoint start_epoch = utils.try_restore(hparams, sess, model_saver) # Get data iterator iterator = data_input.channel_data_iteratior(hparams) next_element = iterator.get_next() # Training for epoch in range(start_epoch + 1, hparams.training_epochs): avg_loss = 0.0 num_batches = hparams.num_samples // hparams.batch_size batch_num = 0 for i in range(num_batches): try: x_batch_val = sess.run(next_element['H_data']) x_batch_val = np.squeeze(x_batch_val) batch_num += 1 feed_dict = {x_ph: x_batch_val} _, loss_val = sess.run([update_op, total_loss], feed_dict=feed_dict) #print(loss_val) avg_loss += loss_val / hparams.num_samples * hparams.batch_size except tf.errors.OutOfRangeError: print("End of dataset") break if epoch % hparams.summary_epoch == 0: print "Epoch:", '%04d' % (epoch), 'Avg loss = {:.9f}'.format( avg_loss) if epoch % hparams.ckpt_epoch == 0: save_path = os.path.join(hparams.ckpt_dir, 'channel_vae_model') model_saver.save(sess, save_path, global_step=epoch) save_path = os.path.join(hparams.ckpt_dir, 'channel_vae_model') model_saver.save(sess, save_path, global_step=hparams.training_epochs - 1)
def main(hparams): # Set up some stuff accoring to hparams hparams.n_input = np.prod(hparams.image_shape) utils.set_num_measurements(hparams) utils.print_hparams(hparams) if hparams.dataset == 'mnist': hparams.n_z = latent_dim elif hparams.dataset == 'celebA': hparams.z_dim = latent_dim # get inputs xs_dict = model_input(hparams) estimators = utils.get_estimators(hparams) utils.setup_checkpointing(hparams) measurement_losses, l2_losses = utils.load_checkpoints(hparams) image_loss_mnist = [] meas_loss_mnist = [] x_hat_mnist = [] x_hats_dict = {model_type : {} for model_type in hparams.model_types} x_batch_dict = {} for key, x in xs_dict.iteritems(): if not hparams.not_lazy: # If lazy, first check if the image has already been # saved before by *all* estimators. If yes, then skip this image. save_paths = utils.get_save_paths(hparams, key) is_saved = all([os.path.isfile(save_path) for save_path in save_paths.values()]) if is_saved: continue x_batch_dict[key] = x if len(x_batch_dict) < hparams.batch_size: continue # Reshape input x_batch_list = [x.reshape(1, hparams.n_input) for _, x in x_batch_dict.iteritems()] x_batch = np.concatenate(x_batch_list) # Construct noise and measurements A = utils.get_A(hparams) noise_batch = hparams.noise_std * np.random.randn(hparams.batch_size, hparams.num_measurements) if hparams.measurement_type == 'project': y_batch = x_batch + noise_batch else: measure = np.matmul(x_batch, A) y_batch = np.absolute(measure) + noise_batch # Construct estimates using each estimator for model_type in hparams.model_types: x_main_batch = 10000*np.ones_like(x_batch) for k in range(num_restarts): print "Restart #", str(k+1) # Solve deep pr problem with random initial iterate init_iter = np.random.randn(hparams.batch_size, latent_dim) # First gradient descent z_opt_batch = init_iter estimator = estimators[model_type] items = estimator(A, y_batch, z_opt_batch, hparams) x_hat_batch1 = items[0] z_opt_batch1 = items[1] losses_val1 = items[2] x_hat_batch = x_hat_batch1 x_hat_batch = utils.resolve_ambiguity(x_hat_batch, x_batch, hparams.batch_size) # Use reflection of initial iterate z_opt_batch2 = -1*init_iter items = estimator(A, y_batch, z_opt_batch2, hparams) x_hat_batch2 = items[0] z_opt_batch2 = items[1] losses_val2 = items[2] x_hat_batch2 = utils.resolve_ambiguity(x_hat_batch2, x_batch, hparams.batch_size) x_hat_batchnew = utils.get_optimal_x_batch(x_hat_batch, x_hat_batch2, x_batch, hparams.batch_size) x_main_batch = utils.get_optimal_x_batch(x_hat_batchnew, x_main_batch, x_batch, hparams.batch_size) x_hat_batch = x_main_batch if hparams.dataset == 'mnist': utils.print_stats(x_hat_batch, x_batch, hparams.batch_size) for i, key in enumerate(x_batch_dict.keys()): x = xs_dict[key] y = y_batch[i] x_hat = x_hat_batch[i] # Save the estimate x_hats_dict[model_type][key] = x_hat # Compute and store measurement and l2 loss measurement_losses[model_type][key] = utils.get_measurement_loss(x_hat, A, y) meas_loss_mnist.append(utils.get_measurement_loss(x_hat, A, y)) l2_losses[model_type][key] = utils.get_l2_loss(x_hat, x) image_loss_mnist.append(utils.get_l2_loss(x_hat,x)) print 'Processed upto image {0} / {1}'.format(key+1, len(xs_dict)) # Checkpointing if (hparams.save_images) and ((key+1) % hparams.checkpoint_iter == 0): utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) x_hats_dict = {model_type : {} for model_type in hparams.model_types} print '\nProcessed and saved first ', key+1, 'images\n' x_batch_dict = {} # Final checkpoint if hparams.save_images: utils.checkpoint(x_hats_dict, measurement_losses, l2_losses, save_image, hparams) print '\nProcessed and saved all {0} image(s)\n'.format(len(xs_dict)) if hparams.print_stats: for model_type in hparams.model_types: mean_m_loss = np.mean(measurement_losses[model_type].values()) mean_l2_loss = np.mean(l2_losses[model_type].values()) print 'mean measurement loss = {0}'.format(mean_m_loss) print 'mean l2 loss = {0}'.format(mean_l2_loss) if hparams.image_matrix > 0: utils.image_matrix(xs_dict, x_hats_dict, view_image, hparams) # Warn the user that some things were not processsed if len(x_batch_dict) > 0: print '\nDid NOT process last {} images because they did not fill up the last batch.'.format(len(x_batch_dict)) print 'Consider rerunning lazily with a smaller batch size.'