def generate_argmax_samples_and_gt_samples_class(session, inv_charmap, fake_inputs, disc_fake, gen, real_inputs_discrete, real_classes_discrete, feed_gt=True,iteration=None,seq_length=None,disc_class=None): scores = [] samples = [] samples_probabilites = [] samples_class_scores = [] if feed_gt: #train for i in range(10): argmax_samples, real_samples, samples_scores, class_scores = generate_samples(session, inv_charmap, fake_inputs, disc_fake, gen, real_inputs_discrete,real_classes_discrete, feed_gt=feed_gt,disc_class=disc_class) samples.extend(argmax_samples) scores.extend(samples_scores) samples_probabilites.extend(real_samples) samples_class_scores.extend(class_scores) else: #infrance per label for _label in gen.class_dict.keys(): for i in range(10): argmax_samples, real_samples, samples_scores, class_scores = generate_samples(session, inv_charmap, fake_inputs, disc_fake, gen, real_inputs_discrete,real_classes_discrete, feed_gt=feed_gt, single_label=_label,disc_class=disc_class) samples.extend(argmax_samples) scores.extend(samples_scores) samples_probabilites.extend(real_samples) samples_class_scores.extend(class_scores) #write label visualization log_samples(samples, scores, iteration, seq_length, "test_" + gen.class_dict[_label], class_scores=samples_class_scores) scores = [] samples = [] samples_probabilites = [] samples_class_scores = [] return samples, samples_probabilites, scores, class_scores
def run(iterations, seq_length, is_first, charmap, inv_charmap, prev_seq_length): if len(DATA_DIR) == 0: raise Exception( 'Please specify path to data directory in single_length_train.py!') # lines, _, _ = model_and_data_serialization.load_dataset(seq_length=seq_length, b_charmap=False, b_inv_charmap=False, # n_examples=FLAGS.MAX_N_EXAMPLES) # instance data handler data_handler = Runtime_data_handler( h5_path=FLAGS.H5_PATH, json_path=FLAGS.H5_PATH.replace('.h5', '.json'), seq_len=seq_length, # max_len=self.seq_len, # teacher_helping_mode='th_extended', use_var_len=False, batch_size=BATCH_SIZE, use_labels=False) #define placeholders real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length]) real_classes_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE]) global_step = tf.Variable(0, trainable=False) # build graph according to arch if FLAGS.ARCH == 'default': disc_cost, gen_cost, fake_inputs, disc_fake, disc_real, disc_on_inference, inference_op = define_objective( charmap, real_inputs_discrete, seq_length) disc_fake_class = None disc_real_class = None disc_on_inference_class = None visualize_text = generate_argmax_samples_and_gt_samples elif FLAGS.ARCH == 'class_conditioned': disc_cost, gen_cost, fake_inputs, disc_fake, disc_fake_class, disc_real, disc_real_class,\ disc_on_inference, disc_on_inference_class, inference_op = define_class_objective(charmap, real_inputs_discrete, real_classes_discrete, seq_length, num_classes=len(data_handler.class_dict)) visualize_text = generate_argmax_samples_and_gt_samples_class merged, train_writer = define_summaries(disc_cost, gen_cost, seq_length) disc_train_op, gen_train_op = get_optimization_ops(disc_cost, gen_cost, global_step) saver = tf.train.Saver(tf.trainable_variables()) with tf.Session() as session: session.run(tf.initialize_all_variables()) if not is_first: print("Loading previous checkpoint...") internal_checkpoint_dir = model_and_data_serialization.get_internal_checkpoint_dir( prev_seq_length) model_and_data_serialization.optimistic_restore( session, latest_checkpoint(internal_checkpoint_dir, "checkpoint")) restore_config.set_restore_dir( load_from_curr_session=True ) # global param, always load from curr session after finishing the first seq # gen = inf_train_gen(lines, charmap) data_handler.epoch_start(seq_len=seq_length) for iteration in range(iterations): start_time = time.time() # Train critic for i in range(CRITIC_ITERS): _data, _labels = data_handler.get_batch() if FLAGS.ARCH == 'class_conditioned': _disc_cost, _, real_scores, real_class_scores = session.run( [disc_cost, disc_train_op, disc_real, disc_real_class], feed_dict={ real_inputs_discrete: _data, real_classes_discrete: _labels }) else: _disc_cost, _, real_scores = session.run( [disc_cost, disc_train_op, disc_real], feed_dict={ real_inputs_discrete: _data, real_classes_discrete: _labels }) real_class_scores = None # Train G for i in range(GEN_ITERS): # _data = next(gen) _data, _labels = data_handler.get_batch() _ = session.run(gen_train_op, feed_dict={ real_inputs_discrete: _data, real_classes_discrete: _labels }) print("iteration %s/%s" % (iteration, iterations)) print("disc cost %f" % _disc_cost) # Summaries if iteration % 1000 == 999: # if iteration % 100 == 99: _data, _labels = data_handler.get_batch() summary_str = session.run(merged, feed_dict={ real_inputs_discrete: _data, real_classes_discrete: _labels }) train_writer.add_summary(summary_str, global_step=iteration) fake_samples, samples_real_probabilites, fake_scores, fake_class_scores = visualize_text( session, inv_charmap, fake_inputs, disc_fake, data_handler, real_inputs_discrete, real_classes_discrete, feed_gt=True, iteration=iteration, seq_length=seq_length, disc_class=disc_fake_class) log_samples(fake_samples, fake_scores, iteration, seq_length, "train", class_scores=fake_class_scores) log_samples(decode_indices_to_string(_data, inv_charmap), real_scores, iteration, seq_length, "gt", class_scores=real_class_scores) # inference test_samples, _, fake_scores, fake_class_scores = visualize_text( session, inv_charmap, inference_op, disc_on_inference, data_handler, real_inputs_discrete, real_classes_discrete, feed_gt=False, iteration=iteration, seq_length=seq_length, disc_class=disc_on_inference_class) # disc_on_inference, inference_op if not FLAGS.ARCH == 'class_conditioned': log_samples(test_samples, fake_scores, iteration, seq_length, "test") if iteration % FLAGS.SAVE_CHECKPOINTS_EVERY == FLAGS.SAVE_CHECKPOINTS_EVERY - 1: saver.save( session, model_and_data_serialization.get_internal_checkpoint_dir( seq_length) + "/ckp") data_handler.epoch_end() saver.save( session, model_and_data_serialization.get_internal_checkpoint_dir( seq_length) + "/ckp") session.close()
def run(iterations, seq_length, is_first, wordmap, inv_wordmap, prev_seq_length, meta_iter, max_meta_iter): ''' Performs a single run of a single meta iter of curriculum training also performs reallocate at the end iterations = the number of minibatches seq_length = the length of the sequences to create is_first = if we need to load from ckpt wordmap = for getting the naw from inv_wordmap = for decoding the samples prev_seq_len = for chosing the ckpt to load from meta_iter = the round of the seq_length training max_meta_iter = used to make sure the tensorboard graphs log using the correct global step ''' # make sure tf does not take up all the GPU mem (model size is not that large so there is unlikely to be fragmentation problems) config = tf.ConfigProto(log_device_placement=False) config.gpu_options.allow_growth = True # first one so copy from the initial location file if is_first and os.path.isfile('locations/word-0.locations') or FLAGS.WORDVECS is not None: copyfile('locations/word-0.locations', 'locations/word-%d.locations' % seq_length) # load the lines from the dataset along with the current wordmap and inv_wordmap lines, wordmap, inv_wordmap = load_dataset(seq_length=seq_length, n_examples=FLAGS.MAX_N_EXAMPLES) if not os.path.isfile('locations/word-0.locations'): copyfile('locations/word-%d.locations' % seq_length, 'locations/word-0.locations') # placeholders for the input from the datset real_inputs_discrete = [tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length]), \ tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length])] # global step global_step = tf.Variable(iterations, trainable=False, name='global-step') # indec of <naw> in the map naw_r, naw_c = wordmap['<naw>'][0], wordmap['<naw>'][1] session = tf.Session(config=config) # start the generator to get minibatches from the dataset gen = inf_train_gen(lines, wordmap, seq_length) # define the network #disc_cost, gen_cost, fake_inputs, disc_fake, disc_real, disc_on_inference, inference_op, realloc_op, d_cost_fake, d_cost_real = define_objective(session, wordmap, real_inputs_discrete, seq_length, naw_r, naw_c, gen) # get the summaries, optimizers, and saver optim_costs, discs, ops, log_costs, embeddings = define_objective(session, wordmap, real_inputs_discrete, seq_length, naw_r, naw_c, gen) #embed_config = tf.contrib.tensorboard.plugins.projector.ProjectorConfig() #r_embedding = embed_config.embeddings.add() #r_embedding.tensor_name = embeddings.name #r_embedding.metadata_path = '../../loations/metadata.tsv' disc_cost, gen_cost = optim_costs disc_fake, disc_real, disc_on_inference = discs fake_inputs, inference_op, realloc_op = ops d_cost_fake, d_cost_real = log_costs merged, train_writer = define_summaries(d_cost_real, d_cost_fake, gen_cost, disc_cost) disc_train_op, gen_train_op = get_optimization_ops(disc_cost, gen_cost, global_step) saver = tf.train.Saver(tf.trainable_variables()) with session.as_default(): # write the graph and init vars train_writer.add_graph(session.graph) session.run(tf.global_variables_initializer()) # if not is_first, we need to load from ckpt if is_first and PRETRAIN: optimistic_restore(session, tf.train.latest_checkpoint('pretrain', 'checkpoint')) restore_config.set_restore_dir(load_from_curr_session=True) elif not is_first: internal_checkpoint_dir = get_internal_checkpoint_dir(prev_seq_length) optimistic_restore(session, tf.train.latest_checkpoint(internal_checkpoint_dir, "checkpoint")) restore_config.set_restore_dir(load_from_curr_session=True) # cool progress bar with tqdm(total=iterations, ncols=150) as pbar: # train loop for iteration in range(iterations): # Train critic first for _ in range(CRITIC_ITERS): _data = next(gen) _disc_cost, _, _ = session.run([disc_cost, disc_train_op, disc_real], \ feed_dict={real_inputs_discrete[0]: _data[0], \ real_inputs_discrete[1]: _data[1]}) # Train generator for _ in range(GEN_ITERS): _data = next(gen) _, _g_cost = session.run([gen_train_op, gen_cost], feed_dict={real_inputs_discrete[0]:_data[0], \ real_inputs_discrete[1]: _data[1]}) # update progress bat with costs and inc its counter by 1 pbar.set_description("disc cost %f \t gen cost %f \t"% (_disc_cost, _g_cost)) pbar.update(1) # write sumamries if iteration % 100 == 99: _data = next(gen) summary_str = session.run(merged, feed_dict={real_inputs_discrete[0]:_data[0], \ real_inputs_discrete[1]:_data[1]}) iii = meta_iter*iterations + iteration + ((seq_length-1)*iterations*max_meta_iter) train_writer.add_summary(summary_str, \ global_step=iii) # generate and log ouput from training fake_samples, _, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_wordmap, fake_inputs, disc_fake, gen, real_inputs_discrete, feed_gt=True, method='argmax') log_samples(fake_samples, fake_scores, iii, seq_length, "gen-w-gt") # generate and log output from inference test_samples, _, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_wordmap, inference_op, disc_on_inference, gen, real_inputs_discrete, feed_gt=False, method='argmax') log_samples(test_samples, fake_scores, iii, seq_length, "gen-no-gt") #********************************************************************************* #if seq_length >= 40: # copy current location file to next sequence_length as we are going to gen sequences of length seq_len +1 for realloc if FLAGS.WORDVECS is None: copyfile('locations/word-%d.locations' % (seq_length), 'locations/word-%d.locations' % (seq_length+1)) if seq_length >= 0: # get the lines, note that wordmap and inv_wormap stay the same lines, _, _ = load_dataset(seq_length=48, n_examples=FLAGS.MAX_N_EXAMPLES,\ no_write=True) # start generator and perform the reallocation gen = inf_realloc_gen(lines, wordmap, seq_length+1) perf_reallocate(int(1000000), session, inv_wordmap, realloc_op, \ gen, seq_length, real_inputs_discrete, naw_r, naw_c) # realloc creates the new location file in seq_length +1 so we move it back if we are not at the last meta_iter if meta_iter != max_meta_iter -1: copyfile('locations/word-%d.locations' % (seq_length+1), 'locations/word-%d.locations' % (seq_length)) #os.remove('locations/word-%d.locations' % (seq_length+1)) os.remove('locations/word-%d.locations.string' % (seq_length+1)) # save the ckpt and close the session because we need to reset the graph saver.save(session, get_internal_checkpoint_dir(seq_length) + "/ckp") session.close()
def run(iterations, seq_length, is_first, charmap, inv_charmap, prev_seq_length): if len(DATA_DIR) == 0: raise Exception('Please specify path to data directory in single_length_train.py!') lines, _, _ = model_and_data_serialization.load_dataset(seq_length=seq_length, b_charmap=False, b_inv_charmap=False, n_examples=FLAGS.MAX_N_EXAMPLES) real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length]) global_step = tf.Variable(0, trainable=False) disc_cost, gen_cost, fake_inputs, disc_fake, disc_real, disc_on_inference, inference_op, other_ops = define_objective(charmap,real_inputs_discrete, seq_length, gan_type=FLAGS.GAN_TYPE, rnn_cell=RNN_CELL) merged, train_writer = define_summaries(disc_cost, gen_cost, seq_length) disc_train_op, gen_train_op = get_optimization_ops( disc_cost, gen_cost, global_step, FLAGS.DISC_LR, FLAGS.GEN_LR) saver = tf.train.Saver(tf.trainable_variables()) # Use JIT XLA compilation to speed up calculations config=tf.ConfigProto( log_device_placement=False, allow_soft_placement=True) config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 with tf.Session(config=config) as session: session.run(tf.initialize_all_variables()) if not is_first: print("Loading previous checkpoint...") internal_checkpoint_dir = model_and_data_serialization.get_internal_checkpoint_dir(prev_seq_length) model_and_data_serialization.optimistic_restore(session, latest_checkpoint(internal_checkpoint_dir, "checkpoint")) restore_config.set_restore_dir( load_from_curr_session=True) # global param, always load from curr session after finishing the first seq gen = inf_train_gen(lines, charmap) _gen_cost_list = [] _disc_cost_list = [] _step_time_list = [] for iteration in range(iterations): start_time = time.time() # Train critic for i in range(CRITIC_ITERS): _data = next(gen) if FLAGS.GAN_TYPE.lower() == "fgan": _disc_cost, _, real_scores, _ = session.run( [disc_cost, disc_train_op, disc_real, other_ops["alpha_optimizer_op"]], feed_dict={real_inputs_discrete: _data} ) elif FLAGS.GAN_TYPE.lower() == "wgan": _disc_cost, _, real_scores = session.run( [disc_cost, disc_train_op, disc_real], feed_dict={real_inputs_discrete: _data} ) else: raise ValueError( "Appropriate gan type not selected: {}".format(FLAGS.GAN_TYPE)) _disc_cost_list.append(_disc_cost) # Train G for i in range(GEN_ITERS): _data = next(gen) # in Fisher GAN, paper measures convergence by gen_cost instead of disc_cost # To measure convergence, gen_cost should start at a positive number and decrease # to zero. The lower, the better. _gen_cost, _ = session.run([gen_cost, gen_train_op], feed_dict={real_inputs_discrete: _data}) _gen_cost_list.append(_gen_cost) _step_time_list.append(time.time() - start_time) if FLAGS.PRINT_EVERY_STEP: print("iteration %s/%s"%(iteration, iterations)) print("disc cost {}"%_disc_cost) print("gen cost {}".format(_gen_cost)) print("total step time {}".format(time.time() - start_time)) # Summaries if iteration % FLAGS.PRINT_ITERATION == FLAGS.PRINT_ITERATION-1: _data = next(gen) summary_str = session.run( merged, feed_dict={real_inputs_discrete: _data} ) tf.logging.warn("iteration %s/%s"%(iteration, iterations)) tf.logging.warn("disc cost {} gen cost {} average step time {}".format( np.mean(_disc_cost_list), np.mean(_gen_cost_list), np.mean(_step_time_list))) _gen_cost_list, _disc_cost_list, _step_time_list = [], [], [] train_writer.add_summary(summary_str, global_step=iteration) fake_samples, samples_real_probabilites, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_charmap, fake_inputs, disc_fake, gen, real_inputs_discrete,feed_gt=True) log_samples(fake_samples, fake_scores, iteration, seq_length, "train") log_samples(decode_indices_to_string(_data, inv_charmap), real_scores, iteration, seq_length, "gt") test_samples, _, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_charmap, inference_op, disc_on_inference, gen, real_inputs_discrete, feed_gt=False) # disc_on_inference, inference_op log_samples(test_samples, fake_scores, iteration, seq_length, "test") if iteration % FLAGS.SAVE_CHECKPOINTS_EVERY == FLAGS.SAVE_CHECKPOINTS_EVERY-1: saver.save(session, model_and_data_serialization.get_internal_checkpoint_dir(seq_length) + "/ckp") saver.save(session, model_and_data_serialization.get_internal_checkpoint_dir(seq_length) + "/ckp") session.close()
def run(iterations, seq_length, is_first, charmap, inv_charmap, prev_seq_length): if len(DATA_DIR) == 0: raise Exception('Please specify path to data directory in single_length_train.py!') lines, _, _ = model_and_data_serialization.load_dataset(seq_length=seq_length, b_charmap=False, b_inv_charmap=False, n_examples=FLAGS.MAX_N_EXAMPLES) real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length]) global_step = tf.Variable(0, trainable=False) disc_cost, gen_cost, fake_inputs, disc_fake, disc_real, disc_on_inference, inference_op = define_objective(charmap, real_inputs_discrete, seq_length) merged, train_writer = define_summaries(disc_cost, gen_cost, seq_length) disc_train_op, gen_train_op = get_optimization_ops(disc_cost, gen_cost, global_step) saver = tf.train.Saver(tf.trainable_variables()) with tf.Session() as session: session.run(tf.initialize_all_variables()) if not is_first: print("Loading previous checkpoint...") internal_checkpoint_dir = model_and_data_serialization.get_internal_checkpoint_dir(prev_seq_length) model_and_data_serialization.optimistic_restore(session, latest_checkpoint(internal_checkpoint_dir, "checkpoint")) restore_config.set_restore_dir( load_from_curr_session=True) # global param, always load from curr session after finishing the first seq gen = inf_train_gen(lines, charmap) for iteration in range(iterations): start_time = time.time() # Train critic for i in range(CRITIC_ITERS): _data = next(gen) _disc_cost, _, real_scores = session.run( [disc_cost, disc_train_op, disc_real], feed_dict={real_inputs_discrete: _data} ) # Train G for i in range(GEN_ITERS): _data = next(gen) _ = session.run(gen_train_op, feed_dict={real_inputs_discrete: _data}) print("iteration %s/%s"%(iteration, iterations)) print("disc cost %f"%_disc_cost) # Summaries if iteration % 100 == 99: _data = next(gen) summary_str = session.run( merged, feed_dict={real_inputs_discrete: _data} ) train_writer.add_summary(summary_str, global_step=iteration) fake_samples, samples_real_probabilites, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_charmap, fake_inputs, disc_fake, gen, real_inputs_discrete, feed_gt=True) log_samples(fake_samples, fake_scores, iteration, seq_length, "train") log_samples(decode_indices_to_string(_data, inv_charmap), real_scores, iteration, seq_length, "gt") test_samples, _, fake_scores = generate_argmax_samples_and_gt_samples(session, inv_charmap, inference_op, disc_on_inference, gen, real_inputs_discrete, feed_gt=False) # disc_on_inference, inference_op log_samples(test_samples, fake_scores, iteration, seq_length, "test") if iteration % FLAGS.SAVE_CHECKPOINTS_EVERY == FLAGS.SAVE_CHECKPOINTS_EVERY-1: saver.save(session, model_and_data_serialization.get_internal_checkpoint_dir(seq_length) + "/ckp") saver.save(session, model_and_data_serialization.get_internal_checkpoint_dir(seq_length) + "/ckp") session.close()
def evaluate(EVAL_FLAGS): # load dict & params _, charmap, inv_charmap = model_and_data_serialization.load_dataset( seq_length=32, b_lines=False) seq_length = EVAL_FLAGS.seq_len N = EVAL_FLAGS.num_samples ckp_list, config_list = get_models_list() print("ALL MODELS: %0s" % ckp_list) for ckp_path, config_path in zip(ckp_list, config_list): model_name = "%0s_%0s" % (ckp_path.split('/')[2], ckp_path.split('/')[4]) #filter by model name if not model_name.startswith(EVAL_FLAGS.prefix_filter): print("NOT EVALUATING [%0s]" % model_name) continue tf.reset_default_graph() print("EVALUATING [%0s]" % model_name) print("restoring config:") FLAGS.DISC_STATE_SIZE = restore_param_from_config( config_path, 'DISC_STATE_SIZE') FLAGS.GEN_STATE_SIZE = restore_param_from_config( config_path, 'GEN_STATE_SIZE') print("DISC_STATE_SIZE [%0d]" % FLAGS.DISC_STATE_SIZE) print("GEN_STATE_SIZE [%0d]" % FLAGS.GEN_STATE_SIZE) lines, _, _ = model_and_data_serialization.load_dataset( seq_length=seq_length, b_charmap=False, b_inv_charmap=False, n_examples=FLAGS.MAX_N_EXAMPLES, dataset='heldout') real_inputs_discrete = tf.placeholder(tf.int32, shape=[BATCH_SIZE, seq_length]) global_step = tf.Variable(0, trainable=False) disc_cost, gen_cost, train_pred, train_pred_for_eval, disc_fake, disc_real, disc_on_inference, inference_op = define_objective( charmap, real_inputs_discrete, seq_length) # train_pred -> run session train_pred_all = np.zeros( [N, BATCH_SIZE, seq_length, train_pred.shape[2]], dtype=np.float32) sess = tf.Session() sess.run(tf.initialize_all_variables()) # load checkpoints # internal_checkpoint_dir = model_and_data_serialization.get_internal_checkpoint_dir(0) internal_checkpoint_dir = ckp_path model_and_data_serialization.optimistic_restore( sess, latest_checkpoint(internal_checkpoint_dir, "checkpoint")) restore_config.set_restore_dir( load_from_curr_session=True ) # global param, always load from curr session after finishing the first seq # create samples print('start creating samples..') test_samples, _, fake_scores = generate_argmax_samples_and_gt_samples( sess, inv_charmap, inference_op, disc_on_inference, None, real_inputs_discrete, feed_gt=False) log_samples(test_samples, fake_scores, 666, seq_length, "test") print('starting BPC eval...') BPC_list = [] if EVAL_FLAGS.short_run: end_line = BATCH_SIZE * 100 # 100 batches else: end_line = len(lines) - BATCH_SIZE + 1 # all data for start_line in range(0, end_line, BATCH_SIZE): t0 = time.time() _data = np.array([[charmap[c] for c in l] for l in lines[start_line:start_line + BATCH_SIZE]]) # rand N noise vectors and for each one - calculate train_pred. for i in range(N): train_pred_i = sess.run( train_pred_for_eval, feed_dict={real_inputs_discrete: _data}) train_pred_all[i, :, :, :] = train_pred_i # take average on each time step (first dimension) train_pred_average = np.mean(train_pred_all, axis=0) # compute BPC (char-based perplexity) train_pred_average_2d = train_pred_average.reshape([ train_pred_average.shape[0] * train_pred_average.shape[1], train_pred_average.shape[2] ]) real_data = _data.reshape([_data.shape[0] * _data.shape[1]]) BPC = 0 epsilon = 1e-20 for i in range(real_data.shape[0]): BPC -= np.log2(train_pred_average_2d[i, real_data[i]] + epsilon) BPC /= real_data.shape[0] print("BPC of start_line %d/%d = %.2f" % (start_line, len(lines), BPC)) print("t_iter = %.2f" % (time.time() - t0)) BPC_list.append(BPC) np.save('BPC_list_temp.npy', BPC_list) BPC_final = np.mean(BPC_list) print("[%0s]BPC_final = %.2f\n" % (ckp_path, BPC_final)) np.save("%0s_BPC_list.npy" % model_name, BPC_list) np.save("%0s_BPC_final.npy" % model_name, BPC_final)