def evaluate(epoch_id, model, train, test, batch_size=4, min_res_size=16): gen = batch_generator(batch_size, min_res_size, train, test, False) batch_n = next(gen) evalMetrics = EvaluationMetrics(batch_n) model.zero_grad() #s = perf_counter() for n in range(batch_n): X, y, _, bboxes = next(gen) with torch.no_grad(): prediction = model(X) # evaluate this batch #print(y, prediction) evalMetrics.evaluateBatch(y, prediction) del X, y, prediction, bboxes gc.collect() torch.cuda.empty_cache() loading(n+1, batch_n) #print("total time = ", perf_counter() - s) evalMetrics.getEvaluation(epoch_id) return evalMetrics.pixel_acc, evalMetrics.iou, evalMetrics.dice_coeff
def test(val_model, session): its = 0 val_summaries = [] val_loss = 0 true_labels = [] predicted_labels = [] for X_gene_seqs, X_regulator_expression, Y_gene_expression in dataset.batch_generator( dataset.getTest(), val_model.batch_size): feed_dict = { val_model.X_gene_seqs: X_gene_seqs, val_model.X_regulator_expression: X_regulator_expression / 10, val_model.Y_gene_expression: discretize(Y_gene_expression) } vl, val_summary, pred_lab = session.run([ val_model.expression_loss, val_model.summary, val_model.predicted_labels ], feed_dict=feed_dict) val_loss += vl val_summaries.append(val_summary) disc = discretize(Y_gene_expression) for k in range(len(pred_lab)): predicted_labels.append(pred_lab[k]) true_labels.append(disc[k]) its += 1 return val_loss / its, val_summaries, confusion_matrix( np.array(true_labels), np.array(predicted_labels))
def train(name, labels, unlabels, aug, resume): numpy.set_printoptions( precision=2, linewidth=140) # paths log_path = "logs/{}.json".format(name) out_path = "snapshots/" + name + ".{epoch:06d}.h5" echo('log path', log_path) echo('out path', out_path) lib.log.info(log_path, {'_commandline': { 'name': name, 'labels': labels, 'unlabels': unlabels, 'aug': aug, 'resume': resume}}) # init echo('train', (name, resume)) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) # dataset echo('dataset loading...') batch_size = 100 gen_train, gen_test = dataset.batch_generator(labels, unlabels, batch_size=batch_size, aug=aug) # model building echo('model building...') model = lib.model.build() model.summary() if resume: echo('Resume Learning from {}'.format(resume)) model.load_weights(resume, by_name=True) # x_tensor = model.layers[2].get_input_at(0) # z_tensor = model.layers[2].get_output_at(0) # embedding = K.function([x_tensor], [z_tensor]) # z_tensor = model.layers[5].get_input_at(0) # y_tensor = model.layers[5].get_output_at(0) # predict = K.function([z_tensor], [y_tensor]) # training echo('start learning...') callbacks = [ lib.log.JsonLog(log_path), # keras.callbacks.ModelCheckpoint(out_path, monitor='val_loss', save_weights_only=True) ] model.fit_generator( gen_train, epochs=50, steps_per_epoch=(59900 // batch_size), validation_data=gen_test, validation_steps=(1000 // batch_size), callbacks=callbacks)
def train(name, resume, batch_size, epochs, verbose): # paths log_path = "logs/{}.json".format(name) out_path = "snapshots/" + name + ".{epoch:06d}.h5" echo('log path', log_path) echo('out path', out_path) # init echo('train', locals()) logging.info(log_path, {'train': locals()}) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) # dataset echo('dataset loading...') seq_train, seq_valid = dataset.batch_generator(batch_size) # model building echo('model building...') model = Model.build() model.summary() if resume: echo('Resume Learning from {}'.format(resume)) model.load_weights(resume, by_name=True) # training echo('start learning...') callbacks = [ logging.JsonLog(log_path), keras.callbacks.ModelCheckpoint( out_path, monitor='val_loss', save_weights_only=True, save_best_only=True, ) ] model.fit_generator( seq_train, validation_data=seq_valid, shuffle=True, epochs=epochs, verbose=verbose, callbacks=callbacks, workers=1, use_multiprocessing=True, )
def test(model, session): its = 0 test_summaries = [] test_loss = 0 for X_gene_seqs, X_regulator_expression, Y_gene_expression in dataset.batch_generator( dataset.getTest(), model.batch_size): feed_dict = { model.X_gene_seqs: X_gene_seqs, model.X_regulator_expression: X_regulator_expression, model.Y_gene_expression: Y_gene_expression } tl, test_summary = session.run([model.expression_loss, model.summary], feed_dict=feed_dict) test_loss += tl test_summaries.append(test_summary) its += 1 return test_loss / its, test_summaries
def validate(val_model, session): its = 0 val_summaries = [] val_loss = 0 for X_gene_seqs, X_regulator_expression, Y_gene_expression in dataset.batch_generator( dataset.getVal(), val_model.batch_size): feed_dict = { val_model.X_gene_seqs: X_gene_seqs, val_model.X_regulator_expression: X_regulator_expression, val_model.Y_gene_expression: Y_gene_expression } vl, val_summary = session.run( [val_model.expression_loss, val_model.summary], feed_dict=feed_dict) val_loss += vl val_summaries.append(val_summary) its += 1 return val_loss / its, val_summaries
def train_epoch(train_model, val_model, session, global_iters, epoch, val_every=4000): summary_writer = tf.train.SummaryWriter(FLAGS.log_dir) i = 0 print("training") for X_gene_seqs, X_regulator_expression, Y_gene_expression in dataset.batch_generator( dataset.getTrain(), train_model.batch_size): feed_dict = { train_model.X_gene_seqs: X_gene_seqs, train_model.X_regulator_expression: X_regulator_expression, train_model.Y_gene_expression: Y_gene_expression } expression_loss, train_summary, _, seq, pred = session.run( [ train_model.expression_loss, train_model.summary, train_model.train_op, train_model.seq_features, train_model.expression_prediction ], feed_dict=feed_dict) summary_writer.add_summary(train_summary, global_iters) if i % 50 == 49: print "expression loss: %.4f" % expression_loss print "approximately %.4f percent remaining in epoch %d" % ( (100 * (1 - i * 32 / float(633906))), epoch) print pred if i % val_every == val_every - 1: val_loss, val_summaries = validate(val_model, session) for summary in val_summaries: summary_writer.add_summary(summary, global_iters) summary_writer.flush() print "validation loss was: %.4f" % val_loss i += 1 global_iters += 1 return global_iters
def model_train(model): callback = get_callbacks("predict_labels_validacc", model) # 自定义验证 my_callback = ValidCallback() callbacks = [my_callback] + callback print('!!!!!', model.metrics_names) tfrecode_file = FLAGS.train_data batch_size = FLAGS.batch_size n_classes = FLAGS.class_number suffle_buffer = 5000 train_gen = batch_generator(tfrecode_file, batch_size, suffle_buffer) steps_per_epoch = int((5822652 / batch_size) / 30) model.fit_generator( train_gen, # steps_per_epoch=30, # steps_per_epoch=steps_per_epoch, steps_per_epoch=FLAGS.steps_per_epoch, epochs=FLAGS.epochs, callbacks=callbacks, verbose=1)
def test(snapshot, batch_size): # init echo('test', locals()) session = tf.Session('') K.set_session(session) K.set_learning_phase(0) # model loading echo('model loading...') model = Model.build() model.load_weights(snapshot) # testing data echo('testing dataset loading...') seq_test = dataset.batch_generator(batch_size, test=True) # testing results = model.evaluate_generator(seq_test) for metrics, value in zip(model.metrics_names, results): print(f"{metrics}: {value}")
def train_epoch(train_model, val_model, session, global_iters, val_every=4000): summary_writer = tf.train.SummaryWriter(FLAGS.log_dir) i = 0 print("training") for X_gene_seqs, X_regulator_expression, Y_gene_expression in dataset.batch_generator( dataset.getTrain(), train_model.batch_size): feed_dict = { train_model.X_gene_seqs: X_gene_seqs, train_model.X_regulator_expression: X_regulator_expression / 10, train_model.Y_gene_expression: discretize(Y_gene_expression) } expression_loss, train_summary, _ = session.run([ train_model.expression_loss, train_model.summary, train_model.train_op ], feed_dict=feed_dict) summary_writer.add_summary(train_summary, global_iters) if i % 200 == 199: print discretize(Y_gene_expression) print "expression loss: %.4f" % expression_loss print "approximately %.4f percent remaining" % ( 100 * (1 - i * 32 / float(633906))) if i % val_every == val_every - 1: val_loss, val_summaries, val_confusion = validate( val_model, session) for summary in val_summaries: summary_writer.add_summary(summary, global_iters) summary_writer.flush() print "====VALIDATING====" print "validation loss was: %.4f" % val_loss print "confusion matrix", val_confusion print "====VALIDATING====" i += 1 global_iters += 1 return global_iters
def test(name, snapshot): config = Config(name) if snapshot is None: snapshots = sorted(glob(f"snapshots/{name}.*.h5"), reverse=True) if len(snapshots) == 0: echo(f"[Error] No snapshots found for name={name}", fg='red') return snapshot = snapshots[0] del snapshots # running parameters run_params = locals() del run_params['config'] run_params.update(dict(config)) # init echo('test', run_params) session = tf.Session('') K.set_session(session) K.set_learning_phase(0) # model loading echo(f'model loading from {snapshot}...') model = build(config) model.load_weights(snapshot) # testing data echo('test dataset loading...') seq_test = dataset.batch_generator(config('batch_size'), test=True) # testing echo('testing...') results = model.evaluate_generator(seq_test) for metrics, value in zip(model.metrics_names, results): print(f"{metrics}: {value}")
def train(name, critics, clip, batch_size, epochs, out, verbose): # paths log_path = "logs/{}.json".format(name) out_path = "snapshots/" + name + ".{epoch:06d}.h5" echo('log path', log_path) echo('out path', out_path) # init echo('train', locals()) logging.info(log_path, {'train': locals()}) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) # dataset echo('dataset loading...') data = dataset.batch_generator(batch_size) # model building echo('model building...') models = Model.build() echo('- Generator:') models[0].summary() echo('- Critic:') models[1].summary() # training echo('start learning...') Model.train(models, data, epochs, batch_size, critics, clip, out=out, log=lambda data: logging.info(log_path, data))
def train(name, resume, verbose): config = Config(name) # paths log_path = f"logs/{name}.json" out_path = f"snapshots/{name}.{{epoch:06d}}.h5" echo('log path', log_path) echo('out path', out_path) # running parameters run_params = locals() del run_params['config'] run_params.update(dict(config)) # init echo('train', run_params) log = logging.Logger(log_path) log({'train': run_params}) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) if len(list(glob(f"snapshots/{name}.*.h5"))) > 0: echo( f"Error: Some snapshots for name ({name}) already exists. Use another name", fg='red') return # dataset echo('dataset loading...') seq_train, seq_valid = dataset.batch_generator(config('batch_size')) # model building echo('model building...') model = build(config) model.summary() if resume: echo('Resume Learning from {}'.format(resume)) model.load_weights(resume, by_name=True) # training echo('start learning...') callbacks = [ logging.JsonLog(log_path), keras.callbacks.ModelCheckpoint( out_path, monitor='val_loss', save_weights_only=True, save_best_only=True, ), ] model.fit_generator( seq_train, validation_data=seq_valid, shuffle=True, epochs=config('epochs'), verbose=verbose, callbacks=callbacks, workers=1, use_multiprocessing=True, )
log = logging.Logger(log_path) log({'train': run_params}) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) if len(list(glob(f"snapshots/{name}.*.h5"))) > 0: echo(f"Error: Some snapshots for name ({name}) already exists. Use another name", fg='red') return # dataset echo('dataset loading...') {%- if cookiecutter.fit_generator == "no" %} X, y = dataset.load() {%- else %} seq_train, seq_valid = dataset.batch_generator(config('batch_size')) {%- endif %} # model building echo('model building...') model = build(config) model.summary() if resume: echo('Resume Learning from {}'.format(resume)) model.load_weights(resume, by_name=True) # training echo('start learning...') callbacks = [ logging.JsonLog(log_path), keras.callbacks.ModelCheckpoint(out_path,
dev = "cpu" device = torch.device(dev) m = IOGnet() m.to(device) min_res = 16 optimizer = torch.optim.Adam(m.parameters(), lr=lr) loss_fce = F.binary_cross_entropy epoch_losses = [] mean_epoch_losses = [] accuracies = [] g_train = batch_generator(batch_size, min_res, args.train, args.test ,False, False) batch_n = next(g_train) for n in range(21): epoch_losses = [] # First train model on dataset for i in range(batch_n): X, y, _ = next(g_train) # Predict result y_pred = m(X) loss = loss_fce(y_pred, y) loss.backward() optimizer.step()
def train(name, resume, batch_size, epochs, verbose): # paths log_path = None if name is None else f"logs/{name}.json" out_path = None if name is None else f"snapshots/{name}.{{epoch:06d}}.h5" echo('log path', log_path) echo('out path', out_path) result_dir = "result/{}".format(name) echo('result images', result_dir) if not os.path.exists(result_dir): os.makedirs(result_dir) # running parameters run_params = locals() # init echo('train', run_params) log = logging.Logger(log_path) log({'train': run_params}) session = tf.Session('') K.set_session(session) K.set_learning_phase(1) if name is None: echo("Warning: name is None. Models will never be saved.", fg='red') # dataset echo('dataset loading...') seq_train, seq_valid = dataset.batch_generator(batch_size) # model building echo('model building...') enc, dec, auto, gen, dis, gan_fake, gan_real, clf = Model.build() echo('Encoder', fg='yellow') enc.summary() echo('Decoder', fg='yellow') dec.summary() echo('Generator', fg='yellow') gen.summary() echo('Discriminator', fg='yellow') dis.summary() echo('GANs', fg='yellow') gan_fake.summary() gan_real.summary() echo('Classifier', fg='yellow') clf.summary() # training echo('start learning...') eps = 0.001 TRUE = numpy.ones((batch_size, )) - eps FALSE = numpy.zeros((batch_size, )) + eps def make_noise(): """() -> U""" x = numpy.random.randn(batch_size, Model.U_DIM) return x def interpolating(): U = make_noise() Z = gen.predict_on_batch(U) u = Z[0] v = Z[1] + 0.001 X = numpy.array( [a * u + (1 - a) * v for a in numpy.linspace(0, 3, batch_size)]) return X for epoch in range(epochs): INTERVAL = 20 auto_loss = 0 dis_fake_loss = 0 dis_real_loss = 0 gen_loss = 0 enc_loss = 0 clf_loss = 0 clf_acc = 0 last_metrics = () # Training for i, (X, y) in enumerate(seq_train): xsize = len(X) # Training autoencodfer auto.trainable = True enc.trainable = True dec.trainable = True auto_loss += auto.train_on_batch(X, X) # Training GAN (dis for real) clf.trainable = True dis.trainable = True enc.trainable = False gen.trainable = False z_real = enc.predict_on_batch(X) dis_real_loss += dis.train_on_batch(z_real, TRUE[:xsize]) # Training GAN (dis for fake) clf.trainable = True dis.trainable = True enc.trainable = False gen.trainable = False u = make_noise() z_fake = gen.predict_on_batch(u) dis_fake_loss += dis.train_on_batch(z_fake, FALSE) # Training GAN (gen) clf.trainable = False dis.trainable = False gen.trainable = True u = make_noise() gen_loss += gan_fake.train_on_batch(u, TRUE) # Training GAN (enc) dis.trainable = False clf.trainable = False enc.trainable = True enc_loss += gan_real.train_on_batch(X, TRUE[:xsize]) # Training Classifier (clf for real data) clf.trainable = True dis.trainable = True enc.trainable = False gen.trainable = False z_real = enc.predict_on_batch(X) _loss, _acc = clf.train_on_batch(z_real, y) clf_loss += _loss clf_acc += _acc if i % INTERVAL == INTERVAL - 1: if i > INTERVAL: click.echo('\r', nl=False) auto_loss /= INTERVAL dis_fake_loss /= INTERVAL dis_real_loss /= INTERVAL gen_loss /= INTERVAL enc_loss /= INTERVAL clf_loss /= INTERVAL clf_acc /= INTERVAL click.echo( f"Epoch:{epoch:4d} " f"Train: " f"auto={auto_loss:.4f} " f"dis_fake={dis_fake_loss:.4f} " f"dis_real={dis_real_loss:.4f} " f"gen={gen_loss:.4f} " f"enc={enc_loss:.4f} " f"clf={clf_loss:.4f},{clf_acc:.4f} ", nl=False) last_metrics = (auto_loss, dis_fake_loss, dis_real_loss, gen_loss, enc_loss, clf_loss, clf_acc) auto_loss = 0 dis_fake_loss = 0 dis_real_loss = 0 gen_loss = 0 enc_loss = 0 clf_loss = 0 clf_acc = 0 click.echo('') # Logging auto_loss, dis_fake_loss, dis_real_loss, \ gen_loss, enc_loss, clf_loss, clf_acc = last_metrics log({ 'epoch': epoch, 'loss': { 'auto': auto_loss, 'dis_fake_loss': dis_fake_loss, 'dis_real_loss': dis_real_loss, 'gen_loss': gen_loss, 'enc_loss': enc_loss, 'clf_loss': clf_loss, 'clf_acc': clf_acc, } }) # Testing X, _ = seq_valid[0] X_rec = auto.predict_on_batch(X) testing.imgsave(X_rec, f"{result_dir}/rec.{epoch:03d}.png") u = make_noise() z = gen.predict_on_batch(u) X_smp = dec.predict_on_batch(z) testing.imgsave(X_smp, f"{result_dir}/smp.{epoch:03d}.png") z = interpolating() X_int = dec.predict_on_batch(z) testing.imgsave(X_int, f"{result_dir}/int.{epoch:03d}.png")
x1 = decoder_block(x2, x1, self.decoder_block3, self.relu) x0 = decoder_block(x1, x0, self.decoder_block4, self.relu) # # START OF FINE NET # x4 = finenet_block(x4, self.fine_block1) x3 = finenet_block(x3, self.fine_block2) x2 = finenet_block(x2, self.fine_block3) x1 = finenet_block(x1, self.fine_block4) x0 = finenet_block(x0, self.fine_block5) x0 = self.final_conv1(torch.cat([x4, x3, x2, x1, x0], 1)) x0 = self.relu(self.final_bn1(x0)) x0 = self.relu(self.final_conv2(x0)) x0 = self.sigmoid(self.final_conv3(x0)) return x0 from dataset import batch_generator if __name__ == "__main__": m = IOGnet() g = batch_generator(1, 16, False, False) print(next(g)) X, _, _ = next(g) print(X.shape) y_pred = m(X) print(y_pred.shape)
training_model = Model(inputs=input_tensor, outputs=feature_tensors) training_model.summary() def save_int_image(epoch_idx, logs): if epoch_idx % 100 == 99: flattened_image_data = image_layer.get_weights()[0] # flattened_image_data = K.eval(flattened_image_tensor) image_data = np.reshape(flattened_image_data, (768, 1024, 3)) save_image(f'./images/result{epoch_idx:04}.jpeg', image_data) save_model = ModelCheckpoint('weights.{epoch:03d}-{loss:.2e}.hdf5', monitor='loss') optimizer = Adam(lr=0.001) #changed to 10 from 0.001 training_model.compile(loss='mean_squared_error', optimizer=optimizer, loss_weights=[2.5, *([1] * 5)]) training_model.fit_generator( #one example, one feature, the value is 1 since 1s function batch_generator(featurization_model, style_values), #batch gen gives x and y values steps_per_epoch=(num_batches // 32), epochs=3000 * 32, # callbacks=[LambdaCallback(on_epoch_end=save_int_image)] callbacks=[save_model]) # ls | head | parallel