def _predict(self, data_generator_function, steps_per_epoch, include_datum=False): data_generator = data_generator_function(include_datum=True) enqueuer = GeneratorEnqueuer(data_generator) enqueuer.start(workers=self._WORKERS, max_queue_size=self._MAX_Q_SIZE) caption_results = [] datum_results = [] for _ in range(steps_per_epoch): generator_output = None while enqueuer.is_running(): if not enqueuer.queue.empty(): generator_output = enqueuer.queue.get() break else: sleep(self._WAIT_TIME) x, y, datum_batch = generator_output[1] captions_pred_str = self._predict_batch(x, y) caption_results += captions_pred_str datum_results += datum_batch enqueuer.stop() if include_datum: return zip(caption_results, datum_results) else: return caption_results
def test_generator_enqueuer_threadsafe(): enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs( DummySequence([3, 10, 10, 3])), use_multiprocessing=False) enqueuer.start(3, 10) gen_output = enqueuer.get() with pytest.raises(RuntimeError) as e: [next(gen_output) for _ in range(10)] assert 'thread-safe' in str(e.value) enqueuer.stop()
def test_finite_generator_enqueuer_threads(): enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_threads( DummySequence([3, 200, 200, 3])), use_multiprocessing=False) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for output in gen_output: acc.append(int(output[0, 0, 0, 0])) assert set(acc) == set(range(100)), "Output is not the same" enqueuer.stop()
def test_finite_generator_enqueuer_processes(): enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_pcs( TestSequence([3, 200, 200, 3])), use_multiprocessing=True) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for output in gen_output: acc.append(int(output[0, 0, 0, 0])) assert acc != list(range(100)), "Order was keep in GeneratorEnqueuer with processes" enqueuer.stop()
def test_finite_generator_enqueuer_threads(): enqueuer = GeneratorEnqueuer(create_finite_generator_from_sequence_threads( TestSequence([3, 200, 200, 3])), use_multiprocessing=False) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for output in gen_output: acc.append(int(output[0, 0, 0, 0])) assert len(set(acc) - set(range(100))) == 0, "Output is not the same" enqueuer.stop()
def train_gan(gan, data_generator, cf): check = _checkpoint(cf.output_path) logs = _logs_manager(cf.output_path, gan) train_datagen = GeneratorEnqueuer( data_generator, use_multiprocessing=cf.use_multiprocessing) train_datagen.start(cf.workers, cf.max_queue_size) train_generator = train_datagen.get() for epoch in range(cf.epochs): print('Epoch %d/%d' % (epoch + 1, cf.epochs)) progbar = generic_utils.Progbar(data_generator.nb_steps) # Training loop for step in range(data_generator.nb_steps): batch_real = next(train_generator) batch_gray = np.expand_dims(batch_real[:, :, :, 0], -1) batch_chroma = batch_real[:, :, :, 1:] batch_fake = gan.combined.predict(batch_gray)[-1] dis_real_X = np.concatenate((batch_gray, batch_chroma), axis=-1) dis_fake_X = np.concatenate((batch_gray, batch_fake), axis=-1) d_loss = [] for d, real_Y, fake_Y in zip(gan.discriminator, gan.dis_real_Y, gan.dis_fake_Y): d_real = d.train_on_batch(dis_real_X, real_Y) d_fake = d.train_on_batch(dis_fake_X, fake_Y) d_loss.append(0.5 * np.add(d_real, d_fake)) g_loss = gan.combined.train_on_batch( batch_gray, gan.gen_real_Y + [batch_chroma]) if step % (data_generator.nb_steps // cf.plots_per_epoch) == 0 and step != 0: logs.save_plots(epoch, step, data_generator.decoder(dis_fake_X), data_generator.decoder(batch_real)) if step % (data_generator.nb_steps // cf.weights_per_epoch) == 0 and step != 0 and epoch != 0: check.save_weights(gan, epoch, step) d_names = ['d_loss_%d' % i for i in range(cf.d_scales)] g_names = ['g_cgan_%d' % i for i in range(cf.d_scales)] logs.update( names=['d_loss'] + d_names + ['g_loss'] + g_names + ['g_l1'], values=[sum(d_loss)] + d_loss + [g_loss[0]] + g_loss[1:-1] + [g_loss[-1]], progbar=progbar, display_step=cf.display_step) train_datagen.stop()
def test_generator_enqueuer_processes(): enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs( DummySequence([3, 200, 200, 3])), use_multiprocessing=True) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for i in range(100): acc.append(int(next(gen_output)[0, 0, 0, 0])) assert acc != list(range(100)), ('Order was keep in GeneratorEnqueuer ' 'with processes') enqueuer.stop()
def test_generator_enqueuer_processes(): enqueuer = GeneratorEnqueuer(create_generator_from_sequence_pcs( DummySequence([3, 10, 10, 3])), use_multiprocessing=True) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for i in range(100): acc.append(int(next(gen_output)[0, 0, 0, 0])) assert acc != list(range(100)), ('Order was keep in GeneratorEnqueuer ' 'with processes') enqueuer.stop()
def test_generator_enqueuer_threads(): enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads( DummySequence([3, 200, 200, 3])), use_multiprocessing=False) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for i in range(100): acc.append(int(next(gen_output)[0, 0, 0, 0])) """ Not comparing the order since it is not guaranteed. It may get ordered, but not a lot, one thread can take the GIL before he was supposed to. """ assert len(set(acc) - set(range(100))) == 0, "Output is not the same" enqueuer.stop()
def test_generator_enqueuer_threads(): enqueuer = GeneratorEnqueuer(create_generator_from_sequence_threads( TestSequence([3, 200, 200, 3])), use_multiprocessing=False) enqueuer.start(3, 10) gen_output = enqueuer.get() acc = [] for i in range(100): acc.append(int(next(gen_output)[0, 0, 0, 0])) """ Not comparing the order since it is not guarantee. It may get ordered, but not a lot, one thread can take the GIL before he was supposed to. """ assert len(set(acc) - set(range(100))) == 0, "Output is not the same" enqueuer.stop()
def auc_eval(self): if isinstance(self.validation_data, Generator): assert self.validation_steps is not None, \ 'If validation data is a generator, validation steps must be provided' y_pred = [] y_true = [] enqueuer = GeneratorEnqueuer(self.validation_data, use_multiprocessing=False, wait_time=.01) enqueuer.start(workers=1, max_queue_size=10) output_generator = enqueuer.get() for _ in range(self.validation_steps): generator_output = next(output_generator) if not hasattr(generator_output, '__len__'): raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) if len(generator_output) == 2: x, y = generator_output elif len(generator_output) == 3: x, y, _ = generator_output else: raise ValueError('Output of generator should be a tuple ' '(x, y, sample_weight) ' 'or (x, y). Found: ' + str(generator_output)) outs = self.model.predict_on_batch(x) y_pred += outs.tolist() y_true += y.tolist() enqueuer.stop() else: y_pred = self.model.predict(self.validation_data[0]) y_true = self.validation_data[1].astype(np.bool) roc_auc = roc_auc_score(y_true=y_true, y_score=y_pred) self.auc.append(roc_auc) print('AUC Score is %s' % self.auc[-1])
class ModelDiagnoser(Callback): def __init__(self, data_generator, m_batch_size, num_samples, output_dir, normalization_mean): super().__init__() self.epoch_index = 0 self.data_generator = data_generator self.batch_size = m_batch_size self.num_samples = num_samples self.tensorboard_writer = TensorBoardWriter(output_dir) self.normalization_mean = normalization_mean is_sequence = isinstance(self.data_generator, Sequence) if is_sequence: self.enqueuer = OrderedEnqueuer(self.data_generator, use_multiprocessing=True, shuffle=False) else: self.enqueuer = GeneratorEnqueuer(self.data_generator, use_multiprocessing=False, # todo: how to 'True' ? wait_time=0.01) # todo: integrate the Sequence generator properly # import multiprocessing # self.enqueuer.start(workers=multiprocessing.cpu_count(), max_queue_size=4) self.enqueuer.start(workers=1, max_queue_size=4) def on_epoch_end(self, epoch, logs=None): output_generator = self.enqueuer.get() steps_done = 0 total_steps = int(np.ceil(np.divide(self.num_samples, self.batch_size))) sample_index = 0 while steps_done < total_steps: generator_output = next(output_generator) x, y = generator_output[:2] x = next(iter(x.values())) y = next(iter(y.values())) y_pred = self.model.predict(x) self.epoch_index += 1 for i in range(0, len(y_pred)): n = steps_done * self.batch_size + i if n >= self.num_samples: return # rearranging images for visualization img_x = self.__reformat_img(x, i) img_y = self.__reformat_img(y, i) img_p = self.__reformat_img(y_pred, i) self.tensorboard_writer.save_image("Epoch-{}/{}/x" .format(self.epoch_index, sample_index), img_x) self.tensorboard_writer.save_image("Epoch-{}/{}/y" .format(self.epoch_index, sample_index), img_y) self.tensorboard_writer.save_image("Epoch-{}/{}/y_pred" .format(self.epoch_index, sample_index), img_p) sample_index += 1 steps_done += 1 def __reformat_img(self, img_np_array, img_index): img = np.squeeze(img_np_array[img_index, :, :, :]) img = 255. * (img + self.normalization_mean) # mean is the training images normalization mean img = img[:, :, [2, 1, 0]] # reordering of channels return img def on_train_end(self, logs=None): self.enqueuer.stop() self.tensorboard_writer.close()
def validate(config, model, val_client, validation_steps, metrics_id, epoch): val_di = val_client.gen() from keras.utils import GeneratorEnqueuer val_thre = GeneratorEnqueuer(val_di) val_thre.start() model_metrics = [] inhouse_metrics = [] for i in range(validation_steps): X, GT = next(val_thre.get()) Y = model.predict(X) model_losses = [(np.sum((gt - y)**2) / gt.shape[0] / 2) for gt, y in zip(GT, Y)] mm = sum(model_losses) if config.paf_layers > 0 and config.heat_layers > 0: GTL6 = np.concatenate([GT[-2], GT[-1]], axis=3) YL6 = np.concatenate([Y[-2], Y[-1]], axis=3) mm6l1 = model_losses[-2] mm6l2 = model_losses[-1] elif config.paf_layers == 0 and config.heat_layers > 0: GTL6 = GT[-1] YL6 = Y[-1] mm6l1 = None mm6l2 = model_losses[-1] else: assert False, "Wtf or not implemented" m = calc_batch_metrics(i, GTL6, YL6, range(config.heat_start, config.bkg_start)) inhouse_metrics += [m] model_metrics += [(i, mm, mm6l1, mm6l2, m["MAE"].sum() / GTL6.shape[0], m["RMSE"].sum() / GTL6.shape[0], m["DIST"].mean())] print( "Validating[BATCH: %d] LOSS: %0.4f, S6L1: %0.4f, S6L2: %0.4f, MAE: %0.4f, RMSE: %0.4f, DIST: %0.2f" % model_metrics[-1]) inhouse_metrics = pd.concat(inhouse_metrics) inhouse_metrics['epoch'] = epoch inhouse_metrics.to_csv("logs/val_scores.%s.%04d.txt" % (metrics_id, epoch), sep="\t") model_metrics = pd.DataFrame(model_metrics, columns=("batch", "loss", "stage6l1", "stage6l2", "mae", "rmse", "dist")) model_metrics['epoch'] = epoch del model_metrics['batch'] model_metrics = model_metrics.groupby('epoch').mean() with open('%s.val.tsv' % metrics_id, 'a') as f: model_metrics.to_csv(f, header=(epoch == 1), sep="\t", float_format='%.4f') val_thre.stop()
def validate(config, model, multi_model, val_client, validation_steps, metrics_id, epoch): # 得到的X是包含image, confidence mask, paf mask的list,得到的Y是包含6个stage一共12个groundtruth的heapmap # 网络一共有三个输入(对于训练时的评估,指标为了反映出训练的效果在测试时网络模型就不用考虑对feature map的输出进行mask了, # 可以对所有区域预测),即原始image,以及在训练模型过程中评估时,去除没有标记区域的confidence和paf的mask1, mask2 val_di = val_client.gen() val_thre = GeneratorEnqueuer( val_di ) # The provided generator can be finite in which case the class will throw # a `StopIteration` exception. 但是这里实现的gen貌似不存在这种问题。不过这个函数提供了multiprocess的封装 val_thre.start() model_metrics = [] inhouse_metrics = [] t0 = time() for i in range(validation_steps ): # 分成很多个batch进行预测估计的,为了减少validation耗时,在计算validation部分数据 # validation_steps = val_samples//batch_size 为了防止内存OOM,所以要分batch预测 # if random.randint(0, 9) < 5: # 只计算20%的数据 # continue X, GT = next(val_thre.get()) Y = multi_model.predict(X) model_losses = [(np.sum((gt - y)**2) / gt.shape[0] / 2) for gt, y in zip(GT, Y)] # 与模型定义时的loss保持一致,除以2的好处是,平方项的微分会出现2,抵消,可以减少乘法操作 mm = sum(model_losses) if config.paf_layers > 0 and config.heat_layers > 0: GTL6 = np.concatenate([GT[-2], GT[-1]], axis=3) YL6 = np.concatenate([Y[-2], Y[-1]], axis=3) mm6l1 = model_losses[-2] # NOTICE! 计算的是模型最后一个阶段的预测和groundtruth的距离 mm6l2 = model_losses[-1] elif config.paf_layers == 0 and config.heat_layers > 0: GTL6 = GT[-1] YL6 = Y[-1] mm6l1 = None mm6l2 = model_losses[-1] else: assert False, "Wtf or not implemented" m = calc_batch_metrics(i, GTL6, YL6, range(config.heat_start, config.bkg_start)) inhouse_metrics += [m] model_metrics += [(i, mm, mm6l1, mm6l2, m["MAE"].sum() / GTL6.shape[0], m["RMSE"].sum() / GTL6.shape[0], m["DIST"].mean())] # 以epoch为key,group之后取平均值 print( "Validating[BATCH: %d] LOSS: %0.4f, S6L1: %0.4f, S6L2: %0.4f, MAE: %0.4f, RMSE: %0.4f, DIST: %0.2f" % model_metrics[-1]) t1 = time() print('The CNN prediction time during validation is : ', t1 - t0) # inhouse_metrics = pd.concat(inhouse_metrics) # inhouse_metrics['epoch'] = epoch # inhouse_metrics.to_csv("logs/val_scores.%s.%04d.csv" % (metrics_id, epoch)) # , sep="\t" 默认的不是\t,而是',' # # 保存的是每个层的细节 # # model_metrics = pd.DataFrame(model_metrics, columns=("batch","loss","stage6l1","stage6l2","mae","rmse","dist") ) # model_metrics['epoch'] = epoch # del model_metrics['batch'] # model_metrics = model_metrics.groupby('epoch').mean() # with open('%s.val.tsv' % metrics_id, 'a') as f: # model_metrics.to_csv(f, header=(epoch==1), float_format='%.4f') # sep="\t", # # print(inhouse_metrics[["layer", "epoch", "MAE", "RMSE", "DIST"]].groupby(["layer", "epoch"]).mean()) val_thre.stop()