class ProgbarLogger(Callback): def on_train_begin(self, logs=None): #print('Epoch %d/%d' % (epoch + 1, self.epochs)) self.target = self.params['epochs'] self.stateful_metrics = ['loss', 'roc_auc', 'roc_auc_max'] self.roc_auc_max = 0 self.progbar = Progbar(self.target, verbose=1, stateful_metrics=self.stateful_metrics) self.seen = 0 def on_epoch_begin(self, epoch, logs=None): if self.seen < self.target: self.log_values = [] def on_epoch_end(self, epoch, logs=None): self.seen += 1 logs = logs or {} for k in logs: if k in ['loss', 'roc_auc', 'roc_auc_max']: self.log_values.append((k, logs[k])) if self.seen < self.target: self.progbar.update(self.seen, self.log_values) def on_train_end(self, logs=None): # Necessary to end line print('') return
def evaluate(model, dataset, num_val, num_cls): # if use fastnms # if use cross class nms # if eval image # if eval images # if eval video # if not display or benchmark # For mAP evaluation, creating AP_Object for every class per iou_threshold ap_data = { 'box': [[APObject() for _ in range(num_cls)] for _ in iou_thresholds], 'mask': [[APObject() for _ in range(num_cls)] for _ in iou_thresholds] } # detection object made from prediction output. for the purpose of creating json detections = Detections() # iterate the whole dataset to save TP, FP, FN i = 0 progbar = Progbar(num_val) tf.print("Evaluating...") for image, labels in dataset: i += 1 output = model(image, training=False) dets = model.detect(output) # update ap_data or detection depends if u want to save it to json or just for validation table prep_metrics(ap_data, dets, image, labels, detections) progbar.update(i) # if to json # save detection to json # Todo if not training, save ap_data, else calc_map return calc_map(ap_data, num_cls)
def get_hessian(self, X, U, X_df): ''' Get the full hessian by repeated calls to Hessian_Matvec Since PINNs are often small, this is feasible. Warning! This operation scales quadratically in time and space! ''' print( "Warning, trying to calculate the full Hessian is infeasible for large networks!" ) if self.use_differential_points: feed_dict = {self.X: X, self.U: U, self.X_df: X_df} else: feed_dict = {self.X: X, self.U: U} # We use repeated runs to avoid adding gradient ops for every # element of the hessian n = int(self.grads_flat.shape[0]) H = np.empty((n, n)) progbar = Progbar(n) for i in range(n): vec = np.zeros(n, dtype=np.float32) vec[i] = 1.0 feed_dict[self.hessian_vector] = vec h_row = self.sess.run(self.hessian_matvec, feed_dict) h_row = util.unwrap(h_row) H[i, :] = h_row[:] progbar.update(i + 1) # Explicitly diagonalize so that e.g. eigenvalues are always real for i in range(n): for j in range(i + 1, n): H[j, i] = H[i, j] return H
def train(self, train_gen, val_gen, epochs=10): """Trains the gan Arguments: train_gen {Sequence} -- Train data generator val_gen {Sequence} -- Validation data generator Keyword Arguments: epochs {int} -- Epochs (default: {10}) Returns: tuple -- (train loss history, validation loss history) """ for epoch in range(1, epochs + 1): print(f"Epoch {epoch}/{epochs}") num_batches = len(train_gen) progress_bar = Progbar(target=num_batches) for index, (X_train, y_train) in enumerate(train_gen): self.train_step(X_train, y_train) progress_bar.update(index + 1) display.clear_output(wait=True) self.generate_and_plot_images(epoch, val_gen[epoch][0])
def train_Adam(self, X: np.ndarray, U: np.ndarray, X_df=None, epochs=2000, learning_rate=1e-3): ''' Train using Full-Batch Adam for the given number of iterations Parameters: X (np.ndarray) : (N,d_in) array of domain points U (np.ndarray) : (N,d_out) array of solution points such that U = F(X) X_df (Optional[np.ndarray]) : (M,d_in) array of domain points where U is unknown but the PINN residual should still be evaluated. epochs (int) : Number of epochs to train for learning_rate (float) : If use_dynamic_learning_rate=True, this will be the learning rate used by the optimizer ''' if self.use_differential_points: feed_dict = {self.X: X, self.U: U, self.X_df: X_df} else: feed_dict = {self.X: X, self.U: U} if self.learning_rate is not None: feed_dict[self.learning_rate] = learning_rate progbar = Progbar(epochs) for i in range(epochs): _, loss = self.sess.run([self.optimizer_Adam, self.loss], feed_dict) progbar.update(i + 1, [("loss", loss)])
def train_population(population, x, y, batch_size, steps, steps_save=100, validation_split=0.3): # Split data in train and validation. Set seed to get same splits in # consequent calls x_train, x_val, y_train, y_val = train_test_split( x, y, test_size=validation_split, random_state=42) population_size = len(population) batch_generator = BatchGenerator(x_train, y_train, batch_size) results = defaultdict(lambda: []) stateful_metrics = ['min_loss', 'max_loss', 'mean_loss'] for metric, _ in population[0].eval_metrics: stateful_metrics.extend( [m.format(metric) for m in ['min_{}', 'max_{}', 'mean_{}']]) progbar = Progbar(steps, stateful_metrics=stateful_metrics) for step in range(1, steps + 1): x, y = batch_generator.next() for idx, member in enumerate(population): # One step of optimisation using hyperparameters of 'member' member.step_on_batch(x, y) # Model evaluation loss = member.eval_on_batch(x_val, y_val) # If optimised for 'STEPS_READY' steps if member.ready(): # Use the rest of population to find better solutions exploited = member.exploit(population) # If new weights != old weights if exploited: # Produce new hyperparameters for 'member' member.explore() loss = member.eval_on_batch(x_val, y_val) if step % steps_save == 0 or step == steps: results['model_id'].append(str(member)) results['step'].append(step) results['loss'].append(loss) results['loss_smoothed'].append(member.loss_smoothed()) for metric, value in member.eval_metrics: results[metric].append(value) for h, v in member.get_hyperparameter_config().items(): results[h].append(v) # Get recently added losses to show in the progress bar all_losses = results['loss'] recent_losses = all_losses[-population_size:] if recent_losses: metrics = _statistics(recent_losses, 'loss') for metric, _ in population[0].eval_metrics: metrics.extend( _statistics(results[metric][-population_size:], metric)) progbar.update(step, metrics) return pd.DataFrame(results)
def predict_depth_generator(self, data_iterator, depth, steps, progbar: Progbar = None): """ Args: data_iterator: should provide data in the form of a dict containing keys IterativeARTResNet.imgs_input_name and IterativeARTResNet.sinos_input_name """ if progbar is not None: progbar.add(1) if depth == 0: return data_iterator new_actual = [] new_sino = [] new_good_reco = [] # Lots of optimisation needed. Outputting sinograms and good_reconstructions could be optimized. nr_steps = steps or len(data_iterator) progbar_sublevel = Progbar(target=nr_steps, verbose=1) for i in range(nr_steps): data_batch = next(data_iterator) reconstructions_output, bad_sinograms, good_reconstructions = \ self._predict_depth_generator_step(data_batch) new_actual.append(reconstructions_output.numpy()) new_sino.append(bad_sinograms.numpy()) new_good_reco.append(good_reconstructions.numpy()) progbar_sublevel.update(i + 1) new_data_iterator = RecSinoArrayIterator(new_actual, new_sino, new_good_reco) return self.predict_depth_generator(new_data_iterator, depth - 1, steps=None, progbar=progbar)
def load_data(self): sample_filenames = self._get_sample_filenames() progress_bar, data = Progbar(len(sample_filenames)), [] for sample_arg, sample_id in enumerate(sample_filenames): data.append(self._load_sample(sample_id)) progress_bar.update(sample_arg + 1) return data
def load_data(self): scene_names = glob(self.path + 'mixed/*') image_paths, label_paths = [], [] for scene_name in scene_names: scene_image_paths, scene_label_paths = [], [] for image_side in ['left', 'right']: image_names = glob(scene_name + '/*%s.jpg' % image_side) side_image_paths = sorted(image_names, key=self._base_number) label_names = glob(scene_name + '/0*%s.json' % image_side) side_label_paths = sorted(label_names, key=self._base_number) scene_image_paths = scene_image_paths + side_image_paths scene_label_paths = scene_label_paths + side_label_paths image_paths = image_paths + scene_image_paths label_paths = label_paths + scene_label_paths self.data = [] progress_bar = Progbar(len(image_paths)) for sample_arg, sample in enumerate(zip(image_paths, label_paths)): image_path, label_path = sample if not self._valid_name_match(image_path, label_path): raise ValueError('Invalid name match:', image_path, label_path) boxes = self._extract_boxes(label_path) if boxes is None: continue self.data.append({'image': image_path, 'boxes': boxes}) progress_bar.update(sample_arg + 1) return self.data
def _train_depth(self, iterator, epochs, steps_per_epoch, weights_filepath): print(f"Training network {self.name}.") for epoch in range(epochs): print(f"Epoch {epoch}:") progbar = Progbar( steps_per_epoch, verbose=1, stateful_metrics=[m.name for m in self._all_metrics]) for step in range(1, steps_per_epoch + 1): data = next(iterator) self._train_depth_step(data) if step % 1 == 0: progbar.update(step, values=[(m.name, m.result().numpy()) for m in self._all_metrics]) print("Saving model") self._model.save_weights( filepath=weights_filepath + '-' + self._monitored_metric.name + '-' + '{a:.3f}'.format( a=self._monitored_metric.result().numpy()[0]) + '.hdf5') for m in self._all_metrics: m.reset_states()
def on_epoch_end(self, epoch, logs=None): generator = iter(self.validation_data) y_true, y_pred = [], [] print(f'\nValidation {epoch+1}') pbar = Progbar(self.validation_steps) for it in range(self.validation_steps): x, y = next(generator) y_pred.extend(self.adj_fx(self.model.predict(x))) y_true.extend(self.adj_fy(y)) pbar.update(current=it) print('\nClassification report:') print(classification_report(y_true, y_pred, zero_division=0)) print('\nConfusion matrix:') print(confusion_matrix(y_true, y_pred, normalize='pred')) # verify if f1 score improved report = classification_report(y_true, y_pred, output_dict=True, zero_division=0) macro_f1 = report['macro avg']['precision'] if macro_f1 > self.best: print( f'\nEpoch {epoch+1}: precision improved from {self.best:.3f} to {macro_f1:.3f}\n' ) self.best = macro_f1 self.model.save(self.path_to_save)
def train_network(self, dataset, test_data, test_data_size, loss_function, epochs, tree_loss_weight, opt, size, interpret=False): # Inference has to be done on the original network and the full NBDT # The network should be pretrained on the dataset of interest # iterate through dataset # for each member of the dataset # make prediction with model # make prediction with NBDT # compute the loss # update the neural network parameters training_loss_results = [] #tree_loss = TreeSupLoss(loss_function, tree_loss_weight) self.model.layers[-1].trainable = False for epoch in range(epochs): epoch_loss_avg = tf.keras.metrics.Mean() epoch_nbdt_loss_avg = tf.keras.metrics.Mean() epoch_net_loss_avg = tf.keras.metrics.Mean() epoch_acc_avg = tf.keras.metrics.CategoricalAccuracy() i = 0 progress = Progbar(target=size) for x, y in dataset.take(size).batch(1): i = i + 1 sample = x nbdt_loss, net_loss, loss, grad = self.gradient(sample, y, loss_function, tree_loss_weight) opt.apply_gradients(zip(grad[0], self.model.trainable_variables)) self.backbone = Sequential(self.model.layers[:-1]) epoch_loss_avg.update_state(loss) epoch_nbdt_loss_avg.update_state(nbdt_loss) epoch_net_loss_avg.update_state(net_loss) nbdt_pred = self.nbdt_predict(sample, interpret) epoch_acc_avg.update_state(y, nbdt_pred) progress.update(i, values=[('nbdt_loss:', epoch_nbdt_loss_avg.result()), ('net_loss:', epoch_net_loss_avg.result()), ('loss:', epoch_loss_avg.result()), ('acc:', epoch_acc_avg.result()), ]) #training_loss_results.append(epoch_loss_avg.result().numpy()) print() print("Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}".format(epoch, epoch_loss_avg.result(), epoch_acc_avg.result()), ) test_acc, test_auc = self.evaluate(test_data.batch(1), size=test_data_size) print('VAL ACC: {:.3%} VAL AUC: {:.3}'.format(test_acc, test_auc)) """ if test_acc >= .90: print("SAVING MODEL") self.model.save("nn_nbdt_test_acc-{:.3f}_epoch-{:03d}_adam".format(test_acc, epoch)) print() self.model.save("nn_nbdt_epoch-{:03d}_adam".format(epoch)) """ return training_loss_results
def evaluate(self, test_iterator, steps): progbar = Progbar(steps, verbose=1, stateful_metrics=[m.name for m in self._all_metrics]) for step in range(1, steps + 1): data = next(test_iterator) self._test_step(data) if step % 1 == 0: progbar.update(step, values=[(m.name, m.result().numpy()) for m in self._all_metrics])
def _train_stochastic_optimizer(self, optimizer_opp, X, U, X_df=None, batch_size=128, epochs=10): ''' Generic custom training loop for stochastic optimizers. Replace optimizer_opp with e.g. RMSProp.minimize() for a different stochastic optimizer. ''' if self.use_differential_points: assert (X_df is not None) assert (X_df.shape[0] >= X.shape[0]) progbar = Progbar(epochs, stateful_metrics=["loss_full"]) for epoch in range(epochs): X_s, U_s = shuffle(X, U) if X_df is not None: X_df_s = shuffle(X_df) dataset_size = X_df.shape[0] else: dataset_size = X.shape[0] b_c = 0 for b in range(0, dataset_size, batch_size): if X_df is not None: b_c_last = b_c b_c = b % X_s.shape[0] # X and X_df are typically different sizes, # so we shuffle them at different times if b_c_last > b_c: X_s, U_s = shuffle(X, U) X_b = X_s[b_c:(b_c + batch_size), :] U_b = U_s[b_c:(b_c + batch_size), :] X_df_b = X_df_s[b:(b + batch_size), :] feed_dict = {self.X: X_b, self.U: U_b, self.X_df: X_df_b} else: X_b = X_s[b:(b + batch_size), :] U_b = U_s[b:(b + batch_size), :] feed_dict = {self.X: X_b, self.U: U_b} _, loss = self.sess.run([optimizer_opp, self.loss], feed_dict) if X_df is not None: feed_dict = {self.X: X, self.U: U, self.X_df: X_df} else: feed_dict = {self.X: X, self.U: U} progbar.update(epoch + 1, [("loss", loss)])
def save(ds_X, ds_Y, gen): prog = Progbar(len(gen)) cur_idx = 0 for idx, (x, y) in enumerate(gen): rows = x.shape[0] assert(rows == y.shape[0]) ds_X[cur_idx:(cur_idx+rows), :] = x ds_Y[cur_idx:(cur_idx+rows), :] = y cur_idx += rows prog.update(idx) print()
def inference(self, test_dataset, L=1): '''Get \(p(c_i|Y_i,X_i)\). Parameters ---------- test_dataset : tf.Dataset The dataset object. L : int The number of MC samples. Returns ---------- pi_norm : np.array \([1, K]\) The estimated \(\\pi\). mu : np.array \([d, k]\) The estimated \(\\mu\). p_c_x : np.array \([N, ]\) The estimated \(p(c_i|Y_i,X_i)\). w_tilde : np.array \([N, k]\) The estimated \(E(\\tilde{w}_i|Y_i,X_i)\). var_w_tilde : np.array \([N, k]\) The estimated \(Var(\\tilde{w}_i|Y_i,X_i)\). z_mean : np.array \([N, d]\) The estimated latent mean. ''' if self.latent_space is None: raise ReferenceError('Have not initialized the latent space.') print('Computing posterior estimations over mini-batches.') progbar = Progbar(test_dataset.cardinality().numpy()) pi_norm = tf.nn.softmax(self.latent_space.pi).numpy() mu = self.latent_space.mu.numpy() z_mean = [] p_c_x = [] w_tilde = [] var_w_tilde = [] for step, (x, c_score) in enumerate(test_dataset): x = tf.concat([x, c_score], -1) if self.has_cov else x _z_mean, _, z = self.encoder(x, L, False) res = self.latent_space(z, inference=True) z_mean.append(_z_mean.numpy()) p_c_x.append(res['p_c_x']) w_tilde.append(res['w_tilde']) var_w_tilde.append(res['var_w_tilde']) progbar.update(step + 1) z_mean = np.concatenate(z_mean) p_c_x = np.concatenate(p_c_x) w_tilde = np.concatenate(w_tilde) var_w_tilde = np.concatenate(var_w_tilde) return pi_norm, mu, p_c_x, w_tilde, var_w_tilde, z_mean
def train(BATCH_SIZE, X_train): ### model define d = discriminator_model() g = generator_model() d_on_g = generator_containing_discriminator(g, d) d_optim = RMSprop(lr=0.0004) g_optim = RMSprop(lr=0.0002) g.compile(loss='mse', optimizer=g_optim) d_on_g.compile(loss='mse', optimizer=g_optim) d.trainable = True d.compile(loss='mse', optimizer=d_optim) for epoch in range(10): print ("Epoch is", epoch) n_iter = int(X_train.shape[0]/BATCH_SIZE) progress_bar = Progbar(target=n_iter) for index in range(n_iter): # create random noise -> U(0,1) 10 latent vectors noise = np.random.uniform(0, 1, size=(BATCH_SIZE, 10)) # load real data & generate fake data image_batch = X_train[index*BATCH_SIZE:(index+1)*BATCH_SIZE] generated_images = g.predict(noise, verbose=0) # visualize training results if index % 20 == 0: image = combine_images(generated_images) image = image*127.5+127.5 cv2.imwrite('./result/'+str(epoch)+"_"+str(index)+".png", image) # attach label for training discriminator X = np.concatenate((image_batch, generated_images)) y = np.array([1] * BATCH_SIZE + [0] * BATCH_SIZE) # training discriminator d_loss = d.train_on_batch(X, y) # training generator d.trainable = False g_loss = d_on_g.train_on_batch(noise, np.array([1] * BATCH_SIZE)) d.trainable = True progress_bar.update(index, values=[('g',g_loss), ('d',d_loss)]) print ('') # save weights for each epoch g.save_weights('weights/generator.h5', True) d.save_weights('weights/discriminator.h5', True) return d, g
def predict_from_model(sess, graph, test_dataloader, data_config): instance_collection_pred = [] progbar = Progbar(target=test_dataloader.size) while True: input_feed, excluded_count, targets = get_feed(graph.data_feeds, test_dataloader, data_config, False) fetches = [ graph.data_feeds[0]['pred'], graph.loss, graph.global_step, graph.perplexity, graph.losses_eval ] preds, loss, _, perplexity, losses_eval = sess.run(fetches, input_feed) progbar.update(current=targets[-1]['line_id'], values=[('loss', loss), ('ppl', perplexity)]) for example_id in range(data_config.batch_size - excluded_count): gt_target = targets[example_id] pred = preds[example_id] instance_collection_pred.append( InstancePred(index=gt_target['inst_id'], abbr=gt_target['abbr'], sense_pred=test_dataloader.id2sense[pred[0]] if gt_target['abbr_id'] else None)) if excluded_count > 0: break # sort collection list by the global instance idx instance_collection_pred = sorted(instance_collection_pred, key=lambda x: x.index) ''' # some instances might have been skipped, thus we add non-included instances before returning # Rui: not necessary now as no data is skipped instance_collection = [] temp_idx = 0 for idx in range(len(test_true)): temp_instance_pred = instance_collection_pred[temp_idx] if temp_instance_pred.index == idx: instance_collection.append(temp_instance_pred) temp_idx += 1 else: instance_collection.append(InstancePred(index=idx, abbr=None, sense_pred=None)) ''' return instance_collection_pred
def tag_dataset(dataset): correctLabels = [] predLabels = [] b = Progbar(len(dataset)) for i,data in enumerate(dataset): tokens, casing,char, labels = data tokens = np.asarray([tokens]) casing = np.asarray([casing]) char = np.asarray([char]) pred = model.predict([tokens, casing,char], verbose=False)[0] pred = pred.argmax(axis=-1) #Predict the classes correctLabels.append(labels) predLabels.append(pred) b.update(i) b.update(i+1) return predLabels, correctLabels
class ProgressBar: '''Wrapper for tf.keras.progbar that accounts for unknown length of game For simplicity, the interface is slightly changed and not all features are supported. Methods ------- step: increment the bar by one step. Will extend the target length if the bar is more than 90% of the way done. terminate: step the bar AND cause it to end Attributes ---------- move_num: number of moves made so far ''' def __init__(self, estimated_length, verbose = True, expandable = True): self._bar = Progbar(estimated_length) self._move_num = 0 self.verbose = verbose self.expandable = expandable @property def move_num(self): return self._move_num def step(self, **kwargs): self._move_num += 1 curr_target = self._bar.target if self._move_num >= curr_target * 0.9 and self.expandable: self._bar.target += curr_target // 10 + 1 if self.verbose: self._bar.update(self._move_num, **kwargs) def terminate(self): self._move_num += 1 self._bar.target = self._move_num if self.verbose: self._bar.update(self._move_num)
def evaluate(self, dataset, size, interpret=False, background=None, sample=None): count = 0 total_samples = 0 progress = Progbar(target=size) acc_avg = tf.keras.metrics.CategoricalAccuracy() auc_avg = tf.keras.metrics.AUC() for x, y in dataset: total_samples = total_samples + 1 nbdt_pred = self.nbdt_predict(x, interpret, background, sample) acc_avg.update_state(y, nbdt_pred) auc_avg.update_state(y, nbdt_pred.numpy().reshape((1, nbdt_pred.numpy().shape[0]))) progress.update(total_samples, values=[('acc: ', acc_avg.result()), ('AUC: ', auc_avg.result())]) print() print("Accuracy: {}".format(acc_avg.result())) print("AUC: {}".format(auc_avg.result())) return acc_avg.result(), auc_avg.result()
def _training_loop_optimizer(self, X, U, X_df, epochs, optimizer): X = tf.convert_to_tensor(X, dtype=tf.float32) U = tf.convert_to_tensor(U, dtype=tf.float32) if self.use_differential_points: X_df = tf.convert_to_tensor(X_df, dtype=tf.float32) else: X_df = None progbar = Progbar(epochs, stateful_metrics=["loss"]) for i in range(epochs): with tf.GradientTape() as param_tape: loss, _ = self._loss(X, U, X_df) params = self.get_trainable_parameters() grads = param_tape.gradient(loss, params) optimizer.apply_gradients(zip(grads, params)) progbar.update(i + 1, [("loss", loss)])
def train(self): @tf.function def train_step(image_batch): noise = tf.random.normal((self.configs.data.batch_size, eval(self.configs.noise_dim)[0])) with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape: generated_images = self.generator(noise, training=True) real_output = self.discriminator(image_batch, training=True) fake_output = self.discriminator(generated_images, training=True) generator_loss = self.get_generator_loss(fake_output) discriminator_loss = self.get_discriminator_loss( real_output, fake_output) gradients_of_generator = gen_tape.gradient( generator_loss, self.generator.trainable_variables) self.generator_opt.apply_gradients( zip(gradients_of_generator, self.generator.trainable_variables)) gradients_of_discriminator = disc_tape.gradient( discriminator_loss, self.discriminator.trainable_variables) self.discriminator_opt.apply_gradients( zip(gradients_of_discriminator, self.discriminator.trainable_variables)) return [('generator_loss', generator_loss.numpy()), ('discriminator_loss', discriminator_loss.numpy())] data = get_data(self.configs.data.path, self.configs.data.batch_size) for i in range(self.num_epochs): print("\nepoch {}/{}".format(i + 1, self.num_epochs)) prog_bar = Progbar( None, stateful_metrics=['generator_loss', 'discriminator_loss']) for idx, im_batch in enumerate(data): losses = train_step(im_batch) prog_bar.update(idx + 1, values=losses) self.save_results(i) if self.__save_model and (i + 1) % self.__save_interval == 0: self.save_models()
def resample(self, num_interpolation=200): examples = self.examples print("interpolate") bar = Progbar(len(examples)) # 进度条 if num_interpolation and num_interpolation is not None: for i in range(len(examples)): range_len = examples[i][0][-1] - examples[i][0][0] range_start = examples[i][0][0] range_interval = range_len / num_interpolation interp_x = [ range_start + range_interval * i for i in range(num_interpolation) ] interp_data = [interp_x] for feature_id in range(1, len(self.names)): try: interp_f = interp(examples[i][0], examples[i][feature_id], kind="cubic") interp_data.append([interp_f(x) for x in interp_x]) except: raise ValueError("%d %d" % (i, feature_id), examples[i]) bar.update(i) examples[i] = np.array(interp_data) print("\ndone") # 数据预处理 preprocession = Preprocessing(with_label=self.with_label) examples = preprocession(examples) self.examples = np.array(examples, dtype="float32") if self.with_label: self.y = self.examples[::, -1, 0].tolist() self.x = self.examples[::, 1:-1, ::] else: self.x = self.examples[::, 1:, ::] return self
def explain(self, background, samples, save=True, filename=None): print("Running DeepLIFT on each node...") print("getting shapley values for prediction paths...") pred_i = 0 progress = Progbar(target=samples.shape[0]) shaps = np.zeros((1, samples.shape[1])) #shaps = [] for sample in samples: progress.update(pred_i) x = sample.reshape((1, samples.shape[1])) self.nbdt_predict(x, interpret=True, background=background, sample=x, shaps=shaps) pred_i += 1 #shaps = np.array(shaps) #print(shaps) mean_abs_shaps = shaps / pred_i #mean_abs_shaps = abs(mean_shaps) #print(mean_abs_shaps) df = pd.DataFrame(mean_abs_shaps) #shaps = np.array(shaps) #shaps = shaps.reshape((shaps.shape[0], shaps.shape[2])) #print(shaps.shape) #df = pd.DataFrame(shaps) #df.columns = ['SNP_' + str(i) for i in range(shaps.shape[1])] print(mean_abs_shaps.shape) df.columns = ['SNP_' + str(i) for i in range(mean_abs_shaps.shape[1])] if save: df.to_csv(filename) return df #print(shaps.shape) #print(mean_shaps.shape) #print(mean_abs_shaps.shape)
def fit(train_ds, epochs, test_ds, num_training_samples): checkpoint.restore(tf.train.latest_checkpoint(config.checkpoint_dir)) metrics_names = ['acc', 'pr'] pb_i = Progbar(num_training_samples, stateful_metrics=metrics_names) for epoch in range(epochs): start = time.time() for example_input, example_target in test_ds.take(1): model.generate_images(generator, example_input, example_target) print("Epoch: ", epoch) # Train for n, (input_image, target) in train_ds.enumerate(): train_step(input_image, target, epoch) pb_i.update(int(n + 1) * config.BATCH_SIZE) # saving (checkpoint) the model every 20 epochs if (epoch + 1) % 20 == 0: checkpoint.save(file_prefix=config.checkpoint_prefix) print('Time taken for epoch {} is {} sec\n'.format( epoch + 1, time.time() - start)) checkpoint.save(file_prefix=config.checkpoint_prefix)
def ctc_decode(Y_pred, batch_size=32, greedy=False, beam_width=10, top_paths=1, verbose=0): ''' Use beam/greedy search to find most probable prediction. ''' if verbose == 1: progbar = Progbar(target=np.ceil(len(Y_pred) / batch_size).astype(int)) preds = [] probs = [] for i, batch in enumerate(range(0, len(Y_pred), batch_size)): y_pred = Y_pred[batch: batch + batch_size] decoded = K.ctc_decode(y_pred=y_pred, input_length=np.ones(y_pred.shape[0]) * y_pred.shape[1], greedy=greedy, beam_width=beam_width, top_paths=top_paths) preds.extend([[c for c in y if c != -1] for y in decoded[0][0]]) probs.extend(np.exp(decoded[1]).flatten()) if verbose == 1: progbar.update(i+1) return preds, probs
def predict(self, x, batch_size=None, verbose=0, steps=1, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False, ctc_decode=True): """ Model predicting on data yielded (predict function has support to generator). A predict() abstration function of TensorFlow 2. Provide x parameter of the form: yielding [x]. :param: See tensorflow.keras.Model.predict() :return: raw data on `ctc_decode=False` or CTC decode on `ctc_decode=True` (both with probabilities) """ self.model._make_predict_function() if verbose == 1: print("Model Predict") out = self.model.predict(x=x, batch_size=batch_size, verbose=verbose, steps=steps, callbacks=callbacks, max_queue_size=max_queue_size, workers=workers, use_multiprocessing=use_multiprocessing) if not ctc_decode: return np.log(out) steps_done = 0 if verbose == 1: print("CTC Decode") progbar = Progbar(target=steps) batch_size = int(np.ceil(len(out) / steps)) input_length = len(max(out, key=len)) predicts, probabilities = [], [] while steps_done < steps: index = steps_done * batch_size until = index + batch_size x_test = np.asarray(out[index:until]) x_test_len = np.asarray([input_length for _ in range(len(x_test))]) decode, log = K.ctc_decode(x_test, x_test_len, greedy=self.greedy, beam_width=self.beam_width, top_paths=self.top_paths) probabilities.extend([np.exp(x) for x in log]) decode = [[[int(p) for p in x if p != -1] for x in y] for y in decode] predicts.extend(np.swapaxes(decode, 0, 1)) steps_done += 1 if verbose == 1: progbar.update(steps_done) return (predicts, probabilities)
loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] iter_num += 1 progbar.update(iter_num, [('rpn_cls', np.mean(losses[:iter_num, 0])), ('rpn_regr', np.mean(losses[:iter_num, 1])), ('final_cls', np.mean(losses[:iter_num, 2])), ('final_regr', np.mean(losses[:iter_num, 3]))]) if iter_num == epoch_length: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float( sum(rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose:
class History(tf.keras.callbacks.Callback): def __init__(self): super(History, self).__init__() self.loss = [] self.val_loss = [] self.eff_loss = [] self.eff_val_loss = [] self.i = 0 def on_train_begin(self, logs=None): self.epochs = self.params['epochs'] K.set_value( self.model.optimizer.num_steps, math.ceil(self.params["samples"] / self.params["batch_size"]) * self.params["epochs"]) def on_epoch_begin(self, epoch, logs={}): self.loss_buff = [] self.val_loss_buff = [] print('Epoch : %d/%d, Effective Epoch : %d/%d' % (epoch + 1, self.epochs, (epoch + 1) // self.model.optimizer.L + 1, self.epochs // self.model.optimizer.L)) self.target = self.params['samples'] self.progbar = Progbar(target=self.target, verbose=1, stateful_metrics=['loss', 'val_loss']) self.seen = 0 def on_train_batch_begin(self, batch, logs=None): if self.seen < self.target: self.log_values = [] def on_train_batch_end(self, batch, logs={}): self.i = self.i + 1 batch_size = logs.get('size', 0) self.seen += batch_size if K.eval(self.model.optimizer.state_counter) == 0: self.loss_buff.append(logs.get('loss')) self.log_values.append(('loss', np.mean(self.loss_buff))) # Skip progbar update for the last batch; # will be handled by on_epoch_end. if self.seen < self.target: self.progbar.update(self.seen, self.log_values) else: self.progbar.update(self.target - 1, self.log_values) def on_test_batch_end(self, batch, logs={}): self.val_loss_buff.append(logs.get('loss')) def on_epoch_end(self, epoch, logs): self.loss.append(np.mean(self.loss_buff)) self.val_loss.append(np.mean(self.val_loss_buff)) if (epoch + 1) % self.model.optimizer.L == 0: self.eff_loss.append(np.mean(self.loss[-self.model.optimizer.L:])) self.eff_val_loss.append( np.mean(self.val_loss[-self.model.optimizer.L:])) self.log_values.append(('val_loss', np.mean(self.val_loss_buff))) self.progbar.update(self.target, self.log_values)