def test_online(model: keras.models.Model): cam = cv2.VideoCapture(0) while True: ret_val, img = cam.read() if ret_val: pred = model.predict(np.array([cv2.resize(img, (224, 224))])) pred = keras.applications.vgg16.decode_predictions(pred) labels = [label[1] for label in pred[0]] show_image("Webcam", img, labels)
def evaluate_model(model: keras.models.Model, input_data, cr_codes, classes=DEFAULT_CLASSES): ''' Convenience function to quickly evaluate model performance ''' predictions = model.predict(input_data) params = dict(epochs=0, lr=0) result = Result.from_predictions(predictions, cr_codes, params, 'AUTO_EVAL', '') print(result.describe())
def rmse_score(self, model: keras.models.Model, Xtrain, Xtest, ytrain, ytest, index=0): """ root mean squared error """ pred_train, pred_test = model.predict(Xtrain), model.predict(Xtest) pred_list_train = np.array([pred_train[:, index]]) pred_list_test = np.array([pred_test[:, index]]) from sklearn.metrics import mean_squared_error rmse_train = [ np.sqrt(mean_squared_error(ytrain[:, index], _y)) for _y in pred_list_train ] rmse_test = [ np.sqrt(mean_squared_error(ytest[:, index], _y)) for _y in pred_list_test ] return rmse_train, rmse_test
def predict_with_model(tokenizer, text, model: keras.models.Model): input_text = [i for i in text if chinese_regex.match(i)] input_text = input_text[:tokenizer.max_length-2] input_token = tokenizer.tokenize(input_text) input_x = keras.preprocessing.sequence.pad_sequences([input_token], maxlen=tokenizer.max_length, padding='post') predict_idx = model.predict(input_x)[0].argmax(1) labels = tokenizer.label_de_tokenize(predict_idx, length=len(input_text)) final = '' for i in range(len(input_text)): final += input_text[i] if labels[i] != 'O': final += '{}'.format(labels[i]) return final
def test_offline(model: keras.models.Model): images = load_data() pred_images = images_to_pred_images(images) pred = model.predict(np.array(pred_images)) pred = keras.applications.vgg16.decode_predictions(pred) labels = list() for p in pred: l = list() for label in p: l.append(label[1]) labels.append(l) show_images("image", images, labels)
def predict_gene(model: keras.models.Model, eder: Simple_ED, maxlen, pre_str=None, str_len=None): if pre_str == None: seed = np.random.randint(0, eder.text_len - maxlen) pre_str = eder.text[seed:seed + maxlen] if str_len == None: str_len = maxlen * 10 gene_str = [] prefix = pre_str for i in range(str_len): code = eder.encode(prefix) pred = model.predict(code.reshape((1, ) + code.shape), verbose=0) ci = sample(pred[0]) c = eder.characters[ci] gene_str.append(c) prefix = prefix[1:] + c print("pre_str:{}\n{}".format(pre_str, ''.join(gene_str)))
def get_and_save_model_output_on_batches( model: keras.models.Model, batches: image.DirectoryIterator, model_output_to="path/to/output/to.bc", labels_output_to="optional/path/to/output/onehot/labels/to.bc", num_batches_to_save: int = None): # valid_batches = ut.get_batches(path_data_valid, shuffle=False) if num_batches_to_save is None: num_batches_to_save = int(batches.n / batches.batch_size) if model_output_to == "path/to/output/to.bc": print( 'Must provide a path to output data to, such as: path_data_model + "valid_model_out.bc"' ) output_labels = True if labels_output_to == "optional/path/to/output/onehot/labels/to.bc": output_labels = False model_out, labels_out = [], [] for i in range(num_batches_to_save): print(f"batch {i} of {num_batches_to_save}") imgs, labels = next(batches) model_out.append(model.predict( imgs, batch_size=8, verbose=1)) # May want to change batch_size on a powerful GPU! labels_out.append(labels) model_out = np.concatenate(model_out, axis=0) labels_out = np.concatenate( labels_out, axis=0) # This is automatically converted to one-hot! print( f"\nmodel_out.shape, labels_out.shape is [[{model_out.shape, labels_out.shape}]]" ) save_array(model_output_to, model_out) print(f"Saved model's output to {model_output_to}") if output_labels: save_array(labels_output_to, labels_out) print(f"Saved labels's in one-hot format to {labels_output_to}")
def eval_dist(dp: utils.DataPoint, model: keras.models.Model, datagen: keras.preprocessing.image.ImageDataGenerator): """returns the distance between predicted location and true location""" x = preproc_x(dp.x, datagen) y_pred = model.predict(x, batch_size=1).squeeze() return K.eval(mode_distance(dp.y, y_pred))
def test_model(model: keras.models.Model, dataX: np.array, dataY: np.array) -> float: preds = model.predict(dataX, batch_size=1, verbose=1) preds = np.argmax(preds, axis=-1) accu = np.sum(preds == np.argmax(dataY, axis=-1)) / len(preds) return accu
def HMC_ensemble_run(model: keras.models.Model, x_train: np.array, y_train: np.array, N_mc: int, ep: float, tau: int, burn_in: int, sample_every: int, return_extra=False, verbose=True, verbose_n = 100, ): """ Takes a keras model and a dataset (x_train, y_train) and returns a list of numpy arrays to be used as weights for an ensemble predictor. """ step_size = ep n_steps = tau Ws = model.weights lossfn = model.loss #this is kinda cheeky X = model.input Y_ = model.output Y = K.placeholder(shape=Y_.shape) L = K.sum(lossfn(Y, Y_)) #the sum accross the batch Gs = K.gradients(L, Ws) eval_loss_and_grads = K.function([X,Y], [L] + Gs) def get_loss_and_grads(x, y): res = eval_loss_and_grads([x, y]) return res[0], res[1:] losses = [] i = 0 weights = [] ensemble = [] accept_n = 0 ep_lo = 0.8 * step_size ep_hi = 1.2 * step_size tau_lo = int(0.5 * n_steps) tau_hi = int(1.5 * n_steps) obj = 0 while len(ensemble) < N_mc: ep = ep_lo + (ep_hi - ep_lo) * np.random.random() tau = np.random.randint(tau_lo, high=tau_hi) obj, gs = get_loss_and_grads(x_train, y_train) losses.append(obj) if verbose and i % verbose_n == 0: acc = np.mean(model.predict(x_train).argmax(axis=1) == y_train.argmax(axis=1)) accept_ratio = accept_n / i if i > 0 else 0 print("iter: ", i, 'accuracy :', acc, 'loss: ', obj, 'accept_ratio: ', accept_ratio) i += 1 ps = [np.random.normal(size=w.shape) for w in Ws] # momentum variables H = .5 * sum([np.sum(p ** 2) for p in ps]) + obj ws = [K.eval(w) for w in Ws] weights.append(ws) ws_old = [K.eval(w) for w in Ws] # store the values of the weights in case we need to go back for t in range(tau): for p, g in zip(ps, gs): p -= .5 * ep * g for w, p in zip(ws, ps): w += ep * p # evaluate new weights for (weight, value) in zip(Ws, ws): K.set_value(weight, value) obj, gs = get_loss_and_grads(x_train, y_train) for p, g in zip(ps, gs): p -= .5 * ep * g H_new = .5 * sum([np.sum(p ** 2) for p in ps]) + obj dH = H_new - H if (dH < 0) or np.random.rand() < np.exp(-dH): # in this case, we acccept the new values accept_n += 1 pass else: # reverse the step for weight, value in zip(Ws, ws_old): K.set_value(weight, value) if i > burn_in and i % sample_every == 0: ensemble.append([K.eval(w) for w in Ws]) if return_extra: weights = np.array(weights) return ensemble, losses, weights, accept_n / i else: return ensemble
def predict_experiences(model: keras.models.Model, experiences: Iterable[Experience]) -> np.ndarray: x = np.stack(exp.from_state.data for exp in experiences) return model.predict(x)
def full_multiclass_report(model: keras.models.Model, x: np.ndarray, y_true: np.ndarray, classes: typing.Sequence, cm_path: pathlib.Path, tp_path: pathlib.Path, tn_path: pathlib.Path, fp_path: pathlib.Path, fn_path: pathlib.Path, id_valid: np.ndarray = None, chunk=False): """ Builds a report containing the following: - accuracy - AUC - classification report - confusion matrix - 7/31/2018 metrics The output is the report as a string. The report also generates a confusion matrix plot and tp/fp examples. :param model: :param x: :param y_true: :param classes: :param id_valid :param chunk :return: """ # TODO(luke): Split this into separate functions. y_proba = model.predict(x, batch_size=8) assert y_true.shape == y_proba.shape if y_proba.shape[-1] == 1: y_pred = (y_proba > 0.5).astype('int32') else: y_pred = y_proba.argmax(axis=1) y_true = y_true.argmax(axis=1) assert y_pred.shape == y_true.shape, \ f'y_pred.shape: {y_pred.shape} must equal y_true.shape: {y_true.shape}' comment = "Accuracy: " + str(sklearn.metrics.accuracy_score( y_true, y_pred)) comment += '\n' # Assuming 0 is the negative label y_true_binary = y_true > 0 y_pred_binary = y_pred > 0 score = sklearn.metrics.roc_auc_score(y_true_binary, y_pred_binary) # Do not change the line below, it affects reporting._extract_auc comment += f'AUC: {score}\n' comment += f'Assuming {0} is the negative label' comment += '\n\n' comment += "Classification Report\n" comment += sklearn.metrics.classification_report(y_true, y_pred, digits=5) cnf_matrix = sklearn.metrics.confusion_matrix(y_true, y_pred) comment += '\n' comment += str(cnf_matrix) save_confusion_matrix(cnf_matrix, classes=classes, cm_path=cm_path) # Compute additional metrics for the 7/31 paper try: tn, fp, fn, tp = cnf_matrix.ravel() comment += f'\n\nAdditional statistics:\n' sensitivity = tp / (tp + fn) comment += f'Sensitivity: {sensitivity}\n' specificity = tn / (tn + fp) comment += f'Specificity: {tn / (tn + fp)}\n' comment += f'Precision: {tp / (tp + fp)}\n' total_acc = (tp + tn) / (tp + tn + fp + fn) random_acc = (((tn + fp) * (tn + fn) + (fn + tp) * (fp + tp)) / (tp + tn + fp + fn)**2) comment += f'\n\nNamed statistics:\n' kappa = (total_acc - random_acc) / (1 - random_acc) comment += f'Cohen\'s Kappa: {kappa}\n' youdens = sensitivity - (1 - specificity) comment += f'Youden\'s index: {youdens}\n' comment += f'\n\nOther sklearn statistics:\n' log_loss = sklearn.metrics.classification.log_loss(y_true, y_pred) comment += f'Log loss: {log_loss}\n' comment += f'F-1: {sklearn.metrics.f1_score(y_true, y_pred)}\n' except ValueError as e: comment += '\nCould not add additional statistics (tp, fp, etc.)' comment += str(e) save_misclassification_plots(x, y_true_binary, y_pred_binary, id_valid=id_valid, chunk=chunk, tp_path=tp_path, fp_path=fp_path, tn_path=tn_path, fn_path=fn_path) return comment