def test_online(model: keras.models.Model):
cam = cv2.VideoCapture(0)
while True:
ret_val, img = cam.read()
if ret_val:
pred = model.predict(np.array([cv2.resize(img, (224, 224))]))
pred = keras.applications.vgg16.decode_predictions(pred)
labels = [label[1] for label in pred[0]]
show_image("Webcam", img, labels)
def evaluate_model(model: keras.models.Model,
input_data,
cr_codes,
classes=DEFAULT_CLASSES):
'''
Convenience function to quickly evaluate model performance
'''
predictions = model.predict(input_data)
params = dict(epochs=0, lr=0)
result = Result.from_predictions(predictions, cr_codes, params,
'AUTO_EVAL', '')
print(result.describe())
def rmse_score(self,
model: keras.models.Model,
Xtrain,
Xtest,
ytrain,
ytest,
index=0):
"""
root mean squared error
"""
pred_train, pred_test = model.predict(Xtrain), model.predict(Xtest)
pred_list_train = np.array([pred_train[:, index]])
pred_list_test = np.array([pred_test[:, index]])
from sklearn.metrics import mean_squared_error
rmse_train = [
np.sqrt(mean_squared_error(ytrain[:, index], _y))
for _y in pred_list_train
]
rmse_test = [
np.sqrt(mean_squared_error(ytest[:, index], _y))
for _y in pred_list_test
]
return rmse_train, rmse_test
def predict_with_model(tokenizer,
text,
model: keras.models.Model):
input_text = [i for i in text if chinese_regex.match(i)]
input_text = input_text[:tokenizer.max_length-2]
input_token = tokenizer.tokenize(input_text)
input_x = keras.preprocessing.sequence.pad_sequences([input_token],
maxlen=tokenizer.max_length,
padding='post')
predict_idx = model.predict(input_x)[0].argmax(1)
labels = tokenizer.label_de_tokenize(predict_idx, length=len(input_text))
final = ''
for i in range(len(input_text)):
final += input_text[i]
if labels[i] != 'O':
final += '{}'.format(labels[i])
return final
def test_offline(model: keras.models.Model):
images = load_data()
pred_images = images_to_pred_images(images)
pred = model.predict(np.array(pred_images))
pred = keras.applications.vgg16.decode_predictions(pred)
labels = list()
for p in pred:
l = list()
for label in p:
l.append(label[1])
labels.append(l)
show_images("image", images, labels)
def predict_gene(model: keras.models.Model,
eder: Simple_ED,
maxlen,
pre_str=None,
str_len=None):
if pre_str == None:
seed = np.random.randint(0, eder.text_len - maxlen)
pre_str = eder.text[seed:seed + maxlen]
if str_len == None:
str_len = maxlen * 10
gene_str = []
prefix = pre_str
for i in range(str_len):
code = eder.encode(prefix)
pred = model.predict(code.reshape((1, ) + code.shape), verbose=0)
ci = sample(pred[0])
c = eder.characters[ci]
gene_str.append(c)
prefix = prefix[1:] + c
print("pre_str:{}\n{}".format(pre_str, ''.join(gene_str)))
def get_and_save_model_output_on_batches(
model: keras.models.Model,
batches: image.DirectoryIterator,
model_output_to="path/to/output/to.bc",
labels_output_to="optional/path/to/output/onehot/labels/to.bc",
num_batches_to_save: int = None):
# valid_batches = ut.get_batches(path_data_valid, shuffle=False)
if num_batches_to_save is None:
num_batches_to_save = int(batches.n / batches.batch_size)
if model_output_to == "path/to/output/to.bc":
print(
'Must provide a path to output data to, such as: path_data_model + "valid_model_out.bc"'
)
output_labels = True
if labels_output_to == "optional/path/to/output/onehot/labels/to.bc":
output_labels = False
model_out, labels_out = [], []
for i in range(num_batches_to_save):
print(f"batch {i} of {num_batches_to_save}")
imgs, labels = next(batches)
model_out.append(model.predict(
imgs, batch_size=8,
verbose=1)) # May want to change batch_size on a powerful GPU!
labels_out.append(labels)
model_out = np.concatenate(model_out, axis=0)
labels_out = np.concatenate(
labels_out, axis=0) # This is automatically converted to one-hot!
print(
f"\nmodel_out.shape, labels_out.shape is [[{model_out.shape, labels_out.shape}]]"
)
save_array(model_output_to, model_out)
print(f"Saved model's output to {model_output_to}")
if output_labels:
save_array(labels_output_to, labels_out)
print(f"Saved labels's in one-hot format to {labels_output_to}")
def eval_dist(dp: utils.DataPoint, model: keras.models.Model,
datagen: keras.preprocessing.image.ImageDataGenerator):
"""returns the distance between predicted location and true location"""
x = preproc_x(dp.x, datagen)
y_pred = model.predict(x, batch_size=1).squeeze()
return K.eval(mode_distance(dp.y, y_pred))
def test_model(model: keras.models.Model, dataX: np.array,
dataY: np.array) -> float:
preds = model.predict(dataX, batch_size=1, verbose=1)
preds = np.argmax(preds, axis=-1)
accu = np.sum(preds == np.argmax(dataY, axis=-1)) / len(preds)
return accu
def HMC_ensemble_run(model: keras.models.Model,
x_train: np.array,
y_train: np.array,
N_mc: int,
ep: float,
tau: int,
burn_in: int,
sample_every: int,
return_extra=False,
verbose=True,
verbose_n = 100,
):
"""
Takes a keras model and a dataset (x_train, y_train) and returns a list of numpy
arrays to be used as weights for an ensemble predictor.
"""
step_size = ep
n_steps = tau
Ws = model.weights
lossfn = model.loss #this is kinda cheeky
X = model.input
Y_ = model.output
Y = K.placeholder(shape=Y_.shape)
L = K.sum(lossfn(Y, Y_)) #the sum accross the batch
Gs = K.gradients(L, Ws)
eval_loss_and_grads = K.function([X,Y], [L] + Gs)
def get_loss_and_grads(x, y):
res = eval_loss_and_grads([x, y])
return res[0], res[1:]
losses = []
i = 0
weights = []
ensemble = []
accept_n = 0
ep_lo = 0.8 * step_size
ep_hi = 1.2 * step_size
tau_lo = int(0.5 * n_steps)
tau_hi = int(1.5 * n_steps)
obj = 0
while len(ensemble) < N_mc:
ep = ep_lo + (ep_hi - ep_lo) * np.random.random()
tau = np.random.randint(tau_lo, high=tau_hi)
obj, gs = get_loss_and_grads(x_train, y_train)
losses.append(obj)
if verbose and i % verbose_n == 0:
acc = np.mean(model.predict(x_train).argmax(axis=1) == y_train.argmax(axis=1))
accept_ratio = accept_n / i if i > 0 else 0
print("iter: ", i, 'accuracy :', acc, 'loss: ', obj, 'accept_ratio: ', accept_ratio)
i += 1
ps = [np.random.normal(size=w.shape) for w in Ws] # momentum variables
H = .5 * sum([np.sum(p ** 2) for p in ps]) + obj
ws = [K.eval(w) for w in Ws]
weights.append(ws)
ws_old = [K.eval(w) for w in Ws]
# store the values of the weights in case we need to go back
for t in range(tau):
for p, g in zip(ps, gs):
p -= .5 * ep * g
for w, p in zip(ws, ps):
w += ep * p
# evaluate new weights
for (weight, value) in zip(Ws, ws):
K.set_value(weight, value)
obj, gs = get_loss_and_grads(x_train, y_train)
for p, g in zip(ps, gs):
p -= .5 * ep * g
H_new = .5 * sum([np.sum(p ** 2) for p in ps]) + obj
dH = H_new - H
if (dH < 0) or np.random.rand() < np.exp(-dH):
# in this case, we acccept the new values
accept_n += 1
pass
else:
# reverse the step
for weight, value in zip(Ws, ws_old):
K.set_value(weight, value)
if i > burn_in and i % sample_every == 0:
ensemble.append([K.eval(w) for w in Ws])
if return_extra:
weights = np.array(weights)
return ensemble, losses, weights, accept_n / i
else:
return ensemble
def predict_experiences(model: keras.models.Model,
experiences: Iterable[Experience]) -> np.ndarray:
x = np.stack(exp.from_state.data for exp in experiences)
return model.predict(x)
def full_multiclass_report(model: keras.models.Model,
x: np.ndarray,
y_true: np.ndarray,
classes: typing.Sequence,
cm_path: pathlib.Path,
tp_path: pathlib.Path,
tn_path: pathlib.Path,
fp_path: pathlib.Path,
fn_path: pathlib.Path,
id_valid: np.ndarray = None,
chunk=False):
"""
Builds a report containing the following:
- accuracy
- AUC
- classification report
- confusion matrix
- 7/31/2018 metrics
The output is the report as a string.
The report also generates a confusion matrix plot and tp/fp examples.
:param model:
:param x:
:param y_true:
:param classes:
:param id_valid
:param chunk
:return:
"""
# TODO(luke): Split this into separate functions.
y_proba = model.predict(x, batch_size=8)
assert y_true.shape == y_proba.shape
if y_proba.shape[-1] == 1:
y_pred = (y_proba > 0.5).astype('int32')
else:
y_pred = y_proba.argmax(axis=1)
y_true = y_true.argmax(axis=1)
assert y_pred.shape == y_true.shape, \
f'y_pred.shape: {y_pred.shape} must equal y_true.shape: {y_true.shape}'
comment = "Accuracy: " + str(sklearn.metrics.accuracy_score(
y_true, y_pred))
comment += '\n'
# Assuming 0 is the negative label
y_true_binary = y_true > 0
y_pred_binary = y_pred > 0
score = sklearn.metrics.roc_auc_score(y_true_binary, y_pred_binary)
# Do not change the line below, it affects reporting._extract_auc
comment += f'AUC: {score}\n'
comment += f'Assuming {0} is the negative label'
comment += '\n\n'
comment += "Classification Report\n"
comment += sklearn.metrics.classification_report(y_true, y_pred, digits=5)
cnf_matrix = sklearn.metrics.confusion_matrix(y_true, y_pred)
comment += '\n'
comment += str(cnf_matrix)
save_confusion_matrix(cnf_matrix, classes=classes, cm_path=cm_path)
# Compute additional metrics for the 7/31 paper
try:
tn, fp, fn, tp = cnf_matrix.ravel()
comment += f'\n\nAdditional statistics:\n'
sensitivity = tp / (tp + fn)
comment += f'Sensitivity: {sensitivity}\n'
specificity = tn / (tn + fp)
comment += f'Specificity: {tn / (tn + fp)}\n'
comment += f'Precision: {tp / (tp + fp)}\n'
total_acc = (tp + tn) / (tp + tn + fp + fn)
random_acc = (((tn + fp) * (tn + fn) + (fn + tp) * (fp + tp)) /
(tp + tn + fp + fn)**2)
comment += f'\n\nNamed statistics:\n'
kappa = (total_acc - random_acc) / (1 - random_acc)
comment += f'Cohen\'s Kappa: {kappa}\n'
youdens = sensitivity - (1 - specificity)
comment += f'Youden\'s index: {youdens}\n'
comment += f'\n\nOther sklearn statistics:\n'
log_loss = sklearn.metrics.classification.log_loss(y_true, y_pred)
comment += f'Log loss: {log_loss}\n'
comment += f'F-1: {sklearn.metrics.f1_score(y_true, y_pred)}\n'
except ValueError as e:
comment += '\nCould not add additional statistics (tp, fp, etc.)'
comment += str(e)
save_misclassification_plots(x,
y_true_binary,
y_pred_binary,
id_valid=id_valid,
chunk=chunk,
tp_path=tp_path,
fp_path=fp_path,
tn_path=tn_path,
fn_path=fn_path)
return comment
