def speed(img1, img2, start_point, end_point): # compute target feature map # f1 = resnet.preprocess_input(img1) f1 = resnet.preprocess_input(img1) f1 = model(f1) f1 = tf.math.l2_normalize(f1, axis=1) f1 = tf.keras.layers.Flatten()(f1) # compute patch feature map # f2 = resnet.preprocess_input(img2) f2 = resnet.preprocess_input(img2) f2 = model(f2) f2 = tf.math.l2_normalize(f2, axis=1) f2 = tf.keras.layers.Flatten()(f2) # compute similarity dist = tf.reduce_sum(tf.square(f1-f2), axis=1) sim = tf.reduce_sum(f1*f2, axis=1) # extract pass samples # chosen_idx = tf.math.logical_and(sim > 0.75, dist < 0.45) # chosen_idx = tf.where(chosen_idx) chosen_idx = tf.where(sim > 0.73) chosen_idx = tf.keras.backend.flatten(chosen_idx) chosen_img = tf.gather(img2, chosen_idx) chosen_start = tf.gather(start_point, chosen_idx) chosen_end = tf.gather(end_point, chosen_idx) chosen_dist = tf.gather(dist, chosen_idx) chosen_sim = tf.gather(sim, chosen_idx) return (chosen_img, chosen_start, chosen_end, chosen_dist, chosen_sim)
def prediction(model_pruned, x_val_paths, y_val_one_hot):
y_pred = None
for i, x_val_path in enumerate(x_val_paths):
x_val = np.load(x_val_path).astype('float32') # loaded as RGB
x_val = resnet.preprocess_input(x_val) # converted to BGR
y_pred_sharded = model_pruned.predict(x_val,
verbose=0,
use_multiprocessing=True,
batch_size=64,
callbacks=None)
try:
y_pred = np.concatenate([y_pred, y_pred_sharded])
except ValueError:
y_pred = y_pred_sharded
del x_val
gc.collect()
completed_percentage = (i + 1) * 100 / len(x_val_paths)
if completed_percentage % 5 == 0:
print("{:5.1f}% completed.".format(completed_percentage))
return top_k_accuracy(y_val_one_hot, y_pred,
k=1), top_k_accuracy(y_val_one_hot, y_pred, k=5)
def func_resnet(
pretrain_dataset: str = None,
pooling: str = "max",
task: str = "orig_labels",
):
"""A ResNet50 model that can be pretrained or trained from scratch,
adapting to each relevant variation in this project. This is the
functional API version.
Works well with Weights and Biases' callback.
Args:
pretrain_dataset (str, optional): The dataset in which the model
is pretrained. If left unspecified, the model starts with random
weights. Available options are "imagenet" and "bigearthnet".
Defaults to None.
pooling (str, optional): The type of global pooling to perform
after the ResNet layers. Available options are "max" and "avg".
Defaults to "max".
task (str, optional): The task on which the model will be trained
or fine-tuned on. Available options are "orig_labels" (original
labels from the Kaggle challenge) and "deforestation".
Defaults to "orig_labels".
Raises:
Exception: [description]
"""
inputs = layers.Input(shape=(256, 256, 3))
if task == "orig_labels":
n_outputs = 17
elif task == "deforestation":
n_outputs = 1
else:
raise Exception(
f'ERROR: Unrecognized task "{task}". Please select one of "orig_labels" or "deforestation".'
)
if pretrain_dataset == "bigearthnet":
# TensorFlow Hub modules require data in a [0, 1] range
# stats estimated from subset of data in `02_eda_amazon_planet` notebook
x = Normalization(
mean=[79.67114306, 87.08461826, 76.46177919],
variance=[1857.54070494, 1382.94249315, 1266.69265399],
)(inputs)
x = data_augmentation(x)
x = hub.KerasLayer(
"https://tfhub.dev/google/remote_sensing/bigearthnet-resnet50/1")(
x)
else:
# Using TensorFlow's ResNet-specific preprocessing
x = preprocess_input(x)
x = data_augmentation(x)
x = ResNet50(
include_top=False,
weights=pretrain_dataset,
pooling=pooling,
)(x)
outputs = layers.Dense(n_outputs, activation="sigmoid")(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
def process_image(img_name):
imgA = dataset.get_image(img_name)
imgA = Image.fromarray(imgA)
imgA = imgA.resize((self.image_shape[1], self.image_shape[0]))
imgA = np.array(imgA)
imgA = preprocess_input(imgA)
imgA /= 255
return imgA
def predict(img):
model = ResNet152()
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = resnet.preprocess_input(x)
predictions = model.predict(x)
all_cat_pred = predictions[0]
predicted_classes = resnet.decode_predictions(predictions, top=9)
return predictions[0]
def read_and_prep_images(img_path, img_width=IMAGE_WIDTH, img_height=IMAGE_HEIGHT):
img_names = [ f for f in os.listdir(img_path) if (os.path.isfile(os.path.join(img_path, f)) and f.endswith(".jpg")) ]
img_names = img_names[:2500]
imgs = [ load_img(os.path.join(img_path, img_name), target_size=(img_height, img_width)) for img_name in img_names ]
imgs = np.array([img_to_array(img) for img in imgs])
imgs = preprocess_input(imgs)
imgs = { k.split(".")[0]: v for k, v in zip(img_names, imgs) }
return imgs
def dask_test(split: int, epochs: int):
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras import models
import tensorflow_datasets as tfds
from tensorflow.keras.applications.resnet import preprocess_input
import numpy as np
import math
import time
from typing import Tuple
temp = tf.zeros([4, 32, 32, 3])
preprocess_input(temp)
image_size = (64, 64)
@tf.function
def load_image(datapoint, image_size: Tuple[int, int], num_classes: int):
input_image, label = tf.image.resize(datapoint["image"], image_size), datapoint['label']
input_image = preprocess_input(input_image)
return input_image, tf.one_hot(label, depth=num_classes)
model = models.load_model("/tf/notebooks/cifar10.h5")
dataset = tfds.load('cifar10', split=f'test[:{split}%]')
dataset = dataset.map(lambda x: load_image(x, image_size, 10))
dataset = dataset.batch(64)
dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
model.compile(optimizer="rmsprop",
loss='categorical_crossentropy',
metrics=['accuracy'])
tik = time.time()
for epoch in range(epochs):
og = model.evaluate(dataset, verbose=0)
tok = time.time()
return og, tok - tik, get_worker().name
def picture(patch_path):
model = load_model(model_path)
img = image.load_img(patch_path)
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
preds = model.predict(x)
return preds
def create_model():
embedding = create_embedding()
anchor_input = layers.Input(name="anchor", shape=target_shape + (3, ))
positive_input = layers.Input(name="positive", shape=target_shape + (3, ))
negative_input = layers.Input(name="negative", shape=target_shape + (3, ))
distances = DistanceLayer()(
embedding(resnet.preprocess_input(anchor_input)),
embedding(resnet.preprocess_input(positive_input)),
embedding(resnet.preprocess_input(negative_input)),
)
siamese_network = Model(
inputs=[anchor_input, positive_input, negative_input],
outputs=distances)
return embedding, siamese_network
def _read_image(self, index):
""" Reads images from directory.
:param index: image index to read
:return ndarray representation of image batch
"""
image = imread(os.path.join(self.config.data_source, index))
image = self._preprocess(image)
image = preprocess_input(image)
return image.transpose(2, 0, 1)
def preProcessImage(imagePath, model, input_height, input_width):
"""
Reshape the image to input_height and input_widht as it is a models' constraint
"""
image = I.load_img(imagePath, target_size=(input_height, input_width))
image = I.img_to_array(image)
image = np.reshape(image, (1, input_height, input_width, 3))
if model == 'VGG16':
image = preprocess_input(image)
elif model == 'ResNet50':
image = resnet.preprocess_input(image)
return image
def preprocess_image(filename):
"""
Load the specified file as a JPEG image, preprocess it and
resize it to the target shape.
"""
image_string = tf.io.read_file(filename)
image = tf.image.decode_jpeg(image_string, channels=3)
image = tf.image.convert_image_dtype(image, tf.float32)
# Resize the image to match the target size
image = tf.image.resize(image, target_shape)
image = resnet.preprocess_input(image)
return image
def predict():
classes = ['Class0','Class1','Class10','Class11','Class12','Class2','Class3','Class4',
'Class5','Class6','Class7','Class8',
'Class9']
target_names = ['Хороший класс', 'Волосяные трещины', 'Сломанный и раздавленный', "Тонкий слой",
'Хвостики', 'Затертая поверхность', 'Значительное искажение',
'Отверстия в шоколаде', 'Открытый центр', 'Плохая декорация', 'Плохое донышко','Повреждение упаковочной машины']
# with graph.as_default():
validation_img_paths = [os.path.join(os.getcwd(),"test",'test.JPG')]
img_list = [Image.open(img_path) for img_path in validation_img_paths]
validation_batch = np.stack([preprocess_input(np.array(img.resize((224,224))))
for img in img_list])
# print(validation_batch)
model._make_predict_function()
pred_probs = model.predict(validation_batch)
print("\r\n")
# print(pred_probs)
# top_values= [pred_probs[0][i] for i in np.argsort(class_prob)[-3:]]
top_values= [pred_probs[0][i] for i in np.argsort(pred_probs[0])[-3:]]
top = pred_probs.argsort()[0][-3:][::-1]
resp = [{
"Name":target_names[top[0]],
"Accuracy":pred_probs[0][top[0]],
},
{
"Name":target_names[top[1]],
"Accuracy":pred_probs[0][top[1]],
},
{
"Name":target_names[top[2]],
"Accuracy":pred_probs[0][top[2]],
}]
# y_pred = np.argmax(pred_probs, axis=1)
print(resp)
# img_list = [Image.open(input_path + img_path) for img_path in validation_img_paths]
# print(os.path.join(os.getcwd(),"test"))
# test_generator = test_datagen.flow_from_directory(
# os.path.join(os.getcwd(), "test"),
# target_size=(224, 224),
# color_mode="rgb",
# shuffle = False,
# class_mode='binary',
# batch_size=1)
# Y_pred=model.predict(test_generator)
# y_pred = np.argmax(Y_pred, axis=1)
# print( Y_pred)
# print(classification_report(classes, y_pred, target_names=target_names))
return resp
def classify_unseen_image(self, labels, model):
"""
This method takes an unseen image, performs some preprocessing to prepare it for the model, and predicts the class of the image using the model.
"""
# Define path
img_path = os.path.join("..", "data", "unseen_images",
self.unseen_image)
# Load unseen image
image = load_img(img_path, target_size=(
224, 224
)) # using the same size as the images the model has been trained on
# Convert the image to a numpy array
image = img_to_array(image)
# Reshape the image, because the model expects a tensor of rank 4. The image goes from being 3-dimensional to 4-dimensional: (1, 224, 224, 3)
image = image.reshape(
(1, image.shape[0], image.shape[1], image.shape[2]))
# Prepare the image for the ResNet50 model
image = preprocess_input(image)
# Predict the class of the image
prediction = np.argmax(model.predict(image))
# Convert labels to be a dictionary which is needed to extract the label that corresponds to the prediction
labels = dict(zip(labels, range(len(labels))))
# Define function that finds the key (letter) that corresponds to the predicted value
def find_key(dictionary, value):
return {k for k, v in dictionary.items() if v == value}
# Extract letter that corresponds to the predicted value from the label dictionary
label = find_key(labels, prediction)
# Print the predicted class to the terminal
print(
f"\nThe model predicts {self.unseen_image} to be the letter {label}"
)
# Save prediction as txt-file to output directory
with open(
os.path.join("..", "output",
f"{self.unseen_image}_prediction.txt"), "w") as f:
f.write(
f"The predicted class of the {self.unseen_image} made by the model is {label}"
)
return label
def predict(image_path):
model = None
prediction = None
model = load_model('model.h5')
img = image.load_img(image_path, target_size=(224, 224))
#processing the image
x = image.img_to_array(img)
x = x / 255 #scaling
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
prediction = model.predict(x)
clear_session()
prediction = np.argmax(prediction, axis=1)
return prediction
def get_resnet101_feature_local(imgs):
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
tf.config.experimental.set_memory_growth(gpus[0], True)
feat_extractor = keras.applications.ResNet101(weights='imagenet',
include_top=False)
IMAGE_SHAPE = (224, 224)
imgs = [np.array(img.resize(IMAGE_SHAPE)) for img in imgs]
img_data = preprocess_input(np.array(imgs))
s1 = time.time()
features = feat_extractor.predict(img_data)
features = np.array(features).reshape(len(imgs), -1)
logging.info('inference time:{}'.format(time.time() - s1))
return features
def preprocess_images(im_path):
# load the original image from gdrive (in OpenCV format)
# resize the image to its target dimensions
orig = cv2.imread(im_path)
resized = cv2.resize(orig, (224, 224))
# load the input image from gdrive (in Keras/TensorFlow format)
# basic image pre-processing
image = load_img(im_path, target_size=(224, 224))
image = img_to_array(image)
image = np.expand_dims(image, axis=0)
image = resnet.preprocess_input(image)
return resized, image, orig
def generate(self, images, show=False):
y = []
for img in images:
img = np.array(img)
img = cv.resize(img, (self.n, self.n))
img = preprocess_input(img)
if show:
cv.imshow("Keypoint", img)
cv.waitKey(0)
img = img.reshape(1, self.n, self.n, 3)
feature_vec = self.model.predict(img)[0]
y.append(feature_vec)
return np.array(y)
def load_image(path, target_size):
try:
image = load_img(path, target_size=target_size)
# convert the image pixels to a numpy array
image = img_to_array(image)
except:
print(path)
image = cv2.imread(path)
image = cv2.resize(image, target_size, interpolation=cv2.INTER_AREA)
image = Image.fromarray(image)
image = img_to_array(image)
#image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = preprocess_input(image)
#image = np.expand_dims(image,axis=-1)
#image = image.astype("float32")/255.0
return image
def model_preprocess_fn(inputs, max_dimension, min_dimension):
image_resizer_fn = functools.partial(preprocessor.resize_to_range,
min_dimension=min_dimension,
max_dimension=max_dimension,
method=tf.image.ResizeMethod.BILINEAR,
pad_to_max_dimension=True,
per_channel_pad_value=(0, 0, 0))
with tf.name_scope('Preprocessor'):
outputs = tf.map_fn(image_resizer_fn,
elems=inputs,
dtype=[tf.float32, tf.int32])
resized_inputs = outputs[0]
true_image_shapes = outputs[1]
return (preprocess_input(resized_inputs), true_image_shapes)
def preprocess(self, resized_inputs):
"""Faster R-CNN Resnet V1 preprocessing.
VGG style channel mean subtraction as described here:
https://gist.github.com/ksimonyan/211839e770f7b538e2d8#file-readme-md
Note that if the number of channels is not equal to 3, the mean subtraction
will be skipped and the original resized_inputs will be returned.
Args:
resized_inputs: A [batch, height_in, width_in, channels] float32 tensor
representing a batch of images with values between 0 and 255.0.
Returns:
preprocessed_inputs: A [batch, height_out, width_out, channels] float32
tensor representing a batch of images.
"""
return preprocess_input(resized_inputs)
def test_tl_model(file_name, model, picture_size):
"""Test transfer learning model on single picture
Plot face with predicted point
"""
x, scale_x, scale_y = preprocess_picture(file_name, picture_size)
x = preprocess_input(x)
x = np.expand_dims(x, axis=0)
y_pred = model.predict(x)
x, y_pred = x * MAX_RGB, y_pred * picture_size
# load original image
img = Image.open(file_name)
# Covert into RGB
img = change_to_rgb(img)
img_arr = np.asarray(img)
# Re scale
y_pred[0, 0] = y_pred[0, 0] / scale_x
y_pred[0, 1] = y_pred[0, 1] / scale_y
plot_prediction(img_arr, y_pred[0, :])
def transforms(self, img):
width, height = img.size
if width < height:
img = img.resize((224, int(height * (224 / width))))
else:
img = img.resize((int(width * (224 / height)), 224))
width, height = img.size
crop_width = max(width - 224, 0)
crop_height = max(height - 224, 0)
cropArea = (crop_width // 2, crop_height // 2, 224 + crop_width // 2,
224 + crop_height // 2)
img = img.crop(cropArea)
img_np = img_to_array(img)
img_np = preprocess_input(img_np)
return img_np
def get_resnet101_feature_grpc(url, imgs, timeout=20):
IMAGE_SHAPE = (224, 224)
imgs = [np.array(img.resize(IMAGE_SHAPE)) for img in imgs]
img_data = preprocess_input(np.array(imgs))
channel = grpc.insecure_channel(
url,
options=[('grpc.max_receive_message_length', 2000 * 1024 * 1024)]
# ,options=[('grpc.default_authority','sku_retrieval')]
)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'resnet101'
request.model_spec.signature_name = 'serving_default'
s0 = time.time()
request.inputs['input_1'].CopyFrom(tf.make_tensor_proto(img_data))
s1 = time.time()
response = stub.Predict(request, timeout)
logging.info('data copy time:{:.4f}s inference time:{:.4f}s'.format(
s1 - s0,
time.time() - s1))
features = np.asarray(response.outputs['conv5_block3_out'].float_val)
features = np.reshape(features, (len(img_data), -1))
return features
def extract(self, video_path, start=None, duration=None):
print("extract video", video_path, "from", start, duration)
# feature_path = video_path.replace(
# ".mkv", f"_{self.feature}_{self.back_end}.npy")
feature_path = video_path[:-4] + f"_{self.feature}_{self.back_end}.npy"
if os.path.exists(feature_path) and not self.overwrite:
return
if "TF2" in self.back_end:
if self.grabber=="skvideo":
videoLoader = Frame(video_path, FPS=self.FPS, transform=self.transform, start=start, duration=duration)
elif self.grabber=="opencv":
videoLoader = FrameCV(video_path, FPS=self.FPS, transform=self.transform, start=start, duration=duration)
# create numpy aray (nb_frames x 224 x 224 x 3)
# frames = np.array(videoLoader.frames)
# if self.preprocess:
frames = preprocess_input(videoLoader.frames)
if duration is None:
duration = videoLoader.time_second
# time_second = duration
if self.verbose:
print("frames", frames.shape, "fps=", frames.shape[0]/duration)
# predict the featrues from the frames (adjust batch size for smalled GPU)
features = self.model.predict(frames, batch_size=64, verbose=1)
if self.verbose:
print("features", features.shape, "fps=", features.shape[0]/duration)
# save the featrue in .npy format
os.makedirs(os.path.dirname(feature_path), exist_ok=True)
np.save(feature_path, features)
def normalize(input_image):
return preprocess_input(input_image)
import tensorflow as tf
from tensorflow.keras.applications.resnet import preprocess_input
import matplotlib.pyplot as plt
import numpy as np
import tensorflow_datasets as tfds
from typing import Tuple
import math
temp = tf.zeros([4, 32, 32, 3]) # Or tf.zeros
preprocess_input(temp)
def flip(x: tf.Tensor) -> tf.Tensor:
"""Flip augmentation
Args:
x: Image to flip
Returns:
Augmented image
"""
x = tf.image.random_flip_left_right(x)
x = tf.image.random_flip_up_down(x)
return x
def color(x: tf.Tensor) -> tf.Tensor:
"""Color augmentation
Args:
for img in pyr:
# calculate the scale (to upsample bbox locations)
scale = W / float(img.shape[1])
# slide the window through the current img
for (x, y, orig_roi) in sliding_window(img, WIN_STEP, ROI_SIZE):
# calculate the scaled dimensions
x = int(x * scale)
y = int(y * scale)
w = int(ROI_SIZE[0] * scale)
h = int(ROI_SIZE[1] * scale)
# preprocess the ROI
roi = cv2.resize(orig_roi, INPUT_SIZE)
roi = img_to_array(roi)
roi = preprocess_input(roi)
# update the ROI list along with their correspoding coordinates
rois.append(roi)
locs.append((x, y, x + w, y + h))
# check if the sliding process is to be visualized
if args["visualize"] > 0:
# clone the original image
clone = orig.copy()
cv2.rectangle(clone, (x, y), ((x + w), (y + h)), (0, 0, 255), 2)
# show the visualization and the current ROI
cv2.imshow("Visualization", clone)
cv2.imshow("ROI", roi)
cv2.waitKey(0)
def load_image(datapoint, image_size: Tuple[int, int], num_classes: int):
input_image, label = tf.image.resize(datapoint["image"], image_size), datapoint['label']
input_image = preprocess_input(input_image)
return input_image, tf.one_hot(label, depth=num_classes)
def preprocessing_func(image):
'''Function that preprocesses the input'''
preproc_img = img_to_array(image)
preproc_img = np.expand_dims(image, axis=0)
preproc_img = preprocess_input(image)
return preproc_img
