def tta_predict(learner, im_arr):
"""Use test-time augmentation to make predictions for a single image.
This uses the dihedral transform to make 8 flipped/rotated version of the
input, makes a prediction for each one, and averages the predictive
distributions together. This will take 8x the time for a small accuracy
improvement.
Args:
learner: fastai Learner object for semantic segmentation
im_arr: (Tensor) of shape (nb_channels, height, width)
Returns:
(numpy.ndarray) of shape (height, width) containing predicted class ids
"""
# Note: we are not using the TTA method built into fastai because it only
# works on image classification problems (and this is undocumented).
# We should consider contributing this upstream to fastai.
probs = []
for k in range(8):
trans_im = dihedral(Image(im_arr), k)
o = learner.predict(trans_im)[2]
# https://forums.fast.ai/t/how-best-to-have-get-preds-or-tta-apply-specified-transforms/40731/9
o = Image(o)
if k == 5:
o = dihedral(o, 6)
elif k == 6:
o = dihedral(o, 5)
else:
o = dihedral(o, k)
probs.append(o.data)
label_arr = torch.stack(probs).mean(0).argmax(0).numpy()
return label_arr
def get_spec_images(self):
sg = self.spectro
if sg is None: return []
if torch.all(torch.eq(sg[0], sg[1])) and torch.all(
torch.eq(sg[0], sg[2])):
return [Image(sg[0].unsqueeze(0))]
else:
return [Image(s.unsqueeze(0)) for s in sg]
def get_prediction_image(path, sz=256):
bgr_img = imread(path)
b, g, r = split(bgr_img)
rgb_img = merge([r, g, b])
rgb_img = rgb_img / 255.0
img = Image(px=pil2tensor(rgb_img, np.float32))
img = img.resize((3, sz, sz))
return img.px.reshape(1, 3, sz, sz)
def reconstruct(self, t: Tensor, x: Tensor = None):
if self.feed_type == FEED_TYPE_IMAGE:
return MarkovDecisionProcessSlice(state=Image(t), state_prime=Image(x[0]),
alt_state=Floats(x[1]), action=Floats(x[1]),
reward=Floats(x[2]), done=x[3], feed_type=self.feed_type)
else:
return MarkovDecisionProcessSlice(state=Floats(t), state_prime=Floats(x[0]),
alt_state=Image(x[1]), action=Floats(x[1]),
reward=Floats(x[2]), done=x[3], feed_type=self.feed_type)
def __getitem__(self, index):
if self.indices:
index = self.indices[index]
uri = self.df['name'][index]
sample = CloudSample(pattern=self.pattern, uri=uri)
t1 = torch.tensor(sample.x(), dtype=torch.float32)
t2 = torch.tensor(sample.y(), dtype=torch.int64)
return Image(t1), Image(t2)
def get_list_from_model(learn:Learner, ds_type:DatasetType, batch:Tuple)->[]:
"Factory method to convert a batch of model images to a list of ModelImageSet."
image_sets = []
x,y = batch[0],batch[1]
preds = learn.pred_batch(ds_type=ds_type, batch=(x,y), reconstruct=True)
for orig_px, real_px, gen in zip(x,y,preds):
orig, real = Image(px=orig_px), Image(px=real_px)
image_set = ModelImageSet(orig=orig, real=real, gen=gen)
image_sets.append(image_set)
return image_sets
def show(self, title: [str] = None, **kwargs):
print(f"File: {self.path}")
self.hear(title=title)
sg = self.spectro
if sg is not None:
if torch.all(torch.eq(sg[0], sg[1])) and torch.all(torch.eq(sg[0], sg[2])):
display(Image(sg[0].unsqueeze(0)))
else:
display(Image(sg[0].unsqueeze(0)))
display(Image(sg[1].unsqueeze(0)))
display(Image(sg[2].unsqueeze(0)))
def predict(self, chips, windows, tmp_dir):
"""Return predictions for a chip using model.
Args:
chips: [[height, width, channels], ...] numpy array of chips
windows: List of boxes that are the windows aligned with the chips.
Return:
Labels object containing predictions
"""
self.load_model(tmp_dir)
# TODO get it to work on a whole batch at a time
chip = torch.Tensor(chips[0]).permute((2, 0, 1)) / 255.
im = Image(chip)
label_arr = self.inf_learner.predict(im)[1].squeeze().numpy()
# Return "trivial" instance of SemanticSegmentationLabels that holds a single
# window and has ability to get labels for that one window.
def label_fn(_window):
if _window == windows[0]:
return label_arr
else:
raise ValueError('Trying to get labels for unknown window.')
return SemanticSegmentationLabels(windows, label_fn)
def inference_classification(frame):
'''
All the steps, one by one:
frame_rgb = frame[...,::-1] #convert bgr to rgb
t = Image(pil2tensor(PIL.Image.fromarray(frame_rgb).convert("RGB"), np.float32).div_(255))
prediction = learn.predict(t)
p = prediction[1].squeeze() #prediction data
mask = np.array(p).astype(np.uint8)
'''
pred_class, pred_idx, outputs = classifier.predict(
Image(
pil2tensor(
PIL.Image.fromarray(frame[..., ::-1]).convert("RGB"),
np.float32).div_(255))) #[1].squeeze()
main = "{0:.2f}".format(float("{0:.2f}".format(max(outputs) * 100.0)))
second = "{0:.2f}".format(float(np.sort(outputs)[-2] * 100.0))
outputs = np.array(outputs).tolist()
return (classifier_class_dict[outputs.index(np.max(outputs))],
main), (classifier_class_dict[outputs.index(np.sort(outputs)[-2])],
second)
def predict(self, chips, windows, tmp_dir):
"""Return a prediction for a single chip.
Args:
chips: (numpy.ndarray) of shape (1, height, width, nb_channels)
containing a single imagery chip
windows: List containing a single (Box) window which is aligned
with the chip
Return:
(SemanticSegmentationLabels) containing predictions
"""
self.load_model(tmp_dir)
chip = torch.Tensor(chips[0]).permute((2, 0, 1)) / 255.
im = Image(chip)
self.inf_learner.data.single_ds.tfmargs['size'] = self.task_config.predict_chip_size
self.inf_learner.data.single_ds.tfmargs_y['size'] = self.task_config.predict_chip_size
if self.train_opts.tta:
label_arr = tta_predict(self.inf_learner, chip)
else:
label_arr = self.inf_learner.predict(im)[1].squeeze().numpy()
# TODO better explanation
# Return "trivial" instance of SemanticSegmentationLabels that holds a single
# window and has ability to get labels for that one window.
def label_fn(_window):
if _window == windows[0]:
return label_arr
else:
raise ValueError('Trying to get labels for unknown window.')
return SemanticSegmentationLabels(windows, label_fn)
def to_one(self):
img = torch.stack(self.data, 1)
img = img.view(3, -1, self.N, self.sz,
self.sz).permute(0, 1, 3, 2, 4).contiguous().view(
3, -1, self.sz * self.N)
return Image(1.0 - (self.mean[..., None, None] +
img * self.std[..., None, None]))
def show(self):
img = Image.open(self.info['path'])
for text in self.info['text']:
draw_rectangle(img, text['xmin'], text['ymin'], text['xmax'],
text['ymax'], 'text')
display(img)
def classify_frame(capture, model, label):
"""Use the model to predict the class label.
"""
_, frame = capture.read() # Capture frame-by-frame
_, ind, prob = model.predict(Image(utils.cv2torch(frame)))
utils.put_text(frame, f"{label[ind]} ({prob[ind]:.2f})")
return utils.cv2matplotlib(frame)
def handle_aws_lambda_event(self, event, func):
if event["headers"].get("Content-Type", "").startswith("images/"):
# decodebytes introduced at python3.1
try:
image_data = imread(base64.decodebytes(event["body"]),
pilmode=self.pilmode)
except AttributeError:
image_data = imread(
base64.decodestring(event["body"]), # pylint: disable=W1505
pilmode=self.convert_mode,
)
else:
raise BentoMLException(
"BentoML currently doesn't support Content-Type: {content_type} for "
"AWS Lambda".format(
content_type=event["headers"]["Content-Type"]))
if self.after_open:
image_data = self.after_open(image_data)
image_data = pil2tensor(image_data, np.float32)
if self.div:
image_data = image_data.div_(255)
if self.cls:
image_data = self.cls(image_data)
else:
image_data = Image(image_data)
result = func(image_data)
result = get_output_str(result, event["headers"].get("output", "json"))
return {"statusCode": 200, "body": result}
def open(self, fn):
"""TODO"""
rgb_nir_swir = ["B04", "B03", "B02", "B08", "B11", "B12"]
indices = [rgb_nir_swir.index(band) for band in self.bands]
data = np.load(fn)[:, :, indices]
data = pil2tensor(data, np.float32)
return Image(data)
def predict(self, chips, windows, tmp_dir):
"""Return predictions for a chip using model.
Args:
chips: [[height, width, channels], ...] numpy array of chips
windows: List of boxes that are the windows aligned with the chips.
Return:
Labels object containing predictions
"""
self.load_model(tmp_dir)
chip = torch.Tensor(chips[0]).permute((2, 0, 1)) / 255.
im = Image(chip)
self.inf_learner.data.single_ds.tfmargs[
'size'] = self.task_config.predict_chip_size
self.inf_learner.data.single_ds.tfmargs_y[
'size'] = self.task_config.predict_chip_size
if self.train_opts.tta:
probs = []
for k in range(8):
trans_im = dihedral(Image(chip), k)
o = self.inf_learner.predict(trans_im)[2]
# https://forums.fast.ai/t/how-best-to-have-get-preds-or-tta-apply-specified-transforms/40731/9
o = Image(o)
if k == 5:
o = dihedral(o, 6)
elif k == 6:
o = dihedral(o, 5)
else:
o = dihedral(o, k)
probs.append(o.data)
label_arr = torch.stack(probs).mean(0).argmax(0).numpy()
else:
label_arr = self.inf_learner.predict(im)[1].squeeze().numpy()
# Return "trivial" instance of SemanticSegmentationLabels that holds a single
# window and has ability to get labels for that one window.
def label_fn(_window):
if _window == windows[0]:
return label_arr
else:
raise ValueError('Trying to get labels for unknown window.')
return SemanticSegmentationLabels(windows, label_fn)
def __init__(self, img, model, path_to_learner="./models"):
self.path_to_learner = path_to_learner
self.learn = load_learner(path=self.path_to_learner, file=model)
"Convert np.ndarray to torch style Image"
self._img = np.copy(img)
self.img = Image(pil2tensor(img, np.float32).div_(255))
def open(self, fn):
try:
image = Image(grayloader(fn))
except:
print('Error {}'.format(fn))
image = 0.0
return image
def open(self, uri: str):
uri = uri.replace('./', '')
dataset_id = 'train'
root_path = ROOT_PATH + '/38-Cloud_{dataset_id}'.format(dataset_id=dataset_id)
pattern = root_path + '/{dataset_id}_{band}/{band}_{uri}.TIF'.replace('{dataset_id}', dataset_id)
sample = CloudSample(pattern=pattern, uri=uri)
return Image(torch.tensor(sample.x()))
def open(self, fn):
img_data = np.load(fn)
img_lists = []
for j in range(img_data.shape[0]):
imgs = []
for i in range(img_data.shape[1]):
imgs.append(Image(tensor(img_data[j, i, :, :][None])))
img_lists.append(imgs)
return TransformableLists(img_lists)
def handle_request(self, request, func):
try:
from fastai.vision import Image, pil2tensor
except ImportError:
raise ImportError(
"fastai package is required to use FastaiImageHandler")
try:
from imageio import imread
except ImportError:
raise ImportError(
"imageio package is required to use FastaiImageHandler")
if request.method != "POST":
return Response(response="Only accept POST request", status=400)
input_streams = []
for filename in self.input_names:
file = request.files.get(filename)
if file is not None:
file_name = secure_filename(file.filename)
check_file_format(file_name, self.accept_file_extensions)
input_streams.append(BytesIO(file.read()))
if len(input_streams) == 0:
if request.data:
input_streams = (request.data, )
else:
raise ValueError(
"BentoML#ImageHandler unexpected HTTP request: %s" %
request)
input_data = []
for input_stream in input_streams:
data = imread(input_stream, pilmode=self.convert_mode)
if self.after_open:
data = self.after_open(data)
data = pil2tensor(data, np.float32)
if self.div:
data = data.div_(255)
if self.cls:
data = self.cls(data)
else:
data = Image(data)
input_data.append(data)
result = func(*input_data)
result = get_output_str(result, request.headers.get("output", "json"))
return Response(response=result,
status=200,
mimetype="application/json")
def convert_to_fastai(frame):
""" Makes an opencv::mat image into a fastai compatible image. """
swapped_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
img_fastai = Image(pil2tensor(swapped_image, dtype=np.float32).div_(255))
# using Umat. I found it to be slow, but I'm leaving it here for posterity.
# mat_image = cv2.UMat.get(swapped_image)
# img_fastai = Image(pil2tensor(mat_image, dtype=np.float32).div_(255))
return img_fastai
def predict(self, img: np.ndarray) -> float:
"""Return probability of BirdHome."""
img_tensor = torch.from_numpy(img).float() / 255
# MPL has colors last, while torch expects colors first. Swap:
img_tensor = img_tensor.permute(2, 0, 1)
img = Image(img_tensor)
preds = self.learn.predict(img)
pred_bird_home = preds[2][0]
return pred_bird_home
def show_xys(self, xs, ys, xs_processed=None, figsize=None, **kwargs):
if xs_processed is None:
for x, y in zip(xs, ys):
x.show(title=str(y), figsize=figsize, **kwargs)
else:
for x, y, x_processed in zip(xs, ys, xs_processed):
x.show(title=str(y), figsize=figsize, **kwargs)
for channel in range(x_processed.size(0)):
Image(x_processed[channel, :, :].unsqueeze(0)).show(
figsize=figsize)
def __getitem__(self, idx):
image = self.images[idx]
if self.fastai_transform:
image = Image(image).apply_tfms(self.fastai_transform).px
elif self.torchvision_transform:
image = self.torchvision_transform(tpi(image))
if self.labels is None:
return image
else:
label = torch.LongTensor([self.labels[idx]]).squeeze()
return image, label
def open(self, fn):
image = np.load(fn)['arr_0']
image = np.transpose(image, (2, 0, 1))
image[1] /= 128.0
image[0] /= 128.0
image[3] /= 5.0
image = torch.Tensor(image)
#print('{} {} {}'.format(image.min(), image.max(), image.mean()))
image = Image(image)
return image
def from_df(cls,
df: DataFrame,
path,
cols=0,
folder=None,
suffix: str = '',
**kwargs) -> 'ItemList':
"Get the filenames in `cols` of `df` with `folder` in front of them, `suffix` at the end."
values = df.iloc[:, 1:].values
labels = df.iloc[:, 0].values
values = torch.tensor(values).float()
items = []
for y, v in zip(labels, values):
t = Image(v.view(28, 28).unsqueeze(0))
t.y = y
items.append(t)
# res = cls(items, label_cls=CategoryList)
res = cls(items)
return res
def open(self, item):
filename, pos = item
if self.current_image_filename != filename:
self.current_image = open_image(filename)
self.current_image_filename = filename
# pos is (x, y) center position of a cell on the provided image,
# we will take a 32x32px patch from this location:
x = int(pos[0])
y = int(pos[1])
# note: fast.ai Image dimensions are (c, y, x)
return Image(self.current_image.data[:, y - 16:y + 16, x - 16:x + 16])
def _get_array_from_image(self, img: Image) -> np.array:
"""
Convert an Image object into numpy array.
:param img: fastai Image with dimensions: (channels, height, width)
:return: numpy array with dimensions: (height, width, channels)
"""
img = img.data.numpy() # (channels, height, width)
img = img.transpose(1, 2, 0) # (height, width, channels)
return img
def my_cl_int_plot_top_losses(self, k, largest=True, figsize=(25,7), heatmap:bool=True, heatmap_thresh:int=16,
return_fig:bool=None)->Optional[plt.Figure]:
"Show images in `top_losses` along with their prediction, actual, loss, and probability of actual class."
tl_val,tl_idx = self.top_losses(k, largest)
classes = self.data.classes
cols = math.ceil(math.sqrt(k))
rows = math.ceil(k/cols)
fig,axes = plt.subplots(rows, cols, figsize=figsize)
fig.suptitle('prediction/actual/loss/probability', weight='bold', size=14)
for i,idx in enumerate(tl_idx):
audio, cl = self.data.dl(self.ds_type).dataset[idx]
audio = audio.clone()
m = self.learn.model.eval()
x, _ = self.data.one_item(audio) # Process one audio into prediction
x_consolidated = x.sum(dim=1, keepdim=True) # Sum accross all channels to ease the interpretation
im = Image(x_consolidated[0, :, :, :].cpu()) # Extract the processed image from the prediction (after dl_tfms) and keep it into CPU
cl = int(cl)
title = f'{classes[self.pred_class[idx]]}/{classes[cl]} / {self.losses[idx]:.2f} / {self.probs[idx][cl]:.2f}'
title = title + f'\n {audio.fn}'
im.show(ax=axes.flat[i], title=title)
if heatmap:
# Related paper http://openaccess.thecvf.com/content_ICCV_2017/papers/Selvaraju_Grad-CAM_Visual_Explanations_ICCV_2017_paper.pdf
with hook_output(m[0]) as hook_a: # hook activations from CNN module
with hook_output(m[0], grad= True) as hook_g: # hook gradients from CNN module
preds = m(x) # Forward pass to get activations
preds[0,cl].backward() # Backward pass to get gradients
acts = hook_a.stored[0].cpu()
if (acts.shape[-1]*acts.shape[-2]) >= heatmap_thresh:
grad = hook_g.stored[0][0].cpu() # Hook the gradients from the CNN module and extract the first one (because one item only)
grad_chan = grad.mean(1).mean(1) # Mean accross image to keep mean gradients per channel
mult = F.relu(((acts*grad_chan[...,None,None])).sum(0)) # Multiply activation with gradients (add 1 dim for height and width)
sz = list(im.shape[-2:])
axes.flat[i].imshow(mult, alpha=0.35, extent=(0,*sz[::-1],0), interpolation='bilinear', cmap='magma')
if ifnone(return_fig, defaults.return_fig): return fig
