class MaskRCNN(ArcGISModel):
"""
Creates a ``MaskRCNN`` Instance segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `MaskRCNN`, which
is `resnet50` by default.
Compatible backbones: 'resnet50'
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: ``MaskRCNN`` Object
"""
def __init__(self, data, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
self._backbone = models.resnet50
#if not self._check_backbone_support(self._backbone):
# raise Exception (f"Enter only compatible backbones from {', '.join(self.supported_backbones)}")
self._code = instance_detector_prf
model = models.detection.maskrcnn_resnet50_fpn(pretrained=True,
min_size=data.chip_size)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, data.c)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(
in_features_mask, hidden_layer, data.c)
self.learn = Learner(data, model, loss_func=mask_rcnn_loss)
self.learn.callbacks.append(train_callback(self.learn))
self.learn.model = self.learn.model.to(self._device)
# fixes for zero division error when slice is passed
self.learn.layer_groups = split_model_idx(self.learn.model, [28])
self.learn.create_opt(lr=3e-3)
if pretrained_path is not None:
self.load(pretrained_path)
def __str__(self):
return self.__repr__()
def __repr__(self):
return '<%s>' % (type(self).__name__)
@property
def supported_backbones(self):
return [models.detection.maskrcnn_resnet50_fpn.__name__]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``MaskRCNN`` Instance segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `MaskRCNN` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
empty_data = _EmptyData(path=tempfile.TemporaryDirectory().name,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
empty_data.class_mapping = class_mapping
empty_data.color_mapping = color_mapping
return cls(empty_data,
**model_params,
pretrained_path=str(model_file))
else:
return cls(data, **model_params, pretrained_path=str(model_file))
def _create_emd(self, path):
import random
super()._create_emd(path)
self._emd_template["Framework"] = "arcgis.learn.models._inferencing"
self._emd_template["ModelConfiguration"] = "_maskrcnn_inferencing"
self._emd_template["InferenceFunction"] = "ArcGISInstanceDetector.py"
self._emd_template["ExtractBands"] = [0, 1, 2]
self._emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
self._emd_template['Classes'].append(class_data.copy())
json.dump(self._emd_template,
open(path.with_suffix('.emd'), 'w'),
indent=4)
return path.stem
@property
def _model_metrics(self):
return {}
def _predict_results(self, xb):
self.learn.model.eval()
predictions = self.learn.model(xb.cuda())
predictionsf = []
for i in range(len(predictions)):
predictionsf.append({})
predictionsf[i]['masks'] = predictions[i]['masks'].detach().cpu(
).numpy()
predictionsf[i]['boxes'] = predictions[i]['boxes'].detach().cpu(
).numpy()
predictionsf[i]['labels'] = predictions[i]['labels'].detach().cpu(
).numpy()
predictionsf[i]['scores'] = predictions[i]['scores'].detach().cpu(
).numpy()
del predictions[i]['masks']
del predictions[i]['boxes']
del predictions[i]['labels']
del predictions[i]['scores']
del xb
torch.cuda.empty_cache()
return predictionsf
def _predict_postprocess(self,
predictions,
threshold=0.5,
box_threshold=0.5):
pred_mask = []
pred_box = []
for i in range(len(predictions)):
out = predictions[i]['masks'].squeeze()
pred_box.append([])
if out.shape[0] != 0: # handle for prediction with n masks
if len(
out.shape
) == 2: # for out dimension hxw (in case of only one predicted mask)
out = out[None]
ymask = np.where(out[0] > threshold, 1, 0)
#if torch.max(out[0]) > threshold:
if predictions[i]['scores'][0] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][0])
for j in range(1, out.shape[0]):
ym1 = np.where(out[j] > threshold, j + 1, 0)
ymask += ym1
#if torch.max(out[j]) > threshold:
if predictions[i]['scores'][j] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][j])
else:
ymask = np.zeros(
(self._data.chip_size,
self._data.chip_size)) # handle for not predicted masks
pred_mask.append(ymask)
return pred_mask, pred_box
def show_results(self,
mode='mask',
mask_threshold=0.5,
box_threshold=0.7,
nrows=None,
imsize=5,
index=0,
alpha=0.5,
cmap='tab20'):
"""
Displays the results of a trained model on a part of the validation set.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mode Required arguments within ['bbox', 'mask', 'bbox_mask'].
* ``bbox`` - For visualizing only boundig boxes.
* ``mask`` - For visualizing only mask
* ``bbox_mask`` - For visualizing both mask and bounding boxes.
--------------------- -------------------------------------------
mask_threshold Optional float. The probabilty above which
a pixel will be considered mask.
--------------------- -------------------------------------------
box_threshold Optional float. The pobabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nrows Optional int. Number of rows of results
to be displayed.
===================== ===========================================
"""
if mode not in ['bbox', 'mask', 'bbox_mask']:
raise Exception("mode can be only ['bbox', 'mask', 'bbox_mask']")
# Get Number of items
if nrows is None:
nrows = self._data.batch_size
ncols = 2
# Get Batch
xb, yb = self._data.one_batch('DatasetType.Valid')
predictions = self._predict_results(xb)
pred_mask, pred_box = self._predict_postprocess(
predictions, mask_threshold, box_threshold)
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(ncols * imsize, nrows * imsize))
fig.suptitle('Ground Truth / Predictions', fontsize=20)
for i in range(nrows):
ax[i][0].imshow(xb[i].numpy().transpose(1, 2, 0))
ax[i][0].axis('off')
if mode in ['mask', 'bbox_mask']:
yb_mask = yb[i][0].numpy()
for j in range(1, yb[i].shape[0]):
max_unique = np.max(np.unique(yb_mask))
yb_j = np.where(yb[i][j] > 0, yb[i][j] + max_unique,
yb[i][j])
yb_mask += yb_j
ax[i][0].imshow(yb_mask, cmap=cmap, alpha=alpha)
ax[i][0].axis('off')
ax[i][1].imshow(xb[i].numpy().transpose(1, 2, 0))
ax[i][1].axis('off')
if mode in ['mask', 'bbox_mask']:
ax[i][1].imshow(pred_mask[i], cmap=cmap, alpha=alpha)
if mode in ['bbox', 'bbox']:
if pred_box[i] != []:
for num_boxes in pred_box[i]:
rect = patches.Rectangle((num_boxes[0], num_boxes[1]),
num_boxes[2] - num_boxes[0],
num_boxes[3] - num_boxes[1],
linewidth=1,
edgecolor='r',
facecolor='none')
ax[i][1].add_patch(rect)
ax[i][1].axis('off')
plt.subplots_adjust(top=0.95)
torch.cuda.empty_cache()
class DeepLab(ArcGISModel):
"""
Creates a ``DeepLab`` Semantic segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `DeepLab`, which
is `resnet101` by default since it is pretrained in
torchvision. It supports the ResNet,
DenseNet, and VGG families.
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: ``DeepLab`` Object
"""
def __init__(self, data, backbone=None, pretrained_path=None):
# Set default backbone to be 'resnet101'
if backbone is None:
backbone = models.resnet101
super().__init__(data, backbone)
_backbone = self._backbone
if hasattr(self, '_orig_backbone'):
_backbone = self._orig_backbone
if not self._check_backbone_support(_backbone):
raise Exception(
f"Enter only compatible backbones from {', '.join(self.supported_backbones)}"
)
self._code = image_classifier_prf
if self._backbone.__name__ is 'resnet101':
model = _create_deeplab(data.c)
if self._is_multispectral:
model = _change_tail(model, data)
else:
model = Deeplab(data.c, self._backbone, data.chip_size)
self.learn = Learner(data, model, metrics=self._accuracy)
self.learn.loss_func = self._deeplab_loss
self.learn.model = self.learn.model.to(self._device)
self._freeze()
self._arcgis_init_callback() # make first conv weights learnable
if pretrained_path is not None:
self.load(pretrained_path)
@property
def supported_backbones(self):
return DeepLab._supported_backbones()
@staticmethod
def _supported_backbones():
return [*_resnet_family, *_densenet_family, *_vgg_family]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``DeepLab`` semantic segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `DeepLab` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
empty_data = _EmptyData(path=emd_path.parent.parent,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
empty_data.class_mapping = class_mapping
empty_data.color_mapping = color_mapping
empty_data = get_multispectral_data_params_from_emd(
empty_data, emd)
empty_data.emd_path = emd_path
empty_data.emd = emd
return cls(empty_data,
**model_params,
pretrained_path=str(model_file))
else:
return cls(data, **model_params, pretrained_path=str(model_file))
def _get_emd_params(self):
import random
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["ModelConfiguration"] = "_deeplab_infrencing"
_emd_template["InferenceFunction"] = "ArcGISImageClassifier.py"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
def accuracy(self):
return self.learn.validate()[-1].tolist()
@property
def _model_metrics(self):
return {'accuracy': '{0:1.4e}'.format(self._get_model_metrics())}
def _get_model_metrics(self, **kwargs):
checkpoint = kwargs.get('checkpoint', True)
if not hasattr(self.learn, 'recorder'):
return 0.0
model_accuracy = self.learn.recorder.metrics[-1][0]
if checkpoint:
model_accuracy = np.max(self.learn.recorder.metrics)
return float(model_accuracy)
def _deeplab_loss(self, outputs, targets):
targets = targets.squeeze(1).detach()
criterion = nn.CrossEntropyLoss().to(self._device)
if self.learn.model.training:
out = outputs[0]
aux = outputs[1]
else: # validation
out = outputs
main_loss = criterion(out, targets)
if self.learn.model.training:
aux_loss = criterion(aux, targets)
total_loss = main_loss + 0.4 * aux_loss
return total_loss
else:
return main_loss
def _freeze(self):
"Freezes the pretrained backbone."
for idx, i in enumerate(flatten_model(self.learn.model)):
if isinstance(i, (nn.BatchNorm2d)):
continue
if hasattr(i, 'dilation'):
dilation = i.dilation
dilation = dilation[0] if isinstance(dilation,
tuple) else dilation
if dilation > 1:
break
for p in i.parameters():
p.requires_grad = False
self.learn.layer_groups = split_model_idx(
self.learn.model, [idx]
) ## Could also call self.learn.freeze after this line because layer groups are now present.
self.learn.create_opt(lr=3e-3)
def unfreeze(self):
for _, param in self.learn.model.named_parameters():
param.requires_grad = True
def show_results(self, rows=5, **kwargs):
"""
Displays the results of a trained model on a part of the validation set.
"""
self._check_requisites()
if rows > len(self._data.valid_ds):
rows = len(self._data.valid_ds)
self.learn.show_results(rows=rows, **kwargs)
def _show_results_multispectral(self,
rows=5,
alpha=0.7,
**kwargs): # parameters adjusted in kwargs
ax = show_results_multispectral(self,
nrows=rows,
alpha=alpha,
**kwargs)
def _accuracy(self, input, target, void_code=0, class_mapping=None):
if self.learn.model.training: # while training
input = input[0]
target = target.squeeze(1)
return (input.argmax(dim=1) == target).float().mean()
def mIOU(self, mean=False, show_progress=True):
"""
Computes mean IOU on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
mean IOU, otherwise returns mean iou of all
classes combined.
--------------------- -------------------------------------------
show_progress Optional bool. Displays the prgress bar if
True.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
num_classes = torch.arange(self._data.c)
miou = compute_miou(self, self._data.valid_dl, mean, num_classes,
show_progress)
if mean:
return np.mean(miou)
return dict(zip(['0'] + self._data.classes[1:], miou))
class MaskRCNN(ArcGISModel):
"""
Creates a ``MaskRCNN`` Instance segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `MaskRCNN`, which
is `resnet50` by default.
Compatible backbones: 'resnet50'
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: ``MaskRCNN`` Object
"""
def __init__(self, data, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
if self._is_multispectral:
self._backbone_ms = self._backbone
self._backbone = self._orig_backbone
scaled_mean_values = data._scaled_mean_values[
data._extract_bands].tolist()
scaled_std_values = data._scaled_std_values[
data._extract_bands].tolist()
if backbone is None:
self._backbone = models.resnet50
elif type(backbone) is str:
self._backbone = getattr(models, backbone)
else:
self._backbone = backbone
if not self._check_backbone_support(self._backbone):
raise Exception(
f"Enter only compatible backbones from {', '.join(self.supported_backbones)}"
)
self._code = instance_detector_prf
if self._backbone.__name__ is 'resnet50':
model = models.detection.maskrcnn_resnet50_fpn(
pretrained=True,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size)
if self._is_multispectral:
model.backbone = _change_tail(model.backbone, data)
model.transform.image_mean = scaled_mean_values
model.transform.image_std = scaled_std_values
elif self._backbone.__name__ in ['resnet18', 'resnet34']:
if self._is_multispectral:
backbone_small = create_body(self._backbone_ms,
cut=_get_backbone_meta(
backbone_fn.__name__)['cut'])
backbone_small.out_channels = 512
model = models.detection.MaskRCNN(
backbone_small,
91,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size,
image_mean=scaled_mean_values,
image_std=scaled_std_values)
else:
backbone_small = create_body(self._backbone)
backbone_small.out_channels = 512
model = models.detection.MaskRCNN(backbone_small,
91,
min_size=1.5 *
data.chip_size,
max_size=2 * data.chip_size)
else:
backbone_fpn = resnet_fpn_backbone(self._backbone.__name__, True)
if self._is_multispectral:
backbone_fpn = _change_tail(backbone_fpn, data)
model = models.detection.MaskRCNN(
backbone_fpn,
91,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size,
image_mean=scaled_mean_values,
image_std=scaled_std_values)
else:
model = models.detection.MaskRCNN(backbone_fpn,
91,
min_size=1.5 *
data.chip_size,
max_size=2 * data.chip_size)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, data.c)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(
in_features_mask, hidden_layer, data.c)
self.learn = Learner(data, model, loss_func=mask_rcnn_loss)
self.learn.callbacks.append(train_callback(self.learn))
self.learn.model = self.learn.model.to(self._device)
self.learn.c_device = self._device
# fixes for zero division error when slice is passed
idx = 27
if self._backbone.__name__ in ['resnet18', 'resnet34']:
idx = self._freeze()
self.learn.layer_groups = split_model_idx(self.learn.model, [idx])
self.learn.create_opt(lr=3e-3)
# make first conv weights learnable
self._arcgis_init_callback()
if pretrained_path is not None:
self.load(pretrained_path)
if self._is_multispectral:
self._orig_backbone = self._backbone
self._backbone = self._backbone_ms
def unfreeze(self):
for _, param in self.learn.model.named_parameters():
param.requires_grad = True
def _freeze(self):
"Freezes the pretrained backbone."
for idx, i in enumerate(flatten_model(self.learn.model.backbone)):
if isinstance(i, (torch.nn.BatchNorm2d)):
continue
for p in i.parameters():
p.requires_grad = False
return idx
def __str__(self):
return self.__repr__()
def __repr__(self):
return '<%s>' % (type(self).__name__)
@property
def supported_backbones(self):
"""
Supported torchvision backbones for this model.
"""
return [*self._resnet_family]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``MaskRCNN`` Instance segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `MaskRCNN` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
data = _EmptyData(path=emd_path.parent.parent,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
data.class_mapping = class_mapping
data.color_mapping = color_mapping
data.emd_path = emd_path
data.emd = emd
data = get_multispectral_data_params_from_emd(data, emd)
return cls(data, **model_params, pretrained_path=str(model_file))
def _get_emd_params(self):
import random
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["ModelConfiguration"] = "_maskrcnn_inferencing"
_emd_template["InferenceFunction"] = "ArcGISInstanceDetector.py"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
@property
def _model_metrics(self):
return {
'average_precision_score':
self.average_precision_score(show_progress=False)
}
def _predict_results(self, xb):
self.learn.model.eval()
xb_l = xb.to(self._device)
predictions = self.learn.model(list(xb_l))
xb_l = xb_l.detach().cpu()
del xb_l
predictionsf = []
for i in range(len(predictions)):
predictionsf.append({})
predictionsf[i]['masks'] = predictions[i]['masks'].detach().cpu(
).numpy()
predictionsf[i]['boxes'] = predictions[i]['boxes'].detach().cpu(
).numpy()
predictionsf[i]['labels'] = predictions[i]['labels'].detach().cpu(
).numpy()
predictionsf[i]['scores'] = predictions[i]['scores'].detach().cpu(
).numpy()
del predictions[i]['masks']
del predictions[i]['boxes']
del predictions[i]['labels']
del predictions[i]['scores']
if self._device == torch.device('cuda'):
torch.cuda.empty_cache()
return predictionsf
def _predict_postprocess(self,
predictions,
threshold=0.5,
box_threshold=0.5):
pred_mask = []
pred_box = []
for i in range(len(predictions)):
out = predictions[i]['masks'].squeeze()
pred_box.append([])
if out.shape[0] != 0: # handle for prediction with n masks
if len(
out.shape
) == 2: # for out dimension hxw (in case of only one predicted mask)
out = out[None]
ymask = np.where(out[0] > threshold, 1, 0)
if predictions[i]['scores'][0] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][0])
for j in range(1, out.shape[0]):
ym1 = np.where(out[j] > threshold, j + 1, 0)
ymask += ym1
if predictions[i]['scores'][j] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][j])
else:
ymask = np.zeros(
(self._data.chip_size,
self._data.chip_size)) # handle for not predicted masks
pred_mask.append(ymask)
return pred_mask, pred_box
def show_results(self,
rows=4,
mode='mask',
mask_threshold=0.5,
box_threshold=0.7,
imsize=5,
index=0,
alpha=0.5,
cmap='tab20',
**kwargs):
"""
Displays the results of a trained model on a part of the validation set.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mode Required arguments within ['bbox', 'mask', 'bbox_mask'].
* ``bbox`` - For visualizing only boundig boxes.
* ``mask`` - For visualizing only mask
* ``bbox_mask`` - For visualizing both mask and bounding boxes.
--------------------- -------------------------------------------
mask_threshold Optional float. The probabilty above which
a pixel will be considered mask.
--------------------- -------------------------------------------
box_threshold Optional float. The pobabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nrows Optional int. Number of rows of results
to be displayed.
===================== ===========================================
"""
self._check_requisites()
if mode not in ['bbox', 'mask', 'bbox_mask']:
raise Exception("mode can be only ['bbox', 'mask', 'bbox_mask']")
# Get Number of items
nrows = rows
ncols = 2
type_data_loader = kwargs.get(
'data_loader',
'validation') # options : traininig, validation, testing
if type_data_loader == 'training':
data_loader = self._data.train_dl
elif type_data_loader == 'validation':
data_loader = self._data.valid_dl
elif type_data_loader == 'testing':
data_loader = self._data.test_dl
else:
e = Exception(f'could not find {type_data_loader} in data.')
raise (e)
statistics_type = kwargs.get(
'statistics_type', 'dataset') # Accepted Values `dataset`, `DRA`
cmap_fn = getattr(matplotlib.cm, cmap)
title_font_size = 16
if kwargs.get('top', None) is not None:
top = kwargs.get('top')
else:
top = 1 - (math.sqrt(title_font_size) /
math.sqrt(100 * nrows * imsize))
x_batch, y_batch = [], []
i = 0
dl_iterater = iter(data_loader)
while i < nrows:
x, y = next(dl_iterater)
x_batch.append(x)
y_batch.append(y)
i += self._data.batch_size
x_batch = torch.cat(x_batch)
y_batch = torch.cat(y_batch)
# Get Predictions
prediction_store = []
for i in range(0, x_batch.shape[0], self._data.batch_size):
prediction_store.extend(
self._predict_results(x_batch[i:i + self._data.batch_size]))
pred_mask, pred_box = self._predict_postprocess(
prediction_store, mask_threshold, box_threshold)
if self._is_multispectral:
rgb_bands = kwargs.get('rgb_bands',
self._data._symbology_rgb_bands)
e = Exception(
'`rgb_bands` should be a valid band_order, list or tuple of length 3 or 1.'
)
symbology_bands = []
if not (len(rgb_bands) == 3 or len(rgb_bands) == 1):
raise (e)
for b in rgb_bands:
if type(b) == str:
b_index = self._bands.index(b)
elif type(b) == int:
self._bands[
b] # To check if the band index specified by the user really exists.
b_index = b
else:
raise (e)
b_index = self._data._extract_bands.index(b_index)
symbology_bands.append(b_index)
# Denormalize X
if self._data._do_normalize:
x_batch = (self._data._scaled_std_values[
self._data._extract_bands].view(1, -1, 1, 1).to(x_batch) *
x_batch) + self._data._scaled_mean_values[
self._data._extract_bands].view(1, -1, 1,
1).to(x_batch)
# Extract RGB Bands
symbology_x_batch = x_batch[:, symbology_bands]
if statistics_type == 'DRA':
shp = symbology_x_batch.shape
min_vals = symbology_x_batch.view(shp[0], shp[1],
-1).min(dim=2)[0]
max_vals = symbology_x_batch.view(shp[0], shp[1],
-1).max(dim=2)[0]
symbology_x_batch = symbology_x_batch / (
max_vals.view(shp[0], shp[1], 1, 1) -
min_vals.view(shp[0], shp[1], 1, 1) + .001)
# Channel first to channel last for plotting
symbology_x_batch = symbology_x_batch.permute(0, 2, 3, 1)
# Clamp float values to range 0 - 1
if symbology_x_batch.mean() < 1:
symbology_x_batch = symbology_x_batch.clamp(0, 1)
else:
symbology_x_batch = x_batch.permute(0, 2, 3, 1)
# Squeeze channels if single channel (1, 224, 224) -> (224, 224)
if symbology_x_batch.shape[-1] == 1:
symbology_x_batch = symbology_x_batch.squeeze()
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(ncols * imsize, nrows * imsize))
fig.suptitle('Ground Truth / Predictions', fontsize=title_font_size)
for i in range(nrows):
if nrows == 1:
ax_i = ax
else:
ax_i = ax[i]
# Ground Truth
ax_i[0].imshow(symbology_x_batch[i])
ax_i[0].axis('off')
if mode in ['mask', 'bbox_mask']:
n_instance = y_batch[i].unique().shape[0]
y_merged = y_batch[i].max(dim=0)[0].cpu().numpy()
y_rgba = cmap_fn._resample(n_instance)(y_merged)
y_rgba[y_merged == 0] = 0
y_rgba[:, :, -1] = alpha
ax_i[0].imshow(y_rgba)
ax_i[0].axis('off')
# Predictions
ax_i[1].imshow(symbology_x_batch[i])
ax_i[1].axis('off')
if mode in ['mask', 'bbox_mask']:
n_instance = np.unique(pred_mask[i]).shape[0]
p_rgba = cmap_fn._resample(n_instance)(pred_mask[i])
p_rgba[pred_mask[i] == 0] = 0
p_rgba[:, :, -1] = alpha
ax_i[1].imshow(p_rgba)
if mode in ['bbox_mask', 'bbox']:
if pred_box[i] != []:
for num_boxes in pred_box[i]:
rect = patches.Rectangle((num_boxes[0], num_boxes[1]),
num_boxes[2] - num_boxes[0],
num_boxes[3] - num_boxes[1],
linewidth=1,
edgecolor='r',
facecolor='none')
ax_i[1].add_patch(rect)
ax_i[1].axis('off')
plt.subplots_adjust(top=top)
if self._device == torch.device('cuda'):
torch.cuda.empty_cache()
def average_precision_score(self,
detect_thresh=0.5,
iou_thresh=0.5,
mean=False,
show_progress=True):
"""
Computes average precision on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
detect_thresh Optional float. The probabilty above which
a detection will be considered for computing
average precision.
--------------------- -------------------------------------------
iou_thresh Optional float. The intersection over union
threshold with the ground truth mask, above
which a predicted mask will be
considered a true positive.
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
average precision otherwise returns mean
average precision.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
self._check_requisites()
if mean:
aps = compute_class_AP(self, self._data.valid_dl, 1, show_progress,
detect_thresh, iou_thresh, mean)
return aps
else:
aps = compute_class_AP(self, self._data.valid_dl, self._data.c - 1,
show_progress, detect_thresh, iou_thresh)
return dict(zip(self._data.classes[1:], aps))
class DeepLab(ArcGISModel):
"""
Creates a ``DeepLab`` Semantic segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `DeepLab`, which
is `resnet101` by default since it is pretrained in
torchvision. It supports the ResNet,
DenseNet, and VGG families.
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
**kwargs**
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
class_balancing Optional boolean. If True, it will balance the
cross-entropy loss inverse to the frequency
of pixels per class. Default: False.
--------------------- -------------------------------------------
mixup Optional boolean. If True, it will use mixup
augmentation and mixup loss. Default: False
--------------------- -------------------------------------------
focal_loss Optional boolean. If True, it will use focal loss.
Default: False
--------------------- -------------------------------------------
ignore_classes Optional list. It will contain the list of class
values on which model will not incur loss.
Default: []
===================== ===========================================
:returns: ``DeepLab`` Object
"""
def __init__(self,
data,
backbone=None,
pretrained_path=None,
*args,
**kwargs):
# Set default backbone to be 'resnet101'
if backbone is None:
backbone = models.resnet101
super().__init__(data, backbone)
self._ignore_classes = kwargs.get('ignore_classes', [])
if self._ignore_classes != [] and len(data.classes) <= 3:
raise Exception(
f"`ignore_classes` parameter can only be used when the dataset has more than 2 classes."
)
data_classes = list(self._data.class_mapping.keys())
if 0 not in list(data.class_mapping.values()):
self._ignore_mapped_class = [
data_classes.index(k) + 1 for k in self._ignore_classes
if k != 0
]
else:
self._ignore_mapped_class = [
data_classes.index(k) + 1 for k in self._ignore_classes
]
if self._ignore_classes != []:
if 0 not in self._ignore_mapped_class:
self._ignore_mapped_class.insert(0, 0)
global accuracy
accuracy = partial(accuracy,
ignore_mapped_class=self._ignore_mapped_class)
self.mixup = kwargs.get('mixup', False)
self.class_balancing = kwargs.get('class_balancing', False)
self.focal_loss = kwargs.get('focal_loss', False)
_backbone = self._backbone
if hasattr(self, '_orig_backbone'):
_backbone = self._orig_backbone
if not self._check_backbone_support(_backbone):
raise Exception(
f"Enter only compatible backbones from {', '.join(self.supported_backbones)}"
)
self._code = image_classifier_prf
if self._backbone.__name__ is 'resnet101':
model = _create_deeplab(data.c)
if self._is_multispectral:
model = _change_tail(model, data)
else:
model = Deeplab(data.c, self._backbone, data.chip_size)
if not _isnotebook() and os.name == 'posix':
_set_ddp_multigpu(self)
if self._multigpu_training:
self.learn = Learner(data, model,
metrics=accuracy).to_distributed(
self._rank_distributed)
else:
self.learn = Learner(data, model, metrics=accuracy)
else:
self.learn = Learner(data, model, metrics=accuracy)
self.learn.loss_func = self._deeplab_loss
## setting class_weight if present in data
if self.class_balancing and self._data.class_weight is not None:
class_weight = torch.tensor(
[self._data.class_weight.mean()] +
self._data.class_weight.tolist()).float().to(self._device)
else:
class_weight = None
## Raising warning in apropriate case
if self.class_balancing:
if self._data.class_weight is None:
logger.warning(
"Could not find 'NumPixelsPerClass' in 'esri_accumulated_stats.json'. Ignoring `class_balancing` parameter."
)
elif getattr(data, 'overflow_encountered', False):
logger.warning(
"Overflow Encountered. Ignoring `class_balancing` parameter."
)
class_weight = [1] * len(data.classes)
## Setting class weights for ignored classes
if self._ignore_classes != []:
if not self.class_balancing:
class_weight = torch.tensor([1] * data.c).float().to(
self._device)
class_weight[self._ignore_mapped_class] = 0.
else:
class_weight = None
self._final_class_weight = class_weight
if self.focal_loss:
self.learn.loss_func = FocalLoss(self.learn.loss_func)
if self.mixup:
self.learn.callbacks.append(MixUpCallback(self.learn))
self.learn.model = self.learn.model.to(self._device)
self._freeze()
self._arcgis_init_callback() # make first conv weights learnable
if pretrained_path is not None:
self.load(pretrained_path)
@property
def supported_backbones(self):
return DeepLab._supported_backbones()
@staticmethod
def _supported_backbones():
return [*_resnet_family, *_densenet_family, *_vgg_family]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``DeepLab`` semantic segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `DeepLab` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
empty_data = _EmptyData(path=emd_path.parent.parent,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
empty_data.class_mapping = class_mapping
empty_data.color_mapping = color_mapping
empty_data = get_multispectral_data_params_from_emd(
empty_data, emd)
empty_data.emd_path = emd_path
empty_data.emd = emd
return cls(empty_data,
**model_params,
pretrained_path=str(model_file))
else:
return cls(data, **model_params, pretrained_path=str(model_file))
def _get_emd_params(self):
import random
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["ModelConfiguration"] = "_deeplab_infrencing"
_emd_template["InferenceFunction"] = "ArcGISImageClassifier.py"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template["ignore_mapped_class"] = self._ignore_mapped_class
_emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
def accuracy(self):
return self.learn.validate()[-1].tolist()
@property
def _model_metrics(self):
return {'accuracy': '{0:1.4e}'.format(self._get_model_metrics())}
def _get_model_metrics(self, **kwargs):
checkpoint = kwargs.get('checkpoint', True)
if not hasattr(self.learn, 'recorder'):
return 0.0
model_accuracy = self.learn.recorder.metrics[-1][0]
if checkpoint:
model_accuracy = np.max(self.learn.recorder.metrics)
return float(model_accuracy)
def _deeplab_loss(self, outputs, targets, **kwargs):
targets = targets.squeeze(1).detach()
criterion = nn.CrossEntropyLoss(weight=self._final_class_weight).to(
self._device)
if self.learn.model.training:
out = outputs[0]
aux = outputs[1]
else: # validation
out = outputs
main_loss = criterion(out, targets)
if self.learn.model.training:
aux_loss = criterion(aux, targets)
total_loss = main_loss + 0.4 * aux_loss
return total_loss
else:
return main_loss
def _freeze(self):
"Freezes the pretrained backbone."
for idx, i in enumerate(flatten_model(self.learn.model)):
if isinstance(i, (nn.BatchNorm2d)):
continue
if hasattr(i, 'dilation'):
dilation = i.dilation
dilation = dilation[0] if isinstance(dilation,
tuple) else dilation
if dilation > 1:
break
for p in i.parameters():
p.requires_grad = False
self.learn.layer_groups = split_model_idx(
self.learn.model, [idx]
) ## Could also call self.learn.freeze after this line because layer groups are now present.
self.learn.create_opt(lr=3e-3)
def unfreeze(self):
for _, param in self.learn.model.named_parameters():
param.requires_grad = True
def show_results(self, rows=5, **kwargs):
"""
Displays the results of a trained model on a part of the validation set.
"""
self._check_requisites()
if rows > len(self._data.valid_ds):
rows = len(self._data.valid_ds)
self.learn.show_results(rows=rows,
ignore_mapped_class=self._ignore_mapped_class,
**kwargs)
def _show_results_multispectral(self,
rows=5,
alpha=0.7,
**kwargs): # parameters adjusted in kwargs
ax = show_results_multispectral(self,
nrows=rows,
alpha=alpha,
**kwargs)
def mIOU(self, mean=False, show_progress=True):
"""
Computes mean IOU on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
mean IOU, otherwise returns mean iou of all
classes combined.
--------------------- -------------------------------------------
show_progress Optional bool. Displays the prgress bar if
True.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
num_classes = torch.arange(self._data.c)
miou = compute_miou(self, self._data.valid_dl, mean, num_classes,
show_progress, self._ignore_mapped_class)
if mean:
return np.mean(miou)
if self._ignore_mapped_class == []:
return dict(zip(['0'] + self._data.classes[1:], miou))
else:
class_values = [0] + list(self._data.class_mapping.keys())
return {
class_values[i]: miou[i]
for i in range(len(miou)) if i not in self._ignore_mapped_class
}
def per_class_metrics(self):
"""
Computer per class precision, recall and f1-score on validation set.
"""
## Calling imported function `per_class_metrics`
return per_class_metrics(self,
ignore_mapped_class=self._ignore_mapped_class)
