def get_learn(data):
# create model
model = nb_resnet_unet.get_unet_res18(1, True)
model.load_state_dict(torch.load('./models/unet_res18_allres_init.pth'))
# create learner
learn = Learner(data, model)
# split model
learn.layer_groups = split_model(learn.model)
# set multi-gpu
if data.device.type == 'cuda':
learn.model = torch.nn.DataParallel(learn.model,
device_ids=[0, 1, 2, 3])
# set loss func
# learn.loss_func = partial(nb_loss_metrics.combo_loss, balance_ratio=1)
# learn.loss_func = nb_loss_metrics.dice_loss
learn.loss_func = partial(nb_loss_metrics.balance_bce, balance_ratio=1)
# 添加metrics
learn.metrics += [nb_loss_metrics.dice_loss]
learn.metrics += [partial(nb_loss_metrics.balance_bce, balance_ratio=1)]
learn.metrics += [nb_loss_metrics.mask_iou]
return learn
def get_learn_detectsym_17clas(data, gaf, clas_weights=weights):
'''
用的符号检测的17个类别的数据集
'''
# create model
model = resnet_ssd.get_resnet18_1ssd(num_classes=17)
model.load_state_dict(torch.load('./models/pretrained_res18_1ssd.pth'))
# create learner
learn = Learner(data, model)
# split model
learn.layer_groups = split_model(learn.model)
# set multi-gpu
if data.device.type == 'cuda':
learn.model = torch.nn.DataParallel(
learn.model, device_ids=device_ids) #device_ids=[0,1,2,3,4,5])
# set loss func
learn.loss_func = partial(anchors_loss_metrics.yolo_L,
gaf=gaf,
conf_th=1,
clas_weights=clas_weights,
lambda_nconf=10)
# 添加metrics
learn.metrics += [
partial(anchors_loss_metrics.clas_L,
gaf=gaf,
clas_weights=clas_weights)
]
learn.metrics += [
partial(anchors_loss_metrics.cent_L,
gaf=gaf,
clas_weights=clas_weights)
]
learn.metrics += [
partial(anchors_loss_metrics.pConf_L,
gaf=gaf,
clas_weights=clas_weights)
]
learn.metrics += [
partial(anchors_loss_metrics.nConf_L, gaf=gaf, conf_th=1)
]
learn.metrics += [partial(anchors_loss_metrics.clas_acc, gaf=gaf)]
learn.metrics += [partial(anchors_loss_metrics.cent_d, gaf=gaf)]
return learn
class Sequential:
def __init__(self, model=None):
self.layers = []
self.last_dim = None
self.model = model
self.device = torch.device('cpu')
if torch.cuda.is_available():
self.device = torch.device('cuda')
def add(self, layer):
layer = layer.get_layer(self.last_dim)
self.last_dim = layer['output_dim']
self.layers.extend(layer['layers'])
def compile(self, loss, optimizer=None):
if len(self.layers) > 0:
self.model = nn.Sequential(*self.layers)
self.loss = loss
def fit(self, x, y, bs, epochs, lr=1e-3, one_cycle=True, get_lr=True):
db = create_db(x, y, bs=bs)
self.learn = Learner(db, self.model, loss_func=self.loss)
if one_cycle:
self.learn.fit_one_cycle(epochs, lr)
else:
self.learn.fit(epochs, lr)
def lr_find(self, x, y, bs):
db = create_db(x, y, bs=bs)
learn = Learner(db, self.model, loss_func=self.loss)
learn.lr_find()
clear_output()
learn.recorder.plot(suggestion=True)
def predict(self, x):
self.learn.model.eval()
with torch.no_grad():
y_preds = self.learn.model(torch.Tensor(x).to(device))
return y_preds.cpu().numpy()
num_workers=NUM_WORKERS,
normalization=STATISTICS,
)
# init model
swa_model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
model = MODEL(num_classes=N_CLASSES, dropout_p=DROPOUT)
# nullify all swa model parameters
swa_params = swa_model.parameters()
for swa_param in swa_params:
swa_param.data = torch.zeros_like(swa_param.data)
# average model
n_swa = len(os.listdir(MODELS_FOLDER))
print(f"Averaging {n_swa} models")
for file in os.listdir(MODELS_FOLDER):
model.load_state_dict(torch.load(f'{MODELS_FOLDER}/{file}')['model'])
model_params = model.parameters()
for model_param, swa_param in zip(model_params, swa_params):
swa_param.data += model_param.data / n_swa
# fix batch norm
print("Fixing batch norm")
swa_model.to(DEVICE)
learn = Learner(data, model, model_dir=MODELS_FOLDER, loss_func=CRITERION, opt_func=OPTIMIZER, wd=WD)
learn.model = convert_model(learn.model)
learn.model = nn.DataParallel(learn.model).to(DEVICE)
fix_batchnorm(learn.model, learn.data.train_dl)
learn.save('swa_model')
class MaskRCNN(ArcGISModel):
"""
Creates a ``MaskRCNN`` Instance segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `MaskRCNN`, which
is `resnet50` by default.
Compatible backbones: 'resnet50'
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: ``MaskRCNN`` Object
"""
def __init__(self, data, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
self._backbone = models.resnet50
#if not self._check_backbone_support(self._backbone):
# raise Exception (f"Enter only compatible backbones from {', '.join(self.supported_backbones)}")
self._code = instance_detector_prf
model = models.detection.maskrcnn_resnet50_fpn(pretrained=True,
min_size=data.chip_size)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, data.c)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(
in_features_mask, hidden_layer, data.c)
self.learn = Learner(data, model, loss_func=mask_rcnn_loss)
self.learn.callbacks.append(train_callback(self.learn))
self.learn.model = self.learn.model.to(self._device)
# fixes for zero division error when slice is passed
self.learn.layer_groups = split_model_idx(self.learn.model, [28])
self.learn.create_opt(lr=3e-3)
if pretrained_path is not None:
self.load(pretrained_path)
def __str__(self):
return self.__repr__()
def __repr__(self):
return '<%s>' % (type(self).__name__)
@property
def supported_backbones(self):
return [models.detection.maskrcnn_resnet50_fpn.__name__]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``MaskRCNN`` Instance segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `MaskRCNN` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
empty_data = _EmptyData(path=tempfile.TemporaryDirectory().name,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
empty_data.class_mapping = class_mapping
empty_data.color_mapping = color_mapping
return cls(empty_data,
**model_params,
pretrained_path=str(model_file))
else:
return cls(data, **model_params, pretrained_path=str(model_file))
def _create_emd(self, path):
import random
super()._create_emd(path)
self._emd_template["Framework"] = "arcgis.learn.models._inferencing"
self._emd_template["ModelConfiguration"] = "_maskrcnn_inferencing"
self._emd_template["InferenceFunction"] = "ArcGISInstanceDetector.py"
self._emd_template["ExtractBands"] = [0, 1, 2]
self._emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
self._emd_template['Classes'].append(class_data.copy())
json.dump(self._emd_template,
open(path.with_suffix('.emd'), 'w'),
indent=4)
return path.stem
@property
def _model_metrics(self):
return {}
def _predict_results(self, xb):
self.learn.model.eval()
predictions = self.learn.model(xb.cuda())
predictionsf = []
for i in range(len(predictions)):
predictionsf.append({})
predictionsf[i]['masks'] = predictions[i]['masks'].detach().cpu(
).numpy()
predictionsf[i]['boxes'] = predictions[i]['boxes'].detach().cpu(
).numpy()
predictionsf[i]['labels'] = predictions[i]['labels'].detach().cpu(
).numpy()
predictionsf[i]['scores'] = predictions[i]['scores'].detach().cpu(
).numpy()
del predictions[i]['masks']
del predictions[i]['boxes']
del predictions[i]['labels']
del predictions[i]['scores']
del xb
torch.cuda.empty_cache()
return predictionsf
def _predict_postprocess(self,
predictions,
threshold=0.5,
box_threshold=0.5):
pred_mask = []
pred_box = []
for i in range(len(predictions)):
out = predictions[i]['masks'].squeeze()
pred_box.append([])
if out.shape[0] != 0: # handle for prediction with n masks
if len(
out.shape
) == 2: # for out dimension hxw (in case of only one predicted mask)
out = out[None]
ymask = np.where(out[0] > threshold, 1, 0)
#if torch.max(out[0]) > threshold:
if predictions[i]['scores'][0] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][0])
for j in range(1, out.shape[0]):
ym1 = np.where(out[j] > threshold, j + 1, 0)
ymask += ym1
#if torch.max(out[j]) > threshold:
if predictions[i]['scores'][j] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][j])
else:
ymask = np.zeros(
(self._data.chip_size,
self._data.chip_size)) # handle for not predicted masks
pred_mask.append(ymask)
return pred_mask, pred_box
def show_results(self,
mode='mask',
mask_threshold=0.5,
box_threshold=0.7,
nrows=None,
imsize=5,
index=0,
alpha=0.5,
cmap='tab20'):
"""
Displays the results of a trained model on a part of the validation set.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mode Required arguments within ['bbox', 'mask', 'bbox_mask'].
* ``bbox`` - For visualizing only boundig boxes.
* ``mask`` - For visualizing only mask
* ``bbox_mask`` - For visualizing both mask and bounding boxes.
--------------------- -------------------------------------------
mask_threshold Optional float. The probabilty above which
a pixel will be considered mask.
--------------------- -------------------------------------------
box_threshold Optional float. The pobabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nrows Optional int. Number of rows of results
to be displayed.
===================== ===========================================
"""
if mode not in ['bbox', 'mask', 'bbox_mask']:
raise Exception("mode can be only ['bbox', 'mask', 'bbox_mask']")
# Get Number of items
if nrows is None:
nrows = self._data.batch_size
ncols = 2
# Get Batch
xb, yb = self._data.one_batch('DatasetType.Valid')
predictions = self._predict_results(xb)
pred_mask, pred_box = self._predict_postprocess(
predictions, mask_threshold, box_threshold)
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(ncols * imsize, nrows * imsize))
fig.suptitle('Ground Truth / Predictions', fontsize=20)
for i in range(nrows):
ax[i][0].imshow(xb[i].numpy().transpose(1, 2, 0))
ax[i][0].axis('off')
if mode in ['mask', 'bbox_mask']:
yb_mask = yb[i][0].numpy()
for j in range(1, yb[i].shape[0]):
max_unique = np.max(np.unique(yb_mask))
yb_j = np.where(yb[i][j] > 0, yb[i][j] + max_unique,
yb[i][j])
yb_mask += yb_j
ax[i][0].imshow(yb_mask, cmap=cmap, alpha=alpha)
ax[i][0].axis('off')
ax[i][1].imshow(xb[i].numpy().transpose(1, 2, 0))
ax[i][1].axis('off')
if mode in ['mask', 'bbox_mask']:
ax[i][1].imshow(pred_mask[i], cmap=cmap, alpha=alpha)
if mode in ['bbox', 'bbox']:
if pred_box[i] != []:
for num_boxes in pred_box[i]:
rect = patches.Rectangle((num_boxes[0], num_boxes[1]),
num_boxes[2] - num_boxes[0],
num_boxes[3] - num_boxes[1],
linewidth=1,
edgecolor='r',
facecolor='none')
ax[i][1].add_patch(rect)
ax[i][1].axis('off')
plt.subplots_adjust(top=0.95)
torch.cuda.empty_cache()
class MaskRCNN(ArcGISModel):
"""
Creates a ``MaskRCNN`` Instance segmentation object
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- -------------------------------------------
backbone Optional function. Backbone CNN model to be used for
creating the base of the `MaskRCNN`, which
is `resnet50` by default.
Compatible backbones: 'resnet50'
--------------------- -------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ===========================================
:returns: ``MaskRCNN`` Object
"""
def __init__(self, data, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
if self._is_multispectral:
self._backbone_ms = self._backbone
self._backbone = self._orig_backbone
scaled_mean_values = data._scaled_mean_values[
data._extract_bands].tolist()
scaled_std_values = data._scaled_std_values[
data._extract_bands].tolist()
if backbone is None:
self._backbone = models.resnet50
elif type(backbone) is str:
self._backbone = getattr(models, backbone)
else:
self._backbone = backbone
if not self._check_backbone_support(self._backbone):
raise Exception(
f"Enter only compatible backbones from {', '.join(self.supported_backbones)}"
)
self._code = instance_detector_prf
if self._backbone.__name__ is 'resnet50':
model = models.detection.maskrcnn_resnet50_fpn(
pretrained=True,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size)
if self._is_multispectral:
model.backbone = _change_tail(model.backbone, data)
model.transform.image_mean = scaled_mean_values
model.transform.image_std = scaled_std_values
elif self._backbone.__name__ in ['resnet18', 'resnet34']:
if self._is_multispectral:
backbone_small = create_body(self._backbone_ms,
cut=_get_backbone_meta(
backbone_fn.__name__)['cut'])
backbone_small.out_channels = 512
model = models.detection.MaskRCNN(
backbone_small,
91,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size,
image_mean=scaled_mean_values,
image_std=scaled_std_values)
else:
backbone_small = create_body(self._backbone)
backbone_small.out_channels = 512
model = models.detection.MaskRCNN(backbone_small,
91,
min_size=1.5 *
data.chip_size,
max_size=2 * data.chip_size)
else:
backbone_fpn = resnet_fpn_backbone(self._backbone.__name__, True)
if self._is_multispectral:
backbone_fpn = _change_tail(backbone_fpn, data)
model = models.detection.MaskRCNN(
backbone_fpn,
91,
min_size=1.5 * data.chip_size,
max_size=2 * data.chip_size,
image_mean=scaled_mean_values,
image_std=scaled_std_values)
else:
model = models.detection.MaskRCNN(backbone_fpn,
91,
min_size=1.5 *
data.chip_size,
max_size=2 * data.chip_size)
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, data.c)
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
model.roi_heads.mask_predictor = MaskRCNNPredictor(
in_features_mask, hidden_layer, data.c)
self.learn = Learner(data, model, loss_func=mask_rcnn_loss)
self.learn.callbacks.append(train_callback(self.learn))
self.learn.model = self.learn.model.to(self._device)
self.learn.c_device = self._device
# fixes for zero division error when slice is passed
idx = 27
if self._backbone.__name__ in ['resnet18', 'resnet34']:
idx = self._freeze()
self.learn.layer_groups = split_model_idx(self.learn.model, [idx])
self.learn.create_opt(lr=3e-3)
# make first conv weights learnable
self._arcgis_init_callback()
if pretrained_path is not None:
self.load(pretrained_path)
if self._is_multispectral:
self._orig_backbone = self._backbone
self._backbone = self._backbone_ms
def unfreeze(self):
for _, param in self.learn.model.named_parameters():
param.requires_grad = True
def _freeze(self):
"Freezes the pretrained backbone."
for idx, i in enumerate(flatten_model(self.learn.model.backbone)):
if isinstance(i, (torch.nn.BatchNorm2d)):
continue
for p in i.parameters():
p.requires_grad = False
return idx
def __str__(self):
return self.__repr__()
def __repr__(self):
return '<%s>' % (type(self).__name__)
@property
def supported_backbones(self):
"""
Supported torchvision backbones for this model.
"""
return [*self._resnet_family]
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``MaskRCNN`` Instance segmentation object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `MaskRCNN` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
model_params = emd['ModelParameters']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
data = _EmptyData(path=emd_path.parent.parent,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
data.class_mapping = class_mapping
data.color_mapping = color_mapping
data.emd_path = emd_path
data.emd = emd
data = get_multispectral_data_params_from_emd(data, emd)
return cls(data, **model_params, pretrained_path=str(model_file))
def _get_emd_params(self):
import random
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["ModelConfiguration"] = "_maskrcnn_inferencing"
_emd_template["InferenceFunction"] = "ArcGISInstanceDetector.py"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template['Classes'] = []
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)] if is_no_color(self._data.color_mapping) else \
self._data.color_mapping[inverse_class_mapping[class_name]]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
@property
def _model_metrics(self):
return {
'average_precision_score':
self.average_precision_score(show_progress=False)
}
def _predict_results(self, xb):
self.learn.model.eval()
xb_l = xb.to(self._device)
predictions = self.learn.model(list(xb_l))
xb_l = xb_l.detach().cpu()
del xb_l
predictionsf = []
for i in range(len(predictions)):
predictionsf.append({})
predictionsf[i]['masks'] = predictions[i]['masks'].detach().cpu(
).numpy()
predictionsf[i]['boxes'] = predictions[i]['boxes'].detach().cpu(
).numpy()
predictionsf[i]['labels'] = predictions[i]['labels'].detach().cpu(
).numpy()
predictionsf[i]['scores'] = predictions[i]['scores'].detach().cpu(
).numpy()
del predictions[i]['masks']
del predictions[i]['boxes']
del predictions[i]['labels']
del predictions[i]['scores']
if self._device == torch.device('cuda'):
torch.cuda.empty_cache()
return predictionsf
def _predict_postprocess(self,
predictions,
threshold=0.5,
box_threshold=0.5):
pred_mask = []
pred_box = []
for i in range(len(predictions)):
out = predictions[i]['masks'].squeeze()
pred_box.append([])
if out.shape[0] != 0: # handle for prediction with n masks
if len(
out.shape
) == 2: # for out dimension hxw (in case of only one predicted mask)
out = out[None]
ymask = np.where(out[0] > threshold, 1, 0)
if predictions[i]['scores'][0] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][0])
for j in range(1, out.shape[0]):
ym1 = np.where(out[j] > threshold, j + 1, 0)
ymask += ym1
if predictions[i]['scores'][j] > box_threshold:
pred_box[i].append(predictions[i]['boxes'][j])
else:
ymask = np.zeros(
(self._data.chip_size,
self._data.chip_size)) # handle for not predicted masks
pred_mask.append(ymask)
return pred_mask, pred_box
def show_results(self,
rows=4,
mode='mask',
mask_threshold=0.5,
box_threshold=0.7,
imsize=5,
index=0,
alpha=0.5,
cmap='tab20',
**kwargs):
"""
Displays the results of a trained model on a part of the validation set.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
mode Required arguments within ['bbox', 'mask', 'bbox_mask'].
* ``bbox`` - For visualizing only boundig boxes.
* ``mask`` - For visualizing only mask
* ``bbox_mask`` - For visualizing both mask and bounding boxes.
--------------------- -------------------------------------------
mask_threshold Optional float. The probabilty above which
a pixel will be considered mask.
--------------------- -------------------------------------------
box_threshold Optional float. The pobabilty above which
a detection will be considered valid.
--------------------- -------------------------------------------
nrows Optional int. Number of rows of results
to be displayed.
===================== ===========================================
"""
self._check_requisites()
if mode not in ['bbox', 'mask', 'bbox_mask']:
raise Exception("mode can be only ['bbox', 'mask', 'bbox_mask']")
# Get Number of items
nrows = rows
ncols = 2
type_data_loader = kwargs.get(
'data_loader',
'validation') # options : traininig, validation, testing
if type_data_loader == 'training':
data_loader = self._data.train_dl
elif type_data_loader == 'validation':
data_loader = self._data.valid_dl
elif type_data_loader == 'testing':
data_loader = self._data.test_dl
else:
e = Exception(f'could not find {type_data_loader} in data.')
raise (e)
statistics_type = kwargs.get(
'statistics_type', 'dataset') # Accepted Values `dataset`, `DRA`
cmap_fn = getattr(matplotlib.cm, cmap)
title_font_size = 16
if kwargs.get('top', None) is not None:
top = kwargs.get('top')
else:
top = 1 - (math.sqrt(title_font_size) /
math.sqrt(100 * nrows * imsize))
x_batch, y_batch = [], []
i = 0
dl_iterater = iter(data_loader)
while i < nrows:
x, y = next(dl_iterater)
x_batch.append(x)
y_batch.append(y)
i += self._data.batch_size
x_batch = torch.cat(x_batch)
y_batch = torch.cat(y_batch)
# Get Predictions
prediction_store = []
for i in range(0, x_batch.shape[0], self._data.batch_size):
prediction_store.extend(
self._predict_results(x_batch[i:i + self._data.batch_size]))
pred_mask, pred_box = self._predict_postprocess(
prediction_store, mask_threshold, box_threshold)
if self._is_multispectral:
rgb_bands = kwargs.get('rgb_bands',
self._data._symbology_rgb_bands)
e = Exception(
'`rgb_bands` should be a valid band_order, list or tuple of length 3 or 1.'
)
symbology_bands = []
if not (len(rgb_bands) == 3 or len(rgb_bands) == 1):
raise (e)
for b in rgb_bands:
if type(b) == str:
b_index = self._bands.index(b)
elif type(b) == int:
self._bands[
b] # To check if the band index specified by the user really exists.
b_index = b
else:
raise (e)
b_index = self._data._extract_bands.index(b_index)
symbology_bands.append(b_index)
# Denormalize X
if self._data._do_normalize:
x_batch = (self._data._scaled_std_values[
self._data._extract_bands].view(1, -1, 1, 1).to(x_batch) *
x_batch) + self._data._scaled_mean_values[
self._data._extract_bands].view(1, -1, 1,
1).to(x_batch)
# Extract RGB Bands
symbology_x_batch = x_batch[:, symbology_bands]
if statistics_type == 'DRA':
shp = symbology_x_batch.shape
min_vals = symbology_x_batch.view(shp[0], shp[1],
-1).min(dim=2)[0]
max_vals = symbology_x_batch.view(shp[0], shp[1],
-1).max(dim=2)[0]
symbology_x_batch = symbology_x_batch / (
max_vals.view(shp[0], shp[1], 1, 1) -
min_vals.view(shp[0], shp[1], 1, 1) + .001)
# Channel first to channel last for plotting
symbology_x_batch = symbology_x_batch.permute(0, 2, 3, 1)
# Clamp float values to range 0 - 1
if symbology_x_batch.mean() < 1:
symbology_x_batch = symbology_x_batch.clamp(0, 1)
else:
symbology_x_batch = x_batch.permute(0, 2, 3, 1)
# Squeeze channels if single channel (1, 224, 224) -> (224, 224)
if symbology_x_batch.shape[-1] == 1:
symbology_x_batch = symbology_x_batch.squeeze()
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(ncols * imsize, nrows * imsize))
fig.suptitle('Ground Truth / Predictions', fontsize=title_font_size)
for i in range(nrows):
if nrows == 1:
ax_i = ax
else:
ax_i = ax[i]
# Ground Truth
ax_i[0].imshow(symbology_x_batch[i])
ax_i[0].axis('off')
if mode in ['mask', 'bbox_mask']:
n_instance = y_batch[i].unique().shape[0]
y_merged = y_batch[i].max(dim=0)[0].cpu().numpy()
y_rgba = cmap_fn._resample(n_instance)(y_merged)
y_rgba[y_merged == 0] = 0
y_rgba[:, :, -1] = alpha
ax_i[0].imshow(y_rgba)
ax_i[0].axis('off')
# Predictions
ax_i[1].imshow(symbology_x_batch[i])
ax_i[1].axis('off')
if mode in ['mask', 'bbox_mask']:
n_instance = np.unique(pred_mask[i]).shape[0]
p_rgba = cmap_fn._resample(n_instance)(pred_mask[i])
p_rgba[pred_mask[i] == 0] = 0
p_rgba[:, :, -1] = alpha
ax_i[1].imshow(p_rgba)
if mode in ['bbox_mask', 'bbox']:
if pred_box[i] != []:
for num_boxes in pred_box[i]:
rect = patches.Rectangle((num_boxes[0], num_boxes[1]),
num_boxes[2] - num_boxes[0],
num_boxes[3] - num_boxes[1],
linewidth=1,
edgecolor='r',
facecolor='none')
ax_i[1].add_patch(rect)
ax_i[1].axis('off')
plt.subplots_adjust(top=top)
if self._device == torch.device('cuda'):
torch.cuda.empty_cache()
def average_precision_score(self,
detect_thresh=0.5,
iou_thresh=0.5,
mean=False,
show_progress=True):
"""
Computes average precision on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
detect_thresh Optional float. The probabilty above which
a detection will be considered for computing
average precision.
--------------------- -------------------------------------------
iou_thresh Optional float. The intersection over union
threshold with the ground truth mask, above
which a predicted mask will be
considered a true positive.
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
average precision otherwise returns mean
average precision.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
self._check_requisites()
if mean:
aps = compute_class_AP(self, self._data.valid_dl, 1, show_progress,
detect_thresh, iou_thresh, mean)
return aps
else:
aps = compute_class_AP(self, self._data.valid_dl, self._data.c - 1,
show_progress, detect_thresh, iou_thresh)
return dict(zip(self._data.classes[1:], aps))
# +
model = Dnet_1ch()
learn = Learner(data,
model,
loss_func=Loss_combine(),
opt_func=Over9000,
metrics=[
Metric_grapheme(),
Metric_vowel(),
Metric_consonant(),
Metric_tot()
])
learn.model = nn.DataParallel(learn.model)
logger = CSVLogger(learn, f'log{fold}')
learn.clip_grad = 1.0
learn.split([model.head1])
learn.unfreeze()
# -
learn.summary()
# +
# learn.fit_one_cycle(32, max_lr=slice(0.2e-2,1e-2), wd=[1e-3,0.1e-1], pct_start=0.0,
# div_factor=100, callbacks = [logger, SaveModelCallback(learn,monitor='metric_tot',
# mode='max',name=f'model_{fold}'),MixUpCallback(learn)])
# changed config
class ModelExtension(ArcGISModel):
"""
Creates a ``ModelExtension`` object, object detection model to train a model from your own source.
===================== ============================================================
**Argument** **Description**
--------------------- ------------------------------------------------------------
data Required fastai Databunch. Returned data object from
``prepare_data`` function.
--------------------- ------------------------------------------------------------
model_conf A class definition contains the following methods:
* ``get_model(self, data, backbone=None)``: for model definition,
* ``on_batch_begin(self, learn, model_input_batch, model_target_batch)``: for
feeding input to the model during training,
* ``transform_input(self, xb)``: for feeding input to the model during
inferencing/validation,
* ``transform_input_multispectral(self, xb)``: for feeding input to the
model during inferencing/validation in case of multispectral data,
* ``loss(self, model_output, *model_target)``: to return loss value of the model, and
* ``post_process(self, pred, nms_overlap, thres, chip_size, device)``: to post-process
the output of the model.
--------------------- ------------------------------------------------------------
backbone Optional function. If custom model requires any backbone.
--------------------- ------------------------------------------------------------
pretrained_path Optional string. Path where pre-trained model is
saved.
===================== ============================================================
:return: ``ModelExtension`` Object
"""
def __init__(self, data, model_conf, backbone=None, pretrained_path=None):
super().__init__(data, backbone)
self.model_conf = model_conf()
self.model_conf_class = model_conf
self._backend = 'pytorch'
model = self.model_conf.get_model(data, backbone)
if self._is_multispectral:
model = _change_tail(model, data)
if not _isnotebook() and os.name == 'posix':
_set_ddp_multigpu(self)
if self._multigpu_training:
self.learn = Learner(
data, model,
loss_func=self.model_conf.loss).to_distributed(
self._rank_distributed)
else:
self.learn = Learner(data,
model,
loss_func=self.model_conf.loss)
else:
self.learn = Learner(data, model, loss_func=self.model_conf.loss)
self.learn.callbacks.append(
self.train_callback(self.learn, self.model_conf.on_batch_begin))
self._code = code
self._arcgis_init_callback() # make first conv weights learnable
if pretrained_path is not None:
self.load(pretrained_path)
if HAS_FASTAI:
class train_callback(LearnerCallback):
def __init__(self, learn, on_batch_begin_fn):
super().__init__(learn)
self.on_batch_begin_fn = on_batch_begin_fn
def on_batch_begin(self, last_input, last_target, train, **kwargs):
last_input, last_target = self.on_batch_begin_fn(
self.learn, last_input, last_target)
return {'last_input': last_input, 'last_target': last_target}
def _analyze_pred(self,
pred,
thresh=0.5,
nms_overlap=0.1,
ret_scores=True,
device=None):
return self.model_conf.post_process(pred, nms_overlap, thresh,
self.learn.data.chip_size, device)
def _get_emd_params(self):
import random
_emd_template = {}
_emd_template["Framework"] = "arcgis.learn.models._inferencing"
_emd_template["InferenceFunction"] = "ArcGISObjectDetector.py"
_emd_template["ModelConfiguration"] = "_model_extension_inferencing"
_emd_template["ModelType"] = "ObjectDetection"
_emd_template["ExtractBands"] = [0, 1, 2]
_emd_template['Classes'] = []
_emd_template['ModelConfigurationFile'] = "ModelConfiguration.py"
_emd_template['ModelFileConfigurationClass'] = type(
self.model_conf).__name__
class_data = {}
for i, class_name in enumerate(
self._data.classes[1:]): # 0th index is background
inverse_class_mapping = {
v: k
for k, v in self._data.class_mapping.items()
}
class_data["Value"] = inverse_class_mapping[class_name]
class_data["Name"] = class_name
color = [random.choice(range(256)) for i in range(3)]
class_data["Color"] = color
_emd_template['Classes'].append(class_data.copy())
return _emd_template
@property
def _is_model_extension(self):
return True
@classmethod
def from_model(cls, emd_path, data=None):
"""
Creates a ``ModelExtension`` object from an Esri Model Definition (EMD) file.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
emd_path Required string. Path to Esri Model Definition
file.
--------------------- -------------------------------------------
data Required fastai Databunch or None. Returned data
object from ``prepare_data`` function or None for
inferencing.
===================== ===========================================
:returns: `ModelExtension` Object
"""
emd_path = Path(emd_path)
with open(emd_path) as f:
emd = json.load(f)
model_file = Path(emd['ModelFile'])
if not model_file.is_absolute():
model_file = emd_path.parent / model_file
modelconf = Path(emd['ModelConfigurationFile'])
if not modelconf.is_absolute():
modelconf = emd_path.parent / modelconf
modelconfclass = emd['ModelFileConfigurationClass']
sys.path.append(os.path.dirname(modelconf))
model_configuration = getattr(
importlib.import_module('{}'.format(modelconf.name[0:-3])),
modelconfclass)
backbone = emd['ModelParameters']['backbone']
try:
class_mapping = {i['Value']: i['Name'] for i in emd['Classes']}
color_mapping = {i['Value']: i['Color'] for i in emd['Classes']}
except KeyError:
class_mapping = {
i['ClassValue']: i['ClassName']
for i in emd['Classes']
}
color_mapping = {
i['ClassValue']: i['Color']
for i in emd['Classes']
}
if data is None:
data = _EmptyData(path=emd_path.parent.parent,
loss_func=None,
c=len(class_mapping) + 1,
chip_size=emd['ImageHeight'])
data.class_mapping = class_mapping
data.color_mapping = color_mapping
data.emd_path = emd_path
data.emd = emd
data.classes = ['background']
for k, v in class_mapping.items():
data.classes.append(v)
data = get_multispectral_data_params_from_emd(data, emd)
return cls(data,
model_configuration,
backbone,
pretrained_path=str(model_file))
@property
def _model_metrics(self):
return {
'average_precision_score':
self.average_precision_score(show_progress=False)
}
def _get_y(self, bbox, clas):
try:
bbox = bbox.view(-1, 4)
except Exception:
bbox = torch.zeros(size=[0, 4])
bb_keep = ((bbox[:, 2] - bbox[:, 0]) > 0).nonzero()[:, 0]
return bbox[bb_keep], clas[bb_keep]
def _intersect(self, box_a, box_b):
max_xy = torch.min(box_a[:, None, 2:], box_b[None, :, 2:])
min_xy = torch.max(box_a[:, None, :2], box_b[None, :, :2])
inter = torch.clamp((max_xy - min_xy), min=0)
return inter[:, :, 0] * inter[:, :, 1]
def _box_sz(self, b):
return (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])
def _jaccard(self, box_a, box_b):
inter = self._intersect(box_a, box_b)
union = self._box_sz(box_a).unsqueeze(1) + self._box_sz(
box_b).unsqueeze(0) - inter
return inter / union
def show_results(self, rows=5, thresh=0.5, nms_overlap=0.1):
"""
Displays the results of a trained model on a part of the validation set.
"""
self._check_requisites()
if rows > len(self._data.valid_ds):
rows = len(self._data.valid_ds)
self._show_results_modified(rows=rows,
thresh=thresh,
nms_overlap=nms_overlap,
model=self)
def _show_results_multispectral(self,
rows=5,
thresh=0.3,
nms_overlap=0.1,
alpha=1,
**kwargs):
ax = show_results_multispectral(self,
nrows=rows,
thresh=thresh,
nms_overlap=nms_overlap,
alpha=alpha,
**kwargs)
def _show_results_modified(self, rows=5, **kwargs):
if rows > len(self._data.valid_ds):
rows = len(self._data.valid_ds)
ds_type = DatasetType.Valid
n_items = rows**2 if self.learn.data.train_ds.x._square_show_res else rows
if self.learn.dl(ds_type).batch_size < n_items:
n_items = self.learn.dl(ds_type).batch_size
ds = self.learn.dl(ds_type).dataset
xb, yb = self.learn.data.one_batch(ds_type, detach=False, denorm=False)
self.learn.model.eval()
preds = self.learn.model(self.model_conf.transform_input(xb))
x, y = to_cpu(xb), to_cpu(yb)
norm = getattr(self.learn.data, 'norm', False)
if norm:
x = self.learn.data.denorm(x)
if norm.keywords.get('do_y', False):
y = self.learn.data.denorm(y, do_x=True)
preds = self.learn.data.denorm(preds, do_x=True)
analyze_kwargs, kwargs = split_kwargs_by_func(kwargs,
ds.y.analyze_pred)
preds = ds.y.analyze_pred(preds, **analyze_kwargs)
xs = [ds.x.reconstruct(grab_idx(x, i)) for i in range(n_items)]
if has_arg(ds.y.reconstruct, 'x'):
ys = [
ds.y.reconstruct(grab_idx(y, i), x=x) for i, x in enumerate(xs)
]
zs = [ds.y.reconstruct(z, x=x) for z, x in zip(preds, xs)]
else:
ys = [ds.y.reconstruct(grab_idx(y, i)) for i in range(n_items)]
zs = [ds.y.reconstruct(z) for z in preds]
ds.x.show_xyzs(xs, ys, zs, **kwargs)
def average_precision_score(self,
detect_thresh=0.2,
iou_thresh=0.1,
mean=False,
show_progress=True):
"""
Computes average precision on the validation set for each class.
===================== ===========================================
**Argument** **Description**
--------------------- -------------------------------------------
detect_thresh Optional float. The probabilty above which
a detection will be considered for computing
average precision.
--------------------- -------------------------------------------
iou_thresh Optional float. The intersection over union
threshold with the ground truth labels, above
which a predicted bounding box will be
considered a true positive.
--------------------- -------------------------------------------
mean Optional bool. If False returns class-wise
average precision otherwise returns mean
average precision.
===================== ===========================================
:returns: `dict` if mean is False otherwise `float`
"""
self._check_requisites()
aps = compute_class_AP(self,
self._data.valid_dl,
self._data.c - 1,
show_progress,
detect_thresh=detect_thresh,
iou_thresh=iou_thresh)
if mean:
import statistics
return statistics.mean(aps)
else:
return dict(zip(self._data.classes[1:], aps))
