def __init__(self, anchor_ratios=(0.5, 1, 2), **kwargs):
super().__init__(
2,
SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE,
from_logits=True),
L1Loss(reduction=tf.keras.losses.Reduction.NONE),
multiples=len(anchor_ratios),
kernel_initializer_classification_head=initializers.RandomNormal(stddev=0.01),
kernel_initializer_box_prediction_head=initializers.RandomNormal(stddev=0.01),
**kwargs)
#Force each ground_truths to match to at least one anchor
matcher = Matcher([0.3, 0.7], [0, -1, 1], allow_low_quality_matches=True)
self.target_assigner = TargetAssigner(IoUSimilarity(),
matcher,
encode_boxes_faster_rcnn,
dtype=self._compute_dtype)
anchor_strides = (4, 8, 16, 32, 64)
anchor_zises = (32, 64, 128, 256, 512)
self._anchor_ratios = anchor_ratios
# Precompute a deterministic grid of anchors for each layer of the pyramid.
# We will extract a subpart of the anchors according to
self._anchors = [
Anchors(stride, size, self._anchor_ratios)
for stride, size in zip(anchor_strides, anchor_zises)
]
def __init__(self, num_classes, backbone, num_queries=300, **kwargs):
super().__init__(**kwargs)
self.num_classes = num_classes
self.num_queries = num_queries
self.hidden_dim = 256
self.backbone = backbone
self.input_proj = tf.keras.layers.Conv2D(self.hidden_dim, 1)
self.pos_embed = PositionEmbeddingSine(output_dim=self.hidden_dim)
num_heads = 8
self.transformer_num_layers = 6
self.transformer = Transformer(num_layers=self.transformer_num_layers,
d_model=self.hidden_dim,
num_heads=num_heads,
dim_feedforward=2048)
# MCMA layers
self.dyn_weight_map = DynamicalWeightMaps()
self.ref_points = SMCAReferencePoints(self.hidden_dim, num_heads)
self.bbox_embed = tf.keras.models.Sequential([
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(4, dtype=tf.float32) # (x1, y1, x2, y2)
])
self.class_embed = tf.keras.layers.Dense(num_classes + 1, dtype=tf.float32)
# Will create a learnable embedding matrix for all our queries
# It is a matrix of [num_queries, self.hidden_dim]
# The embedding layers
self.query_embed = tf.keras.layers.Embedding(
num_queries,
self.hidden_dim,
embeddings_initializer=tf.keras.initializers.RandomNormal(mean=0., stddev=1.))
self.all_the_queries = tf.range(num_queries)
# Loss computation
self.weight_class, self.weight_l1, self.weight_giou = 2, 5, 2
similarity_func = DetrSimilarity(self.weight_class, self.weight_l1, self.weight_giou)
self.target_assigner = TargetAssigner(similarity_func,
hungarian_matching,
lambda gt, pred: gt,
negative_class_weight=1.0)
# Losses
self.giou = tfa.losses.GIoULoss(reduction=tf.keras.losses.Reduction.NONE)
self.l1 = L1Loss(reduction=tf.keras.losses.Reduction.NONE)
self.focal_loss = tfa.losses.SigmoidFocalCrossEntropy(
alpha=0.25, gamma=2, reduction=tf.keras.losses.Reduction.NONE, from_logits=True)
# Metrics
self.giou_metric = tf.keras.metrics.Mean(name="giou_last_layer")
self.l1_metric = tf.keras.metrics.Mean(name="l1_last_layer")
self.focal_loss_metric = tf.keras.metrics.Mean(name="focal_loss_last_layer")
self.loss_metric = tf.keras.metrics.Mean(name="loss")
self.precision_metric = tf.keras.metrics.SparseCategoricalAccuracy()
# Object recall = foreground
self.recall_metric = tf.keras.metrics.Mean(name="object_recall")
def __init__(self, num_classes: int, backbone, num_queries=100, **kwargs):
super().__init__(**kwargs)
self.num_classes = num_classes
self.num_queries = num_queries
self.hidden_dim = 256
self.backbone = backbone
self.input_proj = tf.keras.layers.Conv2D(self.hidden_dim, 1)
self.pos_embed = PositionEmbeddingSine(output_dim=self.hidden_dim)
self.transformer_num_layers = 6
self.transformer = Transformer(num_layers=self.transformer_num_layers,
d_model=self.hidden_dim,
num_heads=8,
dim_feedforward=2048)
self.bbox_embed = tf.keras.models.Sequential([
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(4, activation='sigmoid',
dtype=tf.float32) # (x1, y1, x2, y2)
])
self.class_embed = tf.keras.layers.Dense(num_classes + 1,
dtype=tf.float32)
# Will create a learnable embedding matrix for all our queries
# It is a matrix of [num_queries, self.hidden_dim]
# The embedding layers
self.query_embed = tf.keras.layers.Embedding(num_queries,
self.hidden_dim)
self.all_the_queries = tf.range(num_queries)
# Loss computation
self.weight_class, self.weight_l1, self.weight_giou = 1, 5, 2
similarity_func = DetrSimilarity(self.weight_class, self.weight_l1,
self.weight_giou)
self.target_assigner = TargetAssigner(similarity_func,
hungarian_matching,
lambda gt, pred: gt,
negative_class_weight=1.0)
# Relative classification weight applied to the no-object category
# It down-weight the log-probability term of a no-object
# by a factor 10 to account for class imbalance
self.non_object_weight = tf.constant(0.1, dtype=self.compute_dtype)
# Losses
self.giou = GIoULoss(reduction=tf.keras.losses.Reduction.NONE)
self.l1 = L1Loss(reduction=tf.keras.losses.Reduction.NONE)
self.scc = SparseCategoricalCrossentropy(
reduction=tf.keras.losses.Reduction.NONE, from_logits=True)
# Metrics
self.giou_metric = tf.keras.metrics.Mean(name="giou_last_layer")
self.l1_metric = tf.keras.metrics.Mean(name="l1_last_layer")
self.scc_metric = tf.keras.metrics.Mean(name="scc_last_layer")
self.loss_metric = tf.keras.metrics.Mean(name="loss")
self.precision_metric = tf.keras.metrics.SparseCategoricalAccuracy()
# Object recall = foreground
self.recall_metric = tf.keras.metrics.Mean(name="object_recall")
def __init__(self, num_classes, **kwargs):
super().__init__(
num_classes,
SparseCategoricalCrossentropy(
reduction=tf.keras.losses.Reduction.NONE, from_logits=True),
L1Loss(reduction=tf.keras.losses.Reduction.NONE
), # like in tensorpack
kernel_initializer_classification_head=initializers.RandomNormal(
stddev=0.01),
kernel_initializer_box_prediction_head=initializers.RandomNormal(
stddev=0.001),
**kwargs)
matcher = Matcher([0.5], [0, 1])
# The same scale_factors is used in decoding as well
encode = functools.partial(encode_boxes_faster_rcnn,
scale_factors=(10.0, 10.0, 5.0, 5.0))
self.target_assigner = TargetAssigner(IoUSimilarity(),
matcher,
encode,
dtype=self._compute_dtype)
class RegionProposalNetwork(AbstractDetectionHead):
"""It has been introduced in the [Faster R-CNN paper](https://arxiv.org/abs/1506.01497) and
use the parameters from [Feature Pyramidal Networks for Object Detection](https://arxiv.org/abs/1612.03144).
Arguments:
- *anchor_ratios*: The ratios are the different shapes that you want to apply on your anchors.
e.g: (0.5, 1, 2)
"""
def __init__(self, anchor_ratios=(0.5, 1, 2), **kwargs):
super().__init__(
2,
SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE,
from_logits=True),
L1Loss(reduction=tf.keras.losses.Reduction.NONE),
multiples=len(anchor_ratios),
kernel_initializer_classification_head=initializers.RandomNormal(stddev=0.01),
kernel_initializer_box_prediction_head=initializers.RandomNormal(stddev=0.01),
**kwargs)
#Force each ground_truths to match to at least one anchor
matcher = Matcher([0.3, 0.7], [0, -1, 1], allow_low_quality_matches=True)
self.target_assigner = TargetAssigner(IoUSimilarity(),
matcher,
encode_boxes_faster_rcnn,
dtype=self._compute_dtype)
anchor_strides = (4, 8, 16, 32, 64)
anchor_zises = (32, 64, 128, 256, 512)
self._anchor_ratios = anchor_ratios
# Precompute a deterministic grid of anchors for each layer of the pyramid.
# We will extract a subpart of the anchors according to
self._anchors = [
Anchors(stride, size, self._anchor_ratios)
for stride, size in zip(anchor_strides, anchor_zises)
]
def build(self, input_shape):
self.rpn_conv2d = KL.Conv2D(512, (3, 3),
padding='same',
kernel_initializer=self._kernel_initializer_classification_head,
kernel_regularizer=self._kernel_regularizer)
super().build(input_shape)
def build_rpn_head(self, inputs):
"""Predictions for the classification and the regression
Arguments:
- *inputs*: A tensor of shape [batch_size, width, height, channel]
Returns:
A tuple of tensors of shape ([batch_size, num_anchors, 2], [batch_size, num_anchors, 4])
"""
batch_size = tf.shape(inputs)[0]
rpn_conv2d = self.rpn_conv2d(inputs)
classification_head, localization_head = self.build_detection_head(rpn_conv2d)
classification_head = tf.reshape(classification_head, (batch_size, -1, 2))
localization_head = tf.reshape(localization_head, (batch_size, -1, 4))
return classification_head, localization_head
def call(self, inputs: List[tf.Tensor]):
"""Create the computation graph for the rpn inference
Argument:
*inputs*: A List of tensors the output of the pyramid
Returns:
- *localization_pred*: A list of logits 3-D tensor of shape [batch_size, num_anchors, 4]
- *classification_pred*: A lost of logits 3-D tensor of shape [batch_size, num_anchors, 2]
- *anchors*: A list of tensors of shape [batch_size, num_anchors, (y_min, x_min, y_max, x_max)]
"""
anchors = [anchors(tensor) for tensor, anchors in zip(inputs, self._anchors)]
rpn_predictions = [self.build_rpn_head(tensor) for tensor in inputs]
localization_pred = [prediction[1] for prediction in rpn_predictions]
classification_pred = [prediction[0] for prediction in rpn_predictions]
return localization_pred, classification_pred, anchors
def compute_loss(self, localization_pred, classification_pred, anchors, ground_truths):
"""Compute the loss
Arguments:
- *localization_pred*: A list of tensors of shape [batch_size, num_anchors, 4].
- *classification_pred*: A list of tensors of shape [batch_size, num_anchors, 2]
- *anchors*: A list of tensors of shape [num_anchors, (y_min, x_min, y_max, x_max)]
- *ground_truths*: A dict with BoxField as key and a tensor as value.
```python
ground_truths = {
BoxField.BOXES:
tf.constant([[[0, 0, 1, 1], [0, 0, 2, 2]], [[0, 0, 3, 3], [0, 0, 0, 0]]], tf.float32),
BoxField.LABELS:
tf.constant([[1, 0], [1, 0]], tf.float32),
BoxField.WEIGHTS:
tf.constant([[1, 0], [1, 1]], tf.float32),
BoxField.NUM_BOXES:
tf.constant([[2], [1]], tf.int32)
}
```
where `NUM_BOXES` allows to remove the padding created by tf.Data.
Returns:
- *classification_loss*: A scalar in tf.float32
- *localization_loss*: A scalar in tf.float32
"""
localization_pred = tf.concat(localization_pred, 1)
classification_pred = tf.concat(classification_pred, 1)
anchors = tf.concat(anchors, 0)
ground_truths = {
# We add one because the background is not counted in ground_truths[BoxField.LABELS]
BoxField.LABELS:
ground_truths[BoxField.LABELS] + 1,
BoxField.BOXES:
ground_truths[BoxField.BOXES],
BoxField.WEIGHTS:
ground_truths[BoxField.WEIGHTS],
BoxField.NUM_BOXES:
ground_truths[BoxField.NUM_BOXES]
}
# anchors are deterministic duplicate them to create a batch
anchors = tf.tile(anchors[None], (tf.shape(ground_truths[BoxField.BOXES])[0], 1, 1))
y_true, weights = self.target_assigner.assign({BoxField.BOXES: anchors}, ground_truths)
y_true[BoxField.LABELS] = tf.minimum(y_true[BoxField.LABELS], 1)
## Compute metrics
recall = compute_rpn_metrics(y_true[BoxField.LABELS], classification_pred,
weights[BoxField.LABELS])
self.add_metric(recall, name='rpn_recall', aggregation='mean')
# All the boxes which are not -1 can be sampled
labels = y_true[BoxField.LABELS] > 0
sample_idx = batch_sample_balanced_positive_negative(
weights[BoxField.LABELS],
SAMPLING_SIZE,
labels,
positive_fraction=SAMPLING_POSITIVE_RATIO,
dtype=self._compute_dtype)
weights[BoxField.LABELS] = sample_idx * weights[BoxField.LABELS]
weights[BoxField.BOXES] = sample_idx * weights[BoxField.BOXES]
y_pred = {BoxField.LABELS: classification_pred, BoxField.BOXES: localization_pred}
return self.compute_losses(y_true, y_pred, weights)
def get_config(self):
base_config = super().get_config()
base_config['anchor_ratios'] = self._anchor_ratios
return base_config
class DeTr(tf.keras.Model):
"""Build a DeTr model according to the paper
[End-to-End Object Detection with Transformers](https://arxiv.org/abs/2005.12872)
You can use it as follow:
```python
model = DeTrResnet50Pytorch(80)
base_lr = 0.1
optimizer = tf.keras.optimizers.SGD(learning_rate=base_lr)
model.compile(optimizer=optimizer, loss=None)
model.fit(ds_train, validation_data=ds_test, epochs=11,)
```
Arguments:
num_classes: The number of classes of your dataset
(**do not include the background class** it is handle for you)
backbone: A vision model like ResNet50.
num_queries: number of object queries, ie detection slot.
This is the maximal number of objects
DETR can detect in a single image. For COCO, we recommend 100 queries.
Call arguments:
inputs: Tuple
1. images: A 4-D tensor of float32 and shape [batch_size, None, None, 3]
2. image_informations: A 1D tensor of float32 and shape [(height, width),].
It contains the shape of the image without any padding.
3. images_padding_mask: A 3D tensor of int8 and shape [batch_size, None, None]
composed of 0 and 1 which allows to know where a padding has been applied.
training: Is automatically set to `True` in train mode
Call returns:
Tuple:
- `logits`: A Tensor of shape [batch_size, h, num_classes + 1] class logits
- `boxes`: A Tensor of shape [batch_size, h, 4]
where h is num_queries * transformer_decoder.transformer_num_layers if
training is true and num_queries otherwise.
"""
def __init__(self, num_classes: int, backbone, num_queries=100, **kwargs):
super().__init__(**kwargs)
self.num_classes = num_classes
self.num_queries = num_queries
self.hidden_dim = 256
self.backbone = backbone
self.input_proj = tf.keras.layers.Conv2D(self.hidden_dim, 1)
self.pos_embed = PositionEmbeddingSine(output_dim=self.hidden_dim)
self.transformer_num_layers = 6
self.transformer = Transformer(num_layers=self.transformer_num_layers,
d_model=self.hidden_dim,
num_heads=8,
dim_feedforward=2048)
self.bbox_embed = tf.keras.models.Sequential([
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(4, activation='sigmoid',
dtype=tf.float32) # (x1, y1, x2, y2)
])
self.class_embed = tf.keras.layers.Dense(num_classes + 1,
dtype=tf.float32)
# Will create a learnable embedding matrix for all our queries
# It is a matrix of [num_queries, self.hidden_dim]
# The embedding layers
self.query_embed = tf.keras.layers.Embedding(
num_queries,
self.hidden_dim,
embeddings_initializer=tf.keras.initializers.RandomNormal(
mean=0., stddev=1.))
self.all_the_queries = tf.range(num_queries)
# Loss computation
self.weight_class, self.weight_l1, self.weight_giou = 1, 5, 2
similarity_func = DetrSimilarity(self.weight_class, self.weight_l1,
self.weight_giou)
self.target_assigner = TargetAssigner(similarity_func,
hungarian_matching,
lambda gt, pred: gt,
negative_class_weight=1.0)
# Relative classification weight applied to the no-object category
# It down-weight the log-probability term of a no-object
# by a factor 10 to account for class imbalance
self.non_object_weight = tf.constant(0.1, dtype=self.compute_dtype)
# Losses
self.giou = GIoULoss(reduction=tf.keras.losses.Reduction.NONE)
self.l1 = L1Loss(reduction=tf.keras.losses.Reduction.NONE)
self.scc = SparseCategoricalCrossentropy(
reduction=tf.keras.losses.Reduction.NONE, from_logits=True)
# Metrics
self.giou_metric = tf.keras.metrics.Mean(name="giou_last_layer")
self.l1_metric = tf.keras.metrics.Mean(name="l1_last_layer")
self.scc_metric = tf.keras.metrics.Mean(name="scc_last_layer")
self.loss_metric = tf.keras.metrics.Mean(name="loss")
self.precision_metric = tf.keras.metrics.SparseCategoricalAccuracy()
# Object recall = foreground
self.recall_metric = tf.keras.metrics.Mean(name="object_recall")
@property
def metrics(self):
return [
self.loss_metric, self.giou_metric, self.l1_metric,
self.scc_metric, self.precision_metric, self.recall_metric
]
def call(self, inputs, training=None):
"""Perform an inference in training.
Arguments:
inputs: Tuple
1. images: A 4-D tensor of float32 and shape [batch_size, None, None, 3]
2. image_informations: A 1D tensor of float32 and shape [(height, width),]. It contains the shape
of the image without any padding.
3. images_padding_mask: A 3D tensor of int8 and shape
[batch_size, None, None] composed of 0 and 1 which
allows to know where a padding has been applied.
training: Is automatically set to `True` in train mode
Returns:
Tuple:
- `logits`: A Tensor of shape [batch_size, h, num_classes + 1] class logits
- `boxes`: A Tensor of shape [batch_size, h, 4]
where h is num_queries * transformer_decoder.transformer_num_layers if
training is true and num_queries otherwise.
"""
images = inputs[DatasetField.IMAGES]
images_padding_masks = inputs[DatasetField.IMAGES_PMASK]
batch_size = tf.shape(images)[0]
# The preprocessing dedicated to the backbone is done inside the model.
x = self.backbone(images)[-1]
features_mask = tf.image.resize(
tf.cast(images_padding_masks[..., None], tf.float32),
tf.shape(x)[1:3],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
features_mask = tf.cast(features_mask, tf.bool)
# Positional_encoding for the backbone
pos_embed = self.pos_embed(features_mask)
# [batch_size, num_queries, self.hidden_dim]
all_the_queries = tf.tile(self.all_the_queries[None], (batch_size, 1))
# [batch_size, num_queries, self.hidden_dim]
query_embed = self.query_embed(all_the_queries)
# add positional_encoding to x [batch_size, h, w, self.hidden_dim]
x = self.input_proj(x)
# Flatten the position embedding and the spatial tensor
# to allow the preprocessing by the Transformer
# [batch_size, h * w, self.hidden_dim]
x = tf.reshape(x, (batch_size, -1, self.hidden_dim))
pos_embed = tf.reshape(pos_embed, (batch_size, -1, self.hidden_dim))
# Flatten the padding masks
features_mask = tf.reshape(features_mask, (batch_size, -1))
decoder_out, _ = self.transformer(x,
pos_embed,
query_embed,
key_padding_mask=features_mask,
training=training)
boxes = self.bbox_embed(decoder_out)
logits = self.class_embed(decoder_out)
return {
BoxField.SCORES: logits,
BoxField.BOXES: boxes,
}
def compute_loss(
self,
ground_truths: Dict[str, tf.Tensor],
y_pred: Dict[str, tf.Tensor],
input_shape: tf.Tensor,
) -> int:
"""Apply the GIoU, L1 and SCC to each layers of the transformer decoder
Arguments:
ground_truths: see output kerod.dataset.preprocessing for the doc
y_pred: A dict
- *scores: A Tensor of shape [batch_size, num_queries, num_classes + 1] class logits
- *bbox*: A Tensor of shape [batch_size, num_queries, 4]
input_shape: [height, width] of the input tensor.
It is the shape of the images will all the padding included.
It is used to normalize the ground_truths boxes.
"""
normalized_boxes = ground_truths[BoxField.BOXES] / tf.tile(
input_shape[None], [1, 2])
centered_normalized_boxes = convert_to_center_coordinates(
normalized_boxes)
ground_truths = {
# We add one because the background is not counted in ground_truths [BoxField.LABELS]
BoxField.LABELS:
ground_truths[BoxField.LABELS] + 1,
BoxField.BOXES:
centered_normalized_boxes,
BoxField.WEIGHTS:
ground_truths[BoxField.WEIGHTS],
BoxField.NUM_BOXES:
ground_truths[BoxField.NUM_BOXES]
}
boxes_per_lvl = tf.split(y_pred[BoxField.BOXES],
self.transformer_num_layers,
axis=1)
logits_per_lvl = tf.split(y_pred[BoxField.SCORES],
self.transformer_num_layers,
axis=1)
y_pred_per_lvl = [{
BoxField.BOXES: boxes,
BoxField.SCORES: logits
} for boxes, logits in zip(boxes_per_lvl, logits_per_lvl)]
num_boxes = tf.cast(tf.reduce_sum(ground_truths[BoxField.NUM_BOXES]),
tf.float32)
loss = 0
# Compute the Giou, L1 and SCC at each layers of the transformer decoder
for i, y_pred in enumerate(y_pred_per_lvl):
# Logs the metrics for the last layer of the decoder
compute_metrics = i == self.transformer_num_layers - 1
loss += self._compute_loss(y_pred,
ground_truths,
num_boxes,
compute_metrics=compute_metrics)
return loss
def _compute_loss(
self,
y_pred: Dict[str, tf.Tensor],
ground_truths: Dict[str, tf.Tensor],
num_boxes: int,
compute_metrics=False,
):
y_true, weights = self.target_assigner.assign(y_pred, ground_truths)
# Reduce the class imbalanced by applying to the weights
# self.non_object_weight for the non object (pos 0)
weights[BoxField.LABELS] = item_assignment(
weights[BoxField.LABELS], y_true[BoxField.LABELS] == 0,
self.non_object_weight)
# Caveats GIoU is buggy and if the batch_size is 1 and the sample_weight
# is provided will raise an error
giou = self.giou(convert_to_xyxy_coordinates(y_true[BoxField.BOXES]),
convert_to_xyxy_coordinates(y_pred[BoxField.BOXES]),
sample_weight=weights[BoxField.BOXES])
l1 = self.l1(y_true[BoxField.BOXES],
y_pred[BoxField.BOXES],
sample_weight=weights[BoxField.BOXES])
# SparseCategoricalCrossentropy
scc = self.scc(y_true[BoxField.LABELS],
y_pred[BoxField.SCORES],
sample_weight=weights[BoxField.LABELS])
giou = self.weight_giou * tf.reduce_sum(giou) / num_boxes
l1 = self.weight_l1 * tf.reduce_sum(l1) / num_boxes
scc = self.weight_class * tf.reduce_sum(scc) / tf.reduce_sum(
weights[BoxField.LABELS])
if compute_metrics:
self.giou_metric.update_state(giou)
self.l1_metric.update_state(l1)
self.scc_metric.update_state(scc)
self.precision_metric.update_state(
y_true[BoxField.LABELS],
y_pred[BoxField.SCORES],
sample_weight=weights[BoxField.LABELS])
recall = compute_detr_metrics(y_true[BoxField.LABELS],
y_pred[BoxField.SCORES])
self.recall_metric.update_state(recall)
return giou + l1 + scc
def train_step(self, data):
data = data_adapter.expand_1d(data)
x, ground_truths, _ = data_adapter.unpack_x_y_sample_weight(data)
with tf.GradientTape() as tape:
y_pred = self(x, training=True)
input_shape = tf.cast(
tf.shape(x[DatasetField.IMAGES])[1:3], self.compute_dtype)
loss = self.compute_loss(ground_truths, y_pred, input_shape)
loss += self.compiled_loss(None,
y_pred,
None,
regularization_losses=self.losses)
self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
self.loss_metric.update_state(loss)
return {m.name: m.result() for m in self.metrics}
def test_step(self, data):
data = data_adapter.expand_1d(data)
x, ground_truths, _ = data_adapter.unpack_x_y_sample_weight(data)
# To compute the loss we need to get the results of each decoder layer
# Setting training to True will provide it
y_pred = self(x, training=True)
input_shape = tf.cast(
tf.shape(x[DatasetField.IMAGES])[1:3], self.compute_dtype)
loss = self.compute_loss(ground_truths, y_pred, input_shape)
loss += self.compiled_loss(None,
y_pred,
None,
regularization_losses=self.losses)
self.loss_metric.update_state(loss)
return {m.name: m.result() for m in self.metrics}
def predict_step(self, data):
"""Perform an inference and returns the boxes, scores and labels associated.
Background is discarded the max and argmax operation are performed.
It means that if background was predicted the second maximum score would
be outputed.
Example: background + 3 classes
[0.54, 0.40, 0.03, 0.03] => score = 0.40, label = 0 (1 - 1)
"To optimize for AP, we override the prediction of these slots
with the second highest scoring class, using the corresponding confidence"
Part 4. Experiments of Object Detection with Transformers
Returns:
boxes: A Tensor of shape [batch_size, self.num_queries, (y1,x1,y2,x2)]
containing the boxes with the coordinates between 0 and 1.
scores: A Tensor of shape [batch_size, self.num_queries] containing
the score of the boxes.
classes: A Tensor of shape [batch_size, self.num_queries]
containing the class of the boxes [0, num_classes).
"""
data = data_adapter.expand_1d(data)
x, _, _ = data_adapter.unpack_x_y_sample_weight(data)
y_pred = self(x, training=False)
boxes_without_padding, scores, labels = detr_postprocessing(
y_pred[BoxField.BOXES],
y_pred[BoxField.SCORES],
x[DatasetField.IMAGES_INFO],
tf.shape(x[DatasetField.IMAGES])[1:3],
)
return boxes_without_padding, scores, labels
class FastRCNN(AbstractDetectionHead):
"""Build the Fast-RCNN on top of the FPN. The parameters used
are from [Feature Pyramidal Networks for Object Detection](https://arxiv.org/abs/1612.03144).
Arguments:
num_classes: The number of classes that predict the classification head (N+1) where N
is the number of classes of your dataset and 1 is the background.
Call arguments:
inputs: A Tuple
1. `pyramid`: A List of tensors the output of the pyramid
2. `anchors`: A tensor of shape [batch_size, num_boxes, (y_min, x_min, y_max, x_max)]
Call returns:
Tuple:
`classification_pred`: A logit Tensor of shape [batch_size, num_boxes, num_classes]
`localization_pred`: A Tensor of shape [batch_size, num_boxes, 4 * (num_classes - 1)]
`anchors`: A Tensor of shape [batch_size, num_boxes, 4]
"""
def __init__(self, num_classes, **kwargs):
super().__init__(
num_classes,
SparseCategoricalCrossentropy(
reduction=tf.keras.losses.Reduction.NONE, from_logits=True),
L1Loss(reduction=tf.keras.losses.Reduction.NONE
), # like in tensorpack
kernel_initializer_classification_head=initializers.RandomNormal(
stddev=0.01),
kernel_initializer_box_prediction_head=initializers.RandomNormal(
stddev=0.001),
**kwargs)
matcher = Matcher([0.5], [0, 1])
# The same scale_factors is used in decoding as well
encode = functools.partial(encode_boxes_faster_rcnn,
scale_factors=(10.0, 10.0, 5.0, 5.0))
self.target_assigner = TargetAssigner(IoUSimilarity(),
matcher,
encode,
dtype=self._compute_dtype)
def build(self, input_shape):
self.denses = [
KL.Dense(1024,
kernel_initializer=initializers.VarianceScaling(),
kernel_regularizer=self._kernel_regularizer,
activation='relu') for _ in range(2)
]
super().build(input_shape)
def call(self, inputs):
"""Build the computational graph of the fast RCNN HEAD.
It performs a raw prediction of the FastRCNN head you can post_process them using:
```python
from kerod.layers.post_processing import post_process_fast_rcnn_boxes
outputs = post_process_fast_rcnn_boxes(classification_pred, localization_pred, anchors,
images_information, num_classes)
```
where `images_information` is provided as input of your model and `num_classes` includes
the background.
Arguments:
inputs: A Tuple
1. `pyramid`: A List of tensors the output of the pyramid
2. `anchors`: A tensor of shape [batch_size, num_boxes, (y_min, x_min, y_max, x_max)]
Returns:
Tuple:
`classification_pred`: A logit Tensor of shape [batch_size, num_boxes, num_classes]
`localization_pred`: A Tensor of shape [batch_size, num_boxes, 4 * (num_classes - 1)]
`anchors`: A Tensor of shape [batch_size, num_boxes, 4]
"""
# Remove P6
pyramid = inputs[0][:-1]
anchors = inputs[1]
# We can compute the original image shape regarding
# TODO compute it more automatically without knowing that the last layer is stride 32
image_shape = tf.cast(tf.shape(pyramid[-1])[1:3] * 32,
dtype=self._compute_dtype)
boxe_tensors = multilevel_roi_align(pyramid,
anchors,
image_shape,
crop_size=7)
l = KL.Flatten()(boxe_tensors)
for dense in self.denses:
l = dense(l)
classification_pred, localization_pred = self.build_detection_head(
tf.reshape(l, (-1, 1, 1, 1024)))
batch_size = tf.shape(anchors)[0]
classification_pred = tf.reshape(classification_pred,
(batch_size, -1, self._num_classes))
localization_pred = tf.reshape(localization_pred,
(batch_size, -1,
(self._num_classes - 1) * 4))
return classification_pred, localization_pred
def sample_boxes(self,
anchors: tf.Tensor,
ground_truths: Dict[str, tf.Tensor],
sampling_size: int = 512,
sampling_positive_ratio: float = 0.25):
"""Perform the sampling of the target anchors.
During the training a set of RoIs is detected by the RPN.
However, you do not want to analyse all the set. You only want
to analyse the anchors that you sampled with this method.
Arguments:
anchors: A tensor of shape [batch_size, num_boxes, (y_min, x_min, y_max, x_max)]
ground_truths: A dict
- `BoxField.LABELS`: A 3-D tensor of shape [batch_size, num_gt, num_classes],
- `BoxField.BOXES`: A 3-D tensor of shape [batch_size, num_gt, (y1, x1, y2, x2)]
- `BoxField.LABELS`: A 3-D tensor of int32 and shape [batch_size, num_gt]
- `BoxField.WEIGHTS`: A 3-D tensor of float and shape [batch_size, num_gt]
- `BoxField.NUM_BOXES`: A 2-D tensor of int32 and shape [batch_size, 1]
which allows to remove the padding created by tf.Data.
Example: if batch_size=2 and this field equal tf.constant([[2], [1]], tf.int32)
then my second box has a padding of 1
sampling_size: Desired sampling size. If None, keeps all positive samples and
randomly selects negative samples so that the positive sample fraction
matches positive_fraction.
sampling_positive_ratio: Desired fraction of positive examples (scalar in [0,1])
in the batch.
Returns:
Tuple:
1. y_true: A dict with :
- `BoxField.LABELS`: A 3-D tensor of shape [batch_size, num_anchors,
num_classes],
- `BoxField.BOXES`: A 3-D tensor of shape [batch_size, num_anchors,
box_code_dimension]
2. weights: A dict with:
- `BoxField.LABELS`: A 2-D tensor of shape [batch_size, num_anchors],
- `BoxField.BOXES`: A 2-D tensor of shape [batch_size, num_anchors]
Raises:
ValueError: If the batch_size is None.
ValueError: If the batch_size between your ground_truths and the anchors does not match.
"""
ground_truths = {
# We add one because the background is not counted in ground_truths[BoxField.LABELS]
BoxField.LABELS:
ground_truths[BoxField.LABELS] + 1,
BoxField.BOXES:
ground_truths[BoxField.BOXES],
BoxField.WEIGHTS:
ground_truths[BoxField.WEIGHTS],
BoxField.NUM_BOXES:
ground_truths[BoxField.NUM_BOXES]
}
y_true, weights = self.target_assigner.assign(
{BoxField.BOXES: anchors}, ground_truths)
labels = y_true[BoxField.LABELS] > 0
sample_idx = batch_sample_balanced_positive_negative(
weights[BoxField.LABELS],
sampling_size,
labels,
positive_fraction=sampling_positive_ratio,
dtype=self._compute_dtype)
weights[BoxField.LABELS] = sample_idx * weights[BoxField.LABELS]
weights[BoxField.BOXES] = sample_idx * weights[BoxField.BOXES]
selected_boxes_idx = tf.where(sample_idx == 1)
batch_size = tf.shape(sample_idx)[0]
# Extract the selected anchors corresponding anchors
# tf.gather_nd collaps the batch_together so we reshape with the proper batch_size
anchors = tf.reshape(tf.gather_nd(anchors, selected_boxes_idx),
(batch_size, -1, 4))
y_true[BoxField.BOXES] = tf.reshape(
tf.gather_nd(y_true[BoxField.BOXES], selected_boxes_idx),
(batch_size, -1, 4))
y_true[BoxField.LABELS] = tf.reshape(
tf.gather_nd(y_true[BoxField.LABELS], selected_boxes_idx),
(batch_size, -1))
for key in y_true.keys():
weights[key] = tf.reshape(
tf.gather_nd(weights[key], selected_boxes_idx),
(batch_size, -1))
weights[key] = tf.stop_gradient(weights[key])
y_true[key] = tf.stop_gradient(y_true[key])
return y_true, weights, anchors
def compute_loss(self, y_true: dict, weights: dict,
classification_pred: tf.Tensor,
localization_pred: tf.Tensor):
"""Compute the loss of the FastRCNN
Arguments:
y_true: A dict with :
- `BoxField.LABELS`: A 3-D tensor of shape [batch_size, num_anchors, num_classes]
- `BoxField.BOXES`: A 3-D tensor of shape [batch_size, num_anchors, 4]
weights: A dict with:
- `BoxField.LABELS`: A 3-D tensor of shape [batch_size, num_anchors, num_classes]
- `BoxField.BOXES`: A 2-D tensor of shape [batch_size, num_anchors]
classification_pred: A 3-D tensor of float and shape
[batch_size, num_anchors, num_classes]
localization_pred: A 3-D tensor of float and shape
[batch_size, num_anchors, (num_classes - 1) * 4]
Returns:
Tuple:
- `classification_loss`: A scalar
- `localization_loss`: A scalar
"""
y_true_classification = tf.cast(y_true[BoxField.LABELS], tf.int32)
accuracy, fg_accuracy, false_negative = compute_fast_rcnn_metrics(
y_true_classification, classification_pred)
self.add_metric(accuracy, name='accuracy', aggregation='mean')
self.add_metric(fg_accuracy, name='fg_accuracy', aggregation='mean')
self.add_metric(false_negative,
name='false_negative',
aggregation='mean')
# y_true[BoxField.LABELS] is just 1 and 0 we are using it as mask to extract
# the corresponding target anchors
batch_size = tf.shape(classification_pred)[0]
# We create a boolean mask to extract the desired localization prediction to compute
# the loss
one_hot_targets = tf.one_hot(y_true_classification,
self._num_classes,
dtype=tf.int8)
one_hot_targets = tf.reshape(one_hot_targets, [-1])
# We need to insert a fake background classes at the position 0
localization_pred = tf.pad(localization_pred, [[0, 0], [0, 0], [4, 0]])
localization_pred = tf.reshape(localization_pred, [-1, 4])
extracted_localization_pred = tf.boolean_mask(localization_pred,
one_hot_targets > 0)
extracted_localization_pred = tf.reshape(extracted_localization_pred,
(batch_size, -1, 4))
y_pred = {
BoxField.LABELS: classification_pred,
BoxField.BOXES: extracted_localization_pred
}
return self.compute_losses(y_true, y_pred, weights)
class SMCA(tf.keras.Model):
"""Build a single scale SCMA model according to the paper
[Fast Convergence of DETR with Spatially Modulated Co-Attention](https://arxiv.org/pdf/2101.07448.pdf).
In what is it different from DETR ?
Just imagine that your object queries are learned anchors.
Those learned "anchors" will modulate the attention map during
the coattention stage of the decoder. They will help to target
faster some sweet spots which leads to a speed up by 10
of the training. It maintains the same performance than DETR.
You can use it as follow:
```python
model = SMCAR50(80)
base_lr = 0.1
optimizer = tf.keras.optimizers.SGD(learning_rate=base_lr)
model.compile(optimizer=optimizer, loss=None)
model.fit(ds_train, validation_data=ds_test, epochs=11,)
```
Arguments:
num_classes: The number of classes of your dataset
(**do not include the background class** it is handle for you)
backbone: A vision model like ResNet50.
num_queries: number of object queries, ie detection slot.
This is the maximal number of objects
SCMA can detect in a single image. For COCO, we recommend 300 queries.
Call arguments:
inputs: Tuple
1. images: A 4-D tensor of float32 and shape [batch_size, None, None, 3]
2. image_informations: A 1D tensor of float32 and shape [(height, width),].
It contains the shape of the image without any padding.
3. images_padding_mask: A 3D tensor of int8 and shape [batch_size, None, None]
composed of 0 and 1 which allows to know where a padding has been applied.
training: Is automatically set to `True` in train mode
Call returns:
logits: A Tensor of shape [batch_size, h, num_classes + 1] class logits
boxes: A Tensor of shape [batch_size, h, 4]
where h is num_queries * transformer_decoder.num_layers if
training is true and num_queries otherwise.
"""
def __init__(self, num_classes, backbone, num_queries=300, **kwargs):
super().__init__(**kwargs)
self.num_classes = num_classes
self.num_queries = num_queries
self.hidden_dim = 256
self.backbone = backbone
self.input_proj = tf.keras.layers.Conv2D(self.hidden_dim, 1)
self.pos_embed = PositionEmbeddingSine(output_dim=self.hidden_dim)
num_heads = 8
self.transformer_num_layers = 6
self.transformer = Transformer(num_layers=self.transformer_num_layers,
d_model=self.hidden_dim,
num_heads=num_heads,
dim_feedforward=2048)
# MCMA layers
self.dyn_weight_map = DynamicalWeightMaps()
self.ref_points = SMCAReferencePoints(self.hidden_dim, num_heads)
self.bbox_embed = tf.keras.models.Sequential([
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(self.hidden_dim, activation='relu'),
tf.keras.layers.Dense(4, dtype=tf.float32) # (x1, y1, x2, y2)
])
self.class_embed = tf.keras.layers.Dense(num_classes + 1,
dtype=tf.float32)
# Will create a learnable embedding matrix for all our queries
# It is a matrix of [num_queries, self.hidden_dim]
# The embedding layers
self.query_embed = tf.keras.layers.Embedding(num_queries,
self.hidden_dim)
self.all_the_queries = tf.range(num_queries)
# Loss computation
self.weight_class, self.weight_l1, self.weight_giou = 2, 5, 2
similarity_func = DetrSimilarity(self.weight_class, self.weight_l1,
self.weight_giou)
self.target_assigner = TargetAssigner(similarity_func,
hungarian_matching,
lambda gt, pred: gt,
negative_class_weight=1.0)
# Losses
self.giou = tfa.losses.GIoULoss(
reduction=tf.keras.losses.Reduction.NONE)
self.l1 = L1Loss(reduction=tf.keras.losses.Reduction.NONE)
self.focal_loss = tfa.losses.SigmoidFocalCrossEntropy(
alpha=0.25,
gamma=2,
reduction=tf.keras.losses.Reduction.NONE,
from_logits=True)
# Metrics
self.giou_metric = tf.keras.metrics.Mean(name="giou_last_layer")
self.l1_metric = tf.keras.metrics.Mean(name="l1_last_layer")
self.focal_loss_metric = tf.keras.metrics.Mean(
name="focal_loss_last_layer")
self.loss_metric = tf.keras.metrics.Mean(name="loss")
self.precision_metric = tf.keras.metrics.SparseCategoricalAccuracy()
# Object recall = foreground
self.recall_metric = tf.keras.metrics.Mean(name="object_recall")
@property
def metrics(self):
return [
self.loss_metric, self.giou_metric, self.l1_metric,
self.focal_loss_metric, self.precision_metric, self.recall_metric
]
def call(self, inputs, training=None):
"""Perform an inference in training.
Arguments:
- *inputs*: Tuple
1. images: A 4-D tensor of float32 and shape [batch_size, None, None, 3]
2. image_informations: A 1D tensor of float32 and shape [(height, width),]. It contains the shape
of the image without any padding.
3. images_padding_mask: A 3D tensor of int8 and shape [batch_size, None, None] composed of 0 and 1 which allows to know where a padding has been applied.
- *training*: Is automatically set to `True` in train mode
Returns:
- *logits*: A Tensor of shape [batch_size, num_queries, num_classes + 1] class logits
- *boxes*: A Tensor of shape [batch_size, num_queries, 4]
where h is num_queries * transformer_decoder.num_layers if
training is true and num_queries otherwise.
"""
images = inputs[DatasetField.IMAGES]
images_padding_masks = inputs[DatasetField.IMAGES_PMASK]
batch_size = tf.shape(images)[0]
# The preprocessing dedicated to the backbone is done inside the model.
x = self.backbone(images)[-1]
features_mask = tf.image.resize(
tf.cast(images_padding_masks[..., None], tf.float32),
tf.shape(x)[1:3],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
features_mask = tf.cast(features_mask, tf.bool)
# Positional_encoding for the backbone
pos_embed = self.pos_embed(features_mask)
# [batch_size, num_queries, self.hidden_dim]
all_the_queries = tf.tile(self.all_the_queries[None], (batch_size, 1))
# [batch_size, num_queries, self.hidden_dim]
query_embed = self.query_embed(all_the_queries)
h_backbone_out, w_backbone_out = tf.shape(x)[1], tf.shape(x)[2]
x = self.input_proj(x)
# Flatten the position embedding and the spatial tensor
# to allow the preprocessing by the Transformer
# [batch_size, h * w, self.hidden_dim]
x = tf.reshape(x, (batch_size, -1, self.hidden_dim))
pos_embed = tf.reshape(pos_embed, (batch_size, -1, self.hidden_dim))
# Flatten the padding masks
features_mask = tf.reshape(features_mask, (batch_size, -1))
ref_points, ref_points_presigmoid = self.ref_points(query_embed)
# dyn_weight_map_per_head = G in the paper
dyn_weight_map_per_head = self.dyn_weight_map(h_backbone_out,
w_backbone_out,
ref_points)
dyn_weight_map_per_head = tf.math.log(dyn_weight_map_per_head +
10e-4) # log G
decoder_out, _ = self.transformer(x,
pos_embed,
query_embed,
key_padding_mask=features_mask,
coattn_mask=dyn_weight_map_per_head,
training=training)
logits = self.class_embed(decoder_out)
boxes = self.bbox_embed(decoder_out)
if training:
# In training all the outputs of the decoders are stacked together.
# We tile the reference_points to match those outputs
ref_points_presigmoid = tf.tile(
ref_points_presigmoid, (1, self.transformer_num_layers, 1))
# Add initial center to constrain the bounding boxes predictions
offset = tf.concat([
ref_points_presigmoid,
tf.zeros((batch_size, tf.shape(ref_points_presigmoid)[1], 2))
],
axis=-1)
boxes = tf.nn.sigmoid(boxes + offset)
return {
BoxField.SCORES: logits,
BoxField.BOXES: boxes,
}
def compute_loss(
self,
ground_truths: Dict[str, tf.Tensor],
y_pred: Dict[str, tf.Tensor],
input_shape: tf.Tensor,
) -> int:
"""Apply the GIoU, L1 and SCC to each layers of the transformer decoder
Args:
ground_truths: see output kerod.dataset.preprocessing for the doc
y_pred: A dict
- *scores: A Tensor of shape [batch_size, num_queries, num_classes + 1] class logits
- *bbox*: A Tensor of shape [batch_size, num_queries, 4]
input_shape: [height, width] of the input tensor.
It is the shape of the images will all the padding included.
It is used to normalize the ground_truths boxes.
"""
normalized_boxes = ground_truths[BoxField.BOXES] / tf.tile(
input_shape[None], [1, 2])
centered_normalized_boxes = convert_to_center_coordinates(
normalized_boxes)
ground_truths = {
# We add one because the background is not counted in ground_truths [BoxField.LABELS]
BoxField.LABELS:
ground_truths[BoxField.LABELS] + 1,
BoxField.BOXES:
centered_normalized_boxes,
BoxField.WEIGHTS:
ground_truths[BoxField.WEIGHTS],
BoxField.NUM_BOXES:
ground_truths[BoxField.NUM_BOXES]
}
boxes_per_lvl = tf.split(y_pred[BoxField.BOXES],
self.transformer_num_layers,
axis=1)
logits_per_lvl = tf.split(y_pred[BoxField.SCORES],
self.transformer_num_layers,
axis=1)
y_pred_per_lvl = [{
BoxField.BOXES: boxes,
BoxField.SCORES: logits
} for boxes, logits in zip(boxes_per_lvl, logits_per_lvl)]
num_boxes = tf.cast(tf.reduce_sum(ground_truths[BoxField.NUM_BOXES]),
tf.float32)
loss = 0
# Compute the Giou, L1 and SCC at each layers of the transformer decoder
for i, y_pred in enumerate(y_pred_per_lvl):
# Logs the metrics for the last layer of the decoder
compute_metrics = i == self.transformer_num_layers - 1
loss += self._compute_loss(y_pred,
ground_truths,
num_boxes,
compute_metrics=compute_metrics)
return loss
def _compute_loss(
self,
y_pred: Dict[str, tf.Tensor],
ground_truths: Dict[str, tf.Tensor],
num_boxes: int,
compute_metrics=False,
):
y_true, weights = self.target_assigner.assign(y_pred, ground_truths)
# Caveats GIoU is buggy and if the batch_size is 1 and the sample_weight
# is provided will raise an error
giou = self.giou(convert_to_xyxy_coordinates(y_true[BoxField.BOXES]),
convert_to_xyxy_coordinates(y_pred[BoxField.BOXES]),
sample_weight=weights[BoxField.BOXES])
l1 = self.l1(y_true[BoxField.BOXES],
y_pred[BoxField.BOXES],
sample_weight=weights[BoxField.BOXES])
cls_labels = tf.one_hot(
y_true[BoxField.LABELS],
depth=self.num_classes + 1,
dtype=tf.float32,
)
focal_loss = self.focal_loss(cls_labels,
y_pred[BoxField.SCORES],
sample_weight=weights[BoxField.LABELS])
giou = self.weight_giou * tf.reduce_sum(giou) / num_boxes
l1 = self.weight_l1 * tf.reduce_sum(l1) / num_boxes
focal_loss = self.weight_class * tf.reduce_sum(
focal_loss) / tf.reduce_sum(weights[BoxField.LABELS])
if compute_metrics:
self.giou_metric.update_state(giou)
self.l1_metric.update_state(l1)
self.focal_loss_metric.update_state(focal_loss)
self.precision_metric.update_state(
y_true[BoxField.LABELS],
y_pred[BoxField.SCORES],
sample_weight=weights[BoxField.LABELS])
recall = compute_detr_metrics(y_true[BoxField.LABELS],
y_pred[BoxField.SCORES])
self.recall_metric.update_state(recall)
return giou + l1 + focal_loss
def train_step(self, data):
data = data_adapter.expand_1d(data)
x, ground_truths, _ = data_adapter.unpack_x_y_sample_weight(data)
with tf.GradientTape() as tape:
y_pred = self(x, training=True)
input_shape = tf.cast(
tf.shape(x[DatasetField.IMAGES])[1:3], self.compute_dtype)
loss = self.compute_loss(ground_truths, y_pred, input_shape)
loss += self.compiled_loss(None,
y_pred,
None,
regularization_losses=self.losses)
self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
self.loss_metric.update_state(loss)
return {m.name: m.result() for m in self.metrics}
def test_step(self, data):
data = data_adapter.expand_1d(data)
x, ground_truths, _ = data_adapter.unpack_x_y_sample_weight(data)
# To compute the loss we need to get the results of each decoder layer
# Setting training to True will provide it
y_pred = self(x, training=True)
input_shape = tf.cast(
tf.shape(x[DatasetField.IMAGES])[1:3], self.compute_dtype)
loss = self.compute_loss(ground_truths, y_pred, input_shape)
loss += self.compiled_loss(None,
y_pred,
None,
regularization_losses=self.losses)
self.loss_metric.update_state(loss)
return {m.name: m.result() for m in self.metrics}
def predict_step(self, data):
"""Perform an inference and returns the boxes, scores and labels associated.
Background is discarded the max and argmax operation are performed.
It means that if background was predicted the second maximum score would
be outputed.
Example: background + 3 classes
[0.54, 0.40, 0.03, 0.03] => score = 0.40, label = 0 (1 - 1)
"To optimize for AP, we override the prediction of these slots
with the second highest scoring class, using the corresponding confidence"
Part 4. Experiments of Object Detection with Transformers
Returns:
boxes: A Tensor of shape [batch_size, self.num_queries, (y1,x1,y2,x2)]
containing the boxes with the coordinates between 0 and 1.
scores: A Tensor of shape [batch_size, self.num_queries] containing
the score of the boxes.
classes: A Tensor of shape [batch_size, self.num_queries]
containing the class of the boxes [0, num_classes).
"""
data = data_adapter.expand_1d(data)
x, _, _ = data_adapter.unpack_x_y_sample_weight(data)
y_pred = self(x, training=False)
boxes_without_padding, scores, labels = detr_postprocessing(
y_pred[BoxField.BOXES],
y_pred[BoxField.SCORES],
x[DatasetField.IMAGES_INFO],
tf.shape(x[DatasetField.IMAGES])[1:3],
)
return boxes_without_padding, scores, labels
