def __init__(self, phase, size, base, extras, head, num_classes):
super(S1_SSD, self).__init__()
self.phase = phase
self.num_classes = num_classes
self.cfg = voc
self.priorbox = PriorBox(self.cfg)
# self.priors = Variable(self.priorbox.forward(), volatile=True)
self.priors = self.priorbox.forward()
self.priors.requires_grad = False
self.size = size
# S1_SSD network
# input base is not a list, so pack all layers into a list
self.s1_bb = nn.ModuleList([*list(base.children())])
# Layer learns to scale the l2 normalized features from stage 2
self.L2Norm = L2Norm(272, 20)
self.extras = nn.ModuleList(extras)
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
if phase == 'test':
self.softmax = nn.Softmax()
# self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)
self.detect = Detect()
def __init__(self, phase: str = 'train', num_classes: int = 21):
super(SSD, self).__init__()
self.phase = phase
self.num_classes = num_classes
self.vgg: nn.ModuleList = make_vgg()
self.extras: nn.ModuleList = make_extras()
self.L2Norm = L2Norm()
self.loc: nn.ModuleList = make_loc(num_classes=num_classes)
self.conf: nn.ModuleList = make_conf(num_classes=num_classes)
dbox = PriorBox()
self.priors = dbox.forward()
if phase == 'test':
self.detect = Detect()
def __init__(self, img_h, img_w, fpn_channels, num_class, num_mask, aspect_ratio, scales):
super(Yolact, self).__init__()
out = ['conv3_block4_out', 'conv4_block6_out', 'conv5_block3_out']
# use pre-trained ResNet50
# Keras BatchNormalization problem
# https://github.com/keras-team/keras/pull/9965#issuecomment-501933060
tf.keras.layers.BatchNormalization = FrozenBatchNormalization
base_model = tf.keras.applications.ResNet50(input_shape=(img_h, img_w, 3),
include_top=False,
layers=tf.keras.layers,
weights='imagenet')
# extract certain feature maps for FPN
self.backbone_resnet = tf.keras.Model(inputs=base_model.input,
outputs=[base_model.get_layer(x).output for x in out])
# Calculating feature map size
# https://stackoverflow.com/a/44242277/4582711
# https://github.com/tensorflow/tensorflow/issues/4297#issuecomment-246080982
self.feature_map_size = np.array([list(base_model.get_layer(x).output.shape[1:3]) for x in out])
out_height_p6 = np.ceil((self.feature_map_size[-1, 0]).astype(np.float32) / float(2))
out_width_p6 = np.ceil((self.feature_map_size[-1, 1]).astype(np.float32) / float(2))
out_height_p7 = np.ceil(out_height_p6 / float(2))
out_width_p7 = np.ceil(out_width_p6/ float(2))
self.feature_map_size = np.concatenate((self.feature_map_size, [[out_height_p6, out_width_p6], [out_height_p7, out_width_p7]]), axis=0)
self.protonet_out_size = self.feature_map_size[0]*2 # Only one upsampling on p3
self.backbone_fpn = FeaturePyramidNeck(fpn_channels)
self.protonet = ProtoNet(num_mask)
# semantic segmentation branch to boost feature richness
self.semantic_segmentation = tf.keras.layers.Conv2D(num_class-1, (1, 1), 1, padding="same",
kernel_initializer=tf.keras.initializers.glorot_uniform())
anchorobj = anchor.Anchor(img_size_h=img_h,img_size_w=img_w,
feature_map_size=self.feature_map_size,
aspect_ratio=aspect_ratio,
scale=scales)
self.num_anchors = anchorobj.num_anchors
self.priors = anchorobj.anchors
# print("prior shape:", self.priors.shape)
# print("num anchor per feature map: ", self.num_anchor)
# shared prediction head
# Here, len(aspect_ratio) is passed as during prior calculations, individula scale is selected for each layer.
# So, when scale are [24, 48, 96, 130, 192] that means 24 is for p3; 48 is for p4 and so on.
# So, number of priors for that layer will be HxWxlen(aspect_ratio)
# Hence, passing len(aspect_ratio)
# This implementation differs from the original used in yolact
self.predictionHead = PredictionModule(256, len(aspect_ratio), num_class, num_mask)
# post-processing for evaluation
self.detect = Detect(num_class, bkg_label=0, top_k=200,
conf_thresh=0.05, nms_thresh=0.5)
self.max_output_size = 300
def __init__(self, args):
if args.ctx and torch.cuda.is_available():
self.use_cuda = True
else:
self.use_cuda = False
if self.use_cuda:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
else:
torch.set_default_tensor_type('torch.FloatTensor')
self.loadmodel(args.modelpath)
self.threshold = args.threshold
self.img_dir = args.img_dir
self.detect = Detect(cfg)
# self.detect = DetectIou(cfg)
# self.detect = Detect_demo(cfg)
self.Prior = PriorBox()
with torch.no_grad():
self.priors = self.Prior()
self.num_classes = cfg.NUM_CLASSES
def __init__(self, config):
super(Retina, self).__init__()
self.anchors_per_grid_cell = len(config.anchor_ratios) * len(
config.anchor_scales)
self.classes = config.classes
self.num_classes = len(self.classes) + 1
self._backbone = resnet101(pretrained=True)
names, layers = zip(*list(
self._backbone.named_children())[:-2]) # leave off avgpool and fc
self.backbone = []
i = 0
while i < len(names):
j = i + 1
while j < len(names) and not (names[j].startswith('layer')):
j += 1
self.backbone.append(torch.nn.Sequential(*layers[i:j]))
i = j
self.conv6 = torch.nn.Conv2d(2048, 256, 3, stride=2, padding=1)
self.conv7 = torch.nn.Conv2d(256, 256, 3, stride=2, padding=1)
self.conv5 = torch.nn.Conv2d(2048, 256, 3, padding=1)
self.conv4 = torch.nn.Conv2d(1024, 256, 1)
self.conv3 = torch.nn.Conv2d(512, 256, 1)
self.conv2 = torch.nn.Conv2d(256, 256, 1)
self.loc = self.mk_subnet(4, include_sigmoid=False)
self.conf = self.mk_subnet(self.num_classes, include_sigmoid=False)
self.anchors = Anchors(config)
self.detect = Detect(self.num_classes, 200, 0.01, 0.45, self.anchors)
self.config = config
def __init__(self, mode, backbone, size, num_classes, with_fpn=True):
super(SSD, self).__init__()
assert mode in ["test", "train"]
assert backbone in ['mobilenetv3_large', 'mobilenetv3_small']
self.mode = mode
self.num_classes = num_classes
self.cfg = (coco_config, voc_config)[num_classes == 21]
self.priorbox = PriorBox(self.cfg)
self.priors = self.priorbox.forward()
self.size = size
self.with_fpn = with_fpn
# SSD network
if self.with_fpn:
self.basenet, self.topnet, self.conv_layers, self.fpn_layers, self.loc_layers, self.conf_layers =\
self.build_ssd_with_fpn(backbone, self.size, self.num_classes)
else:
self.basenet, self.topnet, self.loc_layers, self.conf_layers =\
self.build_ssd(backbone, self.size, self.num_classes)
if mode == 'test':
self.softmax = nn.Softmax(dim=-1)
self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)
def __init__(self, phase, size, base, extras, head, num_classes):
super(S_SSD, self).__init__()
self.phase = phase
if self.phase != "test" and self.phase != "train":
raise Exception(
"phase must be train or test, got {} instead".format(
self.phase))
self.size = size
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
self.cfg = voc
self.priorbox = PriorBox(self.cfg)
self.priors = self.priorbox.forward()
self.priors.requires_grad = False
self.base = base
self.extras = extras
self.num_classes = num_classes
if phase == 'test':
self.softmax = nn.Softmax(dim=-1)
self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)
class SSD(nn.Module):
if IMPORT_COMPLETED:
def __init__(self,
phase: Literal['train', 'test'] = 'train',
num_classes: int = 21):
super(SSD, self).__init__()
self.phase = phase
self.num_classes = num_classes
self.vgg: nn.ModuleList = make_vgg()
self.extras: nn.ModuleList = make_extras()
self.L2Norm = L2Norm()
self.loc: nn.ModuleList = make_loc(num_classes=num_classes)
self.conf: nn.ModuleList = make_conf(num_classes=num_classes)
dbox = PriorBox()
self.priors = dbox.forward()
if phase == 'test':
self.detect = Detect()
else:
def __init__(self, phase: str = 'train', num_classes: int = 21):
super(SSD, self).__init__()
self.phase = phase
self.num_classes = num_classes
self.vgg: nn.ModuleList = make_vgg()
self.extras: nn.ModuleList = make_extras()
self.L2Norm = L2Norm()
self.loc: nn.ModuleList = make_loc(num_classes=num_classes)
self.conf: nn.ModuleList = make_conf(num_classes=num_classes)
dbox = PriorBox()
self.priors = dbox.forward()
if phase == 'test':
self.detect = Detect()
def forward(self, x: torch.Tensor):
bs = len(x)
out, lout, cout = list(), list(), list()
for i in range(23):
x = self.vgg[i](x)
x1 = x
out.append(self.L2Norm(x1))
for i in range(23, len(self.vgg)):
x = self.vgg[i](x)
out.append(x)
for i in range(0, 8, 2):
x = F.relu(self.extras[i](x), inplace=True)
x = F.relu(self.extras[i + 1](x), inplace=True)
out.append(x)
for (x, l, c) in zip(out, self.loc, self.conf):
lx = l(x).permute(0, 2, 3, 1).contiguous()
cx = c(x).permute(0, 2, 3, 1).contiguous()
lout.append(lx)
cout.append(cx)
lout = torch.cat([o.view(o.size(0), -1) for o in lout], 1)
cout = torch.cat([o.view(o.size(0), -1) for o in cout], 1)
lout = lout.view(lout.size(0), -1, 4)
cout = cout.view(cout.size(0), -1, self.num_classes)
output = (lout, cout, self.priors)
if self.phase == 'test':
return self.detect.apply(output, self.num_classes)
else:
return output
def main(cam_state, pt_state, mot_state):
processes = []
# Pan and Tilt Servo angles are determined by the HS-422 Servos and Adafruit ServoKit library used for this project.
start_pan_angle = 100
start_tilt_angle = 140
try:
with Manager() as manager:
# Initialize the multiprocessing queue buffers. Setting a maxsize of 1 for the buffers currently results in
# optimal performance as there is some latency due to the Raspberry Pi environment.
cam_buffer = Queue(maxsize=1)
detection_buffer = Queue(maxsize=1)
area_buffer = Queue(maxsize=1)
center_buffer = Queue(maxsize=1)
# Initialize the multiprocessing manager values for the pan and tilt process to provide input to the
# pan/tilt and motor processes.
pan = manager.Value("i", start_pan_angle)
tilt = manager.Value("i", start_tilt_angle)
if cam_state == 1:
cam = Camera()
det = Detect(myriad=True)
camera_process = Process(target=cam.start,
args=(cam_buffer, detection_buffer,
center_buffer, area_buffer),
daemon=True)
camera_process.start()
processes.append(camera_process)
detection_process = Process(target=det.start,
args=(cam_buffer,
detection_buffer),
daemon=True)
detection_process.start()
processes.append(detection_process)
if pt_state == 1:
servo = Servos(start_pan_angle, start_tilt_angle)
pan_tilt_process = Process(target=servo.follow,
args=(center_buffer, pan, tilt),
daemon=True)
pan_tilt_process.start()
processes.append(pan_tilt_process)
if mot_state == 1:
mot = Motors()
motor_process = Process(target=mot.follow,
args=(area_buffer, pan),
daemon=True)
motor_process.start()
processes.append(motor_process)
for process in processes:
process.join()
except:
print("Unexpected error: ", sys.exc_info()[0])
finally:
for p in range(len(processes)):
processes[p].terminate()
# Ensure the motors are stopped before exiting.
saber = Sabertooth('/dev/ttyS0')
saber.stop()
sys.exit(0)
def __init__(self):
super().__init__()
self.backbone = construct_backbone(cfg.backbone)
if cfg.freeze_bn:
self.freeze_bn()
if cfg.mask_type == mask_type.direct:
cfg.mask_dim = cfg.mask_size**2
elif cfg.mask_type == mask_type.lincomb:
if cfg.mask_proto_use_grid:
self.grid = torch.Tensor(np.load(cfg.mask_proto_grid_file))
self.num_grids = self.grid.size(0)
else:
self.num_grids = 0
self.proto_src = cfg.mask_proto_src
if self.proto_src is None:
in_channels = 3
elif cfg.fpn is not None:
in_channels = cfg.fpn.num_features
else:
in_channels = self.backbone.channels[self.proto_src]
in_channels += self.num_grids
self.proto_net, cfg.mask_dim = make_net(in_channels,
cfg.mask_proto_net,
include_last_relu=False)
if cfg.mask_proto_bias:
cfg.mask_dim += 1
self.selected_layers = cfg.backbone.selected_layers
src_channels = self.backbone.channels
if cfg.fpn is not None:
# Some hacky rewiring to accomodate the FPN
self.fpn = FPN([src_channels[i] for i in self.selected_layers])
self.selected_layers = list(
range(len(self.selected_layers) + cfg.fpn.num_downsample))
src_channels = [cfg.fpn.num_features] * len(self.selected_layers)
self.prediction_layers = nn.ModuleList()
cfg.num_heads = len(self.selected_layers)
for idx, layer_idx in enumerate(self.selected_layers):
parent = None
if cfg.share_prediction_module and idx > 0:
parent = self.prediction_layers[0]
pred = PredictionModule(
src_channels[layer_idx],
src_channels[layer_idx],
aspect_ratios=cfg.backbone.pred_aspect_ratios[idx],
scales=cfg.backbone.pred_scales[idx],
parent=parent,
index=idx,
)
self.prediction_layers.append(pred)
# Extra parameters for the extra losses
if cfg.use_class_existence_loss:
# This comes from the smallest layer selected
# Also note that cfg.num_classes includes background
self.class_existence_fc = nn.Linear(src_channels[-1],
cfg.num_classes - 1)
if cfg.use_semantic_segmentation_loss:
self.semantic_seg_conv = nn.Conv2d(src_channels[0],
cfg.num_classes - 1,
kernel_size=1)
# For use in evaluation
self.detect = Detect(
cfg.num_classes,
bkg_label=0,
top_k=cfg.nms_top_k,
conf_thresh=cfg.nms_conf_thresh,
nms_thresh=cfg.nms_thresh,
)
class S1_SSD(nn.Module):
"""
changes include:
def __init__: self.detect
def forward: if phase == 'test'
"""
def __init__(self, phase, size, base, extras, head, num_classes):
super(S1_SSD, self).__init__()
self.phase = phase
self.num_classes = num_classes
self.cfg = voc
self.priorbox = PriorBox(self.cfg)
# self.priors = Variable(self.priorbox.forward(), volatile=True)
self.priors = self.priorbox.forward()
self.priors.requires_grad = False
self.size = size
# S1_SSD network
# input base is not a list, so pack all layers into a list
self.s1_bb = nn.ModuleList([*list(base.children())])
# Layer learns to scale the l2 normalized features from stage 2
self.L2Norm = L2Norm(272, 20)
self.extras = nn.ModuleList(extras)
self.loc = nn.ModuleList(head[0])
self.conf = nn.ModuleList(head[1])
if phase == 'test':
self.softmax = nn.Softmax()
# self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)
self.detect = Detect()
def forward(self, x):
"""Applies network layers and ops on input image(s) x.
Args:
x: input image or batch of images. Shape: [batch,3,300,300].
"""
sources = list()
loc = list()
conf = list()
# apply s1_backbone up to stage 2, output shape: [b, 272, 38, 38]
for k in range(3):
x = self.s1_bb[k](x)
s = self.L2Norm(x)
sources.append(s)
# apply s1_bb up to stage 3
# output shape: [b, 544, 19, 19]
x = self.s1_bb[3](x)
sources.append(x)
# apply extra layers and cache source layer outputs
for k, v in enumerate(self.extras):
x = F.relu(v(x), inplace=True)
if k % 2 == 1:
sources.append(x)
# apply multibox head to source layers
for (x, l, c) in zip(sources, self.loc, self.conf):
loc.append(l(x).permute(0, 2, 3, 1).contiguous())
conf.append(c(x).permute(0, 2, 3, 1).contiguous())
loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
# if self.phase == "test":
# output = self.detect(
# loc.view(loc.size(0), -1, 4), # loc preds
# self.softmax(conf.view(conf.size(0), -1,
# self.num_classes)), # conf preds
# self.priors.type(type(x.data)) # default boxes
# )
if self.phase == "test":
output = self.detect.apply(
21,
0,
200,
0.01,
0.45,
loc.view(loc.size(0), -1, 4), # loc preds
self.softmax(conf.view(-1, 21)), # conf preds
self.priors.type(type(x.data)) # default boxes
)
else:
output = (loc.view(loc.size(0), -1,
4), conf.view(conf.size(0), -1,
self.num_classes), self.priors)
return output
def load_weights(self, base_file):
other, ext = os.path.splitext(base_file)
if ext == '.pkl' or '.pth':
print('Loading weights into state dict...')
self.load_state_dict(
torch.load(base_file,
map_location=lambda storage, loc: storage))
print('Finished!')
else:
print('Sorry only .pth and .pkl files supported.')
'checkpoints/EXP_01_SSD300_VOC_BS-16_EP-100_ChkPt_0.0523.hdf5',
by_name=True)
prediction = model.predict(input_img)
#%%****************************************************************************
loc_data = prediction[:, :, :4]
conf_data = prediction[:, :, 4:]
loc_data = torch.from_numpy(loc_data).float()
conf_data = torch.from_numpy(conf_data).float()
priors = torch.from_numpy(priors).float()
Detector = Detect(num_classes=21,
bkg_label=0,
top_k=200,
conf_thresh=0.01,
nms_thresh=0.2)
detections = Detector.forward(loc_data=loc_data,
conf_data=conf_data,
prior_data=priors)
#print(np.unique(detections[:,:,:,0]))
labelmap = ( # always index 0
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person',
'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor')
coordinates = []