def __init__(self,
block,
num_blocks,
layer,
num_classes=10,
quantizer=24,
codec='h264'):
super(ResNetMPEG, self).__init__()
encoder_name = 'avc_encoder' if codec == 'h264' else 'heif_encoder'
decoder_name = 'avc_decoder' if codec == 'h264' else 'heif_decoder'
self.in_planes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512 * block.expansion, num_classes)
self.timing = False
self.layer = layer
self.jpeg_image_compression_layer = CompressionLayer(
encoder_name=encoder_name,
encoder_params_dict={'quantizer': quantizer},
decoder_name=decoder_name,
decoder_params_dict={})
def main(options):
compression_layer = None
compressed_sizes = []
compression_times = []
if options.codec:
encoder_name, decoder_name = ('%s_encoder' % options.codec,
'%s_decoder' % options.codec)
compression_layer = CompressionLayer(
encoder_name=encoder_name,
encoder_params_dict=create_params_dict(options.encoder_params),
decoder_name=decoder_name,
decoder_params_dict=create_params_dict(options.decoder_params))
def progress_callback(model, batch_number, batch_count, map_score):
if options.codec:
compression_times.append(model.timelogger.points[0][1])
compressed_sizes.extend(
[x[0] for x in model.get_compressed_sizes()])
print('[%d/%d] mAP: %.6f' % (batch_number, batch_count, map_score))
model = yolo.load_model(
compression_layer_index=options.compression_layer_index,
compression_layer=compression_layer,
log_time=True)
map_score = do_test(model, options.images, options.labels,
options.batch_size, progress_callback)
data = collections.OrderedDict([
('codec', options.codec),
('compression_at', options.compression_layer_index),
('batch_size', options.batch_size),
('encoder_params', options.encoder_params),
('decoder_params', options.decoder_params),
('images', options.images),
('labels', options.labels),
('map_score', map_score),
('compressed_size', {
('mean', np.mean(compressed_sizes)),
('median', np.median(compressed_sizes)),
('stdev', np.std(compressed_sizes)),
}),
('compression_time', {
('mean', np.mean(compression_times)),
('median', np.median(compression_times)),
('stdev', np.std(compression_times)),
}),
])
if options.output:
with open(options.output, "w") as outfile:
json.dump(data, outfile)
else:
pprint(data)
def __init__(self, inplanes):
super(Compressor, self).__init__()
# self.compression_layer = CompressionLayer(encoder_name='jpeg_encoder',
# encoder_params_dict={'quantizer' : 36},
# decoder_name='jpeg_decoder',
# decoder_params_dict={})
# self.compression_layer = CompressionLayer(encoder_name='avc_encoder',
# encoder_params_dict={'quantizer' : 42},
# decoder_name='avc_decoder',
# decoder_params_dict={})
self.compression_layer = CompressionLayer(encoder_name='nnfc2_encoder',
encoder_params_dict={},
decoder_name='nnfc2_decoder',
decoder_params_dict={})
# a = torch.arange(0,128).reshape((1, 1, 16, 8)).float()
# print(a.shape, a)
# b = self.compression_layer(a)
# print(b.shape, b)
# print(a == b)
self.sizes = []
self.pad = nn.ReplicationPad2d(2)
# define the bottleneck layers
# expland to 6x the size with 3x3
# mix with 1x1
# bottleneck to same spatial dims, but less channels
#planes = int(inplanes / 2)
# encoder
#t = 12
#self.encoder = LinearBottleneck(inplanes, planes, t=t)
# decoder
#self.decoder = LinearBottleneck(planes, inplanes, t=t)
print('inPlanes', inplanes)
class ResNetNNFC1(nn.Module):
def __init__(self, block, num_blocks, num_classes=10, quantizer=87):
super(ResNetNNFC1, self).__init__()
self.in_planes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512 * block.expansion, num_classes)
self.timing = False
self.nnfc_compression_layer = CompressionLayer(
encoder_name='nnfc1_encoder',
encoder_params_dict={'quantizer': quantizer},
decoder_name='nnfc1_decoder',
decoder_params_dict={})
def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, stride))
self.in_planes = planes * block.expansion
return nn.Sequential(*layers)
def get_compressed_sizes(self):
return self.nnfc_compression_layer.get_compressed_sizes()
def forward(self, x):
out = x
out = F.relu(self.bn1(self.conv1(out)))
if self.timing:
return out
out = self.layer1(out)
out = self.layer2(out)
out = self.nnfc_compression_layer(out)
out = self.layer3(out)
out = self.layer4(out)
out = F.avg_pool2d(out, 4)
out = out.view(out.size(0), -1)
out = self.linear(out)
return out
def __init__(self, block, num_blocks, num_classes=10, quantizer=87):
super(ResNetNNFC1, self).__init__()
self.in_planes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512 * block.expansion, num_classes)
self.timing = False
self.nnfc_compression_layer = CompressionLayer(
encoder_name='nnfc1_encoder',
encoder_params_dict={'quantizer': quantizer},
decoder_name='nnfc1_decoder',
decoder_params_dict={})
class Compressor(nn.Module):
def __init__(self, inplanes):
super(Compressor, self).__init__()
# self.compression_layer = CompressionLayer(encoder_name='jpeg_encoder',
# encoder_params_dict={'quantizer' : 36},
# decoder_name='jpeg_decoder',
# decoder_params_dict={})
# self.compression_layer = CompressionLayer(encoder_name='avc_encoder',
# encoder_params_dict={'quantizer' : 42},
# decoder_name='avc_decoder',
# decoder_params_dict={})
self.compression_layer = CompressionLayer(encoder_name='nnfc2_encoder',
encoder_params_dict={},
decoder_name='nnfc2_decoder',
decoder_params_dict={})
# a = torch.arange(0,128).reshape((1, 1, 16, 8)).float()
# print(a.shape, a)
# b = self.compression_layer(a)
# print(b.shape, b)
# print(a == b)
self.sizes = []
self.pad = nn.ReplicationPad2d(2)
# define the bottleneck layers
# expland to 6x the size with 3x3
# mix with 1x1
# bottleneck to same spatial dims, but less channels
#planes = int(inplanes / 2)
# encoder
#t = 12
#self.encoder = LinearBottleneck(inplanes, planes, t=t)
# decoder
#self.decoder = LinearBottleneck(planes, inplanes, t=t)
print('inPlanes', inplanes)
# print('compressPlanes', planes)
# print('t =', t)
def get_compressed_sizes(self):
return self.compression_layer.get_compressed_sizes()
def forward(self, x):
# x = self.encoder(x)
# x_min = float(x.min())
# x_max = float(x.max())
# print(x.max(), x.min())
# thres = 0.5
# x_top = x.clone()
# x_top[x_top <= thres] = 0
# x_bot = x.clone()
# x_bot[x_bot >= -thres] = 0
# x = x_top + x_bot
# density = np.histogram(x.cpu().detach().numpy(), bins=10)
# print(density)
# print(x.max(), x.min())
# visualization code
# print(x.shape)
# d = x[0,:,:,:].cpu().detach().numpy()
# dmin = np.min(d)
# dmax = np.max(d)
# for i in range(x.shape[1]):
# d = x[0,i,:,:].cpu().detach().numpy()
# print(d.shape)
# print(dmin, dmax)
# img = ((255 * (d - dmin)) / (dmax - dmin)).astype(np.uint8)
# imgmin = np.min(img)
# imgmax = np.max(img)
# img = Image.fromarray(img)
# img.save('/home/jemmons/intermediates/intermediate{}.png'.format(i))
# print(x.shape)
x = self.pad(x)
x = self.compression_layer(x)
x = x[:, :, 1:-1, 1:-1]
self.sizes += self.compression_layer.get_compressed_sizes()
# x = self.decoder(x)
# print(np.mean(np.asarray(self.sizes)))
# print(np.median(np.asarray(self.sizes)))
return x
def __init__(self):
super(AutoEncoder1, self).__init__()
stride = 1
self.compression_layer = CompressionLayer(
encoder_name='nnfc1_encoder',
encoder_params_dict={'quantizer': -1},
decoder_name='nnfc1_decoder',
decoder_params_dict={})
# encoder
self.conv1 = nn.Conv2d(256,
256,
kernel_size=1,
stride=1,
padding=0,
bias=False)
self.conv_bn1 = nn.BatchNorm2d(256)
self.conv2 = nn.Conv2d(256,
128,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
self.conv_bn2 = nn.BatchNorm2d(128)
self.conv3 = nn.Conv2d(128,
128,
kernel_size=1,
stride=1,
padding=0,
bias=False)
self.conv_bn3 = nn.BatchNorm2d(128)
self.conv4 = nn.Conv2d(128,
64,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
self.conv_bn4 = nn.BatchNorm2d(64)
# decoder
self.deconv1 = nn.ConvTranspose2d(64,
64,
kernel_size=1,
stride=1,
padding=0,
bias=False)
self.deconv_bn1 = nn.BatchNorm2d(64)
self.deconv2 = nn.ConvTranspose2d(64,
128,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.deconv_bn2 = nn.BatchNorm2d(128)
self.deconv3 = nn.ConvTranspose2d(128,
128,
kernel_size=1,
stride=1,
padding=0,
bias=False)
self.deconv_bn3 = nn.BatchNorm2d(128)
self.deconv4 = nn.ConvTranspose2d(128,
256,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.deconv_bn4 = nn.BatchNorm2d(256)
def __init__(self):
super(YoloV3, self).__init__()
# self.compression_layer = CompressionLayer(encoder_name='jpeg_image_encoder',
# encoder_params_dict={'quantizer' : 87},
# decoder_name='jpeg_image_decoder',
# decoder_params_dict={})
# self.compression_layer = CompressionLayer(encoder_name='jpeg_encoder',
# encoder_params_dict={'quantizer' : 39},
# decoder_name='jpeg_decoder',
# decoder_params_dict={})
self.compression_layer = CompressionLayer(
encoder_name='nnfc1_encoder',
encoder_params_dict={'quantizer': -1},
decoder_name='nnfc1_decoder',
decoder_params_dict={})
self.compression_layer.register_weights = lambda x, y: None
# darknet53 layers (the first 52 conv layers are present)
self.dn53_standalone = [[DarknetConv(0, 3, 32, stride=1)],
[DarknetConv(1, 32, 64)],
[DarknetConv(2, 64, 128)],
[DarknetConv(3, 128, 256)],
[DarknetConv(4, 256, 512)],
[DarknetConv(5, 512, 1024)]]
self.dn53_block = []
# darknet53_block_structure = [1, 2, 8, 8, 4]
for i, k in enumerate([1, 2, 8, 8, 4]):
self.dn53_block += [[
DarknetBlock('darknet53_block{}_instance{}'.format(i, j),
2**(i + 6), 2**(i + 5)) for j in range(k)
]]
# YOLO extracts features from intermediate points in Darknet53
# The blocks below are layer where the features are taken from
self.layers = [None] * 13
self.layers[0] = (self.dn53_standalone[0] +
self.dn53_standalone[1] +
self.dn53_block[0] +
# [self.compression_layer] +
self.dn53_standalone[2] +
self.dn53_block[1] +
# [self.compression_layer] +
self.dn53_standalone[3] +
self.dn53_block[2] \
+ [self.compression_layer]
)
# self.layers[0] = (self.dn53_standalone[0] +
# self.dn53_standalone[1] +
# self.dn53_block[0] +
# self.dn53_standalone[2] +
# self.dn53_block[1]
# + [self.compression_layer]
# )
# self.layers[0] = (self.dn53_standalone[0] +
# self.dn53_standalone[1] +
# self.dn53_block[0]
# + [self.compression_layer]
# )
self.layers[1] = (self.dn53_standalone[4] + self.dn53_block[3])
self.layers[2] = (self.dn53_standalone[5] + self.dn53_block[4])
# yolo detection layers
# detection 1
self.layers[3] = [
YoloBlock(0, 1024, 512, 1, 1, 0),
YoloBlock(1, 512, 1024, 3, 1, 1),
YoloBlock(2, 1024, 512, 1, 1, 0),
YoloBlock(3, 512, 1024, 3, 1, 1),
YoloBlock(4, 1024, 512, 1, 1, 0)
]
self.layers[4] = [YoloBlock(5, 512, 1024, 3, 1, 1)]
self.layers[5] = [YoloConv(6, 1024, 255)]
# detection 2
self.layers[6] = [YoloBlock(7, 512, 256, 1, 1, 0), YoloUpsample()]
self.layers[7] = [
YoloBlock(8, 768, 256, 1, 1, 0),
YoloBlock(9, 256, 512, 3, 1, 1),
YoloBlock(10, 512, 256, 1, 1, 0),
YoloBlock(11, 256, 512, 3, 1, 1),
YoloBlock(12, 512, 256, 1, 1, 0)
]
self.layers[8] = [YoloBlock(13, 256, 512, 3, 1, 1)]
self.layers[9] = [YoloConv(14, 512, 255)]
# detection 3
self.layers[10] = [YoloBlock(15, 256, 128, 1, 1, 0), YoloUpsample()]
self.layers[11] = [
YoloBlock(16, 384, 128, 1, 1, 0),
YoloBlock(17, 128, 256, 3, 1, 1),
YoloBlock(18, 256, 128, 1, 1, 0),
YoloBlock(19, 128, 256, 3, 1, 1),
YoloBlock(20, 256, 128, 1, 1, 0),
YoloBlock(21, 128, 256, 3, 1, 1)
]
self.layers[12] = [YoloConv(22, 256, 255)]
#register the layers
for layers in self.layers:
for layer in layers:
layer.register_weights(self.register_parameter,
self.register_buffer)