def test_not_trainable(self, m_cost):
"""Create an untrainable network.
This is purley the feed forward network for prediction purposes and
contains no optimisation nodes.
"""
sess = self.sess
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Must not create cost nodes.
assert not m_cost.called
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, None, train=False)
assert not m_cost.called
# Further sanity checks.
assert net.trainable() is False
assert net.costNodes() == {}
# Training must be impossible.
with pytest.raises(AssertionError):
net.train(None, None, None, None, None, None)
del net
# Must create cost nodes.
m_cost.return_value = (None, None)
assert not m_cost.called
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, None, train=True)
assert m_cost.called
# Network must consider itself trainable.
assert net.trainable() is True
def test_nonMaxSuppression(self):
"""Ensure the 'nonMaxSuppression' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when provided with valid parameters shapes and types.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create predictor network (parameters do not matter).
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
# Dummy input for NMS.
N = 100
bb_rects = np.random.uniform(-10, 10, (N, 4)).astype(np.float32)
scores = np.random.uniform(0, 1, N).astype(np.float32)
# Function must return a list of integers. Each integer is an index
# into the first bb_rects dimension to indicate which ones survived the
# NMS operation.
out = net.nonMaxSuppression(bb_rects, scores)
assert out.dtype == np.int32
assert 0 <= len(out) < N
# All indices must be unique.
assert len(set(out)) == len(out)
def test_train(self):
"""Ensure the 'train' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when the provided parameters have the correct shape and
type.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create trainable network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, True)
self.sess.run(tf.global_variables_initializer())
assert net.trainable() is True
# Create dummy learning rate, image and training output.
lrate = 1E-5
ft_dim = net.outputShape()
y = np.random.uniform(0, 256, ft_dim.chw()).astype(np.uint8)
img = np.random.randint(0, 256, im_dim.hwc()).astype(np.uint8)
# Create dummy masks.
mask_cls = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
mask_bbox = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
mask_isFg = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
# 'Train' method must complete without error and return the costs.
costs = net.train(img, y, lrate, mask_cls, mask_bbox, mask_isFg)
assert isinstance(costs, dict)
assert set(costs.keys()) == {'cls', 'bbox', 'isFg', 'total'}
def test_basic_attributes(self):
"""Setup network and check basic parameters like TF variable names,
number of layers, size of last feature map...
"""
ft_dim = Shape(None, 64, 64)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
assert net.session() is self.sess
# The feature size must be 1/8 of the image size because the network
# downsamples every second layer, and we specified 7 layers.
assert num_layers == net.numLayers() == 7
assert net.outputShape().hw() == ft_dim.hw()
# Ensure we can query all biases and weights. Also verify the data type
# inside the network.
g = tf.get_default_graph().get_tensor_by_name
for i in range(num_layers):
# These must exist in the graph.
assert g(f'orpac/W{i}:0') is not None
assert g(f'orpac/b{i}:0') is not None
assert net.getBias(i).dtype == np.float32
assert net.getWeight(i).dtype == np.float32
def test_feature_sizes(self):
"""Ensure all the various size getters agree."""
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
# The network must report the correct number of classes.
assert num_cls == net.numClasses()
# The feature channels encode 4 BBox parameters, is-foreground (2
# parameters because the binary choice is hot-label encoded), and the
# number of classes. This convenience function must return that value.
assert net.numOutputChannels(num_cls) == (4 + 2 + num_cls)
# Must return the output tensor shape excluding batch dimension.
ft_shape = net.outputShape()
assert isinstance(ft_shape, Shape)
assert ft_shape.chan == net.numOutputChannels(num_cls)
# Verify the image shape expected by the class for eg `pred` and `train`.
assert net.imageShape() == im_dim
# The input tensor shape for the network differs from the image shape
# passed to the eg `train` and `predict` methods because it holds
# the Wavelet transformed image, not the original image itself.
wl_dim = orpac_net.imageToWaveletDim(im_dim)
assert net._xin.shape == (1, *wl_dim.chw())
# The feature map size must derive from the size of the input image and
# the number of Wavelet transforms. Each WL transform halves the
# dimensions.
assert net.outputShape().hw() == wl_dim.hw()
def test_addOptimiser(self):
"""Create an trainable network and ensure the cost nodes exist.
"""
train = True
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Must call 'cost' function to create cost nodes.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, train)
# Network must identify itself as trainable and return the cost nodes.
assert net.trainable() is True
assert set(net.costNodes().keys()) == {'cls', 'bbox', 'isFg', 'total'}
def setup_class(cls):
# Feature dimension will only be 2x2 to simplify testing and debugging.
ft_dim = Shape(None, 64, 64)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
# Create Tensorflow session and dummy network. The network is such that
# the feature size is only 2x2 because this makes testing easier.
cls.sess = tf.Session()
cls.net = orpac_net.Orpac(
cls.sess, im_dim, num_layers, num_cls, None, train=False)
assert cls.net.outputShape().hw() == ft_dim.hw()
def test_imageToInput(self):
"""Uint8 image must becomes a valid network input."""
num_layers = 7
im_dim = Shape(3, 512, 512)
img = 100 * np.ones(im_dim.hwc(), np.uint8)
# Create a network (parameters do not matter).
net = orpac_net.Orpac(self.sess, im_dim, num_layers, 10, None, False)
# Image must be converted to float32 CHW image with leading
# batch dimension of 1. All values must have been divided by 255.
img_wl = net._imageToInput(img)
dim_xin = Shape(int(net._xin.shape[1]), *net.outputShape().hw())
assert img_wl.dtype == np.float32
assert img_wl.shape == (1, *dim_xin.chw())
def test_non_max_suppresion_setup(self):
"""Ensure the network creates the NMS nodes."""
im_dim = Shape(3, 512, 512)
g = tf.get_default_graph().get_tensor_by_name
# NMS nodes must not yet exist.
try:
assert g('non-max-suppression/op:0') is not None
except KeyError:
pass
# Create a network (parameters do not matter).
orpac_net.Orpac(self.sess, im_dim, 7, 10, None, False)
# All NMS nodes must now exist.
assert g('non-max-suppression/op:0') is not None
assert g('non-max-suppression/scores:0') is not None
assert g('non-max-suppression/bb_rects:0') is not None
def main():
param = parseCmdline()
sess = tf.Session()
netstate_path = 'netstate'
fnames = {
'meta': os.path.join(netstate_path, 'orpac-meta.pickle'),
'orpac-net': os.path.join(netstate_path, 'orpac-net.pickle'),
}
# Load configuration file for latest network.
fname = fnames['meta']
try:
conf = pickle.load(open(fname, 'rb'))['conf']
bw_init = pickle.load(open(fnames['orpac-net'], 'rb'))
except FileNotFoundError:
print(f'\nError: Configuration {fname} does not exist.')
return
del fname
# Overwrite the number of samples to load, and put all of them into the
# 'test' set.
conf = conf._replace(path=param.src, num_samples=param.N)
# Load the BBox training data.
print('\n----- Data Set -----')
ds = data_loader.ORPAC(conf.path, conf.ft_dim, conf.num_samples, conf.seed)
ds.printSummary()
num_classes = len(ds.int2name())
im_dim = ds.imageShape()
# Build the shared layers and connect it to ORPAC.
print('\n----- Network Setup -----')
net = orpac_net.Orpac(sess, im_dim, conf.num_layers, num_classes, bw_init,
False)
sess.run(tf.global_variables_initializer())
print('Output feature map size: ', net.outputShape())
# Predict each image and produce a new image with BBoxes and labels in it.
try:
predictImagesInEpoch(net, ds, param.dst)
plt.show()
except KeyboardInterrupt:
print('User Abort')
def test_serialise(self):
""" Create a network and serialise its biases and weights."""
num_layers = 7
num_cls = 10
im_dim = Shape(3, 512, 512)
# Setup default network. Variables are random.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
# Serialise the network biases and weights.
data = net.serialise()
assert isinstance(data, dict)
assert set(data.keys()) == {'weight', 'bias', 'num-layers'}
assert set(data['bias'].keys()) == set(range(net.numLayers()))
assert set(data['weight'].keys()) == set(range(net.numLayers()))
assert data['num-layers'] == num_layers
# Verify the variables.
for i in range(net.numLayers()):
assert np.array_equal(net.getBias(i), data['bias'][i])
assert np.array_equal(net.getWeight(i), data['weight'][i])
def test_predict(self):
"""Ensure the 'predict' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when the provided parameters have the correct shape and
type.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create predictor-only network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
assert net.trainable() is not True
# Create dummy learning rate, image and training output.
img = np.random.randint(0, 256, im_dim.hwc()).astype(np.uint8)
# 'Train' method must complete without error and return the costs.
y = net.predict(img)
assert isinstance(y, np.ndarray) and y.dtype == np.float32
assert y.shape == net.outputShape().chw()
def test_weights_and_biases(self):
"""Create default network and test various accessor methods"""
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Create network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
# First layer must be compatible with input.
assert net.getBias(0).shape == (64, 1, 1)
assert net.getWeight(0).shape == (3, 3, net._xin.shape[1], 64)
# The last filter is responsible for creating the various features we
# train the network on. Its dimension must be 33x33 to achieve a large
# receptive field on the input image.
num_ft_chan = net.outputShape().chan
net.getBias(num_layers - 1).shape == (num_ft_chan, 1, 1)
net.getWeight(num_layers - 1).shape == (33, 33, 64, num_ft_chan)
# The output layer must have the correct number of features and
# feature map size. This excludes the batch dimension.
assert net.output().shape[1:] == net.outputShape().chw()
def test_restore(self):
""" Restore a network.
This test cannot be combined with `test_serialise` because of TFs
idiosyncrasies with (not) sharing Tensor names. Therefore, specify
dummy values for three layers, pass them to the Ctor, and verify the
values are correct.
"""
sess = self.sess
num_cls, num_layers = 10, 3
im_dim = Shape(3, 512, 512)
# Use utility functions to determine the number channels of the network
# output layer. Also determine the number of ...
num_ft_chan = orpac_net.Orpac.numOutputChannels(num_cls)
dim_xin = orpac_net.imageToWaveletDim(im_dim)
# Create variables for first, middle and last layer. The first layer
# must be adapted to the input, the middle layer is fixed and the last
# layer must encode the features (ie BBox, isFg, Class).
bw_init = {'bias': {}, 'weight': {}}
bw_init['bias'][0] = 0 * np.ones((64, 1, 1), np.float32)
bw_init['weight'][0] = 0 * np.ones((3, 3, dim_xin.chan, 64), np.float32)
bw_init['bias'][1] = 1 * np.ones((64, 1, 1), np.float32)
bw_init['weight'][1] = 1 * np.ones((3, 3, 64, 64), np.float32)
bw_init['bias'][2] = 2 * np.ones((num_ft_chan, 1, 1), np.float32)
bw_init['weight'][2] = 2 * np.ones((33, 33, 64, num_ft_chan), np.float32)
bw_init['num-layers'] = 3
# Create a new network and restore its weights.
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, bw_init, False)
sess.run(tf.global_variables_initializer())
# Ensure the weights are as specified.
for i in range(net.numLayers()):
assert np.array_equal(net.getBias(i), bw_init['bias'][i])
assert np.array_equal(net.getWeight(i), bw_init['weight'][i])
def setup_class(cls):
# Feature dimension will only be 2x2 to simplify testing and debugging.
ft_dim = Shape(None, 2, 2)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
# Create Tensorflow session and dummy network. The network is such that
# the feature size is only 2x2 because this makes testing easier.
cls.sess = tf.Session()
cls.net = orpac_net.Orpac(
cls.sess, im_dim, num_layers, num_cls, None, train=False)
assert cls.net.outputShape().hw() == (2, 2)
# A dummy feature tensor that we will populate it with our own data to
# simulate the network output. To create it we simply "clone" the
# genuine network output tensor.
cls.y_pred_in = tf.placeholder(tf.float32, cls.net.output().shape)
# Setup cost computation. This will create a node for `y_true`.
cls.total_cost = orpac_net.createCostNodes(cls.y_pred_in)
g = tf.get_default_graph().get_tensor_by_name
cls.y_true_in = g('orpac-cost/y_true:0')
def main():
param = parseCmdline()
sess = tf.Session()
# File names.
netstate_path = 'netstate'
os.makedirs(netstate_path, exist_ok=True)
fnames = {
'meta': os.path.join(netstate_path, 'orpac-meta.pickle'),
'orpac-net': os.path.join(netstate_path, 'orpac-net.pickle'),
'checkpt': os.path.join(netstate_path, 'tf-checkpoint.pickle'),
}
del netstate_path
# Restore the configuration if it exists, otherwise create a new one.
print('\n----- Simulation Parameters -----')
restore = os.path.exists(fnames['meta'])
if restore:
meta = pickle.load(open(fnames['meta'], 'rb'))
conf, log = meta['conf'], meta['log']
bw_init = pickle.load(open(fnames['orpac-net'], 'rb'))
else:
log = collections.defaultdict(list)
conf = config.NetConf(
seed=0, epoch=0, num_layers=7, path=os.path.join('data', '3dflight'),
ft_dim=Shape(None, 64, 64), num_samples=None
)
bw_init = None
print(f'Restored from <{None}>')
print('\n', conf)
# Load the BBox training data.
print('\n----- Data Set -----')
ds = data_loader.ORPAC(conf.path, conf.ft_dim, conf.num_samples, conf.seed)
ds.printSummary()
int2name = ds.int2name()
num_classes = len(int2name)
im_dim = ds.imageShape()
# Input/output/parameter tensors for network.
print('\n----- Network Setup -----')
# Create input tensor and trainable ORPAC net.
net = orpac_net.Orpac(sess, im_dim, conf.num_layers, num_classes, bw_init, True)
# Select cost function and optimiser, then initialise the TF graph.
sess.run(tf.global_variables_initializer())
# Ensure the network output shape matches the training output.
assert net.outputShape() == ds.featureShape()
print('Output feature map size: ', net.outputShape())
# Restore the network from Tensorflow's checkpoint file.
saver = tf.train.Saver()
if restore:
print('\nRestored Tensorflow graph from checkpoint file')
saver.restore(sess, fnames['checkpt'])
else:
print('Starting with untrained network')
print(f'\n----- Training for another {param.N} Epochs -----')
try:
epoch_ofs = conf.epoch + 1
lrates = np.logspace(np.log10(param.lr0), np.log10(param.lr1), param.N)
t0_all = time.time()
for epoch, lrate in enumerate(lrates):
t0_epoch = time.time()
tot_epoch = epoch + epoch_ofs
print(f'\nEpoch {tot_epoch} ({epoch+1}/{param.N} in this training cycle)')
ds.reset()
trainEpoch(ds, net, log, lrate)
# Save the network state and log data.
pickle.dump(net.serialise(), open(fnames['orpac-net'], 'wb'))
conf = conf._replace(epoch=epoch + epoch_ofs)
meta = {'conf': conf, 'int2name': int2name, 'log': log}
pickle.dump(meta, open(fnames['meta'], 'wb'))
saver.save(sess, fnames['checkpt'])
# Determine training time for epoch
etime = str(datetime.timedelta(seconds=int(time.time() - t0_epoch)))
et_h, et_m, et_s = etime.split(':')
etime_str = f' Training time: {et_h}h {et_m}m {et_s}s'
# Print basic stats about epoch.
print(f'{etime_str} Learning Rate: {lrate:.1E}')
etime = str(datetime.timedelta(seconds=int(time.time() - t0_all)))
et_h, et_m, et_s = etime.split(':')
print(f'\nTotal training time: {et_h}h {et_m}m {et_s}s\n')
except KeyboardInterrupt:
pass
