def test_imageToInput(self):
"""Uint8 image must becomes a valid network input."""
num_layers = 7
im_dim = Shape(3, 512, 512)
img = 100 * np.ones(im_dim.hwc(), np.uint8)
# Create a network (parameters do not matter).
net = orpac_net.Orpac(self.sess, im_dim, num_layers, 10, None, False)
# Image must be converted to float32 CHW image with leading
# batch dimension of 1. All values must have been divided by 255.
img_wl = net._imageToInput(img)
dim_xin = Shape(int(net._xin.shape[1]), *net.outputShape().hw())
assert img_wl.dtype == np.float32
assert img_wl.shape == (1, *dim_xin.chw())
def imageToWaveletDim(shape):
"""Return the image shape that resulted in a given wavelet feature map.
If the `shape.chan` is None then the returned Shape will have its `chan`
attribute set to None as well. If it is not None then it must be 3 since
the Orpac class only accepts RGB images.
Input:
shape: Shape
Shape: shape of input image.
Return:
Shape of wavelet decomposed tensor.
"""
N = Orpac._NUM_WAVELET_DECOMPOSITIONS
width = shape.width // (2**N)
height = shape.height // (2**N)
if shape.chan is not None:
assert shape.chan == 3
# Every decomposition yields 4 images -> 4 ** N images. Since we have to
# repeat the decomposition for each of the three RGB channels we get a
# total of 3 * (4 ** N) channels.
chan = 3 * (4**N)
else:
chan = None
return Shape(chan=chan, height=height, width=width)
def test_nonMaxSuppression(self):
"""Ensure the 'nonMaxSuppression' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when provided with valid parameters shapes and types.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create predictor network (parameters do not matter).
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
# Dummy input for NMS.
N = 100
bb_rects = np.random.uniform(-10, 10, (N, 4)).astype(np.float32)
scores = np.random.uniform(0, 1, N).astype(np.float32)
# Function must return a list of integers. Each integer is an index
# into the first bb_rects dimension to indicate which ones survived the
# NMS operation.
out = net.nonMaxSuppression(bb_rects, scores)
assert out.dtype == np.int32
assert 0 <= len(out) < N
# All indices must be unique.
assert len(set(out)) == len(out)
def test_not_trainable(self, m_cost):
"""Create an untrainable network.
This is purley the feed forward network for prediction purposes and
contains no optimisation nodes.
"""
sess = self.sess
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Must not create cost nodes.
assert not m_cost.called
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, None, train=False)
assert not m_cost.called
# Further sanity checks.
assert net.trainable() is False
assert net.costNodes() == {}
# Training must be impossible.
with pytest.raises(AssertionError):
net.train(None, None, None, None, None, None)
del net
# Must create cost nodes.
m_cost.return_value = (None, None)
assert not m_cost.called
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, None, train=True)
assert m_cost.called
# Network must consider itself trainable.
assert net.trainable() is True
def test_train(self):
"""Ensure the 'train' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when the provided parameters have the correct shape and
type.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create trainable network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, True)
self.sess.run(tf.global_variables_initializer())
assert net.trainable() is True
# Create dummy learning rate, image and training output.
lrate = 1E-5
ft_dim = net.outputShape()
y = np.random.uniform(0, 256, ft_dim.chw()).astype(np.uint8)
img = np.random.randint(0, 256, im_dim.hwc()).astype(np.uint8)
# Create dummy masks.
mask_cls = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
mask_bbox = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
mask_isFg = np.random.randint(0, 2, ft_dim.hw()).astype(np.float32)
# 'Train' method must complete without error and return the costs.
costs = net.train(img, y, lrate, mask_cls, mask_bbox, mask_isFg)
assert isinstance(costs, dict)
assert set(costs.keys()) == {'cls', 'bbox', 'isFg', 'total'}
def test_basic_attributes(self):
"""Setup network and check basic parameters like TF variable names,
number of layers, size of last feature map...
"""
ft_dim = Shape(None, 64, 64)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
assert net.session() is self.sess
# The feature size must be 1/8 of the image size because the network
# downsamples every second layer, and we specified 7 layers.
assert num_layers == net.numLayers() == 7
assert net.outputShape().hw() == ft_dim.hw()
# Ensure we can query all biases and weights. Also verify the data type
# inside the network.
g = tf.get_default_graph().get_tensor_by_name
for i in range(num_layers):
# These must exist in the graph.
assert g(f'orpac/W{i}:0') is not None
assert g(f'orpac/b{i}:0') is not None
assert net.getBias(i).dtype == np.float32
assert net.getWeight(i).dtype == np.float32
def waveletToImageDim(shape):
"""Return the wavelet feature shape for a given image shape.
If the `shape.chan` is None then the returned Shape will have its `chan`
attribute set to None as well. If it is not None then it must have the
correct value based on the number of Wavelet decompositions specified in
the class variables of `Orpac`.
Input:
shape: Shape
Shape of wavelet decomposed tensor.
Return:
Shape: shape of input image.
"""
N = Orpac._NUM_WAVELET_DECOMPOSITIONS
# The number of Wavelet decomposed channels is constant for a given
# number of Wavelet decompositions and colour channels. Here we ensure that
# the input does indeed make sense if it was specified.
if shape.chan is not None:
assert shape.chan == 3 * (4**N)
chan = 3
else:
chan = None
width = shape.width * (2**N)
height = shape.height * (2**N)
return Shape(chan=chan, height=height, width=width)
def test_feature_sizes(self):
"""Ensure all the various size getters agree."""
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
# The network must report the correct number of classes.
assert num_cls == net.numClasses()
# The feature channels encode 4 BBox parameters, is-foreground (2
# parameters because the binary choice is hot-label encoded), and the
# number of classes. This convenience function must return that value.
assert net.numOutputChannels(num_cls) == (4 + 2 + num_cls)
# Must return the output tensor shape excluding batch dimension.
ft_shape = net.outputShape()
assert isinstance(ft_shape, Shape)
assert ft_shape.chan == net.numOutputChannels(num_cls)
# Verify the image shape expected by the class for eg `pred` and `train`.
assert net.imageShape() == im_dim
# The input tensor shape for the network differs from the image shape
# passed to the eg `train` and `predict` methods because it holds
# the Wavelet transformed image, not the original image itself.
wl_dim = orpac_net.imageToWaveletDim(im_dim)
assert net._xin.shape == (1, *wl_dim.chw())
# The feature map size must derive from the size of the input image and
# the number of Wavelet transforms. Each WL transform halves the
# dimensions.
assert net.outputShape().hw() == wl_dim.hw()
def __init__(self, sess, im_dim, num_layers, num_classes, bw_init, train):
# Decide if we want to create cost nodes or not.
assert isinstance(train, bool)
# Backup basic variables.
self._trainable = train
self.sess = sess
self.num_layers = num_layers
self.num_classes = num_classes
self.im_dim = im_dim
# Create placeholder variable for Wavelet decomposed image.
self._xin = self._createInputTensor(im_dim)
# Setup the NMS nodes and Orpac network.
self._setupNonMaxSuppression()
with tf.variable_scope('orpac'):
self.out = self._setupNetwork(self._xin, bw_init, np.float32)
# Store shape of the output tensor.
self.ft_dim = Shape(*self.out.shape.as_list()[1:])
# Define the cost nodes and compile them into a dictionary if this
# network is trainable, otherwise do nothing.
if self._trainable:
self._cost_nodes, self._optimiser = self._addOptimiser()
else:
self._cost_nodes, self._optimiser = {}, None
def test_addOptimiser(self):
"""Create an trainable network and ensure the cost nodes exist.
"""
train = True
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Must call 'cost' function to create cost nodes.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, train)
# Network must identify itself as trainable and return the cost nodes.
assert net.trainable() is True
assert set(net.costNodes().keys()) == {'cls', 'bbox', 'isFg', 'total'}
def test_feature_to_image_conversions(self):
ft_dim = Shape(chan=None, height=2, width=2)
# The two methods must be inverses of each other.
im_dim = orpac_net.waveletToImageDim(ft_dim)
assert orpac_net.imageToWaveletDim(im_dim) == ft_dim
# Manually check the value as well. For each Wavelet decomposition of
# the original input image the dimensions must have been halved.
# Conversely, for any given feature map size, the corresponding image
# size must be 2 ** num_decompositions in each dimension.
rat = 2 ** orpac_net.Orpac._NUM_WAVELET_DECOMPOSITIONS
assert im_dim.chw() == (None, ft_dim.height * rat, ft_dim.width * rat)
def setup_class(cls):
# Feature dimension will only be 2x2 to simplify testing and debugging.
ft_dim = Shape(None, 64, 64)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
# Create Tensorflow session and dummy network. The network is such that
# the feature size is only 2x2 because this makes testing easier.
cls.sess = tf.Session()
cls.net = orpac_net.Orpac(
cls.sess, im_dim, num_layers, num_cls, None, train=False)
assert cls.net.outputShape().hw() == ft_dim.hw()
def test_non_max_suppresion_setup(self):
"""Ensure the network creates the NMS nodes."""
im_dim = Shape(3, 512, 512)
g = tf.get_default_graph().get_tensor_by_name
# NMS nodes must not yet exist.
try:
assert g('non-max-suppression/op:0') is not None
except KeyError:
pass
# Create a network (parameters do not matter).
orpac_net.Orpac(self.sess, im_dim, 7, 10, None, False)
# All NMS nodes must now exist.
assert g('non-max-suppression/op:0') is not None
assert g('non-max-suppression/scores:0') is not None
assert g('non-max-suppression/bb_rects:0') is not None
def test_serialise(self):
""" Create a network and serialise its biases and weights."""
num_layers = 7
num_cls = 10
im_dim = Shape(3, 512, 512)
# Setup default network. Variables are random.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
# Serialise the network biases and weights.
data = net.serialise()
assert isinstance(data, dict)
assert set(data.keys()) == {'weight', 'bias', 'num-layers'}
assert set(data['bias'].keys()) == set(range(net.numLayers()))
assert set(data['weight'].keys()) == set(range(net.numLayers()))
assert data['num-layers'] == num_layers
# Verify the variables.
for i in range(net.numLayers()):
assert np.array_equal(net.getBias(i), data['bias'][i])
assert np.array_equal(net.getWeight(i), data['weight'][i])
def main(data_path=None):
data_path = data_path or parseCmdline().fname
# Dummy Net configuration. We only fill in the values for the Loader.
conf = config.NetConf(seed=0,
epoch=None,
num_layers=None,
path=data_path,
ft_dim=Shape(None, 64, 64),
num_samples=None)
# Load the data set and request a sample.
t0 = time.time()
ds = data_loader.ORPAC(conf.path, conf.ft_dim, conf.num_samples, conf.seed)
etime = time.time() - t0
print(f'Loaded dataset in {etime:,.1f}s')
ds.printSummary()
train, uuid = ds.next()
plotMasksAndFeatures(train, ds.int2name(), conf.ft_dim)
plt.show()
def loadRawData(self, path, ft_dim, num_samples):
"""Return feature and label vector for data set of choice.
Returns:
im_dim: Shape
Image shape
ft_dim: Shape
Dimensions of training data.
int2name: dict[int:str]
A LUT to translate machine labels to human readable strings.
For instance {0: 'None', 1: 'Cube 0', 2: 'Cube 1'}.
train: N-List[TrainingSample]
Training data.
"""
# Compile a list of JPG images in the source folder. Then verify that
# a) each is a valid JPG file and b) all images have the same size.
fnames = self.findTrainingFiles(path, num_samples)
# Load and verify that the pickled meta data for each JPG file
# specifies the same set of class labels.
int2name = self.getLabelData(fnames)
num_cls = len(int2name)
# Compute the height and width that input images must have to be
# compatible with the selected output feature size.
im_dim = orpac_net.waveletToImageDim(Shape(None, *ft_dim.hw()))
# Fill in channel information: Images must always be RGB and the
# feature output channels are available via a utility method.
im_dim.chan = 3
ft_dim.chan = orpac_net.Orpac.numOutputChannels(num_cls)
# Compile all the features that have not been compiled already.
self.compileMissingFeatures(fnames, ft_dim)
# Load the compiled training data alongside each image.
train = self.loadTrainingData(fnames, im_dim, ft_dim, num_cls)
return im_dim, ft_dim, int2name, train
def test_predict(self):
"""Ensure the 'predict' method succeeds.
This test does not assess the numerical output but merely ensures the
method works when the provided parameters have the correct shape and
type.
"""
im_dim = Shape(3, 64, 64)
num_cls, num_layers = 10, 7
# Create predictor-only network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
assert net.trainable() is not True
# Create dummy learning rate, image and training output.
img = np.random.randint(0, 256, im_dim.hwc()).astype(np.uint8)
# 'Train' method must complete without error and return the costs.
y = net.predict(img)
assert isinstance(y, np.ndarray) and y.dtype == np.float32
assert y.shape == net.outputShape().chw()
def test_weights_and_biases(self):
"""Create default network and test various accessor methods"""
im_dim = Shape(3, 512, 512)
num_cls, num_layers = 10, 7
# Create network with random weights.
net = orpac_net.Orpac(self.sess, im_dim, num_layers, num_cls, None, False)
self.sess.run(tf.global_variables_initializer())
# First layer must be compatible with input.
assert net.getBias(0).shape == (64, 1, 1)
assert net.getWeight(0).shape == (3, 3, net._xin.shape[1], 64)
# The last filter is responsible for creating the various features we
# train the network on. Its dimension must be 33x33 to achieve a large
# receptive field on the input image.
num_ft_chan = net.outputShape().chan
net.getBias(num_layers - 1).shape == (num_ft_chan, 1, 1)
net.getWeight(num_layers - 1).shape == (33, 33, 64, num_ft_chan)
# The output layer must have the correct number of features and
# feature map size. This excludes the batch dimension.
assert net.output().shape[1:] == net.outputShape().chw()
def test_restore(self):
""" Restore a network.
This test cannot be combined with `test_serialise` because of TFs
idiosyncrasies with (not) sharing Tensor names. Therefore, specify
dummy values for three layers, pass them to the Ctor, and verify the
values are correct.
"""
sess = self.sess
num_cls, num_layers = 10, 3
im_dim = Shape(3, 512, 512)
# Use utility functions to determine the number channels of the network
# output layer. Also determine the number of ...
num_ft_chan = orpac_net.Orpac.numOutputChannels(num_cls)
dim_xin = orpac_net.imageToWaveletDim(im_dim)
# Create variables for first, middle and last layer. The first layer
# must be adapted to the input, the middle layer is fixed and the last
# layer must encode the features (ie BBox, isFg, Class).
bw_init = {'bias': {}, 'weight': {}}
bw_init['bias'][0] = 0 * np.ones((64, 1, 1), np.float32)
bw_init['weight'][0] = 0 * np.ones((3, 3, dim_xin.chan, 64), np.float32)
bw_init['bias'][1] = 1 * np.ones((64, 1, 1), np.float32)
bw_init['weight'][1] = 1 * np.ones((3, 3, 64, 64), np.float32)
bw_init['bias'][2] = 2 * np.ones((num_ft_chan, 1, 1), np.float32)
bw_init['weight'][2] = 2 * np.ones((33, 33, 64, num_ft_chan), np.float32)
bw_init['num-layers'] = 3
# Create a new network and restore its weights.
net = orpac_net.Orpac(sess, im_dim, num_layers, num_cls, bw_init, False)
sess.run(tf.global_variables_initializer())
# Ensure the weights are as specified.
for i in range(net.numLayers()):
assert np.array_equal(net.getBias(i), bw_init['bias'][i])
assert np.array_equal(net.getWeight(i), bw_init['weight'][i])
def setup_class(cls):
# Feature dimension will only be 2x2 to simplify testing and debugging.
ft_dim = Shape(None, 2, 2)
num_cls, num_layers = 10, 7
# Compute the image dimensions required for a 2x2 feature size.
im_dim = orpac_net.waveletToImageDim(ft_dim)
# Create Tensorflow session and dummy network. The network is such that
# the feature size is only 2x2 because this makes testing easier.
cls.sess = tf.Session()
cls.net = orpac_net.Orpac(
cls.sess, im_dim, num_layers, num_cls, None, train=False)
assert cls.net.outputShape().hw() == (2, 2)
# A dummy feature tensor that we will populate it with our own data to
# simulate the network output. To create it we simply "clone" the
# genuine network output tensor.
cls.y_pred_in = tf.placeholder(tf.float32, cls.net.output().shape)
# Setup cost computation. This will create a node for `y_true`.
cls.total_cost = orpac_net.createCostNodes(cls.y_pred_in)
g = tf.get_default_graph().get_tensor_by_name
cls.y_true_in = g('orpac-cost/y_true:0')
def main():
param = parseCmdline()
sess = tf.Session()
# File names.
netstate_path = 'netstate'
os.makedirs(netstate_path, exist_ok=True)
fnames = {
'meta': os.path.join(netstate_path, 'orpac-meta.pickle'),
'orpac-net': os.path.join(netstate_path, 'orpac-net.pickle'),
'checkpt': os.path.join(netstate_path, 'tf-checkpoint.pickle'),
}
del netstate_path
# Restore the configuration if it exists, otherwise create a new one.
print('\n----- Simulation Parameters -----')
restore = os.path.exists(fnames['meta'])
if restore:
meta = pickle.load(open(fnames['meta'], 'rb'))
conf, log = meta['conf'], meta['log']
bw_init = pickle.load(open(fnames['orpac-net'], 'rb'))
else:
log = collections.defaultdict(list)
conf = config.NetConf(
seed=0, epoch=0, num_layers=7, path=os.path.join('data', '3dflight'),
ft_dim=Shape(None, 64, 64), num_samples=None
)
bw_init = None
print(f'Restored from <{None}>')
print('\n', conf)
# Load the BBox training data.
print('\n----- Data Set -----')
ds = data_loader.ORPAC(conf.path, conf.ft_dim, conf.num_samples, conf.seed)
ds.printSummary()
int2name = ds.int2name()
num_classes = len(int2name)
im_dim = ds.imageShape()
# Input/output/parameter tensors for network.
print('\n----- Network Setup -----')
# Create input tensor and trainable ORPAC net.
net = orpac_net.Orpac(sess, im_dim, conf.num_layers, num_classes, bw_init, True)
# Select cost function and optimiser, then initialise the TF graph.
sess.run(tf.global_variables_initializer())
# Ensure the network output shape matches the training output.
assert net.outputShape() == ds.featureShape()
print('Output feature map size: ', net.outputShape())
# Restore the network from Tensorflow's checkpoint file.
saver = tf.train.Saver()
if restore:
print('\nRestored Tensorflow graph from checkpoint file')
saver.restore(sess, fnames['checkpt'])
else:
print('Starting with untrained network')
print(f'\n----- Training for another {param.N} Epochs -----')
try:
epoch_ofs = conf.epoch + 1
lrates = np.logspace(np.log10(param.lr0), np.log10(param.lr1), param.N)
t0_all = time.time()
for epoch, lrate in enumerate(lrates):
t0_epoch = time.time()
tot_epoch = epoch + epoch_ofs
print(f'\nEpoch {tot_epoch} ({epoch+1}/{param.N} in this training cycle)')
ds.reset()
trainEpoch(ds, net, log, lrate)
# Save the network state and log data.
pickle.dump(net.serialise(), open(fnames['orpac-net'], 'wb'))
conf = conf._replace(epoch=epoch + epoch_ofs)
meta = {'conf': conf, 'int2name': int2name, 'log': log}
pickle.dump(meta, open(fnames['meta'], 'wb'))
saver.save(sess, fnames['checkpt'])
# Determine training time for epoch
etime = str(datetime.timedelta(seconds=int(time.time() - t0_epoch)))
et_h, et_m, et_s = etime.split(':')
etime_str = f' Training time: {et_h}h {et_m}m {et_s}s'
# Print basic stats about epoch.
print(f'{etime_str} Learning Rate: {lrate:.1E}')
etime = str(datetime.timedelta(seconds=int(time.time() - t0_all)))
et_h, et_m, et_s = etime.split(':')
print(f'\nTotal training time: {et_h}h {et_m}m {et_s}s\n')
except KeyboardInterrupt:
pass