def __init__(self,
vector_dimension,
projected_dimension,
position,
masked=False,
dropout_percent=0.1):
'''
:param ni: Vector Dimension of embed layer - this will be the vector dimension from our embeding / number of
heads used in our multi-headed attention
:param no: Output Dimensionality of the model - this could be any reduction of vector dimension, make it % 0
for simplicity's sake
'''
self.ni = vector_dimension
self.no = projected_dimension
self.dropout_percent = dropout_percent
self.head_id = position
self.activation = 'linear'
#TODO: change from this layer strucutre to keep the persisent input X, and not copy inputs multiple times
# Refer to the drawing for residual connections
self.query_layer = layers.Layer(self.ni, self.no, self.activation)
self.key_layer = layers.Layer(self.ni, self.no, self.activation)
self.value_layer = layers.Layer(self.ni, self.no, self.activation)
self.dropout = layers.DropoutLayer(self.ni, self.no,
self.dropout_percent)
# TODO: add masking
if masked:
self.mask = np.ma.masked_array()
self.training_now = False
def __init__(self,
num_heads,
num_hiddens,
vector_dimension,
projected_dimension,
dropout=0.01):
self.attention_block = AttentionNeck(num_heads, vector_dimension,
projected_dimension, dropout)
self.projected_dimension = projected_dimension
#self.vector_dimension = vector_dimension
self.loss_func = 'mse'
self.activation = 'tanh'
self.nhs = num_hiddens
self.brain_1 = layers.Layer(self.projected_dimension, self.nhs,
self.activation)
self.brain_2 = layers.Layer(self.nhs, self.nhs, self.activation)
self.brain_3 = layers.Layer(self.nhs,
self.projected_dimension,
'linear',
is_output=True)
self.brain_layers = [self.brain_1, self.brain_2, self.brain_3]
self.RESIDUAL_X = np.array()
def make_standalone(self, n_classes, num_hidden):
self.standalone = True
self.brain_layers.append(
layers.Layer(self.projected_dimension, num_hidden,
self.activation))
self.brain_layers.append(
layers.Layer(num_hidden, n_classes, activation_type='softmax'))
self.loss_func = 'cross-entropy'
print('added final nonlinear and softmax output layer to model')
def __init__(self,
num_heads,
vector_dimension,
projected_dimension,
dropout=0.01):
'''
The attention "neck" is the connection structure around all multi-heads.
'''
self.num_heads = num_heads # 8
self.vector_dimension = vector_dimension
self.projected_dimension = projected_dimension
self.activation = 'linear'
# each attention head is shape (vectordim, projecteddim)
self.heads = []
pos = 0
# TODO: look into explicitly distributing the head processing?
while len(self.heads) < self.num_heads:
self.heads.append(
AttentionHead(vector_dimension=self.vector_dimension,
projected_dimension=self.projected_dimension,
position=pos,
masked=False,
dropout_percent=dropout))
pos += 1
# neck - single projection of all concatenated attention head outputs
#TODO: figure out where the issue is with layer size in bottleneck - could need -1???
self.bottleneck = layers.Layer(ni=projected_dimension * num_heads,
no=projected_dimension,
activation_type=self.activation)
def maxrects_single_layer_offline(superitems_pool,
pallet_dims,
superitems_in_layer=None):
"""
Given a superitems pool and the maximum dimensions to pack them into,
try to fit each superitem in a single layer (if not possible, return an error)
"""
logger.debug("MR-SL-Offline starting")
# Set all superitems in layer
if superitems_in_layer is None:
superitems_in_layer = np.arange(len(superitems_pool))
logger.debug(
f"MR-SL-Offline {superitems_in_layer}/{len(superitems_pool)} superitems to place"
)
# Iterate over each placement strategy
ws, ds, _ = superitems_pool.get_superitems_dims()
for strategy in MAXRECTS_PACKING_STRATEGIES:
# Create the maxrects packing algorithm
packer = newPacker(
mode=PackingMode.Offline,
bin_algo=PackingBin.Global,
pack_algo=strategy,
sort_algo=SORT_AREA,
rotation=False,
)
# Add one bin representing one layer
packer.add_bin(pallet_dims.width, pallet_dims.depth, count=1)
# Add superitems to be packed
for i in superitems_in_layer:
packer.add_rect(ws[i], ds[i], rid=i)
# Start the packing procedure
packer.pack()
# Feasible packing with a single layer
if len(packer) == 1 and len(packer[0]) == len(superitems_in_layer):
spool = superitems.SuperitemPool(
superitems=[superitems_pool[s.rid] for s in packer[0]])
layer = layers.Layer(
spool, [utils.Coordinate(s.x, s.y) for s in packer[0]],
pallet_dims)
logger.debug(
f"MR-SL-Offline generated a new layer with {len(layer)} superitems "
f"and {layer.get_density(two_dims=False)} 3D density")
return layer
return None
def build_layer_from_model_output(superitems_pool, superitems_in_layer,
solution, pallet_dims):
"""
Return a single layer from the given model solution (either baseline or column generation).
The 'solution' parameter should be a dictionary of the form
{
'c_{s}_x': ...,
'c_{s}_y: ...,
...
}
"""
spool, scoords = [], []
for s in superitems_in_layer:
spool += [superitems_pool[s]]
scoords += [Coordinate(x=solution[f"c_{s}_x"], y=solution[f"c_{s}_y"])]
spool = superitems.SuperitemPool(superitems=spool)
return layers.Layer(spool, scoords, pallet_dims)
def load(self, prefix):
"""
This function reads in the files beginning with prefix, assuming the structure
described in save(). This assumes that all of the files being loaded are in the
current working directory for simplicity.
"""
# Get current working directory (assuming that's where network files are)
cwd = os.getcwd()
# Get properties file
props_file = os.path.join(cwd, prefix + '-network-props.txt')
if os.path.isfile(props_file) is False:
raise ('Error, named network does not exist in current directory!')
self._layers = []
# Read properties and weights simultaneously
with open(props_file) as pf:
# Get the number of layers in the network
nlayers = int(pf.readline())
# Loop over the number of layers
for i in range(nlayers):
# Get the number of nodes in the layer
nNodes = int(pf.readline())
# Layer activation function
act = pf.readline().strip()
# Layer weight init method, in case of re-training
weight_init = pf.readline().strip()
# Get file for current layer's weights
wt_file = os.path.join(
cwd, prefix + '-network-layer-' + str(i) + '.npy')
if os.path.isfile(wt_file) is False:
raise ('Error, could not find a weights file!')
# Load weights file
weights = np.load(wt_file)
# Build layer
self._layers.append(layers.Layer(nNodes, act, weight_init))
self._layers[i]._weights = weights.copy()
# Set the activation, loss, and optimizer functions for the network
self.loss = 'SSE'
self.optimization = 'SGD'
self._lossFunction = None
self._optimizer = None
self.E_T = None
self._progress = None
self.learning_rate = 0.1
self._setLossFunction()
self._setOptimizer()
def maxrects_single_layer_online(superitems_pool,
pallet_dims,
superitems_duals=None):
"""
Given a superitems pool and the maximum dimensions to pack them into, try to fit
the greatest number of superitems in a single layer following the given order
"""
logger.debug("MR-SL-Online starting")
# If no duals are given use superitems' heights as a fallback
ws, ds, hs = superitems_pool.get_superitems_dims()
if superitems_duals is None:
superitems_duals = np.array(hs)
# Sort rectangles by duals
indexes = utils.argsort(list(zip(superitems_duals, hs)), reverse=True)
logger.debug(
f"MR-SL-Online {sum(superitems_duals[i] > 0 for i in indexes)} non-zero duals to place"
)
# Iterate over each placement strategy
generated_layers, num_duals = [], []
for strategy in MAXRECTS_PACKING_STRATEGIES:
# Create the maxrects packing algorithm
packer = newPacker(
mode=PackingMode.Online,
pack_algo=strategy,
rotation=False,
)
# Add one bin representing one layer
packer.add_bin(pallet_dims.width, pallet_dims.depth, count=1)
# Online packing procedure
n_packed, non_zero_packed, layer_height = 0, 0, 0
for i in indexes:
if superitems_duals[i] > 0 or hs[i] <= layer_height:
packer.add_rect(ws[i], ds[i], i)
if len(packer[0]) > n_packed:
n_packed = len(packer[0])
if superitems_duals[i] > 0:
non_zero_packed += 1
if hs[i] > layer_height:
layer_height = hs[i]
num_duals += [non_zero_packed]
# Build layer after packing
spool, coords = [], []
for s in packer[0]:
spool += [superitems_pool[s.rid]]
coords += [utils.Coordinate(s.x, s.y)]
layer = layers.Layer(superitems.SuperitemPool(spool), coords,
pallet_dims)
generated_layers += [layer]
# Find the best layer by taking into account the number of
# placed superitems with non-zero duals and density
layer_indexes = utils.argsort(
[(duals, layer.get_density(two_dims=False))
for duals, layer in zip(num_duals, generated_layers)],
reverse=True,
)
layer = generated_layers[layer_indexes[0]]
logger.debug(
f"MR-SL-Online generated a new layer with {len(layer)} superitems "
f"(of which {num_duals[layer_indexes[0]]} with non-zero dual) "
f"and {layer.get_density(two_dims=False)} 3D density")
return layer
def maxrects_multiple_layers(superitems_pool, pallet_dims, add_single=True):
"""
Given a superitems pool and the maximum dimensions to pack them into,
return a layer pool with warm start placements
"""
logger.debug("MR-ML-Offline starting")
logger.debug(
f"MR-ML-Offline {'used' if add_single else 'not_used'} as warm_start")
logger.debug(f"MR-ML-Offline {len(superitems_pool)} superitems to place")
# Return a layer with a single item if only one is present in the superitems pool
if len(superitems_pool) == 1:
layer_pool = layers.LayerPool(superitems_pool,
pallet_dims,
add_single=True)
uncovered = 0
else:
generated_pools = []
for strategy in MAXRECTS_PACKING_STRATEGIES:
# Build initial layer pool
layer_pool = layers.LayerPool(superitems_pool,
pallet_dims,
add_single=add_single)
# Create the maxrects packing algorithm
packer = newPacker(
mode=PackingMode.Offline,
bin_algo=PackingBin.Global,
pack_algo=strategy,
sort_algo=SORT_AREA,
rotation=False,
)
# Add an infinite number of layers (no upper bound)
packer.add_bin(pallet_dims.width,
pallet_dims.depth,
count=float("inf"))
# Add superitems to be packed
ws, ds, _ = superitems_pool.get_superitems_dims()
for i, (w, d) in enumerate(zip(ws, ds)):
packer.add_rect(w, d, rid=i)
# Start the packing procedure
packer.pack()
# Build a layer pool
for layer in packer:
spool, scoords = [], []
for superitem in layer:
spool += [superitems_pool[superitem.rid]]
scoords += [utils.Coordinate(superitem.x, superitem.y)]
spool = superitems.SuperitemPool(superitems=spool)
layer_pool.add(layers.Layer(spool, scoords, pallet_dims))
layer_pool.sort_by_densities(two_dims=False)
# Add the layer pool to the list of generated pools
generated_pools += [layer_pool]
# Find the best layer pool by considering the number of placed superitems,
# the number of generated layers and the density of each layer dense
uncovered = [
len(pool.not_covered_superitems()) for pool in generated_pools
]
n_layers = [len(pool) for pool in generated_pools]
densities = [
pool[0].get_density(two_dims=False) for pool in generated_pools
]
pool_indexes = utils.argsort(list(zip(uncovered, n_layers, densities)),
reverse=True)
layer_pool = generated_pools[pool_indexes[0]]
uncovered = uncovered[pool_indexes[0]]
logger.debug(
f"MR-ML-Offline generated {len(layer_pool)} layers with 3D densities {layer_pool.get_densities(two_dims=False)}"
)
logger.debug(
f"MR-ML-Offline placed {len(superitems_pool) - uncovered}/{len(superitems_pool)} superitems"
)
return layer_pool
training_inputs, training_outputs, validation_inputs, validation_outputs = audio.LoadAudioTrainingDataFromFile(
csv_file_name=file_name,
validation_size=25,
nmfcc=13,
nfft=4096,
output_type='spectrum')
elif mode == 'FEATURE':
training_inputs, training_outputs, validation_inputs, validation_outputs = audio.LoadAudioTrainingDataFromFile(
csv_file_name=file_name,
validation_size=25,
nmfcc=13,
output_type='mfcc')
#need to null activation layers
input_layer = layers.Layer(inputs=training_inputs.shape[0],
neurons=training_inputs.shape[1] + 1)
if mode == 'E2E':
hidden_layer = layers.Layer(
inputs=training_inputs.shape[1] + 1,
neurons=2048,
activation=activationfunctions.Tanh_Activation,
activation_derivative=activationfunctions.Tanh_Activation_Deriv)
hidden_layer.Initialize_Synaptic_Weights()
output_layer = layers.Layer(
inputs=2048,
neurons=training_outputs.shape[1],
activation=activationfunctions.Sigmoid_Activation,
activation_derivative=activationfunctions.
Sigmoid_Activation_Derivative)
test_ims = preprocess(test_ims)
# Convert the labels to a one-hot representation
train_labs = one_hot(train_labs)
# Set up the network. The original tensorflow tutorial used 128 nodes in the hidden
# layer, which I'm guessing is optimal. Doing that would take ~3.3 hours on my
# laptop and this (super unoptimized) code for 1 epoch. So, yeah. Slow. Also, they
# used the full 60,000 sample training set. I'm not going to do that as, again, slow.
# The purpose of this exercise is less to have a network that has been trained really
# well and more to gain a bit more experience about how a NN works on the gut level.
train_ims = train_ims[0:10000]
train_labs = train_labs[0:10000]
test_ims = test_ims[0:1000]
print('Constructing network...')
hidden_layer = layers.Layer(nNodes=32, activation='logistic')
output_layer = layers.Layer(nNodes=10, activation='softmax')
net_layers = [hidden_layer, output_layer]
net = network.Network(net_layers, learning_rate=0.1)
# Train the network
print('Training...')
net.train(train_ims, train_labs, epochs=5)
# Save the network
net.save('digits', 28 * 28)
# Load the network
#net = network.Network(load='digits')
# Test the network
print('Testing...')
net.test(test_ims, test_labs)