def load(name):
with open(name + "/p.pkl", "rb") as file:
print(file)
params = pickle.load(file)
layers = []
for index in range(params["no_of_layers"] - 1):
path = name + "/l" + str(index) + ".npy"
weights = np.load(path)
layers.append(
Layer(f_activation=params["f_activation"], weights=weights))
path = name + "/l" + str(params["no_of_layers"] - 1) + ".npy"
weights = np.load(path)
layers.append(
OutputLayer(f_activation=params["f_activation"],
f_cost=params["f_cost"],
weights=weights))
return Network(f_activation=params["f_activation"],
f_cost=params["f_cost"],
layers=layers)
def __init__(self, cfg):
self.model = cfg["model"]
self.cost = cfg["cost"]
self.struct = cfg["struct"]
self.activation = cfg["activation"]
# size is the number of weight matrices
self.size = len(cfg["struct"]) - 1
# init of layers
self.layers = []
layer_cfg = {
"model": cfg["model"],
"load_weights": cfg["load_weights"],
"activation": cfg["activation"],
"cost": cfg["cost"]
}
for l in range(self.size):
layer_cfg["shape"] = [
cfg["struct"][l], # input size
cfg["struct"][l + 1] # output size
]
layer_cfg["ID"] = l
if l == self.size - 1:
self.layers.append(OutputLayer(layer_cfg))
else:
self.layers.append(Layer(layer_cfg))
def Mininet_test():
from functions import Sigmoid, Quadratic
w_h = np.ones((2, 2)) * 0.5
w_o = np.ones((1, 3)) * 0.5
f_activation = Sigmoid
f_cost = Quadratic
layers = [
Layer(f_activation, weights=w_h),
OutputLayer(f_activation, f_cost, weights=w_o)
]
net = Network(f_activation, f_cost, layers=layers)
input = [1]
act, cost = net.train(input=input, target=input, learning_rate=1)
print(act, cost)
for l in net.layers:
print(l.delta_w)
def init_net_without_loading_data(self, config):
"""This should be called after loading all required data."""
self.config = config
if config.is_output and (not os.path.exists(config.output_dir)):
os.makedirs(config.output_dir)
[num_total_cases, input_dim] = self.train_data.X.shape
self.num_total_cases = num_total_cases
self.input_dim = input_dim
self.num_minibatches = num_total_cases / config.minibatch_size
if self.num_minibatches < 1:
self.num_minibatches = 1
# initialize the network
self.num_layers = config.num_layers
self.layer = []
in_dim = input_dim
for i in range(self.num_layers):
layer_spec = config.layer[i]
self.layer.append(Layer(
in_dim, layer_spec.out_dim, layer_spec.act_type,
layer_spec.weight_decay, layer_spec.weight_constraint,
layer_spec.dropout))
in_dim = layer_spec.out_dim
self.output = OutputLayer(in_dim, config.output.out_dim,
config.output.output_type, config.output.weight_decay,
config.output.weight_constraint, config.output.dropout)
# if not linear output (regression) load task loss function
if not isinstance(self.output.act_type, act.LinearOutput):
if config.task_loss_file != None:
self.task_loss = self.read_loss(config.task_loss_file)
print 'Loading task loss from %s' % config.task_loss_file
else:
self.task_loss = 1 - np.eye(self.train_data.K)
print 'No task loss specified, using 0-1 loss.'
# To use multi-class hinge output, a training loss function is required
if isinstance(self.output.act_type, act.MulticlassHingeOutput):
if config.train_loss_file != None:
self.train_loss = self.read_loss(config.train_loss_file)
print 'Loading surrogate loss from %s' % config.train_loss_file
else:
self.train_loss = 1 - np.eye(self.train_data.K)
print 'No surrogate loss specified, using 0-1 loss.'
self.output.act_type.set_loss(self.train_loss)
# initialize the weights in every layer
self._init_weights(config.init_scale, config.random_seed)
def __init_layers(self, layer_spec):
self.layers = []
last_index = len(layer_spec) - 1
for i, size in enumerate(layer_spec):
if i == 0:
self.layers.append(InputLayer(size, self.activation_fn))
elif i == last_index:
self.layers.append(OutputLayer(size, self.activation_fn))
else:
self.layers.append(HiddenLayer(size, self.activation_fn))
for i in range(len(self.layers) - 1):
self.__join_layers(self.layers[i], self.layers[i+1])
def __init__(self,
architecture=[784, 100, 10],
activation='sigmoid',
learning_rate=0.1,
momentum=0.5,
weight_decay=1e-4,
dropout=0.5,
early_stopping=True,
seed=99):
"""
Neural network model initializer.
"""
# Attributes
self.architecture = architecture
self.activation = activation
self.learning_rate = learning_rate
self.momentum = momentum
self.weight_decay = weight_decay
self.dropout = dropout
self.early_stopping = early_stopping
self.seed = seed
# Turn `activation` and `learning_rate` to class instances
if not isinstance(self.activation, Activation):
self.activation = Activation(self.activation)
if not isinstance(self.learning_rate, LearningRate):
self.learning_rate = LearningRate(self.learning_rate)
# Initialize a list of layers
self.layers = []
for i, (n_in,
n_out) in enumerate(zip(architecture[:-2],
architecture[1:-1])):
l = HiddenLayer('layer{}'.format(i), n_in, n_out, self.activation,
self.learning_rate, self.momentum,
self.weight_decay, self.dropout, self.seed + i)
self.layers.append(l)
# Output layer
n_in, n_out = architecture[-2], architecture[-1]
l = OutputLayer('output_layer', n_in, n_out, self.learning_rate,
self.momentum, self.weight_decay, self.dropout,
self.seed + i + 1)
self.layers.append(l)
# Training updates
self.epoch = 0
self.training_error = []
self.validation_error = []
self.training_loss = []
self.validation_loss = []
def create_layers(f_activation, f_cost, struct):
layers = []
for input, output in zip(struct, struct[1:-1]):
layer = Layer(f_activation, size=[output, input])
layers.append(layer)
layer = OutputLayer(f_activation,
f_cost,
size=[struct[-1], struct[-2]])
layers.append(layer)
return layers
def forward_time(self, ix, prev_cell):
"""Forward in time t given current input and previous hidden cell state."""
# FIXME: There maybe multiple layers here.
# Compute the hidden state
(hidden, output) = self.create_cell()
hidden.forward(self.Wz, self.Wi, self.Wf, self.Wo, self.Rz, self.Ri,
self.Rf, self.Ro, self.pi, self.pf, self.po, self.bz,
self.bi, self.bf, self.bo, ix, prev_cell[0].c,
prev_cell[0].h)
# Compute the output
output = OutputLayer(self.hidden_size)
output.forward(self.V, hidden.h, self.c)
return (hidden, output)
def init_net(self, config):
"""config is an instance of class Config"""
import os
self.config = config
if config.is_output and (not os.path.exists(config.output_dir)):
os.makedirs(config.output_dir)
self.train_data = self.read_data(config.train_data_file)
if config.is_val:
self.val_data = self.read_data(config.val_data_file)
if config.is_test:
self.test_data = self.read_data(config.test_data_file)
[num_total_cases, input_dim] = self.train_data.X.shape
self.num_total_cases = num_total_cases
self.input_dim = input_dim
self.num_minibatches = num_total_cases / config.minibatch_size
if self.num_minibatches < 1:
self.num_minibatches = 1
# initialize the network
self.num_layers = config.num_layers
self.layer = []
in_dim = input_dim
for i in range(0, self.num_layers):
self.layer.append(
Layer(in_dim, config.layer[i].out_dim,
config.layer[i].act_type))
in_dim = config.layer[i].out_dim
self.output = OutputLayer(in_dim, config.output.out_dim,
config.output.output_type)
# To use multi-class hinge output, we need to specify the loss function
if isinstance(self.output.act_type, act.MulticlassHingeOutput):
if config.loss_file != None:
self.output.act_type.set_loss(self.read_loss(config.loss_file))
else:
self.output.act_type.set_loss(1 - np.eye(self.train_data.K))
# initialize the weights in every layer
self._init_weights(config.init_scale, config.random_seed)
def __init__(self,
model,
struct,
activation="sigmoid",
cost="quadratic",
load_weights=False,
weight_range=[-1, 1],
bias_range=[0.5, 0.95]):
self.model = model
self.struct = struct
self.activation = activation
self.cost = cost
# size is the number of weight matrices
self.size = len(self.struct) - 1
# init of layers
self.layers = []
for l in range(self.size):
if l == self.size - 1:
self.layers.append(
OutputLayer(model=self.model,
ID=l,
activation=self.activation,
load_weights=load_weights,
in_size=self.struct[l],
out_size=self.struct[l + 1],
weight_range=weight_range,
bias_range=bias_range,
cost=self.cost))
else:
self.layers.append(
Layer(model=self.model,
ID=l,
activation=self.activation,
load_weights=load_weights,
in_size=self.struct[l],
out_size=self.struct[l + 1],
weight_range=weight_range,
bias_range=bias_range))
def main():
sess = tf.Session()
image = read_image('../data/heart.jpg')
image = np.reshape(image, [1, 224, 224, 3]) # type numpy.ndarray
image.astype(np.float32)
parser = Parser('../data/alexnet.cfg')
network_builder = NetworkBuilder("test") # type: NetworkBuilder
network_builder.set_parser(parser)
network = network_builder.build() # type: Network
network.add_input_layer(InputLayer(tf.float32, [None, 224, 224, 3]))
network.add_output_layer(OutputLayer())
network.connect_each_layer()
sess.run(tf.global_variables_initializer())
fc_layer = sess.run(network.output, feed_dict={network.input: image})
def setUpClass( self ):
from opencl import OpenCL
from layer import InputLayer, OutputLayer, ExecutionContext
self.ocl = OpenCL( pyopencl.create_some_context() )
self.i = InputLayer( 2, self.ocl )
self.o = OutputLayer( 1, self.ocl )
self.i.link_next( self.o )
self.nnc = ExecutionContext( self.i, self.o, allow_training = True )
self.i.set_weights( numpy.array( [ 0.1 ] * self.i.weights_count, numpy.float32 ) )
self.o.set_weights( numpy.array( [ 0.3 ] * self.o.weights_count, numpy.float32 ) )
self.tr = TrainingResults()
self._create_method()
def test():
data = pd.read_csv("train.csv").values
x = data[:, 1:]
t = np.identity(10, dtype=np.uint8)[data[:, 0]]
x = (x - x.min()) / x.max()
x = (x - x.mean()) / x.std()
validate = int(x.shape[0] * 0.75)
x_train, t_train = x[:validate], t[:validate]
x_test, t_test = x[validate:], t[validate:]
network = NeuralNetwork(784, error="R2error", optimizer="Gradient")
network.add(Layer(100, activation="Sigmoid"))
network.add(Layer(50, activation="Sigmoid"))
network.add(OutputLayer(10, activation="Softmax"))
network.fit(x_train, t_train, epoch_time=EPOCH_TIME, batch_size=BATCH_SIZE)
network.print_accurate(x_test, t_test)
def build_net_from_copy(self, copy):
"""
Rebuild the net from a copy made by make_copy.
"""
nnstore = copy
self.num_layers = len(nnstore.layer)
self.layer = []
for i in range(self.num_layers):
in_dim, out_dim = nnstore.layer[i].W.shape
new_layer = Layer(in_dim, out_dim, nnstore.layer[i].act_type)
new_layer.load_weight(nnstore.layer[i].W, nnstore.layer[i].b)
self.layer.append(new_layer)
in_dim, out_dim = nnstore.output.W.shape
new_layer = OutputLayer(in_dim, out_dim, nnstore.output.act_type)
new_layer.load_weight(nnstore.output.W, nnstore.output.b)
self.output = new_layer
if self.num_layers > 0:
self.input_dim = self.layer[0].W.shape[0]
else:
self.input_dim = self.output.W.shape[0]
epoch = 1
batch_size = 8
interval = 1 # 経過の表示間隔
n_sample = 7 # 誤差計測のサンプル数
n_flt = 6 # n_flt:フィルタ数
flt_h = 3 # flt_h:フィルタ高さ
flt_w = 3 # flt_w:フィルタ幅
# stride:ストライド幅, pad:パディング幅
# -- 各層の初期化 --
cl_1 = ConvLayer(img_ch, img_h, img_w, n_flt, flt_h, flt_w, 1, 1)
pl_1 = PoolingLayer(cl_1.y_ch, cl_1.y_h, cl_1.y_w, 2, 0)
n_fc_in = pl_1.y_ch * pl_1.y_h * pl_1.y_w
ml_1 = MiddleLayer(n_fc_in, 10)
ol_1 = OutputLayer(10, 10)
# -- 順伝播 --
def forward_propagation(x):
n_bt = x.shape[0]
print("forward_propagation()")
print(" x.shape", x.shape)
images = x.reshape(n_bt, img_ch, img_h, img_w)
print(" x.reshape", images.shape)
cl_1.forward(images)
pl_1.forward(cl_1.y)
print(" pl_1.y.shape", pl_1.y.shape, "バッチ数,フィルタ数,img縦,img横")
def create_cell(self):
# FIXME: Should create cells base on network structure
hidden = HiddenLayer(self.hidden_size)
output = OutputLayer(self.hidden_size)
return (hidden, output)
