def main():
mndata = MNIST('MNIST_data')
mnist_images_train, mnist_labels_train = mndata.load_training()
mnist_images_test, mnist_labels_test = mndata.load_testing()
rested_size = 60000
mnist_images_train = mnist_images_train[:rested_size]
mnist_labels_train = mnist_labels_train[:rested_size]
batch_size = 1000
hm_epochs = 1
input_layer_size = 784
hidden_layer_size = 500
output_layer_size = 10
nn = network.Network(input_layer_size, [
network.Layer(hidden_layer_size, input_layer_size, 'tanh'),
network.Layer(hidden_layer_size, hidden_layer_size, 'tanh'),
network.Layer(output_layer_size, hidden_layer_size)
])
train(nn, mnist_images_train, mnist_labels_train, batch_size, hm_epochs)
def train_network(neuron_list, epochs, mini_batch_size, learn_rate,
reg_lambda):
"""
Initializes a neural network and begins the SGD process
:param neuron_list: list with number of neurons per layer
:param epochs: number of training rounds
:param mini_batch_size: number of images to use per epoch
:param learn_rate: float that adjusts how fast the network converges
:param reg_lambda: float that determines network's self-regularization
:return: Network object representing the neural network
"""
training_data, validation_data, test_data = format_data()
neural_network = network.Network(neuron_list)
neural_network.stochastic_gradient_descent(training_data,
validation_data,
epochs,
mini_batch_size,
learn_rate,
reg_lambda,
TRAINING_DATA_SIZE,
test_data=test_data)
return neural_network
def train():
networks = network.Network()
datas = data.cifar_data()
output = networks.generator(config.img_depth, 1)
g_loss, d_loss = networks.loss(config.img_depth)
g_opt, d_opt = networks.optimizer(g_loss, d_loss)
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
for epoh in range(config.epoh):
for step in range(500):
X, Y = datas.next_batch()
X = X * 2 - 1
#img_show(X)
noise = np.random.uniform(-1,
1,
size=(config.batch_size,
config.noise_size))
_ = sess.run(g_opt,
feed_dict={
networks.x: X,
networks.y: noise
})
_ = sess.run(d_opt,
feed_dict={
networks.x: X,
networks.y: noise
})
if step % 10 == 0:
samples = get_samples(sess, networks, output)
img_show(samples)
loss_d = d_loss.eval({networks.x: X, networks.y: noise})
loss_g = g_loss.eval({networks.x: X, networks.y: noise})
print('\repoh: {}, steps: {}, d_loss: {}, g_loss: {}'.format(
epoh, step, loss_d, loss_g),
end="")
def initialize(self):
"""Here is were all the relative vars get initialized. """
# see if we can sniff out network info
netinfo = network.Network()
devices = netinfo.netdevices
active_devs = network.getActiveNetDevs()
if active_devs != []:
dev = devices[active_devs[0]]
try:
devname = dev.get("DEVICE")
ips = (isys.getIPAddresses(devname, version=4) +
isys.getIPAddresses(devname, version=6))
self.ip = ips[0]
log.info("IPs (using first) of device %s: %s" % (devname, ips))
if self.ip == "127.0.0.1" or self.ip == "::1":
self.ip = None
except Exception, e:
log.warning("Got an exception trying to get the self.ip addr "
"of %s: %s" % (devname, e))
def train(new_train = 1):
networks = network.Network()
datas = data.cifar_data()
output = networks.work()
predicts = tf.nn.softmax(output)
#print(predicts.shape)
predicts = tf.reshape(predicts,(config.batch_size,10))
predicts = tf.argmax(predicts, axis = 1)
actual_y = tf.argmax(networks.y, axis = 1)
#print(networks.y.shape)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicts,actual_y), dtype = tf.float32))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = output, labels = networks.y))
opt = tf.train.AdamOptimizer(learning_rate = 0.0001)
train_step = opt.minimize(loss)
saver = tf.train.Saver()
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
if new_train == 0:
saver.restore(sess,config.model_path + config.model_name)
print('Start training...')
for epoh in range(config.epoh):
all_loss = 0
all_acc = 0
for steps in range(500):
trainX,trainY = datas.next_batch()
#print(trainX.shape,trainY.shape)
_, losses, acc= sess.run([train_step, loss, accuracy],
feed_dict = {
networks.x:trainX,
networks.y:trainY})
all_loss = all_loss + losses
all_acc = all_acc + acc
print('\repoh:{}, step:{}, loss: {}, acc:{}'.format(epoh,steps,all_loss / (steps+1), all_acc / (steps+1)),end="")
print('\n')
saver.save(sess,os.path.join(config.model_path,config.model_name))
def __init__(self, save_path, is_verbose=False,
batch_size=1, class_counts=None,
smooth=350): # TODO, awni, setup way to x-val smoothing param
config_file = os.path.join(save_path, "config.json")
with open(config_file, 'r') as fid:
config = json.load(fid)
config['model']['batch_size'] = batch_size
self.model = network.Network(is_verbose)
self.graph = tf.Graph()
self.session = sess = tf.Session(graph=self.graph)
with self.graph.as_default():
self.model.init_inference(config['model'])
tf.global_variables_initializer().run(session=sess)
saver = tf.train.Saver(tf.global_variables())
saver.restore(sess, os.path.join(save_path, "best_model.epoch"))
if class_counts is not None:
counts = np.array(class_counts)[None, :]
total = np.sum(counts) + counts.shape[1]
self.prior = (counts + smooth) / total
else:
self.prior = None
def testOutputValue(self):
number_of_nodes = [2, 2, 1]
n = network.Network(number_of_nodes)
n.ClearValues()
output_layer = n.layers[2]
hidden_layer = n.layers[1]
input_layer = n.layers[0]
n.SetWeight(hidden_layer.nodes[0], input_layer.nodes[0], 0.23)
n.SetWeight(hidden_layer.nodes[0], input_layer.nodes[1], 0.18)
n.SetWeight(hidden_layer.nodes[1], input_layer.nodes[0], 0.47)
n.SetWeight(hidden_layer.nodes[1], input_layer.nodes[1], 0.14)
n.SetWeight(output_layer.nodes[0], hidden_layer.nodes[0], 0.56)
n.SetWeight(output_layer.nodes[0], hidden_layer.nodes[1], 0.71)
pattern = [3.5, 2.8]
n.SetPattern(pattern)
n.FeedForward(pattern)
retrieved = output_layer.nodes[0].value
expected = 0.56 * network.Sigmoid(1.31) + 0.71 * network.Sigmoid(2.04)
expected = network.Sigmoid(expected)
self.assertAlmostEqual(retrieved, expected, places=3)
def main():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network.Network([784, 30, 10])
if sys.argv < 2:
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
else:
net.SGD(training_data, 1, 1, float(sys.argv[1]), test_data=test_data)
while (True):
a = input("Ingrese una prueba: ")
#print(len(training_data))
#print(len(training_data[0][0]))
print(int(a))
l = net.feedforward(test_data[int(a)][0])
print("Respuesta dada")
for i, x in enumerate(l):
print(i, str(x * 100) + "%")
print("")
print("Respuesta verdadera")
print(test_data[int(a)][1])
detector = TargetDetector()
processor = TargetProcessor()
camera = VideoDevice()
#cmdinp = ["main.py", "-d", "0", "--no-networking", "--isDebug"]
#interface = CmdLineInterface(cmdinp)
interface = CmdLineInterface(sys.argv) #cmdline input
config = interface.getConfig(
) #AppConfig object that contains all values from cmdline inputs
gui = GUIManager()
networking = config.getIsNetworking()
if (networking):
global network1
network1 = network.Network()
network1.userServer()
#creates class instances
camera.startCapture(config.getDeviceID()) #inputs device id from config
if (config.getIsDebug()):
print("Camera is ready\n")
#gui.threshWindow()
loop = 1
while (cv2.waitKey(30) != 27):
print("While Loop " + str(loop) + " \n")
thIn = [53, 58, 5, 255, 228, 238] #thresholding values subject to change
detector.threshInputs(thIn)
import input_handler
import network
import network_metadata
network_structure = [155, 25, 1]
net = network.Network(network_structure)
net_tolerance = 0.6
net.layers = network_metadata.layers
net.biases = network_metadata.biases
net.weights = network_metadata.weights
# Main function to check for harassment
def is_harassment(phrase):
current_hash = input_handler.phrase_to_hash(phrase)
network_output = net.eval(current_hash)
if network_output[0] > net_tolerance:
return 1
else:
return 0
import load_word_data
# code for training network on MINST data
'''
training_data, validation_data, test_data = mnist_loader_python3.load_data_wrapper()
net = network.Network([784, 30, 10])
net.sgd(training_data, 30, 10, 3.0, test_data=test_data)
net.save()
'''
# code for training network on shape data
# the data used for this is from https://www.kaggle.com/smeschke/four-shapes/home#process_data.py
'''
training_data, test_data = load_image_data.load_data_wrapper('shape_data.npy')
net = network.Network([625, 30, 4])
net.sgd(training_data, 15, 10, 1.0, test_data)
net.save('shape_detection_network.json')
'''
# code for training network on language data
training_data, test_data = load_word_data.load_data_wrapper('word_data.npy')
net = network.Network([10*26, 30, 11])
net.sgd(training_data, 8, 10, 3.0, test_data)
net.save('language_detection_network.json')
def main1():
training_data, validation_data, test_data = mnist_loader.load_data_wrapper(
)
net = network.Network([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
### loading data
training_data, _, test_data = mnist_loader.load_data_wrapper()
#
# print("training data")
# print(type(training_data))
# print(len(training_data))
# print(training_data[0][0].shape)
# print(training_data[0][1].shape)
# print(training_data[0])
#
# print("test data")
# print(len(test_data))
begin = datetime.datetime.now()
# 设置每层神经元个数，第一层和最后一层必须是784和10
net = network.Network([784, 30, 10], cost=network.CrossEntropyCost)
# 参数，第一个参数表示训练集，第二个参数开始分别表示 epcho, mini batch size，eta学习率，lmbda正则化参数
net.SGD(training_data, 30, 10, 0.5, 5.0, evaluation_data=test_data)
end = datetime.datetime.now()
print "训练耗时：", (end - begin)
print
print "最优训练集错误率为：%.4f" % net.low_training_error
print "最优测试集错误率为：%.4f" % net.low_test_error
'''
# 导入一张图片，输出预测结果
one_picture_data = Image.open(" ")
one_picture_input = np.reshape(one_picture_data, (784, 1))
print "this picture is %s"%net.feedforward(one_picture_input)
'''
def __init__(self, layer_sizes, n_samples, alpha, learning_rate, v_prior, batch_size, X_train, y_train, N_train, \
X_val, y_val, N_val, mean_y_train, std_y_train):
# 29 October 17:55 - y_min and y_max added as arguments on 23rd October. Instance variables or just supply to the function?
# 11 November 08:12 - They will need to be instance variables of the class.
layer_sizes[0] = layer_sizes[0] + 1
self.batch_size = batch_size
self.N_train = N_train
self.X_train = X_train
self.y_train = y_train
# 11 November 08:17, removing y_min and y_max as parameters to the constructor above and creating them as instance variables here.
self.y_min = np.min(y_train.eval())
self.y_max = np.max(y_train.eval())
self.N_val = N_val
self.X_val = X_val
self.y_val = y_val
self.mean_y_train = mean_y_train
self.std_y_train = std_y_train
self.n_samples = n_samples
# We create the network
self.network = network.Network(layer_sizes, n_samples, v_prior,
N_train)
# index to a batch
index = T.lscalar()
# We create the input and output variables. The input will be a minibatch replicated n_samples times
self.x = T.matrix('x')
self.y = T.matrix('y', dtype='float32')
# The logarithm of the values for the likelihood factors
ll_train = self.network.log_likelihood_values(self.x, self.y, 0.0, 1.0)
ll_val = self.network.log_likelihood_values(self.x, self.y,
mean_y_train, std_y_train)
# The energy function for black-box alpha
self.estimate_marginal_ll = -1.0 * N_train / (self.x.shape[0] * alpha) * \
T.sum(LogSumExp(alpha * (T.sum(ll_train, 2) - self.network.log_f_hat()), 0) + \
T.log(1.0 / n_samples)) - self.network.log_normalizer_q() + \
self.network.log_Z_prior()
# We create a theano function for updating q
self.process_minibatch = theano.function([index], self.estimate_marginal_ll, \
updates=adam(self.estimate_marginal_ll, self.network.params,
learning_rate), \
givens={
self.x: self.X_train[index * batch_size: (index + 1) * batch_size],
self.y: self.y_train[
index * batch_size: (index + 1) * batch_size]})
# We create a theano function for making predictions
self.error_minibatch_train = theano.function(
[index],
T.sum(
(T.mean(self.network.output(self.x), 0, keepdims=True)[0, :, :]
- self.y)**2) / layer_sizes[-1],
givens={
self.x:
self.X_train[index * batch_size:(index + 1) * batch_size],
self.y:
self.y_train[index * batch_size:(index + 1) * batch_size]
})
self.error_minibatch_val = theano.function(
[index],
T.sum(
(T.mean(self.network.output(self.x), 0, keepdims=True)[0, :, :]
* std_y_train + mean_y_train - self.y)**2) / layer_sizes[-1],
givens={
self.x:
self.X_val[index * batch_size:(index + 1) * batch_size],
self.y: self.y_val[index * batch_size:(index + 1) * batch_size]
})
self.ll_minibatch_val = theano.function([index], T.sum(LogSumExp(T.sum(ll_val, 2), 0) + T.log(1.0 / n_samples)), \
givens={
self.x: self.X_val[index * batch_size: (index + 1) * batch_size],
self.y: self.y_val[index * batch_size: (index + 1) * batch_size]})
self.ll_minibatch_train = theano.function([index],
T.sum(LogSumExp(T.sum(ll_train, 2), 0) + T.log(1.0 / n_samples)), \
givens={self.x: self.X_train[
index * batch_size: (index + 1) * batch_size],
self.y: self.y_train[
index * batch_size: (index + 1) * batch_size]})
self.target_lbfgs_grad = theano.function([],
T.grad(
self.estimate_marginal_ll,
self.network.params),
givens={
self.x: self.X_train,
self.y: self.y_train
})
self.target_lbfgs_objective = theano.function(
[],
self.estimate_marginal_ll,
givens={
self.x: self.X_train,
self.y: self.y_train
})
# 26 November 16:51 - Will subbing prediction in directly change anything?
prediction = self.network.output(
self.x) * std_y_train[None, None, :] + mean_y_train[None, None, :]
self.predict = theano.function([self.x], prediction)
self.function_grid = theano.function([self.x], prediction[0, :, 0])
self.function_scalar = theano.function([self.x], prediction[0, 0, 0])
self.function_scalar_gradient = theano.function(
[self.x], T.grad(prediction[0, 0, 0], self.x))
self.network.update_randomness()
import readtr as rt
import readf as rd
import network as nt
import numpy as np
n = nt.Network([2500, 30, 30, 10])
a, b = rt.readnn(0, 62)
def init(i=0, j=62):
global n
global a
global b
a, b = rt.readnn(i, j)
len = j - i
n = nt.Network([2500, 30, 30, len])
print "Training\n"
n.SGD(a, 10, 5, 3.0, b)
def main(testfile=None):
global n
global a
global b
if testfile:
tr = rd.read(testfile)
ti = np.array(tr)
tir = [np.reshape(x, (2500, 1)) for x in ti]
return (n.evaluate(tir))
else:
n.SGD(a, 20, 20, 3, 0, c)
###############################################################################
# Import the necessary modules
import time
import numpy as np
import network
from network_params import net_dict
from sim_params import sim_dict
from stimulus_params import stim_dict
###############################################################################
# Initialize the network and pass parameters to it.
tic = time.time()
net = network.Network(sim_dict, net_dict, stim_dict)
toc = time.time() - tic
print("Time to initialize the network: %.2f s" % toc)
# Connect all nodes.
tic = time.time()
net.setup()
toc = time.time() - tic
print("Time to create the connections: %.2f s" % toc)
# Simulate.
tic = time.time()
net.simulate()
toc = time.time() - tic
print("Time to simulate: %.2f s" % toc)
###############################################################################
# Plot a raster plot of the spikes of the simulated neurons and the average
def train_gan(separate_funcs=False,
D_training_repeats=1,
G_learning_rate_max=0.0010,
D_learning_rate_max=0.0010,
G_smoothing=0.999,
adam_beta1=0.0,
adam_beta2=0.99,
adam_epsilon=1e-8,
minibatch_default=16,
minibatch_overrides={},
rampup_kimg=40,
rampdown_kimg=0,
lod_initial_resolution=4,
lod_training_kimg=400,
lod_transition_kimg=400,
total_kimg=10000,
dequantize_reals=False,
gdrop_beta=0.9,
gdrop_lim=0.5,
gdrop_coef=0.2,
gdrop_exp=2.0,
drange_net=[-1, 1],
drange_viz=[-1, 1],
image_grid_size=None,
tick_kimg_default=50,
tick_kimg_overrides={
32: 20,
64: 10,
128: 10,
256: 5,
512: 2,
1024: 1
},
image_snapshot_ticks=4,
network_snapshot_ticks=40,
image_grid_type='default',
resume_network_pkl=None,
resume_kimg=0.0,
resume_time=0.0):
# Load dataset and build networks.
training_set, drange_orig = load_dataset()
if resume_network_pkl:
print 'Resuming', resume_network_pkl
G, D, _ = misc.load_pkl(
os.path.join(config.result_dir, resume_network_pkl))
else:
G = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.G)
D = network.Network(num_channels=training_set.shape[1],
resolution=training_set.shape[2],
label_size=training_set.labels.shape[1],
**config.D)
Gs = G.create_temporally_smoothed_version(beta=G_smoothing,
explicit_updates=True)
misc.print_network_topology_info(G.output_layers)
misc.print_network_topology_info(D.output_layers)
# Setup snapshot image grid.
if image_grid_type == 'default':
if image_grid_size is None:
w, h = G.output_shape[3], G.output_shape[2]
image_grid_size = np.clip(1920 / w, 3,
16), np.clip(1080 / h, 2, 16)
example_real_images, snapshot_fake_labels = training_set.get_random_minibatch(
np.prod(image_grid_size), labels=True)
snapshot_fake_latents = random_latents(np.prod(image_grid_size),
G.input_shape)
elif image_grid_type == 'category':
W = training_set.labels.shape[1]
H = W if image_grid_size is None else image_grid_size[1]
image_grid_size = W, H
snapshot_fake_latents = random_latents(W * H, G.input_shape)
snapshot_fake_labels = np.zeros((W * H, W),
dtype=training_set.labels.dtype)
example_real_images = np.zeros((W * H, ) + training_set.shape[1:],
dtype=training_set.dtype)
for x in xrange(W):
snapshot_fake_labels[x::W, x] = 1.0
indices = np.arange(
training_set.shape[0])[training_set.labels[:, x] != 0]
for y in xrange(H):
example_real_images[x + y * W] = training_set.h5_lods[0][
np.random.choice(indices)]
else:
raise ValueError('Invalid image_grid_type', image_grid_type)
# Theano input variables and compile generation func.
print 'Setting up Theano...'
real_images_var = T.TensorType('float32', [False] *
len(D.input_shape))('real_images_var')
real_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('real_labels_var')
fake_latents_var = T.TensorType('float32', [False] *
len(G.input_shape))('fake_latents_var')
fake_labels_var = T.TensorType(
'float32', [False] * len(training_set.labels.shape))('fake_labels_var')
G_lrate = theano.shared(np.float32(0.0))
D_lrate = theano.shared(np.float32(0.0))
gen_fn = theano.function([fake_latents_var, fake_labels_var],
Gs.eval_nd(fake_latents_var,
fake_labels_var,
ignore_unused_inputs=True),
on_unused_input='ignore')
# Misc init.
resolution_log2 = int(np.round(np.log2(G.output_shape[2])))
initial_lod = max(
resolution_log2 - int(np.round(np.log2(lod_initial_resolution))), 0)
cur_lod = 0.0
min_lod, max_lod = -1.0, -2.0
fake_score_avg = 0.0
if config.D.get('mbdisc_kernels', None):
print 'Initializing minibatch discrimination...'
if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(initial_lod))
D.eval(real_images_var, deterministic=False, init=True)
init_layers = lasagne.layers.get_all_layers(D.output_layers)
init_updates = [
update for layer in init_layers
for update in getattr(layer, 'init_updates', [])
]
init_fn = theano.function(inputs=[real_images_var],
outputs=None,
updates=init_updates)
init_reals = training_set.get_random_minibatch(500, lod=initial_lod)
init_reals = misc.adjust_dynamic_range(init_reals, drange_orig,
drange_net)
init_fn(init_reals)
del init_reals
# Save example images.
snapshot_fake_images = gen_fn(snapshot_fake_latents, snapshot_fake_labels)
result_subdir = misc.create_result_subdir(config.result_dir,
config.run_desc)
misc.save_image_grid(example_real_images,
os.path.join(result_subdir, 'reals.png'),
drange=drange_orig,
grid_size=image_grid_size)
misc.save_image_grid(snapshot_fake_images,
os.path.join(result_subdir, 'fakes%06d.png' % 0),
drange=drange_viz,
grid_size=image_grid_size)
# Training loop.
cur_nimg = int(resume_kimg * 1000)
cur_tick = 0
tick_start_nimg = cur_nimg
tick_start_time = time.time()
tick_train_out = []
train_start_time = tick_start_time - resume_time
while cur_nimg < total_kimg * 1000:
# Calculate current LOD.
cur_lod = initial_lod
if lod_training_kimg or lod_transition_kimg:
tlod = (cur_nimg / 1000.0) / (lod_training_kimg +
lod_transition_kimg)
cur_lod -= np.floor(tlod)
if lod_transition_kimg:
cur_lod -= max(
1.0 + (np.fmod(tlod, 1.0) - 1.0) *
(lod_training_kimg + lod_transition_kimg) /
lod_transition_kimg, 0.0)
cur_lod = max(cur_lod, 0.0)
# Look up resolution-dependent parameters.
cur_res = 2**(resolution_log2 - int(np.floor(cur_lod)))
minibatch_size = minibatch_overrides.get(cur_res, minibatch_default)
tick_duration_kimg = tick_kimg_overrides.get(cur_res,
tick_kimg_default)
# Update network config.
lrate_coef = misc.rampup(cur_nimg / 1000.0, rampup_kimg)
lrate_coef *= misc.rampdown_linear(cur_nimg / 1000.0, total_kimg,
rampdown_kimg)
G_lrate.set_value(np.float32(lrate_coef * G_learning_rate_max))
D_lrate.set_value(np.float32(lrate_coef * D_learning_rate_max))
if hasattr(G, 'cur_lod'): G.cur_lod.set_value(np.float32(cur_lod))
if hasattr(D, 'cur_lod'): D.cur_lod.set_value(np.float32(cur_lod))
# Setup training func for current LOD.
new_min_lod, new_max_lod = int(np.floor(cur_lod)), int(
np.ceil(cur_lod))
if min_lod != new_min_lod or max_lod != new_max_lod:
print 'Compiling training funcs...'
min_lod, max_lod = new_min_lod, new_max_lod
# Pre-process reals.
real_images_expr = real_images_var
if dequantize_reals:
rnd = theano.sandbox.rng_mrg.MRG_RandomStreams(
lasagne.random.get_rng().randint(1, 2147462579))
epsilon_noise = rnd.uniform(size=real_images_expr.shape,
low=-0.5,
high=0.5,
dtype='float32')
real_images_expr = T.cast(
real_images_expr, 'float32'
) + epsilon_noise # match original implementation of Improved Wasserstein
real_images_expr = misc.adjust_dynamic_range(
real_images_expr, drange_orig, drange_net)
if min_lod > 0: # compensate for shrink_based_on_lod
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=2)
real_images_expr = T.extra_ops.repeat(real_images_expr,
2**min_lod,
axis=3)
# Optimize loss.
G_loss, D_loss, real_scores_out, fake_scores_out = evaluate_loss(
G, D, min_lod, max_lod, real_images_expr, real_labels_var,
fake_latents_var, fake_labels_var, **config.loss)
G_updates = adam(G_loss,
G.trainable_params(),
learning_rate=G_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
D_updates = adam(D_loss,
D.trainable_params(),
learning_rate=D_lrate,
beta1=adam_beta1,
beta2=adam_beta2,
epsilon=adam_epsilon).items()
# Compile training funcs.
if not separate_funcs:
GD_train_fn = theano.function([
real_images_var, real_labels_var, fake_latents_var,
fake_labels_var
], [G_loss, D_loss, real_scores_out, fake_scores_out],
updates=G_updates + D_updates +
Gs.updates,
on_unused_input='ignore')
else:
D_train_fn = theano.function([
real_images_var, real_labels_var, fake_latents_var,
fake_labels_var
], [G_loss, D_loss, real_scores_out, fake_scores_out],
updates=D_updates,
on_unused_input='ignore')
G_train_fn = theano.function(
[fake_latents_var, fake_labels_var], [],
updates=G_updates + Gs.updates,
on_unused_input='ignore')
# Invoke training funcs.
if not separate_funcs:
assert D_training_repeats == 1
mb_reals, mb_labels = training_set.get_random_minibatch(
minibatch_size,
lod=cur_lod,
shrink_based_on_lod=True,
labels=True)
mb_train_out = GD_train_fn(
mb_reals, mb_labels,
random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
cur_nimg += minibatch_size
tick_train_out.append(mb_train_out)
else:
for idx in xrange(D_training_repeats):
mb_reals, mb_labels = training_set.get_random_minibatch(
minibatch_size,
lod=cur_lod,
shrink_based_on_lod=True,
labels=True)
mb_train_out = D_train_fn(
mb_reals, mb_labels,
random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
cur_nimg += minibatch_size
tick_train_out.append(mb_train_out)
G_train_fn(random_latents(minibatch_size, G.input_shape),
random_labels(minibatch_size, training_set))
# Fade in D noise if we're close to becoming unstable
fake_score_cur = np.clip(np.mean(mb_train_out[1]), 0.0, 1.0)
fake_score_avg = fake_score_avg * gdrop_beta + fake_score_cur * (
1.0 - gdrop_beta)
gdrop_strength = gdrop_coef * (max(fake_score_avg - gdrop_lim, 0.0)**
gdrop_exp)
if hasattr(D, 'gdrop_strength'):
D.gdrop_strength.set_value(np.float32(gdrop_strength))
# Perform maintenance operations once per tick.
if cur_nimg >= tick_start_nimg + tick_duration_kimg * 1000 or cur_nimg >= total_kimg * 1000:
cur_tick += 1
cur_time = time.time()
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = cur_time - tick_start_time
tick_start_time = cur_time
tick_train_avg = tuple(
np.mean(np.concatenate([np.asarray(v).flatten()
for v in vals]))
for vals in zip(*tick_train_out))
tick_train_out = []
# Print progress.
print 'tick %-5d kimg %-8.1f lod %-5.2f minibatch %-4d time %-12s sec/tick %-9.1f sec/kimg %-6.1f Dgdrop %-8.4f Gloss %-8.4f Dloss %-8.4f Dreal %-8.4f Dfake %-8.4f' % (
(cur_tick, cur_nimg / 1000.0, cur_lod, minibatch_size,
misc.format_time(cur_time - train_start_time), tick_time,
tick_time / tick_kimg, gdrop_strength) + tick_train_avg)
# Visualize generated images.
if cur_tick % image_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
snapshot_fake_images = gen_fn(snapshot_fake_latents,
snapshot_fake_labels)
misc.save_image_grid(snapshot_fake_images,
os.path.join(
result_subdir,
'fakes%06d.png' % (cur_nimg / 1000)),
drange=drange_viz,
grid_size=image_grid_size)
# Save network snapshot every N ticks.
if cur_tick % network_snapshot_ticks == 0 or cur_nimg >= total_kimg * 1000:
misc.save_pkl(
(G, D, Gs),
os.path.join(
result_subdir,
'network-snapshot-%06d.pkl' % (cur_nimg / 1000)))
# Write final results.
misc.save_pkl((G, D, Gs), os.path.join(result_subdir, 'network-final.pkl'))
training_set.close()
print 'Done.'
with open(os.path.join(result_subdir, '_training-done.txt'), 'wt'):
pass
def main():
nameOfTest = input("What is the name of the test? ")
transformationsAddMore = transforms.Compose(
[transforms.RandomRotation(degrees=15),
transforms.ToTensor()])
trainingDataset = torch.utils.data.ConcatDataset([
datasets.MNIST(root='data/train',
transform=transformationsAddMore,
download=True),
datasets.MNIST(root='data/train', transform=transformationsAddMore),
datasets.MNIST(root='data/train', transform=transformationsAddMore)
])
testingDataset = datasets.MNIST(root='data/test',
transform=transformationsAddMore,
train=False,
download=True)
trainingDataLoader = torch.utils.data.DataLoader(trainingDataset,
batch_size=32,
shuffle=True)
testingDataLoader = torch.utils.data.DataLoader(
testingDataset, batch_size=len(testingDataset))
testingResults = []
for i in range(5):
net = network.Network()
optimizer = torch.optim.SGD(net.parameters(), lr=1e-02)
lossFunction = nn.CrossEntropyLoss()
print("Round {}".format(i + 1))
print("Training!")
net.train()
for epoch in range(0, 10):
print("Epoch {}".format(epoch))
for batchNum, (data, target) in enumerate(trainingDataLoader):
optimizer.zero_grad()
output = net(data)
loss = lossFunction(output, target)
loss.backward()
optimizer.step()
if batchNum % 50 == 0:
print(
"Training loss at epoch {} batch number {}: {}".format(
epoch, batchNum, loss))
print("Testing!")
net.eval()
correct = 0
for _, (data, target) in enumerate(testingDataLoader):
output = net(data)
pred = output.max(
1, keepdim=True)[1] # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
print("Number of correctness is {}".format(correct))
print("Percentage of correctness is {}%".format(
correct / len(testingDataset) * 100.))
testingResults.append(float(correct) / float(len(testingDataset)))
resultsFile = open("results.txt", mode="a+")
resultsFile.write(
"\n" + nameOfTest + ": " +
str(float(sum(testingResults)) / float(len(testingResults)) * 100) +
"%")
resultsFile.close()
# -*- coding: utf-8 -*-
import data
training_data,test_data,begin_end,index = data.load_data_wrapper()
import network
net = network.Network([1024,30,10])
d=1.4
for i in range(10):
d=d+0.1
print(d)
net.SGD(training_data,35,begin_end,index,d,test_data = test_data)
import cPickle
write_file = open('weight.pkl','wb')
cPickle.dump([net.weights[:],net.biases[:]],write_file,-1)
write_file.close()
for b in xrange(labels.shape[0]):
for c in xrange(labels.shape[1]):
if labels[b, c] > 0:
correct += 1 if labels[b, c] == index_output[b, c] else 0
count += 1
print "Percent correct = ", correct * 100.0 / count
collector = []
for b in xrange(alphas.shape[0]):
read_index = 0
converted = gen.indices_to_unicode(index_output[b])
read_word = u""
for c in xrange(alphas.shape[1]):
if alphas[b, c] > 0.5:
read_word = read_word + converted[read_index]
read_index = read_index + 1
print read_word
collector.append(read_word)
return collector
words, imgs = gen.get_tuples(range(100))
word_mat, img_mat = gen.prepare_input_tensors(words, imgs)
nn = ann.Network(img_mat.shape, word_mat.shape, gen.get_default_total_code(), 100)
nn.load_session("../artifacts/" + "test_weight")
eval(nn, img_mat, word_mat)
# raw_input()
def __init__(self, layer_sizes, n_samples, alpha, learning_rate, v_prior,
batch_size, X_train, y_train, N_train):
layer_sizes = copy.copy(layer_sizes)
layer_sizes[0] = layer_sizes[0] + 1
print layer_sizes
self.batch_size = batch_size
self.N_train = N_train
self.X_train = X_train
self.y_train = y_train
self.rate = learning_rate
# We create the network
self.network = network.Network(layer_sizes, n_samples, v_prior,
N_train)
# index to a batch
index = T.lscalar()
self.indexes = T.vector('index', dtype='int32')
indexes_train = theano.shared(value=np.array(range(0, N_train),
dtype=np.int32),
borrow=True)
self.x = T.tensor3('x', dtype=theano.config.floatX)
self.y = T.matrix('y', dtype=theano.config.floatX)
self.lr = T.fscalar()
# The logarithm of the values for the likelihood factors
sampl = T.bscalar()
self.fwpass = theano.function(outputs=self.network.output(
self.x, False, samples=sampl, use_indices=False),
inputs=[self.x, sampl],
allow_input_downcast=True)
ll_train = self.network.log_likelihood_values(self.x, self.y,
self.indexes, 0.0, 1.0)
self.estimate_marginal_ll = (-1.0 * N_train / (self.x.shape[ 1 ] * alpha) * \
T.sum(LogSumExp(alpha * (T.sum(ll_train, 2) - self.network.log_f_hat() - self.network.log_f_hat_z()), 0)+ \
T.log(1.0 / n_samples)) - self.network.log_normalizer_q() - 1.0 * N_train / self.x.shape[ 1 ] * self.network.log_normalizer_q_z() + \
self.network.log_Z_prior())
# We create a theano function for updating q
upd = adam(self.estimate_marginal_ll,
self.network.params,
indexes_train[index * batch_size:(index + 1) * batch_size],
self.rate,
rescale_local=np.float32(N_train / batch_size))
self.process_minibatch = theano.function([ index], self.estimate_marginal_ll, \
updates = upd, \
givens = { self.x: T.tile(self.X_train[ index * batch_size: (index + 1) * batch_size] , [ n_samples, 1, 1 ]),
self.y: self.y_train[ index * batch_size: (index + 1) * batch_size ],
self.indexes: indexes_train[ index * batch_size : (index + 1) * batch_size ] })
# We create a theano function for making predictions
self.error_minibatch_train = theano.function(
[index],
T.sum((T.mean(
self.network.output(self.x, self.indexes), 0,
keepdims=True)[0, :, :] - self.y)**2) / layer_sizes[-1],
givens={
self.x:
T.tile(
self.X_train[index * batch_size:(index + 1) * batch_size],
[n_samples, 1, 1]),
self.y:
self.y_train[index * batch_size:(index + 1) * batch_size],
self.indexes:
indexes_train[index * batch_size:(index + 1) * batch_size]
})
self.ll_minibatch_train = theano.function([ index ], T.sum(LogSumExp(T.sum(ll_train, 2), 0) + T.log(1.0 / n_samples)), \
givens = { self.x: T.tile(self.X_train[ index * batch_size: (index + 1) * batch_size ], [ n_samples, 1, 1 ]),
self.y: self.y_train[ index * batch_size: (index + 1) * batch_size ],
self.indexes: indexes_train[ index * batch_size : (index + 1) * batch_size ] })
import network import data training_data, test_data = data.create_data() net = network.Network([2500, 50, 2]) net.SGD(training_data, 30, 10, 2.0, test_data=test_data)
import mnist_loader
import network
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
net_1 = network.Network([784, 30, 10])
net_1.SGD(training_data,
30,
10,
3.0,
'bs10_lr3.0_784x30x10.csv',
test_data=test_data)
net_2 = network.Network([784, 128, 10])
net_2.SGD(training_data,
30,
10,
3.0,
'bs10_lr3.0_784x128x10.csv',
test_data=test_data)
net_3 = network.Network([784, 128, 10])
net_3.SGD(training_data,
30,
10,
0.5,
'bs10_lr0.5_784x128x10.csv',
test_data=test_data)
net_4 = network.Network([784, 128, 10])
net_4.SGD(training_data,
import network as NN
import dataLoader as loader
trainingData = loader.training_data()
testingData = loader.testing_data()
net = NN.Network([784, 100, 30, 10])
steps = 30
net.StochasticGradientDescent(trainingData, steps, test_data=testingData)
net.save('trainedNet')
import numpy as np
# Learn the OR function
# 0 0 = 0
# 0 1 = 1
# 1 0 = 1
# 1 1 = 1
x = np.array([[0, 0],
[0, 1],
[1, 0],
[1, 1]])
y = np.array([[0],
[1],
[1],
[1]])
# 1 input layer with 2 neurons + 1 bias
# 1 hidden layer with 2 neurons + 1 bias
# 1 output layer with 1 neuron
nn = net.Network(2,2,1)
# Learn by drawing 100000 samples randomly from our inputs
# Learning rate if too large will fail miserably, but there seems to
# be other local minima
nn.train(x,y,learn_rate=0.1,epochs=100000)
# Show our output
for sample, answer in zip(x,y):
print (sample, nn.predict(sample))
#before you run this code comment out the plot section in network.py
#this code let's you choose the best eta for a particular activation function and a particular adaptive learning algorithm
import mnist_loader
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
import network
etas = [5.0, 3.0, 1.0, 0.5, 0.1, 0.06, 0.01, 0.005]
for eta in etas:
net = network.Network([784, 30, 10], 'tanh', 'softmax')
net.SGD(training_data, 5, 10, eta, 'adagrad', test_data=validation_data)
for eta in etas:
net = network.Network([784, 30, 10], 'tanh', 'softmax')
net.SGD(training_data, 5, 10, eta, 'rmsprop', test_data=validation_data)
for eta in etas:
net = network.Network([784, 30, 10], 'ReLU', 'softmax')
net.SGD(training_data, 5, 10, eta, 'adagrad', test_data=validation_data)
for eta in etas:
net = network.Network([784, 30, 10], 'ReLU', 'softmax')
net.SGD(training_data, 5, 10, eta, 'rmsprop', test_data=validation_data)
for az in current_segment.az:
train_data_az.append(az)
labels_train.append(current_segment.group_label)
num_segments_train = len(train_segments)
num_segments_test = len(test_segments)
labels_train, labels_test = np.array(labels_train), np.array(labels_test)
train_data = np.array([train_data_ax, train_data_ay, train_data_az])
test_data = np.array([test_data_ax, test_data_ay, test_data_az])
#%% Cell 9: Train and test Reservoir Computer Network using previously generated segments as input data.
import copy
import network as Network
Network = Network.Network()
num_nodes = 100
input_probability = 0.3
reservoir_probability = 0.3
classifier = "log"
Network.T = sum(len_segments_train)
Network.n_min = 1
Network.K = 3
Network.N = num_nodes
Network.setup_network(train_data, num_nodes, input_probability, reservoir_probability, num_groups, num_segments_train)
Network.train_network(num_groups, classifier, num_segments_train, len_segments_train, labels_train, num_nodes)
Network.mean_test_matrix = np.zeros([Network.N, num_segments_test])
def __init__(self, ip, mask):
self.__network = network.Network(ip, mask)
import mnist_loader training_data, validation_data, test_data = mnist_loader.load_data_wrapper() import network net = network.Network([784, 30, 10]) net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
# Georgia Institute of Technology # ---------------------------------------------------- import tensorflow as tf import network as net import pandas as pd import random as rd import numpy as np import time import math import csv import os # ---------------------------------------------------- # Instantiate Network Class # ---------------------------------------------------- lstm = net.Network() # ---------------------------------------------------- # User-Defined Constants # ---------------------------------------------------- # Training NUM_TRAINING = 1000 # Number of training batches (balanced minibatches) NUM_VALIDATION = 100 # Number of validation batches (balanced minibatches) WINDOW_INT_t = 1 # Rolling window step interval for training (rolling window) WINDOW_INT_v = 1000 # Rolling window step interval for validation (rolling window) # Load File LOAD_FILE = False # Load initial LSTM model from saved checkpoint? # Input Pipeline # Enter "True" for balanced mini-batching and "False" for rolling-window
