def volumize(dist):
global words
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
return V
def __init__(self, opt={}):
self.out_depth = opt['filters']
self.sx = opt['sx'] # filter size: should be odd if possible
self.in_depth = opt['in_depth']
self.in_sx = opt['in_sx']
self.in_sy = opt['in_sy']
# optional
self.sy = getopt(opt, 'sy', self.sx)
self.stride = getopt(
opt, 'stride',
1) # stride at which we apply filters to input volume
self.pad = getopt(opt, 'pad', 0) # padding to borders of input volume
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
"""
Note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
final application.
"""
self.out_sx = int(
floor((self.in_sx - self.sx + 2 * self.pad) / self.stride + 1))
self.out_sy = int(
floor((self.in_sy - self.sy + 2 * self.pad) / self.stride + 1))
self.layer_type = 'conv'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [
Vol(self.sx, self.sx, self.in_depth)
for i in xrange(self.out_depth)
]
self.biases = Vol(1, 1, self.out_depth, bias)
def volumize(dist):
global words
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
return V
def forward(self, V, is_training):
self.in_act = V
N = self.out_depth
V2 = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
if self.out_sx == 1 and self.out_sy == 1:
for i in xrange(N):
offset = i * self.group_size
m = max(V.w[offset:])
index = V.w[offset:].index(m)
V2.w[i] = m
self.switches[i] = offset + index
else:
switch_counter = 0
for x in xrange(V.sx):
for y in xrange(V.sy):
for i in xrange(N):
ix = i * self.group_size
elem = V.get(x, y, ix)
elem_i = 0
for j in range(1, self.group_size):
elem2 = V.get(x, y, ix + j)
if elem2 > elem:
elem = elem2
elem_i = j
V2.set(x, y, i, elem)
self.switches[i] = ix + elem_i
switch_counter += 1
self.out_act = V2
return self.out_act
def load_data(training=True):
"""Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/"""
path = "./data"
if training:
fname_img = os.path.join(path, 'train-images-idx3-ubyte')
fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte')
else:
fname_img = os.path.join(path, 't10k-images-idx3-ubyte')
fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte')
# Inputs
fimg = open(fname_img, 'rb')
magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16))
imgs = pyarray("B", fimg.read())
fimg.close()
imgs = [imgs[n:n+784] for n in xrange(0, len(imgs), 784)]
inputs = []
for img in imgs:
V = Vol(28, 28, 1, 0.0)
V.w = [ (px / 255.0) for px in img ]
inputs.append(V)
# Outputs
flbl = open(fname_lbl, 'rb')
magic_nr, size = struct.unpack(">II", flbl.read(8))
labels = pyarray("b", flbl.read())
flbl.close()
return zip(inputs, labels)
def load_data(training=True):
"""Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/"""
path = './data'
if training:
fname_img = os.path.join(path, 'train-images-idx3-ubyte')
fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte')
else:
fname_img = os.path.join(path, 't10k-images-idx3-ubyte')
fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte')
# Inputs
fimg = open(fname_img, 'rb')
magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16))
imgs = pyarray("B", fimg.read())
fimg.close()
imgs = [imgs[n:n + 784] for n in xrange(0, len(imgs), 784)]
inputs = []
V = Vol(28, 28, 1, 0.0)
for img in imgs:
V.w = [(px / 255.0) for px in img]
inputs.append(augment(V, 24))
# Outputs
flbl = open(fname_lbl, 'rb')
magic_nr, size = struct.unpack(">II", flbl.read(8))
labels = pyarray("b", flbl.read())
flbl.close()
return zip(inputs, labels)
def test():
global N, words, network
print 'In testing.'
gettysburg = """Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we can not dedicate -- we can not consecrate -- we can not hallow -- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us -- that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion -- that we here highly resolve that these dead shall not have died in vain -- that this nation, under God, shall have a new birth of freedom -- and that government of the people, by the people, for the people, shall not perish from the earth."""
tokenizer = RegexpTokenizer('\w+')
gettysburg_tokens = tokenizer.tokenize(gettysburg)
samples = []
for token in gettysburg_tokens:
word = token.lower()
if word not in ENGLISH_STOP_WORDS and word not in punctuation:
samples.append(word)
dist = FreqDist(samples)
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
pred = network.forward(V).w
topics = []
while len(topics) != 5:
max_act = max(pred)
topic_idx = pred.index(max_act)
topic = words[topic_idx]
if topic in gettysburg_tokens:
topics.append(topic)
del pred[topic_idx]
print 'Topics of the Gettysburg Address:'
print topics
def makeTerrainData(n_points=1000):
global training_data, testing_data
###############################################################################
### from: https://github.com/udacity/ud120-projects/blob/master/choose_your_own/prep_terrain_data.py
### make the toy dataset
random.seed(42)
grade = [random.random() for ii in range(0, n_points)]
bumpy = [random.random() for ii in range(0, n_points)]
error = [random.random() for ii in range(0, n_points)]
y = [
round(grade[ii] * bumpy[ii] + 0.3 + 0.1 * error[ii])
for ii in range(0, n_points)
]
for ii in range(0, len(y)):
if grade[ii] > 0.8 or bumpy[ii] > 0.8:
y[ii] = 1.0
### split into train/test sets
X = [[gg, ss] for gg, ss in zip(grade, bumpy)]
split = int(0.75 * n_points)
X_train = X[0:split]
X_test = X[split:]
y_train = y[0:split]
y_test = y[split:]
for x, y in zip(X_train, y_train):
training_data.append((Vol(x), int(y)))
for x, y in zip(X_test, y_test):
testing_data.append((Vol(x), int(y)))
def load_data(train=True):
global N, frequencies
with open('./data/big.txt', 'r') as infile:
text = infile.read()
skip = 3
size = skip * N
start = randint(0, len(text) - size)
content = text[start:start+size]
data = []
for i in range(0, len(content), skip):
x1, x2, y = content[i:i+skip]
l1 = ord(x1)
l2 = ord(x2)
frequencies[l1] += 1
frequencies[l2] += 1
V = Vol(1, 1, 255, 0.0)
V.w[l1] = 1.0
V.w[l2] = 1.0
label = ord(y)
data.append((V, label))
normalize()
return data
def forward(self, V, in_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
Vw = V.w
def norm(vec):
return sqrt(sum(c * c for c in vec))
normv = norm(Vw)
# compute cos sim between V and filters
for i in xrange(self.out_depth):
sum_a = 0.0
fiw = self.filters[i].w
for d in xrange(self.num_inputs):
sum_a += Vw[d] * fiw[d]
sum_a += self.biases.w[i] # dot(W, v) + b
normf = norm(fiw)
try:
A.w[i] = sum_a / (normv * normf)
except:
A.w[i] = 0
self.out_act = A
return self.out_act
def forward(self, V, is_training):
self.in_act = V
N = self.out_depth
V2 = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
if self.out_sx == 1 and self.out_sy == 1:
for i in xrange(N):
offset = i * self.group_size
m = max(V.w[offset:])
index = V.w[offset:].index(m)
V2.w[i] = m
self.switches[i] = offset + index
else:
switch_counter = 0
for x in xrange(V.sx):
for y in xrange(V.sy):
for i in xrange(N):
ix = i * self.group_size
elem = V.get(x, y, ix)
elem_i = 0
for j in range(1, self.group_size):
elem2 = V.get(x, y, ix + j)
if elem2 > elem:
elem = elem2
elem_i = j
V2.set(x, y, i, elem)
self.switches[i] = ix + elem_i
switch_counter += 1
self.out_act = V2
return self.out_act
def fromJSON(self, json):
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.num_inputs = json['num_inputs']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.filters = [Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters']]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
def augment(V, crop, grayscale=False):
# note assumes square outputs of size crop x crop
# randomly sample a crop in the input volume
if crop == V.sx: return V
dx = randi(0, V.sx - crop)
dy = randi(0, V.sy - crop)
W = Vol(crop, crop, V.depth)
for x in xrange(crop):
for y in xrange(crop):
if x + dx < 0 or x + dx >= V.sx or \
y + dy < 0 or y + dy >= V.sy:
continue
for d in xrange(V.depth):
W.set(x, y, d, V.get(x + dx, y + dy, d))
if grayscale:
#flatten into depth=1 array
G = Vol(crop, crop, 1, 0.0)
for i in xrange(crop):
for j in xrange(crop):
G.set(i, j, 0, W.get(i, j, 0))
W = G
return W
def forward(self, V, is_training):
self.in_act = V
V2 = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
sexp = 0.0
for i in xrange(len(V2.w)):
sexp += exp(V.w[i])
V2.w[i] = log((sexp / (i + 1))) / self.zeta
self.out_act = V2
return self.out_act
def load_data():
global training_data, testing_data
train = [
line.split(',') for line in file(
'./data/titanic-kaggle/train.csv').read().split('\n')[1:]
]
for ex in train:
PassengerId, Survived, Pclass, Name, NameRest, Sex, Age, SibSp, Parch, Ticket, Fare, Cabin, Embarked = ex
# Fixing
sex = 0.0 if Sex == 'male' else 1.0
age = 0 if Age == '' else float(Age)
Embarked = Embarked.replace('\r', '')
if Embarked == 'C':
emb = 0.0
elif Embarked == 'Q':
emb = 1.0
else:
emb = 2.0
vec = [
float(Pclass), sex, age,
float(SibSp),
float(Parch),
float(Fare), emb
]
v = Vol(vec)
training_data.append((v, int(Survived)))
test = [
line.split(',') for line in file(
'./data/titanic-kaggle/test.csv').read().split('\n')[1:]
]
for ex in test:
PassengerId, Pclass, Name, NameRest, Sex, Age, SibSp, Parch, Ticket, Fare, Cabin, Embarked = ex
# Fixing
sex = 0.0 if Sex == 'male' else 1.0
age = 0 if Age == '' else float(Age)
Embarked = Embarked.replace('\r', '')
if Embarked == 'C':
emb = 0.0
elif Embarked == 'Q':
emb = 1.0
else:
emb = 2.0
fare = 0 if Fare == '' else float(Fare)
vec = [float(Pclass), sex, age, float(SibSp), float(Parch), fare, emb]
testing_data.append(Vol(vec))
print 'Data loaded...'
def __init__(self, opt={}):
self.out_depth = opt['num_neurons']
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth']
self.out_sx = 1
self.out_sy = 1
self.layer_type = 'sim'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def fromJSON(self, json):
self.sx = json['sx']
self.sy = json['sy']
self.stride = json['stride']
self.in_depth = json['in_depth']
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.pad = json['pad']
self.filters = [Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters']]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
def forward(self, V, in_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
Vw = V.w
# dot(W, x) + b
for i in xrange(self.out_depth):
sum_a = 0.0
fiw = self.filters[i].w
for d in xrange(self.num_inputs):
sum_a += Vw[d] * fiw[d]
sum_a += self.biases.w[i]
A.w[i] = sum_a
self.out_act = A
return self.out_act
def forward(self, V, is_training):
self.in_act = V
A = Vol(1, 1, self.num_inputs, 0.0)
applied = 0
for n in xrange(self.num_inputs):
if n < self.skip:
A.w[n] = V.w[n]
else:
A.w[n] = V.w[n] + self.delta[n - self.skip]
applied += 1
if applied == self.num_neurons:
break
self.out_act = A
return self.out_act
def fill():
global embeddings
output = 'PhraseId,Sentiment\n'
raw = file('./data/sentiment-kaggle/test.tsv').read().split('\n')[1:]
for idx, line in enumerate(raw):
try:
values = line.split('\t')
phrase_id = values[0]
phrase = values[2]
x = []
for word in phrase.split():
if word in embeddings:
x.append(embeddings[word])
avgs = [0.0] * 80
for n in xrange(80):
for vec in x:
avgs[n] += vec[n]
try:
avgs[n] /= float(len(x))
except:
avgs[n] = 0.0
network.forward(Vol(avgs))
output += '{},{}\n'.format(phrase_id, network.getPrediction() + 1)
print idx
except:
continue
with open('./data/sentiment-kaggle/out1.csv', 'w') as outfile:
outfile.write(output)
print 'Done'
def forward(self, V, in_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
Vw = V.w
# dot(W, x) + b
for i in xrange(self.out_depth):
sum_a = 0.0
fiw = self.filters[i].w
for d in xrange(self.num_inputs):
sum_a += Vw[d] * fiw[d]
sum_a += self.biases.w[i]
A.w[i] = sum_a
self.out_act = A
return self.out_act
def forward(self, V, is_training):
self.in_act = V
A = Vol(1, 1, self.num_inputs, 0.0)
applied = 0
for n in xrange(self.num_inputs):
if n < self.skip:
A.w[n] = V.w[n]
else:
A.w[n] = V.w[n] + self.delta[n - self.skip]
applied += 1
if applied == self.num_neurons:
break
self.out_act = A
return self.out_act
def load_data():
global N, words
raw = list(word for fileid in corpus.fileids()
for word in corpus.words(fileid))
words = list(
token
for token in RegexpTokenizer('\w+').tokenize(' '.join(raw)))[100:1000]
tokens = set(words)
tokens_l = list(tokens)
N = len(tokens)
print 'Corpus size: {} words'.format(N)
step = 4
data = []
for gram in ngrams(words, step):
w1, w2, w3, pred = gram
V = Vol(1, 1, N, 0.0)
V.w[tokens_l.index(w1)] = 1
V.w[tokens_l.index(w2)] = 1
V.w[tokens_l.index(w3)] = 1
label = tokens_l.index(pred)
data.append((V, label))
return data
def __init__(self, opt={}):
self.out_depth = opt['filters']
self.sx = opt['sx'] # filter size: should be odd if possible
self.in_depth = opt['in_depth']
self.in_sx = opt['in_sx']
self.in_sy = opt['in_sy']
# optional
self.sy = getopt(opt, 'sy', self.sx)
self.stride = getopt(opt, 'stride', 1) # stride at which we apply filters to input volume
self.pad = getopt(opt, 'pad', 0) # padding to borders of input volume
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
"""
Note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
final application.
"""
self.out_sx = int(floor((self.in_sx - self.sx + 2 * self.pad) / self.stride + 1))
self.out_sy = int(floor((self.in_sy - self.sy + 2 * self.pad) / self.stride + 1))
self.layer_type = 'conv'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(self.sx, self.sy, self.in_depth) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def backward(self, reward):
self.latest_reward = reward
self.average_reward_window.add(reward)
self.reward_window.pop(0)
self.reward_window.append(reward)
if not self.learning:
return
self.age += 1
#it is time t+1 and we have to store (s_t, a_t, r_t, s_{t+1}) as new experience
#(given that an appropriate number of state measurements already exist, of course)
if self.forward_passes > self.temporal_window + 1:
n = self.window_size
e = Experience(
self.net_window[n - 2],
self.action_window[n - 2],
self.reward_window[n - 2],
self.net_window[n - 1]
)
if len(self.experience) < self.experience_size:
self.experience.append(e)
else:
ri = randi(0, self.experience_size)
self.experience[ri] = e
#learn based on experience, once we have some samples to go on
#this is where the magic happens...
if len(self.experience) > self.start_learn_threshold:
avcost = 0.0
for k in range(self.tdtrainer.batch_size):
re = randi(0, len(self.experience))
e = self.experience[re]
x = Vol(1, 1, self.net_inputs)
x.w = e.state0
maxact = self.policy(e.state1)
r = e.reward0 + self.gamma * maxact['value']
ystruct = {'dim': e.action0, 'val': r}
stats = self.tdtrainer.train(x, ystruct)
# print(stats['accuracy'])
avcost += stats['loss']
avcost /= self.tdtrainer.batch_size
self.average_loss_window.add(avcost)
def augment(V, crop, grayscale=False):
# note assumes square outputs of size crop x crop
# randomly sample a crop in the input volume
if crop == V.sx: return V
dx = randi(0, V.sx - crop)
dy = randi(0, V.sy - crop)
W = Vol(crop, crop, V.depth)
for x in xrange(crop):
for y in xrange(crop):
if x + dx < 0 or x + dx >= V.sx or \
y + dy < 0 or y + dy >= V.sy: continue
for d in xrange(V.depth):
W.set(x, y, d, V.get(x + dx, y + dy, d))
if grayscale:
#flatten into depth=1 array
G = Vol(crop, crop, 1, 0.0)
for i in xrange(crop):
for j in xrange(crop):
G.set(i, j, 0, W.get(i, j, 0))
W = G
return W
def forward(self, V, is_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
# max activation
max_act = max(V.w)
# compute exponentials (carefully to not blow up)
# normalize
exps = [exp(w - max_act) for w in V.w]
exps_sum = float(sum(exps))
exps_norm = [elem / exps_sum for elem in exps]
self.es = exps_norm
A.w = exps_norm
self.out_act = A
return self.out_act
def forward(self, V, is_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
# max activation
max_act = max(V.w)
# compute exponentials (carefully to not blow up)
# normalize
exps = [ exp(w - max_act) for w in V.w ]
exps_sum = float(sum(exps))
exps_norm = [ elem / exps_sum for elem in exps ]
self.es = exps_norm
A.w = exps_norm
self.out_act = A
return self.out_act
def fromJSON(self, json):
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.num_inputs = json['num_inputs']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
def load_data():
global N, words
freqs = [ FreqDist(corpus.words(fileid)) for fileid in corpus.fileids() ]
words = list(set(word
for dist in freqs
for word in dist.keys()
if word not in ENGLISH_STOP_WORDS and
word not in punctuation))
data = []
N = len(words)
for dist in freqs:
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
data.append((V, V.w))
return data
def backward(self, reward):
self.latest_reward = reward
self.average_reward_window.add(reward)
self.reward_window.pop(0)
self.reward_window.append(reward)
if not self.learning:
return
self.age += 1
#it is time t+1 and we have to store (s_t, a_t, r_t, s_{t+1}) as new experience
#(given that an appropriate number of state measurements already exist, of course)
if self.forward_passes > self.temporal_window + 1:
n = self.window_size
e = Experience(self.net_window[n - 2], self.action_window[n - 2],
self.reward_window[n - 2], self.net_window[n - 1])
if len(self.experience) < self.experience_size:
self.experience.append(e)
else:
ri = randi(0, self.experience_size)
self.experience[ri] = e
#learn based on experience, once we have some samples to go on
#this is where the magic happens...
if len(self.experience) > self.start_learn_threshold:
avcost = 0.0
for k in xrange(self.tdtrainer.batch_size):
re = randi(0, len(self.experience))
e = self.experience[re]
x = Vol(1, 1, self.net_inputs)
x.w = e.state0
maxact = self.policy(e.state1)
r = e.reward0 + self.gamma * maxact['value']
ystruct = {'dim': e.action0, 'val': r}
stats = self.tdtrainer.train(x, ystruct)
avcost += stats['loss']
avcost /= self.tdtrainer.batch_size
print avcost
self.average_loss_window.add(avcost)
def policy(self, s):
"""
compute the value of doing any action in this state
and return the argmax action and its value
"""
V = Vol(s)
action_values = self.value_net.forward(V)
weights = action_values.w
max_val = max(weights)
max_k = weights.index(maxval)
return {'action': max_k, 'value': max_val}
def __init__(self, opt={}):
self.out_depth = opt['num_neurons']
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth']
self.out_sx = 1
self.out_sy = 1
self.layer_type = 'fc'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def load_data():
global training_data, testing_data
lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4)
xs = lfw_people.data
ys = lfw_people.target
inputs = []
labels = list(ys)
for face in xs:
V = Vol(50, 37, 1, 0.0)
V.w = list(face)
inputs.append(augment(V, 30))
x_tr, x_te, y_tr, y_te = train_test_split(inputs, labels, test_size=0.25)
training_data = zip(x_tr, y_tr)
testing_data = zip(x_te, y_te)
print 'Dataset made...'
def load_data():
global iris_data
data = load_iris()
xs = data.data
ys = data.target
inputs = [Vol(list(row)) for row in xs]
labels = list(ys)
iris_data = zip(inputs, labels)
print 'Data loaded...'
def load_data():
global N, words, labels
posts = corpus.xml_posts()[:10000]
freqs = [ FreqDist(post.text) for post in posts ]
words = list(set(word
for dist in freqs
for word in dist.keys()
if word not in ENGLISH_STOP_WORDS and
word not in punctuation))
labels = list(set([ post.get('class') for post in posts ]))
data = []
N = len(words)
for post, dist in zip(posts, freqs):
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
data.append((V, labels.index(post.get('class'))))
return data
def load_data():
global training_data, testing_data
lfw_people = fetch_lfw_people(min_faces_per_person=70, resize=0.4)
xs = lfw_people.data
ys = lfw_people.target
inputs = []
labels = list(ys)
for face in xs:
V = Vol(50, 37, 1, 0.0)
V.w = list(face)
inputs.append(augment(V, 30))
x_tr, x_te, y_tr, y_te = train_test_split(inputs, labels, test_size=0.25)
training_data = zip(x_tr, y_tr)
testing_data = zip(x_te, y_te)
print 'Dataset made...'
def load_data(crop, gray, training=True):
filename = './data/cifar10_'
if training:
filename += 'train.npz'
else:
filename += 'test.npz'
data = numpy.load(filename)
xs = data['x']
ys = data['y']
for i in xrange(len(xs)):
V = Vol(32, 32, 3, 0.0)
for d in xrange(3):
for x in xrange(32):
for y in xrange(32):
px = xs[i][x * 32 + y, d] / 255.0 - 0.5
V.set(x, y, d, px)
if crop:
V = augment(V, 24, gray)
y = ys[i]
yield V, y
def load_data(crop, gray, training=True):
filename = './data/cifar10_'
if training:
filename += 'train.npz'
else:
filename += 'test.npz'
data = numpy.load(filename)
xs = data['x']
ys = data['y']
for i in xrange(len(xs)):
V = Vol(32, 32, 3, 0.0)
for d in xrange(3):
for x in xrange(32):
for y in xrange(32):
px = xs[i][x * 32 + y, d] / 255.0 - 0.5
V.set(x, y, d, px)
if crop:
V = augment(V, 24, gray)
y = ys[i]
yield V, y
def fromJSON(self, json):
self.sx = json['sx']
self.sy = json['sy']
self.stride = json['stride']
self.in_depth = json['in_depth']
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.pad = json['pad']
self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
def train():
global training_data, n, t, training_data2
print 'In training...'
print 'k', 'time\t\t ', 'loss\t '
print '----------------------------------------------------'
training_data2 = []
try:
for x, y in training_data:
stats = t.train(x, x.w)
print stats['k'], stats['time'], stats['loss']
training_data2.append((Vol(n.forward(x).w), y))
except: #hit control-c or other
return
def forward(self, V, is_training):
self.in_act = V
A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
switch_counter = 0
for d in xrange(self.out_depth):
x = -self.pad
y = -self.pad
for ax in xrange(self.out_sx):
y = -self.pad
for ay in xrange(self.out_sy):
# convolve centered at this particular location
max_a = -99999
win_x, win_y = -1, -1
for fx in xrange(self.sx):
for fy in xrange(self.sy):
off_x = x + fx
off_y = y + fy
if off_y >= 0 and off_y < V.sy \
and off_x >= 0 and off_x < V.sx:
v = V.get(off_x, off_y, d)
# max pool
if v > max_a:
max_a = v
win_x = off_x
win_y = off_y
self.switch_x[switch_counter] = win_x
self.switch_y[switch_counter] = win_y
switch_counter += 1
A.set(ax, ay, d, max_a)
y += self.stride
x += self.stride
self.out_act = A
return self.out_act
def load_data(training=True):
"""Adapted from http://g.sweyla.com/blog/2012/mnist-numpy/"""
path = "./data"
if training:
fname_img = os.path.join(path, 'train-images-idx3-ubyte')
fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte')
else:
fname_img = os.path.join(path, 't10k-images-idx3-ubyte')
fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte')
# Inputs
fimg = open(fname_img, 'rb')
magic_nr, size, rows, cols = struct.unpack(">IIII", fimg.read(16))
imgs = pyarray("B", fimg.read())
fimg.close()
imgs = [imgs[n:n + 784] for n in xrange(0, len(imgs), 784)]
inputs = []
for img in imgs:
V = Vol(28, 28, 1, 0.0)
pxs = [0.0] * 784
for n in xrange(784):
if n % 28 == 0:
pxs[n] = 0
else:
pxs[n] = img[n - 1]
V.w = pxs
inputs.append(V)
# Outputs
flbl = open(fname_lbl, 'rb')
magic_nr, size = struct.unpack(">II", flbl.read(8))
labels = pyarray("b", flbl.read())
flbl.close()
return zip(inputs, labels)
def forward(self, V, is_training):
self.in_act = V
A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
v_sx = V.sx
v_sy = V.sy
xy_stride = self.stride
for d in xrange(self.out_depth):
f = self.filters[d]
x = -self.pad
y = -self.pad
for ay in xrange(self.out_sy):
x = -self.pad
for ax in xrange(self.out_sx):
# convolve centered at this particular location
sum_a = 0.0
for fy in xrange(f.sy):
off_y = y + fy
for fx in xrange(f.sx):
# coordinates in the original input array coordinates
off_x = x + fx
if off_y >= 0 and off_y < V.sy and off_x >= 0 and off_x < V.sx:
for fd in xrange(f.depth):
sum_a += f.w[((f.sx * fy) + fx) * f.depth + fd] \
* V.w[((V.sx * off_y) + off_x) * V.depth + fd]
sum_a += self.biases.w[d]
A.set(ax, ay, d, sum_a)
x += xy_stride
y += xy_stride
self.out_act = A
return self.out_act
def forward(self, V, is_training):
self.in_act = V
A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
v_sx = V.sx
v_sy = V.sy
xy_stride = self.stride
for d in xrange(self.out_depth):
f = self.filters[d]
x = -self.pad
y = -self.pad
for ay in xrange(self.out_sy):
x = -self.pad
for ax in xrange(self.out_sx):
# convolve centered at this particular location
sum_a = 0.0
for fy in xrange(f.sy):
off_y = y + fy
for fx in xrange(f.sx):
# coordinates in the original input array coordinates
off_x = x + fx
if off_y >= 0 and off_y < V.sy and off_x >= 0 and off_x < V.sx:
for fd in xrange(f.depth):
sum_a += f.w[((f.sx * fy) + fx) * f.depth + fd] \
* V.w[((V.sx * off_y) + off_x) * V.depth + fd]
sum_a += self.biases.w[d]
A.set(ax, ay, d, sum_a)
x += xy_stride
y += xy_stride
self.out_act = A
return self.out_act
def test():
global n, frequencies
y = weightedSample(lst=frequencies, count=2)
x = Vol(1, 1, 255, 0.0)
x.w[y[0]] = 1.0
x.w[y[1]] = 1.0
s = ''
for i in xrange(50):
n.forward(x)
pred = n.getPrediction()
pred2 = weightedSample(lst=frequencies, count=1)[0]
s += chr(pred) + chr(pred2)
x.w[pred] = 1.0
x.w[y[0]] = 0.0
x.w[y[1]] = 0.0
y = (pred, pred2)
print s
username = "******"
mdp = "playmote"
parawing.connect(username,mdp)
#Recuperation des ailes du pilote enregistre sur Parawing
liste_aile = parawing.listAile(username)
#Ouverture du fichier csv passe en argument
fic = open(argument,"rb")
try:
reader = csv.DictReader(fic)
for row in reader:
print row
v = Vol(row['day'],row['deco'],row['atterro'],"ZuluXP",row['duree'],row['typev'],row['hdeco'],row['conddeco'],"",row['condatterro'],row['km'],row['altmax'],row['variomax'],row['altgain'],row['variomin'],True,"Autonome","",row['story'],"","Pas mal")
# Test aile
print v.compareAile(liste_aile)
# Test deco
# ex : parawing.testdeco
# Test atterro
# ex : parawing.testatterro
#ajout du vol a un objet list
#liste_vol.append(v)
except csv.Error, e:
sys.exit('file %s, line %d: %s' % (argument, reader.line_num, e))
class SimilarityLayer(object):
"""
Computes similarity measures, generalization of dot products
http://arxiv.org/pdf/1410.0781.pdf
"""
def __init__(self, opt={}):
self.out_depth = opt['num_neurons']
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth']
self.out_sx = 1
self.out_sy = 1
self.layer_type = 'sim'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def forward(self, V, in_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
Vw = V.w
def norm(vec):
return sqrt(sum(c * c for c in vec))
normv = norm(Vw)
# compute cos sim between V and filters
for i in xrange(self.out_depth):
sum_a = 0.0
fiw = self.filters[i].w
for d in xrange(self.num_inputs):
sum_a += Vw[d] * fiw[d]
sum_a += self.biases.w[i] # dot(W, v) + b
normf = norm(fiw)
try:
A.w[i] = sum_a / (normv * normf)
except:
A.w[i] = 0
self.out_act = A
return self.out_act
def backward(self):
V = self.in_act
V.dw = zeros(len(V.w)) # zero out gradient
# compute gradient wrt weights and data
for i in xrange(self.out_depth):
fi = self.filters[i]
chain_grad = self.out_act.dw[i]
for d in xrange(self.num_inputs):
V.dw[d] += fi.w[d] * chain_grad #grad wrt input data
fi.dw[d] += V.w[d] * chain_grad #grad wrt params
self.biases.dw[i] += chain_grad
def getParamsAndGrads(self):
response = []
for d in xrange(self.out_depth):
response.append({
'params': self.filters[d].w,
'grads': self.filters[d].dw,
'l2_decay_mul': self.l2_decay_mul,
'l1_decay_mul': self.l1_decay_mul
})
response.append({
'params': self.biases.w,
'grads': self.biases.dw,
'l2_decay_mul': 0.0,
'l1_decay_mul': 0.0
})
return response
def toJSON(self):
return {
'out_depth' : self.out_depth,
'out_sx' : self.out_sx,
'out_sy' : self.out_sy,
'layer_type' : self.layer_type,
'num_inputs' : self.num_inputs,
'l1_decay_mul': self.l1_decay_mul,
'l2_decay_mul': self.l2_decay_mul,
'filters' : [ f.toJSON() for f in self.filters ],
'biases' : self.biases.toJSON()
}
def fromJSON(self, json):
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.num_inputs = json['num_inputs']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
class FullyConnectedLayer(object):
"""
Fully connected dot products, ie. multi-layer perceptron net
Building block for most networks
http://www.deeplearning.net/tutorial/mlp.html
"""
def __init__(self, opt={}):
self.out_depth = opt['num_neurons']
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
self.num_inputs = opt['in_sx'] * opt['in_sy'] * opt['in_depth']
self.out_sx = 1
self.out_sy = 1
self.layer_type = 'fc'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(1, 1, self.num_inputs) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def forward(self, V, in_training):
self.in_act = V
A = Vol(1, 1, self.out_depth, 0.0)
Vw = V.w
# dot(W, x) + b
for i in xrange(self.out_depth):
sum_a = 0.0
fiw = self.filters[i].w
for d in xrange(self.num_inputs):
sum_a += Vw[d] * fiw[d]
sum_a += self.biases.w[i]
A.w[i] = sum_a
self.out_act = A
return self.out_act
def backward(self):
V = self.in_act
V.dw = zeros(len(V.w)) # zero out gradient
# compute gradient wrt weights and data
for i in xrange(self.out_depth):
fi = self.filters[i]
chain_grad = self.out_act.dw[i]
for d in xrange(self.num_inputs):
V.dw[d] += fi.w[d] * chain_grad #grad wrt input data
fi.dw[d] += V.w[d] * chain_grad #grad wrt params
self.biases.dw[i] += chain_grad
def getParamsAndGrads(self):
response = []
for d in xrange(self.out_depth):
response.append({
'params': self.filters[d].w,
'grads': self.filters[d].dw,
'l2_decay_mul': self.l2_decay_mul,
'l1_decay_mul': self.l1_decay_mul
})
response.append({
'params': self.biases.w,
'grads': self.biases.dw,
'l2_decay_mul': 0.0,
'l1_decay_mul': 0.0
})
return response
def toJSON(self):
return {
'out_depth' : self.out_depth,
'out_sx' : self.out_sx,
'out_sy' : self.out_sy,
'layer_type' : self.layer_type,
'num_inputs' : self.num_inputs,
'l1_decay_mul': self.l1_decay_mul,
'l2_decay_mul': self.l2_decay_mul,
'filters' : [ f.toJSON() for f in self.filters ],
'biases' : self.biases.toJSON()
}
def fromJSON(self, json):
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.num_inputs = json['num_inputs']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
class ConvLayer(object):
"""
Performs convolutions: spatial weight sharing
http://deeplearning.net/tutorial/lenet.html
"""
def __init__(self, opt={}):
self.out_depth = opt['filters']
self.sx = opt['sx'] # filter size: should be odd if possible
self.in_depth = opt['in_depth']
self.in_sx = opt['in_sx']
self.in_sy = opt['in_sy']
# optional
self.sy = getopt(opt, 'sy', self.sx)
self.stride = getopt(opt, 'stride', 1) # stride at which we apply filters to input volume
self.pad = getopt(opt, 'pad', 0) # padding to borders of input volume
self.l1_decay_mul = getopt(opt, 'l1_decay_mul', 0.0)
self.l2_decay_mul = getopt(opt, 'l2_decay_mul', 1.0)
"""
Note we are doing floor, so if the strided convolution of the filter doesnt fit into the input
volume exactly, the output volume will be trimmed and not contain the (incomplete) computed
final application.
"""
self.out_sx = int(floor((self.in_sx - self.sx + 2 * self.pad) / self.stride + 1))
self.out_sy = int(floor((self.in_sy - self.sy + 2 * self.pad) / self.stride + 1))
self.layer_type = 'conv'
bias = getopt(opt, 'bias_pref', 0.0)
self.filters = [ Vol(self.sx, self.sy, self.in_depth) for i in xrange(self.out_depth) ]
self.biases = Vol(1, 1, self.out_depth, bias)
def forward(self, V, is_training):
self.in_act = V
A = Vol(self.out_sx, self.out_sy, self.out_depth, 0.0)
v_sx = V.sx
v_sy = V.sy
xy_stride = self.stride
for d in xrange(self.out_depth):
f = self.filters[d]
x = -self.pad
y = -self.pad
for ay in xrange(self.out_sy):
x = -self.pad
for ax in xrange(self.out_sx):
# convolve centered at this particular location
sum_a = 0.0
for fy in xrange(f.sy):
off_y = y + fy
for fx in xrange(f.sx):
# coordinates in the original input array coordinates
off_x = x + fx
if off_y >= 0 and off_y < V.sy and off_x >= 0 and off_x < V.sx:
for fd in xrange(f.depth):
sum_a += f.w[((f.sx * fy) + fx) * f.depth + fd] \
* V.w[((V.sx * off_y) + off_x) * V.depth + fd]
sum_a += self.biases.w[d]
A.set(ax, ay, d, sum_a)
x += xy_stride
y += xy_stride
self.out_act = A
return self.out_act
def backward(self):
# compute gradient wrt weights, biases and input data
V = self.in_act
V.dw = zeros(len(V.w)) # zero out gradient
V_sx = V.sx
V_sy = V.sy
xy_stride = self.stride
for d in xrange(self.out_depth):
f = self.filters[d]
x = -self.pad
y = -self.pad
for ay in xrange(self.out_sy):
x = -self.pad
for ax in xrange(self.out_sx):
# convolve and add up the gradients
chain_grad = self.out_act.get_grad(ax, ay, d) # gradient from above, from chain rule
for fy in xrange(f.sy):
off_y = y + fy
for fx in xrange(f.sx):
off_x = x + fx
if off_y >= 0 and off_y < V_sy and off_x >= 0 and off_x < V_sx:
# forward prop calculated: a += f.get(fx, fy, fd) * V.get(ox, oy, fd)
#f.add_grad(fx, fy, fd, V.get(off_x, off_y, fd) * chain_grad)
#V.add_grad(off_x, off_y, fd, f.get(fx, fy, fd) * chain_grad)
for fd in xrange(f.depth):
ix1 = ((V.sx * off_y) + off_x) * V.depth + fd
ix2 = ((f.sx * fy) + fx) * f.depth + fd
f.dw[ix2] += V.w[ix1] * chain_grad
V.dw[ix1] += f.w[ix2] * chain_grad
self.biases.dw[d] += chain_grad
x += xy_stride
y += xy_stride
def getParamsAndGrads(self):
response = []
for d in xrange(self.out_depth):
response.append({
'params': self.filters[d].w,
'grads': self.filters[d].dw,
'l2_decay_mul': self.l2_decay_mul,
'l1_decay_mul': self.l1_decay_mul
})
response.append({
'params': self.biases.w,
'grads': self.biases.dw,
'l2_decay_mul': 0.0,
'l1_decay_mul': 0.0
})
return response
def toJSON(self):
return {
'sx' : self.sx,
'sy' : self.sy,
'stride' : self.stride,
'in_depth' : self.in_depth,
'out_depth' : self.out_depth,
'out_sx' : self.out_sx,
'out_sy' : self.out_sy,
'layer_type' : self.layer_type,
'l1_decay_mul': self.l1_decay_mul,
'l2_decay_mul': self.l2_decay_mul,
'pad' : self.pad,
'filters' : [ f.toJSON() for f in self.filters ],
'biases' : self.biases.toJSON()
}
def fromJSON(self, json):
self.sx = json['sx']
self.sy = json['sy']
self.stride = json['stride']
self.in_depth = json['in_depth']
self.out_depth = json['out_depth']
self.out_sx = json['out_sx']
self.out_sy = json['out_sy']
self.layer_type = json['layer_type']
self.l1_decay_mul = json['l1_decay_mul']
self.l2_decay_mul = json['l2_decay_mul']
self.pad = json['pad']
self.filters = [ Vol(0, 0, 0, 0).fromJSON(f) for f in json['filters'] ]
self.biases = Vol(0, 0, 0, 0).fromJSON(json['biases'])
