def __backward(self, dout):
dbeta = dout.sum(axis=0)
dgammax = dout
dgamma = ve.sum(ve.mul(self.xhat, dgammax), axis=0)
dxhat = dgammax * self.gamma
divar = ve.sum(ve.mul(dxhat, self.xmu), axis=0)
dxmu1 = dxhat * self.ivar
dsqrtvar = divar * (1 / (self.sqrtvar**2)) * (-1)
dvar = (1 / ve.sqrt(self.var + 10e-7)) * dsqrtvar * 0.5
dsq = ve.arange([
1 for i in range(reduce(lambda x, y: x * y, dout.get_demension()))
]).reshape(dout.get_demension()) * (dvar / dout.get_demension()[1])
dxmu2 = ve.mul(self.xmu, dsq) * 2
dx1 = dxmu1 + dxmu2
dmu = ve.sum(dxmu1 + dxmu2, axis=0) * (-1)
dx2 = ve.arange([
1 for i in range(reduce(lambda x, y: x * y, dout.get_demension()))
]).reshape(dout.get_demension()) * (dmu / dout.get_demension()[1])
dx = dx1 + dx2
return dx
def AND(x1, x2, b=-0.7):
x = np.arange([x1, x2])
w = np.arange([0.5, 0.5]).reshape(2, 1)
tmp = x * w + b
if tmp <= 0:
return 0
else:
return 1
def _f(func, x):
if (type(x) == type(np.arange([
1
]))) and (len(x.get_demension()) >= 2) and (x.get_demension()[0] == 1):
#TODO
lst = list(map(func, x.get_origin_list()))
#print lst
return np.arange(lst).reshape(1, len(lst))
elif (type(x) == type(1)) or (type(x) == type(1.0)):
return func(x)
else:
#print type(x)
raise ValueError
def _classification(x):
lst = list(map(lambda x: e._e**x, x.get_origin_list()))
full_lst = [1 for _ in range(x.get_demension()[1])]
a = ve.arange(lst).reshape(1, x.get_demension()[1])
full = ve.arange(full_lst).reshape(x.get_demension()[1], 1)
total = a * full
#print a * (1 / total.get_origin_list())
#print "^^^^^^^^^^^^^^^^^^^^^^^^^^^"
return a * (1 / total.get_origin_list())
def create_contexts_target(corpus, window_size=1):
corpus_lst = corpus.get_origin_list()
target = corpus_lst[window_size:-window_size]
contexts = []
for idx in range(window_size, len(corpus_lst) - window_size):
cs = []
for t in range(-window_size, window_size + 1):
if t == 0:
continue
cs.append(corpus_lst[idx + 1])
contexts.append(cs)
return np.arange(contexts), np.arange(target)
def backward(self, dout):
if (type(dout) == type(1)) or (type(dout) == type(1.0)):
if x >= 0:
dx = dout
else:
dx = 0
elif (type(dout) == type(ve.arange([1]))):
dmen = dout.get_demension()
dout.reshape(1, reduce(lambda x, y: x * y, dmen))
dout_lst = dout.get_origin_list()
dx = ve.arange(list(map(lambda x: x
if x >= 0 else 0, dout_lst))).reshape(dmen)
return dx
def forward(self, x):
if (type(x) == type(1)) or (type(x) == type(1.0)):
if x >= 0:
out = x
else:
out = 0
elif (type(x) == type(ve.arange([1]))):
#print "**********"
dmen = x.get_demension()
x.reshape(1, reduce(lambda x, y: x * y, dmen))
x_lst = x.get_origin_list()
out = ve.arange(list(map(lambda x: x
if x >= 0 else 0, x_lst))).reshape(dmen)
return out
def forward(self, x):
if (type(x) == type(1)) or (type(x) == type(1.0)):
out = 1 / (1 + e._e**(x * (-1)))
elif (type(x) == type(ve.arange([1]))):
print "%% Sigmoid forward %%"
print x
dmen = x.get_demension()
x.reshape(1, reduce(lambda x, y: x * y, dmen))
x_lst = x.get_origin_list()
out = ve.arange(
list(map(lambda x: (1 / (1 + e._e**(x * (-1)))),
x_lst))).reshape(dmen)
print out
self.out = out
return out
def _numerical_gradient(f, x):
h = 1e-4
dmen = x.get_demension()
tlen = reduce(lambda x, y: x * y, dmen)
grad_lst = []
it = np.nditer(x)
while not it.finished:
idx = it.multi_index
tmp_val = it.iterget(idx)
it.iterset(idx, ((tmp_val * 1.0) + h))
print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
print x
fxh1 = f(x)
it.iterset(idx, ((tmp_val * 1.0) - h))
print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
print x
fxh2 = f(x)
grad_lst.append((fxh1 - fxh2) / (2 * h))
#grad_lst[it._get_cur_ptr(list(idx))] = (fxh1 - fxh2) / (2 * h)
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
print it._get_cur_ptr(list(idx))
print grad_lst
print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
it.iterset(idx, tmp_val)
it.iternext()
grad = np.arange(grad_lst).reshape(dmen)
print grad
print "!>>>>>>>>>>>>>>>>>>>>>"
return grad
def backward(self, dout=1):
batch_size = self.t.get_demension()[1]
#TODO
print "*********xxx**********************"
print self.y
print self.t
#dx = (self.y - self.t) * (1.0/batch_size)
dmen = self.y.get_demension()
self.y.reshape(1, reduce(lambda x, y: x * y, dmen))
dx_lst = self.y.get_origin_list()
dx = ve.arange(dx_lst).reshape(dmen)
self.y.reshape(dmen)
dx = (dx - self.t)
print "**********yyy**********************"
print self.y
print self.t
print self.loss
print dx
print "**********zzz*********************"
return dx
def update(self, params, grads):
if self.v is None:
self.v = {}
for key, val in params.items():
dmen = val.get_demension()
self.v[key] = np.arange([
0 for _ in range(reduce(lambda x, y: x * y, dmen))
]).reshape(dmen)
for key in params.keys():
self.v[key] = (self.v[key] * self.momentum) + (grads[key] *
self.lr)
params[key] = params[key] - self.v[key]
def backward(self, dout):
if (type(dout) == type(1)) or (type(dout) == type(1.0)):
dx = dout * (1.0 - self.out) * self.out
elif (type(dout) == type(ve.arange([1]))):
print "%% Sigmoid backward %%"
print dout
print self.out
dout_dmen = dout.get_demension()
out_dmen = self.out.get_demension()
dout.reshape(1, reduce(lambda x, y: x * y, dout_dmen))
self.out.reshape(1, reduce(lambda x, y: x * y, out_dmen))
dout_lst = dout.get_origin_list()
out_lst = self.out.get_origin_list()
dx = ve.arange(
list(map(lambda x, y: (x * (1.0 - y) * y), dout_lst,
out_lst))).reshape(dout_dmen)
self.out.reshape(out_dmen)
print dx
return dx
def update(self, params, grads):
if self.v is None:
self.v = {}
for key, val in params.items():
self.v[key] = np.arange([
0 for _ in range(reduce(lambda x, y: x * y, dmen))
]).reshape(dmen)
for key in params.keys():
# v(t) = momentum * v(t-1) - learning_rate * dL/dw
# w(t) = w(t-1) + momentum * v(t+1) - learning_rate * dL/dw
self.v[key] = (self.v[key] * self.momentum) - grads[key] * self.lr
params[key] = params[key] + self.v[
key] * self.momentum * self.momentum - grads[key] * (
1 + self.momentum) * self.lr
def init_network():
network = {}
network['W1'] = ve.arange([0.1, 0.3, 0.5, 0.2, 0.4, 0.6]).reshape(2, 3)
network['b1'] = ve.arange([0.1, 0.2, 0.3])
network['W2'] = ve.arange([0.1, 0.4, 0.2, 0.5, 0.3, 0.6]).reshape(3, 2)
network['b2'] = ve.arange([0.1, 0.2])
network['W3'] = ve.arange([0.1, 0.3, 0.2, 0.4]).reshape(2, 2)
network['b3'] = ve.arange([0.1, 0.2])
return network
def softmax(a):
print "=========================================="
dmen = a.get_demension()
c = ve.max(a)
print c
b = a - c
print b
exp_a = ve.exp(b)
print exp_a
y = ve.arange(exp_a).reshape(dmen)
print y
sum_exp_a = sum(exp_a)
print sum_exp_a
y = y * (1 / sum_exp_a)
print y
print "========================================="
return y
def create_co_matrix(corpus, vocab_size, window_size=1):
corpus_lst = corpus.get_origin_list()
corpus_size = len(corpus_lst)
co_matrix = np.arange([0 for _ in range(vocab_size * vocab_size)
]).reshape(vocab_size, vocab_size)
for idx, word_id in enumerate(corpus_lst):
for i in range(1, window_size + 1):
left_idx = idx - 1
right_idx = idx + 1
if left_idx >= 0:
left_word_id = corpus_lst[left_idx]
co_matrix.get_origin_list()[word_id][left_word_id] += 1
if right_idx < corpus_size:
right_word_id = corpus_lst[right_idx]
co_matrix.get_origin_list()[word_id][right_word_id] += 1
return co_matrix
def _check_vector(a):
return type(a) == type(np.arange([2]))
