def one_hot_lists(data, n):
T = max([len(l) for l in data])
data_1h = np.zeros((n, T, len(data)))
for b in xrange(len(data)):
for t in xrange(len(data[b])):
data_1h[data[b][t], t, b] = 1
return array(data_1h)
def from_file(self, fin):
logger.info('Loading state')
loaded_params = pickle.load(fin)
self.params = dict(zip(self.param_keys, [array(param) for param in loaded_params]))
if self.train:
self.opt.from_file(fin)
self.params_loaded = True
def one_hot_lists(data, n):
T = max([len(l) for l in data])
data_1h = np.zeros((n, T, len(data)))
for b in xrange(len(data)):
for t in xrange(len(data[b])):
data_1h[data[b][t], t, b] = 1
return array(data_1h)
def from_file(self, fin):
logger.info('Loading state')
loaded_params = pickle.load(fin)
self.params = dict(
zip(self.param_keys, [array(param) for param in loaded_params]))
if self.train:
self.opt.from_file(fin)
self.params_loaded = True
def from_file(self, fin):
self.iters = pickle.load(fin)
self.costs = pickle.load(fin)
self.expcosts = pickle.load(fin)
if self.mom > 0:
loaded_vels = pickle.load(fin)
# Put back on gpu
self.vel = zip(self.model.param_keys, [array(v) for v in loaded_vels])
def from_file(self, fin):
self.iters = pickle.load(fin)
self.costs = pickle.load(fin)
self.expcosts = pickle.load(fin)
if self.mom > 0:
loaded_vels = pickle.load(fin)
# Put back on gpu
self.vel = zip(self.model.param_keys,
[array(v) for v in loaded_vels])
def sample_continuation(s, model, order, alpha=1.0):
if MODEL_TYPE == 'rnn':
data = array(np.array([char_inds[w] for w in s[-1:]])).reshape(-1, 1)
else:
data = array(np.array([char_inds[w] for w in s[-order+1:]])).reshape(-1, 1)
data = one_hot(data, model.hps.output_size)
if MODEL_TYPE == 'rnn':
_, probs = model.cost_and_grad(data, None, prev_h0=model.last_h)
probs = np.squeeze(as_np(probs))
else:
data = data.reshape((-1, data.shape[2]))
_, probs = model.cost_and_grad(data, None)
probs = probs.ravel()
# Higher alpha -> more and more like most likely sequence
probs = probs ** alpha
probs = probs / sum(probs)
w = np.random.choice(range(model.hps.output_size), p=probs)
char = chars[w]
return char
def one_hot(data, n):
data_1h = np.zeros((n, data.shape[0], data.shape[1]))
for t in xrange(data.shape[0]):
for b in xrange(data.shape[1]):
data_1h[data[t, b], t, b] = 1
return array(data_1h)
def cost_and_grad(self, data, labels, back=True, prev_h0=None):
hps = self.hps
T = data.shape[1]
bsize = data.shape[2]
# FIXME gnumpy reallocates if try and use same parameters?
#us = self.us[:, 0:T, 0:bsize]
#dus = self.dus[:, 0:T, 0:bsize]
#hs = self.hs[:, 0:T, 0:bsize]
#dhs = self.dhs[:, 0:T, 0:bsize]
#probs = self.probs[:, 0:T, 0:bsize]
#dprobs = self.dprobs[:, 0:T, 0:bsize]
#costs = self.costs[0:T, 0:bsize]
us = list()
dus = list()
hs = list()
dhs = list()
h0 = list()
for k in xrange(hps.hidden_layers):
us.append(list())
dus.append(list())
hs.append(list())
dhs.append(list())
h0.append(empty((hps.hidden_size, bsize)))
for t in xrange(T):
us[k].append(zeros((hps.hidden_size, bsize)))
dus[k].append(zeros((hps.hidden_size, bsize)))
hs[k].append(zeros((hps.hidden_size, bsize)))
dhs[k].append(zeros((hps.hidden_size, bsize)))
probs = list()
for t in xrange(T):
probs.append(zeros((hps.output_size, bsize)))
costs = np.zeros((T, bsize))
if prev_h0 is not None:
h0 = prev_h0
else:
for k in xrange(hps.hidden_layers):
h0[k] = tile(self.params['h0'][:, k].reshape(-1, 1), bsize)
bih = self.params['bih']
Wih = self.params['Wih']
Whh = self.params['Whh']
bhh = self.params['bhh']
Who = self.params['Who']
bho = self.params['bho']
# Forward prop
for t in xrange(T):
for k in xrange(hps.hidden_layers):
if t == 0:
hprev = h0[k]
else:
hprev = hs[k][t-1]
if k == 0:
us[k][t] = mult(Wih, data[:, t, :]) + bih
else:
us[k][t] = mult(self.params['Wh%d' % k], hs[k-1][t])
if k == hps.recurrent_layer - 1:
us[k][t] += mult(Whh, hprev) + bhh
# Clip maximum activation
mask = us[k][t] < hps.max_act
us[k][t] = us[k][t] * mask + hps.max_act * (1 - mask)
elif k != 0:
us[k][t] += self.params['bh%d' % k]
hs[k][t] = self.nl(us[k][t])
probs[t] = softmax(mult(Who, hs[-1][t]) + bho)
self.last_h = list()
for k in xrange(hps.hidden_layers):
self.last_h.append(hs[k][-1])
if labels is None:
return None, probs
probs_neg_log = list()
dprobs = list()
for t in xrange(T):
probs_neg_log.append(as_np(-1 * log(probs[t])))
dprobs.append(as_np(probs[t].copy()))
for k in xrange(bsize):
for t in xrange(len(labels[k])):
costs[t, k] = probs_neg_log[t][labels[k][t], k]
dprobs[t][labels[k][t], k] -= 1
for t in xrange(T):
dprobs[t] = array(dprobs[t])
# NOTE Summing costs over time
# NOTE FIXME Dividing by T to get better sense if objective
# is decreasing, remove for grad checking
cost = costs.sum() / bsize / float(T)
if not back:
return cost, probs
# Backprop
for k in self.grads:
self.grads[k][:] = 0
for t in reversed(xrange(T)):
self.grads['bho'] += dprobs[t][:, :].sum(axis=-1).reshape((-1, 1)) / bsize
self.grads['Who'] += mult(dprobs[t], hs[-1][t].T) / bsize
for k in reversed(xrange(hps.hidden_layers)):
if k == hps.hidden_layers - 1:
dhs[k][t] += mult(Who.T, dprobs[t])
else:
dhs[k][t] += mult(self.params['Wh%d' % (k+1)].T, dhs[k+1][t])
dus[k][t] += get_nl_grad(self.hps.nl, us[k][t]) * dhs[k][t]
if k > 0:
self.grads['Wh%d' % k] += mult(dus[k][t], hs[k-1][t].T) / bsize
self.grads['bh%d' % k] += dus[k][t].sum(axis=-1).reshape((-1, 1)) / bsize
if k == hps.recurrent_layer - 1:
if t == 0:
hprev = h0[k]
self.grads['h0'][:, k] = mult(Whh.T, dus[k][t]).sum(axis=-1) / bsize
else:
hprev = hs[k][t-1]
dhs[k][t-1] = mult(Whh.T, dus[k][t])
self.grads['Whh'] += mult(dus[k][t], hprev.T) / bsize
self.grads['bhh'] += dus[k][t].sum(axis=-1).reshape((-1, 1)) / bsize
self.grads['Wih'] += mult(dus[0][t], data[:, t, :].T) / bsize
self.grads['bih'] += dus[0][t].sum(axis=-1).reshape((-1, 1)) / bsize
return cost, self.grads
def cost_and_grad(self, data, labels, back=True, prev_h0=None):
hps = self.hps
T = data.shape[1]
bsize = data.shape[2]
# FIXME gnumpy reallocates if try and use same parameters?
#us = self.us[:, 0:T, 0:bsize]
#dus = self.dus[:, 0:T, 0:bsize]
#hs = self.hs[:, 0:T, 0:bsize]
#dhs = self.dhs[:, 0:T, 0:bsize]
#probs = self.probs[:, 0:T, 0:bsize]
#dprobs = self.dprobs[:, 0:T, 0:bsize]
#costs = self.costs[0:T, 0:bsize]
us = list()
dus = list()
hs = list()
dhs = list()
h0 = list()
for k in xrange(hps.hidden_layers):
us.append(list())
dus.append(list())
hs.append(list())
dhs.append(list())
h0.append(empty((hps.hidden_size, bsize)))
for t in xrange(T):
us[k].append(zeros((hps.hidden_size, bsize)))
dus[k].append(zeros((hps.hidden_size, bsize)))
hs[k].append(zeros((hps.hidden_size, bsize)))
dhs[k].append(zeros((hps.hidden_size, bsize)))
probs = list()
for t in xrange(T):
probs.append(zeros((hps.output_size, bsize)))
costs = np.zeros((T, bsize))
if prev_h0 is not None:
h0 = prev_h0
else:
for k in xrange(hps.hidden_layers):
h0[k] = tile(self.params['h0'][:, k].reshape(-1, 1), bsize)
bih = self.params['bih']
Wih = self.params['Wih']
Whh = self.params['Whh']
bhh = self.params['bhh']
Who = self.params['Who']
bho = self.params['bho']
# Forward prop
for t in xrange(T):
for k in xrange(hps.hidden_layers):
if t == 0:
hprev = h0[k]
else:
hprev = hs[k][t - 1]
if k == 0:
us[k][t] = mult(Wih, data[:, t, :]) + bih
else:
us[k][t] = mult(self.params['Wh%d' % k], hs[k - 1][t])
if k == hps.recurrent_layer - 1:
us[k][t] += mult(Whh, hprev) + bhh
# Clip maximum activation
mask = us[k][t] < hps.max_act
us[k][t] = us[k][t] * mask + hps.max_act * (1 - mask)
elif k != 0:
us[k][t] += self.params['bh%d' % k]
hs[k][t] = self.nl(us[k][t])
probs[t] = softmax(mult(Who, hs[-1][t]) + bho)
self.last_h = list()
for k in xrange(hps.hidden_layers):
self.last_h.append(hs[k][-1])
if labels is None:
return None, probs
probs_neg_log = list()
dprobs = list()
for t in xrange(T):
probs_neg_log.append(as_np(-1 * log(probs[t])))
dprobs.append(as_np(probs[t].copy()))
for k in xrange(bsize):
for t in xrange(len(labels[k])):
costs[t, k] = probs_neg_log[t][labels[k][t], k]
dprobs[t][labels[k][t], k] -= 1
for t in xrange(T):
dprobs[t] = array(dprobs[t])
# NOTE Summing costs over time
# NOTE FIXME Dividing by T to get better sense if objective
# is decreasing, remove for grad checking
cost = costs.sum() / bsize / float(T)
if not back:
return cost, probs
# Backprop
for k in self.grads:
self.grads[k][:] = 0
for t in reversed(xrange(T)):
self.grads['bho'] += dprobs[t][:, :].sum(axis=-1).reshape(
(-1, 1)) / bsize
self.grads['Who'] += mult(dprobs[t], hs[-1][t].T) / bsize
for k in reversed(xrange(hps.hidden_layers)):
if k == hps.hidden_layers - 1:
dhs[k][t] += mult(Who.T, dprobs[t])
else:
dhs[k][t] += mult(self.params['Wh%d' % (k + 1)].T,
dhs[k + 1][t])
dus[k][t] += get_nl_grad(self.hps.nl, us[k][t]) * dhs[k][t]
if k > 0:
self.grads['Wh%d' %
k] += mult(dus[k][t], hs[k - 1][t].T) / bsize
self.grads['bh%d' % k] += dus[k][t].sum(axis=-1).reshape(
(-1, 1)) / bsize
if k == hps.recurrent_layer - 1:
if t == 0:
hprev = h0[k]
self.grads['h0'][:, k] = mult(
Whh.T, dus[k][t]).sum(axis=-1) / bsize
else:
hprev = hs[k][t - 1]
dhs[k][t - 1] = mult(Whh.T, dus[k][t])
self.grads['Whh'] += mult(dus[k][t], hprev.T) / bsize
self.grads['bhh'] += dus[k][t].sum(axis=-1).reshape(
(-1, 1)) / bsize
self.grads['Wih'] += mult(dus[0][t], data[:, t, :].T) / bsize
self.grads['bih'] += dus[0][t].sum(axis=-1).reshape(
(-1, 1)) / bsize
return cost, self.grads
def one_hot(data, n):
data_1h = np.zeros((n, data.shape[0], data.shape[1]))
for t in xrange(data.shape[0]):
for b in xrange(data.shape[1]):
data_1h[data[t, b], t, b] = 1
return array(data_1h)
def cost_and_grad(self, data, labels, back=True):
hps = self.hps
grads = self.grads
# May not be full batch size if at end of dataset
bsize = data.shape[-1]
p = ParamStruct(**self.params)
# Forward prop
acts = list()
acts.append(self.nl(mult(p.Wih, data) + p.bih))
for k in xrange(hps.hidden_layers - 1):
W = self.params['W%d' % (k + 1)]
b = self.params['b%d' % (k + 1)]
acts.append(self.nl(mult(W, acts[-1]) + b))
y = mult(p.Who, acts[-1]) + p.bho
probs = softmax(y)
if labels is None:
return None, probs
# NOTE For more precision if necessary convert to nparray early
cost_array = np.empty(bsize, dtype=np.float64)
# Speed things up by doing assignments off gpu
neg_log_prob = -1 * np.log(as_np(probs))
for k in xrange(bsize):
cost_array[k] = neg_log_prob[labels[k], k]
cost = cost_array.sum() / bsize
if not back:
return cost, probs
# Backprop
for k in self.grads:
self.grads[k][:] = 0
# Do assignments off GPU to speed things up
dLdy = as_np(probs)
# NOTE This changes probs
for k in xrange(bsize):
dLdy[labels[k], k] -= 1
dLdy = array(dLdy)
grads['bho'] = dLdy.sum(axis=1).reshape((-1, 1))
grads['Who'] = mult(dLdy, acts[-1].T)
Ws = [p.Wih] + [
self.params['W%d' % (k + 1)] for k in xrange(hps.hidden_layers - 1)
] + [p.Who]
deltas = [dLdy]
for k in reversed(xrange(hps.hidden_layers - 1)):
delta = get_nl_grad(self.hps.nl, acts[k + 1]) * mult(
Ws[k + 2].T, deltas[-1])
deltas.append(delta)
grads['b%d' % (k + 1)] = delta.sum(axis=1).reshape((-1, 1))
grads['W%d' % (k + 1)] = mult(delta, acts[k].T)
delta = get_nl_grad(self.hps.nl, acts[0]) * mult(Ws[1].T, deltas[-1])
grads['bih'] = delta.sum(axis=1).reshape((-1, 1))
grads['Wih'] = mult(delta, data.T)
# Normalize
for k in self.grads:
self.grads[k] /= bsize
return cost, self.grads
def cost_and_grad(self, data, labels, back=True):
hps = self.hps
grads = self.grads
# May not be full batch size if at end of dataset
bsize = data.shape[-1]
p = ParamStruct(**self.params)
# Forward prop
acts = list()
acts.append(self.nl(mult(p.Wih, data) + p.bih))
for k in xrange(hps.hidden_layers - 1):
W = self.params['W%d' % (k+1)]
b = self.params['b%d' % (k+1)]
acts.append(self.nl(mult(W, acts[-1]) + b))
y = mult(p.Who, acts[-1]) + p.bho
probs = softmax(y)
if labels is None:
return None, probs
# NOTE For more precision if necessary convert to nparray early
cost_array = np.empty(bsize, dtype=np.float64)
# Speed things up by doing assignments off gpu
neg_log_prob = -1 * np.log(as_np(probs))
for k in xrange(bsize):
cost_array[k] = neg_log_prob[labels[k], k]
cost = cost_array.sum() / bsize
if not back:
return cost, probs
# Backprop
for k in self.grads:
self.grads[k][:] = 0
# Do assignments off GPU to speed things up
dLdy = as_np(probs)
# NOTE This changes probs
for k in xrange(bsize):
dLdy[labels[k], k] -= 1
dLdy = array(dLdy)
grads['bho'] = dLdy.sum(axis=1).reshape((-1, 1))
grads['Who'] = mult(dLdy, acts[-1].T)
Ws = [p.Wih] + [self.params['W%d' % (k+1)] for k in xrange(hps.hidden_layers - 1)] + [p.Who]
deltas = [dLdy]
for k in reversed(xrange(hps.hidden_layers - 1)):
delta = get_nl_grad(self.hps.nl, acts[k+1]) * mult(Ws[k + 2].T, deltas[-1])
deltas.append(delta)
grads['b%d' % (k+1)] = delta.sum(axis=1).reshape((-1, 1))
grads['W%d' % (k+1)] = mult(delta, acts[k].T)
delta = get_nl_grad(self.hps.nl, acts[0]) * mult(Ws[1].T, deltas[-1])
grads['bih'] = delta.sum(axis=1).reshape((-1, 1))
grads['Wih'] = mult(delta, data.T)
# Normalize
for k in self.grads:
self.grads[k] /= bsize
return cost, self.grads