bprop_len = 35 # length of truncated BPTT
grad_clip = 5 # gradient norm threshold to clip
INPUT_LEN = 20
OUTPUT_LEN = 20
TRAINING_DIV = 200
# Prepare RNNLM model
# model = FunctionSet(embed=F.EmbedID(INPUT_LEN, n_units),
# l1_x =F.Linear(n_units, 4 * n_units),
# l1_h =F.Linear(n_units, 4 * n_units),
# l2_x =F.Linear(n_units, 4 * n_units),
# l2_h =F.Linear(n_units, 4 * n_units),
# l3 =F.Linear(n_units, OUTPUT_LEN))
model = FunctionSet(embed=F.EmbedID(4, n_units),
l1_x =F.Linear(n_units, 4 * n_units),
l1_h =F.Linear(n_units, 4 * n_units),
l2_x =F.Linear(n_units, 4 * n_units),
l2_h =F.Linear(n_units, 4 * n_units),
l3 =F.Linear(n_units, 4))
for param in model.parameters:
param[:] = np.random.uniform(-0.1, 0.1, param.shape)
# Neural net architecture
def forward_one_step(x_data, y_data, state, train=True):
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
h0 = model.embed(x)
h1_in = model.l1_x(F.dropout(h0, train=train)) + model.l1_h(state['h1'])
c1, h1 = F.lstm(state['c1'], h1_in)
h2_in = model.l2_x(F.dropout(h1, train=train)) + model.l2_h(state['h2'])
print('loading ROI dataset')
x_train, _ = fetch_roidata(which_set='train')
x_test, _ = fetch_roidata(which_set='test')
N, N_test = len(x_train), len(x_test)
x_train = x_train.astype(np.float32)
# y_train = y_train.astype(np.float32)
x_test = x_test.astype(np.float32)
# y_test = y_test.astype(np.float32)
print('N:', N)
print('N_test:', N_test)
# Prepare multi-layer perceptron model
model = chainer.FunctionSet(
l1=F.Linear(8712, 4000),
l2=F.Linear(4000, 1000),
l3=F.Linear(1000, 200),
l4=F.Linear(200, 20),
l5=F.Linear(20, 200),
l6=F.Linear(200, 1000),
l7=F.Linear(1000, 4000),
l8=F.Linear(4000, 8712),
)
if args.gpu >= 0:
print('converting model to gpu')
cuda.get_device(args.gpu).use()
model.to_gpu()
def forward(x_data, y_data, train=True):
def __init__(self, n_input, n_output):
super(Autoencoder, self).__init__(encoder=F.Linear(n_input, n_output),
decoder=F.Linear(n_output, n_input))
n_units = 30
# Prepare dataset
print 'fetch diabetes dataset'
diabetes = load_diabetes()
data = diabetes['data'].astype(np.float32)
target = diabetes['target'].astype(np.float32).reshape(len(diabetes['target']),
1)
N = batchsize * 30
x_train, x_test = np.split(data, [N])
y_train, y_test = np.split(target, [N])
N_test = y_test.size
# Prepare multi-layer perceptron model
model = FunctionSet(l1=F.Linear(10, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, 1))
if args.gpu >= 0:
cuda.init(args.gpu)
model.to_gpu()
# Neural net architecture
def forward(x_data, y_data, train=True):
x, t = Variable(x_data), Variable(y_data)
h1 = F.dropout(F.relu(model.l1(x)), train=train)
h2 = F.dropout(F.relu(model.l2(h1)), train=train)
y = model.l3(h2)
return F.mean_squared_error(y, t), y
# coding: UTF-8
# Chainerを使ってXORを学習する
import chainer
import chainer.functions as F
from chainer import optimizers
import numpy as np
np.random.seed(0)
TIME = 3000
middle_units = 2
# ネットワークモデル
model = chainer.FunctionSet(l1=F.Linear(2, middle_units),
l2=F.Linear(middle_units, 1))
def forward(x_data, y_data):
x = chainer.Variable(x_data)
y = chainer.Variable(y_data)
X1 = model.l1(x)
out1 = F.sigmoid(X1)
X2 = model.l2(out1)
out2 = F.sigmoid(X2)
return F.mean_squared_error(out2, y), out2
x_train = np.array([[0,0], [0,1], [1,0], [1,1]], dtype=np.float32)
y_train = np.array([[0], [1], [1], [0]], dtype=np.float32)
datasize = len(x_train)
acc_node = 100.0 * result['correct_node'] / result['total_node']
acc_root = 100.0 * result['correct_root'] / result['total_root']
print(' Node accuracy: {0:.2f} %% ({1:,d}/{2:,d})'.format(
acc_node, result['correct_node'], result['total_node']))
print(' Root accuracy: {0:.2f} %% ({1:,d}/{2:,d})'.format(
acc_root, result['correct_root'], result['total_root']))
vocab = {}
train_trees = read_corpus('trees/train.txt', vocab)
test_trees = read_corpus('trees/test.txt', vocab)
develop_trees = read_corpus('trees/dev.txt', vocab)
model = chainer.FunctionSet(
embed=F.EmbedID(len(vocab), n_units),
l=F.Linear(n_units * 2, n_units),
w=F.Linear(n_units, n_label),
)
if args.gpu >= 0:
cuda.init()
model.to_gpu()
# Setup optimizer
optimizer = optimizers.AdaGrad(lr=0.1)
optimizer.setup(model)
accum_loss = 0
count = 0
start_at = time.time()
cur_at = start_at
length = int(np.sqrt(imagesize))
cross_perm = np.random.permutation(N)
for opt in optimizer_list:
print '========================='
print 'Set Optimizer : ' + opt
cross_acc_sum = 0
cross_train_loss = []
cross_train_acc = []
cross_test_loss = []
cross_test_acc = []
for k in range(cross):
print '-------------------------'
print 'Cross Validation : ' + str(k + 1)
model = FunctionSet(conv1=F.Convolution2D(1, 28, 5),
conv2=F.Convolution2D(28, 28, 5),
l3=F.Linear(252, 252),
l4=F.Linear(252, 10))
optimizer = cross_optimizers(opt)
optimizer.setup(model)
x_train, x_test, y_train, y_test = cross_split(
mnist.data, mnist.target, cross, k, cross_perm, N, length)
N_train = x_train.shape[0]
cross_acc = 0
train_loss = []
train_acc = []
test_loss = []
test_acc = []
for epoch in range(1, n_epoch + 1):
print 'epoch' + str(epoch)
loss_sum, acc_sum = 0, 0
perm = np.random.permutation(N_train)
mnist.data = mnist.data.astype(np.float32)
mnist.data /= 255
mnist.target = mnist.target.astype(np.int32)
N = 60000
x_train, x_test = np.split(mnist.data, [N])
y_train, y_test = np.split(mnist.target, [N])
N_test = y_test.size
# Prepare the multi-layer perceptron model
# Note that the model splits into two GPUs at the first layer,
# and share their activations only at third and sixth layers.
cuda.init()
wscale = math.sqrt(2)
model = FunctionSet(gpu0=FunctionSet(l1=F.Linear(784,
n_units // 2,
wscale=wscale),
l2=F.Linear(n_units // 2,
n_units // 2,
wscale=wscale),
l3=F.Linear(n_units // 2,
n_units,
wscale=wscale),
l4=F.Linear(n_units,
n_units // 2,
wscale=wscale),
l5=F.Linear(n_units // 2,
n_units // 2,
wscale=wscale),
l6=F.Linear(n_units // 2,
10,
def __init__(self, n_input, n_hidden, n_output):
super(LSTMmodel, self).__init__(l1_x=F.Linear(n_input, 4 * n_hidden),
l1_h=F.Linear(n_hidden, 4 * n_hidden),
l2_x=F.Linear(n_hidden, 4 * n_hidden),
l2_h=F.Linear(n_hidden, 4 * n_hidden),
l3=F.Linear(n_hidden, n_output))
rawim = np.copy(im).astype('uint8')
# Shuffle axes to c01
im = np.swapaxes(np.swapaxes(im, 1, 2), 0, 1)
# Convert to BGR
im = im[::-1, :, :]
im = im - MEAN_VALUES
return rawim.transpose(2, 0, 1).astype(np.float32)
#Model Preparation
print "preparing caption generation models"
model = FunctionSet()
model.img_feature2vec = F.Linear(image_feature_dim,
n_units) #CNN(I)の最後のレイヤーに相当。#parameter W,b
model.embed = F.EmbedID(len(vocab), n_units) #W_e*S_tに相当 #parameter W
model.l1_x = F.Linear(n_units, 4 * n_units) #parameter W,b
model.l1_h = F.Linear(n_units, 4 * n_units) #parameter W,b
model.out = F.Linear(n_units, len(vocab)) #parameter W,b
serializers.load_hdf5(model_place, model)
#To GPU
if gpu_id >= 0:
model.to_gpu()
print "done"
#Define Newtowork (Forward)
test_data = mnist.data[60000:]
##############################################################################
depth = 2 ###
order = (('enc1', 'enc2'), ('dec2', 'dec1')) ###
#model = chainer.FunctionSet(
# enc1=F.Linear(n_units[0], n_units[1]),
# enc2=F.Linear(n_units[1], n_units[2]),
# dec2=F.Linear(n_units[2], n_units[1]),
# dec1=F.Linear(n_units[1], n_units[0])
#)
if args.model == '': ###
model = chainer.FunctionSet(
enc1=F.Linear(n_units[0], n_units[1]),
enc2=F.Linear(n_units[1], n_units[2]),
dec2=F.Linear(n_units[2], n_units[1]),
dec1=F.Linear(n_units[1], n_units[0])
)
else: ###
model = pickle.load(open(args.model, 'rb')) ###
##############################################################################
# GPU使用の時はGPUにモデルを転送
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
##############################################################################
def __init__(self):
print "Initializing DQN..."
# Initialization of Chainer 1.1.0 or older.
# print "CUDA init"
# cuda.init()
print "Model Building"
# self.model = FunctionSet(
# l1=F.Convolution2D(4, 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
# l2=F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
# l3=F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2)),
# l4=F.Linear(3136, 512, wscale=np.sqrt(2)),
# q_value=F.Linear(512, self.num_of_actions,
# initialW=np.zeros((self.num_of_actions, 512),
# dtype=np.float32))
# ).to_gpu()
# self.critic = FunctionSet(
# l1=F.Linear(self.num_of_actions+self.num_of_states,512),
# l2=F.Linear(512,256),
# l3=F.Linear(256,128),
# q_value=F.Linear(128,1,initialW=np.zeros((1,128),dtype=np.float32))
# ).to_gpu()
#
# self.actor = FunctionSet(
# l1=F.Linear(self.num_of_states,512),
# l2=F.Linear(512,256),
# l3=F.Linear(256,128),
# a_value=F.Linear(128,self.num_of_actions,initialW=np.zeros((1,128),dtype=np.float32))
# ).to_gpu()
self.critic = FunctionSet(
l1=F.Linear(self.num_of_actions+self.num_of_states,1024),
l2=F.Linear(1024,512),
l3=F.Linear(512,256),
l4=F.Linear(256,128),
q_value=F.Linear(128,1,initialW=np.zeros((1,128),dtype=np.float32))
).to_gpu()
self.actor = FunctionSet(
l1=F.Linear(self.num_of_states,1024),
l2=F.Linear(1024,512),
l3=F.Linear(512,256),
l4=F.Linear(256,128),
a_value=F.Linear(128,self.num_of_actions,initialW=np.zeros((1,128),dtype=np.float32))
).to_gpu()
# self.critic = FunctionSet(
# l1=F.Linear(self.num_of_actions+self.num_of_states,1024,wscale=0.01*math.sqrt(self.num_of_actions+self.num_of_states)),
# l2=F.Linear(1024,512,wscale=0.01*math.sqrt(1024)),
# l3=F.Linear(512,256,wscale=0.01*math.sqrt(512)),
# l4=F.Linear(256,128,wscale=0.01*math.sqrt(256)),
# q_value=F.Linear(128,1,wscale=0.01*math.sqrt(128))
# ).to_gpu()
#
# self.actor = FunctionSet(
# l1=F.Linear(self.num_of_states,1024,wscale=0.01*math.sqrt(self.num_of_states)),
# l2=F.Linear(1024,512,wscale=0.01*math.sqrt(1024)),
# l3=F.Linear(512,256,wscale=0.01*math.sqrt(512)),
# l4=F.Linear(256,128,wscale=0.01*math.sqrt(256)),
# a_value=F.Linear(128,self.num_of_actions,wscale=0.01*math.sqrt(128))
# ).to_gpu()
self.critic_target = copy.deepcopy(self.critic)
self.actor_target = copy.deepcopy(self.actor)
print "Initizlizing Optimizer"
#self.optim_critic = optimizers.RMSpropGraves(lr=0.0001, alpha=0.95, momentum=0.95, eps=0.0001)
#self.optim_actor = optimizers.RMSpropGraves(lr=0.0001, alpha=0.95, momentum=0.95, eps=0.0001)
self.optim_critic = optimizers.Adam(alpha=0.00001)
self.optim_actor = optimizers.Adam(alpha=0.00001)
self.optim_critic.setup(self.critic)
self.optim_actor.setup(self.actor)
# self.optim_critic.add_hook(chainer.optimizer.WeightDecay(0.00001))
# self.optim_critic.add_hook(chainer.optimizer.GradientClipping(10))
# self.optim_actor.add_hook(chainer.optimizer.WeightDecay(0.00001))
# self.optim_actor.add_hook(chainer.optimizer.GradientClipping(10))
# History Data : D=[s, a, r, s_dash, end_episode_flag]
self.D = [np.zeros((self.data_size, self.num_of_states), dtype=np.float32),
np.zeros((self.data_size, self.num_of_actions), dtype=np.float32),
np.zeros((self.data_size, 1), dtype=np.float32),
np.zeros((self.data_size, self.num_of_states), dtype=np.float32),
np.zeros((self.data_size, 1), dtype=np.bool)]
from chainer import Chain, Variable, optimizers
import chainer.functions as F
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.externals import joblib
import numpy as np
# 学習データとテストデータに分ける
data_train, data_test, label_train, label_test = joblib.load("mnist")
data_train = np.asarray(data_train, np.float32)
data_test = np.asarray(data_test, np.float32)
label_train = np.asarray(label_train, np.int32)
label_test = np.asarray(label_test, np.int32)
# 層のパラメータ
model = Chain(
l1=F.Linear(784, 200),
l2=F.Linear(200, 100),
l3=F.Linear(100, 10))
# 伝播のさせかた
def forward(x, is_train=True):
h1 = F.dropout(F.relu(model.l1(x)), train=is_train)
h2 = F.dropout(F.relu(model.l2(h1)), train=is_train)
p = model.l3(h2)
return p
# 学習のさせかた
optimizer = optimizers.Adam()
optimizer.setup(model)
def Main():
import argparse
import numpy as np
from chainer import cuda, Variable, FunctionSet, optimizers
import chainer.functions as F
parser = argparse.ArgumentParser(description='Chainer example: regression')
parser.add_argument('--gpu', '-g', default=-1, type=int,
help='GPU ID (negative value indicates CPU)')
args = parser.parse_args()
batchsize = 10
n_epoch = NEpoch
n_units = 300 #TEST
# Prepare dataset
data_x, data_y = LoadData()
batchsize= max(1,min(batchsize, len(data_y)/20)) #TEST: adjust batchsize
#dx2,dy2=GenData(300, noise=0.0); data_x.extend(dx2); data_y.extend(dy2)
data = np.array(data_x).astype(np.float32)
target = np.array(data_y).astype(np.int32) #DIFF_REG
N= len(data) #batchsize * 30
x_train= data
y_train= target
#For test:
mi,ma,me= GetStat(data_x)
f_reduce=lambda xa:[xa[0],xa[1]]
f_repair=lambda xa:[xa[0],xa[1]]
nt= 20+1
N_test= nt*nt
x_test= np.array(sum([[f_repair([x1,x2]) for x2 in FRange1(f_reduce(mi)[1],f_reduce(ma)[1],nt)] for x1 in FRange1(f_reduce(mi)[0],f_reduce(ma)[0],nt)],[])).astype(np.float32)
y_test= np.array([0.0 for x in x_test]).astype(np.int32) #DIFF_REG
#No true test data (just for plotting)
print 'Num of samples for train:',len(y_train),'batchsize:',batchsize
# Dump data for plot:
DumpData('/tmp/nn/smpl_train.dat', x_train, [[y] for y in y_train], f_reduce) #DIFF_REG
# Prepare multi-layer perceptron model
model = FunctionSet(l1=F.Linear(2, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, 3))
#TEST: Random bias initialization
#, bias=Rand()
#model.l1.b[:]= [Rand() for k in range(n_units)]
#model.l2.b[:]= [Rand() for k in range(n_units)]
#model.l3.b[:]= [Rand() for k in range(1)]
#print model.l2.__dict__
if args.gpu >= 0:
cuda.init(args.gpu)
model.to_gpu()
# Neural net architecture
def forward(x_data, y_data, train=True):
#train= False #TEST: Turn off dropout
dratio= 0.2 #0.5 #TEST: Dropout ratio
x, t = Variable(x_data), Variable(y_data)
h1 = F.dropout(F.relu(model.l1(x)), ratio=dratio, train=train)
h2 = F.dropout(F.relu(model.l2(h1)), ratio=dratio, train=train)
#h1 = F.dropout(F.leaky_relu(model.l1(x),slope=0.2), ratio=dratio, train=train)
#h2 = F.dropout(F.leaky_relu(model.l2(h1),slope=0.2), ratio=dratio, train=train)
#h1 = F.dropout(F.sigmoid(model.l1(x)), ratio=dratio, train=train)
#h2 = F.dropout(F.sigmoid(model.l2(h1)), ratio=dratio, train=train)
#h1 = F.dropout(F.tanh(model.l1(x)), ratio=dratio, train=train)
#h2 = F.dropout(F.tanh(model.l2(h1)), ratio=dratio, train=train)
#h1 = F.dropout(model.l1(x), ratio=dratio, train=train)
#h2 = F.dropout(model.l2(h1), ratio=dratio, train=train)
#h1 = F.relu(model.l1(x))
#h2 = F.relu(model.l2(h1))
#h1 = model.l1(x)
#h2 = model.l2(h1)
y = model.l3(h2)
#return F.mean_squared_error(y, t), y
return F.softmax_cross_entropy(y, t), F.softmax(y) #DIFF_REG
# Setup optimizer
optimizer = optimizers.AdaDelta(rho=0.9)
#optimizer = optimizers.AdaGrad(lr=0.5)
#optimizer = optimizers.RMSprop()
#optimizer = optimizers.MomentumSGD()
#optimizer = optimizers.SGD(lr=0.8)
optimizer.setup(model.collect_parameters())
# Learning loop
for epoch in xrange(1, n_epoch+1):
print 'epoch', epoch
# training
perm = np.random.permutation(N)
sum_loss = 0
for i in xrange(0, N, batchsize):
x_batch = x_train[perm[i:i+batchsize]]
y_batch = y_train[perm[i:i+batchsize]]
if args.gpu >= 0:
x_batch = cuda.to_gpu(x_batch)
y_batch = cuda.to_gpu(y_batch)
optimizer.zero_grads()
loss, pred = forward(x_batch, y_batch)
loss.backward() #Computing gradients
optimizer.update()
sum_loss += float(cuda.to_cpu(loss.data)) * batchsize
print 'train mean loss={}'.format(
sum_loss / N)
if epoch%10==0:
#'''
# testing all data
preds = []
x_batch = x_test[:]
y_batch = y_test[:]
if args.gpu >= 0:
x_batch = cuda.to_gpu(x_batch)
y_batch = cuda.to_gpu(y_batch)
loss, pred = forward(x_batch, y_batch, train=False)
preds = cuda.to_cpu(pred.data)
sum_loss = float(cuda.to_cpu(loss.data)) * len(y_test)
#'''
print 'test mean loss={}'.format(
sum_loss / N_test)
# Dump data for plot:
y_pred= [[y.index(max(y))]+y for y in preds.tolist()] #DIFF_REG
DumpData('/tmp/nn/nn_test%04i.dat'%epoch, x_test, y_pred, f_reduce, lb=nt+1)
return input_data, label
# stack results
lstm_errors_mean = np.zeros(len(list_n_units))
lstm_errors_se = np.zeros(len(list_n_units))
svm_errors_mean = np.zeros(len(list_n_units))
svm_errors_se = np.zeros(len(list_n_units))
# for loop on different n_units
for j in range(len(list_n_units)):
n_units = list_n_units[j]
# model
model = chainer.FunctionSet(x_to_h=F.Linear(64, n_units * 4),
h_to_h=F.Linear(n_units, n_units * 4),
h_to_y=F.Linear(n_units, 60))
if args.gpu >= 0:
cuda.check_cuda_available()
cuda.get_device(args.gpu).use()
model.to_gpu()
# optimizer
optimizer = optimizers.SGD(lr=1.)
optimizer.setup(model.collect_parameters())
# loop initialization
jump = whole_len // batchsize # = whole len
cur_log_perp = mod.zeros(())
start_at = time.time()
type=int,
help='GPU ID (negative value indicates CPU)')
args = parser.parse_args()
mod = cuda.cupy if args.gpu >= 0 else np
# validation dataset
valid_data_stack = []
for i in range(valid_iter):
valid_data = DatasetGenerator(maze_size).generate_seq(100, offset_timing)
valid_data_stack.append(valid_data)
# test dataset
test_data = DatasetGenerator(maze_size).generate_seq(100, offset_timing)
# model
model = chainer.FunctionSet(x_to_h=F.Linear(64, n_units * 4),
h_to_h=F.Linear(n_units, n_units * 4),
h_to_y=F.Linear(n_units,
maze_size[0] * maze_size[1]))
if args.gpu >= 0:
cuda.check_cuda_available()
cuda.get_device(args.gpu).use()
model.to_gpu()
# optimizer
optimizer = optimizers.SGD(lr=1.)
optimizer.setup(model.collect_parameters())
# one-step forward propagation
def forward_one_step(data, targets, state, train=True):
def init_model():
#Make models
if use_pre2 == 'pre': pre_unit = 4
else: pre_unit = 0
if use_null == 'null': null_unit = 6
else: null_unit = 0
if args.phrase == 'phrase':
phrase_unit = 4
model = chainer.FunctionSet(
trainable=chainer.FunctionSet(
w0=F.Linear(n_units * 2 + null_unit * 2, n_label),
ww0=F.Linear(
n_units * 2 + pre_unit + null_unit * 2 + phrase_unit,
n_units + null_unit),
ww1=F.Linear(
n_units * 2 + pre_unit + null_unit * 2 + phrase_unit,
n_units + null_unit),
),
w1_f=F.Linear(n_units * 2 + null_unit * 2,
n_units + null_unit), #source input
w2_f=F.Linear(n_units + null_unit,
n_units * 2 + null_unit * 2), #source output
w1_e=F.Linear(n_units * 2 + null_unit * 2,
n_units + null_unit), #target input
w2_e=F.Linear(n_units + null_unit,
n_units * 2 + null_unit * 2), #target output
embed_f=F.EmbedID(vocab_f['len_vocab'],
n_units), #source word embedding
embed_e=F.EmbedID(vocab_e['len_vocab'],
n_units), #target word embedding
)
else:
model = chainer.FunctionSet(
trainable=chainer.FunctionSet(w0=F.Linear(
n_units * 4 + null_unit * 4, n_label), ),
w1_f=F.Linear(n_units * 2 + null_unit * 2,
n_units + null_unit), #source input
w2_f=F.Linear(n_units + null_unit,
n_units * 2 + null_unit * 2), #source output
w1_e=F.Linear(n_units * 2 + null_unit * 2,
n_units + null_unit), #target input
w2_e=F.Linear(n_units + null_unit,
n_units * 2 + null_unit * 2), #target output
embed_f=F.EmbedID(vocab_f['len_vocab'],
n_units), #source word embedding
embed_e=F.EmbedID(vocab_e['len_vocab'],
n_units), #target word embedding
)
if opt_name == 'SGD':
optimizer = optimizers.SGD(lr=0.02) # (lr=opt_score) # lr=0.01
elif opt_name == 'AdaGrad':
optimizer = optimizers.AdaGrad(lr=0.001) # (lr=opt_score) # lr=0.001
elif opt_name == 'AdaDelta':
optimizer = optimizers.AdaDelta(rho=0.9) # (rho=opt_score) # rho=0.9
elif opt_name == 'Adam':
optimizer = optimizers.Adam(
alpha=0.0001) # (alpha=opt_score) # alpha=0.0001
optimizer.setup(model) # .collect_parameters()
return model, optimizer
def __init__(self, input_dim=748, n_units=1000):
super(NN3_Model, self).__init__()
self.n_units = n_units
self.model = FunctionSet(l1=F.Linear(input_dim, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, 2))
with open('../work/attr_triple_id2image_id.pkl', 'r') as f:
attr_triple_id2image_id = pickle.load(f)
with open('../work/index2attribute.pkl', 'r') as f:
index2attribute = pickle.load(f)
attribute2index = dict((v, k) for k, v in index2attribute.iteritems())
#Model Preparation
print "preparing model"
image_feature_dim = 1024 #image feature dimention per image
n_units = 128 # number of units per layer
vocab_size = len(attribute2index)
model = chainer.FunctionSet()
model.img_feature2vec = F.Linear(2 * image_feature_dim,
n_units) #parameter W,b
model.bn_feature = F.BatchNormalization(n_units) #parameter sigma,gamma
model.h1 = F.Linear(n_units, n_units) #hidden unit,#parameter W,b
model.bn1 = F.BatchNormalization(n_units) #parameter gamma,beta
model.out = F.Linear(n_units, vocab_size) #parameter W,b
#To GPU
if gpu_id >= 0:
model.to_gpu()
#Define Newtowork (Forward)
def forward(x_data, y_data, train=True):
x = Variable(x_data, volatile=not train)
t = Variable(y_data, volatile=not train)
feature_input = F.relu(model.bn_feature(model.img_feature2vec(x)))
import chainer
import chainer.functions as F
import chainer.optimizers as Opt
import numpy
from sklearn.datasets import fetch_mldata
from libdnn import StackedAutoEncoder
model = chainer.FunctionSet(enc1=F.Linear(28**2, 200),
enc2=F.Linear(200, 30),
dec2=F.Linear(30, 200),
dec1=F.Linear(200, 28**2))
def encode(self, x, layer, train):
if train:
x = F.dropout(x, ratio=0.2, train=train)
if layer == 0:
return x
x = F.sigmoid(self.model.enc1(x))
if layer == 1:
return x
x = F.sigmoid(self.model.enc2(x))
if layer == 2:
return x
return x
sys.path.append('../')
import chainer
import numpy
import chainer.functions as F
import libdnn.visualizer as V
from libdnn import Classifier
# define 3-layer convolutional neural network
model = chainer.FunctionSet(conv1=F.Convolution2D(1, 15, 5),
bn1=F.BatchNormalization(15),
conv2=F.Convolution2D(15, 30, 3, pad=1),
bn2=F.BatchNormalization(30),
conv3=F.Convolution2D(30, 64, 3, pad=1),
fl4=F.Linear(576, 576),
fl5=F.Linear(576, 10))
def forward(self, x, train):
h = F.max_pooling_2d(F.relu(model.bn1(model.conv1(x))), 2)
h = F.max_pooling_2d(h, 2)
h = F.max_pooling_2d(F.relu(model.bn2(model.conv2(h))), 2)
h = F.max_pooling_2d(F.relu(model.conv3(h)), 2)
h = F.dropout(F.relu(model.fl4(h)), train=True)
y = model.fl5(h)
return y
cnn = Classifier(model, gpu=-1)
# 学習データを画像に変換
def conv_feat_2_image(feats):
data = np.ndarray((len(feats), 1, 28, 28), dtype=np.float32)
for i, f in enumerate(feats):
data[i] = f.reshape(28, 28)
return data
data_train = conv_feat_2_image(data_train)
data_test = conv_feat_2_image(data_test)
# 層のパラメータ
model = Chain(conv1=F.Convolution2D(1, 32, 3),
conv2=F.Convolution2D(32, 64, 3),
l1=F.Linear(576, 200),
l2=F.Linear(200, 100),
l3=F.Linear(100, 10))
# 伝播のさせかた
def forward(x, is_train=True):
h1 = F.max_pooling_2d(F.relu(model.conv1(x)), 3)
h2 = F.max_pooling_2d(F.relu(model.conv2(h1)), 3)
h3 = F.dropout(F.relu(model.l1(h2)), train=is_train)
h4 = F.dropout(F.relu(model.l2(h3)), train=is_train)
p = model.l3(h4)
return p
# 学習のさせかた
x = iris.data.astype(np.float32)
y = iris.target
y = y.flatten().astype(np.int32)
# 学習用データをN個、検証用データを残りの個数と設定
index = np.arange(y.size)
x_train = x[index % 2 != 0, :]
x_test = x[index % 2 == 0, :]
y_train = y[index % 2 != 0]
y_test = y[index % 2 == 0]
N_test = y_test.size
N = y_train.size
# Prepare multi-layerr perceptron model
# 多層パーセプトロンモデルの設定
model = FunctionSet(l1=F.Linear(4, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, 3))
# Neural net architecture
# ニューラルネットワークの構造
def forward(x_data, y_data, train=True):
x, t = Variable(x_data), Variable(y_data)
h1 = F.dropout(F.relu(model.l1(x)), train=train)
h2 = F.dropout(F.relu(model.l2(h1)), train=train)
y = model.l3(h2)
# 他クラス分類なので誤差関数としてソフトマックス関数の
# 交差エントロピー関数を用いて、誤差を導出
return F.softmax_cross_entropy(y, t), F.accuracy(y, t)
x_train = np.array(x[perm[train_idx]], dtype=np.float32)
x_test = np.array(x[perm[test_idx]], dtype=np.float32)
y_train = np.array(Y[perm[train_idx]], dtype=np.int32)
y_test = np.array(Y[perm[test_idx]], dtype=np.int32)
x_train_image = np.array(X[perm[train_idx]], dtype=np.float32)
N = len(x_train)
N_test = len(x_test)
print('train: {}, test: {}'.format(N, N_test))
sys.stdout.flush()
# Prepare multi-layer perceptron model
# 多層パーセプトロンモデルの設定
# 入力 4096次元, 出力 10次元
model = FunctionSet(l1=F.Linear(4096, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, len(labels)))
# Neural net architecture
def forward(x_data, y_data, train=True):
# relu: 負の時は0, 正の時は値をそのまま返す (計算量が小さく学習スピードが速くなることが利点)
# dropout: ランダムに中間層をドロップ(ないものとする)し,過学習を防ぐ
x, t = Variable(x_data,
volatile=not train), Variable(y_data,
volatile=not train)
h1 = F.dropout(F.relu(model.l1(x)), train=train)
h2 = F.dropout(F.relu(model.l2(h1)), train=train)
y = model.l3(h2)
# 多クラス分類なので誤差関数としてソフトマックス関数の
words = open(filename).read().replace('\n', '<eos>').strip().split()
dataset = np.ndarray((len(words),), dtype=np.int32)
for i, word in enumerate(words):
if word not in vocab:
vocab[word] = len(vocab)
dataset[i] = vocab[word]
return dataset
train_data = load_data('ptb.train.txt')
valid_data = load_data('ptb.valid.txt')
test_data = load_data('ptb.test.txt')
print('#vocab =', len(vocab))
# Prepare RNNLM model
model = chainer.FunctionSet(embed=F.EmbedID(len(vocab), n_units),
l1_x=F.Linear(n_units, 4 * n_units),
l1_h=F.Linear(n_units, 4 * n_units),
l2_x=F.Linear(n_units, 4 * n_units),
l2_h=F.Linear(n_units, 4 * n_units),
l3=F.Linear(n_units, len(vocab)))
for param in model.parameters:
param[:] = np.random.uniform(-0.1, 0.1, param.shape)
if args.gpu >= 0:
cuda.get_device(args.gpu).use()
model.to_gpu()
def forward_one_step(x_data, y_data, state, train=True):
# Neural net architecture
x = chainer.Variable(x_data, volatile=not train)
t = chainer.Variable(y_data, volatile=not train)
# main
parser = argparse.ArgumentParser(description='Sandglass: MNIST')
parser.add_argument('--gpu',
'-g',
default=-1,
type=int,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--model', '-m', default=0, help='Trained Model')
args = parser.parse_args()
if args.model != 0:
model = pickle.load(open(args.model, 'rb'))
else:
# モデルを作る
model = chainer.FunctionSet(l1=F.Linear(784, 1000),
l2=F.Linear(1000, 500),
l3=F.Linear(500, 100),
l4=F.Linear(100, 500),
l5=F.Linear(500, 1000),
l6=F.Linear(1000, 784))
if args.gpu >= 0:
cuda.check_cuda_available()
xp = cuda.cupy if args.gpu >= 0 else np
# すでにmnist.pklになっているデータを読み込む
with open('mnist.pkl', 'rb') as mnist_pickle:
mnist = six.moves.cPickle.load(mnist_pickle)
# 画素値を[0.0, 1.0]に正規化する
def __init__(self, n_input, n_output):
super(SLP, self).__init__(transform=F.Linear(n_input, n_output))
lightcurve = dict()
# todo: optimize this by use of sortedcontainers
def goes_future_max(t0, dt):
t = t0
ret = 0
while t <= t0 + dt:
t += datetime.timedelta(minutes=1)
if lightcurve.has_key(t):
ret = max(ret, lightcurve[t])
return ret
# Prepare RNNLM model
model = chainer.FunctionSet(embed=F.Linear(n_inputs, n_units),
l1_x=F.Linear(n_units, 4 * n_units),
l1_h=F.Linear(n_units, 4 * n_units),
l2_x=F.Linear(n_units, 4 * n_units),
l2_h=F.Linear(n_units, 4 * n_units),
l3=F.Linear(n_units, n_outputs))
for param in model.parameters:
param[:] = np.random.uniform(-0.1, 0.1, param.shape)
if args.gpu >= 0:
cuda.init(args.gpu)
model.to_gpu()
def forward_one_step(x_data, state, train=True):
if args.gpu >= 0:
x_data = cuda.to_gpu(x_data)
mnist['data'] /= 255
mnist['target'] = mnist['target'].astype(np.int32)
# トレーニングデータとテストデータに分ける
N = 60000
x_train, x_test = np.split(mnist['data'], [N])
y_train, y_test = np.split(mnist['target'], [N])
N_test = y_test.size
# パラメータ
batchsize = 100
n_epoch = 2
n_units = 1000
# モデルを作る
model = chainer.FunctionSet(l1=F.Linear(784, n_units),
l2=F.Linear(n_units, n_units),
l3=F.Linear(n_units, 10))
# 前向き計算
def forward(x_data, y_data, train=True):
# データの型をchainer.Variableに変換する
x, t = chainer.Variable(x_data), chainer.Variable(y_data)
# train=trueならdropoutする．デフォルトのdropout率は0.5
# 学習の時はtrueにする．学習せずに前向き計算のみのときはfalseにする．
h1 = F.dropout(F.relu(model.l1(x)), train=train)
h2 = F.dropout(F.relu(model.l2(h1)), train=train)
# h2の出力は1000個．それを積和計算しているだけ．
# yは出力層のニューロンの出力．ニューロン数は10．
t_shape = _compute_dimensions(
np.zeros((100, 1, word_vector_size, sentence_length)),
np.zeros((100, )))
# Define elements of the deep convolutionary network
model = FunctionSet(l1=F.Convolution2D(ch_in_1,
ch_out_1,
filter_1,
stride=stride_1,
pad=padding_1),
l2=F.Convolution2D(ch_out_1,
ch_out_2,
filter_2,
stride=stride_2,
pad=padding_2),
l3=F.Linear(t_shape[3] * t_shape[2] * t_shape[1],
linear_out_3))
# Setup GPU if required
if args.gpu >= 0:
print "\ninitializing graphical processing unit..."
cuda.init(args.gpu)
model.to_gpu()
# Setup optimizer
optimizer = optimizers.Adam()
optimizer.setup(model.collect_parameters())
# use momentum SGD first
# is_sgd = False
max_accuracy = 0
