def train_leam(self,gpu_id=0,epoch_size=1000,max_epoch=15,batch_size=128):
train_x, test_x, train_y, test_y = train_test_split(self.id_texts, self.labels_multi, random_state=0)
train = datasets.TupleDataset(train_x, train_y)
test = datasets.TupleDataset(test_x, test_y)
train_iter = iterators.SerialIterator(train, batch_size)
test_iter = iterators.SerialIterator(test, batch_size, False, False)
models = model.LEAM(self.vocab_size, self.n_class)
models.embed.W.copydata(Variable(self.embedding_vectors))
models.embed_class.W.copydata(Variable(self.value_mean))
if gpu_id >= 0:
models.to_gpu(gpu_id)
models = L.Classifier(models, lossfun=self.lossfun_multi_leam, accfun=self.auc_fun_leam)
optimizer = optimizers.Adam(alpha=0.001)
optimizer.setup(models)
updater = training.updaters.StandardUpdater(train_iter, optimizer, device=gpu_id)
trainer = training.Trainer(updater, (epoch_size * max_epoch, 'iteration'), out='./appr-leam')
trainer.extend(extensions.LogReport(trigger=(epoch_size, 'iteration')))
trainer.extend(extensions.snapshot(filename='snapshot_iteration-{.updater.iteration}'), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.snapshot_object(models.predictor, filename='models_iteration-{.updater.iteration}'), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.Evaluator(test_iter, models, device=gpu_id), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.observe_lr(), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.PrintReport(['iteration', 'main/loss', 'validation/main/loss','main/accuracy', 'validation/main/accuracy', 'elapsed_time']), trigger=(epoch_size, 'iteration'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.ProgressBar(update_interval=1000))
trainer.run()
def get_fashion_mnist():
_retrieve_fashion_mnist()
train = np.load(os.path.join(_save_dir, _train_name))
test = np.load(os.path.join(_save_dir, _test_name))
train_dataset = datasets.TupleDataset(train['x'].astype(np.float32), train['y'].astype(np.int32))
test_dataset = datasets.TupleDataset(test['x'].astype(np.float32), test['y'].astype(np.int32))
return train_dataset, test_dataset
def __init__(self, data_name):
# get data from chainer
# images are normalized to [0.0, 1.0]
if data_name == 'mnist':
train_tuple, test_tuple = datasets.get_mnist(ndim=3)
elif data_name == 'fmnist':
train_tuple, test_tuple = get_fmnist(withlabel=True,
ndim=3,
scale=1.0)
elif data_name == 'cifar10':
train_tuple, test_tuple = datasets.get_cifar10()
else:
raise ValueError('Invalid data')
self.data_name = data_name
# preprocess
# convert to array
train_image, train_label = concat_examples(train_tuple)
test_image, test_label = concat_examples(test_tuple)
# set images to [-0.5, 0.5]
self.train_image = np.array(train_image, dtype=np.float32) - 0.5
self.train_label = np.array(train_label, dtype=np.int32)
self.test_image = np.array(test_image, dtype=np.float32) - 0.5
self.test_label = np.array(test_label, dtype=np.int32)
# re-convert to TupleDataset
self.train_tuple = datasets.TupleDataset(self.train_image,
self.train_label)
self.test_tuple = datasets.TupleDataset(self.test_image,
self.test_label)
def make_model(record_X, record_y, mode):
X = np.array(record_X, dtype=np.float32) # 全ゲームの棋譜リスト
y = np.array(record_y, dtype=np.int32) # 全ゲームの石の置き場の正解リスト
train = datasets.TupleDataset(X, y)
train_iter = iterators.SerialIterator(train, batch_size=100)
model = Classifier(MLP()) #MLPという手法で学習
#ここもSDGじゃなくてAdamとかに変えたら精度あがるかも。時間があったら試したい。
optimizer = optimizers.SGD()
optimizer.setup(model)
updater = training.StandardUpdater(train_iter, optimizer)
trainer = training.Trainer(updater, (150, 'epoch'), out='result')
trainer.extend(
extensions.PlotReport(['main/accuracy', 'val/main/accuracy'],
x_key='epoch',
file_name='accuracy.png',
trigger=(1, 'epoch')))
trainer.run()
if mode == "black":
serializers.save_npz('black.npz', model)
files.download('black.npz')
elif mode == "white":
serializers.save_npz('white.npz', model)
files.download('white.npz')
else:
serializers.save_npz('model.npz', model)
files.download('model.npz')
def main():
mushroomsfile = "mushrooms.csv"
data_array = np.genfromtxt(mushroomsfile,
delimiter=',',
dtype=str,
skip_header=1)
n_data, n_featrue = data_array.shape
for col in range(n_featrue):
data_array[:, col] = np.unique(data_array[:, col],
return_inverse=True)[1]
X = data_array[:, 1:].astype(np.float32)
Y = data_array[:, 0].astype(np.int32)[:, None]
train, test = datasets.split_dataset_random(datasets.TupleDataset(X, Y),
int(n_data * 0.7))
train_iter = ch.iterators.SerialIterator(train, 100)
test_iter = ch.iterators.SerialIterator(test,
100,
repeat=False,
shuffle=False)
model = L.Classifier(MLP(44, 1),
lossfun=F.sigmoid_cross_entropy,
accfun=F.binary_accuracy)
optimizer = ch.optimizers.SGD().setup(model)
updater = training.StandardUpdater(train_iter, optimizer, device=-1)
trainer = training.Trainer(updater, (50, 'epoch'), out='result')
trainer.extend(extensions.Evaluator(test_iter, model, device=-1))
trainer.extend(extensions.DumpGraph('main/loss'))
trainer.extend(
extensions.snapshot(filename='trainer_epoch_{.updater.epoch}'),
trigger=(10, 'epoch'))
trainer.extend(extensions.LogReport())
if extensions.PlotReport.available():
trainer.extend(
extensions.PlotReport(['main/loss', 'validation/main/loss'],
'epoch',
file_name='loss.png'))
trainer.extend(
extensions.PlotReport(
['main/accuracy', 'validation/main/accuracy'],
'epoch',
file_name='accuracy.png'))
trainer.extend(
extensions.PrintReport([
'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy',
'validation/main/accuracy', 'elapsed_time'
]))
trainer.run()
x, t = test[np.random.randint(len(test))]
predict = model.predictor(x[None]).array
predict = predict[0][0]
if predict >= 0:
print('Predicted Poisonous, Actual ' + ['Edible', 'Poisonous'][t[0]])
else:
print('Predicted Edible, Actual ' + ['Edible', 'Poisonous'][t[0]])
def load(fname, V, C, T=700, L=57):
a = numpy.load(fname)
N = len(a)
a = a.reshape(N, T, L)
acids_raw = a[..., :V + 1]
structure_labels_raw = a[..., V + 1:V + C + 2]
acids = numpy.full((N, T), numpy.nan, dtype=numpy.int32)
structure_labels = numpy.full((N, T), numpy.nan, dtype=numpy.int32)
for i in six.moves.range(V):
acids[acids_raw[..., i] == 1.0] = i
acids[acids_raw[..., V] == 1.0] = -1
assert not (numpy.isnan(acids)).any()
for i in six.moves.range(C):
structure_labels[structure_labels_raw[..., i] == 1.0] = i
structure_labels[structure_labels_raw[..., C] == 1.0] = -1
assert not (numpy.isnan(structure_labels)).any()
# To reduce the computational time, we reduce the time step
# in this example.
acids = acids[..., :100]
structure_labels = structure_labels[..., :100]
# As opposed to the original papere, we do not use protein profiles
# nor solvency to simplify the model.
# profiles = a[..., -V - 1:-1].astype(numpy.float32)
# absolute_solvent_labels = a[..., V + C + 4].astype(numpy.int32)
# relative_solvent_labels = a[..., V + C + 5].astype(numpy.int32)
# profiles = profiles[:, :100, :]
# absolute_solvent_labels = absolute_solvent_labels[..., :100]
# relative_solvent_labels = relative_solvent_labels[..., :100]
return datasets.TupleDataset(acids, structure_labels)
def conv(data):
# 抽出値を初期化
X = []
Y = []
# numpy変換用
data_array = data.as_matrix()
# 説明変数（20）と目的変数（1）を分割する。
for j in data_array:
x_split = np.hsplit(j, [10, 11])
X.append(x_split[0].astype(np.float32))
Y.append(x_split[1].astype(np.int32))
X = np.array(X)
Y = np.ndarray.flatten(np.array(Y))
# テストデータのスケーリング
scaler = MinMaxScaler()
scaler.fit(X)
X = scaler.transform(X)
# 訓練データ、検証データを無作為に8：2で分割する。
train, test = datasets.split_dataset_random(datasets.TupleDataset(X, Y),
32000)
# 訓練データ、検証データを返却する
return train, test
def create_dataset_board_and_hand_and_result():
'''
input looks like ([board ... hand ...], win/lost)
'''
train_raw = []
score_raw = []
for i in range(10000):
g = Game()
g.play()
result, winner = g.result()
if result == TicTocToe.WIN:
for b, h in g.playing.record(winner):
b_num = [0 if v is None else 1 if v else -1 for v in b]
r_num = [0] * len(b)
r_num[h] = np.float32(1)
train_raw.append(np.array(b_num + r_num, np.float32))
score_raw.append(W_WIN)
for b, h in g.playing.record(
TicTocToe.SIDE_O if winner ==
TicTocToe.SIDE_X else TicTocToe.SIDE_X):
b_num = [0 if v is None else 1 if v else -1 for v in b]
r_num = [0] * len(b)
r_num[h] = np.float32(1)
train_raw.append(np.array(b_num + r_num, np.float32))
score_raw.append(W_LOST)
return datasets.TupleDataset(train_raw, score_raw)
def get_soft_label(model, image):
"""This function calculates soft labelings of input image data.
Args:
model (ClassiferNN) : model which calculates the soft labelings
image (numpy ndarray) : image data which is fed into the model
Return:
TupleDataset (image, soft label) located in GPU if possible
"""
xp_image = xp.array(image)
iterator = iterators.SerialIterator(xp_image,
batch_size=400,
shuffle=False,
repeat=False)
soft_label = xp.empty((0, model.n_classes), dtype=xp.float32)
y = []
# loops until iterator gets end
while iterator.epoch == 0:
x = iterator.next()
y.append(model.predict_proba(x).data)
soft_label = xp.concatenate(y, axis=0)
tuple_data = datasets.TupleDataset(xp_image, soft_label)
return tuple_data
def train():
epoch_num = 100
batchsize = 64
gpu_id = -1
train = load_images('./train_data')
train = datasets.TupleDataset(train, train)
train_iter = chainer.iterators.SerialIterator(train, batchsize)
model = L.Classifier(Autoencoder(256 * 256, 128),
lossfun=F.mean_squared_error)
model.compute_accuracy = False
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
updater = training.StandardUpdater(train_iter, optimizer, device=gpu_id)
trainer = training.Trainer(updater, (epoch_num, 'epoch'), out='result')
trainer.extend(extensions.LogReport())
trainer.extend(
extensions.snapshot(filename='snapshot_epoch-{.updater.epoch}'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.ProgressBar())
trainer.run()
model.to_cpu()
serializers.save_npz('sushi_hotdog.model', model)
def sample_5():
# make a iterator
x = X.copy()
y = Y.copy()
dataset = D.TupleDataset(x, y)
train_iter = Iter.SerialIterator(dataset, batch_size=DATA_SIZE, shuffle=False)
# create a network
model = TwoLayerNet(INPUT_SIZE, HIDDEN_SIZE, OUTPUT_SIZE)
loss_calculator = LossCalculator(model)
# create an optimizer
optimizer = P.SGD(lr=LEARNING_RATE)
# connect the optimizer with the network
optimizer.setup(loss_calculator)
# make a updater
updater = training.StandardUpdater(train_iter, optimizer)
# make a trainer
trainer = training.Trainer(updater, (EPOCHS, 'epoch'), out='result')
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'elapsed_time']))
trainer.run()
def train():
# _/_/_/ load dataset
xs = np.load(XS_PATH)
ys = np.load(YS_PATH)
# _/_/_/ split dataset and make iterators
# for training
train_dataset = D.TupleDataset(xs, ys)
train_iter = Iter.SerialIterator(train_dataset,
batch_size=BATCH_SIZE,
shuffle=False)
# _/_/_/ create a network
model = MyNet(LENGTH_SCALE, INPUT_SIZE, HIDDEN_SIZE, OUTPUT_SIZE,
DROPOUT_RATIO)
loss_calculator = LossCalculator(model)
# _/_/_/ create an optimizer
# optimizer = P.SGD(lr=LEARNING_RATE)
optimizer = P.Adam()
# ptimizer = P.RMSprop(lr=LEARNING_RATE)
# _/_/_/ connect the optimizer with the network
optimizer.setup(loss_calculator)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=WEIGHT_DECAY))
# _/_/_/ make a updater
updater = training.StandardUpdater(train_iter, optimizer)
# _/_/_/ make a trainer
epoch_interval = (1, 'epoch')
model_interval = (EPOCHS, 'epoch')
trainer = training.Trainer(updater, (EPOCHS, 'epoch'), out=OUTPUT_DIR_PATH)
# trainer.extend(extensions.ExponentialShift('lr', 0.99), trigger=epoch_interval)
# save a trainer
trainer.extend(extensions.snapshot(), trigger=model_interval)
# save a model
trainer.extend(extensions.snapshot_object(model, MODEL_NAME),
trigger=model_interval)
trainer.extend(extensions.LogReport(trigger=epoch_interval))
trainer.extend(extensions.PrintReport(['epoch', 'iteration', 'main/loss']),
trigger=epoch_interval)
trainer.extend(
extensions.PlotReport(['main/loss'], 'epoch', file_name='loss.png'))
# _/_/_/ run
trainer.run()
def temp_data_prep():
X = np.random.rand(40, 4, 256, 256, 256).astype(np.float32)
Y = np.random.rand(40, 4).astype(np.float32)
dataset = datasets.TupleDataset(X, Y)
return dataset
def objective(trial):
# Create model instance
model = TripletClassifier(
MyNeuralNetwork(n_mid_units=mid_size, n_out=out_size))
optimizer = create_optimizer(trial, model)
batchsize = trial.suggest_int('batchsize', 10, len(y))
epoch = trial.suggest_int('epoch', 10, 50)
# Assign GPU or CPU to the model
if gpu_id >= 0:
cuda.get_device(gpu_id).use()
model.to_gpu(gpu_id)
# Define Iterator
train_set = datasets.TupleDataset(train_triplet, train_label)
test_set = datasets.TupleDataset(test_triplet, test_label)
train_iter = iterators.SerialIterator(train_set, batchsize)
test_iter = iterators.SerialIterator(test_set,
batchsize,
repeat=False,
shuffle=False)
# Define Trainer
updater = chainer.training.StandardUpdater(train_iter, optimizer)
trainer = chainer.training.Trainer(updater, (epoch, 'epoch'))
trainer.extend(chainer.training.extensions.Evaluator(test_iter, model))
log_report_extension = chainer.training.extensions.LogReport(log_name=None)
trainer.extend(
chainer.training.extensions.PrintReport([
'epoch', 'main/squared_error', 'validation/main/squared_error',
'main/abs_error', 'validation/main/abs_error', 'elapsed_time'
]))
trainer.extend(log_report_extension)
trainer.run()
log_last = log_report_extension.log[-1]
for key, value in log_last.items():
trial.set_user_attr(key, value)
val_err = log_report_extension.log[-1]['validation/main/squared_error']
return val_err
def load_omniglot(section, git_root_dir, resize=None, ndim=2, verbose=False):
"""
----
Args:
section (str):
['train-small1', 'train-small2', 'train', 'eval', 'all'] のいずれか
'all' は 'train' + 'eval' です．
'train-small1' や 'train-small2' は 'train' の部分集合です．
git_root_dir (str or Path):
omniglot の公式リポジトリを clone したディレクトリを指定．
あらかじめ，"python" フォルダ内の zip ファイルを全て展開しておく必要があります．
resize (None or tuple):
画像のリサイズ指定．例えば Finn+ 2017 では (28, 28)．None の場合は (105, 105)
ndim (int):
各画像データの array の ndim 指定．[1, 2, 3] のいずれか．
Returns:
(chainer.datasets.TupleDataset)
クラスラベルは 0 始まりの連続した int32 です
"""
assert section in ['train-small1', 'train-small2', 'train', 'eval', 'all']
assert ndim in [1, 2, 3]
images_dir = Path(git_root_dir) / "python" / {
"train-small1": "images_background_small1",
"train-small2": "images_background_small2",
"train": "images_background",
"eval": "images_evaluation",
"all": "images_*[dn]",
}[section]
class_dirs = sorted(glob.glob(str(images_dir / "*/*/")))
if verbose:
print('{} classes found.'.format(len(class_dirs)))
Xs, ys = [], []
for i, cl in enumerate(class_dirs):
pngs = sorted(glob.glob(str(Path(cl) / "*.png")))
for p in pngs:
img = Image.open(p, 'r')
if resize is not None:
img = img.resize(resize, resample=Image.LANCZOS)
# オリジナルは bool 形式かつ「白地に黒」
# logical_not するのは「黒地に白」に色反転するため
ary = np.logical_not(np.array(img))
if ndim == 3:
ary = ary[None, :, :]
elif ndim == 1:
ary = ary.reshape(-1)
Xs.append(ary)
ys.append(i)
Xs = np.array(Xs, dtype=np.float32)
ys = np.array(ys, dtype=np.int32)
return datasets.TupleDataset(Xs, ys)
def check_tuple_dataset(self, x0, x1):
td = datasets.TupleDataset(x0, x1)
self.assertEqual(len(td), len(x0))
for i in range(len(x0)):
example = td[i]
self.assertEqual(len(example), 2)
numpy.testing.assert_array_equal(
cuda.to_cpu(example[0]), cuda.to_cpu(x0[i]))
numpy.testing.assert_array_equal(
cuda.to_cpu(example[1]), cuda.to_cpu(x1[i]))
def mnist_data_for_chainer(csv_loaded_data):
images = np.array(csv_loaded_data["images"])
labels = np.array(csv_loaded_data["labels"])
X = []
Y = []
n = len(images)
for i in range(n):
d = np.array(images[i].reshape(28, 28), dtype=np.float32)
X.append([d])
Y.append(labels[i].astype(np.int32))
X = np.array(X)
Y = np.array(Y)
return datasets.TupleDataset(X, Y)
def training_vae(data,
hidden,
max_epoch,
batchsize,
act_func='sigmoid',
gpu_device=0,
loss_function='mse',
out_dir='result'):
# 入力サイズ
inputs = data.shape[1]
layers = [inputs] + hidden
# モデルの定義
vae = VariationalAutoEncoder(layers, act_func=act_func)
model = VariationalAutoencoderTrainer(vae,
beta=1.0,
k=1,
loss_function=loss_function)
opt = optimizers.Adam()
opt.setup(model)
# データの形式を変換する
train = datasets.TupleDataset(data, data)
train_iter = iterators.SerialIterator(train, batchsize)
# 学習ループ
updater = training.StandardUpdater(train_iter, opt, device=gpu_device)
trainer = training.Trainer(updater, (max_epoch, 'epoch'), out=out_dir)
trainer.extend(extensions.LogReport())
trainer.extend(
extensions.PrintReport([
'epoch', 'main/loss', 'main/reconst_loss', 'main/kld',
'elapsed_time'
]))
trainer.run()
# GPUを使っていた場合CPUに戻す
if -1 < gpu_device:
vae.to_cpu()
return vae
def create_dataset(assays, smiles, featurizer):
# Merge assay results
assays = pubchem.concat_assays(assays)
df = pd.merge(assays, smiles, on='PUBCHEM_SID', how='inner')
# Convert smiles to fingerprint and drop substances
# that cannot be converted to fingerprint.
print('Creating feature vectors from SMILES...')
df['FINGERPRINT'] = df['SMILES'].apply(featurizer)
df = df[df['FINGERPRINT'] != -1]
fps = np.array(list(df['FINGERPRINT'].values), dtype=np.float32)
# Convert outcome to binary value
assays = df.drop(['PUBCHEM_SID', 'SMILES', 'FINGERPRINT'], axis=1).values
assays[assays == 'Active'] = 0
assays[assays == 'Inactive'] = 1
assays[(assays != 0) & (assays != 1)] = -1
assays = assays.astype(np.int32)
assert len(fps) == len(assays)
return D.TupleDataset(fps, assays)
def main():
epoch = 100
batch_size = 1
n_in = 4
data = generate_data(n_in)
# Convert to set of tuples (target, label).
train = datasets.TupleDataset(*data)
model = L.Classifier(MyModel(4, 1), lossfun=F.mean_squared_error)
# Set compute_accuracy=False when using MSE.
model.compute_accuracy = False
# Define optimizer (Adam, RMSProp, etc)
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# Define iterators.
train_iter = chainer.iterators.SerialIterator(train, batch_size)
updater = training.StandardUpdater(train_iter, optimizer)
trainer = training.Trainer(updater, (epoch, 'epoch'))
# Helper functions (extensions) to monitor progress on stdout.
report_params = [
'epoch',
'main/loss',
]
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(report_params))
trainer.extend(extensions.ProgressBar())
# Run trainer
trainer.run()
# Should print out value close to W.
print(model.predictor(np.ones((1, n_in)).astype(np.float32)).data)
def train():
# train_txt = "/media/common-ns/New Volume/reseach/Dataset/OU-ISIR_by_Setoguchi/CV01.txt"
train_dir = "/media/common-ns/New Volume/reseach/Dataset/OU-ISIR_by_Setoguchi/Gallery/CV01(Gallery)/*"
train = load_OULP(path_dir=train_dir)
# print(train[0])
# 教師データ
# train = train[0:1000]
train = [i[0] for i in train] # dataのパスとラベルのうち、dataだけ抜き出す
train = datasets.TupleDataset(train, train) # 同じパス画像のペアから、dataに変換してタプルにする
batch_size = 195
train_iter = chainer.iterators.SerialIterator(train, batch_size=batch_size)
#model = L.Classifier(Autoencoder(), lossfun=F.mean_squared_error)
model = L.Classifier(CAE(), lossfun=sce_loss)
model.compute_accuracy = False
optimizer = optimizers.Adam()
optimizer.setup(model)
updater = StandardUpdater(train_iter, optimizer, device=0)
trainer = Trainer(
updater,
(1000, 'epoch'),
out="result",
)
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(['epoch', 'main/loss']))
trainer.extend(extensions.snapshot(), trigger=(200, 'epoch'))
trainer.extend(extensions.snapshot_object(
target=model, filename='model_snapshot_{.updater.iteration}'),
trigger=(250, 'epoch'))
trainer.extend(extensions.ProgressBar())
trainer.run()
serializers.save_npz(
"/home/common-ns/setoguchi/chainer_files/Convolutional_Auto_Encoder/CAE_FC_model",
model)
def train(model, number_of_epoch, once=False, show_hidden_layer=False):
#optimizer = optimizers.SGD()
#optimizer = optimizers.SGD(lr = 0.5)
optimizer = optimizers.Adam(alpha=0.1)
#optimizer = optimizers.MomentumSGD(lr = 0.1, momentum = 0.9)
optimizer.setup(model)
train = datasets.TupleDataset(data_axis, data_value)
train_iter = iterators.SerialIterator(train,
batch_size=number_of_data,
shuffle=True)
updater = training.StandardUpdater(train_iter, optimizer)
trainer = training.Trainer(updater, (number_of_epoch, 'epoch'),
out='result')
if (once):
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(['epoch', 'main/loss']))
p = model.predictor
if (not once):
with open("./output/initial-weight.txt", mode='a') as f:
for (a, b) in zip(p.l1.W.data.reshape((number_of_hidden_nodes, )),
p.l1.b.data):
f.write("{0} {1}\n".format(a, b))
trainer.run()
if (not once):
with open("./output/final-weight.txt", mode='a') as f:
for (a, b) in zip(p.l1.W.data.reshape((number_of_hidden_nodes, )),
p.l1.b.data):
f.write("{0} {1}\n".format(a, b))
if (once):
plotGraph(model, show_hidden_layer)
return
def load_svhn(path, fmt='TupleDataset', image_size=[70, 30], num=None):
assert fmt in ['TupleDataset', 'dict']
print('Loading digitStruct.mat...')
dsm = load_digitStruct(str(Path(path) / 'digitStruct.mat'), num=num)
print(' Done.')
print('Loading images...')
xs = []
for name in tqdm(dsm['name']):
xs.append(np.array(Image.open(str(Path(path) / name)).resize(image_size)))
print(' Done.')
xs = np.asarray(xs, dtype=np.float32).transpose([0, 3, 1, 2]) / 256 # (batch, color, height, width)
ys = [''.join(map(str, map(int, b['label']))) for b in dsm['bbox']]
if fmt == 'dict':
return {
"xs": xs,
"ys": ys,
"digitStruct": dsm
}
elif fmt == 'TupleDataset':
return datasets.TupleDataset(xs, ys)
else:
raise ValueError
print('depth must be (9, 17, 29)')
sys.exit()
print('VDCNN setting: emb_dim={} n_out={}, depth={}'.format(
len(char2id) + 1, kind,
sum(depth) * 2 + 1))
gpu_id = args.gpu
model = VDCNN(len(char2id) + 1, kind, depth)
if gpu_id >= 0:
model.to_gpu(gpu_id)
print(mode, train_x.shape, train_y.shape, test_x.shape, test_y.shape)
train = datasets.TupleDataset(train_x, train_y)
test = datasets.TupleDataset(test_x, test_y)
batch_size = 128
train_iter = iterators.SerialIterator(train, batch_size)
test_iter = iterators.SerialIterator(test, batch_size, False, False)
epoch_size = 5000
max_epoch = 15
model = L.Classifier(model)
optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
optimizer.setup(model)
import numpy as np
import matplotlib
matplotlib.use('Agg')
mushroomsfile = 'mushrooms.csv'
data_array = np.genfromtxt(mushroomsfile,
delimiter=',',
dtype=str,
skip_header=1)
for col in range(data_array.shape[1]):
data_array[:, col] = np.unique(data_array[:, col], return_inverse=True)[1]
X = data_array[:, 1:].astype(np.float32)
Y = data_array[:, 0].astype(np.int32)[:, None]
train, test = datasets.split_dataset_random(datasets.TupleDataset(X, Y),
int(data_array.shape[0] * .7))
train_iter = ch.iterators.SerialIterator(train, 100)
test_iter = ch.iterators.SerialIterator(test, 100, repeat=False, shuffle=False)
# Network definition
def MLP(n_units, n_out):
layer = ch.Sequential(L.Linear(n_units), F.relu)
model = layer.repeat(2)
model.append(L.Linear(n_out))
return model
Y = iris.target
Y = Y.flatten().astype(np.int32) # NOTICE: calling flatten
return (X, Y)
verbose = True
report_params = [
'epoch',
'main/loss',
'validation/main/loss',
'main/accuracy',
'validation/main/accuracy',
]
iris_data = iris_data()
train, test = datasets.split_dataset_random(datasets.TupleDataset(*iris_data),
100)
train_iter = iterators.SerialIterator(train, batch_size=10, shuffle=True)
test_iter = iterators.SerialIterator(test,
batch_size=1,
repeat=False,
shuffle=False)
model = L.Classifier(MyModel())
optimizer = optimizers.Adam()
optimizer.setup(model)
updater = training.StandardUpdater(train_iter, optimizer)
trainer = training.Trainer(updater, (20, 'epoch'), out='result')
if verbose:
def test_tuple_dataset_overrun(self):
td = datasets.TupleDataset(self.x0, self.x1)
with self.assertRaises(IndexError):
td[3]
def test_tuple_dataset_len_mismatch(self):
with self.assertRaises(ValueError):
datasets.TupleDataset(self.x0, self.z0)
from chainer import training
from chainer.training import extensions
import numpy as np
mushroomsfile = 'mushrooms.csv'
data_array = np.genfromtxt(
mushroomsfile, delimiter=',', dtype=str, skip_header=1)
for col in range(data_array.shape[1]):
data_array[:, col] = np.unique(data_array[:, col], return_inverse=True)[1]
X = data_array[:, 1:].astype(np.float32)
Y = data_array[:, 0].astype(np.int32)[:, None]
train, test = datasets.split_dataset_random(
datasets.TupleDataset(X, Y), int(data_array.shape[0] * .7))
train_iter = chainer.iterators.SerialIterator(train, 100)
test_iter = chainer.iterators.SerialIterator(
test, 100, repeat=False, shuffle=False)
# Network definition
class MLP(chainer.Chain):
def __init__(self, n_units, n_out):
super(MLP, self).__init__()
with self.init_scope():
# the input size to each layer inferred from the layer before
self.l1 = L.Linear(n_units) # n_in -> n_units
self.l2 = L.Linear(n_units) # n_units -> n_units
self.l3 = L.Linear(n_out) # n_units -> n_out
def main():
# オプション処理
parser = argparse.ArgumentParser(description='話者認識モデルの学習')
parser.add_argument('--batchsize', '-b', type=int, default=50,
help='Number of images in each mini-batch')
parser.add_argument('--epoch', '-e', type=int, default=20,
help='Number of sweeps over the dataset to train')
parser.add_argument('--gpu', '-g', type=int, default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out', '-o', default='result',
help='Directory to output the result')
parser.add_argument('--datadir', '-d', default='train',
help='学習データのディレクトリ')
args = parser.parse_args()
sys.stderr.write('GPU: {}\n'.format(args.gpu))
sys.stderr.write('# minibatch-size: {}\n'.format(args.batchsize))
sys.stderr.write('# epoch: {}\n'.format(args.epoch))
trainf = []
label = 0
print('loading dataset')
mfcc = os.listdir(args.datadir)
for i in [f for f in mfcc if ('mfcc' in f)]:
trainf.append([os.path.join(args.datadir, i), label])
label += 1
#print('{}'.format(trainf))
input = []
target = []
if not os.path.exists(args.out):
os.mkdir(args.out)
log = open(args.out+"/class_log",'w')
for file in trainf:
print('{}'.format(file))
log.write('{},{}\n'.format(file[0].strip().split("/")[-1].split(".")[0],file[1]))
with open(file[0], 'r') as f:
lines = f.readlines()
for i, l in enumerate(lines):
#print('{}'.format(len(lines)))
tmp = []
flag = False
for j in range(3): # 3フレームで評価
if i + 2 < len(lines):
frame = lines[i+j].strip().split(" ")
#print("i:{},file:{}".format(i,file))
np.array(frame,dtype=np.float32)
tmp.extend(frame)
flag = True
#print('{}'.format(tmp))
if flag:
input.append(tmp)
target.append(file[1])
# print('{}'.format(input))
log.close()
#print('{}'.format(input))
#sys.exit()
#print(np.array(input))
input = np.array(input).astype(np.float32)
target = np.array(target).astype(np.int32)
#print('{},{}'.format(len(input), len(target)))
train = D.TupleDataset(input, target)
#sys.stderr.write(train)
#print(len(input)*0.9)
train, test = D.split_dataset_random(train, int(len(input)*0.9))
print('{},{}'.format(len(train), len(test)))
#sys.exit()
model = L.Classifier(DNN(label)) # CNNにする
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current
model.to_gpu() # Copy the model to the GPU
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test, args.batchsize,
repeat=False, shuffle=False)
updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch'))
trainer.extend(extensions.LogReport())
# trainer.extend(
# extensions.PlotReport('main/loss', 'epoch', file_name='loss.png'))
# trainer.extend(
# extensions.PlotReport('main/accuracy', 'epoch', file_name='accuracy.png'))
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'elapsed_time']))
#trainer.extend(extensions.PrintReport(
# ['epoch', 'main/loss', 'main/accuracy', 'elapsed_time']))
trainer.extend(extensions.ProgressBar())
print('training start!')
trainer.run()
# モデルをCPU対応へ
model.to_cpu()
# 保存
modelname = args.out + "/speaker.model"
print('save the trained model: {}'.format(modelname))
chainer.serializers.save_npz(modelname, model)
