def test_backprop_mode_affects_chainerx(self):
# chainer.{no,force}_backprop_mode should affect chainerx's
# counterpart.
assert chainerx.is_backprop_required()
# nobp
with chainer.no_backprop_mode():
assert not chainerx.is_backprop_required()
# nobp > forcebp
with chainer.force_backprop_mode():
assert chainerx.is_backprop_required()
# nobp > nobp
with chainer.no_backprop_mode():
assert not chainerx.is_backprop_required()
assert chainerx.is_backprop_required()
# forcebp
with chainer.force_backprop_mode():
assert chainerx.is_backprop_required()
# forcebp > forcebp
with chainer.force_backprop_mode():
assert chainerx.is_backprop_required()
# forcebp > nobp
with chainer.no_backprop_mode():
assert not chainerx.is_backprop_required()
assert chainerx.is_backprop_required()
def test3(self):
obj = MockDistribution(chainer.Variable(numpy.array([1.])))
h0 = obj.h
with chainer.no_backprop_mode():
h1 = obj.h
h2 = obj.h
with chainer.no_backprop_mode():
h3 = obj.h
assert obj.h_call_count <= 2
assert h0 is h2
assert h0 is not h1
assert h1 is h3
numpy.testing.assert_allclose(h0.array, 2.)
numpy.testing.assert_allclose(h1.array, 2.)
def evaluate(examples, iterator, model, converter, device,
predict_func):
all_results = []
all_features = []
total_iter = len(iterator.dataset) // iterator.batch_size + 1
for prebatch in progressbar.ProgressBar(max_value=total_iter)(iterator):
batch = converter(prebatch, device)
features_list = prebatch
# In `batch`, features is concatenated and to_gpu.
with chainer.no_backprop_mode(), chainer.using_config('train', False):
result = predict_func(*batch)
for i in range(len(prebatch)):
unique_id = int(result["unique_ids"][i])
start_logits = [float(x) for x in result["start_logits"][i]]
end_logits = [float(x) for x in result["end_logits"][i]]
all_results.append(
RawResult(
unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits))
all_features.append(features_list[i])
output_prediction_file = os.path.join(
FLAGS.output_dir, "predictions.json")
output_nbest_file = os.path.join(
FLAGS.output_dir, "nbest_predictions.json")
write_predictions(examples, all_features, all_results,
FLAGS.n_best_size, FLAGS.max_answer_length,
FLAGS.do_lower_case, output_prediction_file,
output_nbest_file)
def _predict_depth_chainer_backend(self, bgr, depth_bgr=None):
bgr_data = np.array([bgr], dtype=np.float32)
depth_bgr_data = np.array([depth_bgr], dtype=np.float32)
if self.gpu != -1:
bgr_data = cuda.to_gpu(bgr_data, device=self.gpu)
depth_bgr_data = cuda.to_gpu(depth_bgr_data, device=self.gpu)
if LooseVersion(chainer.__version__) < LooseVersion('2.0.0'):
bgr = chainer.Variable(bgr_data, volatile=True)
depth_bgr = chainer.Variable(depth_bgr_data, volatile=True)
self.model(bgr, depth_bgr)
else:
with chainer.using_config('train', False):
with chainer.no_backprop_mode():
bgr = chainer.Variable(bgr_data)
depth_bgr = chainer.Variable(depth_bgr_data)
self.model(bgr, depth_bgr)
proba_img = F.softmax(self.model.mask_score)
label_pred = F.argmax(self.model.mask_score, axis=1)
depth_pred = F.sigmoid(self.model.depth_score)
proba_img = F.transpose(proba_img, (0, 2, 3, 1))
max_proba_img = F.max(proba_img, axis=-1)
# squeeze batch axis, gpu -> cpu
proba_img = cuda.to_cpu(proba_img.data)[0]
max_proba_img = cuda.to_cpu(max_proba_img.data)[0]
label_pred = cuda.to_cpu(label_pred.data)[0]
depth_pred = cuda.to_cpu(depth_pred.data)[0]
# uncertain because the probability is low
label_pred[max_proba_img < self.proba_threshold] = self.bg_label
# get depth image
depth_pred = depth_pred[0, :, :]
depth_pred *= (self.model.max_depth - self.model.min_depth)
depth_pred += self.model.min_depth
return label_pred, proba_img, depth_pred
def predict(self, atoms, adjs):
# type: (numpy.ndarray, numpy.ndarray) -> chainer.Variable
with chainer.no_backprop_mode(), chainer.using_config('train', False):
x = self.__call__(atoms, adjs)
if self.label_scaler is not None:
x = self.label_scaler.inverse_transform(x)
return self.postprocess_fn(x)
def predict(
self, data, batchsize=16, converter=concat_examples,
retain_inputs=False, preprocess_fn=None, postprocess_fn=None):
"""Predict label of each category by taking .
Args:
data: input data
batchsize (int): batch size
converter (Callable): convert from `data` to `inputs`
preprocess_fn (Callable): Its input is numpy.ndarray or
cupy.ndarray, it can return either Variable, cupy.ndarray or
numpy.ndarray
postprocess_fn (Callable): Its input argument is Variable,
but this method may return either Variable, cupy.ndarray or
numpy.ndarray.
retain_inputs (bool): If True, this instance keeps inputs in
`self.inputs` or not.
Returns (tuple or numpy.ndarray): Typically, it is 1-dimensional int
array with shape (batchsize, ) which represents each examples
category prediction.
"""
with chainer.no_backprop_mode(), chainer.using_config('train', False):
predict_labels = self._forward(
data, fn=self.predictor, batchsize=batchsize,
converter=converter, retain_inputs=retain_inputs,
preprocess_fn=preprocess_fn, postprocess_fn=postprocess_fn)
return predict_labels
def evaluate(raw_model, iter):
model = raw_model.copy() # to use different state
model.reset_state() # initialize state
sum_perp = 0
count = 0
xt_batch_seq = []
one_pack = args.batchsize * args.bproplen * 2
with chainer.using_config('train', False), chainer.no_backprop_mode():
for batch in copy.copy(iter):
xt_batch_seq.append(batch)
count += 1
if len(xt_batch_seq) >= one_pack:
x_seq_batch, t_seq_batch = utils_pretrain.convert_xt_batch_seq(
xt_batch_seq, args.gpu)
loss = model.forward_seq_batch(
x_seq_batch, t_seq_batch, normalize=1.)
sum_perp += loss.data
xt_batch_seq = []
if xt_batch_seq:
x_seq_batch, t_seq_batch = utils_pretrain.convert_xt_batch_seq(
xt_batch_seq, args.gpu)
loss = model.forward_seq_batch(
x_seq_batch, t_seq_batch, normalize=1.)
sum_perp += loss.data
return np.exp(float(sum_perp) / count)
def get_greedy_action(Q, obs):
"""Get a greedy action wrt a given Q-function."""
dtype = chainer.get_dtype()
obs = Q.xp.asarray(obs[None], dtype=dtype)
with chainer.no_backprop_mode():
q = Q(obs).array[0]
return int(q.argmax())
def evaluate(self):
bt = time.time()
with chainer.no_backprop_mode():
references = []
hypotheses = []
observation = {}
with reporter.report_scope(observation):
for i in range(0, len(self.test_data), self.batch):
src, trg = zip(*self.test_data[i:i + self.batch])
references.extend([[t.tolist()] for t in trg])
src = [chainer.dataset.to_device(self.device, x)
for x in src]
if self.comm.rank == 0:
self.model.translate(src, self.max_length)
elif self.comm.rank == 1:
ys = [y.tolist()
for y in self.model.translate(
src, self.max_length)]
hypotheses.extend(ys)
if self.comm.rank == 1:
bleu = bleu_score.corpus_bleu(
references, hypotheses, smoothing_function=bleu_score.
SmoothingFunction().method1)
reporter.report({'bleu': bleu}, self.model)
et = time.time()
if self.comm.rank == 1:
print("BleuEvaluator(single)::evaluate(): "
"took {:.3f} [s]".format(et - bt))
sys.stdout.flush()
return observation
def translate(self, xs, max_length=100):
batch = len(xs)
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
xs = [x[::-1] for x in xs]
exs = sequence_embed(self.embed_x, xs)
# Initial hidden variable and cell variable
# zero = self.xp.zeros((self.n_layers, batch, self.n_units), self.xp.float32) # NOQA
# h, c, _ = self.encoder(zero, zero, exs, train=False) # NOQA
h, c, _ = self.encoder(None, None, exs)
ys = self.xp.zeros(batch, self.xp.int32)
result = []
for i in range(max_length):
eys = self.embed_y(ys)
eys = chainer.functions.split_axis(
eys, batch, 0, force_tuple=True)
h, c, ys = self.decoder(h, c, eys)
cys = chainer.functions.concat(ys, axis=0)
wy = self.W(cys)
ys = self.xp.argmax(wy.data, axis=1).astype(self.xp.int32)
result.append(ys)
result = cuda.to_cpu(self.xp.stack(result).T)
# Remove EOS taggs
outs = []
for y in result:
inds = numpy.argwhere(y == 0)
if len(inds) > 0:
y = y[:inds[0, 0]]
outs.append(y)
return outs
def predict(self, x, max_length=50):
"""
テスト用の予測メソッド
:param x: 入力文
:param max_length: 出力文の制御
:return: 出力文のindex
"""
with chainer.no_backprop_mode(), chainer.using_config('train', False):
result = []
y = [EOS]
# Embedding Layer
x = x[::-1] # 入力を反転したほうが精度が上がる
emb_x = [self.x_embed(x)]
# Encoder
h, c, a = self.encoder(None, None, emb_x)
# Decoder, Output Layerの計算
for i in range(max_length):
prob, h, c = self.generate_one_step(y, h, c, a) # decoderの計算
y = self.xp.argmax(prob.data)
# 出力がEOS信号かcheck。もしそうならbreak
if y == EOS:
break
result.append(y)
# 次の入力へのVariable化
y = self.xp.array([y], dtype=self.xp.int32)
return result
def translate(self, xs, max_length=100):
batch = len(xs)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
xs = [x[::-1] for x in xs]
exs = sequence_embed(self.embed_x, xs)
h, c, _ = self.encoder(None, None, exs)
ys = self.xp.full(batch, EOS, numpy.int32)
result = []
for i in range(max_length):
eys = self.embed_y(ys)
eys = F.split_axis(eys, batch, 0)
h, c, ys = self.decoder(h, c, eys)
cys = F.concat(ys, axis=0)
wy = self.W(cys)
ys = self.xp.argmax(wy.array, axis=1).astype(numpy.int32)
result.append(ys)
# Using `xp.concatenate(...)` instead of `xp.stack(result)` here to
# support NumPy 1.9.
result = chainer.get_device(numpy).send(
self.xp.concatenate([x[None, :] for x in result]).T)
# Remove EOS taggs
outs = []
for y in result:
inds = numpy.argwhere(y == EOS)
if len(inds) > 0:
y = y[:inds[0, 0]]
outs.append(y)
return outs
def translate(self, xs, max_length=100):
batch = len(xs)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
xs = [x[::-1] for x in xs]
exs = sequence_embed(self.embed_x, xs)
h, c, _ = self.encoder(None, None, exs)
ys = self.xp.full(batch, EOS, 'i')
result = []
for i in range(max_length):
eys = self.embed_y(ys)
eys = F.split_axis(eys, batch, 0)
h, c, ys = self.decoder(h, c, eys)
cys = F.concat(ys, axis=0)
wy = self.W(cys)
ys = self.xp.argmax(wy.data, axis=1).astype('i')
result.append(ys)
result = cuda.to_cpu(self.xp.stack(result).T)
# Remove EOS taggs
outs = []
for y in result:
inds = numpy.argwhere(y == EOS)
if len(inds) > 0:
y = y[:inds[0, 0]]
outs.append(y)
return outs
def predict(self, obs):
with chainer.no_backprop_mode():
if self.concat_time:
ts = np.arange(len(obs)) / self.env_spec.timestep_limit
obs = np.concatenate([obs, ts[:, None]], axis=-1)
values = self.compute_baselines(Variable(obs))
return values.data
def update(Q, target_Q, opt, samples, gamma=0.99, target_type='double_dqn'):
"""Update a Q-function with given samples and a target Q-function."""
dtype = chainer.get_dtype()
xp = Q.xp
obs = xp.asarray([sample[0] for sample in samples], dtype=dtype)
action = xp.asarray([sample[1] for sample in samples], dtype=np.int32)
reward = xp.asarray([sample[2] for sample in samples], dtype=dtype)
done = xp.asarray([sample[3] for sample in samples], dtype=dtype)
obs_next = xp.asarray([sample[4] for sample in samples], dtype=dtype)
# Predicted values: Q(s,a)
y = F.select_item(Q(obs), action)
# Target values: r + gamma * max_b Q(s',b)
with chainer.no_backprop_mode():
if target_type == 'dqn':
next_q = F.max(target_Q(obs_next), axis=1)
elif target_type == 'double_dqn':
next_q = F.select_item(target_Q(obs_next),
F.argmax(Q(obs_next), axis=1))
else:
raise ValueError('Unsupported target_type: {}'.format(target_type))
target = reward + gamma * (1 - done) * next_q
loss = mean_clipped_loss(y, target)
Q.cleargrads()
loss.backward()
opt.update()
def get_action(policy, obs):
"""Get an action by evaluating a given policy."""
dtype = chainer.get_dtype()
obs = policy.xp.asarray(obs[None], dtype=dtype)
with chainer.no_backprop_mode():
action = policy(obs).array[0]
return chainer.backends.cuda.to_cpu(action)
def predict(self, x, max_length=1000):
"""
テスト用の予測メソッド
:param x: 入力文
:param max_length: 出力文の制御
:return: 出力分のindex
"""
with chainer.no_backprop_mode(), chainer.using_config('train', False):
result = []
h = None
c = None
# Decoder, Output Layerの計算
for i in range(max_length):
# Embedding Layer
x = [self.embed(x)]
# Hidden Layer
h, c, output = self.h(h, c, x) # h => hidden, c => cell, a => output
o = self.y(output[0])
# Softmax関数による各単語の生成確率を求める
prob = F.softmax(o)
y = np.argmax(prob.data) # argmaxによってindexを得る
# 出力がEOS信号かcheck。もしそうならbreak
# if y == EOS:
# break
result.append(y)
# 次の入力へのVariable化
x = Variable(np.array([y], dtype=np.int32))
return result
def policy(self, s, eva=False): if self.net_type == "full": s = np.asarray(s.reshape(1, self.input_slides*self.size*self.size), dtype=np.float32) if self.net_type == "conv" or self.net_type == "DQN": s = np.asarray(s.reshape(1, self.input_slides, self.size, self.size), dtype=np.float32) if self.gpu >= 0: s = cuda.to_gpu(s) if chainer.__version__ >= "2.0.0": s = Variable(s) else: s = Variable(s, volatile='auto') if self.exp_policy == "epsilon_greedy": with chainer.no_backprop_mode(): q = self.q(s) q = q.data[0] if self.gpu >= 0: q = cuda.to_cpu(q) q_max = np.amax(q) if eva == True: epsilon = 0.05 else: epsilon = self.epsilon if np.random.rand() < epsilon: action = np.random.randint(0, self.num_of_actions) else: candidate = np.where(q == q_max) action = np.random.choice(candidate[0]) return action, q[action]
def test_no_backprop_mode(self):
xs_data = numpy.random.uniform(-1, 1, (4, 2, 3)).astype(numpy.float32)
t_data = numpy.array([[0, 1], [1, 0]]).astype(numpy.int32)
with chainer.no_backprop_mode():
x = [chainer.Variable(x_data) for x_data in xs_data]
t = chainer.Variable(t_data)
functions.connectionist_temporal_classification(x, t, 2)
def test2(self):
obj = MockDistribution(chainer.Variable(numpy.array([1.])))
with chainer.no_backprop_mode():
h0 = obj.h
h1 = obj.h
assert obj.h_call_count == 1
assert h0 is h1
numpy.testing.assert_allclose(h0.array, 2.)
def plot_clusters():
dataset_train, dataset_test = chainer.datasets.get_mnist()
images_train, labels_train = dataset_train._datasets
images_test, labels_test = dataset_test._datasets
dataset_indices = np.arange(0, len(images_test))
np.random.shuffle(dataset_indices)
model = Model()
assert model.load("model.hdf5")
# normalize
images_train = (images_train - 0.5) * 2
images_test = (images_test - 0.5) * 2
num_clusters = model.ndim_y
num_plots_per_cluster = 11
image_width = 28
image_height = 28
ndim_x = image_width * image_height
pylab.gray()
with chainer.no_backprop_mode() and chainer.using_config("train", False):
# plot cluster head
head_y = np.identity(model.ndim_y, dtype=np.float32)
zero_z = np.zeros((model.ndim_y, model.ndim_z), dtype=np.float32)
head_x = model.decode_yz_x(head_y, zero_z).data
head_x = (head_x + 1.0) / 2.0
for n in range(num_clusters):
pylab.subplot(num_clusters, num_plots_per_cluster + 2, n * (num_plots_per_cluster + 2) + 1)
pylab.imshow(head_x[n].reshape((image_width, image_height)), interpolation="none")
pylab.axis("off")
# plot elements in cluster
counts = [0 for i in range(num_clusters)]
indices = np.arange(len(images_test))
np.random.shuffle(indices)
batchsize = 500
i = 0
x_batch = np.zeros((batchsize, ndim_x), dtype=np.float32)
for n in range(len(images_test) // batchsize):
for b in range(batchsize):
x_batch[b] = images_test[indices[i]]
i += 1
y_batch = model.encode_x_yz(x_batch)[0].data
labels = np.argmax(y_batch, axis=1)
for m in range(labels.size):
cluster = int(labels[m])
counts[cluster] += 1
if counts[cluster] <= num_plots_per_cluster:
x = (x_batch[m] + 1.0) / 2.0
pylab.subplot(num_clusters, num_plots_per_cluster + 2, cluster * (num_plots_per_cluster + 2) + 2 + counts[cluster])
pylab.imshow(x.reshape((image_width, image_height)), interpolation="none")
pylab.axis("off")
fig = pylab.gcf()
fig.set_size_inches(num_plots_per_cluster, num_clusters)
pylab.savefig("clusters.png")
def predict_batch(words_batch):
xs = nlp_utils.transform_to_array(words_batch, vocab, with_label=False)
xs = nlp_utils.convert_seq(xs, device=device, with_label=False)
with chainer.using_config('train', False), chainer.no_backprop_mode():
probs = model.predict(xs, softmax=True)
answers = model.xp.argmax(probs, axis=1)
scores = probs[model.xp.arange(answers.size), answers].tolist()
for words, answer, score in zip(words_batch, answers, scores):
print('{}\t{:.4f}\t{}'.format(answer, score, ' '.join(words)))
def generate_image(self, z):
with chainer.no_backprop_mode(), chainer.using_config('train', False):
h = F.reshape(F.relu(self.bn1(self.l0(z))), (len(z), self.ch, self.bottom_width, self.bottom_width))
h = F.leaky_relu(self.bn2(self.dc1(h)))
h = F.leaky_relu(self.bn3(self.dc2(h)))
h = F.leaky_relu(self.bn4(self.dc3(h)))
x = F.tanh(self.dc4(h))
return x
def test_no_backprop_mode(self):
y = self.x + 1
self.assertTrue(y.creator_node is not None)
with chainer.no_backprop_mode():
y = self.x + 1
self.assertTrue(y.creator_node is None)
y = self.x + 1
self.assertTrue(y.creator_node is not None)
def infer(n_class):
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-g', '--gpu', default=0, type=int, help='GPU id')
parser.add_argument('-m', '--model-file')
parser.add_argument('-i', '--img-files', nargs='+', required=True)
parser.add_argument('-o', '--out-dir', required=True)
args = parser.parse_args()
# model
if args.model_file is None:
args.model_file = fcn.models.FCN8s.download()
match = re.match('^fcn(32|16|8)s.*$', osp.basename(args.model_file))
if match is None:
print('Unsupported model filename: %s' % args.model_file)
quit(1)
model_name = 'FCN%ss' % match.groups()[0]
model_class = getattr(fcn.models, model_name)
model = model_class(n_class=n_class)
chainer.serializers.load_npz(args.model_file, model)
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
# inference
if not osp.exists(args.out_dir):
os.makedirs(args.out_dir)
for file in args.img_files:
# input
img = skimage.io.imread(file)
input, = fcn.datasets.transform_lsvrc2012_vgg16((img,))
input = input[np.newaxis, :, :, :]
if args.gpu >= 0:
input = chainer.cuda.to_gpu(input)
# forward
with chainer.no_backprop_mode():
input = chainer.Variable(input)
with chainer.using_config('train', False):
model(input)
lbl_pred = chainer.functions.argmax(model.score, axis=1)[0]
lbl_pred = chainer.cuda.to_cpu(lbl_pred.data)
# visualize
viz = fcn.utils.visualize_segmentation(
lbl_pred=lbl_pred, img=img, n_class=n_class,
label_names=fcn.datasets.VOC2012ClassSeg.class_names)
out_file = osp.join(args.out_dir, osp.basename(file))
skimage.io.imsave(out_file, viz)
print('==> wrote to: %s' % out_file)
def dump_bilm_embeddings(vocab_file, dataset_file, options_file,
weight_file, outfile, gpu=-1,
batchsize=32):
with open(options_file, 'r') as fin:
options = json.load(fin)
max_word_length = options['char_cnn']['max_characters_per_token']
vocab = UnicodeCharsVocabulary(vocab_file, max_word_length)
batcher = Batcher(vocab_file, max_word_length)
model = Elmo(
options_file,
weight_file,
num_output_representations=1,
requires_grad=False,
do_layer_norm=False,
dropout=0.)
if gpu >= 0:
cuda.get_device_from_id(gpu).use()
model.to_gpu()
# (batch_size, timesteps, 50)
# TODO(sosk): preencoding token embedding for acceleration
with chainer.using_config("train", False), \
chainer.no_backprop_mode():
sentence_id = 0
n_lines = sum([1 for _ in open(dataset_file, 'r')])
with open(dataset_file, 'r') as fin, h5py.File(outfile, 'w') as fout:
for minibatch in minibatch_iterator(tqdm.tqdm(fin, total=n_lines),
batchsize):
sentences = [line.strip().split() for line in minibatch]
char_ids = batcher.batch_sentences(
sentences, add_bos_eos=False)
char_ids = model.xp.asarray(char_ids)
mb_outs = model.forward(char_ids)
mb_embedding_layers = mb_outs['elmo_layers']
# [(batch_size, max_sequence_length, embedding_dim), ..., x n_layers]
# Note that embedding layers have already trushed bos & eos
# But they contains padding
mb_mask = mb_outs['mask']
mb_concat_embedding_layers = cuda.to_cpu(
model.xp.stack([mb_emb.array for mb_emb in mb_embedding_layers], axis=1))
# (batch_size, n_layers=3, max_sequence_length, embedding_dim)
for mask, concat_embedding_layers in zip(mb_mask, mb_concat_embedding_layers):
# remove pads
length = int(mask.sum())
concat_embedding_layers = concat_embedding_layers[:, :length]
# (n_layers=3, sequence_length, embedding_dim)
ds = fout.create_dataset(
'{}'.format(sentence_id),
concat_embedding_layers.shape,
dtype='float32',
data=concat_embedding_layers
)
sentence_id += 1
def predict(self, imgs):
lbls = []
for img in imgs:
with chainer.no_backprop_mode(), \
chainer.using_config('train', False):
x = self.xp.asarray(img[None])
self.__call__(x)
lbl = chainer.functions.argmax(self.score, axis=1)
lbl = chainer.cuda.to_cpu(lbl.array[0])
lbls.append(lbl)
return lbls
def beam_search_predict(self, x, max_length=10, beam_width=3):
"""
ビームサーチを用いたpredict
:param x: 入力文
:param max_length: 出力文の制御
:param beam_width: ビーム幅
:return: 出力文のindex
"""
# import heapq
with chainer.no_backprop_mode(), chainer.using_config('train', False):
result = []
y = [EOS]
# Embedding Layer
x = x[::-1] # 入力を反転したほうが精度が上がる
emb_x = [self.x_embed(x)]
# Encoder
h, c, a = self.encoder(None, None, emb_x)
# beam search
heaps = [[] for _ in range(max_length + 1)]
heaps[0].append((0, y, h, c)) # socre, word, hidden state, cell state
result_score = 1e8
# Decoder, Output Layerの計算
for i in range(max_length):
heaps[i] = sorted(heaps[i], key=lambda t: t[0])[:beam_width]
for score, y, h, c in heaps[i]:
prob, h, c = self.generate_one_step(y, h, c, a) # decoderの計算
for next_index in self.xp.argsort(prob.data)[::-1]:
if prob.data[next_index] < 1e-6:
break
next_score = score - self.xp.log(prob.data[next_index])
if next_score > result_score:
break
next_word = y + [next_index]
next_item = (next_score, next_word, h, c)
if next_index == EOS:
if next_score < result_score:
result = y[1:] # EOS信号の削除
print("result: {}".format(result))
result_score = next_score
else:
heaps[i+1].append(next_item)
return result
def update_Q():
# Predicted values: Q(s,a)
y = F.squeeze(Q(obs, action), axis=1)
# Target values: r + gamma * Q(s,policy(s))
with chainer.no_backprop_mode():
next_q = F.squeeze(target_Q(obs_next, target_policy(obs_next)),
axis=1)
target = reward + gamma * (1 - done) * next_q
loss = F.mean_squared_error(y, target)
Q.cleargrads()
loss.backward()
opt_Q.update()
def evaluate(model, test_trees):
result = collections.defaultdict(lambda: 0)
with chainer.using_config('train', False), chainer.no_backprop_mode():
for tree in test_trees:
model.traverse(tree, evaluate=result)
acc_node = 100.0 * result['correct_node'] / result['total_node']
acc_root = 100.0 * result['correct_root'] / result['total_root']
print(' Node accuracy: {0:.2f} %% ({1:,d}/{2:,d})'.format(
acc_node, result['correct_node'], result['total_node']))
print(' Root accuracy: {0:.2f} %% ({1:,d}/{2:,d})'.format(
acc_root, result['correct_root'], result['total_root']))
def translate(self, xs, max_length=100):
print("Now translating")
batch = len(xs)
print("batch", batch)
#loss_w = 0
#loss_c1 = 0
#loss_c2 = 0
with chainer.no_backprop_mode(), chainer.using_config('train', False):
char_hidden = []
wxs = [
np.array([source_word_ids.get(w, UNK) for w in x],
dtype=np.int32) for x in xs
]
unk_words = list(map(lambda x, y: np.array(y)[x == UNK], wxs, xs))
unk_xs = list(
map(
lambda x: np.array([
np.array(
[source_char_ids.get(c, UNK) for c in list(w)],
dtype=np.int32) for w in x
]), unk_words))
unk_pos = [np.where(x == UNK)[0] for x in wxs]
wx_len = [len(wx) for wx in wxs]
wx_section = np.cumsum(wx_len[:-1])
valid_wx_section = np.insert(wx_section, 0, 0)
concat_wxs = np.concatenate(wxs)
#wys = [np.array([target_word_ids.get(w, UNK) for w in y], dtype=np.int32) for y in ys]
#eos = self.xp.array([EOS], 'i')
#ys_out = [F.concat([y, eos], axis=0) for y in wys]
#concat_ys_out = F.concat(ys_out, axis=0)
#n_words = len(concat_ys_out)
exs = sequence_embed(self.embed_x, wxs)
exs = list(
map(
lambda s, t, u: get_unk_hidden_vector(
s, t, u, self.embed_xc, self.char_encoder, char_hidden
), exs, unk_pos, unk_xs))
exs_f = exs
exs_b = [ex[::-1] for ex in exs]
_, hf = self.encoder_f(None, exs_f)
_, hb = self.encoder_b(None, exs_b)
ht = list(map(lambda x, y: F.concat([x, y], axis=1), hf, hb))
ys = self.xp.full(batch, EOS, 'i')
result = []
h_list = None
for a in range(max_length):
eys = self.embed_y(ys)
eys = F.split_axis(eys, batch, 0)
if h_list == None:
h0 = h_list
else:
h0 = F.transpose_sequence(h_list)[-1]
h0 = F.reshape(h0,
(self.n_layers, h0.shape[0], h0.shape[1]))
#h0 : {type:variable, shape:(n_layers*batch*dimentionality)} or None
h_list, h_bar_list, c_s_list, z_s_list = self.decoder(
h0, ht, eys)
os = h_list
concat_os = F.concat(os, axis=0)
concat_os_out = self.W(concat_os)
concat_pred_w = self.xp.argmax(concat_os_out.data,
axis=1).astype('i')
is_unk = concat_pred_w == UNK
if UNK in concat_pred_w:
N = np.sum(is_unk)
true_wys = concat_ys_out[is_unk]
concat_c_s = F.concat(c_s_list, axis=0)
concat_h_bar = F.concat(h_bar_list, axis=0)
c_ss = concat_c_s[is_unk]
h_bars = concat_h_bar[is_unk]
c = F.concat([c_ss, h_bars], axis=1)
ds_hats = F.relu(self.W_hat(c))
abs_z_s_list = [
z_s_list[i] + valid_wx_section[i]
for i in range(len(z_s_list))
]
concat_z_s = F.concat(abs_z_s_list, axis=0)
z_ss = concat_z_s[is_unk]
#各UNK単語について
results_c = []
bow = self.xp.array([BOW], 'i')
for i in range(N):
wy = true_wys[i]
if wy != UNK and wy != EOS:
cys = np.array([[
target_char_ids[c]
for c in list(target_words[wy])
]], np.int32)
elif wy == UNK:
#本来ありえない
cys = np.array([[target_char_ids['UNK']]],
np.int32)
elif wy == EOS:
cys = np.array([[target_char_ids['BOW']]],
np.int32)
cys_out = [F.concat([y, bow], axis=0) for y in cys]
concat_cys_out = F.concat(cys_out, axis=0)
result_c = []
cy = self.xp.full(1, BOW, 'i')
cy = F.split_axis(cy, 1, 0)
cey = sequence_embed(self.embed_yc, cy)
z_s = int(z_ss[i].data)
ds_hat = F.reshape(ds_hats[i],
(1, 1, ds_hats[i].shape[0]))
cos_out_list = []
if concat_wxs[z_s] != UNK:
for b in range(10):
#attentionなし文字ベースdecoder
ds_hat, cos = self.char_decoder(ds_hat, cey)
cos_out = self.W_char(cos[0])
cos_out_list.append(cos_out)
pred_cos = self.xp.argmax(cos_out.data,
axis=1).astype('i')
cey = self.embed_yc(pred_cos)
print(pred_cos)
print(target_chars[pred_cos])
result_c.append(pred_cos)
#concat_cos_out = F.concat(cos_out_list, axis=0)
#loss_c1= loss_c1 + F.sum(F.softmax_cross_entropy(
# concat_cos_out, concat_cys_out, reduce='no'))
else:
c_ht = char_hidden[z_s]
for b in range(10):
#attentionあり文字ベースdecoder
if b == 0:
c_h0 = ds_hat
else:
c_h0 = F.transpose_sequence(h_list)[-1]
c_h0 = F.reshape(
c_h0, (self.n_layers, c_h0.shape[0],
c_h0.shape[1]))
c_h_list, c_h_bar_list, c_c_s_list, c_z_s_list = self.char_att_decoder(
c_h0, c_ht, cey)
cos_out = self.W_char(h_list[-1])
cos_out_list.append(cos_out)
pred_cos = self.xp.argmax(cos_out.data,
axis=1).astype('i')
cey = self.embed_yc(pred_cos)
print(pred_cos)
print(target_chars[pred_cos])
result_c.append(pred_cos)
#concat_cos_out = F.concat(cos_out_list, axis=0)
#loss_c2 = loss_c2 + F.sum(F.softmax_cross_entropy(
# concat_cos_out, concat_cys_out, reduce='no'))
r = ""
for c in result_c:
if c == BOW:
break
r += target_chars.get(c, UNK)
print(r)
pred_w = target_word_ids.get(r, UNK)
results_c.append(pred_w)
concat_pred_w[is_unk] = results_c
#loss_w = loss_w + F.sum(F.softmax_cross_entropy(
# concat_os_out[is_unk!=1], concat_ys_out[is_unk!=1], reduce='no'))
result.append(concat_pred_w)
#loss = F.sum(loss_w + Alpha * loss_c1 + Beta * loss_c2) / n_words
result = cuda.to_cpu(self.xp.stack(result).T)
# Remove EOS taggs
outs = []
for y in result:
inds = np.argwhere(y == EOS)
if len(inds) > 0:
y = y[:inds[0, 0]]
outs.append(y)
return outs
def predict(self, atoms, adjs):
# type: (numpy.ndarray, numpy.ndarray) -> chainer.Variable
# TODO (nakago): support super_node & is_real_node args.
with chainer.no_backprop_mode(), chainer.using_config('train', False):
x = self.__call__(atoms, adjs)
return self.postprocess_fn(x)
def main():
parser = argparse.ArgumentParser(description='Chainer example: VAE')
parser.add_argument('--initmodel', '-m', default='',
help='Initialize the model from given file')
parser.add_argument('--resume', '-r', default='',
help='Resume the optimization from snapshot')
parser.add_argument('--gpu', '-g', default=-1, type=int,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out', '-o', default='result',
help='Directory to output the result')
parser.add_argument('--epoch', '-e', default=100, type=int,
help='number of epochs to learn')
parser.add_argument('--dimz', '-z', default=20, type=int,
help='dimention of encoded vector')
parser.add_argument('--batchsize', '-b', type=int, default=100,
help='learning minibatch size')
parser.add_argument('--test', action='store_true',
help='Use tiny datasets for quick tests')
args = parser.parse_args()
print('GPU: {}'.format(args.gpu))
print('# dim z: {}'.format(args.dimz))
print('# Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('')
# Prepare VAE model, defined in net.py
model = net.VAE(784, args.dimz, 500)
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# Initialize
if args.initmodel:
chainer.serializers.load_npz(args.initmodel, model)
# Load the MNIST dataset
train, test = chainer.datasets.get_mnist(withlabel=False)
if args.test:
train, _ = chainer.datasets.split_dataset(train, 100)
test, _ = chainer.datasets.split_dataset(test, 100)
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test, args.batchsize,
repeat=False, shuffle=False)
# Set up an updater. StandardUpdater can explicitly specify a loss function
# used in the training with 'loss_func' option
updater = training.updaters.StandardUpdater(
train_iter, optimizer,
device=args.gpu, loss_func=model.get_loss_func())
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu,
eval_func=model.get_loss_func(k=10)))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot(), trigger=(args.epoch, 'epoch'))
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/rec_loss', 'validation/main/rec_loss', 'elapsed_time']))
trainer.extend(extensions.ProgressBar())
if args.resume:
chainer.serializers.load_npz(args.resume, trainer)
# Run the training
trainer.run()
# Visualize the results
def save_images(x, filename):
import matplotlib.pyplot as plt
fig, ax = plt.subplots(3, 3, figsize=(9, 9), dpi=100)
for ai, xi in zip(ax.flatten(), x):
ai.imshow(xi.reshape(28, 28))
fig.savefig(filename)
model.to_cpu()
train_ind = [1, 3, 5, 10, 2, 0, 13, 15, 17]
x = chainer.Variable(np.asarray(train[train_ind]))
with chainer.using_config('train', False), chainer.no_backprop_mode():
x1 = model(x)
save_images(x.array, os.path.join(args.out, 'train'))
save_images(x1.array, os.path.join(args.out, 'train_reconstructed'))
test_ind = [3, 2, 1, 18, 4, 8, 11, 17, 61]
x = chainer.Variable(np.asarray(test[test_ind]))
with chainer.using_config('train', False), chainer.no_backprop_mode():
x1 = model(x)
save_images(x.array, os.path.join(args.out, 'test'))
save_images(x1.array, os.path.join(args.out, 'test_reconstructed'))
# draw images from randomly sampled z
z = chainer.Variable(
np.random.normal(0, 1, (9, args.dimz)).astype(np.float32))
x = model.decode(z)
save_images(x.array, os.path.join(args.out, 'sampled'))
def decode(self, ws, ps=None, label_prediction=False):
self.label_prediction = label_prediction
with chainer.no_backprop_mode():
ret = self.__call__(ws, ps=ps, train=False, calculate_loss=False)
return ret[1:]
def do_paint(sketchimg, referenceimg, hintimg, version, denoise, outputimg):
versionURL = version
denoiseURL = denoise
print('version/denoise : ', versionURL, denoiseURL)
#sketchimg = 'mysketch.png'
sketchDataURL = cv2.imread(sketchimg, cv2.IMREAD_UNCHANGED)
#referenceimg = 'myref.png'
referenceDataURL = cv2.imread(referenceimg, cv2.IMREAD_UNCHANGED)
#hintimg = 'myhint.png'
hintDataURL = cv2.imread(hintimg, cv2.IMREAD_UNCHANGED)
#resultimg='myresult.png'
# end lulu space
low_level_scale = 28
shifter = 2.0
up_level = True
interpolation = cv2.INTER_AREA
high_level_scale = 32
high_interpolation = cv2.INTER_AREA
if versionURL == '1':
low_level_scale = 16
shifter = 1.3
up_level = True
interpolation = cv2.INTER_AREA
high_level_scale = 32
high_interpolation = cv2.INTER_AREA
if versionURL == '2':
low_level_scale = 28
shifter = 1.2
up_level = True
interpolation = cv2.INTER_AREA
high_level_scale = 32
high_interpolation = cv2.INTER_AREA
if versionURL == '3':
low_level_scale = 48
shifter = 1.1
up_level = True
interpolation = cv2.INTER_LANCZOS4
high_level_scale = 64
high_interpolation = cv2.INTER_LANCZOS4
if versionURL == '4':
low_level_scale = 64
shifter = 1.0
up_level = False
interpolation = cv2.INTER_LANCZOS4
high_level_scale = 64
high_interpolation = cv2.INTER_LANCZOS4
raw_sketch = from_png_to_jpg(sketchDataURL)
raw_sketch_shape = raw_sketch.shape
# shape_x = raw_sketch_shape[0]
# shape_y = raw_sketch_shape[1]
# if shape_x > shape_y:
# new_shape_x = 512.0 / shape_y * shape_x
# new_shape_y = 512.0
# else:
# new_shape_x = 512.0
# new_shape_y = 512.0 / shape_x * shape_y
# raw_sketch_shape = (int(new_shape_x),int(new_shape_y))
raw_sketch = cv2.cvtColor(raw_sketch, cv2.COLOR_RGB2GRAY)
normed_sketch = norm_sketch(raw_sketch, denoiseURL)
sketch = unet_resize(normed_sketch, low_level_scale, interpolation)
sketch = sketch.astype(np.float)
sketch = 1 - (sketch / 255.0)
sketch = sketch[None, :, :, None]
reference = from_png_to_jpg(referenceDataURL)
reference = cv2.resize(reference, (224, 224), interpolation=cv2.INTER_AREA)
reference = reference.astype(np.float32)
reference = reference / 255.0
reference = reference.transpose((2, 0, 1))[None, :, :, :]
t = time.time()
if is_GPU:
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
vhint_s57c64_0, vhint_s29c192_0, vhint_s29c256_0, vhint_s29c320_0, vhint_s15c576_0, vhint_s15c576_1, vhint_s15c576_2, vhint_s15c576_3, vhint_s15c576_4, vhint_s8c1024_0, vhint_s8c1024_1, vhint_s8c1024_2 = google_net.forward(
chainer.cuda.to_gpu(reference, chainer_ID))
else:
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
vhint_s57c64_0, vhint_s29c192_0, vhint_s29c256_0, vhint_s29c320_0, vhint_s15c576_0, vhint_s15c576_1, vhint_s15c576_2, vhint_s15c576_3, vhint_s15c576_4, vhint_s8c1024_0, vhint_s8c1024_1, vhint_s8c1024_2 = google_net.forward(
reference)
print(time.time() - t)
hint = hintDataURL[:, :, 0:4]
color = hint[:, :, 0:3]
color = cv2.cvtColor(color, cv2.COLOR_RGB2HSV).astype(np.float)
color[:, :, 1] *= shifter
color = color.clip(0, 255).astype(np.uint8)
color = cv2.cvtColor(color, cv2.COLOR_HSV2RGB)
hint[:, :, 0:3] = color
hint = cv2.resize(hint, (sketch.shape[2], sketch.shape[1]), cv2.INTER_AREA)
hint = hint.astype(np.float)
local_hint = hint[:, :, 0:3]
alpha = hint[:, :, 3] / 255.0
#alpha = 1
local_hint = local_hint - 127
local_hint = local_hint / 128.0
for _ in range(3):
local_hint[:, :, _] = np.multiply(local_hint[:, :, _], alpha)
hint = local_hint[None, :, :, :]
t = time.time()
if is_GPU:
final = session.run(
paint_output,
feed_dict={
sketch_ref_input_448: sketch,
local_hint_input_448: hint,
hint_s57c64_0: chainer.cuda.to_cpu(vhint_s57c64_0.data)[None],
hint_s29c192_0:
chainer.cuda.to_cpu(vhint_s29c192_0.data)[None],
hint_s29c256_0:
chainer.cuda.to_cpu(vhint_s29c256_0.data)[None],
hint_s29c320_0:
chainer.cuda.to_cpu(vhint_s29c320_0.data)[None],
hint_s15c576_0:
chainer.cuda.to_cpu(vhint_s15c576_0.data)[None],
hint_s15c576_1:
chainer.cuda.to_cpu(vhint_s15c576_1.data)[None],
hint_s15c576_2:
chainer.cuda.to_cpu(vhint_s15c576_2.data)[None],
hint_s15c576_3:
chainer.cuda.to_cpu(vhint_s15c576_3.data)[None],
hint_s15c576_4:
chainer.cuda.to_cpu(vhint_s15c576_4.data)[None],
hint_s8c1024_0:
chainer.cuda.to_cpu(vhint_s8c1024_0.data)[None],
hint_s8c1024_1:
chainer.cuda.to_cpu(vhint_s8c1024_1.data)[None],
hint_s8c1024_2: chainer.cuda.to_cpu(vhint_s8c1024_2.data)[None]
})
else:
final = session.run(paint_output,
feed_dict={
sketch_ref_input_448: sketch,
local_hint_input_448: hint,
hint_s57c64_0: vhint_s57c64_0.data[None],
hint_s29c192_0: vhint_s29c192_0.data[None],
hint_s29c256_0: vhint_s29c256_0.data[None],
hint_s29c320_0: vhint_s29c320_0.data[None],
hint_s15c576_0: vhint_s15c576_0.data[None],
hint_s15c576_1: vhint_s15c576_1.data[None],
hint_s15c576_2: vhint_s15c576_2.data[None],
hint_s15c576_3: vhint_s15c576_3.data[None],
hint_s15c576_4: vhint_s15c576_4.data[None],
hint_s8c1024_0: vhint_s8c1024_0.data[None],
hint_s8c1024_1: vhint_s8c1024_1.data[None],
hint_s8c1024_2: vhint_s8c1024_2.data[None]
})
print(time.time() - t)
final = final[0]
final += [103.939, 116.779, 123.68]
final = final[:, :, ::-1]
final = final.clip(0, 255).astype(np.uint8)
t = time.time()
if up_level:
sketch = unet_resize(normed_sketch, high_level_scale,
high_interpolation)
final = cv2.resize(final, (sketch.shape[1], sketch.shape[0]),
cv2.INTER_LANCZOS4)
final = cv2.cvtColor(final, cv2.COLOR_RGB2YUV)
final = final[None, :, :, :]
sketch = sketch[None, :, :, None]
fin = session.run(combined_output,
feed_dict={
sketch_ref_input_448: sketch,
local_hint_input_448: final
})[0].clip(0, 255).astype(np.uint8)
fin = cv2.cvtColor(fin, cv2.COLOR_YUV2RGB)
else:
fin = final
print(time.time() - t)
fin = cv2.cvtColor(fin, cv2.COLOR_RGB2HSV).astype(np.float)
fin[:, :, 1] /= 0.9
fin = fin.clip(0, 255).astype(np.uint8)
fin = cv2.cvtColor(fin, cv2.COLOR_HSV2RGB)
fin = cv2.resize(fin, (raw_sketch_shape[1], raw_sketch_shape[0]),
cv2.INTER_LANCZOS4)
#cv2.imwrite('record/' + dstr + '.fin.jpg', fin)
#result_path = 'results/' + dstr + '.jpg'
cv2.imwrite(outputimg, fin)
def beam_search_translate(model, xs, k_beam=4, max_length=100, use_unk=False):
""" A basic beam search implementation
Parameters:
-----------
model: our seq2seq model
xs: our source sequence of ids
k_beam: size of beam search
max_length: maximum length of sequence
use_unk: if True, UNK could be used as output word
if False, an alternative word should be used instead
"""
sample = []
sample_score = []
live_k = 1
dead_k = 0
hyp_samples = [[]] * live_k
hyp_scores = model.xp.zeros(live_k).astype('float32')
hyp_states = []
batch = len(xs)
assert(batch) == 1, "batch must be 1 in this implementation"
with chainer.no_backprop_mode(), chainer.using_config('train', False):
# obtaining initial state in encoder
xs = [x[::-1] for x in xs]
exs = sequence_embed(model.embed_x, xs)
# h and c are #layers x #batch x # dim
h, c, _ = model.encoder(None, None, exs)
# repeat encoder state till full beam is reached
h = chainer.Variable(h.data.repeat(k_beam, axis=1))
c = chainer.Variable(c.data.repeat(k_beam, axis=1))
# zeroing initial target words
ys = model.xp.full(k_beam, EOS, 'i')
for i in range(max_length):
live_batch = len(ys)
eys = model.embed_y(ys)
eys = F.split_axis(eys, live_batch, 0)
h, c, ys = model.decoder(h, c, eys)
cys = F.concat(ys, axis=0)
wy = model.W(cys)
probs = F.softmax(wy)
probs_data = probs.data - model.xp.finfo(model.xp.float32).epsneg
cand_scores = hyp_scores[:, None] - model.xp.log(probs_data[:live_k, :])
cand_flat = cand_scores.flatten()
# TODO filter UNK words here (before ranking)
voc_size = probs_data.shape[1]
if not use_unk:
for xx in range(int(len(cand_flat) / int(voc_size))):
cand_flat[voc_size * xx + UNK] = 1e20
ranks_flat = cand_flat.argsort()[:(k_beam-dead_k)]
trans_indices = ((ranks_flat / voc_size)).astype('int32')
word_indices = ranks_flat % voc_size
costs = cand_flat[ranks_flat]
new_hyp_samples = []
new_hyp_scores = model.xp.zeros(k_beam-dead_k).astype('float32')
new_hyp_states = []
for idx, [ti, wi] in enumerate(zip(trans_indices, word_indices)):
new_hyp_samples.append(hyp_samples[ti]+[wi])
new_hyp_scores[idx] = copy.copy(costs[idx])
new_hyp_states.append(copy.copy((h[:,ti,:],c[:,ti,:])))
# check the finished samples
new_live_k = 0
hyp_samples = []
hyp_scores = []
hyp_states = []
for idx in range(len(new_hyp_samples)):
if new_hyp_samples[idx][-1] == EOS:
sample.append(new_hyp_samples[idx])
sample_score.append(new_hyp_scores[idx])
dead_k += 1
else:
new_live_k += 1
hyp_samples.append(new_hyp_samples[idx])
hyp_scores.append(new_hyp_scores[idx])
hyp_states.append(new_hyp_states[idx])
hyp_scores = numpy.array(hyp_scores)
live_k = new_live_k
if new_live_k < 1:
break
if dead_k >= k_beam:
break
# Next state
# state is #layers x #batch x #dim
ys = numpy.array([w[-1] for w in hyp_samples]).astype('i')
h = F.stack([hyp_states[k][0] for k in range(len(hyp_states))], axis=1)
c = F.stack([hyp_states[k][1] for k in range(len(hyp_states))], axis=1)
# dump every remaining one
if live_k > 0:
for idx in xrange(live_k):
sample.append(hyp_samples[idx])
sample_score.append(hyp_scores[idx])
return sample, sample_score
def predict(self, x, **kwargs):
x = self.adapt(x)
with chainer.using_config('train', False), chainer.no_backprop_mode():
return F.sigmoid(self.predictor(x, inference=True, **kwargs))
def temperature(self):
if self.entropy_target is None:
return self.initial_temperature
else:
with chainer.no_backprop_mode():
return float(self.temperature_holder().array)
def main():
print("chainer cudnn enabled: {}".format(chainer.cuda.cudnn_enabled))
parser = argparse.ArgumentParser(
description='Action Unit Intensity R-CNN training example:')
parser.add_argument('--pid', '-pp', default='/tmp/AU_R_CNN/')
parser.add_argument('--gpu',
'-g',
default="0",
help='GPU ID, multiple GPU split by comma')
parser.add_argument('--lr', '-l', type=float, default=0.001)
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
parser.add_argument('--database',
default='BP4D',
help='Output directory: BP4D/DISFA/BP4D_DISFA')
parser.add_argument('--seed', '-s', type=int, default=0)
parser.add_argument('--iteration', '-i', type=int, default=70000)
parser.add_argument('--epoch', '-e', type=int, default=20)
parser.add_argument('--batch_size', '-bs', type=int, default=20)
parser.add_argument('--snapshot', '-snap', type=int, default=1000)
parser.add_argument('--mean',
default=config.ROOT_PATH + "BP4D/idx/mean_rgb.npy",
help='image mean .npy file')
parser.add_argument('--feature_model',
default="resnet101",
help="vgg or resnet101 for train")
parser.add_argument('--extract_len', type=int, default=1000)
parser.add_argument('--optimizer',
default='RMSprop',
help='optimizer: RMSprop/AdaGrad/Adam/SGD/AdaDelta')
parser.add_argument('--pretrained_model',
default='resnet101',
help='imagenet/vggface/resnet101/*.npz')
parser.add_argument(
'--use_memcached',
action='store_true',
help='whether use memcached to boost speed of fetch crop&mask') #
parser.add_argument('--memcached_host', default='127.0.0.1')
parser.add_argument("--fold", '-fd', type=int, default=3)
parser.add_argument("--split_idx", '-sp', type=int, default=1)
parser.add_argument("--proc_num", "-proc", type=int, default=1)
parser.add_argument(
"--is_pretrained",
action="store_true",
help="whether is to pretrain BP4D later will for DISFA dataset or not")
parser.add_argument(
"--pretrained_target",
'-pt',
default="",
help="whether pretrain label set will use DISFA or not")
parser.add_argument("--prefix",
'-prefix',
default="",
help="_beta, for example 3_fold_beta")
parser.add_argument('--eval_mode',
action='store_true',
help='Use test datasets for evaluation metric')
parser.add_argument("--img_resolution", type=int, default=512)
args = parser.parse_args()
if not os.path.exists(args.pid):
os.makedirs(args.pid)
pid = str(os.getpid())
pid_file_path = args.pid + os.sep + "{0}_{1}_fold_{2}.pid".format(
args.database, args.fold, args.split_idx)
with open(pid_file_path, "w") as file_obj:
file_obj.write(pid)
file_obj.flush()
config.IMG_SIZE = (args.img_resolution, args.img_resolution)
print('GPU: {}'.format(args.gpu))
if args.is_pretrained:
adaptive_AU_database(args.pretrained_target)
else:
adaptive_AU_database(args.database)
np.random.seed(args.seed)
# 需要先构造一个list的txt文件:id_trainval_0.txt, 每一行是subject + "/" + emotion_seq + "/" frame
mc_manager = None
if args.use_memcached:
from collections_toolkit.memcached_manager import PyLibmcManager
mc_manager = PyLibmcManager(args.memcached_host)
if mc_manager is None:
raise IOError("no memcached found listen in {}".format(
args.memcached_host))
if args.feature_model == 'vgg':
faster_rcnn = FasterRCNNVGG16(
n_fg_class=len(config.AU_INTENSITY_DICT),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len
) # 可改为/home/nco/face_expr/result/snapshot_model.npz
elif args.feature_model == 'resnet101':
faster_rcnn = FasterRCNNResnet101(
n_fg_class=len(config.AU_INTENSITY_DICT),
pretrained_model=args.pretrained_model,
mean_file=args.mean,
min_size=args.img_resolution,
max_size=args.img_resolution,
extract_len=args.extract_len
) # 可改为/home/nco/face_expr/result/snapshot_model.npz
if args.eval_mode:
with chainer.no_backprop_mode(), chainer.using_config("train", False):
test_data = AUDataset(database=args.database,
fold=args.fold,
img_resolution=args.img_resolution,
split_name='test',
split_index=args.split_idx,
mc_manager=mc_manager,
prefix=args.prefix,
pretrained_target=args.pretrained_target)
test_data = TransformDataset(test_data,
Transform(faster_rcnn, mirror=False))
if args.proc_num == 1:
test_iter = SerialIterator(test_data,
1,
repeat=False,
shuffle=True)
else:
test_iter = MultiprocessIterator(test_data,
batch_size=1,
n_processes=args.proc_num,
repeat=False,
shuffle=True,
n_prefetch=10,
shared_mem=10000000)
gpu = int(args.gpu) if "," not in args.gpu else int(
args.gpu[:args.gpu.index(",")])
chainer.cuda.get_device_from_id(gpu).use()
faster_rcnn.to_gpu(gpu)
evaluator = AUEvaluator(
test_iter,
faster_rcnn,
lambda batch, device: concat_examples_not_none(
batch, device, padding=-99),
args.database,
"/home/machen/face_expr",
device=gpu)
observation = evaluator.evaluate()
with open(
args.out + os.path.sep +
"evaluation_intensity_split_{}_result_test_mode.json".
format(args.split_idx), "w") as file_obj:
file_obj.write(
json.dumps(observation, indent=4, separators=(',', ': ')))
file_obj.flush()
return
train_data = AUDataset(database=args.database,
img_resolution=args.img_resolution,
fold=args.fold,
split_name='trainval',
split_index=args.split_idx,
mc_manager=mc_manager,
prefix=args.prefix,
pretrained_target=args.pretrained_target)
train_data = TransformDataset(train_data,
Transform(faster_rcnn, mirror=True))
if args.proc_num == 1:
train_iter = SerialIterator(train_data, args.batch_size, True, True)
else:
train_iter = MultiprocessIterator(train_data,
batch_size=args.batch_size,
n_processes=args.proc_num,
repeat=True,
shuffle=True,
n_prefetch=10,
shared_mem=31457280)
model = FasterRCNNTrainChain(faster_rcnn)
if "," in args.gpu:
for gpu in args.gpu.split(","):
chainer.cuda.get_device_from_id(int(gpu)).use()
else:
chainer.cuda.get_device_from_id(int(args.gpu)).use()
optimizer = None
if args.optimizer == 'AdaGrad':
optimizer = chainer.optimizers.AdaGrad(
lr=args.lr
) # 原本为MomentumSGD(lr=args.lr, momentum=0.9) 由于loss变为nan问题，改为AdaGrad
elif args.optimizer == 'RMSprop':
optimizer = chainer.optimizers.RMSprop(lr=args.lr)
elif args.optimizer == 'Adam':
print("using Adam")
optimizer = chainer.optimizers.Adam(alpha=args.lr)
elif args.optimizer == 'SGD':
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr, momentum=0.9)
elif args.optimizer == "AdaDelta":
print("using AdaDelta")
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(rate=0.0005))
optimizer_name = args.optimizer
if not os.path.exists(args.out):
os.makedirs(args.out)
pretrained_optimizer_file_name = '{0}_fold_{1}_{2}_{3}_{4}_optimizer.npz'.format(
args.fold, args.split_idx, args.feature_model, "linear",
optimizer_name)
pretrained_optimizer_file_name = args.out + os.sep + pretrained_optimizer_file_name
single_model_file_name = args.out + os.sep + '{0}_fold_{1}_{2}_{3}_model.npz'.format(
args.fold, args.split_idx, args.feature_model, "linear")
if os.path.exists(pretrained_optimizer_file_name):
print("loading optimizer snatshot:{}".format(
pretrained_optimizer_file_name))
chainer.serializers.load_npz(pretrained_optimizer_file_name, optimizer)
if os.path.exists(single_model_file_name):
print("loading pretrained snapshot:{}".format(single_model_file_name))
chainer.serializers.load_npz(single_model_file_name, model.faster_rcnn)
if "," in args.gpu:
gpu_dict = {"main": int(args.gpu.split(",")[0])} # many gpu will use
for slave_gpu in args.gpu.split(",")[1:]:
gpu_dict[slave_gpu] = int(slave_gpu)
updater = chainer.training.ParallelUpdater(
train_iter,
optimizer,
devices=gpu_dict,
converter=lambda batch, device: concat_examples(
batch, device, padding=-99))
else:
print("only one GPU({0}) updater".format(args.gpu))
updater = chainer.training.StandardUpdater(
train_iter,
optimizer,
device=int(args.gpu),
converter=lambda batch, device: concat_examples(
batch, device, padding=-99))
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out)
trainer.extend(chainer.training.extensions.snapshot_object(
optimizer, filename=os.path.basename(pretrained_optimizer_file_name)),
trigger=(args.snapshot, 'iteration'))
snap_model_file_name = '{0}_fold_{1}_{2}_model.npz'.format(
args.fold, args.split_idx, args.feature_model)
trainer.extend(chainer.training.extensions.snapshot_object(
model.faster_rcnn, filename=snap_model_file_name),
trigger=(args.snapshot, 'iteration'))
log_interval = 100, 'iteration'
print_interval = 100, 'iteration'
plot_interval = 100, 'iteration'
if args.optimizer != "Adam" and args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.ExponentialShift('lr', 0.1),
trigger=(10, 'epoch'))
elif args.optimizer == "Adam":
# use Adam
trainer.extend(chainer.training.extensions.ExponentialShift(
"alpha", 0.5, optimizer=optimizer),
trigger=(10, 'epoch'))
if args.optimizer != "AdaDelta":
trainer.extend(chainer.training.extensions.observe_lr(),
trigger=log_interval)
trainer.extend(
chainer.training.extensions.LogReport(
trigger=log_interval,
log_name="{0}_fold_{1}.log".format(args.fold, args.split_idx)))
trainer.extend(chainer.training.extensions.PrintReport([
'iteration',
'epoch',
'elapsed_time',
'lr',
'main/loss',
'main/pearson_correlation',
]),
trigger=print_interval)
trainer.extend(
chainer.training.extensions.ProgressBar(update_interval=100))
if chainer.training.extensions.PlotReport.available():
trainer.extend(chainer.training.extensions.PlotReport(
['main/loss'],
file_name='loss_{0}_fold_{1}.png'.format(args.fold,
args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.extend(chainer.training.extensions.PlotReport(
['main/pearson_correlation'],
file_name='pearson_correlation_{0}_fold_{1}.png'.format(
args.fold, args.split_idx),
trigger=plot_interval),
trigger=plot_interval)
trainer.run()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--batchsize", "-b", type=int, default=64)
parser.add_argument("--total-epochs", "-e", type=int, default=5000)
parser.add_argument("--num-labeled-data", "-nl", type=int, default=100)
parser.add_argument("--gpu-device", "-g", type=int, default=0)
parser.add_argument("--grad-clip", "-gc", type=float, default=5)
parser.add_argument("--seed", type=int, default=0)
parser.add_argument("--model", "-m", type=str, default="model.hdf5")
args = parser.parse_args()
np.random.seed(args.seed)
model = Model()
model.load(args.model)
mnist_train, mnist_test = chainer.datasets.get_mnist()
images_train, labels_train = mnist_train._datasets
images_test, labels_test = mnist_test._datasets
# normalize
images_train = (images_train - 0.5) * 2
images_test = (images_test - 0.5) * 2
dataset = Dataset(train=(images_train, labels_train),
test=(images_test, labels_test),
num_labeled_data=args.num_labeled_data,
num_classes=model.ndim_y)
print("#labeled: {}".format(dataset.get_num_labeled_data()))
print("#unlabeled: {}".format(dataset.get_num_unlabeled_data()))
_, labels = dataset.get_labeled_data()
print("labeled data:", labels)
total_iterations_train = len(images_train) // args.batchsize
# optimizers
optimizer_encoder = Optimizer("msgd", 0.01, 0.9)
optimizer_encoder.setup(model.encoder)
if args.grad_clip > 0:
optimizer_encoder.add_hook(GradientClipping(args.grad_clip))
optimizer_semi_supervised = Optimizer("msgd", 0.1, 0.9)
optimizer_semi_supervised.setup(model.encoder)
if args.grad_clip > 0:
optimizer_semi_supervised.add_hook(GradientClipping(args.grad_clip))
optimizer_generator = Optimizer("msgd", 0.1, 0.1)
optimizer_generator.setup(model.encoder)
if args.grad_clip > 0:
optimizer_generator.add_hook(GradientClipping(args.grad_clip))
optimizer_decoder = Optimizer("msgd", 0.01, 0.9)
optimizer_decoder.setup(model.decoder)
if args.grad_clip > 0:
optimizer_decoder.add_hook(GradientClipping(args.grad_clip))
optimizer_discriminator_z = Optimizer("msgd", 0.1, 0.1)
optimizer_discriminator_z.setup(model.discriminator_z)
if args.grad_clip > 0:
optimizer_discriminator_z.add_hook(GradientClipping(args.grad_clip))
optimizer_discriminator_y = Optimizer("msgd", 0.1, 0.1)
optimizer_discriminator_y.setup(model.discriminator_y)
if args.grad_clip > 0:
optimizer_discriminator_y.add_hook(GradientClipping(args.grad_clip))
optimizer_cluster_head = Optimizer("msgd", 0.01, 0.9)
optimizer_cluster_head.setup(model.cluster_head)
if args.grad_clip > 0:
optimizer_cluster_head.add_hook(GradientClipping(args.grad_clip))
using_gpu = False
if args.gpu_device >= 0:
cuda.get_device(args.gpu_device).use()
model.to_gpu()
using_gpu = True
xp = model.xp
# 0 -> true sample
# 1 -> generated sample
class_true = np.zeros(args.batchsize, dtype=np.int32)
class_fake = np.ones(args.batchsize, dtype=np.int32)
if using_gpu:
class_true = cuda.to_gpu(class_true)
class_fake = cuda.to_gpu(class_fake)
training_start_time = time.time()
for epoch in range(args.total_epochs):
sum_loss_generator = 0
sum_loss_discriminator = 0
sum_loss_autoencoder = 0
sum_loss_supervised = 0
sum_loss_cluster_head = 0
sum_discriminator_z_confidence_true = 0
sum_discriminator_z_confidence_fake = 0
sum_discriminator_y_confidence_true = 0
sum_discriminator_y_confidence_fake = 0
epoch_start_time = time.time()
dataset.shuffle()
# training
for itr in range(total_iterations_train):
# update model parameters
with chainer.using_config("train", True):
# sample minibatch
x_u = dataset.sample_unlabeled_minibatch(args.batchsize,
gpu=using_gpu)
x_l, y_l, _ = dataset.sample_labeled_minibatch(args.batchsize,
gpu=using_gpu)
### reconstruction phase ###
if True:
y_onehot_u, z_u = model.encode_x_yz(x_u,
apply_softmax_y=True)
repr_u = model.encode_yz_representation(y_onehot_u, z_u)
x_reconstruction_u = model.decode_representation_x(repr_u)
loss_reconstruction_u = F.mean_squared_error(
x_u, x_reconstruction_u)
y_onehot_l, z_l = model.encode_x_yz(x_l,
apply_softmax_y=True)
repr_l = model.encode_yz_representation(y_onehot_l, z_l)
x_reconstruction_l = model.decode_representation_x(repr_l)
loss_reconstruction_l = F.mean_squared_error(
x_l, x_reconstruction_l)
loss_reconstruction = loss_reconstruction_u + loss_reconstruction_l
model.cleargrads()
loss_reconstruction.backward()
optimizer_encoder.update()
# optimizer_cluster_head.update()
optimizer_decoder.update()
### adversarial phase ###
if True:
y_onehot_fake_u, z_fake_u = model.encode_x_yz(
x_u, apply_softmax_y=True)
z_true = sampler.gaussian(args.batchsize,
model.ndim_z,
mean=0,
var=1)
y_onehot_true = sampler.onehot_categorical(
args.batchsize, model.ndim_y)
if using_gpu:
z_true = cuda.to_gpu(z_true)
y_onehot_true = cuda.to_gpu(y_onehot_true)
dz_true = model.discriminate_z(z_true, apply_softmax=False)
dz_fake = model.discriminate_z(z_fake_u,
apply_softmax=False)
dy_true = model.discriminate_y(y_onehot_true,
apply_softmax=False)
dy_fake = model.discriminate_y(y_onehot_fake_u,
apply_softmax=False)
discriminator_z_confidence_true = float(
xp.mean(F.softmax(dz_true).data[:, 0]))
discriminator_z_confidence_fake = float(
xp.mean(F.softmax(dz_fake).data[:, 1]))
discriminator_y_confidence_true = float(
xp.mean(F.softmax(dy_true).data[:, 0]))
discriminator_y_confidence_fake = float(
xp.mean(F.softmax(dy_fake).data[:, 1]))
loss_discriminator_z = F.softmax_cross_entropy(
dz_true, class_true) + F.softmax_cross_entropy(
dz_fake, class_fake)
loss_discriminator_y = F.softmax_cross_entropy(
dy_true, class_true) + F.softmax_cross_entropy(
dy_fake, class_fake)
loss_discriminator = loss_discriminator_z + loss_discriminator_y
model.cleargrads()
loss_discriminator.backward()
optimizer_discriminator_z.update()
optimizer_discriminator_y.update()
### generator phase ###
if True:
y_onehot_fake_u, z_fake_u = model.encode_x_yz(
x_u, apply_softmax_y=True)
dz_fake = model.discriminate_z(z_fake_u,
apply_softmax=False)
dy_fake = model.discriminate_y(y_onehot_fake_u,
apply_softmax=False)
loss_generator = F.softmax_cross_entropy(
dz_fake, class_true) + F.softmax_cross_entropy(
dy_fake, class_true)
model.cleargrads()
loss_generator.backward()
optimizer_generator.update()
### supervised phase ###
if True:
logit_l, _ = model.encode_x_yz(x_l, apply_softmax_y=False)
loss_supervised = F.softmax_cross_entropy(logit_l, y_l)
model.cleargrads()
loss_supervised.backward()
optimizer_semi_supervised.update()
### additional cost ###
if True:
distance = model.compute_distance_of_cluster_heads()
loss_cluster_head = -F.sum(distance)
model.cleargrads()
loss_cluster_head.backward()
optimizer_cluster_head.update()
sum_loss_discriminator += float(loss_discriminator.data)
sum_loss_supervised += float(loss_supervised.data)
sum_loss_generator += float(loss_generator.data)
sum_loss_autoencoder += float(loss_reconstruction.data)
sum_loss_cluster_head += float(
model.nCr(model.ndim_y, 2) *
model.cluster_head_distance_threshold +
loss_cluster_head.data)
sum_discriminator_z_confidence_true += discriminator_z_confidence_true
sum_discriminator_z_confidence_fake += discriminator_z_confidence_fake
sum_discriminator_y_confidence_true += discriminator_y_confidence_true
sum_discriminator_y_confidence_fake += discriminator_y_confidence_fake
printr("Training ... {:3.0f}% ({}/{})".format(
(itr + 1) / total_iterations_train * 100, itr + 1,
total_iterations_train))
model.save(args.model)
labeled_iter_train = dataset.get_iterator(args.batchsize * 20,
train=True,
labeled=True,
gpu=using_gpu)
unlabeled_iter_train = dataset.get_iterator(args.batchsize * 20,
train=True,
unlabeled=True,
gpu=using_gpu)
average_accuracy_l = 0
average_accuracy_u = 0
for x_l, true_label in labeled_iter_train:
with chainer.no_backprop_mode() and chainer.using_config(
"train", False):
y_onehot_l, _ = model.encode_x_yz(x_l, apply_softmax_y=True)
accuracy = F.accuracy(y_onehot_l, true_label)
average_accuracy_l += float(accuracy.data)
for x_u, true_label in unlabeled_iter_train:
with chainer.no_backprop_mode() and chainer.using_config(
"train", False):
y_onehot_u, _ = model.encode_x_yz(x_u, apply_softmax_y=True)
accuracy = F.accuracy(y_onehot_u, true_label)
average_accuracy_u += float(accuracy.data)
average_accuracy_l /= labeled_iter_train.get_total_iterations()
average_accuracy_u /= unlabeled_iter_train.get_total_iterations()
clear_console()
print(
"Epoch {} done in {} sec - loss: g={:.5g}, d={:.5g}, a={:.5g}, s={:.5g}, c={:.5g} - disc_z: true={:.1f}%, fake={:.1f}% - disc_y: true={:.1f}%, fake={:.1f}% - acc: l={:.2f}%, u={:.2f}% - total {} min"
.format(
epoch + 1, int(time.time() - epoch_start_time),
sum_loss_generator / total_iterations_train,
sum_loss_discriminator / total_iterations_train,
sum_loss_autoencoder / total_iterations_train,
sum_loss_supervised / total_iterations_train,
sum_loss_cluster_head / total_iterations_train,
sum_discriminator_z_confidence_true / total_iterations_train *
100, sum_discriminator_z_confidence_fake /
total_iterations_train * 100,
sum_discriminator_y_confidence_true / total_iterations_train *
100, sum_discriminator_y_confidence_fake /
total_iterations_train * 100, average_accuracy_l * 100,
average_accuracy_u * 100,
int((time.time() - training_start_time) // 60)))
if epoch == 50:
optimizer_encoder.set_learning_rate(0.001)
optimizer_decoder.set_learning_rate(0.001)
optimizer_semi_supervised.set_learning_rate(0.01)
optimizer_generator.set_learning_rate(0.01)
optimizer_discriminator_y.set_learning_rate(0.01)
optimizer_discriminator_z.set_learning_rate(0.01)
if epoch == 1000:
optimizer_encoder.set_learning_rate(0.0001)
optimizer_decoder.set_learning_rate(0.0001)
optimizer_semi_supervised.set_learning_rate(0.001)
optimizer_generator.set_learning_rate(0.001)
optimizer_discriminator_y.set_learning_rate(0.001)
optimizer_discriminator_z.set_learning_rate(0.001)
def _update(self, dataset):
# override func
xp = self.xp
if self.obs_normalizer:
self._update_obs_normalizer(dataset)
dataset_iter = chainer.iterators.SerialIterator(dataset,
self.minibatch_size,
shuffle=True)
loss_mean = 0
while dataset_iter.epoch < self.epochs:
# create batch for this iter
batch = dataset_iter.__next__()
states = self.batch_states([b['state'] for b in batch], xp,
self.phi)
next_states = self.batch_states([b['next_state'] for b in batch],
xp, self.phi)
actions = xp.array([b['action'] for b in batch])
# create batch of expert data for this iter
demonstrations_indexes = np.random.permutation(
len(self.demo_states))[:self.minibatch_size]
# Get random expert state-action pair
demo_states, demo_actions, demo_next_states = [
d[demonstrations_indexes]
for d in (self.demo_states, self.demo_actions,
self.demo_next_states)
]
# Normalize if chosen
states, demo_states, next_states, demo_next_states = [
(self.obs_normalizer(d, update=False)
if self.obs_normalizer else d)
for d in [states, demo_states, next_states, demo_next_states]
]
# Get the probabilities for actions for expert and agent from policy net, i.e. self.model
with chainer.configuration.using_config(
'train', False), chainer.no_backprop_mode():
action_log_probs = self.get_probs(states, actions).data
demo_action_log_probs = self.get_probs(demo_states,
demo_actions).data
# Merge together expert's and agent's states, actions and probabilites and create a target array with ones and zeros
batch_states = np.concatenate((states, demo_states))
batch_next_states = np.concatenate((next_states, demo_next_states))
batch_actions = np.concatenate((actions, demo_actions))
batch_probs = np.concatenate(
(action_log_probs, demo_action_log_probs))
targets = np.ones((len(actions) + len(demo_actions), 1))
targets[:len(
actions
)] = 0 # the target for fake data is 0 and 1 for expert (true) data
loss = self.discriminator.train(states=batch_states,
actions=batch_actions,
action_logprobs=batch_probs,
next_states=batch_next_states,
targets=targets,
xp=self.xp)
loss_mean += loss / (self.epochs * self.minibatch_size)
self.discriminator_loss_record.append(float(loss_mean.array))
super()._update(dataset)
def predict(self, atoms, adjs):
with chainer.no_backprop_mode(), chainer.using_config('train', False):
x = self.__call__(atoms, adjs)
return F.sigmoid(x)
def run(self):
x = chainer.Variable(numpy.array([1], dtype='f'))
with chainer.no_backprop_mode():
y = x + 1
self.creator_is_none = y.creator_node is None
def main():
parser = argparse.ArgumentParser(description='CapsNet: MNIST')
parser.add_argument('--batchsize', '-b', type=int, default=256)
parser.add_argument('--decay', '-d', type=float, default=0.95)
parser.add_argument('--epoch', '-e', type=int, default=500)
parser.add_argument('--gpu', '-g', type=int, default=-1)
parser.add_argument('--seed', '-s', type=int, default=789)
parser.add_argument('--reconstruct', '--recon', action='store_true')
parser.add_argument('--save')
args = parser.parse_args()
print(json.dumps(args.__dict__, indent=2))
# Set up a neural network to train
np.random.seed(args.seed)
model = nets.CapsNet(use_reconstruction=args.reconstruct)
if args.gpu >= 0:
# Make a speciied GPU current
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu() # Copy the model to the GPU
np.random.seed(args.seed)
model.xp.random.seed(args.seed)
# Setup an optimizer
optimizer = chainer.optimizers.Adam(alpha=1e-3)
optimizer.setup(model)
# Load the MNIST dataset
train, test = get_recaptcha()
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test,
10,
repeat=False,
shuffle=False)
def report(epoch, result):
mode = 'train' if chainer.config.train else 'test '
print('epoch {:2d}\t{} mean loss: {}, accuracy: {}'.format(
train_iter.epoch, mode, result['mean_loss'], result['accuracy']))
if args.reconstruct:
print('\t\t\tclassification: {}, reconstruction: {}'.format(
result['cls_loss'], result['rcn_loss']))
best = 0.
best_epoch = 0
print('TRAINING starts')
while train_iter.epoch < args.epoch:
batch = train_iter.next()
x, t = concat_examples(batch, args.gpu)
optimizer.update(model, x, t)
# evaluation
if train_iter.is_new_epoch:
result = model.pop_results()
report(train_iter.epoch, result)
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
for batch in test_iter:
x, t = concat_examples(batch, args.gpu)
loss = model(x, t)
result = model.pop_results()
report(train_iter.epoch, result)
if result['accuracy'] > best:
best, best_epoch = result['accuracy'], train_iter.epoch
serializers.save_npz(args.save, model)
optimizer.alpha *= args.decay
optimizer.alpha = max(optimizer.alpha, 1e-5)
print('\t\t# optimizer alpha', optimizer.alpha)
test_iter.reset()
print('Finish: Best accuray: {} at {} epoch'.format(best, best_epoch))
def main():
parser = argparse.ArgumentParser(
description='chainer example of img2img learning')
parser.add_argument('--batchsize',
'-b',
type=int,
default=1,
help='Number of images in each mini-batch')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out',
'-o',
default='test',
help='Directory to output the test result')
parser.add_argument('--model',
'-m',
default='result/model_final',
help='model to use')
parser.add_argument('--filelist',
'-fl',
default='filelist.dat',
help='filelist of dataset')
args = parser.parse_args()
print('GPU: {}'.format(args.gpu))
print('# batch-size: {}'.format(args.batchsize))
print('# model: {}'.format(args.model))
print('')
if args.gpu >= 0:
chainer.cuda.get_device(args.gpu).use() # Make a specified GPU current
with chainer.no_backprop_mode():
with chainer.using_config('train', False):
cnn = net.AutoENC()
serializers.load_npz(args.model, cnn)
if args.gpu >= 0:
cnn.to_gpu() # Copy the model to the GPU
dataset = Image2ImageDataset(args.filelist)
src, dst = dataset.get_example(0)
x_in = np.zeros(
(args.batchsize, src.shape[0], src.shape[1], src.shape[2])).astype('f')
#x_out = np.zeros((batchsize, dst.shape[1], dst.shape[1], dst.shape[2])).astype('f')
for j in range(args.batchsize):
src, dst = dataset.get_example(j)
x_in[j, :] = src
#x_out[j,:] = dst
if args.gpu >= 0:
x_in = cuda.to_gpu(x_in)
x_out = cnn(x_in)
output = x_out.data.get()
for i in range(args.batchsize):
img = dataset.post_proc(output[i])
cv2.imwrite(args.out + "/" + dataset.get_name(i), img)
def main():
try:
os.makedirs(args.figure_directory)
except:
pass
xp = np
using_gpu = args.gpu_device >= 0
if using_gpu:
cuda.get_device(args.gpu_device).use()
xp = cp
dataset = gqn.data.Dataset(args.dataset_directory)
meter = Meter()
assert meter.load(args.snapshot_directory)
hyperparams = HyperParameters()
assert hyperparams.load(args.snapshot_directory)
model = Model(hyperparams)
assert model.load(args.snapshot_directory, meter.epoch)
if using_gpu:
model.to_gpu()
total_observations_per_scene = 4
fps = 30
black_color = -0.5
image_shape = (3, ) + hyperparams.image_size
axis_observations_image = np.zeros(
(3, image_shape[1], total_observations_per_scene * image_shape[2]),
dtype=np.float32)
#==============================================================================
# Utilities
#==============================================================================
def to_device(array):
if using_gpu:
array = cuda.to_gpu(array)
return array
def fill_observations_axis(observation_images):
axis_observations_image = np.full(
(3, image_shape[1], total_observations_per_scene * image_shape[2]),
black_color,
dtype=np.float32)
num_current_obs = len(observation_images)
total_obs = total_observations_per_scene
width = image_shape[2]
x_start = width * (total_obs - num_current_obs) // 2
for obs_image in observation_images:
x_end = x_start + width
axis_observations_image[:, :, x_start:x_end] = obs_image
x_start += width
return axis_observations_image
def compute_camera_angle_at_frame(t):
horizontal_angle_rad = 2 * t * math.pi / (fps * 2) + math.pi / 4
y_rad_top = math.pi / 3
y_rad_bottom = -math.pi / 3
y_rad_range = y_rad_bottom - y_rad_top
if t < fps * 1.5:
vertical_angle_rad = y_rad_top
elif fps * 1.5 <= t and t < fps * 2.5:
interp = (t - fps * 1.5) / fps
vertical_angle_rad = y_rad_top + interp * y_rad_range
elif fps * 2.5 <= t and t < fps * 4:
vertical_angle_rad = y_rad_bottom
elif fps * 4.0 <= t and t < fps * 5:
interp = (t - fps * 4.0) / fps
vertical_angle_rad = y_rad_bottom - interp * y_rad_range
else:
vertical_angle_rad = y_rad_top
return horizontal_angle_rad, vertical_angle_rad
def rotate_query_viewpoint(horizontal_angle_rad, vertical_angle_rad):
camera_direction = np.array([
math.sin(horizontal_angle_rad), # x
math.sin(vertical_angle_rad), # y
math.cos(horizontal_angle_rad), # z
])
camera_direction = args.camera_distance * camera_direction / np.linalg.norm(
camera_direction)
yaw, pitch = compute_yaw_and_pitch(camera_direction)
query_viewpoints = xp.array(
(
camera_direction[0],
camera_direction[1],
camera_direction[2],
math.cos(yaw),
math.sin(yaw),
math.cos(pitch),
math.sin(pitch),
),
dtype=np.float32,
)
query_viewpoints = xp.broadcast_to(query_viewpoints,
(1, ) + query_viewpoints.shape)
return query_viewpoints
#==============================================================================
# Visualization
#==============================================================================
plt.style.use("dark_background")
fig = plt.figure(figsize=(6, 7))
plt.subplots_adjust(left=0.1, right=0.95, bottom=0.1, top=0.95)
# fig.suptitle("GQN")
axis_observations = fig.add_subplot(2, 1, 1)
axis_observations.axis("off")
axis_observations.set_title("observations")
axis_generation = fig.add_subplot(2, 1, 2)
axis_generation.axis("off")
axis_generation.set_title("neural rendering")
#==============================================================================
# Generating animation
#==============================================================================
file_number = 1
random.seed(0)
np.random.seed(0)
with chainer.no_backprop_mode():
for subset in dataset:
iterator = gqn.data.Iterator(subset, batch_size=1)
for data_indices in iterator:
animation_frame_array = []
# shape: (batch, views, height, width, channels)
images, viewpoints = subset[data_indices]
# (batch, views, height, width, channels) -> (batch, views, channels, height, width)
images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32)
images = preprocess_images(images)
batch_index = 0
total_views = images.shape[1]
random_observation_view_indices = list(range(total_views))
random.shuffle(random_observation_view_indices)
random_observation_view_indices = random_observation_view_indices[:
total_observations_per_scene]
#------------------------------------------------------------------------------
# Observations
#------------------------------------------------------------------------------
observed_images = images[batch_index,
random_observation_view_indices]
observed_viewpoints = viewpoints[
batch_index, random_observation_view_indices]
observed_images = to_device(observed_images)
observed_viewpoints = to_device(observed_viewpoints)
#------------------------------------------------------------------------------
# Generate images with a single observation
#------------------------------------------------------------------------------
# Scene encoder
representation = model.compute_observation_representation(
observed_images[None, :1], observed_viewpoints[None, :1])
# Update figure
observation_index = random_observation_view_indices[0]
observed_image = images[batch_index, observation_index]
axis_observations_image = fill_observations_axis(
[observed_image])
# Rotate camera
for t in range(fps, fps * 6):
artist_array = [
axis_observations.imshow(
make_uint8(axis_observations_image),
interpolation="none",
animated=True)
]
horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame(
t)
query_viewpoints = rotate_query_viewpoint(
horizontal_angle_rad, vertical_angle_rad)
generated_images = model.generate_image(
query_viewpoints, representation)[0]
artist_array.append(
axis_generation.imshow(
make_uint8(generated_images),
interpolation="none",
animated=True))
animation_frame_array.append(artist_array)
#------------------------------------------------------------------------------
# Add observations
#------------------------------------------------------------------------------
for n in range(1, total_observations_per_scene):
observation_indices = random_observation_view_indices[:n +
1]
axis_observations_image = fill_observations_axis(
images[batch_index, observation_indices])
# Scene encoder
representation = model.compute_observation_representation(
observed_images[None, :n + 1],
observed_viewpoints[None, :n + 1])
for t in range(fps // 2):
artist_array = [
axis_observations.imshow(
make_uint8(axis_observations_image),
interpolation="none",
animated=True)
]
horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame(
0)
query_viewpoints = rotate_query_viewpoint(
horizontal_angle_rad, vertical_angle_rad)
generated_images = model.generate_image(
query_viewpoints, representation)[0]
artist_array.append(
axis_generation.imshow(
make_uint8(generated_images),
interpolation="none",
animated=True))
animation_frame_array.append(artist_array)
#------------------------------------------------------------------------------
# Generate images with all observations
#------------------------------------------------------------------------------
# Scene encoder
representation = model.compute_observation_representation(
observed_images[None, :total_observations_per_scene + 1],
observed_viewpoints[None, :total_observations_per_scene +
1])
# Rotate camera
for t in range(0, fps * 6):
artist_array = [
axis_observations.imshow(
make_uint8(axis_observations_image),
interpolation="none",
animated=True)
]
horizontal_angle_rad, vertical_angle_rad = compute_camera_angle_at_frame(
t)
query_viewpoints = rotate_query_viewpoint(
horizontal_angle_rad, vertical_angle_rad)
generated_images = model.generate_image(
query_viewpoints, representation)[0]
artist_array.append(
axis_generation.imshow(
make_uint8(generated_images),
interpolation="none",
animated=True))
animation_frame_array.append(artist_array)
#------------------------------------------------------------------------------
# Write to file
#------------------------------------------------------------------------------
anim = animation.ArtistAnimation(
fig,
animation_frame_array,
interval=1 / fps,
blit=True,
repeat_delay=0)
# anim.save(
# "{}/shepard_metzler_observations_{}.gif".format(
# args.figure_directory, file_number),
# writer="imagemagick",
# fps=fps)
anim.save(
"{}/shepard_metzler_observations_{}.mp4".format(
args.figure_directory, file_number),
writer="ffmpeg",
fps=fps)
file_number += 1
def main():
layer_indexes = [int(x) for x in FLAGS.layers.split(",")]
bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
tokenizer = tokenization.FullTokenizer(
vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)
bert = modeling.BertModel(config=bert_config)
model = modeling.BertExtracter(bert)
ignore_names = ['output/W', 'output/b']
chainer.serializers.load_npz(
FLAGS.init_checkpoint, model,
ignore_names=ignore_names)
if FLAGS.gpu >= 0:
chainer.backends.cuda.get_device_from_id(FLAGS.gpu).use()
model.to_gpu()
examples = read_examples(input_file=FLAGS.input_file)
features = convert_examples_to_features(
examples, FLAGS.max_seq_length, tokenizer)
iterator = chainer.iterators.SerialIterator(
features, FLAGS.batch_size,
repeat=False, shuffle=False)
# converter = converter(is_training=False)
unique_id_to_feature = {}
for feature in features:
unique_id_to_feature[feature.unique_id] = feature
# with codecs.getwriter("utf-8")(open(FLAGS.output_file, "w")) as writer:
with open(FLAGS.output_file, "w") as writer:
for prebatch in iterator:
unique_ids = [f.unique_id for f in prebatch]
batch = make_batch(prebatch, FLAGS.gpu)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
encoder_layers = model.get_all_encoder_layers(
input_ids=batch['input_ids'],
input_mask=batch['input_mask'],
token_type_ids=batch['input_type_ids'])
encoder_layers = [layer.array.tolist() for layer in encoder_layers]
for b in range(len(prebatch)): # batchsize loop
unique_id = unique_ids[b]
feature = unique_id_to_feature[unique_id]
output_json = collections.OrderedDict()
output_json["linex_index"] = unique_id
all_features = []
for (i, token) in enumerate(feature.tokens):
all_layers = []
for (j, layer_index) in enumerate(layer_indexes):
layer_output = encoder_layers[layer_index][b]
layers = collections.OrderedDict()
layers["index"] = layer_index
layers["values"] = [
round(float(x), 6) for x in layer_output[b]
]
all_layers.append(layers)
features = collections.OrderedDict()
features["token"] = token
features["layers"] = all_layers
all_features.append(features)
output_json["features"] = all_features
writer.write(json.dumps(output_json) + "\n")
def main():
xp = np
using_gpu = args.gpu_device >= 0
if using_gpu:
cuda.get_device(args.gpu_device).use()
xp = cupy
hyperparams = Hyperparameters(args.snapshot_path)
hyperparams.print()
num_bins_x = 2.0**hyperparams.num_bits_x
num_pixels = 3 * hyperparams.image_size[0] * hyperparams.image_size[1]
# Get Dataset:
if True:
assert args.dataset_format in ["png", "npy"]
files = Path(args.dataset_path).glob("*.{}".format(
args.dataset_format))
if args.dataset_format == "png":
images = []
for filepath in files:
image = np.array(Image.open(filepath)).astype("float32")
image = preprocess(image, hyperparams.num_bits_x)
images.append(image)
assert len(images) > 0
images = np.asanyarray(images)
elif args.dataset_format == "npy":
images = []
for filepath in files:
array = np.load(filepath).astype("float32")
array = preprocess(array, hyperparams.num_bits_x)
images.append(array)
assert len(images) > 0
num_files = len(images)
images = np.asanyarray(images)
images = images.reshape((num_files * images.shape[1], ) +
images.shape[2:])
else:
raise NotImplementedError
dataset = glow.dataset.Dataset(images)
iterator = glow.dataset.Iterator(dataset, batch_size=1)
print(tabulate([["#image", len(dataset)]]))
encoder = Glow(hyperparams, hdf5_path=args.snapshot_path)
if using_gpu:
encoder.to_gpu()
ori_x = []
enc_z = []
rev_x = []
fw_logdet = []
logpZ = []
logpZ2 = []
i = 0
with chainer.no_backprop_mode() and encoder.reverse() as decoder:
for data_indices in iterator:
i += 1
x = to_gpu(dataset[data_indices]) # 1x3x64x64
x += xp.random.uniform(0, 1.0 / num_bins_x, size=x.shape)
x_img = make_uint8(x[0], num_bins_x)
ori_x.append(x_img)
factorized_z_distribution, fw_ldt = encoder.forward_step(x)
fw_ldt -= math.log(num_bins_x) * num_pixels
fw_logdet.append(cupy.asnumpy(fw_ldt.data))
factor_z = []
ez = []
nll = 0
for (zi, mean, ln_var) in factorized_z_distribution:
nll += cf.gaussian_nll(zi, mean, ln_var)
factor_z.append(zi.data)
ez.append(zi.data.reshape(-1, ))
ez = np.concatenate(ez)
enc_z.append(ez.get())
logpZ.append(cupy.asnumpy(nll.data))
logpZ2.append(
cupy.asnumpy(
cf.gaussian_nll(ez, np.mean(ez), np.log(np.var(ez))).data))
rx, _ = decoder.reverse_step(factor_z)
rx_img = make_uint8(rx.data[0], num_bins_x)
rev_x.append(rx_img)
if i % 100 == 0:
np.save(str(i) + '/ori_x.npy', ori_x)
fw_logdet = np.array(fw_logdet)
np.save(str(i) + '/fw_logdet.npy', fw_logdet)
np.save(str(i) + '/enc_z.npy', enc_z)
logpZ = np.array(logpZ)
np.save(str(i) + '/logpZ.npy', logpZ)
logpZ2 = np.array(logpZ2)
np.save(str(i) + '/logpZ2.npy', logpZ2)
np.save(str(i) + '/rev_x.npy', rev_x)
ori_x = []
enc_z = []
rev_x = []
fw_logdet = []
logpZ = []
logpZ2 = []
return
def main():
with open(args.list) as f:
trained_files = f.readlines()
trained_files = [x.split('\n')[0] for x in trained_files]
trained_list = []
trained_align = []
for x in trained_files:
_file, _align = x.split('\t')
trained_list += [_file]
trained_align += [_align]
if args.untrained == 1:
with open(args.audio_list) as f:
audio_list = f.readlines()
audio_list = [x.split('\n')[0] for x in audio_list]
untrain_list = []
for i in six.moves.range(len(audio_list)):
audioname = os.path.basename(audio_list[i]).split('.')[0]
istrained = False
for j in trained_list:
if audioname in j:
istrained = True
break
if not istrained:
untrain_list.append(audio_list[i])
with h5py.File(args.step_file) as F:
motion_beat_idx = np.array(F['location'], copy=True)
dance_step = np.array(F['dance_step'], copy=True)
logging.info('Evaluation training...')
logging.info('Loading model definition from {}'.format(args.network))
logging.info('Using gpu {}'.format(args.gpu))
net = imp.load_source('Network', args.network)
audionet = imp.load_source('Network', os.path.join('models', 'audio_nets.py'))
model = net.Dancer(args.initOpt, getattr(audionet, args.encoder))
serializers.load_hdf5(args.model, model)
logging.info('Loading pretrained model from {}'.format(args.model))
if args.gpu >= 0:
model.to_gpu()
minmaxfile = os.path.join('exp', 'data', '{}_{}'.format(args.exp, args.rot), 'minmax', 'pos_minmax.h5')
with h5py.File(minmaxfile, 'r') as f:
pos_min = f['minmax'][0:1, :]
pos_max = f['minmax'][1:2, :]
config['out_folder'] = os.path.join(args.folder, 'evaluation')
div = (pos_max - pos_min)
div[div == 0] = 1
config['slope_pos'] = (config['rng_pos'][1] - config['rng_pos'][0]) / div
config['intersec_pos'] = config['rng_pos'][1] - config['slope_pos'] * pos_max
if not os.path.exists(config['out_folder']):
os.makedirs(config['out_folder'])
for i in six.moves.range(2):
if i == 0:
stage = 'trained'
filelist = trained_list
else:
if args.untrained == 0:
logging.info('Only trained data. Exit')
exit()
stage = 'untrained'
filelist = untrain_list
logging.info('Evaluating model for {} audio files'.format(stage))
results = dict()
results_keys = ['filename', 'noise', 'snr', 'fscore', 'precission', 'recall', 'acc',
'forward_time', 'init_beat', 'entropy_beat', 'entropy_step']
for key in results_keys:
results[key] = list()
for j in six.moves.range(len(filelist)):
if i == 0:
mbfile = filelist[j].replace(os.path.join('MOCAP', 'HTR'), os.path.join('Annotations', 'corrected'))
mbfile = mbfile.replace('{}_'.format(args.exp), '')
mbfile = mbfile.replace('.htr', '.txt')
audiofile = filelist[j].replace(os.path.join('MOCAP', 'HTR'), os.path.join('AUDIO', 'MP3'))
audiofile = audiofile.replace('{}_'.format(args.exp), '')
audiofile = audiofile.replace('.htr', '.mp3')
else:
mbfile = filelist[j].replace(os.path.join('AUDIO', 'MP3'), os.path.join('Annotations', 'corrected'))
mbfile = mbfile.replace('.mp3', '.txt')
audiofile = filelist[j]
music_beat = np.unique(np.loadtxt(mbfile))
filename = os.path.basename(mbfile).split('.')[0]
for noise in list_noises:
if noise == 'Clean':
list_snr = [None]
else:
list_snr = snr_lst
noise_mp3 = noise.replace(os.path.join('WAVE', ''), '')
noise_mp3 = noise_mp3.replace('.wav', '.mp3')
noise_name = noise
if os.path.exists(noise_mp3):
noise_name = os.path.basename(noise_mp3).split('.')[0]
if not os.path.exists(noise):
logging.warning('Wavefile not found in folder, converting from mp3 file.')
noise_dir = os.path.dirname(noise)
if not os.path.exists(noise_dir):
os.makedirs(noise_dir)
if platform == 'Windows':
cmmd = 'ffmpeg -loglevel panic -y -i {} -acodec pcm_s16le -ar {} -ac 1 {}'.format(
noise_mp3, args.freq, noise)
subprocess.Popen(cmmd, shell=False).communicate()
elif platform == 'Linux':
os.system('sox {} -c 1 -r {} {}'.format(noise_mp3, args.freq, noise))
else:
raise TypeError('OS not supported')
for snr in list_snr:
logging.info('Forwarding file: {} with noise: {} at snr: {}'.format(audiofile, noise, snr))
audio = format_audio(audiofile, noise, snr, args.freq, config['rng_wav'])
predicted_motion = np.zeros((audio.shape[0], args.initOpt[2]), dtype=np.float32)
feats = np.zeros((audio.shape[0] - 1, args.initOpt[1]), dtype=np.float32)
start = timeit.default_timer()
state = model.state
current_step = Variable(xp.asarray(np.zeros((1, args.initOpt[2]), dtype=np.float32)))
with chainer.no_backprop_mode(), chainer.using_config('train', False):
for k in six.moves.range(audio.shape[0] - 1):
audiofeat = model.audiofeat(Variable(xp.asarray(audio[k:k + 1])))
rnnAudio, state, out_step = model.forward(state, current_step, audiofeat, True)
if args.gpu >= 0:
predicted_motion[k + 1] = chainer.cuda.to_cpu(out_step.data)
feats[k] = chainer.cuda.to_cpu(rnnAudio.data)
else:
predicted_motion[k + 1] = out_step.data
feats[k] = rnnAudio.data
current_step = out_step
del audio
time = timeit.default_timer() - start
predicted_motion = (predicted_motion - config['intersec_pos']) / config['slope_pos']
predicted_motion = render_motion(predicted_motion, args.rot, scale=args.scale)
logging.info('Forwarding time was: {:.02f}s'.format(time))
# Evaluations
fscore, prec, recall, acc, best_init_beat, best_beat_entropy, best_step_entropy = metrics(
predicted_motion, music_beat, motion_beat_idx, dance_step)
results['filename'].append(filename) # TODO(nelson): Separation by genre
results['noise'].append(noise_name)
results['snr'].append(str(snr))
results['fscore'].append(fscore)
results['precission'].append(prec)
results['recall'].append(recall)
results['acc'].append(acc)
results['forward_time'].append(time)
results['init_beat'].append(best_init_beat)
results['entropy_beat'].append(best_beat_entropy)
results['entropy_step'].append(best_step_entropy)
# Save extra data
save_file = os.path.join(args.folder, 'evaluation', '{}_{}_{}_{}_snr{}_ep{}_output.h5'.format(
args.stage, stage, filename, noise_name, str(snr), args.epoch))
if os.path.exists(save_file):
os.remove(save_file)
with h5py.File(save_file, 'w') as f:
f.create_dataset('motion', data=predicted_motion)
f.create_dataset('audiofeats', data=feats)
del feats
del predicted_motion
# Save evaluations results
df = pandas.DataFrame(results, columns=results_keys)
result_file = os.path.join(args.folder, 'results',
'result_{}_{}_ep{}.csv'.format(args.stage, stage, args.epoch))
df.to_csv(result_file, encoding='utf-8')
return
def __generateAction(self, _q, state):
"""Get a greedy action wrt a given Q-function."""
state = _q.xp.asarray(state[None], dtype=np.float32)
with chainer.no_backprop_mode():
_qValue = _q(state).data[0]
return int(_qValue.argmax())
def recog(args):
"""Decode with the given args
:param Namespace args: The program arguments
"""
# display chainer version
logging.info('chainer version = ' + chainer.__version__)
set_deterministic_chainer(args)
# read training config
idim, odim, train_args = get_model_conf(args.model, args.model_conf)
for key in sorted(vars(args).keys()):
logging.info('ARGS: ' + key + ': ' + str(vars(args)[key]))
# specify model architecture
logging.info('reading model parameters from ' + args.model)
# To be compatible with v.0.3.0 models
if hasattr(train_args, "model_module"):
model_module = train_args.model_module
else:
model_module = "espnet.nets.chainer_backend.e2e_asr:E2E"
model_class = dynamic_import(model_module)
model = model_class(idim, odim, train_args)
assert isinstance(model, ASRInterface)
chainer_load(args.model, model)
# read rnnlm
if args.rnnlm:
rnnlm_args = get_model_conf(args.rnnlm, args.rnnlm_conf)
rnnlm = lm_chainer.ClassifierWithState(
lm_chainer.RNNLM(len(train_args.char_list), rnnlm_args.layer,
rnnlm_args.unit))
chainer_load(args.rnnlm, rnnlm)
else:
rnnlm = None
if args.word_rnnlm:
rnnlm_args = get_model_conf(args.word_rnnlm, args.word_rnnlm_conf)
word_dict = rnnlm_args.char_list_dict
char_dict = {x: i for i, x in enumerate(train_args.char_list)}
word_rnnlm = lm_chainer.ClassifierWithState(
lm_chainer.RNNLM(len(word_dict), rnnlm_args.layer,
rnnlm_args.unit))
chainer_load(args.word_rnnlm, word_rnnlm)
if rnnlm is not None:
rnnlm = lm_chainer.ClassifierWithState(
extlm_chainer.MultiLevelLM(word_rnnlm.predictor,
rnnlm.predictor, word_dict,
char_dict))
else:
rnnlm = lm_chainer.ClassifierWithState(
extlm_chainer.LookAheadWordLM(word_rnnlm.predictor, word_dict,
char_dict))
# read json data
with open(args.recog_json, 'rb') as f:
js = json.load(f)['utts']
load_inputs_and_targets = LoadInputsAndTargets(
mode='asr',
load_output=False,
sort_in_input_length=False,
preprocess_conf=train_args.preprocess_conf
if args.preprocess_conf is None else args.preprocess_conf,
preprocess_args={'train': False} # Switch the mode of preprocessing
)
# decode each utterance
new_js = {}
with chainer.no_backprop_mode():
for idx, name in enumerate(js.keys(), 1):
logging.info('(%d/%d) decoding ' + name, idx, len(js.keys()))
batch = [(name, js[name])]
feat = load_inputs_and_targets(batch)[0][0]
nbest_hyps = model.recognize(feat, args, train_args.char_list,
rnnlm)
new_js[name] = add_results_to_json(js[name], nbest_hyps,
train_args.char_list)
with open(args.result_label, 'wb') as f:
f.write(
json.dumps({
'utts': new_js
},
indent=4,
ensure_ascii=False,
sort_keys=True).encode('utf_8'))
def main():
try:
os.mkdir(args.figure_directory)
except:
pass
xp = np
using_gpu = args.gpu_device >= 0
if using_gpu:
cuda.get_device(args.gpu_device).use()
xp = cupy
dataset = gqn.data.Dataset(args.dataset_path)
hyperparams = HyperParameters(snapshot_directory=args.snapshot_path)
model = Model(hyperparams, snapshot_directory=args.snapshot_path)
if using_gpu:
model.to_gpu()
plt.style.use("dark_background")
fig = plt.figure(figsize=(15, 5))
fig.suptitle("GQN")
axis_observations = fig.add_subplot(1, 3, 1)
axis_observations.axis("off")
axis_observations.set_title("Observations")
axis_ground_truth = fig.add_subplot(1, 3, 2)
axis_ground_truth.axis("off")
axis_ground_truth.set_title("Ground Truth")
axis_reconstruction = fig.add_subplot(1, 3, 3)
axis_reconstruction.axis("off")
axis_reconstruction.set_title("Reconstruction")
total_observations_per_scene = 2**2
num_observations_per_column = int(math.sqrt(total_observations_per_scene))
black_color = -0.5
image_shape = (3, ) + hyperparams.image_size
axis_observations_image = np.full(
(3, num_observations_per_column * image_shape[1],
num_observations_per_column * image_shape[2]),
black_color,
dtype=np.float32)
file_number = 1
with chainer.no_backprop_mode():
for subset in dataset:
iterator = gqn.data.Iterator(subset, batch_size=1)
for data_indices in iterator:
animation_frame_array = []
axis_observations_image[...] = black_color
observed_image_array = xp.full(
(total_observations_per_scene, ) + image_shape,
black_color,
dtype=np.float32)
observed_viewpoint_array = xp.zeros(
(total_observations_per_scene, 7), dtype=np.float32)
# shape: (batch, views, height, width, channels)
# range: [-1, 1]
images, viewpoints = subset[data_indices]
# (batch, views, height, width, channels) -> (batch, views, channels, height, width)
images = images.transpose((0, 1, 4, 2, 3)).astype(np.float32)
images = preprocess_images(images)
batch_index = 0
query_index = total_observations_per_scene
query_image = images[batch_index, query_index]
query_viewpoint = to_gpu(viewpoints[None, batch_index,
query_index])
axis_ground_truth.imshow(make_uint8(query_image),
interpolation="none")
for observation_index in range(total_observations_per_scene):
observed_image = images[batch_index, observation_index]
observed_viewpoint = viewpoints[batch_index,
observation_index]
observed_image_array[observation_index] = to_gpu(
observed_image)
observed_viewpoint_array[observation_index] = to_gpu(
observed_viewpoint)
representation = model.compute_observation_representation(
observed_image_array[None, :observation_index + 1],
observed_viewpoint_array[None, :observation_index + 1])
representation = cf.broadcast_to(representation, (1, ) +
representation.shape[1:])
# Update figure
x_start = image_shape[1] * (observation_index %
num_observations_per_column)
x_end = x_start + image_shape[1]
y_start = image_shape[2] * (observation_index //
num_observations_per_column)
y_end = y_start + image_shape[2]
axis_observations_image[:, y_start:y_end,
x_start:x_end] = observed_image
axis_observations.imshow(
make_uint8(axis_observations_image),
interpolation="none",
animated=True)
generated_images = model.generate_image(
query_viewpoint, representation)[0]
axis_reconstruction.imshow(make_uint8(generated_images),
interpolation="none")
plt.pause(1)
def main(args):
####################
# Arguments
tree_sampling = args.tree_sampling # NOTE
trial_name = args.name
# Check
assert len(tree_sampling.split("_")) == 3
for type_ in tree_sampling.split("_"):
assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"]
assert tree_sampling.split("_")[2] != "X"
assert tree_sampling.split("_")[1] != "RB2"
assert tree_sampling.split("_")[2] != "RB2"
if trial_name is None or trial_name == "None":
trial_name = utils.get_current_time()
####################
# Path setting
config = utils.Config()
basename = "%s.%s" \
% (tree_sampling,
trial_name)
utils.mkdir(os.path.join(config.getpath("results"), "baselines"))
path_log = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.log")
path_pred = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.ctrees")
path_eval = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.json")
utils.set_logger(path_log)
####################
# Random seed
random_seed = trial_name
random_seed = utils.hash_string(random_seed)
np.random.seed(random_seed)
cuda.cupy.random.seed(random_seed)
####################
# Log so far
utils.writelog("tree_sampling=%s" % tree_sampling)
utils.writelog("trial_name=%s" % trial_name)
utils.writelog("path_log=%s" % path_log)
utils.writelog("path_pred=%s" % path_pred)
utils.writelog("path_eval=%s" % path_eval)
utils.writelog("random_seed=%d" % random_seed)
####################
# Data preparation
begin_time = time.time()
test_databatch = dataloader.read_rstdt("test",
relation_level="coarse-grained",
with_root=False)
end_time = time.time()
utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time))
####################
# Tree-sampler preparation
sampler = treesamplers.TreeSampler(tree_sampling.split("_")) # NOTE
with chainer.using_config("train", False), chainer.no_backprop_mode():
parse(sampler=sampler, databatch=test_databatch, path_pred=path_pred)
scores = rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
old_scores = old_rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
out = {
"Morey2018": {
"Unlabeled Precision": scores["S"]["Precision"] * 100.0,
"Precision_info": scores["S"]["Precision_info"],
"Unlabeled Recall": scores["S"]["Recall"] * 100.0,
"Recall_info": scores["S"]["Recall_info"],
"Micro F1": scores["S"]["Micro F1"] * 100.0
},
"Marcu2000": {
"Unlabeled Precision": old_scores["S"]["Precision"] * 100.0,
"Precision_info": old_scores["S"]["Precision_info"],
"Unlabeled Recall": old_scores["S"]["Recall"] * 100.0,
"Recall_info": old_scores["S"]["Recall_info"],
"Micro F1": old_scores["S"]["Micro F1"] * 100.0
}
}
utils.write_json(path_eval, out)
utils.writelog(utils.pretty_format_dict(out))
utils.writelog("Done.")
s_x_abs = Variable(s_x_abs)
if args.gpu >= 0:
s_x_abs.to_gpu(args.gpu)
s_x_abs_list.append(s_x_abs)
elif args.single == 1:
audio_data = read_audio(cur_line)
fx, tx, s_x = signal.stft(audio_data, fs=16000, nperseg=512,
noverlap=512-128, nfft=512)
s_x = np.transpose(s_x)
s_x_abs = 20 * log_sp(np.abs(s_x))
s_x_abs = stack_features(s_x_abs.astype(np.float32), 5)
s_x_abs = Variable(s_x_abs)
if args.gpu >= 0:
s_x_abs.to_gpu(args.gpu)
s_x_abs_list.append(s_x_abs)
with chainer.no_backprop_mode():
X = model.predict(s_x_abs_list)
if args.gpu >= 0:
X.to_cpu()
if args.single == 0:
xs = X.data
xs = np.reshape(xs, (8, xs.shape[0] // 8, xs.shape[1]))
taps = 10
xs_power = np.square(exp_sp(xs/20))
Y_hat = np.copy(Y)
D, T, F = xs.shape
for f in range(0, F):
Y_hat[:, :, f] = wpe_filter(Y[:, :, f], xs_power[:, :, f],
taps=taps, delay=3)
def greedy_batch_translate(encdec,
eos_idx,
src_data,
batch_size=80,
gpu=None,
get_attention=False,
nb_steps=50,
reverse_src=False,
reverse_tgt=False,
use_chainerx=False):
with chainer.using_config("train", False), chainer.no_backprop_mode():
if encdec.encdec_type() == "ff":
result = encdec.greedy_batch_translate(src_data,
mb_size=batch_size,
nb_steps=nb_steps)
if get_attention:
dummy_attention = []
for src, tgt in six.moves.zip(src_data, result):
dummy_attention.append(
np.zeros((len(src), len(tgt)), dtype=np.float32))
return result, dummy_attention
else:
return result
nb_ex = len(src_data)
nb_batch = nb_ex // batch_size + (1 if nb_ex % batch_size != 0 else 0)
res = []
attn_all = []
for i in six.moves.range(nb_batch):
current_batch_raw_data = src_data[i * batch_size:(i + 1) *
batch_size]
if reverse_src:
current_batch_raw_data_new = []
for src_side in current_batch_raw_data:
current_batch_raw_data_new.append(src_side[::-1])
current_batch_raw_data = current_batch_raw_data_new
src_batch, src_mask = make_batch_src(current_batch_raw_data,
gpu=gpu,
use_chainerx=use_chainerx)
sample_greedy, score, attn_list = encdec(
src_batch,
nb_steps,
src_mask,
use_best_for_sample=True,
keep_attn_values=get_attention)
deb = de_batch(sample_greedy,
mask=None,
eos_idx=eos_idx,
is_variable=False)
res += deb
if get_attention:
deb_attn = de_batch(attn_list,
mask=None,
eos_idx=None,
is_variable=True,
raw=True,
reverse=reverse_tgt)
attn_all += deb_attn
if reverse_tgt:
new_res = []
for t in res:
if t[-1] == eos_idx:
new_res.append(t[:-1][::-1] + [t[-1]])
else:
new_res.append(t[::-1])
res = new_res
if get_attention:
assert not reverse_tgt, "not implemented"
return res, attn_all
else:
return res
def sample_once(encdec,
src_batch,
tgt_batch,
src_mask,
src_indexer,
tgt_indexer,
eos_idx,
max_nb=None,
s_unk_tag="#S_UNK#",
t_unk_tag="#T_UNK#"):
with chainer.using_config("train", False), chainer.no_backprop_mode():
print("sample")
sample_greedy, score, attn_list = encdec(src_batch,
50,
src_mask,
use_best_for_sample=True,
need_score=True)
# sample, score = encdec(src_batch, 50, src_mask, use_best_for_sample = False)
assert len(src_batch[0].data) == len(tgt_batch[0].data)
assert len(sample_greedy[0]) == len(src_batch[0].data)
debatched_src = de_batch(src_batch,
mask=src_mask,
eos_idx=None,
is_variable=True)
debatched_tgt = de_batch(tgt_batch, eos_idx=eos_idx, is_variable=True)
debatched_sample = de_batch(sample_greedy, eos_idx=eos_idx)
sample_random, score_random, attn_list_random = encdec(
src_batch,
50,
src_mask,
use_best_for_sample=False,
need_score=True)
debatched_sample_random = de_batch(sample_random, eos_idx=eos_idx)
for sent_num in six.moves.range(len(debatched_src)):
if max_nb is not None and sent_num > max_nb:
break
src_idx_seq = debatched_src[sent_num]
tgt_idx_seq = debatched_tgt[sent_num]
sample_idx_seq = debatched_sample[sent_num]
sample_random_idx_seq = debatched_sample_random[sent_num]
print("sent num", sent_num)
for name, seq, unk_tag, indexer, this_eos_idx in six.moves.zip(
"src tgt sample sample_random".split(" "), [
src_idx_seq, tgt_idx_seq, sample_idx_seq,
sample_random_idx_seq
], [s_unk_tag, t_unk_tag, t_unk_tag, t_unk_tag],
[src_indexer, tgt_indexer, tgt_indexer, tgt_indexer],
[None, eos_idx, eos_idx, eos_idx]):
print(name, "idx:", seq)
print(
name, "raw:", " ".join(
indexer.deconvert_swallow(
seq, unk_tag=unk_tag,
eos_idx=this_eos_idx)).encode('utf-8'))
print(
name, "postp:",
indexer.deconvert(seq,
unk_tag=unk_tag,
eos_idx=this_eos_idx).encode('utf-8'))
def CalculateValLoss(self, xs, ys):
with chainer.no_backprop_mode(), chainer.using_config('train', False):
loss, n_w, n_c, n_c_a = self.CalcLoss(xs, ys)
return loss.data
def main(use_gpu=0):
start_time = time.clock()
# select processing unit
if use_gpu >= 0:
import cupy as cp
xp = cp
chainer.cuda.get_device(use_gpu).use()
else:
xp = np
# paths set
training_dataset_path = './samples/sample_dataset/mnist/mnist_training.csv'
validation_dataset_path = './samples/sample_dataset/mnist/mnist_test.csv'
image_path = './samples/sample_dataset/mnist'
# setup network
model = ResidualNetwork()
if use_gpu >= 0:
model.to_gpu()
optimizer = chainer.optimizers.Adam(alpha=0.001,
beta1=0.9,
beta2=0.999,
eps=10**(-8),
weight_decay_rate=0)
optimizer.setup(model)
# setup dataset(training)
train_image_list, train_image_label_list = load_dataset(
training_dataset_path, image_path)
# setup dataset(validation)
validation_image_list, validation_image_label_list = load_dataset(
validation_dataset_path, image_path)
epoch = 10
batchsize = 64
accuracy_train_list, accuracy_val_list = [], []
# learning
for ep in range(0, epoch):
print('epoch', ep + 1)
# before learning, we have to shuffle training data because we want to make network learn different order for
# each epoch.
zipped_train_list = list(zip(train_image_list, train_image_label_list))
random.shuffle(zipped_train_list)
learn_image_list, learn_label_list = zip(*zipped_train_list)
learn_image_list = xp.array(list(learn_image_list))
learn_label_list = xp.array(list(learn_label_list))
batch_times = 0
accuracy_train = 0
for b in range(0, len(learn_image_list), batchsize):
model.cleargrads()
x = chainer.Variable(
xp.asarray(learn_image_list[b:b + batchsize]).astype(
xp.float32))
y = chainer.Variable(
xp.asarray(learn_label_list[b:b + batchsize]).astype(xp.int32))
h = model(x)
# CategorialCrossEntropy doesn't exist in chainer, so, instead of it, I use softmax_cross_entropy.
loss = F.softmax_cross_entropy(h, y)
accuracy_train += F.accuracy(h, y).data
batch_times += 1
loss.backward()
optimizer.update()
accuracy_train_list.append(1 - (accuracy_train / batch_times))
with chainer.using_config('train', False), chainer.no_backprop_mode():
x_valid = chainer.Variable(
xp.asarray(validation_image_list[0:100]).astype(xp.float32))
y_valid_acc = chainer.Variable(
xp.asarray(validation_image_label_list[0:100]).astype(
xp.int32))
h_valid = model(x_valid)
accuracy_val = F.accuracy(h_valid, y_valid_acc)
accuracy_val_list.append(1 - accuracy_val.data)
serializers.save_npz('./models/residual_learning', model)
print("Time to finish learning:" + str(time.clock() - start_time))
# draw accuracy graph
axis_x = np.arange(0, epoch, 1)
y0 = accuracy_train_list
y1 = accuracy_val_list
plt.plot(axis_x, y0, label='train')
plt.plot(axis_x, y1, label='validation')
plt.title('Learning Curve', fontsize=20)
plt.xlabel('epoch', fontsize=16)
plt.ylabel('Error rate')
plt.tick_params(labelsize=14)
plt.grid(True)
plt.legend(loc='upper right')
plt.show()
def get_greedy_action(Q, obs):
"""Get a greedy action wrt a given Q-function."""
obs = Q.xp.asarray(obs[None], dtype=np.float32)
with chainer.no_backprop_mode():
q = Q(obs).data[0]
return int(q.argmax())
if opts['dataset'] == 'h36m':
test = H36M(action=opts['action'],
length=1,
train=False,
use_sh_detection=opts['use_sh_detection'])
elif opts['dataset'] == 'mpii':
test = MPII(train=False, use_sh_detection=opts['use_sh_detection'])
elif opts['dataset'] == 'mpi_inf':
test = MPII3DDataset(train=False)
test_iter = chainer.iterators.SerialIterator(test,
batch_size=row,
shuffle=True,
repeat=False)
with chainer.no_backprop_mode(), chainer.using_config('train', False):
for k in range(col):
batch = test_iter.next()
batchsize = len(batch)
xy_proj, xyz, scale = chainer.dataset.concat_examples(batch)
xy_proj, xyz = xy_proj[:, 0], xyz[:, 0]
xy_real = chainer.Variable(xy_proj)
z_pred = gen(xy_real)
# Rotation by 90 degree
theta = np.array([np.pi / 2] * batchsize, dtype=np.float32)
cos_theta = np.cos(theta)[:, None]
sin_theta = np.sin(theta)[:, None]
# 2D Projection
# Ground Truth