def testReadWeights(self):
memory = 10 * (np.random.rand(BATCH_SIZE, MEMORY_SIZE, WORD_SIZE) - 0.5)
prev_read_weights = np.random.rand(BATCH_SIZE, NUM_READS, MEMORY_SIZE)
prev_read_weights /= prev_read_weights.sum(2, keepdims=True) + 1
link = np.random.rand(BATCH_SIZE, NUM_WRITES, MEMORY_SIZE, MEMORY_SIZE)
# Row and column sums should be at most 1:
link /= np.maximum(link.sum(2, keepdims=True), 1)
link /= np.maximum(link.sum(3, keepdims=True), 1)
# We query the memory on the third location in memory, and select a large
# strength on the query. Then we select a content-based read-mode.
read_content_keys = np.random.rand(BATCH_SIZE, NUM_READS, WORD_SIZE)
read_content_keys[0, 0] = memory[0, 3]
read_content_strengths = tf.constant(
100., shape=[BATCH_SIZE, NUM_READS], dtype=tf.float64)
read_mode = np.random.rand(BATCH_SIZE, NUM_READS, 1 + 2 * NUM_WRITES)
read_mode[0, 0, :] = util.one_hot(1 + 2 * NUM_WRITES, 2 * NUM_WRITES)
inputs = {
'read_content_keys': tf.constant(read_content_keys),
'read_content_strengths': read_content_strengths,
'read_mode': tf.constant(read_mode),
}
read_weights = self.module._read_weights(inputs, memory, prev_read_weights,
link)
with self.test_session():
read_weights = read_weights.eval()
# read_weights for batch 0, read head 0 should be memory location 3
self.assertAllClose(
read_weights[0, 0, :], util.one_hot(MEMORY_SIZE, 3), atol=1e-3)
def load_data_home(path):
Q, P, A_start, A_end, P_raw, A_raw, Q_len, P_len, A_len= [], [], [], [], [], [], [], [], []
with open(path + ".ids.question") as f:
for line in f:
Q.append(line.split())
Q_len.append(len(line.split()))
with open(path + ".ids.context") as f:
for line in f:
P.append(line.split())
P_len.append(len(line.split()))
with open(path + ".span") as f:
for line in f:
start_index, end_index = [int(index) for index in line.split()]
A_start.append(
one_hot(start_index + 1, start_index)
) # [0, 0, ..., start_index] will be padded to 1-hot vec of max_answer_size i.e. max_context_size
A_end.append(
one_hot(end_index + 1, end_index)
) # [0, 0, ..., end_index] will be padded to a 1-hot vec of max_answer_size i.e. max_context_size
A_len.append(end_index - start_index + 1)
with open(path + ".context") as f:
for line in f:
P_raw.append(line)
with open(path + ".answer") as f:
for line in f:
A_raw.append(line)
return Q, P, A_start, A_end, A_len, P_raw, A_raw, Q_len, P_len
def testWriteAllocationWeights(self):
batch_size = 7
memory_size = 23
num_writes = 5
module = addressing.Freeness(memory_size)
usage = np.random.rand(batch_size, memory_size)
write_gates = np.random.rand(batch_size, num_writes)
# Turn off gates for heads 1 and 3 in batch 0. This doesn't scaling down the
# weighting, but it means that the usage doesn't change, so we should get
# the same allocation weightings for: (1, 2) and (3, 4) (but all others
# being different).
write_gates[0, 1] = 0
write_gates[0, 3] = 0
# and turn heads 0 and 2 on for full effect.
write_gates[0, 0] = 1
write_gates[0, 2] = 1
# In batch 1, make one of the usages 0 and another almost 0, so that these
# entries get most of the allocation weights for the first and second heads.
usage[
1] = usage[1] * 0.9 + 0.1 # make sure all entries are in [0.1, 1]
usage[1][4] = 0 # write head 0 should get allocated to position 4
usage[1][3] = 1e-4 # write head 1 should get allocated to position 3
write_gates[1, 0] = 1 # write head 0 fully on
write_gates[1, 1] = 1 # write head 1 fully on
weights = module.write_allocation_weights(
usage=tf.constant(usage),
write_gates=tf.constant(write_gates),
num_writes=num_writes)
with self.test_session():
weights = weights.eval()
# Check that all weights are between 0 and 1
self.assertGreaterEqual(weights.min(), 0)
self.assertLessEqual(weights.max(), 1)
# Check that weights sum to close to 1
self.assertAllClose(np.sum(weights, axis=2),
np.ones([batch_size, num_writes]),
atol=1e-3)
# Check the same / different allocation weight pairs as described above.
self.assertGreater(
np.abs(weights[0, 0, :] - weights[0, 1, :]).max(), 0.1)
self.assertAllEqual(weights[0, 1, :], weights[0, 2, :])
self.assertGreater(
np.abs(weights[0, 2, :] - weights[0, 3, :]).max(), 0.1)
self.assertAllEqual(weights[0, 3, :], weights[0, 4, :])
self.assertAllClose(weights[1][0],
util.one_hot(memory_size, 4),
atol=1e-3)
self.assertAllClose(weights[1][1],
util.one_hot(memory_size, 3),
atol=1e-3)
def mlp_input_cmd(cmd, size):
y, x = cmd['org']
dst = cmd['dst']
cy, cx = size
cmd = cmd['cmd']
return ([1, 0] if cmd == 'move-y' else [0, 1]) + \
utl.one_hot(y, cy) + \
utl.one_hot(x, cx) + \
utl.one_hot(dst, max(cy, cx))
def testWriteWeights(self):
memory = 10 * (np.random.rand(BATCH_SIZE, MEMORY_SIZE, WORD_SIZE) - 0.5)
usage = np.random.rand(BATCH_SIZE, MEMORY_SIZE)
allocation_gate = np.random.rand(BATCH_SIZE, NUM_WRITES)
write_gate = np.random.rand(BATCH_SIZE, NUM_WRITES)
write_content_keys = np.random.rand(BATCH_SIZE, NUM_WRITES, WORD_SIZE)
write_content_strengths = np.random.rand(BATCH_SIZE, NUM_WRITES)
# Check that turning on allocation gate fully brings the write gate to
# the allocation weighting (which we will control by controlling the usage).
usage[:, 3] = 0
allocation_gate[:, 0] = 1
write_gate[:, 0] = 1
inputs = {
'allocation_gate': tf.constant(allocation_gate),
'write_gate': tf.constant(write_gate),
'write_content_keys': tf.constant(write_content_keys),
'write_content_strengths': tf.constant(write_content_strengths)
}
weights = self.module._write_weights(inputs,
tf.constant(memory),
tf.constant(usage))
with self.test_session():
weights = weights.eval()
# Check the weights sum to their target gating.
self.assertAllClose(np.sum(weights, axis=2), write_gate, atol=5e-2)
# Check that we fully allocated to the third row.
weights_0_0_target = util.one_hot(MEMORY_SIZE, 3)
self.assertAllClose(weights[0, 0], weights_0_0_target, atol=1e-3)
def get_sample_embedding(self, production_index):
"""Args: int
Returns: one hot vector of shape [sample_embedding_dim]
"""
return util.one_hot(torch.tensor([production_index]),
self.sample_embedding_dim)[0]
def brier_loss(y, log_pred_prob, rescale=True):
'''Compute (rescaled) Brier loss.
Parameters
----------
y : ndarray of type int or bool, shape (n_samples,)
True labels for each classication data point.
log_pred_prob : ndarray, shape (n_samples, n_labels)
Array of shape ``(len(y), n_labels)``. Each row corresponds to a
categorical distribution with *normalized* probabilities in log scale.
Therefore, the number of columns must be at least 1.
rescale : bool
If True, linearly rescales lost so perfect (P=1) predictions give 0.0
loss and a uniform prediction gives loss of 1.0. False gives the
standard Brier loss.
Returns
-------
loss : ndarray, shape (n_samples,)
Array of the Brier loss for the predictions on each data point in `y`.
'''
n_samples, n_labels = shape_and_validate(y, log_pred_prob)
y_bin = one_hot(y.astype(int), n_labels)
loss = np.sum((np.exp(log_pred_prob) - y_bin)**2, axis=1)
if rescale and n_labels > 1:
# Linearly rescale so perfect is 0.0 and uniform gives 1.0
loss = np.true_divide(n_labels, n_labels - 1) * loss
return loss
def get_deltas(self, O, H):
"""
Returns:
delta_o, delta_1,
"""
from activations import relu_deriv
delta_o = O - util.one_hot(self.labels_active)
delta_h = relu_deriv(H) * (delta_o @ self.W[:, 1:])
return delta_o, delta_h
def featurize(embeddings, word):
"""
Featurize a word given embeddings.
"""
case = casing(word)
word = normalize(word)
case_mapping = {c: one_hot(FDIM, i) for i, c in enumerate(CASES)}
wv = embeddings.get(word, embeddings[UNK])
fv = case_mapping[case]
return np.hstack((wv, fv))
def generate_image(self, batch_image, age, gender):
batch_image = np.repeat(batch_image, 8, axis=0)
batch_image = torch.FloatTensor(batch_image)
batch_image = Variable(batch_image.cuda(), requires_grad=True)
age_code = torch.LongTensor(np.arange(self.Na)).repeat(1, 2)[0]
batch_age_code = util.one_hot(age_code, self.Na)
male_code = torch.LongTensor(np.ones(len(batch_image) // 2) * int(0))
female_code = torch.LongTensor(np.ones(len(batch_image) // 2) * int(1))
gender_code = torch.cat((male_code, female_code), 0)
batch_gender_code = util.one_hot(gender_code, self.Ng)
batch_age_code, batch_gender_code = \
Variable(batch_age_code.cuda()), Variable(batch_gender_code.cuda())
generated = self.G_model(batch_image, batch_age_code,
batch_gender_code)
return generated[age + gender]
def featurize(embeddings, word):
"""
Featurize a word given embeddings.
"""
case = casing(word)
word = normalize(word)
case_mapping = {c: one_hot(FDIM, i) for i, c in enumerate(CASES)}
wv = embeddings.get(word, embeddings[UNK])
fv = case_mapping[case]
return np.hstack((wv, fv))
def load_dataset(name, noise=False):
def featurize(image):
return image.flatten().astype(float) / 255.0
images = np.array([
featurize(image) for image in read_images(name + '-images.idx3-ubyte')
])
labels = np.array([
one_hot(label, 11)
for label in read_labels(name + '-labels.idx1-ubyte')
])
extra_count = int(labels.shape[0] * 0.1)
images = np.append(images, np.random.rand(extra_count, 784), axis=0)
labels = np.append(labels,
np.array([one_hot(10, 11)
for _ in xrange(extra_count)]),
axis=0)
# print images.shape, labels.shape
return images, labels
def process(self, sequence_block):
i,j = sequence_block[0],sequence_block[1]
i = i.astype(np.float32)
j = j.astype(np.uint8)
i = length_norm_crop(i, self.resize_shape, self.crop_x_range, self.crop_y_range)
i,j = _preprocess_oneslice((i, j), False, False, 0)
i = cv2.resize(i, (self.input_shape))
i = np.expand_dims(i, axis=-1)
j = one_hot(j, self.num_class)
return i,j
def test_one_hot():
n_labels = np.random.randint(low=1, high=10)
N = np.random.randint(low=0, high=10)
y = np.random.randint(low=0, high=n_labels, size=N)
z0 = util.one_hot(y, n_labels)
assert (z0.dtype.kind == 'b' and z0.shape == (N, n_labels))
if N >= 1:
enc = OneHotEncoder(n_values=n_labels, sparse=False, dtype=bool)
z1 = enc.fit_transform(y[:, None])
assert (np.all(z0 == z1))
def action_str_to_arr(self, s):
try:
s = s.replace(" ",
"").lower() # remove whitespace and case-invariant
if s.isdigit() and len(
s
) == self.n_species: # if typed like 00030000 for '3 giraffes'
action = np.array([int(c) for c in s],
dtype=int) # change to array
if "*" in s: # if typed like 3*4 for '3 giraffes'
n = int(s[0])
animal_idx = int(s[2])
action = n * util.one_hot(animal_idx, n_dim=self.n_species)
else:
animal_idx = [
i for i in range(self.n_species)
if any([word in s for word in self.animal_names[i]])
]
if len(animal_idx) >= 2:
print(
"you submitted more than 1 animal, we'll ignore that and just select the first"
)
animal_idx = animal_idx[0]
n = [int(c) for c in s if c.isdigit()]
if len(n) > 1:
print(
"you submitted more than 1 number, we'll ignore that and just select the first"
)
if len(n) == 0:
print(
"you submitted no number. We'll assume you meant to play a single card"
)
n = [1]
n = n[0]
action = n * util.one_hot(animal_idx, n_dim=self.n_species)
except:
action = np.zeros(self.n_species)
return action
def allowed_actions(self, player):
hand = self.hands[player]
cards_list = [
n * util.one_hot(idx, n_dim=self.n_species)
for idx in range(self.n_species)
for n in range(1,
int(self.hands[player][idx]) + 1)
]
event_list = [{
"kind": "action",
"who": player,
"cards": cards
} for cards in cards_list]
return event_list
def email_vectors_generator(self, email_examples):
for email in email_examples:
body, label = email['Body'], email['Label']
body_toks = nltk.word_tokenize(body) if self.config.should_tokenize else body
if len(body_toks) + 2 > self.config.len_cutoff:
continue
features, num_feats = self.featurize_email(body_toks)
if label:
if self.config.collapse_classes and label == 2:
label = 1
else:
label = 0
yield features, num_feats, one_hot(self.config.n_classes, label) if self.config.one_hot else label
def random_card_draw(self):
n_hand = np.sum(self.hands[self.whose_turn])
n_to_draw = min(self.max_n_hand - n_hand, np.sum(self.deck))
cards = np.zeros(self.n_species, dtype=int)
for _ in range(n_to_draw):
cards += util.one_hot(np.random.choice(range(self.n_species),
p=(self.deck - cards) /
np.sum(self.deck - cards)),
n_dim=self.n_species)
event = {"kind": "deck_draw", "who": self.whose_turn, "cards": cards}
return event
def recover(patient_mask_predict, shape, ifprocess, num_class, crop_x_range, crop_y_range, resize_shape):
length = len(patient_mask_predict)
w = crop_x_range[1]-crop_x_range[0]
h = crop_y_range[1]-crop_y_range[0]
out = np.array([cv2.resize(x.astype(np.uint8), (h, w)) for x in patient_mask_predict], dtype=np.uint8)
temp = np.zeros(shape=shape, dtype=np.uint8)
for i in range(length):
temp_slice = np.zeros(shape=resize_shape, dtype=np.uint8)
temp_slice[np.ix_(range(*crop_x_range), range(*crop_y_range))] = out[i]
temp[i] = cv2.resize(temp_slice, tuple(temp[i].shape))
# After postprocessing...
temp = one_hot(temp, num_class)
if(ifprocess):
temp = np.array([after_process(temp[i]) for i in range(temp.shape[0])])
return temp
def tensorize_mentions(self, mentions, num_words):
if len(mentions) > 0:
starts, ends = zip(*mentions)
else:
starts, ends = [], []
tag_labels = np.zeros((num_words, 3))
tag_seq = np.zeros(num_words)
tag_loss_label = np.zeros(num_words)
for s, e in zip(starts, ends):
tag_seq[s:e + 1] = np.ones(e - s + 1)
tag_seq[s] = 2
if e < num_words - 1:
span_end = e + 2
tag_loss_label[s:span_end] = np.ones(span_end - s)
tag_loss_label[s] = 1
tag_loss_label[span_end - 1] = 1
tag_labels = [util.one_hot(x, y) for x, y in zip(tag_labels, tag_seq)]
return np.array(starts), np.array(ends), np.array(tag_labels), tag_seq, tag_loss_label
def truncate_example(self,
word_emb,
char_index,
text_len,
speaker_ids,
genre,
is_training,
gold_starts,
gold_ends,
cluster_ids,
tag_labels):
max_training_sentences = self.config["max_training_sentences"]
num_sentences = word_emb.shape[0]
assert num_sentences > max_training_sentences
sentence_offset = random.randint(0, num_sentences - max_training_sentences)
word_offset = text_len[:sentence_offset].sum()
num_words = text_len[sentence_offset:sentence_offset + max_training_sentences].sum()
word_emb = word_emb[sentence_offset:sentence_offset + max_training_sentences,:,:]
char_index = char_index[sentence_offset:sentence_offset + max_training_sentences,:,:]
text_len = text_len[sentence_offset:sentence_offset + max_training_sentences]
speaker_ids = speaker_ids[word_offset: word_offset + num_words]
gold_spans = np.logical_and(gold_ends >= word_offset, gold_starts < word_offset + num_words)
gold_starts = gold_starts[gold_spans] - word_offset
gold_ends = gold_ends[gold_spans] - word_offset
cluster_ids = cluster_ids[gold_spans]
tag_labels = np.zeros((num_words, 3))
tag_seq = np.zeros(num_words)
tag_loss_label = np.zeros(num_words)
for s, e in zip(gold_starts, gold_ends):
tag_seq[s:e + 1] = np.ones(e - s + 1)
tag_seq[s] = 2
if e < num_words - 1:
span_end = e + 2
tag_loss_label[s:span_end] = np.ones(span_end - s)
tag_loss_label[s] = 1
tag_loss_label[span_end - 1] = 1
tag_labels = np.array([util.one_hot(x, y) for x, y in zip(tag_labels, tag_seq)])
return word_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts, gold_ends, cluster_ids, tag_labels, tag_seq, tag_loss_label
def load_all(styles, batch_size, time_steps):
"""
Loads all MIDI files as a piano roll.
(For Keras)
"""
note_data = []
beat_data = []
style_data = []
note_target = []
# TODO: Can speed this up with better parallel loading. Order gaurentee.
styles = [y for x in styles for y in x]
for style_id, style in enumerate(styles):
style_hot = one_hot(style_id, NUM_STYLES)
# Parallel process all files into a list of music sequences
seqs = Parallel(n_jobs=multiprocessing.cpu_count(),
backend='threading')(delayed(load_midi)(f)
for f in get_all_files([style]))
for seq in seqs:
if len(seq) >= time_steps:
# Clamp MIDI to note range
seq = clamp_midi(seq)
# Create training data and labels
train_data, label_data = stagger(seq, time_steps)
note_data += train_data
note_target += label_data
# beats = [compute_beat(i, NOTES_PER_BAR) for i in range(len(seq))]
# beat_data += stagger(beats, time_steps)[0]
note_data = np.array(note_data)
# beat_data = np.array(beat_data)
note_target = np.array(note_target)
#note_data[:,:,:,2] = 0 #note_data[:,:,:,0]
#note_target[:,:,:,2] = 0 #note_target[:,:,:,0]
return note_data, note_target,
def run_game(game):
# Initialize memory
actions = []
policies = []
indices = []
# Run through game
node = game
board = chess.Board()
while not node.is_end():
next_node = node.variation(0)
move = next_node.move
# Get action taken and action list
action = adapter.move_to_label_flat(move)
legal_actions = map(adapter.move_to_label_flat, board.legal_moves)
# Create one-hot probability vector
index = legal_actions.index(action)
probs = util.one_hot(index, len(legal_actions))
assert board.is_legal(move)
# TODO: Look at the validity of this in case of underpromotion
board.push(move)
node = next_node
# Update memory
actions.append(action)
policies.append(probs)
indices.append(legal_actions)
# Get game winner
winner, outcome = {
'1/2-1/2': (chess.WHITE, 0.0),
'1-0': (chess.WHITE, 1.0),
'0-1': (chess.BLACK, 1.0)
}.get(game.headers['Result'], None)
return actions, policies, indices, outcome, winner
def train(self):
experience_indices = list(range(len(self.reward_buffer)))
np.random.shuffle(experience_indices)
reward_mean = np.mean(self.reward_buffer)
reward_std = np.std(self.reward_buffer)
state_input_buffer = np.array(self.state_input_buffer)
legal_action_buffer = np.array(self.legal_action_buffer)
reward_buffer = np.array([[x] for x in self.reward_buffer])
action_taken_buffer = one_hot(self.action_taken_buffer,
self.action_dim)
for batch_index in range(self.number_batches):
if batch_index * self.batch_size < len(experience_indices):
state_input = state_input_buffer[
experience_indices[batch_index *
self.batch_size:(batch_index + 1) *
self.batch_size]]
reward = reward_buffer[
experience_indices[batch_index *
self.batch_size:(batch_index + 1) *
self.batch_size]]
legal_actions = legal_action_buffer[
experience_indices[batch_index *
self.batch_size:(batch_index + 1) *
self.batch_size]]
action_taken = action_taken_buffer[
experience_indices[batch_index *
self.batch_size:(batch_index + 1) *
self.batch_size]]
reward = (reward - reward_mean) / reward_std
self.sess.run(self.train_op,
feed_dict={
self.state_input: state_input,
self.reward: reward,
self.legal_actions: legal_actions,
self.action_taken: action_taken
})
def kernel_ridge_regression_one_hot(join_fn,
train_inputs,
test_inputs,
regularizer=1e-2):
'''
Args:
train_inputs: [b, k, n, *]
test_inputs: [b, q, *]
Returns:
predictions: [b, q, k]
'''
device = train_inputs.device
k, n = train_inputs.shape[1:3]
train_inputs, train_labels = util.flatten_few_shot_examples(train_inputs)
train_labels = train_labels.to(device)
# train_inputs: [b, kn, *]
# train_labels: [b, kn, 1]
train_targets = util.one_hot(k, train_labels,
device=device).type(torch.float)
# train_targets: [b, kn, k]
kernel_matrix = join_fn(
train_inputs.unsqueeze(2), # [b, kn, 1, *]
train_inputs.unsqueeze(1)) # [b, 1, kn, *]
kernel_matrix = kernel_matrix.squeeze(-1)
# kernel_matrix: [b, kn, kn]
kernel_matrix = kernel_matrix.squeeze(-1)
kernel_matrix += regularizer * torch.eye(k * n, device=device)
coeff, _ = torch.gesv(train_targets, kernel_matrix)
# coeff: [b, kn, k]
kernel_train_test = join_fn(
train_inputs.unsqueeze(2), # [b, kn, 1, *]
test_inputs.unsqueeze(1)) # [b, 1, q, *]
kernel_train_test = kernel_train_test.squeeze(-1)
# kernel_train_test: [b, kn, q]
predictions = torch.bmm(kernel_train_test.transpose(-1, -2), coeff)
# predictions: [b, q, k]
return predictions
def load_dataset_with_class(args):
X = []
Y = []
# Load datasets class by class
datasets = args.X.split(',')
num_classes = len(datasets)
parts = np.zeros([num_classes, 2], dtype=np.int32)
count = 0
for i in xrange(num_classes):
x = np.matrix(np.load(datasets[i]), dtype=np.float32)
y = np.tile(one_hot(num_classes, i),
[len(x), 1]).astype(dtype=np.float32)
parts[i, 0] = count
parts[i, 1] = len(x)
X.append(x)
Y.append(y)
count += len(x)
logging.info('Load %d data for class %d from %s' %
(len(x), i, datasets[i]))
X = np.asarray(np.concatenate(X))
Y = np.asarray(np.concatenate(Y))
return X, Y, parts
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Common definitions for QA
"""
from util import one_hot
LBLS = ["ANS", "O", "EMPTY"]
NONE = "EMPTY"
LMAP = {k: one_hot(5, i) for i, k in enumerate(LBLS)}
NUM = "NNNUMMM"
UNK = "UUUNKKK"
EMBED_SIZE = 100
def run():
mode = 'gan'
experiment_name = mode + '_stride_local_discrimination'
filename = 'savestates/cifar_cond_' + experiment_name + '.pickle'
in_filename = filename
in_filename = None
print('experiment_name', experiment_name)
print('in_filename', in_filename)
print('filename', filename)
# Fetch dataset
dataset = dp.dataset.CIFAR10()
x_train, y_train, x_test, y_test = dataset.arrays(dp_dtypes=True)
n_classes = dataset.n_classes
# Normalize pixel intensities
scaler = dp.StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
y_train = one_hot(y_train, n_classes).astype(dp.float_)
y_test = one_hot(y_test, n_classes).astype(dp.float_)
# Setup network
if in_filename is None:
print('Creating new model')
img_shape = x_train.shape[1:]
expressions = model_expressions(img_shape)
else:
print('Starting from %s' % in_filename)
with open(in_filename, 'rb') as f:
expressions = pickle.load(f)
encoder, sampler, generator, discriminator = expressions
model = cond_vaegan.ConditionalVAEGAN(
encoder=encoder,
sampler=sampler,
generator=generator,
discriminator=discriminator,
mode=mode,
)
# Prepare network inputs
batch_size = 64
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size,
epoch_size=150)
# Plotting
n_examples = 100
examples = x_test[:n_examples]
examples_y = y_test[:n_examples]
samples_z = np.random.normal(size=(n_examples, model.sampler.n_hidden))
samples_z = samples_z.astype(dp.float_)
samples_y = ((np.arange(n_examples) // 10) % n_classes)
samples_y = one_hot(samples_y, n_classes).astype(dp.float_)
recon_video = Video('plots/cifar_' + experiment_name +
'_reconstruction.mp4')
sample_video = Video('plots/cifar_' + experiment_name + '_samples.mp4')
sp.misc.imsave('cifar_examples.png', img_tile(dp.misc.to_b01c(examples)))
def plot():
examples_z = model.embed(examples, examples_y)
examples_recon = model.reconstruct(examples_z, examples_y)
examples_recon = clip_range(examples_recon)
recon_video.append(img_tile(dp.misc.to_b01c(examples_recon)))
samples = clip_range(model.reconstruct(samples_z, samples_y))
sample_video.append(img_tile(dp.misc.to_b01c(samples)))
model.setup(**train_input.shapes)
# Train network
runs = [
# (10, dp.RMSProp(learn_rate=0.08)),
# (25, dp.RMSProp(learn_rate=0.12)),
# (100, dp.RMSProp(learn_rate=0.1)),
(150, dp.RMSProp(learn_rate=0.075)),
(150, dp.RMSProp(learn_rate=0.06)),
(150, dp.RMSProp(learn_rate=0.05)),
(150, dp.RMSProp(learn_rate=0.04)),
(25, dp.RMSProp(learn_rate=0.01)),
]
try:
for n_epochs, learn_rule in runs:
if mode == 'vae':
vaegan.train(model, train_input, learn_rule, n_epochs,
epoch_callback=plot)
else:
vaegan.margin_train(model, train_input, learn_rule, n_epochs,
epoch_callback=plot)
except KeyboardInterrupt:
pass
raw_input('\n\nsave model to %s?\n' % filename)
with open(filename, 'wb') as f:
expressions = encoder, sampler, generator, discriminator
pickle.dump(expressions, f)
print('Generating latent space video')
walk_video = Video('plots/cifar_' + experiment_name + '_walk.mp4')
for z in random_walk(samples_z, 500, step_std=0.15):
samples = clip_range(model.reconstruct(z, samples_y))
walk_video.append(img_tile(dp.misc.to_b01c(samples)))
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Common definitions for Lang Detection
"""
from util import one_hot
n_classes = 3
LBLS = [
"Belize Kriol English",
"Hindi",
"Other",
]
NONE = "O"
LMAP = {i: one_hot(n_classes, i) for i, k in enumerate(LBLS)}
LID = {k: i for i, k in enumerate(LBLS)}
def pretrain(self, X_mnb, y_mnb, params):
"""
Pretraining (greedy layer-wise) function for the DBN.
Successively trains the RBMs of the DBN.
:param X_mnb: Training data, split into
minibatches (an iterable of minibatches).
Each minibatch is a numpy array of shape
(N, n_vis) where N is the number of samples
in that minibatch, and n_vis is the number
of neurons in the visible (lowest) layer
of the DBN.
:param y_mnb: Training data labels corresponding to
X_mnb, split into minibatches (an iterable).
Each minibatch is a numpy array of shape
(N, 1).
:param params: Parameters for training the RBMs.
An iterable that has (layer_count - 1) elements.
An element can be 'None', in which case the
corresponding RBM is not trained (thus allowing
for RBM training outside the DBN).
A element is a dict or an iterable that can
be unpacked into calling arguments to the
RBM.train(...) method.
"""
assert len(params) == len(self.rbms)
# pre-train one rbm at a time
rbm_train_res = []
rbm_X = X_mnb
for rbm_ind, rbm in enumerate(self.rbms):
log.info('Training RBM #%d', rbm_ind)
# we only train a rbm if we got params for it
if params[rbm_ind] is None:
log.info('Skipping RBM #%d training, no params', rbm_ind)
else:
# the last rbm needs labels appended
if rbm is self.rbms[-1]:
log.info('Appending one-hot labes to data')
# iterate through the minibatches
for i in range(len(X_mnb)):
# get "one hot" matrix for label indices
y_one_hot = util.one_hot(y_mnb[i], self.class_count)
# append one-hot labels in front of the data
rbm_X[i] = np.append(y_one_hot, rbm_X[i], axis=1)
# train rbm
if isinstance(params[rbm_ind], dict):
rbm_train_res.append(rbm.train(rbm_X, **params[rbm_ind]))
else:
rbm_train_res.append(rbm.train(rbm_X, *params[rbm_ind]))
# convert X to input for the next rbm
rbm_X = [rbm.hid_given_vis(r)[1] for r in rbm_X]
return rbm_train_res
if not os.path.exists('../build/checkpoints/' + target):
os.mkdir('../build/checkpoints/' + target)
if not os.path.exists('../build/Log'):
os.mkdir('../build/Log')
if not os.path.exists('../build/Log/' + target):
os.mkdir('../build/Log/' + target)
# 给出加载数据时间，其实没有太多意义
start = time.time()
x, y = LoadData.data_generator_xd(path, input_height, input_width)
x = np.expand_dims(x, axis=-1)
y = util.one_hot(y, n_classes)
X_train, X_valid, y_train, y_valid = train_test_split(x, y, test_size=0.1)
end = time.time()
print('X_train.shape: {}'.format(X_train.shape))
print('X_valid.shape: {}'.format(X_valid.shape))
print('y_train.shape: {}'.format(y_train.shape))
print('y_valid.shape: {}'.format(y_valid.shape))
print("加载时间:%.2f" % (end - start))
eva_list = ['global_dice']
metrics = []
for item in eva_list:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Common definitions for NER
"""
from util import one_hot
LBLS = [
"PER",
"ORG",
"LOC",
"MISC",
"O",
]
NONE = "O"
LMAP = {k: one_hot(5,i) for i, k in enumerate(LBLS)}
NUM = "NNNUMMM"
UNK = "UUUNKKK"
EMBED_SIZE = 50
def testModule(self):
batch_size = 7
memory_size = 4
num_reads = 11
num_writes = 5
module = addressing.TemporalLinkage(memory_size=memory_size,
num_writes=num_writes)
prev_link_in = tf.placeholder(
tf.float32, (batch_size, num_writes, memory_size, memory_size))
prev_precedence_weights_in = tf.placeholder(
tf.float32, (batch_size, num_writes, memory_size))
write_weights_in = tf.placeholder(
tf.float32, (batch_size, num_writes, memory_size))
state = addressing.TemporalLinkageState(
link=np.zeros([batch_size, num_writes, memory_size, memory_size]),
precedence_weights=np.zeros([batch_size, num_writes, memory_size]))
calc_state = module(
write_weights_in,
addressing.TemporalLinkageState(
link=prev_link_in,
precedence_weights=prev_precedence_weights_in))
with self.test_session() as sess:
num_steps = 5
for i in xrange(num_steps):
write_weights = np.random.rand(batch_size, num_writes,
memory_size)
write_weights /= write_weights.sum(2, keepdims=True) + 1
# Simulate (in final steps) link 0-->1 in head 0 and 3-->2 in head 1
if i == num_steps - 2:
write_weights[0, 0, :] = util.one_hot(memory_size, 0)
write_weights[0, 1, :] = util.one_hot(memory_size, 3)
elif i == num_steps - 1:
write_weights[0, 0, :] = util.one_hot(memory_size, 1)
write_weights[0, 1, :] = util.one_hot(memory_size, 2)
state = sess.run(calc_state,
feed_dict={
prev_link_in: state.link,
prev_precedence_weights_in:
state.precedence_weights,
write_weights_in: write_weights
})
# link should be bounded in range [0, 1]
self.assertGreaterEqual(state.link.min(), 0)
self.assertLessEqual(state.link.max(), 1)
# link diagonal should be zero
self.assertAllEqual(
state.link[:, :, range(memory_size),
range(memory_size)],
np.zeros([batch_size, num_writes, memory_size]))
# link rows and columns should sum to at most 1
self.assertLessEqual(state.link.sum(2).max(), 1)
self.assertLessEqual(state.link.sum(3).max(), 1)
# records our transitions in batch 0: head 0: 0->1, and head 1: 3->2
self.assertAllEqual(state.link[0, 0, :, 0],
util.one_hot(memory_size, 1))
self.assertAllEqual(state.link[0, 1, :, 3],
util.one_hot(memory_size, 2))
# Now test calculation of forward and backward read weights
prev_read_weights = np.random.rand(batch_size, num_reads, memory_size)
prev_read_weights[0, 5, :] = util.one_hot(memory_size,
0) # read 5, posn 0
prev_read_weights[0, 6, :] = util.one_hot(memory_size,
2) # read 6, posn 2
forward_read_weights = module.directional_read_weights(
tf.constant(state.link),
tf.constant(prev_read_weights, dtype=tf.float32),
forward=True)
backward_read_weights = module.directional_read_weights(
tf.constant(state.link),
tf.constant(prev_read_weights, dtype=tf.float32),
forward=False)
with self.test_session():
forward_read_weights = forward_read_weights.eval()
backward_read_weights = backward_read_weights.eval()
# Check directional weights calculated correctly.
self.assertAllEqual(
forward_read_weights[0, 5, 0, :], # read=5, write=0
util.one_hot(memory_size, 1))
self.assertAllEqual(
backward_read_weights[0, 6, 1, :], # read=6, write=1
util.one_hot(memory_size, 3))
def run():
mode = 'vaegan'
vae_grad_scale = 0.025
experiment_name = mode + 'scale_%.5f' % vae_grad_scale
filename = 'savestates/mnist_' + experiment_name + '.pickle'
in_filename = filename
in_filename = None
print('experiment_name', experiment_name)
print('in_filename', in_filename)
print('filename', filename)
# Fetch dataset
dataset = dp.dataset.MNIST()
x_train, y_train, x_test, y_test = dataset.arrays(dp_dtypes=True)
n_classes = dataset.n_classes
img_shape = x_train.shape[1:]
# Normalize pixel intensities
scaler = dp.UniformScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
y_train = one_hot(y_train, n_classes).astype(dp.float_)
y_test = one_hot(y_test, n_classes).astype(dp.float_)
x_train = np.reshape(x_train, (x_train.shape[0], -1))
x_test = np.reshape(x_test, (x_test.shape[0], -1))
# Setup network
if in_filename is None:
print('Creating new model')
expressions = model_expressions(img_shape)
else:
print('Starting from %s' % in_filename)
with open(in_filename, 'rb') as f:
expressions = pickle.load(f)
encoder, sampler, generator, discriminator = expressions
model = cond_vaegan.ConditionalVAEGAN(
encoder=encoder,
sampler=sampler,
generator=generator,
discriminator=discriminator,
mode=mode,
reconstruct_error=expr.nnet.BinaryCrossEntropy(),
vae_grad_scale=vae_grad_scale,
)
# Prepare network inputs
batch_size = 128
train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size,
epoch_size=250)
# Plotting
n_examples = 100
examples = x_test[:n_examples]
examples_y = y_test[:n_examples]
samples_z = np.random.normal(size=(n_examples, model.sampler.n_hidden))
samples_z = samples_z.astype(dp.float_)
samples_y = ((np.arange(n_examples) // 10) % n_classes)
samples_y = one_hot(samples_y, n_classes).astype(dp.float_)
recon_video = Video('plots/mnist_' + experiment_name +
'_reconstruction.mp4')
sample_video = Video('plots/mnist_' + experiment_name + '_samples.mp4')
sp.misc.imsave('plots/mnist_examples.png',
img_tile(to_b01c(examples, img_shape)))
def plot():
model.phase = 'test'
model.sampler.batch_size=100
examples_z = model.embed(examples, examples_y)
examples_recon = model.reconstruct(examples_z, examples_y)
recon_video.append(img_tile(to_b01c(examples_recon, img_shape)))
samples = model.reconstruct(samples_z, samples_y)
sample_video.append(img_tile(to_b01c(samples, img_shape)))
model.setup(**train_input.shapes)
model.phase = 'train'
# Train network
runs = [
(75, dp.RMSProp(learn_rate=0.075)),
(25, dp.RMSProp(learn_rate=0.05)),
(5, dp.RMSProp(learn_rate=0.01)),
(5, dp.RMSProp(learn_rate=0.005)),
]
try:
for n_epochs, learn_rule in runs:
if mode == 'vae':
vaegan.train(model, train_input, learn_rule, n_epochs,
epoch_callback=plot)
else:
vaegan.margin_train(model, train_input, learn_rule, n_epochs,
epoch_callback=plot)
except KeyboardInterrupt:
pass
raw_input('\n\nsave model to %s?\n' % filename)
with open(filename, 'wb') as f:
expressions = encoder, sampler, generator, discriminator
pickle.dump(expressions, f)
model.phase = 'test'
batch_size = 128
model.sampler.batch_size=128
z = []
i = 0
z = model.embed(x_train, y_train)
print(z.shape)
z_mean = np.mean(z, axis=0)
z_std = np.std(z, axis=0)
z_cov = np.cov(z.T)
print(np.mean(z_mean), np.std(z_mean))
print(np.mean(z_std), np.std(z_std))
print(z_mean.shape, z_std.shape, z_cov.shape)
raw_input('\n\ngenerate latent space video?\n')
print('Generating latent space video')
walk_video = Video('plots/mnist_' + experiment_name + '_walk.mp4')
for z in random_walk(samples_z, 500, n_dir_steps=10, mean=z_mean, std=z_cov):
samples = model.reconstruct(z, samples_y)
walk_video.append(img_tile(to_b01c(samples, img_shape)))
print('Generating AdversarialMNIST dataset')
_, y_train, _, y_test = dataset.arrays(dp_dtypes=True)
n = 0
batch_size = 512
advmnist_size = 1e6
x_advmnist = np.empty((advmnist_size, 28*28))
y_advmnist = np.empty((advmnist_size,))
while n < advmnist_size:
samples_z = np.random.multivariate_normal(mean=z_mean, cov=z_cov,
size=batch_size)
samples_z = samples_z.astype(dp.float_)
start_idx = n % len(y_train)
stop_idx = (n + batch_size) % len(y_train)
if start_idx > stop_idx:
samples_y = np.concatenate([y_train[start_idx:], y_train[:stop_idx]])
else:
samples_y = y_train[start_idx:stop_idx]
y_advmnist[n:n+batch_size] = samples_y[:advmnist_size-n]
samples_y = one_hot(samples_y, n_classes).astype(dp.float_)
samples = model.reconstruct(samples_z, samples_y)
x_advmnist[n:n+batch_size] = samples[:advmnist_size-n]
n += batch_size
x_train = x_advmnist
y_train = y_advmnist
import sklearn.neighbors
clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1, algorithm='brute', n_jobs=-1)
clf.fit(x_train, y_train)
print('KNN predict')
step = 2500
errors = []
i = 0
while i < len(x_test):
print(i)
errors.append(clf.predict(x_test[i:i+step]) != y_test[i:i+step])
i += step
error = np.mean(errors)
print('Test error rate: %.4f' % error)
print('DONE ' + experiment_name)
