def testReadWeights(self): memory = 10 * (np.random.rand(BATCH_SIZE, MEMORY_SIZE, WORD_SIZE) - 0.5) prev_read_weights = np.random.rand(BATCH_SIZE, NUM_READS, MEMORY_SIZE) prev_read_weights /= prev_read_weights.sum(2, keepdims=True) + 1 link = np.random.rand(BATCH_SIZE, NUM_WRITES, MEMORY_SIZE, MEMORY_SIZE) # Row and column sums should be at most 1: link /= np.maximum(link.sum(2, keepdims=True), 1) link /= np.maximum(link.sum(3, keepdims=True), 1) # We query the memory on the third location in memory, and select a large # strength on the query. Then we select a content-based read-mode. read_content_keys = np.random.rand(BATCH_SIZE, NUM_READS, WORD_SIZE) read_content_keys[0, 0] = memory[0, 3] read_content_strengths = tf.constant( 100., shape=[BATCH_SIZE, NUM_READS], dtype=tf.float64) read_mode = np.random.rand(BATCH_SIZE, NUM_READS, 1 + 2 * NUM_WRITES) read_mode[0, 0, :] = util.one_hot(1 + 2 * NUM_WRITES, 2 * NUM_WRITES) inputs = { 'read_content_keys': tf.constant(read_content_keys), 'read_content_strengths': read_content_strengths, 'read_mode': tf.constant(read_mode), } read_weights = self.module._read_weights(inputs, memory, prev_read_weights, link) with self.test_session(): read_weights = read_weights.eval() # read_weights for batch 0, read head 0 should be memory location 3 self.assertAllClose( read_weights[0, 0, :], util.one_hot(MEMORY_SIZE, 3), atol=1e-3)
def load_data_home(path): Q, P, A_start, A_end, P_raw, A_raw, Q_len, P_len, A_len= [], [], [], [], [], [], [], [], [] with open(path + ".ids.question") as f: for line in f: Q.append(line.split()) Q_len.append(len(line.split())) with open(path + ".ids.context") as f: for line in f: P.append(line.split()) P_len.append(len(line.split())) with open(path + ".span") as f: for line in f: start_index, end_index = [int(index) for index in line.split()] A_start.append( one_hot(start_index + 1, start_index) ) # [0, 0, ..., start_index] will be padded to 1-hot vec of max_answer_size i.e. max_context_size A_end.append( one_hot(end_index + 1, end_index) ) # [0, 0, ..., end_index] will be padded to a 1-hot vec of max_answer_size i.e. max_context_size A_len.append(end_index - start_index + 1) with open(path + ".context") as f: for line in f: P_raw.append(line) with open(path + ".answer") as f: for line in f: A_raw.append(line) return Q, P, A_start, A_end, A_len, P_raw, A_raw, Q_len, P_len
def testWriteAllocationWeights(self): batch_size = 7 memory_size = 23 num_writes = 5 module = addressing.Freeness(memory_size) usage = np.random.rand(batch_size, memory_size) write_gates = np.random.rand(batch_size, num_writes) # Turn off gates for heads 1 and 3 in batch 0. This doesn't scaling down the # weighting, but it means that the usage doesn't change, so we should get # the same allocation weightings for: (1, 2) and (3, 4) (but all others # being different). write_gates[0, 1] = 0 write_gates[0, 3] = 0 # and turn heads 0 and 2 on for full effect. write_gates[0, 0] = 1 write_gates[0, 2] = 1 # In batch 1, make one of the usages 0 and another almost 0, so that these # entries get most of the allocation weights for the first and second heads. usage[ 1] = usage[1] * 0.9 + 0.1 # make sure all entries are in [0.1, 1] usage[1][4] = 0 # write head 0 should get allocated to position 4 usage[1][3] = 1e-4 # write head 1 should get allocated to position 3 write_gates[1, 0] = 1 # write head 0 fully on write_gates[1, 1] = 1 # write head 1 fully on weights = module.write_allocation_weights( usage=tf.constant(usage), write_gates=tf.constant(write_gates), num_writes=num_writes) with self.test_session(): weights = weights.eval() # Check that all weights are between 0 and 1 self.assertGreaterEqual(weights.min(), 0) self.assertLessEqual(weights.max(), 1) # Check that weights sum to close to 1 self.assertAllClose(np.sum(weights, axis=2), np.ones([batch_size, num_writes]), atol=1e-3) # Check the same / different allocation weight pairs as described above. self.assertGreater( np.abs(weights[0, 0, :] - weights[0, 1, :]).max(), 0.1) self.assertAllEqual(weights[0, 1, :], weights[0, 2, :]) self.assertGreater( np.abs(weights[0, 2, :] - weights[0, 3, :]).max(), 0.1) self.assertAllEqual(weights[0, 3, :], weights[0, 4, :]) self.assertAllClose(weights[1][0], util.one_hot(memory_size, 4), atol=1e-3) self.assertAllClose(weights[1][1], util.one_hot(memory_size, 3), atol=1e-3)
def mlp_input_cmd(cmd, size): y, x = cmd['org'] dst = cmd['dst'] cy, cx = size cmd = cmd['cmd'] return ([1, 0] if cmd == 'move-y' else [0, 1]) + \ utl.one_hot(y, cy) + \ utl.one_hot(x, cx) + \ utl.one_hot(dst, max(cy, cx))
def testWriteWeights(self): memory = 10 * (np.random.rand(BATCH_SIZE, MEMORY_SIZE, WORD_SIZE) - 0.5) usage = np.random.rand(BATCH_SIZE, MEMORY_SIZE) allocation_gate = np.random.rand(BATCH_SIZE, NUM_WRITES) write_gate = np.random.rand(BATCH_SIZE, NUM_WRITES) write_content_keys = np.random.rand(BATCH_SIZE, NUM_WRITES, WORD_SIZE) write_content_strengths = np.random.rand(BATCH_SIZE, NUM_WRITES) # Check that turning on allocation gate fully brings the write gate to # the allocation weighting (which we will control by controlling the usage). usage[:, 3] = 0 allocation_gate[:, 0] = 1 write_gate[:, 0] = 1 inputs = { 'allocation_gate': tf.constant(allocation_gate), 'write_gate': tf.constant(write_gate), 'write_content_keys': tf.constant(write_content_keys), 'write_content_strengths': tf.constant(write_content_strengths) } weights = self.module._write_weights(inputs, tf.constant(memory), tf.constant(usage)) with self.test_session(): weights = weights.eval() # Check the weights sum to their target gating. self.assertAllClose(np.sum(weights, axis=2), write_gate, atol=5e-2) # Check that we fully allocated to the third row. weights_0_0_target = util.one_hot(MEMORY_SIZE, 3) self.assertAllClose(weights[0, 0], weights_0_0_target, atol=1e-3)
def get_sample_embedding(self, production_index): """Args: int Returns: one hot vector of shape [sample_embedding_dim] """ return util.one_hot(torch.tensor([production_index]), self.sample_embedding_dim)[0]
def brier_loss(y, log_pred_prob, rescale=True): '''Compute (rescaled) Brier loss. Parameters ---------- y : ndarray of type int or bool, shape (n_samples,) True labels for each classication data point. log_pred_prob : ndarray, shape (n_samples, n_labels) Array of shape ``(len(y), n_labels)``. Each row corresponds to a categorical distribution with *normalized* probabilities in log scale. Therefore, the number of columns must be at least 1. rescale : bool If True, linearly rescales lost so perfect (P=1) predictions give 0.0 loss and a uniform prediction gives loss of 1.0. False gives the standard Brier loss. Returns ------- loss : ndarray, shape (n_samples,) Array of the Brier loss for the predictions on each data point in `y`. ''' n_samples, n_labels = shape_and_validate(y, log_pred_prob) y_bin = one_hot(y.astype(int), n_labels) loss = np.sum((np.exp(log_pred_prob) - y_bin)**2, axis=1) if rescale and n_labels > 1: # Linearly rescale so perfect is 0.0 and uniform gives 1.0 loss = np.true_divide(n_labels, n_labels - 1) * loss return loss
def get_deltas(self, O, H): """ Returns: delta_o, delta_1, """ from activations import relu_deriv delta_o = O - util.one_hot(self.labels_active) delta_h = relu_deriv(H) * (delta_o @ self.W[:, 1:]) return delta_o, delta_h
def featurize(embeddings, word): """ Featurize a word given embeddings. """ case = casing(word) word = normalize(word) case_mapping = {c: one_hot(FDIM, i) for i, c in enumerate(CASES)} wv = embeddings.get(word, embeddings[UNK]) fv = case_mapping[case] return np.hstack((wv, fv))
def generate_image(self, batch_image, age, gender): batch_image = np.repeat(batch_image, 8, axis=0) batch_image = torch.FloatTensor(batch_image) batch_image = Variable(batch_image.cuda(), requires_grad=True) age_code = torch.LongTensor(np.arange(self.Na)).repeat(1, 2)[0] batch_age_code = util.one_hot(age_code, self.Na) male_code = torch.LongTensor(np.ones(len(batch_image) // 2) * int(0)) female_code = torch.LongTensor(np.ones(len(batch_image) // 2) * int(1)) gender_code = torch.cat((male_code, female_code), 0) batch_gender_code = util.one_hot(gender_code, self.Ng) batch_age_code, batch_gender_code = \ Variable(batch_age_code.cuda()), Variable(batch_gender_code.cuda()) generated = self.G_model(batch_image, batch_age_code, batch_gender_code) return generated[age + gender]
def load_dataset(name, noise=False): def featurize(image): return image.flatten().astype(float) / 255.0 images = np.array([ featurize(image) for image in read_images(name + '-images.idx3-ubyte') ]) labels = np.array([ one_hot(label, 11) for label in read_labels(name + '-labels.idx1-ubyte') ]) extra_count = int(labels.shape[0] * 0.1) images = np.append(images, np.random.rand(extra_count, 784), axis=0) labels = np.append(labels, np.array([one_hot(10, 11) for _ in xrange(extra_count)]), axis=0) # print images.shape, labels.shape return images, labels
def process(self, sequence_block): i,j = sequence_block[0],sequence_block[1] i = i.astype(np.float32) j = j.astype(np.uint8) i = length_norm_crop(i, self.resize_shape, self.crop_x_range, self.crop_y_range) i,j = _preprocess_oneslice((i, j), False, False, 0) i = cv2.resize(i, (self.input_shape)) i = np.expand_dims(i, axis=-1) j = one_hot(j, self.num_class) return i,j
def test_one_hot(): n_labels = np.random.randint(low=1, high=10) N = np.random.randint(low=0, high=10) y = np.random.randint(low=0, high=n_labels, size=N) z0 = util.one_hot(y, n_labels) assert (z0.dtype.kind == 'b' and z0.shape == (N, n_labels)) if N >= 1: enc = OneHotEncoder(n_values=n_labels, sparse=False, dtype=bool) z1 = enc.fit_transform(y[:, None]) assert (np.all(z0 == z1))
def action_str_to_arr(self, s): try: s = s.replace(" ", "").lower() # remove whitespace and case-invariant if s.isdigit() and len( s ) == self.n_species: # if typed like 00030000 for '3 giraffes' action = np.array([int(c) for c in s], dtype=int) # change to array if "*" in s: # if typed like 3*4 for '3 giraffes' n = int(s[0]) animal_idx = int(s[2]) action = n * util.one_hot(animal_idx, n_dim=self.n_species) else: animal_idx = [ i for i in range(self.n_species) if any([word in s for word in self.animal_names[i]]) ] if len(animal_idx) >= 2: print( "you submitted more than 1 animal, we'll ignore that and just select the first" ) animal_idx = animal_idx[0] n = [int(c) for c in s if c.isdigit()] if len(n) > 1: print( "you submitted more than 1 number, we'll ignore that and just select the first" ) if len(n) == 0: print( "you submitted no number. We'll assume you meant to play a single card" ) n = [1] n = n[0] action = n * util.one_hot(animal_idx, n_dim=self.n_species) except: action = np.zeros(self.n_species) return action
def allowed_actions(self, player): hand = self.hands[player] cards_list = [ n * util.one_hot(idx, n_dim=self.n_species) for idx in range(self.n_species) for n in range(1, int(self.hands[player][idx]) + 1) ] event_list = [{ "kind": "action", "who": player, "cards": cards } for cards in cards_list] return event_list
def email_vectors_generator(self, email_examples): for email in email_examples: body, label = email['Body'], email['Label'] body_toks = nltk.word_tokenize(body) if self.config.should_tokenize else body if len(body_toks) + 2 > self.config.len_cutoff: continue features, num_feats = self.featurize_email(body_toks) if label: if self.config.collapse_classes and label == 2: label = 1 else: label = 0 yield features, num_feats, one_hot(self.config.n_classes, label) if self.config.one_hot else label
def random_card_draw(self): n_hand = np.sum(self.hands[self.whose_turn]) n_to_draw = min(self.max_n_hand - n_hand, np.sum(self.deck)) cards = np.zeros(self.n_species, dtype=int) for _ in range(n_to_draw): cards += util.one_hot(np.random.choice(range(self.n_species), p=(self.deck - cards) / np.sum(self.deck - cards)), n_dim=self.n_species) event = {"kind": "deck_draw", "who": self.whose_turn, "cards": cards} return event
def recover(patient_mask_predict, shape, ifprocess, num_class, crop_x_range, crop_y_range, resize_shape): length = len(patient_mask_predict) w = crop_x_range[1]-crop_x_range[0] h = crop_y_range[1]-crop_y_range[0] out = np.array([cv2.resize(x.astype(np.uint8), (h, w)) for x in patient_mask_predict], dtype=np.uint8) temp = np.zeros(shape=shape, dtype=np.uint8) for i in range(length): temp_slice = np.zeros(shape=resize_shape, dtype=np.uint8) temp_slice[np.ix_(range(*crop_x_range), range(*crop_y_range))] = out[i] temp[i] = cv2.resize(temp_slice, tuple(temp[i].shape)) # After postprocessing... temp = one_hot(temp, num_class) if(ifprocess): temp = np.array([after_process(temp[i]) for i in range(temp.shape[0])]) return temp
def tensorize_mentions(self, mentions, num_words): if len(mentions) > 0: starts, ends = zip(*mentions) else: starts, ends = [], [] tag_labels = np.zeros((num_words, 3)) tag_seq = np.zeros(num_words) tag_loss_label = np.zeros(num_words) for s, e in zip(starts, ends): tag_seq[s:e + 1] = np.ones(e - s + 1) tag_seq[s] = 2 if e < num_words - 1: span_end = e + 2 tag_loss_label[s:span_end] = np.ones(span_end - s) tag_loss_label[s] = 1 tag_loss_label[span_end - 1] = 1 tag_labels = [util.one_hot(x, y) for x, y in zip(tag_labels, tag_seq)] return np.array(starts), np.array(ends), np.array(tag_labels), tag_seq, tag_loss_label
def truncate_example(self, word_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts, gold_ends, cluster_ids, tag_labels): max_training_sentences = self.config["max_training_sentences"] num_sentences = word_emb.shape[0] assert num_sentences > max_training_sentences sentence_offset = random.randint(0, num_sentences - max_training_sentences) word_offset = text_len[:sentence_offset].sum() num_words = text_len[sentence_offset:sentence_offset + max_training_sentences].sum() word_emb = word_emb[sentence_offset:sentence_offset + max_training_sentences,:,:] char_index = char_index[sentence_offset:sentence_offset + max_training_sentences,:,:] text_len = text_len[sentence_offset:sentence_offset + max_training_sentences] speaker_ids = speaker_ids[word_offset: word_offset + num_words] gold_spans = np.logical_and(gold_ends >= word_offset, gold_starts < word_offset + num_words) gold_starts = gold_starts[gold_spans] - word_offset gold_ends = gold_ends[gold_spans] - word_offset cluster_ids = cluster_ids[gold_spans] tag_labels = np.zeros((num_words, 3)) tag_seq = np.zeros(num_words) tag_loss_label = np.zeros(num_words) for s, e in zip(gold_starts, gold_ends): tag_seq[s:e + 1] = np.ones(e - s + 1) tag_seq[s] = 2 if e < num_words - 1: span_end = e + 2 tag_loss_label[s:span_end] = np.ones(span_end - s) tag_loss_label[s] = 1 tag_loss_label[span_end - 1] = 1 tag_labels = np.array([util.one_hot(x, y) for x, y in zip(tag_labels, tag_seq)]) return word_emb, char_index, text_len, speaker_ids, genre, is_training, gold_starts, gold_ends, cluster_ids, tag_labels, tag_seq, tag_loss_label
def load_all(styles, batch_size, time_steps): """ Loads all MIDI files as a piano roll. (For Keras) """ note_data = [] beat_data = [] style_data = [] note_target = [] # TODO: Can speed this up with better parallel loading. Order gaurentee. styles = [y for x in styles for y in x] for style_id, style in enumerate(styles): style_hot = one_hot(style_id, NUM_STYLES) # Parallel process all files into a list of music sequences seqs = Parallel(n_jobs=multiprocessing.cpu_count(), backend='threading')(delayed(load_midi)(f) for f in get_all_files([style])) for seq in seqs: if len(seq) >= time_steps: # Clamp MIDI to note range seq = clamp_midi(seq) # Create training data and labels train_data, label_data = stagger(seq, time_steps) note_data += train_data note_target += label_data # beats = [compute_beat(i, NOTES_PER_BAR) for i in range(len(seq))] # beat_data += stagger(beats, time_steps)[0] note_data = np.array(note_data) # beat_data = np.array(beat_data) note_target = np.array(note_target) #note_data[:,:,:,2] = 0 #note_data[:,:,:,0] #note_target[:,:,:,2] = 0 #note_target[:,:,:,0] return note_data, note_target,
def run_game(game): # Initialize memory actions = [] policies = [] indices = [] # Run through game node = game board = chess.Board() while not node.is_end(): next_node = node.variation(0) move = next_node.move # Get action taken and action list action = adapter.move_to_label_flat(move) legal_actions = map(adapter.move_to_label_flat, board.legal_moves) # Create one-hot probability vector index = legal_actions.index(action) probs = util.one_hot(index, len(legal_actions)) assert board.is_legal(move) # TODO: Look at the validity of this in case of underpromotion board.push(move) node = next_node # Update memory actions.append(action) policies.append(probs) indices.append(legal_actions) # Get game winner winner, outcome = { '1/2-1/2': (chess.WHITE, 0.0), '1-0': (chess.WHITE, 1.0), '0-1': (chess.BLACK, 1.0) }.get(game.headers['Result'], None) return actions, policies, indices, outcome, winner
def train(self): experience_indices = list(range(len(self.reward_buffer))) np.random.shuffle(experience_indices) reward_mean = np.mean(self.reward_buffer) reward_std = np.std(self.reward_buffer) state_input_buffer = np.array(self.state_input_buffer) legal_action_buffer = np.array(self.legal_action_buffer) reward_buffer = np.array([[x] for x in self.reward_buffer]) action_taken_buffer = one_hot(self.action_taken_buffer, self.action_dim) for batch_index in range(self.number_batches): if batch_index * self.batch_size < len(experience_indices): state_input = state_input_buffer[ experience_indices[batch_index * self.batch_size:(batch_index + 1) * self.batch_size]] reward = reward_buffer[ experience_indices[batch_index * self.batch_size:(batch_index + 1) * self.batch_size]] legal_actions = legal_action_buffer[ experience_indices[batch_index * self.batch_size:(batch_index + 1) * self.batch_size]] action_taken = action_taken_buffer[ experience_indices[batch_index * self.batch_size:(batch_index + 1) * self.batch_size]] reward = (reward - reward_mean) / reward_std self.sess.run(self.train_op, feed_dict={ self.state_input: state_input, self.reward: reward, self.legal_actions: legal_actions, self.action_taken: action_taken })
def kernel_ridge_regression_one_hot(join_fn, train_inputs, test_inputs, regularizer=1e-2): ''' Args: train_inputs: [b, k, n, *] test_inputs: [b, q, *] Returns: predictions: [b, q, k] ''' device = train_inputs.device k, n = train_inputs.shape[1:3] train_inputs, train_labels = util.flatten_few_shot_examples(train_inputs) train_labels = train_labels.to(device) # train_inputs: [b, kn, *] # train_labels: [b, kn, 1] train_targets = util.one_hot(k, train_labels, device=device).type(torch.float) # train_targets: [b, kn, k] kernel_matrix = join_fn( train_inputs.unsqueeze(2), # [b, kn, 1, *] train_inputs.unsqueeze(1)) # [b, 1, kn, *] kernel_matrix = kernel_matrix.squeeze(-1) # kernel_matrix: [b, kn, kn] kernel_matrix = kernel_matrix.squeeze(-1) kernel_matrix += regularizer * torch.eye(k * n, device=device) coeff, _ = torch.gesv(train_targets, kernel_matrix) # coeff: [b, kn, k] kernel_train_test = join_fn( train_inputs.unsqueeze(2), # [b, kn, 1, *] test_inputs.unsqueeze(1)) # [b, 1, q, *] kernel_train_test = kernel_train_test.squeeze(-1) # kernel_train_test: [b, kn, q] predictions = torch.bmm(kernel_train_test.transpose(-1, -2), coeff) # predictions: [b, q, k] return predictions
def load_dataset_with_class(args): X = [] Y = [] # Load datasets class by class datasets = args.X.split(',') num_classes = len(datasets) parts = np.zeros([num_classes, 2], dtype=np.int32) count = 0 for i in xrange(num_classes): x = np.matrix(np.load(datasets[i]), dtype=np.float32) y = np.tile(one_hot(num_classes, i), [len(x), 1]).astype(dtype=np.float32) parts[i, 0] = count parts[i, 1] = len(x) X.append(x) Y.append(y) count += len(x) logging.info('Load %d data for class %d from %s' % (len(x), i, datasets[i])) X = np.asarray(np.concatenate(X)) Y = np.asarray(np.concatenate(Y)) return X, Y, parts
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Common definitions for QA """ from util import one_hot LBLS = ["ANS", "O", "EMPTY"] NONE = "EMPTY" LMAP = {k: one_hot(5, i) for i, k in enumerate(LBLS)} NUM = "NNNUMMM" UNK = "UUUNKKK" EMBED_SIZE = 100
def run(): mode = 'gan' experiment_name = mode + '_stride_local_discrimination' filename = 'savestates/cifar_cond_' + experiment_name + '.pickle' in_filename = filename in_filename = None print('experiment_name', experiment_name) print('in_filename', in_filename) print('filename', filename) # Fetch dataset dataset = dp.dataset.CIFAR10() x_train, y_train, x_test, y_test = dataset.arrays(dp_dtypes=True) n_classes = dataset.n_classes # Normalize pixel intensities scaler = dp.StandardScaler() x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) y_train = one_hot(y_train, n_classes).astype(dp.float_) y_test = one_hot(y_test, n_classes).astype(dp.float_) # Setup network if in_filename is None: print('Creating new model') img_shape = x_train.shape[1:] expressions = model_expressions(img_shape) else: print('Starting from %s' % in_filename) with open(in_filename, 'rb') as f: expressions = pickle.load(f) encoder, sampler, generator, discriminator = expressions model = cond_vaegan.ConditionalVAEGAN( encoder=encoder, sampler=sampler, generator=generator, discriminator=discriminator, mode=mode, ) # Prepare network inputs batch_size = 64 train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size, epoch_size=150) # Plotting n_examples = 100 examples = x_test[:n_examples] examples_y = y_test[:n_examples] samples_z = np.random.normal(size=(n_examples, model.sampler.n_hidden)) samples_z = samples_z.astype(dp.float_) samples_y = ((np.arange(n_examples) // 10) % n_classes) samples_y = one_hot(samples_y, n_classes).astype(dp.float_) recon_video = Video('plots/cifar_' + experiment_name + '_reconstruction.mp4') sample_video = Video('plots/cifar_' + experiment_name + '_samples.mp4') sp.misc.imsave('cifar_examples.png', img_tile(dp.misc.to_b01c(examples))) def plot(): examples_z = model.embed(examples, examples_y) examples_recon = model.reconstruct(examples_z, examples_y) examples_recon = clip_range(examples_recon) recon_video.append(img_tile(dp.misc.to_b01c(examples_recon))) samples = clip_range(model.reconstruct(samples_z, samples_y)) sample_video.append(img_tile(dp.misc.to_b01c(samples))) model.setup(**train_input.shapes) # Train network runs = [ # (10, dp.RMSProp(learn_rate=0.08)), # (25, dp.RMSProp(learn_rate=0.12)), # (100, dp.RMSProp(learn_rate=0.1)), (150, dp.RMSProp(learn_rate=0.075)), (150, dp.RMSProp(learn_rate=0.06)), (150, dp.RMSProp(learn_rate=0.05)), (150, dp.RMSProp(learn_rate=0.04)), (25, dp.RMSProp(learn_rate=0.01)), ] try: for n_epochs, learn_rule in runs: if mode == 'vae': vaegan.train(model, train_input, learn_rule, n_epochs, epoch_callback=plot) else: vaegan.margin_train(model, train_input, learn_rule, n_epochs, epoch_callback=plot) except KeyboardInterrupt: pass raw_input('\n\nsave model to %s?\n' % filename) with open(filename, 'wb') as f: expressions = encoder, sampler, generator, discriminator pickle.dump(expressions, f) print('Generating latent space video') walk_video = Video('plots/cifar_' + experiment_name + '_walk.mp4') for z in random_walk(samples_z, 500, step_std=0.15): samples = clip_range(model.reconstruct(z, samples_y)) walk_video.append(img_tile(dp.misc.to_b01c(samples)))
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Common definitions for Lang Detection """ from util import one_hot n_classes = 3 LBLS = [ "Belize Kriol English", "Hindi", "Other", ] NONE = "O" LMAP = {i: one_hot(n_classes, i) for i, k in enumerate(LBLS)} LID = {k: i for i, k in enumerate(LBLS)}
def pretrain(self, X_mnb, y_mnb, params): """ Pretraining (greedy layer-wise) function for the DBN. Successively trains the RBMs of the DBN. :param X_mnb: Training data, split into minibatches (an iterable of minibatches). Each minibatch is a numpy array of shape (N, n_vis) where N is the number of samples in that minibatch, and n_vis is the number of neurons in the visible (lowest) layer of the DBN. :param y_mnb: Training data labels corresponding to X_mnb, split into minibatches (an iterable). Each minibatch is a numpy array of shape (N, 1). :param params: Parameters for training the RBMs. An iterable that has (layer_count - 1) elements. An element can be 'None', in which case the corresponding RBM is not trained (thus allowing for RBM training outside the DBN). A element is a dict or an iterable that can be unpacked into calling arguments to the RBM.train(...) method. """ assert len(params) == len(self.rbms) # pre-train one rbm at a time rbm_train_res = [] rbm_X = X_mnb for rbm_ind, rbm in enumerate(self.rbms): log.info('Training RBM #%d', rbm_ind) # we only train a rbm if we got params for it if params[rbm_ind] is None: log.info('Skipping RBM #%d training, no params', rbm_ind) else: # the last rbm needs labels appended if rbm is self.rbms[-1]: log.info('Appending one-hot labes to data') # iterate through the minibatches for i in range(len(X_mnb)): # get "one hot" matrix for label indices y_one_hot = util.one_hot(y_mnb[i], self.class_count) # append one-hot labels in front of the data rbm_X[i] = np.append(y_one_hot, rbm_X[i], axis=1) # train rbm if isinstance(params[rbm_ind], dict): rbm_train_res.append(rbm.train(rbm_X, **params[rbm_ind])) else: rbm_train_res.append(rbm.train(rbm_X, *params[rbm_ind])) # convert X to input for the next rbm rbm_X = [rbm.hid_given_vis(r)[1] for r in rbm_X] return rbm_train_res
if not os.path.exists('../build/checkpoints/' + target): os.mkdir('../build/checkpoints/' + target) if not os.path.exists('../build/Log'): os.mkdir('../build/Log') if not os.path.exists('../build/Log/' + target): os.mkdir('../build/Log/' + target) # 给出加载数据时间,其实没有太多意义 start = time.time() x, y = LoadData.data_generator_xd(path, input_height, input_width) x = np.expand_dims(x, axis=-1) y = util.one_hot(y, n_classes) X_train, X_valid, y_train, y_valid = train_test_split(x, y, test_size=0.1) end = time.time() print('X_train.shape: {}'.format(X_train.shape)) print('X_valid.shape: {}'.format(X_valid.shape)) print('y_train.shape: {}'.format(y_train.shape)) print('y_valid.shape: {}'.format(y_valid.shape)) print("加载时间:%.2f" % (end - start)) eva_list = ['global_dice'] metrics = [] for item in eva_list:
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Common definitions for NER """ from util import one_hot LBLS = [ "PER", "ORG", "LOC", "MISC", "O", ] NONE = "O" LMAP = {k: one_hot(5,i) for i, k in enumerate(LBLS)} NUM = "NNNUMMM" UNK = "UUUNKKK" EMBED_SIZE = 50
def testModule(self): batch_size = 7 memory_size = 4 num_reads = 11 num_writes = 5 module = addressing.TemporalLinkage(memory_size=memory_size, num_writes=num_writes) prev_link_in = tf.placeholder( tf.float32, (batch_size, num_writes, memory_size, memory_size)) prev_precedence_weights_in = tf.placeholder( tf.float32, (batch_size, num_writes, memory_size)) write_weights_in = tf.placeholder( tf.float32, (batch_size, num_writes, memory_size)) state = addressing.TemporalLinkageState( link=np.zeros([batch_size, num_writes, memory_size, memory_size]), precedence_weights=np.zeros([batch_size, num_writes, memory_size])) calc_state = module( write_weights_in, addressing.TemporalLinkageState( link=prev_link_in, precedence_weights=prev_precedence_weights_in)) with self.test_session() as sess: num_steps = 5 for i in xrange(num_steps): write_weights = np.random.rand(batch_size, num_writes, memory_size) write_weights /= write_weights.sum(2, keepdims=True) + 1 # Simulate (in final steps) link 0-->1 in head 0 and 3-->2 in head 1 if i == num_steps - 2: write_weights[0, 0, :] = util.one_hot(memory_size, 0) write_weights[0, 1, :] = util.one_hot(memory_size, 3) elif i == num_steps - 1: write_weights[0, 0, :] = util.one_hot(memory_size, 1) write_weights[0, 1, :] = util.one_hot(memory_size, 2) state = sess.run(calc_state, feed_dict={ prev_link_in: state.link, prev_precedence_weights_in: state.precedence_weights, write_weights_in: write_weights }) # link should be bounded in range [0, 1] self.assertGreaterEqual(state.link.min(), 0) self.assertLessEqual(state.link.max(), 1) # link diagonal should be zero self.assertAllEqual( state.link[:, :, range(memory_size), range(memory_size)], np.zeros([batch_size, num_writes, memory_size])) # link rows and columns should sum to at most 1 self.assertLessEqual(state.link.sum(2).max(), 1) self.assertLessEqual(state.link.sum(3).max(), 1) # records our transitions in batch 0: head 0: 0->1, and head 1: 3->2 self.assertAllEqual(state.link[0, 0, :, 0], util.one_hot(memory_size, 1)) self.assertAllEqual(state.link[0, 1, :, 3], util.one_hot(memory_size, 2)) # Now test calculation of forward and backward read weights prev_read_weights = np.random.rand(batch_size, num_reads, memory_size) prev_read_weights[0, 5, :] = util.one_hot(memory_size, 0) # read 5, posn 0 prev_read_weights[0, 6, :] = util.one_hot(memory_size, 2) # read 6, posn 2 forward_read_weights = module.directional_read_weights( tf.constant(state.link), tf.constant(prev_read_weights, dtype=tf.float32), forward=True) backward_read_weights = module.directional_read_weights( tf.constant(state.link), tf.constant(prev_read_weights, dtype=tf.float32), forward=False) with self.test_session(): forward_read_weights = forward_read_weights.eval() backward_read_weights = backward_read_weights.eval() # Check directional weights calculated correctly. self.assertAllEqual( forward_read_weights[0, 5, 0, :], # read=5, write=0 util.one_hot(memory_size, 1)) self.assertAllEqual( backward_read_weights[0, 6, 1, :], # read=6, write=1 util.one_hot(memory_size, 3))
def run(): mode = 'vaegan' vae_grad_scale = 0.025 experiment_name = mode + 'scale_%.5f' % vae_grad_scale filename = 'savestates/mnist_' + experiment_name + '.pickle' in_filename = filename in_filename = None print('experiment_name', experiment_name) print('in_filename', in_filename) print('filename', filename) # Fetch dataset dataset = dp.dataset.MNIST() x_train, y_train, x_test, y_test = dataset.arrays(dp_dtypes=True) n_classes = dataset.n_classes img_shape = x_train.shape[1:] # Normalize pixel intensities scaler = dp.UniformScaler() x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) y_train = one_hot(y_train, n_classes).astype(dp.float_) y_test = one_hot(y_test, n_classes).astype(dp.float_) x_train = np.reshape(x_train, (x_train.shape[0], -1)) x_test = np.reshape(x_test, (x_test.shape[0], -1)) # Setup network if in_filename is None: print('Creating new model') expressions = model_expressions(img_shape) else: print('Starting from %s' % in_filename) with open(in_filename, 'rb') as f: expressions = pickle.load(f) encoder, sampler, generator, discriminator = expressions model = cond_vaegan.ConditionalVAEGAN( encoder=encoder, sampler=sampler, generator=generator, discriminator=discriminator, mode=mode, reconstruct_error=expr.nnet.BinaryCrossEntropy(), vae_grad_scale=vae_grad_scale, ) # Prepare network inputs batch_size = 128 train_input = dp.SupervisedInput(x_train, y_train, batch_size=batch_size, epoch_size=250) # Plotting n_examples = 100 examples = x_test[:n_examples] examples_y = y_test[:n_examples] samples_z = np.random.normal(size=(n_examples, model.sampler.n_hidden)) samples_z = samples_z.astype(dp.float_) samples_y = ((np.arange(n_examples) // 10) % n_classes) samples_y = one_hot(samples_y, n_classes).astype(dp.float_) recon_video = Video('plots/mnist_' + experiment_name + '_reconstruction.mp4') sample_video = Video('plots/mnist_' + experiment_name + '_samples.mp4') sp.misc.imsave('plots/mnist_examples.png', img_tile(to_b01c(examples, img_shape))) def plot(): model.phase = 'test' model.sampler.batch_size=100 examples_z = model.embed(examples, examples_y) examples_recon = model.reconstruct(examples_z, examples_y) recon_video.append(img_tile(to_b01c(examples_recon, img_shape))) samples = model.reconstruct(samples_z, samples_y) sample_video.append(img_tile(to_b01c(samples, img_shape))) model.setup(**train_input.shapes) model.phase = 'train' # Train network runs = [ (75, dp.RMSProp(learn_rate=0.075)), (25, dp.RMSProp(learn_rate=0.05)), (5, dp.RMSProp(learn_rate=0.01)), (5, dp.RMSProp(learn_rate=0.005)), ] try: for n_epochs, learn_rule in runs: if mode == 'vae': vaegan.train(model, train_input, learn_rule, n_epochs, epoch_callback=plot) else: vaegan.margin_train(model, train_input, learn_rule, n_epochs, epoch_callback=plot) except KeyboardInterrupt: pass raw_input('\n\nsave model to %s?\n' % filename) with open(filename, 'wb') as f: expressions = encoder, sampler, generator, discriminator pickle.dump(expressions, f) model.phase = 'test' batch_size = 128 model.sampler.batch_size=128 z = [] i = 0 z = model.embed(x_train, y_train) print(z.shape) z_mean = np.mean(z, axis=0) z_std = np.std(z, axis=0) z_cov = np.cov(z.T) print(np.mean(z_mean), np.std(z_mean)) print(np.mean(z_std), np.std(z_std)) print(z_mean.shape, z_std.shape, z_cov.shape) raw_input('\n\ngenerate latent space video?\n') print('Generating latent space video') walk_video = Video('plots/mnist_' + experiment_name + '_walk.mp4') for z in random_walk(samples_z, 500, n_dir_steps=10, mean=z_mean, std=z_cov): samples = model.reconstruct(z, samples_y) walk_video.append(img_tile(to_b01c(samples, img_shape))) print('Generating AdversarialMNIST dataset') _, y_train, _, y_test = dataset.arrays(dp_dtypes=True) n = 0 batch_size = 512 advmnist_size = 1e6 x_advmnist = np.empty((advmnist_size, 28*28)) y_advmnist = np.empty((advmnist_size,)) while n < advmnist_size: samples_z = np.random.multivariate_normal(mean=z_mean, cov=z_cov, size=batch_size) samples_z = samples_z.astype(dp.float_) start_idx = n % len(y_train) stop_idx = (n + batch_size) % len(y_train) if start_idx > stop_idx: samples_y = np.concatenate([y_train[start_idx:], y_train[:stop_idx]]) else: samples_y = y_train[start_idx:stop_idx] y_advmnist[n:n+batch_size] = samples_y[:advmnist_size-n] samples_y = one_hot(samples_y, n_classes).astype(dp.float_) samples = model.reconstruct(samples_z, samples_y) x_advmnist[n:n+batch_size] = samples[:advmnist_size-n] n += batch_size x_train = x_advmnist y_train = y_advmnist import sklearn.neighbors clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1, algorithm='brute', n_jobs=-1) clf.fit(x_train, y_train) print('KNN predict') step = 2500 errors = [] i = 0 while i < len(x_test): print(i) errors.append(clf.predict(x_test[i:i+step]) != y_test[i:i+step]) i += step error = np.mean(errors) print('Test error rate: %.4f' % error) print('DONE ' + experiment_name)