def dropout_forward(x, dropout_param): """ Performs the forward pass for (inverted) dropout. Inputs: - x: Input data, of any shape - dropout_param: A dictionary with the following keys: - p: Dropout parameter. We drop each neuron output with probability p. - mode: 'test' or 'train'. If the mode is train, then perform dropout; if the mode is test, then just return the input. - seed: Seed for the random number generator. Passing seed makes this function deterministic, which is needed for gradient checking but not in real networks. Outputs: - out: Array of the same shape as x. - cache: A tuple (dropout_param, mask). In training mode, mask is the dropout mask that was used to multiply the input; in test mode, mask is None. """ p, mode = dropout_param['p'], dropout_param['mode'] if 'seed' in dropout_param: random.seed(dropout_param['seed']) mask = None out = None if mode == 'train': mask = np.random.rand(*x.shape) > p #drop mask! out = x * mask #drop! elif mode == 'test': out = x return out
def dropout(x, prob, mode='train', seed=None): """ Performs the forward pass for (inverted) dropout. Inputs: - x: Input data, of any shape - prob: Dropout parameter. We drop each neuron output with probability prob. - mode: 'test' or 'train'. If the mode is train, then perform dropout; if the mode is test, then just return the input. - seed: Seed for the random number generator. Passing seed makes this function deterministic, which is needed for gradient checking but not in real networks. Outputs: - out: Array of the same shape as x. """ if seed is not None: random.seed(seed) mask = None out = None if mode == 'train': #TODO: check implementation of compare operator in mxnet? mask = random.rand(*x.shape) > prob out = x * mask #drop! else: out = x * (1 - prob) return out
return "".join([chr(np.argmax(c)) for c in one_hot_matrix]) def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1): """Loads a text file, and turns each line into an encoded sequence.""" with open(filename) as f: content = f.readlines() content = content[:max_lines] content = [line for line in content if len(line) > 2] # Remove blank lines seqs = np.zeros((sequence_length, len(content), alphabet_size)) for ix, line in enumerate(content): padded_line = (line + " " * sequence_length)[:sequence_length] seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size) return seqs if __name__ == '__main__': npr.seed(1) input_size = output_size = 128 # The first 128 ASCII characters are the common ones. state_size = 40 seq_length = 30 param_scale = 0.01 train_iters = 100 train_inputs = build_dataset(__file__, seq_length, input_size, max_lines=60) pred_fun, loglike_fun, num_weights = build_lstm(input_size, state_size, output_size) def print_training_prediction(weights): print("Training text Predicted text") logprobs = np.asarray(pred_fun(weights, train_inputs)) for t in range(logprobs.shape[1]): training_text = one_hot_to_string(train_inputs[:,t,:])
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1): """Loads a text file, and turns each line into an encoded sequence.""" with open(filename) as f: content = f.readlines() content = content[:max_lines] content = [line for line in content if len(line) > 2] # Remove blank lines seqs = np.zeros((sequence_length, len(content), alphabet_size)) for ix, line in enumerate(content): padded_line = (line + " " * sequence_length)[:sequence_length] seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size) return seqs if __name__ == '__main__': npr.seed(1) input_size = output_size = 128 # The first 128 ASCII characters are the common ones. state_size = 40 seq_length = 30 param_scale = 0.01 train_iters = 100 train_inputs = build_dataset(__file__, seq_length, input_size, max_lines=60) pred_fun, loglike_fun, num_weights = build_lstm(input_size, state_size, output_size) def print_training_prediction(weights):