def dropout_forward(x, dropout_param):
"""
Performs the forward pass for (inverted) dropout.
Inputs:
- x: Input data, of any shape
- dropout_param: A dictionary with the following keys:
- p: Dropout parameter. We drop each neuron output with probability p.
- mode: 'test' or 'train'. If the mode is train, then perform dropout;
if the mode is test, then just return the input.
- seed: Seed for the random number generator. Passing seed makes this
function deterministic, which is needed for gradient checking but not in
real networks.
Outputs:
- out: Array of the same shape as x.
- cache: A tuple (dropout_param, mask). In training mode, mask is the dropout
mask that was used to multiply the input; in test mode, mask is None.
"""
p, mode = dropout_param['p'], dropout_param['mode']
if 'seed' in dropout_param:
random.seed(dropout_param['seed'])
mask = None
out = None
if mode == 'train':
mask = np.random.rand(*x.shape) > p #drop mask!
out = x * mask #drop!
elif mode == 'test':
out = x
return out
def dropout(x, prob, mode='train', seed=None):
"""
Performs the forward pass for (inverted) dropout.
Inputs:
- x: Input data, of any shape
- prob: Dropout parameter. We drop each neuron output with probability prob.
- mode: 'test' or 'train'. If the mode is train, then perform dropout;
if the mode is test, then just return the input.
- seed: Seed for the random number generator. Passing seed makes this
function deterministic, which is needed for gradient checking but not in
real networks.
Outputs:
- out: Array of the same shape as x.
"""
if seed is not None:
random.seed(seed)
mask = None
out = None
if mode == 'train':
#TODO: check implementation of compare operator in mxnet?
mask = random.rand(*x.shape) > prob
out = x * mask #drop!
else:
out = x * (1 - prob)
return out
def dropout_forward(x, dropout_param):
"""
Performs the forward pass for (inverted) dropout.
Inputs:
- x: Input data, of any shape
- dropout_param: A dictionary with the following keys:
- p: Dropout parameter. We drop each neuron output with probability p.
- mode: 'test' or 'train'. If the mode is train, then perform dropout;
if the mode is test, then just return the input.
- seed: Seed for the random number generator. Passing seed makes this
function deterministic, which is needed for gradient checking but not in
real networks.
Outputs:
- out: Array of the same shape as x.
- cache: A tuple (dropout_param, mask). In training mode, mask is the dropout
mask that was used to multiply the input; in test mode, mask is None.
"""
p, mode = dropout_param['p'], dropout_param['mode']
if 'seed' in dropout_param:
random.seed(dropout_param['seed'])
mask = None
out = None
if mode == 'train':
mask = np.random.rand(*x.shape) > p #drop mask!
out = x * mask #drop!
elif mode == 'test':
out = x
return out
def dropout(x, prob, mode='train', seed=None):
"""
Performs the forward pass for (inverted) dropout.
Inputs:
- x: Input data, of any shape
- prob: Dropout parameter. We drop each neuron output with probability prob.
- mode: 'test' or 'train'. If the mode is train, then perform dropout;
if the mode is test, then just return the input.
- seed: Seed for the random number generator. Passing seed makes this
function deterministic, which is needed for gradient checking but not in
real networks.
Outputs:
- out: Array of the same shape as x.
"""
if seed is not None:
random.seed(seed)
mask = None
out = None
if mode == 'train':
#TODO: check implementation of compare operator in mxnet?
mask = random.rand(*x.shape) > prob
out = x * mask #drop!
else:
out = x * (1 - prob)
return out
return "".join([chr(np.argmax(c)) for c in one_hot_matrix])
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
"""Loads a text file, and turns each line into an encoded sequence."""
with open(filename) as f:
content = f.readlines()
content = content[:max_lines]
content = [line for line in content if len(line) > 2] # Remove blank lines
seqs = np.zeros((sequence_length, len(content), alphabet_size))
for ix, line in enumerate(content):
padded_line = (line + " " * sequence_length)[:sequence_length]
seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
return seqs
if __name__ == '__main__':
npr.seed(1)
input_size = output_size = 128 # The first 128 ASCII characters are the common ones.
state_size = 40
seq_length = 30
param_scale = 0.01
train_iters = 100
train_inputs = build_dataset(__file__, seq_length, input_size, max_lines=60)
pred_fun, loglike_fun, num_weights = build_lstm(input_size, state_size, output_size)
def print_training_prediction(weights):
print("Training text Predicted text")
logprobs = np.asarray(pred_fun(weights, train_inputs))
for t in range(logprobs.shape[1]):
training_text = one_hot_to_string(train_inputs[:,t,:])
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
"""Loads a text file, and turns each line into an encoded sequence."""
with open(filename) as f:
content = f.readlines()
content = content[:max_lines]
content = [line for line in content if len(line) > 2] # Remove blank lines
seqs = np.zeros((sequence_length, len(content), alphabet_size))
for ix, line in enumerate(content):
padded_line = (line + " " * sequence_length)[:sequence_length]
seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
return seqs
if __name__ == '__main__':
npr.seed(1)
input_size = output_size = 128 # The first 128 ASCII characters are the common ones.
state_size = 40
seq_length = 30
param_scale = 0.01
train_iters = 100
train_inputs = build_dataset(__file__,
seq_length,
input_size,
max_lines=60)
pred_fun, loglike_fun, num_weights = build_lstm(input_size, state_size,
output_size)
def print_training_prediction(weights):
