def trainEpoch(self, inputs, targets, sess, train_data, train_step, error, accuracy, extraFeedDict={}):
train_error = 0.
train_accuracy = 0.
num_batches = train_data.num_batches
for step, (input_batch, target_batch) in enumerate(train_data):
# curOutputs, \
_, batch_error, batch_acc = sess.run(
[
# self.outputs,
train_step, error, accuracy,
],
feed_dict=merge_dicts({inputs: input_batch, targets: target_batch}, extraFeedDict)
)
train_error += batch_error
train_accuracy += batch_acc
# print curOutputs[:5, :3]
train_error /= num_batches
train_accuracy /= num_batches
return train_error, train_accuracy
def trainEpoch(self,
inputs,
targets,
sess,
train_data,
train_step,
error,
accuracy,
extraFeedDict={}):
train_error = 0.
train_accuracy = 0.
num_batches = train_data.num_batches
for step, (input_batch, target_batch) in enumerate(train_data):
sess_out = sess.run([
self.firstRnnOut, self.initialState, self.lastRnnOut,
self.finalState, train_step, error, accuracy
] + self.allRnnOuts,
feed_dict=merge_dicts(
{
inputs: input_batch,
targets: target_batch
}, extraFeedDict))
first_rnn_out, cur_initial_state, last_rnn_out, cur_fin_state, _, batch_error, batch_acc = sess_out[:
7]
all_rnn_outs = sess_out[7:]
if step % 100 == 0:
one_like_initial_state = False
one_like_final_state = False
for counter, rnn_out in enumerate(all_rnn_outs):
cur_check = np.allclose(cur_initial_state, rnn_out)
one_like_initial_state = one_like_initial_state or cur_check
if cur_check:
print "for initial state counter: {}".format(counter)
for counter, rnn_out in enumerate(all_rnn_outs):
cur_check = np.allclose(cur_fin_state, rnn_out)
one_like_final_state = one_like_final_state or cur_check
if cur_check:
print "for final state counter: {}".format(counter)
#assert np.allclose(last_rnn_out, cur_fin_state) # fails
#assert np.allclose(first_rnn_out, cur_initial_state) #fails as well
print "one_like_initial_state"
print one_like_initial_state
print "one_like_final_state"
print one_like_final_state
train_error += batch_error
train_accuracy += batch_acc
train_error /= num_batches
train_accuracy /= num_batches
return train_error, train_accuracy
def validateEpoch(self, state_size, sess, valid_data, extraFeedDict={}):
valid_error = 0.
valid_accuracy = 0.
zeroState = lambda: np.zeros([self.batch_size, state_size])
cur_state = zeroState()
for step, ((input_batch, target_batch),
segmentPartCounter) in enumerate(valid_data):
cur_state, batch_error, batch_acc = sess.run(
[self.outputs, self.error, self.accuracy],
feed_dict=merge_dicts(
{
self.inputs: input_batch,
self.targets: target_batch,
self.init_state: cur_state
}, extraFeedDict))
if (segmentPartCounter + 1) % valid_data.segment_part_count == 0:
cur_state = zeroState()
# only care about the last output of the training error and acc of the rnn
# so include it in if-statement
valid_error += batch_error
valid_accuracy += batch_acc
num_batches = valid_data.num_batches
valid_error /= num_batches
valid_accuracy /= num_batches
return valid_error, valid_accuracy
def createKaggleFile(self, batch_size, data_provider, sess, state_size, verbose=True):
assert data_provider.shuffle_order == False
def zeroState():
return np.zeros([batch_size, state_size])
# chopping off to make batch size fit exactly
length = len(data_provider.inputs) - (len(data_provider.inputs) % batch_size)
if verbose and length == len(data_provider.inputs):
print "data_provider is divided exactly by batch size"
else:
print "data_provider is NOT divided exactly by batch size"
# np.empty
test_predictions = np.zeros((length, data_provider.num_classes))
cur_state = zeroState()
step = 0
# instances_order = data_provider._current_order
for (input_batch, target_batch), segmentPartCounter in data_provider:
batch_predictions, cur_state = sess.run(
[self.softmax, self.outputs],
feed_dict=merge_dicts({self.inputs: input_batch,
self.targets: target_batch,
self.init_state: cur_state,
},
{self.training: False} if hasattr(self, "training") else {}
)
)
if (segmentPartCounter + 1) % data_provider.segment_part_count == 0:
cur_state = zeroState()
test_predictions[step * batch_size:(step + 1) * batch_size, :] = batch_predictions
step += 1
# instances_order, test_predictions
# re order not necessary because shuffle was set to false
assert np.all(test_predictions != 0) # all expected to be something else than zero
create_kaggle_submission_file(test_predictions, output_file=self.filename, overwrite=True)
def trainEpoch(self, state_size, sess, train_data, extraFeedDict={}):
train_error = 0.
train_accuracy = 0.
# train_error = []
# train_accuracy = []
num_batches = train_data.num_batches
zeroState = lambda: np.zeros([self.batch_size, state_size])
cur_state = zeroState()
for step, ((input_batch, target_batch),
segmentPartCounter) in enumerate(train_data):
cur_state, _, batch_error, batch_acc = sess.run(
[self.outputs, self.train_step, self.error, self.accuracy],
feed_dict=merge_dicts(
{
self.inputs: input_batch,
self.targets: target_batch,
self.init_state: cur_state
}, extraFeedDict))
if (segmentPartCounter + 1) % train_data.segment_part_count == 0:
cur_state = zeroState()
# only care about the last output of the training error and acc of the rnn
# so include it in if-statement
train_error += batch_error
train_accuracy += batch_acc
# train_error.append(batch_error)
# train_accuracy.append(batch_acc)
train_error /= num_batches
train_accuracy /= num_batches
# assert len(train_error) == num_batches
# assert len(train_accuracy) == num_batches
# train_error = np.mean(train_error)
# train_accuracy = np.mean(train_accuracy)
return train_error, train_accuracy
def trainAndValidate_selfPaced(graph,
init,
error,
accuracy,
inputs,
targets,
training,
keep_probs,
train_step,
input_keep_prob,
hidden_keep_prob,
train_data,
valid_data,
num_curr_levels,
verbose=True):
if verbose:
print "input_keep_prob: %f" % input_keep_prob
print "hidden_keep_prob: %f" % hidden_keep_prob
keep_prob_dict = constructProbs(keep_probs,
cur_input_prob=input_keep_prob,
cur_hidden_prob=hidden_keep_prob)
with tf.Session(graph=graph, config=config) as sess:
sess.run(init)
dynStats = DynStats()
epoch_counter = 0
for cur_curr_level_index in range(num_curr_levels):
if verbose:
print "=====curr level: %d" % train_data.curriculum_level
valid_error, prevValidError = sys.maxint - 1, sys.maxint #dummy initialization
while valid_error < prevValidError:
(train_error, train_accuracy), runTime = getRunTime(
lambda: trainEpoch(inputs,
targets,
sess,
train_data,
train_step,
error,
accuracy,
extraFeedDict=merge_dicts(
keep_prob_dict, {training: True})))
if (epoch_counter + 1) % 1 == 0:
prevValidError = valid_error
valid_error, valid_accuracy = validateEpoch(
inputs,
targets,
sess,
valid_data,
error,
accuracy,
extraFeedDict={training: False},
keep_prob_keys=keep_probs)
if verbose:
print 'End epoch %02d (%.3f secs): err(train)=%.2f, acc(train)=%.2f, err(valid)=%.2f, acc(valid)=%.2f, ' % (
epoch_counter + 1, runTime, train_error,
train_accuracy, valid_error, valid_accuracy)
dynStats.gatherStats(train_error, train_accuracy, valid_error,
valid_accuracy)
epoch_counter += 1
train_data.updateCurriculumLevel()
if verbose:
print
return dynStats.stats, dynStats.keys
def trainAndValidate(graph,
init,
error,
accuracy,
inputs,
targets,
training,
keep_probs,
train_step,
input_keep_prob,
hidden_keep_prob,
train_data,
valid_data,
epochs=35,
verbose=True):
if verbose:
print "epochs: %d" % epochs
print "input_keep_prob: %f" % input_keep_prob
print "hidden_keep_prob: %f" % hidden_keep_prob
keep_prob_dict = constructProbs(keep_probs,
cur_input_prob=input_keep_prob,
cur_hidden_prob=hidden_keep_prob)
with tf.Session(graph=graph, config=config) as sess:
sess.run(init)
stats, keys = initStats(epochs)
for e in range(epochs):
(train_error,
train_accuracy), runTime = getRunTime(lambda: trainEpoch(
inputs,
targets,
sess,
train_data,
train_step,
error,
accuracy,
extraFeedDict=merge_dicts(keep_prob_dict, {training: True})))
#print 'End epoch %02d (%.3f secs): err(train)=%.2f acc(train)=%.2f' % (e+1, runTime, train_error,train_accuracy)
if (e + 1) % 1 == 0:
valid_error, valid_accuracy = validateEpoch(
inputs,
targets,
sess,
valid_data,
error,
accuracy,
extraFeedDict={training: False},
keep_prob_keys=keep_probs)
#print((' ' * 27) + 'err(valid)={0:.2f} acc(valid)={1:.2f}'.format(valid_error, valid_accuracy))
if verbose:
print 'End epoch %02d (%.3f secs): err(train)=%.2f, acc(train)=%.2f, err(valid)=%.2f, acc(valid)=%.2f, ' % (
e + 1, runTime, train_error, train_accuracy, valid_error,
valid_accuracy)
stats = gatherStats(e, train_error, train_accuracy, valid_error,
valid_accuracy, stats)
if verbose:
print
return stats, keys
def getLogits(self,
batch_size,
data_provider,
sess,
state_size,
extraFeedDict={},
verbose=True):
# batch_size = 97 #factors of 9991: [1, 103, 97, 9991]
total_error = 0.
total_accuracy = 0.
def zeroState():
return np.zeros([batch_size, state_size])
# chopping off to make batch size fit exactly
length = len(
data_provider.inputs) - (len(data_provider.inputs) % batch_size)
if verbose and length == len(data_provider.inputs):
print "data_provider is divided exactly by batch size"
else:
print "data_provider is NOT divided exactly by batch size"
# np.empty
all_logits = np.zeros((length, data_provider.num_classes))
cur_state = zeroState()
step = 0
instances_order = data_provider._current_order
#for step, ((input_batch, target_batch), segmentPartCounter) in enumerate(data_provider):
for (input_batch, target_batch), segmentPartCounter in data_provider:
batch_logits, cur_state, batch_error, batch_acc = sess.run(
[self.logits, self.outputs, self.error, self.accuracy],
feed_dict=merge_dicts(
{
self.inputs: input_batch,
self.targets: target_batch,
self.init_state: cur_state
}, extraFeedDict))
if (segmentPartCounter +
1) % data_provider.segment_part_count == 0:
cur_state = zeroState()
# only care about the last output of the training error and acc of the rnn
# so include it in if-statement
total_error += batch_error
total_accuracy += batch_acc
all_logits[step * len(batch_logits):(step + 1) *
len(batch_logits), :] = batch_logits
assert np.all(instances_order == data_provider._current_order)
step += 1
num_batches = data_provider.num_batches
total_error /= num_batches
total_accuracy /= num_batches
assert np.all(all_logits !=
0) # all logits expected to be something else than zero
logits_dict = OrderedDict(zip(instances_order, all_logits))
return logits_dict, total_error, total_accuracy
