def testOnDataset(self):
dataset = GeneratedExpressionDataset(
'../data/expressions_positive_integer_answer_shallow/train.txt',
'../data/expressions_positive_integer_answer_shallow/test.txt',
preload=False,
finishExpressions=True)
_, _, _, train_expressions = \
dataset.loadFile(dataset.sources[dataset.TRAIN],
location_index=dataset.TRAIN,
file_length=dataset.lengths[dataset.TRAIN])
print("Now querying %d expressions..." % (len(train_expressions)))
profiler.start('querying')
for exp in train_expressions:
lookup = dataset.expressionsByPrefix.get(exp[:5], len(exp[:5]))
profiler.stop('querying')
profiler.profile()
onehot_indices[:input_sample_length]] = 1.0
data[i, range(input_sample_length, input_length), eos_index] = 1.0
targets[i,
range(input_sample_length),
onehot_indices[:input_sample_length]] = 1.0
targets[i, range(input_sample_length, input_length), eos_index] = 1.0
labels = np.argmax(targets, 2)
print("Training model...")
# Train model
for i in range(repetitions):
print("Batch %d (repetition %d of %d, dataset 1 of 1)" %
(i + 1, i + 1, repetitions))
# Shuffle data indices
inds = range(len(data))
#np.random.shuffle(inds);
# Train one repetition of minibatches
profiler.start('training')
for j in range(0, len(inds), minibatch_size):
train_data = np.swapaxes(data[j:j + minibatch_size], 0, 1)
train_targets = np.swapaxes(targets[j:j + minibatch_size], 0, 1)
sgd(train_data, train_targets, learning_rate)
profiler.stop('training')
# Perform intermediate testing
test()
profiler.profile()
def test(model,
dataset,
parameters,
max_length,
print_samples=False,
sample_size=False,
returnTestSamples=False):
# Test
printF("Testing...", experimentId, currentIteration)
total = dataset.lengths[dataset.TEST]
printing_interval = 1000
if (parameters['max_dataset_size'] is not False):
printing_interval = 100
elif (sample_size != False):
total = sample_size
# Set up statistics
stats = set_up_statistics(dataset.output_dim, model.n_max_digits,
dataset.oneHot.keys())
total_labels_used = {}
# Predict
printed_samples = False
totalError = 0.0
k = 0
testSamples = []
while k < total:
# Get data from batch
test_data, test_targets, test_labels, test_expressions, \
nrSamples, health = get_batch(1, dataset, model, dataset_data, label_index, debug=parameters['debug'])
predictions, other = model.predict(test_data,
test_targets,
nrSamples=nrSamples)
totalError += other['summed_error']
profiler.start("test batch stats")
stats, _, _ = model.batch_statistics(stats,
predictions,
test_labels,
None,
other,
nrSamples,
dataset,
None,
None,
parameters,
data=test_data)
for j in range(nrSamples):
if (test_labels[j] not in total_labels_used):
total_labels_used[test_labels[j]] = True
# Save predictions to testSamples
if (returnTestSamples):
strData = map(
lambda x: dataset.findSymbol[x],
np.argmax(test_data[j, :, :model.data_dim],
len(test_data.shape) - 2))
strPrediction = dataset.findSymbol[predictions[j]]
testSamples.append((strData, strPrediction))
# Print samples
if (print_samples and not printed_samples):
for i in range(nrSamples):
prefix = "# "
printF(
prefix + "Data 1: %s" % "".join(
(map(lambda x: dataset.findSymbol[x],
np.argmax(test_data[i],
len(test_data.shape) - 2)))),
experimentId, currentIteration)
printF(
prefix +
"Prediction 1: %s" % dataset.findSymbol[predictions[i]],
experimentId, currentIteration)
printF(
prefix +
"Used label 1: %s" % dataset.findSymbol[test_labels[i]],
experimentId, currentIteration)
printed_samples = True
if (stats['prediction_size'] % printing_interval == 0):
printF("# %d / %d" % (stats['prediction_size'], total),
experimentId, currentIteration)
profiler.stop("test batch stats")
k += nrSamples
profiler.profile()
print("Test: %d" % k)
printF("Total testing error: %.2f" % totalError, experimentId,
currentIteration)
printF("Mean testing error: %.8f" % (totalError / float(k)), experimentId,
currentIteration)
stats = model.total_statistics(stats,
dataset,
parameters,
total_labels_used=total_labels_used,
digits=False)
print_stats(stats, parameters)
if (returnTestSamples):
return stats, testSamples
else:
return stats
# Print repetition progress and save to raw results file
printF("Batch %d (repetition %d of %d, dataset 1 of 1) (samples processed after batch: %d)" % \
(r+1,r+1,parameters['repetitions'],(r+1)*repetition_size), experimentId, currentIteration)
currentIteration = r + 1
currentDataset = 1
# Train model per minibatch
k = 0
printedProgress = -1
while k < repetition_size:
profiler.start('train batch')
profiler.start('get train batch')
data, target, test_labels, target_expressions, nrSamples, health = \
get_batch(0, dataset, model, dataset_data, label_index,
debug=parameters['debug'])
profiler.stop('get train batch')
# Run training
profiler.start('train sgd')
outputs = model.sgd(dataset,
data,
target,
parameters['learning_rate'],
nrSamples=nrSamples)
total_error += outputs[1]
profiler.stop('train sgd')
# Print batch progress
if ((k+model.minibatch_size) % (model.minibatch_size*4) < model.minibatch_size and \
(k+model.minibatch_size) / (model.minibatch_size*4) > printedProgress):
printedProgress = (k + model.minibatch_size) / (
def test(model,
dataset,
parameters,
max_length,
print_samples=False,
sample_size=False,
returnTestSamples=False):
# Test
print("Testing...")
total = dataset.lengths[dataset.TEST]
printing_interval = 1000
if (parameters['max_testing_size'] is not False):
total = parameters['max_testing_size']
printing_interval = 100
elif (sample_size != False):
total = sample_size
# Set up statistics
stats = set_up_statistics(dataset.output_dim, model.n_max_digits)
total_labels_used = {k: 0
for k in range(30)}
# Predict
printed_samples = False
totalError = 0.0
k = 0
testSamples = []
while k < total:
# Get data from batch
test_data, test_targets, _, test_expressions, \
nrSamples = get_batch(False, dataset, model, debug=parameters['debug'])
predictions, other = model.predict(test_data,
test_targets,
nrSamples=nrSamples)
totalError += other['error']
profiler.start("test batch stats")
stats, _ = model.batch_statistics(
stats,
predictions,
test_expressions,
other,
nrSamples,
dataset,
testInDataset=parameters['test_in_dataset'])
for j in range(nrSamples):
total_labels_used[test_expressions[j]] = True
# Save predictions to testSamples
if (returnTestSamples):
strData = map(
lambda x: dataset.findSymbol[x],
np.argmax(test_targets[j, :, :model.data_dim],
len(test_targets.shape) - 2))
strPrediction = map(lambda x: dataset.findSymbol[x],
predictions[j])
testSamples.append((strData, strPrediction))
# Print samples
if (print_samples and not printed_samples):
for i in range(nrSamples):
prefix = ""
print(prefix + "Data 1: %s" % "".join((map(
lambda x: dataset.findSymbol[x],
np.argmax(test_targets[i, :, :model.data_dim],
len(test_data.shape) - 2)))))
print(prefix + "Prediction 1: %s" % "".join(
map(lambda x: dataset.findSymbol[x], predictions[i])))
print(prefix + "Used label 1: %s" % test_expressions[i])
printed_samples = True
if (stats['prediction_size'] % printing_interval == 0):
print("# %d / %d" % (stats['prediction_size'], total))
profiler.stop("test batch stats")
k += nrSamples
profiler.profile()
print("Total testing error: %.2f" % totalError)
stats = model.total_statistics(stats, total_labels_used=total_labels_used)
print_stats(stats, parameters)
if (returnTestSamples):
return stats, testSamples
else:
return stats
def testExists(self):
params = [
'--finish_subsystems', 'True', '--only_cause_expression', '1',
'--dataset', '../data/subsystems_shallow_simple_topcause',
"--sample_testing_size", "10000", "--n_max_digits", "17",
"--intervention_base_offset", "0", "--intervention_range", "17",
"--nesterov_optimizer", "True", "--decoder", "True",
"--learning_rate", "0.005", "--hidden_dim", "256"
]
params = processCommandLineArguments(params)
datasets, _ = constructModels(params, 1234, {})
dataset = datasets[0]
storage = dataset.expressionsByPrefix
expressions = [
"(3-9)*(0-3)=18", "(4-7)+(6*5)=27", "(0/6)+(2*8)=16",
"(1-4)+(3+6)=6", "(6+0)+(0-1)=5"
]
for i, expr in enumerate(expressions):
self.assertEqual(
storage.exists(expr), True,
"(exists) Failing exists lookup for sample %d" % i)
_, _, _, _, branch = storage.get(expr[:4], alsoGetStructure=True)
closest, _, _, _ = branch.get_closest(expr[4:])
self.assertNotEqual(
closest, False,
"(exists) Branch-based lookup failed with False for sample %d: %s"
% (i, closest))
self.assertEqual(
closest, expr,
"(exists) Failing branch-based lookup for sample %d: %s" %
(i, closest))
# Apply mutations and test if both methods get the same new label
for n in range(20):
intervention_location = np.random.randint(0, len(expr))
new_symbol = np.random.randint(dataset.data_dim)
new_expression = expr[:intervention_location] + dataset.findSymbol[
new_symbol] + expr[intervention_location + 1:]
print("Old: %s\tNew: %s" % (expr, new_expression))
_, _, valids, _, branch = storage.get(
new_expression[:intervention_location + 1],
alsoGetStructure=True)
if (new_expression not in valids and len(valids) > 0):
# Old method: compare all
profiler.start('old')
nearest = -1
nearest_score = 100000
for j, nexpr in enumerate(valids):
score = string_difference(new_expression, nexpr)
if (score < nearest_score):
nearest = j
nearest_score = score
closest_old = valids[nearest]
profiler.stop('old')
profiler.start('new')
# New method:
closest_new, _, _, _ = branch.get_closest(
new_expression[intervention_location + 1:])
profiler.stop('new')
if (closest_old != closest_new):
print(
"(exists) Intervened closest do not match for sample %d: loc %d / orig %s / int %s / old %s / new %s"
% (i, intervention_location, expr, new_expression,
closest_old, closest_new))
# self.assertEqual(closest_old, closest_new,
# "(exists) Intervened closest do not match for sample %d: loc %d / orig %s / int %s / old %s / new %s" %
# (i, intervention_location, expr, new_expression, closest_old, closest_new));
profiler.profile()
def test(model,
dataset_data,
parameters,
print_samples=False,
sample_size=False):
# Test
print("Testing...")
total = parameters['test_size'] * len(dataset_data)
printing_interval = 1000
if (parameters['max_testing_size'] is not False):
total = parameters['max_testing_size']
printing_interval = 100
elif (sample_size != False):
total = sample_size
# Set up statistics
stats = set_up_linear_stats(parameters)
# Predict
printed_samples = False
totalError = 0.0
k = 0
while k < total:
# Get data from batch
test_data = get_batch(False, dataset_data, model)
predictions, other = model.predict(
test_data, test_data, nrSamples=parameters['minibatch_size'])
totalError += other['error']
if (parameters['only_cause_expression']):
prediction_1 = predictions
predictions = [predictions]
else:
prediction_1 = predictions[0]
prediction_2 = predictions[1]
profiler.start("test batch stats")
stats = model.batch_statistics(
stats,
predictions,
test_expressions,
interventionLocations,
other,
nrSamples,
dataset,
test_expressions,
topcause=topcause
or parameters['bothcause'], # If bothcause then topcause = 1
testInDataset=parameters['test_in_dataset'],
bothcause=parameters['bothcause'])
# Print samples
if (print_samples and not printed_samples):
for i in range(nrSamples):
prefix = "# "
print(prefix + "Data 1: %s" % str(test_data[:, :, 0]))
print(prefix + "Prediction 1: %s" % str(prediction_1[i]))
print(prefix + "Data 2: %s" % str(test_data[:, :, 1]))
print(prefix + "Prediction 2: %s" % str(prediction_2[i]))
printed_samples = True
if (stats['prediction_size'] % printing_interval == 0):
print("# %d / %d" % (stats['prediction_size'], total))
profiler.stop("test batch stats")
k += nrSamples
profiler.profile()
print("Total testing error: %.2f" % totalError)
print_stats(stats, parameters)
return stats
