def testOnDataset(self): dataset = GeneratedExpressionDataset( '../data/expressions_positive_integer_answer_shallow/train.txt', '../data/expressions_positive_integer_answer_shallow/test.txt', preload=False, finishExpressions=True) _, _, _, train_expressions = \ dataset.loadFile(dataset.sources[dataset.TRAIN], location_index=dataset.TRAIN, file_length=dataset.lengths[dataset.TRAIN]) print("Now querying %d expressions..." % (len(train_expressions))) profiler.start('querying') for exp in train_expressions: lookup = dataset.expressionsByPrefix.get(exp[:5], len(exp[:5])) profiler.stop('querying') profiler.profile()
onehot_indices[:input_sample_length]] = 1.0 data[i, range(input_sample_length, input_length), eos_index] = 1.0 targets[i, range(input_sample_length), onehot_indices[:input_sample_length]] = 1.0 targets[i, range(input_sample_length, input_length), eos_index] = 1.0 labels = np.argmax(targets, 2) print("Training model...") # Train model for i in range(repetitions): print("Batch %d (repetition %d of %d, dataset 1 of 1)" % (i + 1, i + 1, repetitions)) # Shuffle data indices inds = range(len(data)) #np.random.shuffle(inds); # Train one repetition of minibatches profiler.start('training') for j in range(0, len(inds), minibatch_size): train_data = np.swapaxes(data[j:j + minibatch_size], 0, 1) train_targets = np.swapaxes(targets[j:j + minibatch_size], 0, 1) sgd(train_data, train_targets, learning_rate) profiler.stop('training') # Perform intermediate testing test() profiler.profile()
def test(model, dataset, parameters, max_length, print_samples=False, sample_size=False, returnTestSamples=False): # Test printF("Testing...", experimentId, currentIteration) total = dataset.lengths[dataset.TEST] printing_interval = 1000 if (parameters['max_dataset_size'] is not False): printing_interval = 100 elif (sample_size != False): total = sample_size # Set up statistics stats = set_up_statistics(dataset.output_dim, model.n_max_digits, dataset.oneHot.keys()) total_labels_used = {} # Predict printed_samples = False totalError = 0.0 k = 0 testSamples = [] while k < total: # Get data from batch test_data, test_targets, test_labels, test_expressions, \ nrSamples, health = get_batch(1, dataset, model, dataset_data, label_index, debug=parameters['debug']) predictions, other = model.predict(test_data, test_targets, nrSamples=nrSamples) totalError += other['summed_error'] profiler.start("test batch stats") stats, _, _ = model.batch_statistics(stats, predictions, test_labels, None, other, nrSamples, dataset, None, None, parameters, data=test_data) for j in range(nrSamples): if (test_labels[j] not in total_labels_used): total_labels_used[test_labels[j]] = True # Save predictions to testSamples if (returnTestSamples): strData = map( lambda x: dataset.findSymbol[x], np.argmax(test_data[j, :, :model.data_dim], len(test_data.shape) - 2)) strPrediction = dataset.findSymbol[predictions[j]] testSamples.append((strData, strPrediction)) # Print samples if (print_samples and not printed_samples): for i in range(nrSamples): prefix = "# " printF( prefix + "Data 1: %s" % "".join( (map(lambda x: dataset.findSymbol[x], np.argmax(test_data[i], len(test_data.shape) - 2)))), experimentId, currentIteration) printF( prefix + "Prediction 1: %s" % dataset.findSymbol[predictions[i]], experimentId, currentIteration) printF( prefix + "Used label 1: %s" % dataset.findSymbol[test_labels[i]], experimentId, currentIteration) printed_samples = True if (stats['prediction_size'] % printing_interval == 0): printF("# %d / %d" % (stats['prediction_size'], total), experimentId, currentIteration) profiler.stop("test batch stats") k += nrSamples profiler.profile() print("Test: %d" % k) printF("Total testing error: %.2f" % totalError, experimentId, currentIteration) printF("Mean testing error: %.8f" % (totalError / float(k)), experimentId, currentIteration) stats = model.total_statistics(stats, dataset, parameters, total_labels_used=total_labels_used, digits=False) print_stats(stats, parameters) if (returnTestSamples): return stats, testSamples else: return stats
# Print repetition progress and save to raw results file printF("Batch %d (repetition %d of %d, dataset 1 of 1) (samples processed after batch: %d)" % \ (r+1,r+1,parameters['repetitions'],(r+1)*repetition_size), experimentId, currentIteration) currentIteration = r + 1 currentDataset = 1 # Train model per minibatch k = 0 printedProgress = -1 while k < repetition_size: profiler.start('train batch') profiler.start('get train batch') data, target, test_labels, target_expressions, nrSamples, health = \ get_batch(0, dataset, model, dataset_data, label_index, debug=parameters['debug']) profiler.stop('get train batch') # Run training profiler.start('train sgd') outputs = model.sgd(dataset, data, target, parameters['learning_rate'], nrSamples=nrSamples) total_error += outputs[1] profiler.stop('train sgd') # Print batch progress if ((k+model.minibatch_size) % (model.minibatch_size*4) < model.minibatch_size and \ (k+model.minibatch_size) / (model.minibatch_size*4) > printedProgress): printedProgress = (k + model.minibatch_size) / (
def test(model, dataset, parameters, max_length, print_samples=False, sample_size=False, returnTestSamples=False): # Test print("Testing...") total = dataset.lengths[dataset.TEST] printing_interval = 1000 if (parameters['max_testing_size'] is not False): total = parameters['max_testing_size'] printing_interval = 100 elif (sample_size != False): total = sample_size # Set up statistics stats = set_up_statistics(dataset.output_dim, model.n_max_digits) total_labels_used = {k: 0 for k in range(30)} # Predict printed_samples = False totalError = 0.0 k = 0 testSamples = [] while k < total: # Get data from batch test_data, test_targets, _, test_expressions, \ nrSamples = get_batch(False, dataset, model, debug=parameters['debug']) predictions, other = model.predict(test_data, test_targets, nrSamples=nrSamples) totalError += other['error'] profiler.start("test batch stats") stats, _ = model.batch_statistics( stats, predictions, test_expressions, other, nrSamples, dataset, testInDataset=parameters['test_in_dataset']) for j in range(nrSamples): total_labels_used[test_expressions[j]] = True # Save predictions to testSamples if (returnTestSamples): strData = map( lambda x: dataset.findSymbol[x], np.argmax(test_targets[j, :, :model.data_dim], len(test_targets.shape) - 2)) strPrediction = map(lambda x: dataset.findSymbol[x], predictions[j]) testSamples.append((strData, strPrediction)) # Print samples if (print_samples and not printed_samples): for i in range(nrSamples): prefix = "" print(prefix + "Data 1: %s" % "".join((map( lambda x: dataset.findSymbol[x], np.argmax(test_targets[i, :, :model.data_dim], len(test_data.shape) - 2))))) print(prefix + "Prediction 1: %s" % "".join( map(lambda x: dataset.findSymbol[x], predictions[i]))) print(prefix + "Used label 1: %s" % test_expressions[i]) printed_samples = True if (stats['prediction_size'] % printing_interval == 0): print("# %d / %d" % (stats['prediction_size'], total)) profiler.stop("test batch stats") k += nrSamples profiler.profile() print("Total testing error: %.2f" % totalError) stats = model.total_statistics(stats, total_labels_used=total_labels_used) print_stats(stats, parameters) if (returnTestSamples): return stats, testSamples else: return stats
def testExists(self): params = [ '--finish_subsystems', 'True', '--only_cause_expression', '1', '--dataset', '../data/subsystems_shallow_simple_topcause', "--sample_testing_size", "10000", "--n_max_digits", "17", "--intervention_base_offset", "0", "--intervention_range", "17", "--nesterov_optimizer", "True", "--decoder", "True", "--learning_rate", "0.005", "--hidden_dim", "256" ] params = processCommandLineArguments(params) datasets, _ = constructModels(params, 1234, {}) dataset = datasets[0] storage = dataset.expressionsByPrefix expressions = [ "(3-9)*(0-3)=18", "(4-7)+(6*5)=27", "(0/6)+(2*8)=16", "(1-4)+(3+6)=6", "(6+0)+(0-1)=5" ] for i, expr in enumerate(expressions): self.assertEqual( storage.exists(expr), True, "(exists) Failing exists lookup for sample %d" % i) _, _, _, _, branch = storage.get(expr[:4], alsoGetStructure=True) closest, _, _, _ = branch.get_closest(expr[4:]) self.assertNotEqual( closest, False, "(exists) Branch-based lookup failed with False for sample %d: %s" % (i, closest)) self.assertEqual( closest, expr, "(exists) Failing branch-based lookup for sample %d: %s" % (i, closest)) # Apply mutations and test if both methods get the same new label for n in range(20): intervention_location = np.random.randint(0, len(expr)) new_symbol = np.random.randint(dataset.data_dim) new_expression = expr[:intervention_location] + dataset.findSymbol[ new_symbol] + expr[intervention_location + 1:] print("Old: %s\tNew: %s" % (expr, new_expression)) _, _, valids, _, branch = storage.get( new_expression[:intervention_location + 1], alsoGetStructure=True) if (new_expression not in valids and len(valids) > 0): # Old method: compare all profiler.start('old') nearest = -1 nearest_score = 100000 for j, nexpr in enumerate(valids): score = string_difference(new_expression, nexpr) if (score < nearest_score): nearest = j nearest_score = score closest_old = valids[nearest] profiler.stop('old') profiler.start('new') # New method: closest_new, _, _, _ = branch.get_closest( new_expression[intervention_location + 1:]) profiler.stop('new') if (closest_old != closest_new): print( "(exists) Intervened closest do not match for sample %d: loc %d / orig %s / int %s / old %s / new %s" % (i, intervention_location, expr, new_expression, closest_old, closest_new)) # self.assertEqual(closest_old, closest_new, # "(exists) Intervened closest do not match for sample %d: loc %d / orig %s / int %s / old %s / new %s" % # (i, intervention_location, expr, new_expression, closest_old, closest_new)); profiler.profile()
def test(model, dataset_data, parameters, print_samples=False, sample_size=False): # Test print("Testing...") total = parameters['test_size'] * len(dataset_data) printing_interval = 1000 if (parameters['max_testing_size'] is not False): total = parameters['max_testing_size'] printing_interval = 100 elif (sample_size != False): total = sample_size # Set up statistics stats = set_up_linear_stats(parameters) # Predict printed_samples = False totalError = 0.0 k = 0 while k < total: # Get data from batch test_data = get_batch(False, dataset_data, model) predictions, other = model.predict( test_data, test_data, nrSamples=parameters['minibatch_size']) totalError += other['error'] if (parameters['only_cause_expression']): prediction_1 = predictions predictions = [predictions] else: prediction_1 = predictions[0] prediction_2 = predictions[1] profiler.start("test batch stats") stats = model.batch_statistics( stats, predictions, test_expressions, interventionLocations, other, nrSamples, dataset, test_expressions, topcause=topcause or parameters['bothcause'], # If bothcause then topcause = 1 testInDataset=parameters['test_in_dataset'], bothcause=parameters['bothcause']) # Print samples if (print_samples and not printed_samples): for i in range(nrSamples): prefix = "# " print(prefix + "Data 1: %s" % str(test_data[:, :, 0])) print(prefix + "Prediction 1: %s" % str(prediction_1[i])) print(prefix + "Data 2: %s" % str(test_data[:, :, 1])) print(prefix + "Prediction 2: %s" % str(prediction_2[i])) printed_samples = True if (stats['prediction_size'] % printing_interval == 0): print("# %d / %d" % (stats['prediction_size'], total)) profiler.stop("test batch stats") k += nrSamples profiler.profile() print("Total testing error: %.2f" % totalError) print_stats(stats, parameters) return stats