def audit_directory(directory):
only_files = [f for f in listdir(directory) if isfile(join(directory, f))]
audits = [
"{}/{}".format(directory, f) for f in only_files if f[-6:] == ".audit"
]
measurers = [accuracy, BCR]
for audit in audits:
audit_image_filename = audit + ".png"
graph_audit(audit, measurers, audit_image_filename)
for measurer in measurers:
ranked_graph_filename = "{}/{}.png".format(directory,
measurer.__name__)
graph_audits(audits, measurer, ranked_graph_filename)
ranks = rank_audit_files(audits, measurer)
print(measurer.__name__, ranks)
def __call__(self,
data,
output_dir=None,
dump_all=False,
features_to_audit=None):
start_time = datetime.now()
headers, train_set, test_set, response_header, features_to_ignore, correct_types = data
self._audits_data = {
"headers": headers,
"train": train_set,
"test": test_set,
"response": response_header,
"ignore": features_to_ignore,
"types": correct_types,
"full_audit": True if features_to_audit is None else False
}
if self.trained_model == None:
"""
ModelFactories require a `build` method that accepts some training data
with which to train a brand new model. This `build` method should output
a Model object that has a `test` method -- which, when given test data
in the same format as the training data, yields a confusion table detailing
the correct and incorrect predictions of the model.
"""
all_data = train_set + test_set
model_factory = self.ModelFactory(
all_data,
headers,
response_header,
features_to_ignore=features_to_ignore,
options=self.model_options)
if self.trained_model != None:
model_or_factory = self.trained_model
elif not self.RETRAIN_MODEL_PER_REPAIR:
vprint("Training initial model.", self.verbose)
model = model_factory.build(train_set)
# Check the quality of the initial model on verbose runs.
if self.verbose:
print("Calculating original model statistics on test data:")
print("\tTraining Set:")
train_pred_tuples = model.test(train_set)
train_conf_matrix = get_conf_matrix(train_pred_tuples)
print("\t\tConf-Matrix:", train_conf_matrix)
for measurer in self.measurers:
print("\t\t{}: {}".format(measurer.__name__,
measurer(train_conf_matrix)))
print("\tTesting Set:")
test_pred_tuples = model.test(test_set)
test_conf_matrix = get_conf_matrix(test_pred_tuples)
print("\t\tConf-Matrix", test_conf_matrix)
for measurer in self.measurers:
print("\t\t{}: {}".format(measurer.__name__,
measurer(test_conf_matrix)))
model_or_factory = model
else:
model_or_factory = model_factory
# Translate the headers into indexes for the auditor.
audit_indices_to_ignore = [
headers.index(f) for f in features_to_ignore
]
# Don't audit the response feature.
audit_indices_to_ignore.append(headers.index(response_header))
# Prepare the auditor.
auditor = GradientFeatureAuditor(
model_or_factory,
headers,
train_set,
test_set,
repair_steps=self.REPAIR_STEPS,
kdd=self.kdd,
features_to_ignore=audit_indices_to_ignore,
features_to_audit=features_to_audit,
output_dir=output_dir,
dump_all=dump_all)
# Perform the Gradient Feature Audit and dump the audit results into files.
audit_filenames = auditor.audit(verbose=self.verbose)
# Retrieve repaired data from audit
self._audits_data["rep_test"] = auditor._rep_test
ranked_features = []
for measurer in self.measurers:
vprint("Ranking audit files by {}.".format(measurer.__name__),
self.verbose)
#ranked_graph_filename = "{}/{}.png".format(auditor.OUTPUT_DIR, measurer.__name__)
ranks = rank_audit_files(audit_filenames, measurer)
vprint("\t{}".format(ranks), self.verbose)
ranked_features.append((measurer, ranks))
end_time = datetime.now()
# Store a summary of this experiment.
model_id = model_factory.factory_name if self.trained_model == None else "Pretrained"
model_name = model_factory.verbose_factory_name if self.trained_model == None else "Pretrained"
summary = [
"Audit Start Time: {}".format(start_time),
"Audit End Time: {}".format(end_time),
"Retrained Per Repair: {}".format(self.RETRAIN_MODEL_PER_REPAIR),
"Model Factory ID: {}".format(model_id),
"Model Type: {}".format(model_name),
"Non-standard Model Options: {}".format(self.model_options),
"Train Size: {}".format(len(train_set)),
"Test Size: {}".format(len(test_set)),
"Non-standard Ignored Features: {}".format(features_to_ignore),
"Features: {}\n".format(headers)
]
# Print summary
for line in summary:
print(line)
for ranker, ranks in ranked_features:
print("Ranked Features by {}: {}".format(ranker.__name__, ranks))
groups = group_audit_ranks(audit_filenames, ranker)
print("\tApprox. Trend Groups: {}\n".format(groups))
if ranker.__name__ == "accuracy":
self._audits_data["ranks"] = ranks
# Dump all experiment results if opted
if dump_all:
vprint("Dumping original training data.", self.verbose)
# Dump the train data to the log.
train_dump = "{}/original_train_data".format(auditor.OUTPUT_DIR)
with open(train_dump + ".csv", "w") as f:
writer = csv.writer(f)
writer.writerow(headers)
for row in train_set:
writer.writerow(row)
if self.WRITE_ORIGINAL_PREDICTIONS:
# Dump the predictions on the test data.
with open(train_dump + ".predictions", "w") as f:
writer = csv.writer(f)
file_headers = ["Response", "Prediction"]
writer.writerow(file_headers)
for response, guess in train_pred_tuples:
writer.writerow([response, guess])
vprint("Dumping original testing data.", self.verbose)
# Dump the train data to the log.
test_dump = "{}/original_test_data".format(auditor.OUTPUT_DIR)
with open(test_dump + ".csv", "w") as f:
writer = csv.writer(f)
writer.writerow(headers)
for row in test_set:
writer.writerow(row)
if self.WRITE_ORIGINAL_PREDICTIONS:
# Dump the predictions on the test data.
with open(test_dump + ".predictions", "w") as f:
writer = csv.writer(f)
file_headers = ["Response", "Prediction"]
writer.writerow(file_headers)
for response, guess in test_pred_tuples:
writer.writerow([response, guess])
# Graph the audit files.
vprint("Graphing audit files.", self.verbose)
for audit_filename in audit_filenames:
audit_image_filename = audit_filename + ".png"
graph_audit(audit_filename, self.measurers,
audit_image_filename)
# Store a graph of how many predictions change as features are repaired.
vprint("Graphing prediction changes throughout repair.",
self.verbose)
output_image = auditor.OUTPUT_DIR + "/similarity_to_original_predictions.png"
graph_prediction_consistency(auditor.OUTPUT_DIR, output_image)
for measurer in self.measurers:
ranked_graph_filename = "{}/{}.png".format(auditor.OUTPUT_DIR,
measurer.__name__)
graph_audits(audit_filenames, measurer, ranked_graph_filename)
# Store a summary of this experiment to file.
summary_file = "{}/summary.txt".format(auditor.OUTPUT_DIR)
with open(summary_file, "w") as f:
for line in summary:
f.write(line + '\n')
for ranker, ranks in ranked_features:
f.write("Ranked Features by {}: {}\n".format(
ranker.__name__, ranks))
groups = group_audit_ranks(audit_filenames, ranker)
f.write("\tApprox. Trend Groups: {}\n".format(groups))
vprint("Summary file written to: {}\n".format(summary_file),
self.verbose)