コード例 #1
0
ファイル: accuracy.py プロジェクト: Tank-ACiD/PeopleRL
def crossValidation(initialTrainingDataSet, allConditions, k):
    listOfFolds = makelistOfFolds(initialTrainingDataSet, k)
    kAccuracies = []
    for i in range(k):
        trainingData, testingData = getIthTrainingAndTestingData(listOfFolds, i)
        adtClassifer = adt.classifer(trainingData, allConditions)
        testingDataOutput = adt.evaluate(testingData, adtClassifer)
        kAccuracies.append(getAccuracy(testingDataOutput, initialTrainingDataSet))
    return computeFinalAccuracy(kAccuracies)
def main():
	if len(sys.argv) != 3:
		Usage()
		return

	testing_data_file = sys.argv[1]
	adtree_file = sys.argv[2]

	if os.path.exists(testing_data_file) and os.path.exists(adtree_file):
		adtree = adt_infrastructure.ReCreateADTree(adtree_file, adt)

		(records, keys) = adt_infrastructure.GetTestingData(testing_data_file)
		testingDataSet = {}

		#    n!
		#------------- = number of unique combinations
		#(n - r)! * r!

		num_records = len(records)
		print " ***** Number of records     : %s" % num_records

		print " Computing number of combinations..."
		num_combinations = math.factorial(num_records) / ( math.factorial(num_records - 2) * math.factorial(2) )
		print " ***** Number of combinations: %s" % num_combinations

		# test unique combinations for record linkage (duplicates)
		combination_pairs = []
		total_processed = 0

		number_of_matches = 0
		number_of_nonmatches = 0

		for combination_1 in range(num_records):
			total_processed = total_processed + (num_records - combination_1) - 1
			percent_complete = float(total_processed) / float(num_combinations) * 100
			print " ***** %.2f%% complete ----- total processed: %s" % (percent_complete, total_processed)
			for combination_2 in range(combination_1+1, num_records):
				tempDictionary = {}

				for key in keys[1:]:
					levenshtein_distance = levenshtein.Compute_Levenshtein_Distance(records[combination_1][key], records[combination_2][key])
					tempDictionary[key] = levenshtein_distance

				combination_pair = str(combination_1) + '-' + str(combination_2)
				combination_pairs.append(combination_pair)
				testingDataSet[combination_pair] = tempDictionary

		outputDatabase = adt.evaluate(testingDataSet, adtree)

		for combination_pair in outputDatabase.keys():
			print combination_pair
	else:
		Usage()
		return
コード例 #3
0
def main():
	global trainingDataSet
	
	if len(sys.argv) != 3:
		Usage()
		return
	
	testing_data_file = sys.argv[1]
	adtree_file = sys.argv[2]
	
	if os.path.exists(testing_data_file) and os.path.exists(adtree_file):
		adtree = ReCreateADTree(adtree_file)
		
		(records, keys) = adt_infrastructure.BuildDataStructure(testing_data_file)
		trainingDataSet = {}
		
		#    n!
		#------------- = number of unique combinations
		#(n - r)! * r!
		
		num_records = len(records)
		print " ***** Number of records     : %s" % num_records
		
		print " Computing number of combinations..."
		num_combinations = math.factorial(num_records) / ( math.factorial(num_records - 2) * math.factorial(2) )
		print " ***** Number of combinations: %s" % num_combinations
		
		# test unique combinations for record linkage (duplicates)
		combination_pairs = []
		total_processed = 0
		for combination_1 in range(num_records):
			total_processed = total_processed + (num_records - combination_1) - 1
			percent_complete = float(total_processed) / float(num_combinations) * 100
			print " ***** %.2f%% complete ----- total processed: %s" % (percent_complete, total_processed)
			for combination_2 in range(combination_1+1, num_records):
				tempDictionary = {}
				if levenshtein.Compute_Levenshtein_Distance(records[combination_1]["ID"], records[combination_2]["ID"]) == 0:
					tempDictionary['class'] = 'same'
				else:
					tempDictionary['class'] = 'different'
				for key in keys[1:]:					
					levenshtein_distance = levenshtein.Compute_Levenshtein_Distance(records[combination_1][key], records[combination_2][key])
					tempDictionary[key] = levenshtein_distance
				
				combination_pair = str(combination_1) + '-' + str(combination_2)
				combination_pairs.append(combination_pair)
				trainingDataSet[combination_pair] = tempDictionary
		
		outputDatabase = adt.evaluate(trainingDataSet, adtree)
		
		correct_classifications = 0
		incorrect_classifications = 0
		
		false_positives = 0
		false_negatives = 0
		for combination_pair in outputDatabase.keys():
			pair = combination_pair.split("-")
			if records[int(pair[0])]["ID"] == records[int(pair[1])]["ID"] and outputDatabase[combination_pair]["classification"] == "same":
				correct_classifications = correct_classifications + 1
			elif records[int(pair[0])]["ID"] != records[int(pair[1])]["ID"] and outputDatabase[combination_pair]["classification"] == "different":
				correct_classifications = correct_classifications + 1
			else:
				if outputDatabase[combination_pair]["classification"] == "same":
					print "False Positive:"
					false_positives = false_positives + 1
				elif outputDatabase[combination_pair]["classification"] == "different":
					print "False Negative:"
					false_negatives = false_negatives + 1
				
				print "   record[%s][ID] = %s --- address: %s" % (pair[0], records[int(pair[0])]["ID"], records[int(pair[0])]["FullAddres"])
				print "   record[%s][ID] = %s --- address: %s" % (pair[1], records[int(pair[1])]["ID"], records[int(pair[1])]["FullAddres"])
				print "   classification = %s" % outputDatabase[combination_pair]["classification"]
				
				incorrect_classifications = incorrect_classifications + 1
		
		print ""
		print "Number of records        : %s" % num_records
		print "Number of combinations   : %s" % num_combinations
		
		print ""
		total_classifications = correct_classifications + incorrect_classifications
		print "Correct classifications  : %s" % correct_classifications
		print "Incorrect classifications: %s" % incorrect_classifications
		classification_accuracy_ratio = float(correct_classifications) / float(total_classifications)
		classification_accuracy_percentage = classification_accuracy_ratio * 100.0
		print "Classification Accuracy  : %.2f%%" % classification_accuracy_percentage
		
		print ""
		print "False Positives: %s" % false_positives
		print "False Negatives: %s" % false_negatives
		
	else:
		Usage()
		return
コード例 #4
0
def main():
	global trainingDataSet
	for i, arg in enumerate(sys.argv):
		print "arg: %s: %s" % (i, arg) 
	argc = len(sys.argv)
	
	if argc != 4:
		Usage()
		return
	
	input_file = sys.argv[1]
	conditions_file = sys.argv[2]
	output_file = sys.argv[3]
	
	if os.path.exists(input_file):
		(records, keys) = adt_infrastructure.BuildDataStructure(input_file)
		trainingDataSet = {}
		
		#    n!
		#------------- = number of unique combinations
		#(n - r)! * r!
		
		num_records = len(records)
		print " ***** Number of records     : %s" % num_records
		
		print " Computing number of combinations..."
		num_combinations = math.factorial(num_records) / ( math.factorial(num_records - 2) * math.factorial(2) )
		print " ***** Number of combinations: %s" % num_combinations
		
		# test unique combinations for record linkage (duplicates)
		combination_pairs = []
		total_processed = 0
		for combination_1 in range(num_records):
			total_processed = total_processed + (num_records - combination_1) - 1
			percent_complete = float(total_processed) / float(num_combinations) * 100
			print " ***** %.2f%% complete ----- total processed: %s" % (percent_complete, total_processed)
			for combination_2 in range(combination_1+1, num_records):
				tempDictionary = {}
				if levenshtein.Compute_Levenshtein_Distance(records[combination_1]["ID"], records[combination_2]["ID"]) == 0:
					tempDictionary['class'] = 'same'
				else:
					tempDictionary['class'] = 'different'
				for key in keys[1:]:					
					levenshtein_distance = levenshtein.Compute_Levenshtein_Distance(records[combination_1][key], records[combination_2][key])
					tempDictionary[key] = levenshtein_distance
				
				combination_pair = str(combination_1) + '-' + str(combination_2)
				combination_pairs.append(combination_pair)
				trainingDataSet[combination_pair] = tempDictionary
		
		#keys.remove("ID") # need to remove the key "ID" since it has no bearing anymore (and is not a key in the trainingDataSet dictionary)
		#OutputRecordsToTabulatedFile(trainingDataSet, output_file, combination_pairs, keys)
		
		allConditions = PopulateConditions(conditions_file)
		print("allConditions: %s" % allConditions)
		
		#Produce trained tree and results
		adtClassifier = adt.classifier(trainingDataSet, allConditions, "people")
		print("adtClassifier: %s" % adtClassifier)
		
		adt_infrastructure.WriteTreeToFile(adtClassifier, output_file)
		outputDatabase = adt.evaluate(trainingDataSet, adtClassifier)
		
		#adtClassifier = adt.classifier(trainingDataSet, allConditions)
		#results = run10FoldCrossValidation(output_file, adtClassifier)
		#print("accuracy: " + str(results))
		stop_drawing = open("node_end.txt", 'w')
		stop_drawing.close()
	else:
		Usage()
		return
def main():
	if len(sys.argv) != 4:
		Usage()
		return

	training_comparisons_file = sys.argv[1]
	adtree_file = sys.argv[2]
	output_file_prefix = sys.argv[3]

	if os.path.exists(training_comparisons_file) and os.path.exists(adtree_file):
		adtree = adt_infrastructure.ReCreateADTree(adtree_file, adt)

		number_of_matches, number_of_nonmatches, trainingComparisonPairs = adt_infrastructure.BuildComparisonPairsDataStructure(training_comparisons_file)
		outputDatabase = adt.evaluate(trainingComparisonPairs, adtree)

		number_of_matches = float(number_of_matches)
		number_of_nonmatches = float(number_of_nonmatches)

		correct_classifications = 0.0
		incorrect_classifications = 0.0

		number_of_predicted_matches = 0.0
		number_of_predicted_nonmatches = 0.0

		true_positives = 0.0
		true_negatives = 0.0
		false_positives = 0.0
		false_negatives = 0.0
		for combination_pair in outputDatabase.keys():
			# true positives
			if trainingComparisonPairs[combination_pair]["classification"] == SAME and outputDatabase[combination_pair]["classification"] == SAME:
				true_positives = true_positives + 1.0
				correct_classifications = correct_classifications + 1.0
				number_of_predicted_matches = number_of_predicted_matches + 1.0
			# true negatives
			elif trainingComparisonPairs[combination_pair]["classification"] == DIFFERENT and outputDatabase[combination_pair]["classification"] == DIFFERENT:
				true_negatives = true_negatives + 1.0
				correct_classifications = correct_classifications + 1.0
				number_of_predicted_nonmatches = number_of_predicted_nonmatches + 1.0
			else:
				# false positives
				if outputDatabase[combination_pair]["classification"] == SAME:
					#print "False Positive:", outputDatabase[combination_pair]
					false_positives = false_positives + 1.0
					incorrect_classifications = incorrect_classifications + 1.0
					number_of_predicted_matches = number_of_predicted_matches + 1.0
				# false negatives
				elif outputDatabase[combination_pair]["classification"] == DIFFERENT:
					#print "False Negative:", outputDatabase[combination_pair]
					false_negatives = false_negatives + 1.0
					incorrect_classifications = incorrect_classifications + 1.0
					number_of_predicted_nonmatches = number_of_predicted_nonmatches + 1.0
				else:
					print " **************************** THIS SHOULD NEVER HAPPEN"

		# Precision = TruePositives / (TruePositives + FalseNegatives)
		precision_denominator = true_positives + false_negatives
		if precision_denominator == 0:
			print " ### ERROR: Precision could not be calculated because (true_positives + false_negatives) = 0"
			precision_denominator = 0.001
		precision = true_positives / precision_denominator * 100.0

		# Recall = TruePositives / (TruePositives + FalsePositives)
		recall_denominator = true_positives + false_positives
		if recall_denominator == 0:
			print " ### ERROR: Recall could not be calculated because (true_positives + false_positives) = 0"
			recall_denominator = 0.001
		recall = true_positives / recall_denominator * 100.0

		#             (TruePositives + TrueNegatives)
		# Accuracy = --------------------------------------------------------------------
		#             (TruePositives + True Negatives + FalsePositives + FalseNegatives)
		accuracy_denominator = true_positives + true_negatives + false_positives + false_negatives
		if accuracy_denominator == 0:
			print " ### ERROR: Accuracy could not be calculated because (true_positives + true_negatives + false_positives + false_negatives) = 0"
			accuracy_denominator = 0.001
		accuracy = (true_positives + true_negatives) / accuracy_denominator * 100.0

		print ""
		print " Training Data Record-Pairs    : %s" % len(outputDatabase)
		print ""
		print " Number of matches             : %s" % str(int(number_of_matches))
		print " Number of nonmatches          : %s" % str(int(number_of_nonmatches))
		print ""
		print " Number of predicted matches   : %s" % str(int(number_of_predicted_matches))
		print " Number of predicted nonmatches: %s" % str(int(number_of_predicted_nonmatches))
		print ""
		print " True Positives                : %s detected of %s (%.2f%%)" % (str(int(true_positives)), str(int(number_of_matches)), true_positives / number_of_matches * 100.0)
		print " True Negatives                : %s detected of %s (%.2f%%)" % (str(int(true_negatives)), str(int(number_of_nonmatches)), true_negatives / number_of_nonmatches * 100.0)
		print " False Positives               : %s" % str(int(false_positives))
		print " False Negatives               : %s" % str(int(false_negatives))
		print ""
		print " Recall                        : %s%%" % precision
		print " Precision                     : %s%%" % recall
		print " Accuracy                      : %s%%" % accuracy
		print ""

		with open(output_file_prefix + "-Precision.txt", 'w') as f_precision:
			f_precision.write(str(precision))
			f_precision.write("\n")
		with open(output_file_prefix + "-Recall.txt", 'w') as f_recall:
			f_recall.write(str(recall))
			f_recall.write("\n")
		with open(output_file_prefix + "-Accuracy.txt", 'w') as f_accuracy:
			f_accuracy.write(str(accuracy))
			f_accuracy.write("\n")

	else:
		Usage()
		return