def __init__(self, data):
self.data = data
self.project_file = '/tmp/data_dirty.csv'
self.data.dirty_pd.to_csv(self.project_file, index=False, header=True)
start_time = time.time()
self.setUp()
self.run_transforms()
self.tearDown()
runtime = (time.time() - start_time)
tool = OpenRefine('/tmp/data_clean.tsv', data=data)
print "Runtime: " + str(runtime)
print "Fscore: " + str(tool.calculate_total_fscore())
print "Precision: " + str(tool.calculate_total_precision())
print "Recall: " + str(tool.calculate_total_recall())
from ml.datasets.mohammad import MohammadDataSet
from ml.tools.openrefine.OpenRefine import OpenRefine
#one rule for all columns:
# if(contains(value, "x"), "error", value)
# takes 3 mins to execute
data = MohammadDataSet("tax", 20, 30, 10)
tool = OpenRefine(
"/home/felix/SequentialPatternErrorDetection/OpenRefine/tax/result/tax_o20_r30_p10-csv-with-minus-rule.tsv",
data=data)
print "Fscore: " + str(tool.calculate_total_fscore())
print "Precision: " + str(tool.calculate_total_precision())
print "Recall: " + str(tool.calculate_total_recall())
for c in range(data.shape[1]):
print tool.calculate_fscore_by_column(c)