def main():
print "parsing data... this could take a while..."
dp = DataParser('posts_news')
featuresToResultsAll = dp.getFeatureResultPairs()
# calculate weights
weights, testData = getWeightsAndTestData(featuresToResultsAll)
i = 0
totalError = 0.0
print "\nPrinting Example Results:"
for fv, target in testData:
i += 1
prediction = predict(weights, fv)
if i % 20 == 0:
printResults(prediction, target)
#error = abs(len(str(prediction)) - len(str(target)))
error = abs(prediction - target)
totalError += error
totalError /= i
print "total Error as average difference between prediction and target: %s" % totalError
dp.printMostProvocativeWords(50)
dp.printMostProvocativeBigrams(50)
dp.printMostProvocativeTrigrams(50)
