def selectFeaturesForKeyword(keyword, threshold = 0.223):
import Trainer, PythonVersionHandler
from pyspark.mllib.regression import LabeledPoint
featureList = Trainer.featuresList[:-2]
Trainer.setFeatureVector(featureList)
trainData, testData, weights, accuracy = getTrainedWeights(keyword)
removedFeatures = []
accuracies = [accuracy]
weightsRow = list(weights)
while (not isImportant(weights, threshold = threshold)) and len(weights) > 1:
index, featureList, removedFeature = eliminate(weights, featureList)
removedFeatures.append(removedFeature)
Trainer.setFeatureVector(featureList)
def getReducedVector(lp):
newFeatures = list(lp.features)
newFeatures.pop(index)
return LabeledPoint(lp.label, newFeatures)
trainData = trainData.map(getReducedVector)
testData = testData.map(getReducedVector)
model = Trainer.trainPairWiseData(trainData, dataName = 'TrainData')
accuracy = Trainer.evaluateModelOnData(model, testData, dataName = 'TestData')
accuracies.append(accuracy)
weights = list(model.weights)
weightsRow.append('X')
weightsRow.extend(weights)
PythonVersionHandler.print_('Keyword: ' + keyword)
PythonVersionHandler.print_('Selected features: ' + str(featureList))
PythonVersionHandler.print_('Following features have reduced by order: ' + str(removedFeatures))
PythonVersionHandler.print_('Accuracies from each step: ' + str(accuracies))
row = [keyword]
row.extend(featureList)
row.append('X')
row.extend(removedFeatures)
row.extend(accuracies)
return row, weightsRow
def extractPairs():
import paths, PythonVersionHandler, Trainer, ReadyTests
feature_names = ['photos', 'soldCount', 'feedbackPercentage', 'memberSoldCount', 'memberSegment',
'subtitleFlag', 'brandNew', 'freeCargo', 'dailyOffer', 'windowOptionFlag', 'price', 'productCount']
Trainer.setFeatureVector(feature_names)
keywords = ReadyTests.get27Keywords()[23:]
for c, keyword in enumerate(keywords):
PythonVersionHandler.print_logging(str(c+1)+'.', keyword.upper() + ':')
trainTesting(keyword)
def trainingTest21():
import paths, FinalizedRunners, Trainer
feature_names = ['photos', 'feedbackPercentage', 'memberSoldCount', 'soldCount',
'memberSegment', 'subtitleFlag', 'brandNew', 'freeCargo', 'windowOptionFlag']
Trainer.setFeatureVector(feature_names)
keywords = ['besiktas', 'kol_saati', 'iphone_7', 'iphone_7_kilif']
for keyword in keywords:
folder = paths.joinPath(paths.joinPath(paths.HDFSRootFolder, 'secondWeek'), keyword)
FinalizedRunners.trainForKeyword(keyword, folder, saving = True)
def trainExtendedPairsLoop(onlyFollowings = False, AllPageButId = False):
import paths, PythonVersionHandler, Trainer, ReadyTests
feature_names = ['photos', 'soldCount', 'feedbackPercentage', 'memberSoldCount', 'memberSegment',
'subtitleFlag', 'brandNew', 'freeCargo', 'dailyOffer', 'windowOptionFlag', 'sameDay']
Trainer.setFeatureVector(feature_names)
keywords = ReadyTests.get27Keywords()
for c, keyword in enumerate(keywords):
PythonVersionHandler.print_logging(str(c+1)+'.', keyword.upper() + ':')
trainExtendedPairs(keyword, onlyFollowings = onlyFollowings, AllPageButId = AllPageButId)
Trainer.saveOutputTable()
Trainer.printOutputTable()
def trainingTestAllLoop(feature_names):
import paths, PythonVersionHandler, FinalizedRunners, Trainer, ReadyTests
Trainer.setFeatureVector(feature_names)
keywords = ReadyTests.get27Keywords()
for c, keyword in enumerate(keywords):
PythonVersionHandler.print_logging(str(c+1)+'.', keyword.upper() + ':')
keyword = keyword.replace(' ', '_')
folder = paths.joinPath(paths.joinPath(paths.HDFSRootFolder, 'weekAugust'), keyword)
FinalizedRunners.trainForKeyword(keyword, folder, saving = False)
Trainer.printOutputTable()
Trainer.saveOutputTable()
Trainer.outputTable = []
