"""

测试数据来源Berkeley大学

个人实现，仅供学习交流

"""

sampleOne = [(0, 'mouse'), (1, 'black')]

sampleTwo = [(0, 'cat'), (1, 'tabby'), (2, 'mouse')]

sampleThree = [(0, 'bear'), (1, 'black'), (2, 'salmon')]

sampleDataRDD = sc.parallelize([sampleOne, sampleTwo, sampleThree])

sampleOHEDictManual = {}

sampleOHEDictManual[(0,'bear')] = 0

sampleOHEDictManual[(0,'cat')] = 1

sampleOHEDictManual[(0,'mouse')] = 2

sampleOHEDictManual[(1,'black')] = 3

sampleOHEDictManual[(1,'tabby')] = 4

sampleOHEDictManual[(2,'mouse')] = 5

sampleOHEDictManual[(2,'salmon')] = 6

from test_helper import Test

Test.assertEqualsHashed(sampleOHEDictManual[(0,'bear')],

'b6589fc6ab0dc82cf12099d1c2d40ab994e8410c',

"incorrect value for sampleOHEDictManual[(0,'bear')]")

Test.assertEqualsHashed(sampleOHEDictManual[(0,'cat')],

'356a192b7913b04c54574d18c28d46e6395428ab',

"incorrect value for sampleOHEDictManual[(0,'cat')]")

Test.assertEqualsHashed(sampleOHEDictManual[(0,'mouse')],

'da4b9237bacccdf19c0760cab7aec4a8359010b0',

"incorrect value for sampleOHEDictManual[(0,'mouse')]")

Test.assertEqualsHashed(sampleOHEDictManual[(1,'black')],

'77de68daecd823babbb58edb1c8e14d7106e83bb',

"incorrect value for sampleOHEDictManual[(1,'black')]")

Test.assertEqualsHashed(sampleOHEDictManual[(1,'tabby')],

'1b6453892473a467d07372d45eb05abc2031647a',

"incorrect value for sampleOHEDictManual[(1,'tabby')]")

Test.assertEqualsHashed(sampleOHEDictManual[(2,'mouse')],

'ac3478d69a3c81fa62e60f5c3696165a4e5e6ac4',

"incorrect value for sampleOHEDictManual[(2,'mouse')]")

Test.assertEqualsHashed(sampleOHEDictManual[(2,'salmon')],

'c1dfd96eea8cc2b62785275bca38ac261256e278',

"incorrect value for sampleOHEDictManual[(2,'salmon')]")

Test.assertEquals(len(sampleOHEDictManual.keys()), 7,

'incorrect number of keys in sampleOHEDictManual')

# 稀疏向量处理

import numpy as np

from pyspark.mllib.linalg import SparseVector

aDense = np.array([0., 3., 0., 4.])

aSparse = SparseVector(4, [1,3], [3., 4.])

bDense = np.array([0., 0., 0., 1.])

bSparse = SparseVector(4, [3], [1.])

w = np.array([0.4, 3.1, -1.4, -.5])

print aDense.dot(w)

print aSparse.dot(w)

print bDense.dot(w)

print bSparse.dot(w)

Test.assertTrue(isinstance(aSparse, SparseVector), 'aSparse needs to be an instance of SparseVector')

Test.assertTrue(isinstance(bSparse, SparseVector), 'aSparse needs to be an instance of SparseVector')

Test.assertTrue(aDense.dot(w) == aSparse.dot(w),

'dot product of aDense and w should equal dot product of aSparse and w')

Test.assertTrue(bDense.dot(w) == bSparse.dot(w),

'dot product of bDense and w should equal dot product of bSparse and w')

sampleOneOHEFeatManual = SparseVector(7, [2,3], [1.,1.])

sampleTwoOHEFeatManual = SparseVector(7, [1,4,5], [1.,1.,1.])

sampleThreeOHEFeatManual = SparseVector(7, [0,3,6], [1.,1.,1.])

Test.assertTrue(isinstance(sampleOneOHEFeatManual, SparseVector),

'sampleOneOHEFeatManual needs to be a SparseVector')

Test.assertTrue(isinstance(sampleTwoOHEFeatManual, SparseVector),

'sampleTwoOHEFeatManual needs to be a SparseVector')

Test.assertTrue(isinstance(sampleThreeOHEFeatManual, SparseVector),

'sampleThreeOHEFeatManual needs to be a SparseVector')

Test.assertEqualsHashed(sampleOneOHEFeatManual,

'ecc00223d141b7bd0913d52377cee2cf5783abd6',

'incorrect value for sampleOneOHEFeatManual')

Test.assertEqualsHashed(sampleTwoOHEFeatManual,

'26b023f4109e3b8ab32241938e2e9b9e9d62720a',

'incorrect value for sampleTwoOHEFeatManual')

Test.assertEqualsHashed(sampleThreeOHEFeatManual,

'c04134fd603ae115395b29dcabe9d0c66fbdc8a7',

'incorrect value for sampleThreeOHEFeatManual')

# oneHotEncoding函数

def oneHotEncoding(rawFeats, OHEDict, numOHEFeats):

return SparseVector(numOHEFeats, indice, values)

numSampleOHEFeats = len(sampleOHEDictManual)

sampleOneOHEFeat = oneHotEncoding(sampleOne, sampleOHEDictManual, numSampleOHEFeats)

print sampleOneOHEFeat

Test.assertTrue(sampleOneOHEFeat == sampleOneOHEFeatManual,

'sampleOneOHEFeat should equal sampleOneOHEFeatManual')

Test.assertEquals(sampleOneOHEFeat, SparseVector(7, [2,3], [1.0,1.0]),

'incorrect value for sampleOneOHEFeat')

Test.assertEquals(oneHotEncoding([(1, 'black'), (0, 'mouse')], sampleOHEDictManual,

numSampleOHEFeats), SparseVector(7, [2,3], [1.0,1.0]),

'incorrect definition for oneHotEncoding')

#并行处理数据集

sampleOHEData = sampleDataRDD.map(lambda x:oneHotEncoding(x, sampleOHEDictManual, numSampleOHEFeats))

print sampleOHEData.collect()

sampleOHEDataValues = sampleOHEData.collect()

Test.assertTrue(len(sampleOHEDataValues) == 3, 'sampleOHEData should have three elements')

Test.assertEquals(sampleOHEDataValues[0], SparseVector(7, {2: 1.0, 3: 1.0}),

'incorrect OHE for first sample')

Test.assertEquals(sampleOHEDataValues[1], SparseVector(7, {1: 1.0, 4: 1.0, 5: 1.0}),

'incorrect OHE for second sample')

Test.assertEquals(sampleOHEDataValues[2], SparseVector(7, {0: 1.0, 3: 1.0, 6: 1.0}),

'incorrect OHE for third sample')

# 创建OHE字典

sampleDistinctFeats = (sampleDataRDD

.flatMap(lambda x:x)

.distinct())

Test.assertEquals(sorted(sampleDistinctFeats.collect()),

[(0, 'bear'), (0, 'cat'), (0, 'mouse'), (1, 'black'),

(1, 'tabby'), (2, 'mouse'), (2, 'salmon')],

'incorrect value for sampleDistinctFeats')

sampleOHEDict = (sampleDistinctFeats

.zipWithIndex()

.collectAsMap())

print sampleOHEDict

Test.assertEquals(sorted(sampleOHEDict.keys()),

[(0, 'bear'), (0, 'cat'), (0, 'mouse'), (1, 'black'),

(1, 'tabby'), (2, 'mouse'), (2, 'salmon')],

'sampleOHEDict has unexpected keys')

Test.assertEquals(sorted(sampleOHEDict.values()), range(7), 'sampleOHEDict has unexpected values')

def createOneHotDict(inputData):

return inputData.flatMap(lambda x:x).distinct().zipWithIndex().collectAsMap()

sampleOHEDictAuto = createOneHotDict(sampleDataRDD)

print sampleOHEDictAuto

Test.assertEquals(sorted(sampleOHEDictAuto.keys()),

[(0, 'bear'), (0, 'cat'), (0, 'mouse'), (1, 'black'),

(1, 'tabby'), (2, 'mouse'), (2, 'salmon')],

'sampleOHEDictAuto has unexpected keys')

Test.assertEquals(sorted(sampleOHEDictAuto.values()), range(7),

'sampleOHEDictAuto has unexpected values')

# 解析CTR数据

baseDir = os.path.join('data')

inputPath = os.path.join('dac_sample.txt')

fileName = os.path.join(baseDir, inputPath)

inputDir = os.path.split(fileName)[0]

def extractTar(check = False):

return False

if os.path.isfile(fileName):

print 'File is already available. Nothing to do.'

elif extractTar(check = True):

print 'tar.gz file was already available.'

elif not url.endswith('dac_sample.tar.gz'):

print 'Check your download url. Are you downloading the Sample dataset?'

else:

try:

urllib.urlretrieve(url, os.path.basename(urlparse.urlsplit(url).path))

except IOError:

print 'Unable to download and store: {0}'.format(url)

extractTar()

# In[24]:

import os.path

baseDir = os.path.join('data')

inputPath = os.path.join('cs190', 'dac_sample.txt')

fileName = os.path.join(baseDir, inputPath)

if os.path.isfile(fileName):

rawData = (sc

.textFile(fileName, 2)

.map(lambda x: x.replace('\t', ','))) # work with either ',' or '\t' separated data

print rawData.take(1)

weights = [.8, .1, .1]

seed = 42

#数据集划分解析

rawTrainData, rawValidationData, rawTestData = rawData.randomSplit(weights, seed)

# 缓存数据

rawTrainData.cache()

rawValidationData.cache()

rawTestData.cache()

nTrain = rawTrainData.count()

nVal = rawValidationData.count()

nTest = rawTestData.count()

print nTrain, nVal, nTest, nTrain + nVal + nTest

print rawData.take(1)

Test.assertTrue(all([rawTrainData.is_cached, rawValidationData.is_cached, rawTestData.is_cached]),

'you must cache the split data')

Test.assertEquals(nTrain, 79911, 'incorrect value for nTrain')

Test.assertEquals(nVal, 10075, 'incorrect value for nVal')

Test.assertEquals(nTest, 10014, 'incorrect value for nTest')

def parsePoint(point):

return list(outputFeatures)

parsedTrainFeat = rawTrainData.map(parsePoint)

numCategories = (parsedTrainFeat

.collect())

print numCategories[2][1]

ctrOHEDict = createOneHotDict(parsedTrainFeat)

numCtrOHEFeats = len(ctrOHEDict.keys())

print numCtrOHEFeats

print ctrOHEDict[(0, '')]

from pyspark.mllib.regression import LabeledPoint

def parseOHEPoint(point, OHEDict, numOHEFeats):

return LabeledPoint(label, featuresVector)

OHETrainData = rawTrainData.map(lambda point: parseOHEPoint(point, ctrOHEDict, numCtrOHEFeats))

OHETrainData.cache()

print OHETrainData.take(1)

backupOneHot = oneHotEncoding

oneHotEncoding = None

withOneHot = False

try: parseOHEPoint(rawTrainData.take(1)[0], ctrOHEDict, numCtrOHEFeats)

except TypeError: withOneHot = True

oneHotEncoding = backupOneHot

numNZ = sum(parsedTrainFeat.map(lambda x: len(x)).take(5))

numNZAlt = sum(OHETrainData.map(lambda lp: len(lp.features.indices)).take(5))

Test.assertEquals(numNZ, numNZAlt, 'incorrect implementation of parseOHEPoint')

Test.assertTrue(withOneHot, 'oneHotEncoding not present in parseOHEPoint')

def bucketFeatByCount(featCount):

return -1

featCounts = (OHETrainData

.flatMap(lambda lp: lp.features.indices)

.map(lambda x: (x, 1))

.reduceByKey(lambda x, y: x + y))

featCountsBuckets = (featCounts

.collect())

print featCountsBuckets

import matplotlib.pyplot as plt

x, y = zip(*featCountsBuckets)

x, y = np.log(x), np.log(y)

def preparePlot(xticks, yticks, figsize=(10.5, 6), hideLabels=False, gridColor='#999999',

return fig, ax

fig, ax = preparePlot(np.arange(0, 10, 1), np.arange(4, 14, 2))

ax.set_xlabel(r'$\log_e(bucketSize)$'), ax.set_ylabel(r'$\log_e(countInBucket)$')

plt.scatter(x, y, s=14**2, c='#d6ebf2', edgecolors='#8cbfd0', alpha=0.75)

pass

def oneHotEncoding(rawFeats, OHEDict, numOHEFeats):

return SparseVector(numOHEFeats, indice, values)

OHEValidationData = rawValidationData.map(lambda point: parseOHEPoint(point, ctrOHEDict, numCtrOHEFeats))

OHEValidationData.cache()

print OHEValidationData.take(1)

numNZVal = (OHEValidationData

.map(lambda lp: len(lp.features.indices))

.sum())

Test.assertEquals(numNZVal, 372080, 'incorrect number of features')

# CTR预估和对数损失函数评估，引用MLlib API

from pyspark.mllib.classification import LogisticRegressionWithSGD

numIters = 50

stepSize = 10.

regParam = 1e-6

regType = 'l2'

includeIntercept = True

model0 = LogisticRegressionWithSGD.train(OHETrainData,iterations=numIters,step=stepSize,regParam=regParam,regType=regType,intercept=includeIntercept)

sortedWeights = sorted(model0.weights)

print sortedWeights[:5], model0.intercept

Test.assertTrue(np.allclose(model0.intercept, 0.56455084025), 'incorrect value for model0.intercept')

Test.assertTrue(np.allclose(sortedWeights[0:5],

[-0.45899236853575609, -0.37973707648623956, -0.36996558266753304,

-0.36934962879928263, -0.32697945415010637]), 'incorrect value for model0.weights')

# log损失

from math import log

def computeLogLoss(p, y):

return -log(1-p)

print computeLogLoss(.5, 1)

print computeLogLoss(.5, 0)

print computeLogLoss(.99, 1)

print computeLogLoss(.99, 0)

print computeLogLoss(.01, 1)

print computeLogLoss(.01, 0)

print computeLogLoss(0, 1)

print computeLogLoss(1, 1)

print computeLogLoss(1, 0)

Test.assertTrue(np.allclose([computeLogLoss(.5, 1), computeLogLoss(.01, 0), computeLogLoss(.01, 1)],

[0.69314718056, 0.0100503358535, 4.60517018599]),

'computeLogLoss is not correct')

Test.assertTrue(np.allclose([computeLogLoss(0, 1), computeLogLoss(1, 1), computeLogLoss(1, 0)],

[25.3284360229, 1.00000008275e-11, 25.3284360229]),

'computeLogLoss needs to bound p away from 0 and 1 by epsilon')

classOneFracTrain = OHETrainData.map(lambda x:x.label).filter(lambda x:x==1).reduce(lambda x,y:x+y)/float(OHETrainData.count())

print classOneFracTrain

logLossTrBase = OHETrainData.map(lambda x:computeLogLoss(classOneFracTrain,x.label)).reduce(lambda x,y:x+y)/float(OHETrainData.count())

print 'Baseline Train Logloss = {0:.3f}\n'.format(logLossTrBase)

Test.assertTrue(np.allclose(classOneFracTrain, 0.22717773523), 'incorrect value for classOneFracTrain')

Test.assertTrue(np.allclose(logLossTrBase, 0.535844), 'incorrect value for logLossTrBase')

from math import exp # exp(-t) = e^-t

def getP(x, w, intercept):

return pow((1+exp(-rawPrediction)), -1)

trainingPredictions = OHETrainData.map(lambda x:getP(x.features,model0.weights,model0.intercept))

print trainingPredictions.take(5)

Test.assertTrue(np.allclose(trainingPredictions.sum(), 18135.4834348),

'incorrect value for trainingPredictions')

def evaluateResults(model, data):

return sumLoss/float(data.count())

logLossTrLR0 = evaluateResults(model0, OHETrainData)

print ('OHE Features Train Logloss:\n\tBaseline = {0:.3f}\n\tLogReg = {1:.3f}'

.format(logLossTrBase, logLossTrLR0))

Test.assertTrue(np.allclose(logLossTrLR0, 0.456903), 'incorrect value for logLossTrLR0')

logLossValBase = OHEValidationData.map(lambda x:computeLogLoss(classOneFracTrain,x.label)).reduce(lambda x,y:x+y)/float(OHEValidationData.count())

logLossValLR0 = evaluateResults(model0, OHEValidationData)

print ('OHE Features Validation Logloss:\n\tBaseline = {0:.3f}\n\tLogReg = {1:.3f}'

.format(logLossValBase, logLossValLR0))

Test.assertTrue(np.allclose(logLossValBase, 0.527603), 'incorrect value for logLossValBase')

Test.assertTrue(np.allclose(logLossValLR0, 0.456957), 'incorrect value for logLossValLR0')

# ROC评测指标

labelsAndScores = OHEValidationData.map(lambda lp:

(lp.label, getP(lp.features, model0.weights, model0.intercept)))

labelsAndWeights = labelsAndScores.collect()

labelsAndWeights.sort(key=lambda (k, v): v, reverse=True)

labelsByWeight = np.array([k for (k, v) in labelsAndWeights])

length = labelsByWeight.size

truePositives = labelsByWeight.cumsum()

numPositive = truePositives[-1]

falsePositives = np.arange(1.0, length + 1, 1.) - truePositives

truePositiveRate = truePositives / numPositive

falsePositiveRate = falsePositives / (length - numPositive)

fig, ax = preparePlot(np.arange(0., 1.1, 0.1), np.arange(0., 1.1, 0.1))

ax.set_xlim(-.05, 1.05), ax.set_ylim(-.05, 1.05)

ax.set_ylabel('True Positive Rate (Sensitivity)')

ax.set_xlabel('False Positive Rate (1 - Specificity)')

plt.plot(falsePositiveRate, truePositiveRate, color='#8cbfd0', linestyle='-', linewidth=3.)

plt.plot((0., 1.), (0., 1.), linestyle='--', color='#d6ebf2', linewidth=2.)

pass

# 通过hash函数进行特征降维

from collections import defaultdict

import hashlib

def hashFunction(numBuckets, rawFeats, printMapping=False):

return dict(sparseFeatures)

sampOneFourBuckets = hashFunction(4, sampleOne, True)

sampTwoFourBuckets = hashFunction(4, sampleTwo, True)

sampThreeFourBuckets = hashFunction(4, sampleThree, True)

sampOneHundredBuckets = hashFunction(100, sampleOne, True)

sampTwoHundredBuckets = hashFunction(100, sampleTwo, True)

sampThreeHundredBuckets = hashFunction(100, sampleThree, True)

print '\t\t 4 Buckets \t\t\t 100 Buckets'

print 'SampleOne:\t {0}\t\t {1}'.format(sampOneFourBuckets, sampOneHundredBuckets)

print 'SampleTwo:\t {0}\t\t {1}'.format(sampTwoFourBuckets, sampTwoHundredBuckets)

print 'SampleThree:\t {0}\t {1}'.format(sampThreeFourBuckets, sampThreeHundredBuckets)

Test.assertEquals(sampOneFourBuckets, {2: 1.0, 3: 1.0}, 'incorrect value for sampOneFourBuckets')

Test.assertEquals(sampThreeHundredBuckets, {72: 1.0, 5: 1.0, 14: 1.0},

'incorrect value for sampThreeHundredBuckets')

def parseHashPoint(point, numBuckets):

return LabeledPoint(label, sparseFeaturesVector)

numBucketsCTR = 2 ** 15

hashTrainData = rawTrainData.map(lambda point: parseHashPoint(point,numBucketsCTR))

hashTrainData.cache()

hashValidationData = rawValidationData.map(lambda point: parseHashPoint(point,numBucketsCTR))

hashValidationData.cache()

hashTestData = rawTestData.map(lambda point: parseHashPoint(point,numBucketsCTR))

hashTestData.cache()

print hashTrainData.take(1)

hashTrainDataFeatureSum = sum(hashTrainData

.map(lambda lp: len(lp.features.indices))

.take(20))

hashTrainDataLabelSum = sum(hashTrainData

.map(lambda lp: lp.label)

.take(100))

hashValidationDataFeatureSum = sum(hashValidationData

.map(lambda lp: len(lp.features.indices))

.take(20))

hashValidationDataLabelSum = sum(hashValidationData

.map(lambda lp: lp.label)

.take(100))

hashTestDataFeatureSum = sum(hashTestData

.map(lambda lp: len(lp.features.indices))

.take(20))

hashTestDataLabelSum = sum(hashTestData

.map(lambda lp: lp.label)

.take(100))

Test.assertEquals(hashTrainDataFeatureSum, 772, 'incorrect number of features in hashTrainData')

Test.assertEquals(hashTrainDataLabelSum, 24.0, 'incorrect labels in hashTrainData')

Test.assertEquals(hashValidationDataFeatureSum, 776,

'incorrect number of features in hashValidationData')

Test.assertEquals(hashValidationDataLabelSum, 16.0, 'incorrect labels in hashValidationData')

Test.assertEquals(hashTestDataFeatureSum, 774, 'incorrect number of features in hashTestData')

Test.assertEquals(hashTestDataLabelSum, 23.0, 'incorrect labels in hashTestData')

def computeSparsity(data, d, n):

return data.map(lambda x:len(x.features.indices)/float(d)).reduce(lambda x,y:x+y)/float(n)

averageSparsityHash = computeSparsity(hashTrainData, numBucketsCTR, nTrain)

averageSparsityOHE = computeSparsity(OHETrainData, numCtrOHEFeats, nTrain)

print 'Average OHE Sparsity: {0:.7e}'.format(averageSparsityOHE)

print 'Average Hash Sparsity: {0:.7e}'.format(averageSparsityHash)

Test.assertTrue(np.allclose(averageSparsityOHE, 1.6717677e-04),

'incorrect value for averageSparsityOHE')

Test.assertTrue(np.allclose(averageSparsityHash, 1.1805561e-03),

'incorrect value for averageSparsityHash')

numIters = 500

regType = 'l2'

includeIntercept = True

bestModel = None

bestLogLoss = 1e10

stepSizes = [1,10]

regParams = [1e-6, 1e-3]

for stepSize in stepSizes:

for regParam in regParams:

model = (LogisticRegressionWithSGD

.train(hashTrainData, numIters, stepSize, regParam=regParam, regType=regType,

intercept=includeIntercept))

logLossVa = evaluateResults(model, hashValidationData)

print ('\tstepSize = {0:.1f}, regParam = {1:.0e}: logloss = {2:.3f}'

.format(stepSize, regParam, logLossVa))

if (logLossVa < bestLogLoss):

bestModel = model

bestLogLoss = logLossVa

print ('Hashed Features Validation Logloss:\n\tBaseline = {0:.3f}\n\tLogReg = {1:.3f}'

.format(logLossValBase, bestLogLoss))

Test.assertTrue(np.allclose(bestLogLoss, 0.4481683608), 'incorrect value for bestLogLoss')

from matplotlib.colors import LinearSegmentedColormap

stepSizes = [3, 6, 9, 12, 15, 18]

regParams = [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2]

logLoss = np.array([[ 0.45808431, 0.45808493, 0.45809113, 0.45815333, 0.45879221, 0.46556321],

[ 0.44508948, 0.44509474, 0.44510274, 0.44525007, 0.44738317, 0.46365405]])

numRows, numCols = len(stepSizes), len(regParams)

logLoss = np.array(logLoss)

logLoss.shape = (numRows, numCols)

fig, ax = preparePlot(np.arange(0, numCols, 1), np.arange(0, numRows, 1), figsize=(8, 7),

hideLabels=True, gridWidth=0.)

ax.set_xticklabels(regParams), ax.set_yticklabels(stepSizes)

ax.set_xlabel('Regularization Parameter'), ax.set_ylabel('Step Size')

colors = LinearSegmentedColormap.from_list('blue', ['#0022ff', '#000055'], gamma=.2)

image = plt.imshow(logLoss,interpolation='nearest', aspect='auto',

cmap = colors)

pass

logLossTest = evaluateResults(bestModel, hashTestData)

logLossTestBaseline = hashTestData.map(lambda x:computeLogLoss(classOneFracTrain,x.label)).reduce(lambda x,y:x+y)/float(hashTestData.count())

print ('Hashed Features Test Log Loss:\n\tBaseline = {0:.3f}\n\tLogReg = {1:.3f}'

.format(logLossTestBaseline, logLossTest))

Test.assertTrue(np.allclose(logLossTestBaseline, 0.537438),

'incorrect value for logLossTestBaseline')

Test.assertTrue(np.allclose(logLossTest, 0.455616931), 'incorrect value for logLossTest')