def test_R_implementation_equivalence(self):
data = self.sc.parallelize(
[
1.1626852897838,
-0.585924465893051,
1.78546500331661,
-1.33259371048501,
-0.446566766553219,
0.569606122374976,
-2.88971761441412,
-0.869018343326555,
-0.461702683149641,
-0.555540910137444,
-0.0201353678515895,
-0.150382224136063,
-0.628126755843964,
1.32322085193283,
-1.52135057001199,
-0.437427868856691,
0.970577579543399,
0.0282226444247749,
-0.0857821886527593,
0.389214404984942,
]
)
model = Statistics.kolmogorovSmirnovTest(data, "norm")
self.assertAlmostEqual(model.statistic, 0.189, 3)
self.assertAlmostEqual(model.pValue, 0.422, 3)
model = Statistics.kolmogorovSmirnovTest(data, "norm", 0, 1)
self.assertAlmostEqual(model.statistic, 0.189, 3)
self.assertAlmostEqual(model.pValue, 0.422, 3)
def test_R_implementation_equivalence(self):
data = self.sc.parallelize([
1.1626852897838, -0.585924465893051, 1.78546500331661, -1.33259371048501,
-0.446566766553219, 0.569606122374976, -2.88971761441412, -0.869018343326555,
-0.461702683149641, -0.555540910137444, -0.0201353678515895, -0.150382224136063,
-0.628126755843964, 1.32322085193283, -1.52135057001199, -0.437427868856691,
0.970577579543399, 0.0282226444247749, -0.0857821886527593, 0.389214404984942
])
model = Statistics.kolmogorovSmirnovTest(data, "norm")
self.assertAlmostEqual(model.statistic, 0.189, 3)
self.assertAlmostEqual(model.pValue, 0.422, 3)
model = Statistics.kolmogorovSmirnovTest(data, "norm", 0, 1)
self.assertAlmostEqual(model.statistic, 0.189, 3)
self.assertAlmostEqual(model.pValue, 0.422, 3)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function
from pyspark import SparkContext
# $example on$
from pyspark.mllib.stat import Statistics
# $example off$
if __name__ == "__main__":
sc = SparkContext(appName="HypothesisTestingKolmogorovSmirnovTestExample")
# $example on$
parallelData = sc.parallelize([0.1, 0.15, 0.2, 0.3, 0.25])
# run a KS test for the sample versus a standard normal distribution
testResult = Statistics.kolmogorovSmirnovTest(parallelData, "norm", 0, 1)
# summary of the test including the p-value, test statistic, and null hypothesis
# if our p-value indicates significance, we can reject the null hypothesis
# Note that the Scala functionality of calling Statistics.kolmogorovSmirnovTest with
# a lambda to calculate the CDF is not made available in the Python API
print(testResult)
# $example off$
sc.stop()
def findFeatures(inputFileName, outputFileName):
inpFile = sc.textFile(inputFileName)
numRows = inpFile.count()
print('\nRead ', numRows, ' rows from ', inputFileName, '\n')
print('Print out a few rows read from file')
print('\n', inpFile.take(5), '\n')
# Rectangularize the RDD before vectorizing
# Filter elements to remove quotes to prevent (quote) embedded commas
countFields = inpFile.map(lambda s: removeEmbeddedCommas(s)).map(
lambda s: len(s.split(','))).collect()
print('number of fields in each row (first few): ', countFields[0:4])
RectangularizationNeeded = False
maxCount = 0
maxCountAt = 0
for i in range(len(countFields)):
if (countFields[i] > maxCount):
maxCount = countFields[i]
maxCountAt = i
if (i > 0) and (RectangularizationNeeded == False):
if (countFields[i] != countFields[i - 1]):
RectangularizationNeeded = True
if (RectangularizationNeeded == True):
print('Identified jagged data set; Rectangularization needed')
else:
print('Identified rectangular data set')
print('Inferring longest row(s) has ', maxCount, ' fields at row ',
maxCountAt)
inpFileRe = inpFile.map(lambda s: removeEmbeddedCommas(s)).map(
lambda s: s + ',No Data')
# remove short rows
shortFile = inpFileRe.filter(
lambda row: len(row.split(',')) < maxCount + 1)
print("Short rows will be filtered out")
print('\n', shortFile.take(10), '\n')
# truncate to maxCount+1 columns
inpFileTr = inpFileRe.filter(
lambda row: len(row.split(',')) == maxCount + 1)
print('\n', inpFileTr.take(5), '\n')
header = inpFileTr.first()
hL = header.split(',')
inpFileNh = inpFileTr.filter(lambda row: row != header)
print('Removed the First row as Header')
numRows = inpFileNh.count()
print('number of rows = ', numRows)
from pyspark.mllib.linalg import Matrix, Matrices
from pyspark.mllib.linalg import Vector, Vectors
# parsedData will be org.apache.spark.rdd.RDD[org.apache.spark.mllib.linalg.Vector]
parsedData = inpFileNh.map(
lambda s: Vectors.dense([with0Str(t) for t in s.split(',')]))
print('\nprint out a few vectors after converting from strings\n')
print(parsedData.take(5))
from pyspark.mllib.stat import MultivariateStatisticalSummary, Statistics
summary = Statistics.colStats(parsedData)
print('\nprint out summary statistics, for each column\n')
print('summary.mean')
print(summary.mean())
print('summary.variance')
print(summary.variance())
print('summary.count')
print(summary.count())
print('summary.max')
print(summary.max())
print('summary.min')
print(summary.min())
print('summary.normL1')
print(summary.normL1())
print('summary.normL2')
print(summary.normL2())
print('summary.numnonZeros')
print(summary.numNonzeros())
print()
numCols = len(summary.mean())
typeStrings = [' '] * numCols
# infer columns where normL1, normL2, mean, variance, max and mean are 0 as non-numeric
print('Inferring column data types:')
import math
for j in range(numCols):
if ((summary.normL1()[j] == 0.0) and (summary.normL2()[j] == 0.0)
and (summary.mean()[j] == 0.0)
and (summary.variance()[j] == 0.0)
and (summary.max()[j] == 0.0) and (summary.min()[j] == 0.0)):
typeStrings[j] = 'String'
else:
if ((math.trunc(summary.normL1()[j]) == summary.normL1()[j])
and (math.trunc(summary.max()[j]) == summary.max()[j])
and (math.trunc(summary.min()[j]) == summary.min()[j])):
typeStrings[j] = 'Int'
else:
typeStrings[j] = 'Float'
print(typeStrings[j], end=',')
print('\n\n')
#******************************************************************************
# take out the 'String' columns before calling Statistics.corr()
numNumericCols = 0
for j in range(numCols):
if (typeStrings[j] != 'String'):
numNumericCols = numNumericCols + 1
noStrings = inpFileNh.map(
lambda s: Vectors.dense(removeStrings(s, numNumericCols)))
print(noStrings.take(5))
correlMatrix = Statistics.corr(noStrings, method='pearson')
print('Computing Correlation Matrix on all columns')
print(
'Printing out column names that have correlation coefficient > 0.5 or < -0.5'
)
for i in range(numNumericCols):
for j in range(i):
if (((correlMatrix[i][j] >= 0.5) or (correlMatrix[i][j] <= -0.5))
and (i != j)):
print(hA[i], hA[j], correlMatrix[i][j])
#******************************************************************************
#******************************************************************************
# create a contingency matrix
LoLoF = [[0.0 for x in range(numNumericCols)] for y in range(numRows)]
LoLoF = noStrings.collect()
pdLinArr = [0.0 for x in range(numNumericCols * numRows)]
for i in range(numRows):
for j in range(numNumericCols):
pdLinArr[i * numNumericCols + j] = abs(LoLoF[i][j])
mat = Matrices.dense(numRows, numNumericCols, pdLinArr)
# conduct Pearson's independence test on the input contingency matrix
print(
"Computing Pearson's independence test on the input contingency matrix using chi-square test"
)
independenceTestResult = Statistics.chiSqTest(mat)
# summary of the test including the p-value, degrees of freedom
print('%s\n' % independenceTestResult)
#*******************************************************************************
stdDev = [0.0] * numCols
for j in range(numCols):
stdDev[j] = math.sqrt(summary.variance()[j])
#*******************************************************************************
# test for normal distribution using Kolmogorov-Smirnov test
#
colVec = [0.0] * numRows
#vecRDD = sc.parallelize(colVec)
#testResult = Statistics.kolmogorovSmirnovTest(vecRDD, 'norm', 0, 1)
#print(testResult)
numericMean = [0.0] * numNumericCols
numericSD = [0.0] * numNumericCols
k = 0
for j in range(numCols):
if ((summary.mean()[j] != 0.0) and (summary.variance()[j] != 0.0)):
numericMean[k] = summary.mean()[j]
numericSD[k] = stdDev[j]
k = k + 1
print(
'Checking if column data is normally distributed using Kolmogorov-Smirnov test'
)
for j in range(numNumericCols):
for i in range(numRows):
# see https://issues.apache.org/jira/browse/SPARK-20802
# test fails if data is normally distributed
# kolmogorovSmirnovTest in pyspark.mllib.stat.Statistics throws net.razorvine.pickle.PickleException
# when input data is normally distributed (no error when data is not normally distributed)
colVec[i] = float(i) # LoLoF[i][j]
vecRDD = sc.parallelize(colVec)
print(colVec[0], colVec[numRows - 1], numericMean[j], numericSD[j])
testResult = Statistics.kolmogorovSmirnovTest(vecRDD, 'norm',
numericMean[j],
numericSD[j])
print(testResult)
#*******************************************************************************
#*******************************************************************************
#
# estimate kernel densities
#
from pyspark.mllib.stat import KernelDensity
# colVec = [0.0]*numRows
# vecRDD = sc.parallelize(colVec)
print('Computing kernel densities on all columns using a Bandwidth of 3.0')
kd = KernelDensity()
kd.setSample(vecRDD)
kd.setBandwidth(3.0)
sAS = int(math.sqrt(numRows)) # sample array size
samplePoints = [0.0] * sAS
#samplePoints = [0.0]*numRows
for i in range(sAS):
samplePoints[i] = float(i * sAS)
#for i in range(numRows):
# samplePoints[i] = float(i)
densities = kd.estimate(samplePoints)
print('Estimating kernel densities')
print('Print kernel densities at sample points')
#print('Print kernel densities > 0.01 at sample points')
for j in range(numNumericCols):
# print( hL[j])
for i in range(numRows):
# see https://issues.apache.org/jira/browse/SPARK-20803
# KernelDensity.estimate in pyspark.mllib.stat.KernelDensity throws
# net.razorvine.pickle.PickleException when input data is normally
# distributed (no error when data is not normally distributed)
colVec[i] = float(i) # LoLoF[i][j]
vecRDD = sc.parallelize(colVec)
kd = KernelDensity()
kd.setSample(vecRDD)
kd.setBandwidth(3.0)
# Find density estimates for the given values
densities = kd.estimate(samplePoints)
for i in range(sAS):
print(densities[i], end=',')
print()
#for i in range(numRows):
# if (densities[i] >= 0.01):
# print(i, densities[i], end=',')
print()
#*******************************************************************************
#*******************************************************************************
#
# compute Skewness and Kurtosis for each numeric column
#
skew = [0.0] * numNumericCols
kurt = [0.0] * numNumericCols
term = 0.0
k = 0
for j in range(numCols):
if (typeStrings[j] != 'String'):
skew[k] = 0.0
kurt[k] = 0.0
# extra work: find Ints
typeStrings[j] = 'Int'
meanj = summary.mean()[j]
for i in range(numRows):
if ((typeStrings[j] == 'Int')
and (math.trunc(LoLoF[i][k]) != LoLoF[i][k])):
typeStrings[j] = 'Float'
term = (LoLoF[i][k] - meanj) / stdDev[j]
skew[k] = skew[k] + (term * term * term)
kurt[k] = kurt[k] + (term * term * term * term)
skew[k] = skew[k] / numRows
kurt[k] = (kurt[k] / numRows) - 3.0
k = k + 1
print('Skewness of columns')
k = 0
for j in range(numCols):
if (typeStrings[j] == 'String'):
print('Text', end=',')
else:
print(skew[k], end=',')
k = k + 1
print()
print('Kurtosis of columns')
k = 0
for j in range(numCols):
if (typeStrings[j] == 'String'):
print('Text', end=',')
else:
print(kurt[k], end=',')
k = k + 1
print()
print('Inferring column data types (Text string, Int, Float)')
# numbers that are Int and non-negative and "large" are likely to be numeric labels -- keep checking this heuristic
# columns that are outside Kurtosis limits <-1.2, 3.0> may be numeric labels
print('Attempting to infer if an Int column is a numeric label')
print("If all Ints in a column are >= 0 and 'large', it may be numLabel")
print(
'If all Ints in a column are >= 0 and excess kurtosis is outside [-1.2, 3.0], it may be numLabel'
)
for j in range(numCols):
if ((typeStrings[j] == 'Int') and (summary.min()[j] >= 0)
and ((summary.max()[j] > 10000) or (kurt[j] < -1.2) or
(kurt[j] > 3.0))):
print('column ' + j + ' (' + hA[j] + ') ' +
' may be a numeric label')
typeStrings[j] = 'NumLabel'
#******************************************************************************
#******************************************************************************
#
# Normalize the dataset by shifting by mean and scaling by stdDev
#
normData = [[0.0 for x in range(numNumericCols)] for y in range(numRows)]
rowMaxs = [0.0] * numRows
rowMins = [0.0] * numRows
rowNormL1s = [0.0] * numRows
rowNormL2s = [0.0] * numRows
rowNumZeros = [0] * numRows
means = [0.0] * numCols
for j in range(numCols):
means[j] = summary.mean()[j]
for i in range(numRows):
rowMaxs[i] = -999999.0
rowMins[i] = 999999.0
rowNumZeros[i] = 0
rowNormL1s[i] = 0.0
rowNormL2s[i] = 0.0
k = 0
for j in range(numCols):
if ((typeStrings[j] == 'Int') or (typeStrings[j] == 'Float')):
normData[i][k] = (LoLoF[i][k] - means[j]) / stdDev[j]
if (normData[i][k] > rowMaxs[i]):
rowMaxs[i] = normData[i][k]
if (normData[i][k] < rowMins[i]):
rowMins[i] = normData[i][k]
if (normData[i][k] == 0.0):
rowNumZeros[i] = rowNumZeros
if (abs(normData[i][k]) < 100.0):
rowNormL1s[i] = rowNormL1s[i] + abs(normData[i][k])
rowNormL2s[
i] = rowNormL2s[i] + normData[i][k] * normData[i][k]
# print(i,j,k, LoLoF[i][k], means[j], stdDev[j], normData[i][k], rowNormL1s[i], rowNormL2s[i])
k = k + 1
input = open(inputFileName, 'r')
fileHandle = open('/home/bsrsharma/work/python/rowNormL1L2.csv', 'w')
# Keep upto 6 columns of identifying info
if (numCols > 1):
for j in range(min(5, numCols)):
fileHandle.write(hL[j])
fileHandle.write(',')
fileHandle.write('L1-Norm')
fileHandle.write(",")
fileHandle.write('L2-Norm\n')
s = input.readline() # don't repeat header
for i in range(numRows):
# copy input to output
s = input.readline()
LoS = s.split(',')
for j in range(min(5, numCols)):
fileHandle.write(LoS[j])
fileHandle.write(',')
fileHandle.write('%s' % rowNormL1s[i])
fileHandle.write(',')
fileHandle.write('%s' % math.sqrt(rowNormL2s[i]))
fileHandle.write('\n')
fileHandle.close()
input.close()
print('Wrote ', 'rowNormL1L2.csv')
input = open(inputFileName, 'r')
fileHandle = open(outputFileName, 'w')
# output normalized data
numCols = numCols - 1
# write header row
if (numCols > 1):
for j in range(numCols - 1):
fileHandle.write(hL[j])
fileHandle.write(',')
fileHandle.write(hL[numCols - 1])
fileHandle.write('\n')
s = input.readline() # don't repeat header
for i in range(numRows):
# copy input to output
s = input.readline()
LoS = s.split(',')
k = 0
for j in range(numCols - 1):
if (typeStrings[j] == 'String'):
fileHandle.write(LoS[j])
else:
fileHandle.write('%s' % normData[i][k])
k = k + 1
fileHandle.write(',')
if (typeStrings[numCols - 1] == 'String'):
fileHandle.write(LoS[numCols - 1])
else:
fileHandle.write('%s' % normData[i][k])
fileHandle.write('\n')
fileHandle.close()
input.close()
print('Wrote ', outputFileName, '\n')
#******************************************************************************
# compute median for each column
medians = [0.0] * numNumericCols
aCol = [0.0] * numRows
for j in range(numNumericCols):
for i in range(numRows):
aCol[i] = LoLoF[i][j]
aCol.sort()
medians[j] = aCol[numRows / 2]
print('medians:')
k = 0
for j in range(numCols):
if (typeStrings[j] == 'String'):
print('Text', end=',')
else:
print(medians[k], end=',')
k = k + 1
print('\n\n')
# compute histograms for each column
numBins = int(math.sqrt(numRows))
histogram = [0] * (numBins + 1)
binWidth = 0
mins = [0.0] * numCols
maxs = [0.0] * numCols
print('Computing histograms for numeric columns')
print('choosing ', numBins, ' bins')
k = 0
for j in range(numCols):
mins[j] = summary.min()[j]
maxs[j] = summary.max()[j]
if (typeStrings[j] == 'String'):
print('column ', j, '( ', hL[j], ' ): Text')
else:
binWidth = (maxs[j] - mins[j]) / numBins
for i in range(numBins):
histogram[i] = 0
for i in range(numRows):
histogram[int((LoLoF[i][k] - mins[j]) / binWidth)] += 1
print('column ', j, '( ', hL[j], ' ):')
if (typeStrings[j] == 'NumLabel'):
print('NumLabel')
for i in range(numBins):
print(histogram[i], end=',')
print()
k = k + 1
print('\n\n')
# compute modes
modes = [0.0] * numNumericCols
largestBin = 0
binIndex = 0
print('modes:')
k = 0
for j in range(numCols):
if (typeStrings[j] == 'String'):
print('Text', end=',')
else:
largestBin = 0
binIndex = 0
for i in range(numBins):
# pick the bin with most items
if (histogram[i] > largestBin):
binIndex = i
modes[k] = mins[j] + (maxs[j] - mins[j]) * binIndex / numBins
print(modes[k], end=',')
k = k + 1
print('\n\n')
return 0
