def func(actual):

ksStat, pValue = stats.ks_2samp(actual, expectedDistribution)

return ksStat, ksStat/(math.sqrt((len(actual) + len(expectedDistribution)) / (len(actual) * (len(expectedDistribution))))), pValue

# return ksStat, pValue, len(actual), len(expectedDistribution)

pearsons = []

clusterList = list(cluster)

for i, iso1 in enumerate(clusterList):

for iso2 in clusterList[i+1:]:

pearsons.append(iso1.regionPearson(iso2, region))

# pearsons.append(pearsonMap[iso1, iso2] if iso1 < iso2 else pearsonMap[iso2, iso1])

pearsons = []

for iso1 in cluster1:

for iso2 in cluster2:

pearsons.append(iso1.regionPearson(iso2, region))

pearsons = []

for cluster in clusters:

pearsons.extend(getIntraClusterPearsons(cluster, region))

distribution1 = getCombinedIntraClusterPearsons(clusters1, region)

distribution2 = getCombinedIntraClusterPearsons(clusters2, region)

ksStat, pValue = stats.ks_2samp(distribution1, distribution2)

# return pValue

return ksStat, ksStat/(math.sqrt((len(distribution1) + len(distribution2)) / (len(distribution1) * (len(distribution2))))), pValue

ksStat, weightedKS, pValue = correctnessFunc(getCombinedIntraClusterPearsons(clusters, region))

# ksStat, pValue = stats.kstest(getCombinedIntraClusterPearsons(clusters, region), betaDistribution.cdf)

# return pValue

# TODO: double check that this is statistically valid

# null hypothesis: all individual clusters fit

# reject if any individuals reject

# want chance that none reject. given chance that each doesn't reject

# so all pValues multiplied together

# print()

KSs = []

wKSs = []

pVs = []

# totalCount = 0

# chanceCount = 0

# averageKS = 0.0

# averageWeightedKS = 0.0

# averageP = 0.0

# chanceNoneReject = 1.0

for cluster in clusters:

if len(cluster) >= 8: # otherwise it wont have enough pairwise pearsons to be significant

ksStat, weightedKS, pValue = correctnessFunc(getIntraClusterPearsons(cluster, region))

# ksStat, pValue = stats.kstest(getIntraClusterPearsons(cluster, region), betaDistribution.cdf)

# print(pValue)

# chanceNoneReject *= pValue

# averageP += pValue

# averageKS += ksStat

# averageWeightedKS += weightedKS

# totalCount += 1

# if pValue >= .1:

# chanceCount += 1

KSs.append(ksStat)

wKSs.append(weightedKS)

pVs.append(pValue)

# return chanceCount/totalCount, averageP/totalCount, chanceNoneReject

# return averageKS/totalCount, averageWeightedKS/totalCount, averageP/totalCount

# TODO: double check that this is statistically valid

# null hypothesis: no pairs of clusters fit

# reject if any pairs fit

# want chance that all reject. given chance that each doesn't reject

# so all inverse pValues multiplied together

intraPearsons = [getIntraClusterPearsons(cluster, region) for cluster in clusters]

# print()

KSs = []

wKSs = []

pVs = []

# totalCount = 0

# chanceCount = 0

# averageP = 0.0

# averageKS = 0.0

# chanceAllReject = 1.0

for i, cluster1 in enumerate(clusters):

# print("{}/{}".format(i+1, len(clusters)))

for j, cluster2 in enumerate(clusters[i+1:]):

if len(cluster1) + len(cluster2) >= 8:

pearsons = intraPearsons[i] + intraPearsons[i+1+j] + getInterClusterPearsons(cluster1, cluster2, region)

# ksStat, pValue = stats.kstest(pearsons, betaDistribution.cdf)

ksStat, weightedKS, pValue = correctnessFunc(pearsons)

# print(1-pValue)

# print(pValue)

# print(cluster1)

# print(cluster2)

# chanceAllReject *= 1 - pValue

# averageP += 1 - pValue

# averageKS += ksStat

# totalCount += 1

# if pValue <= .1:

# chanceCount += 1

KSs.append(ksStat)

wKSs.append(weightedKS)

pVs.append(pValue)

# return chanceCount/totalCount, averageP/totalCount, chanceAllReject

# return chanceCount/totalCount, averageKS/totalCount

clustersMap = dict()

multipleCount = 0

for i, cluster in enumerate(clusters):

for isolate in cluster:

# assert isolate not in clustersMap

if isolate in clustersMap:

clustersMap[isolate].add(i)

multipleCount += 1

# print("\n\tisolate {} in multiple clusters".format(isolate))

# print(clustersMap[isolate])

# print(cluster)

clustersMap[isolate] = {i}

print("\t{} isolates in multiple clusters".format(multipleCount))

sameDS = diffDD = totDS = totDD = totSquishy = 0

clusterMap = getClusterMap(clusters)

for i, iso1 in enumerate(isolates):

# print("{}/{}".format(i+1, len(isolates)))

for iso2 in isolates[i+1:]:

# print("{},{}".format(iso1, iso2))

# pearson = pearsonMap[iso1, iso2] if iso1 < iso2 else pearsonMap[iso2, iso1]

regionsPearson = [iso1.regionPearson(iso2, region) for region in regions]

# print(regionsPearson)

if all(pearson >= region.pSimThresholdAlpha for region, pearson in zip(regions, regionsPearson)):

# if pearson >= dsThreshold:

# print("\tDS")

totDS += 1

if iso1 in clusterMap and iso2 in clusterMap and len(clusterMap[iso1] & clusterMap[iso2]) > 0:

# print("Same Cluster")

sameDS += 1

# elif iso1 not in clusterMap or iso2 not in clusterMap:

# print("not in any cluster")

elif any(pearson <= region.pSimThresholdBeta for region, pearson in zip(regions, regionsPearson)):

# elif pearson <= ddThreshold:

# print("\tDD")

totDD += 1

if iso1 not in clusterMap or iso2 not in clusterMap or clusterMap[iso1] != clusterMap[iso2]:

# print("Diff Cluster")

diffDD += 1

else:

totSquishy += 1

correctDS = sameDS/totDS

correctDD = diffDD/totDD

tcScore = (sameDS+diffDD)/(totDS+totDD)

squishyRatio = totSquishy/(totSquishy+totDD+totDS)

# print("correctDS = {}".format(correctDS))

# print("correctDD = {}".format(correctDD))

# print("tcScore = {}".format(tcScore))

# print("squishyRatio = {}".format(squishyRatio))

# print("totDS: {}, totSquishy: {}, totDD: {}".format(totDS, totSquishy, totDD))

bothSame = 0

oneOfEach = 0

bothDiff = 0

clusterMap1 = getClusterMap(clusters1)

clusterMap2 = getClusterMap(clusters2)

for i, iso1 in enumerate(isolates):

# print("{}/{}".format(i+1, len(isolates)))

for iso2 in isolates[i+1:]:

sameClust1 = iso1 in clusterMap1 and iso2 in clusterMap1 and len(clusterMap1[iso1] & clusterMap1[iso2]) > 0

sameClust2 = iso1 in clusterMap2 and iso2 in clusterMap2 and len(clusterMap2[iso1] & clusterMap2[iso2]) > 0

if sameClust1 == sameClust2:

if sameClust1:

bothSame += 1

else:

bothDiff += 1

else:

oneOfEach += 1

bothSame, oneOfEach, bothDiff = getPairCounts(isolates, clusters1, clusters2)

randIndex = (bothSame + bothDiff) / (bothSame + oneOfEach + bothDiff)

jaccardIndex = (bothSame) / (bothSame + oneOfEach)

anotherIndex = (bothDiff) / (bothDiff + oneOfEach)

# print("randIndex: {}".format(randIndex))

# print("jaccardIndex: {}".format(jaccardIndex))

# print("anotherIndex: {}".format(anotherIndex))

print("loading replicate pearsons from DB...")

regionNameLookup = {region.name: region for region in cfg.regions}

with open("mysqlConfig.json", mode='r') as mysqlConfigJson:

mysqlConfig = json.load(mysqlConfigJson)

cnx = mysql.connector.connect(**mysqlConfig)

# cursor = cnx.cursor(buffered=True)

cursor = cnx.cursor()

maxReplicatesPerIsolateRegion = 10 # to prevent bias

query = (

"SELECT r.appliedRegion as region, PearsonMatch(p1.pyroID, p2.pyroID, r.pearsonDispLength) as pearson FROM Pyroprints AS p1 INNER JOIN Pyroprints AS p2 USING(`isoID`, `appliedRegion`) INNER JOIN Regions AS r USING(`appliedRegion`) WHERE (SELECT COUNT(*) FROM Pyroprints AS pyro WHERE pyro.isoID = p1.isoID AND pyro.appliedRegion = p1.appliedRegion AND pyro.pyroID > p1.pyroID AND pyro.isErroneous IS FALSE) < {} AND p1.pyroID < p2.pyroID AND p1.isErroneous IS FALSE AND p2.isErroneous IS FALSE ORDER BY p1.isoID, p1.appliedRegion, p1.pyroID, p2.pyroID".format(maxReplicatesPerIsolateRegion)

)

# Isolates checked with:

# select isoID, appliedRegion, count(*) from pyroprints where isErroneous is false group by isoID, appliedRegion having count(*) > 1 order by isoID, appliedRegion;

regionPearsons = {region.name: [] for region in cfg.regions}

cursor.execute(query)

for (region, pearson) in cursor:

regionPearsons[region].append(pearson)

cursor.close()

cnx.close()

for region in cfg.regions:

print("region {} has {} replicate pairs".format(region.name, len(regionPearsons[region.name])))

cacheFileName = getReplicatePearsonsCacheFileName(cfg)

if os.path.isfile(cacheFileName):

return loadReplicatePearsonsFromFile(cacheFileName)

else:

clusterMap = getClusterMap(clusters)

clusterSizes = [len(cluster) for cluster in clusters]

# sizeRange = min(clusterSizes), max(clusterSizes)

count, minmax, mean, var, _, _ = stats.describe(clusterSizes)

sizeHistogram = (

len([size for size in clusterSizes if size < 4]),

len([size for size in clusterSizes if size >= 4 and size < 8]),

len([size for size in clusterSizes if size >= 8 and size < 16]),

len([size for size in clusterSizes if size >= 16 and size < 32]),

len([size for size in clusterSizes if size >= 32 and size < 64]),

len([size for size in clusterSizes if size >= 64 and size < 128]),

len([size for size in clusterSizes if size >= 128])

)

noise = [isolate for isolate in isolates if isolate not in clusterMap]