def KMeansConfig(dataTable, k, eps = 0.00001, srcDims=100000000000,gpuMemSize = 512, settings = {}):
"""
Creates all the memory/data settings to run GPU accelerated KMeans.
"""
settings = dataConfig(dataTable,settings)
settings["sourceDims"] = min(settings["sourceDims"],srcDims)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize*1024*1024
settings["maxThreads"] = 1024
#set up chunk sizes
memoryPerElement = 4*(settings["sourceDims"]*2+2) + 20*4 #this is an estimated memory used per element
settings["chunkSize"] = min(int(math.ceil(float(settings["memSize"])/memoryPerElement)),settings["dataLength"])
settings["lastChunkSize"] = ((settings["dataLength"]-1) % settings["chunkSize"]) + 1
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"],1,1)
settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"])/settings["maxThreads"])),1),1,1)
#precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(k)
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def APSPConfig(dataTable, knn_m, eps=100000000., gpuMemSize=512, settings={}):
"""
Creates all the memory/data settings to run GPU accelerated APSP.
"""
settings = dataConfig(dataTable, settings)
settings["dataLength"] = len(knn_m) - 1
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize * 1024 * 1024
settings["maxThreads"] = 1024
#set up chunk sizes - in this case degenerate
settings["chunkSize"] = settings["dataLength"]
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"], 1, 1)
settings["grid"] = (max(
int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])),
1), 1, 1)
#precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(settings["sourceDims"])
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def APSPConfig(dataTable, knn_m, eps=100000000.0, gpuMemSize=512, settings={}):
"""
Creates all the memory/data settings to run GPU accelerated APSP.
"""
settings = dataConfig(dataTable, settings)
settings["dataLength"] = len(knn_m) - 1
# XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize * 1024 * 1024
settings["maxThreads"] = 1024
# set up chunk sizes - in this case degenerate
settings["chunkSize"] = settings["dataLength"]
# create kernel gridsize tuples
settings["block"] = (settings["maxThreads"], 1, 1)
settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])), 1), 1, 1)
# precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(settings["sourceDims"])
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def NMDSConfig(dataTable, targetDims, gpuMemSize = 512, settings = {}):
"""
Creates all the memory/data settings to run GPU accelerated APSP.
"""
#XXX: this needs replacing
settings = dataConfig(dataTable,settings)
settings["memSize"] = gpuMemSize*1024*1024
settings["targetDims"] = targetDims
knnMem = (4*(settings["targetDims"]))*(settings["dataLength"]+4) #we have ( k * size( float + int ) * dataLength ) for our neighbour list
chunkMem = settings["memSize"]-knnMem
if chunkMem < settings["sourceDims"]*4*4:
raise "GPU memory too small for KNN list!"
chunkMem /= settings["sourceDims"]*settings["targetDims"]*4
settings["chunkSize"] = min(chunkMem,settings["sourceDims"])
settings["lastChunkSize"] = settings["dataLength"] % settings["chunkSize"]
if settings["lastChunkSize"] > 0:
settings["totalChunks"] = settings["dataLength"]/settings["chunkSize"]+1
else:
settings["totalChunks"] = settings["dataLength"]/settings["chunkSize"]
settings["lastChunkSize"] = settings["chunkSize"]
settings["block"] = (min(max(settings["dataLength"],1),1024),1,1) #XXX: fix this for using the device data on max threads
g1 = int(math.ceil(settings["dataLength"]/512.))
g2 = int(math.ceil(g1/512.))
g3 = int(math.ceil(g2/64.))
settings["grid"] = (max(g1,1),max(g2,1),max(g3,1))
return settings
def KNNConfig(dataTable,
srcDims,
k,
eps=1000000000.,
gpuMemSize=512,
settings={}):
"""
Creates all the memory/data settings to run GPU accelerated KNN.
"""
kmax = 2**int(math.ceil(math.log(k + 1) / math.log(2))) + 1
settings["oldk"] = k
settings["kmax"] = kmax
k = kmax
settings = dataConfig(dataTable, settings)
settings["sourceDims"] = min(settings["sourceDims"], srcDims)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize * 1024 * 1024
settings["maxThreads"] = 1024
#set up chunk sizes
memoryPerElement = k * 4 * 2 + kmax * 4 * 2 + (
settings["sourceDims"] * 4
) * 2 + 30 #+ (k+10)*4*2 #this is an estimated memory used per element
MemoryPerElementSquared = 8 * 2
ctr = 0
while memoryPerElement * ctr + ctr * ctr * 8 * 2 < settings[
"memSize"] and ctr * settings["sourceDims"] < 1000000:
ctr += 1
ctr -= 1
#settings = GPUConfig(settings,memoryPerElement,memoryPerElementsquared,1000000/settings["sourceDims"])
settings["chunkSize"] = min(ctr, settings["dataLength"])
settings["lastChunkSize"] = (
(settings["dataLength"] - 1) % settings["chunkSize"]) + 1
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"], 1, 1)
settings["grid"] = ((settings["chunkSize"] / settings["maxThreads"]) +
(settings["chunkSize"] % settings["maxThreads"] > 0),
1, 1)
#precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(k)
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def KMeansConfig(dataTable,
k,
eps=0.00001,
srcDims=100000000000,
gpuMemSize=512,
settings={}):
"""
Creates all the memory/data settings to run GPU accelerated KMeans.
"""
settings = dataConfig(dataTable, settings)
settings["sourceDims"] = min(settings["sourceDims"], srcDims)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize * 1024 * 1024
settings["maxThreads"] = 1024
#set up chunk sizes
memoryPerElement = 4 * (
settings["sourceDims"] * 2 +
2) + 20 * 4 #this is an estimated memory used per element
settings["chunkSize"] = min(
int(math.ceil(float(settings["memSize"]) / memoryPerElement)),
settings["dataLength"])
settings["lastChunkSize"] = (
(settings["dataLength"] - 1) % settings["chunkSize"]) + 1
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"], 1, 1)
settings["grid"] = (max(
int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])),
1), 1, 1)
#precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(k)
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def QEConfig(dataTable, dims, k = -1, eps = 0.00001,gpuMemSize = 512, settings = {}):
settings = dataConfig(dataTable,settings)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize*1024*1024
settings["maxThreads"] = 1024
#set up chunk sizes - in this case degenerate
settings["chunkSize"] = settings["dataLength"]
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"],1,1)
settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"])/settings["maxThreads"])),1),1,1)
settings["k"] = k
if k < 0:
settings["k"] = min(settings["dataLength"]/125,50)
settings["delta"] = eps
settings["targetDims"] = dims
return settings
def KNNConfig(dataTable,srcDims, k, eps = 1000000000.,gpuMemSize = 512, settings = {}):
"""
Creates all the memory/data settings to run GPU accelerated KNN.
"""
kmax = 2**int(math.ceil(math.log(k+1)/math.log(2)))+1
settings["oldk"] = k
settings["kmax"] = kmax
k = kmax
settings = dataConfig(dataTable,settings)
settings["sourceDims"] = min(settings["sourceDims"],srcDims)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize*1024*1024
settings["maxThreads"] = 1024
#set up chunk sizes
memoryPerElement = k*4*2+kmax*4*2 + (settings["sourceDims"]*4)*2 + 30 #+ (k+10)*4*2 #this is an estimated memory used per element
MemoryPerElementSquared = 8*2
ctr = 0
while memoryPerElement*ctr + ctr*ctr*8*2 < settings["memSize"] and ctr*settings["sourceDims"]<1000000:
ctr += 1
ctr -= 1
#settings = GPUConfig(settings,memoryPerElement,memoryPerElementsquared,1000000/settings["sourceDims"])
settings["chunkSize"] = min(ctr,settings["dataLength"])
settings["lastChunkSize"] = ((settings["dataLength"]-1) % settings["chunkSize"]) + 1
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"],1,1)
settings["grid"] = ((settings["chunkSize"]/settings["maxThreads"])+(settings["chunkSize"]%settings["maxThreads"]>0),1,1)
#precalculate all constant kernel params
settings["dimensions"] = numpy.int64(settings["sourceDims"])
settings["k"] = numpy.int64(k)
settings["eps"] = numpy.float32(eps)
settings["dataSize"] = numpy.int64(settings["dataLength"])
settings["chunkSize"] = numpy.int64(settings["chunkSize"])
settings["maxThreads"] = numpy.int64(settings["maxThreads"])
return settings
def QEConfig(dataTable, dims, k=-1, eps=0.00001, gpuMemSize=512, settings={}):
settings = dataConfig(dataTable, settings)
#XXX: determine memory and thread sizes from device
settings["memSize"] = gpuMemSize * 1024 * 1024
settings["maxThreads"] = 1024
#set up chunk sizes - in this case degenerate
settings["chunkSize"] = settings["dataLength"]
#create kernel gridsize tuples
settings["block"] = (settings["maxThreads"], 1, 1)
settings["grid"] = (max(
int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])),
1), 1, 1)
settings["k"] = k
if k < 0:
settings["k"] = min(settings["dataLength"] / 125, 50)
settings["delta"] = eps
settings["targetDims"] = dims
return settings
