def KMeansConfig(dataTable, k, eps = 0.00001, srcDims=100000000000,gpuMemSize = 512, settings = {}): """ Creates all the memory/data settings to run GPU accelerated KMeans. """ settings = dataConfig(dataTable,settings) settings["sourceDims"] = min(settings["sourceDims"],srcDims) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize*1024*1024 settings["maxThreads"] = 1024 #set up chunk sizes memoryPerElement = 4*(settings["sourceDims"]*2+2) + 20*4 #this is an estimated memory used per element settings["chunkSize"] = min(int(math.ceil(float(settings["memSize"])/memoryPerElement)),settings["dataLength"]) settings["lastChunkSize"] = ((settings["dataLength"]-1) % settings["chunkSize"]) + 1 #create kernel gridsize tuples settings["block"] = (settings["maxThreads"],1,1) settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"])/settings["maxThreads"])),1),1,1) #precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(k) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def APSPConfig(dataTable, knn_m, eps=100000000., gpuMemSize=512, settings={}): """ Creates all the memory/data settings to run GPU accelerated APSP. """ settings = dataConfig(dataTable, settings) settings["dataLength"] = len(knn_m) - 1 #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize * 1024 * 1024 settings["maxThreads"] = 1024 #set up chunk sizes - in this case degenerate settings["chunkSize"] = settings["dataLength"] #create kernel gridsize tuples settings["block"] = (settings["maxThreads"], 1, 1) settings["grid"] = (max( int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])), 1), 1, 1) #precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(settings["sourceDims"]) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def APSPConfig(dataTable, knn_m, eps=100000000.0, gpuMemSize=512, settings={}): """ Creates all the memory/data settings to run GPU accelerated APSP. """ settings = dataConfig(dataTable, settings) settings["dataLength"] = len(knn_m) - 1 # XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize * 1024 * 1024 settings["maxThreads"] = 1024 # set up chunk sizes - in this case degenerate settings["chunkSize"] = settings["dataLength"] # create kernel gridsize tuples settings["block"] = (settings["maxThreads"], 1, 1) settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])), 1), 1, 1) # precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(settings["sourceDims"]) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def NMDSConfig(dataTable, targetDims, gpuMemSize = 512, settings = {}): """ Creates all the memory/data settings to run GPU accelerated APSP. """ #XXX: this needs replacing settings = dataConfig(dataTable,settings) settings["memSize"] = gpuMemSize*1024*1024 settings["targetDims"] = targetDims knnMem = (4*(settings["targetDims"]))*(settings["dataLength"]+4) #we have ( k * size( float + int ) * dataLength ) for our neighbour list chunkMem = settings["memSize"]-knnMem if chunkMem < settings["sourceDims"]*4*4: raise "GPU memory too small for KNN list!" chunkMem /= settings["sourceDims"]*settings["targetDims"]*4 settings["chunkSize"] = min(chunkMem,settings["sourceDims"]) settings["lastChunkSize"] = settings["dataLength"] % settings["chunkSize"] if settings["lastChunkSize"] > 0: settings["totalChunks"] = settings["dataLength"]/settings["chunkSize"]+1 else: settings["totalChunks"] = settings["dataLength"]/settings["chunkSize"] settings["lastChunkSize"] = settings["chunkSize"] settings["block"] = (min(max(settings["dataLength"],1),1024),1,1) #XXX: fix this for using the device data on max threads g1 = int(math.ceil(settings["dataLength"]/512.)) g2 = int(math.ceil(g1/512.)) g3 = int(math.ceil(g2/64.)) settings["grid"] = (max(g1,1),max(g2,1),max(g3,1)) return settings
def KNNConfig(dataTable, srcDims, k, eps=1000000000., gpuMemSize=512, settings={}): """ Creates all the memory/data settings to run GPU accelerated KNN. """ kmax = 2**int(math.ceil(math.log(k + 1) / math.log(2))) + 1 settings["oldk"] = k settings["kmax"] = kmax k = kmax settings = dataConfig(dataTable, settings) settings["sourceDims"] = min(settings["sourceDims"], srcDims) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize * 1024 * 1024 settings["maxThreads"] = 1024 #set up chunk sizes memoryPerElement = k * 4 * 2 + kmax * 4 * 2 + ( settings["sourceDims"] * 4 ) * 2 + 30 #+ (k+10)*4*2 #this is an estimated memory used per element MemoryPerElementSquared = 8 * 2 ctr = 0 while memoryPerElement * ctr + ctr * ctr * 8 * 2 < settings[ "memSize"] and ctr * settings["sourceDims"] < 1000000: ctr += 1 ctr -= 1 #settings = GPUConfig(settings,memoryPerElement,memoryPerElementsquared,1000000/settings["sourceDims"]) settings["chunkSize"] = min(ctr, settings["dataLength"]) settings["lastChunkSize"] = ( (settings["dataLength"] - 1) % settings["chunkSize"]) + 1 #create kernel gridsize tuples settings["block"] = (settings["maxThreads"], 1, 1) settings["grid"] = ((settings["chunkSize"] / settings["maxThreads"]) + (settings["chunkSize"] % settings["maxThreads"] > 0), 1, 1) #precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(k) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def KMeansConfig(dataTable, k, eps=0.00001, srcDims=100000000000, gpuMemSize=512, settings={}): """ Creates all the memory/data settings to run GPU accelerated KMeans. """ settings = dataConfig(dataTable, settings) settings["sourceDims"] = min(settings["sourceDims"], srcDims) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize * 1024 * 1024 settings["maxThreads"] = 1024 #set up chunk sizes memoryPerElement = 4 * ( settings["sourceDims"] * 2 + 2) + 20 * 4 #this is an estimated memory used per element settings["chunkSize"] = min( int(math.ceil(float(settings["memSize"]) / memoryPerElement)), settings["dataLength"]) settings["lastChunkSize"] = ( (settings["dataLength"] - 1) % settings["chunkSize"]) + 1 #create kernel gridsize tuples settings["block"] = (settings["maxThreads"], 1, 1) settings["grid"] = (max( int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])), 1), 1, 1) #precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(k) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def QEConfig(dataTable, dims, k = -1, eps = 0.00001,gpuMemSize = 512, settings = {}): settings = dataConfig(dataTable,settings) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize*1024*1024 settings["maxThreads"] = 1024 #set up chunk sizes - in this case degenerate settings["chunkSize"] = settings["dataLength"] #create kernel gridsize tuples settings["block"] = (settings["maxThreads"],1,1) settings["grid"] = (max(int(math.ceil(float(settings["chunkSize"])/settings["maxThreads"])),1),1,1) settings["k"] = k if k < 0: settings["k"] = min(settings["dataLength"]/125,50) settings["delta"] = eps settings["targetDims"] = dims return settings
def KNNConfig(dataTable,srcDims, k, eps = 1000000000.,gpuMemSize = 512, settings = {}): """ Creates all the memory/data settings to run GPU accelerated KNN. """ kmax = 2**int(math.ceil(math.log(k+1)/math.log(2)))+1 settings["oldk"] = k settings["kmax"] = kmax k = kmax settings = dataConfig(dataTable,settings) settings["sourceDims"] = min(settings["sourceDims"],srcDims) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize*1024*1024 settings["maxThreads"] = 1024 #set up chunk sizes memoryPerElement = k*4*2+kmax*4*2 + (settings["sourceDims"]*4)*2 + 30 #+ (k+10)*4*2 #this is an estimated memory used per element MemoryPerElementSquared = 8*2 ctr = 0 while memoryPerElement*ctr + ctr*ctr*8*2 < settings["memSize"] and ctr*settings["sourceDims"]<1000000: ctr += 1 ctr -= 1 #settings = GPUConfig(settings,memoryPerElement,memoryPerElementsquared,1000000/settings["sourceDims"]) settings["chunkSize"] = min(ctr,settings["dataLength"]) settings["lastChunkSize"] = ((settings["dataLength"]-1) % settings["chunkSize"]) + 1 #create kernel gridsize tuples settings["block"] = (settings["maxThreads"],1,1) settings["grid"] = ((settings["chunkSize"]/settings["maxThreads"])+(settings["chunkSize"]%settings["maxThreads"]>0),1,1) #precalculate all constant kernel params settings["dimensions"] = numpy.int64(settings["sourceDims"]) settings["k"] = numpy.int64(k) settings["eps"] = numpy.float32(eps) settings["dataSize"] = numpy.int64(settings["dataLength"]) settings["chunkSize"] = numpy.int64(settings["chunkSize"]) settings["maxThreads"] = numpy.int64(settings["maxThreads"]) return settings
def QEConfig(dataTable, dims, k=-1, eps=0.00001, gpuMemSize=512, settings={}): settings = dataConfig(dataTable, settings) #XXX: determine memory and thread sizes from device settings["memSize"] = gpuMemSize * 1024 * 1024 settings["maxThreads"] = 1024 #set up chunk sizes - in this case degenerate settings["chunkSize"] = settings["dataLength"] #create kernel gridsize tuples settings["block"] = (settings["maxThreads"], 1, 1) settings["grid"] = (max( int(math.ceil(float(settings["chunkSize"]) / settings["maxThreads"])), 1), 1, 1) settings["k"] = k if k < 0: settings["k"] = min(settings["dataLength"] / 125, 50) settings["delta"] = eps settings["targetDims"] = dims return settings