def main(): idx = 0 if (len(sys.argv) > 1): idx = int(sys.argv[1]) else: sys.stderr.write("Slab not provided\n") sys.exit(1) for z in range(20): x, y = slabs[idx] #print "processing polygons in", z, " for Slab :", x,y query = "project(cross_join(cross_join(normal_groups_dim, store(redimension(project(apply(filter(between(normal_obs_%d,%d,%d,%d,%d),polygon is not null),check,true),oid,check), temp), temp) as b, normal_groups_dim.oid,b.oid) as A,normal_groups_centers as C,A.group,C.group),x_avg,y_avg)" % ( z, x, y, x + U3, y + U3) #print query result = db.executeQuery(query, "afl") xlist = [] ylist = [] ### desc = result.array.getArrayDesc() dims = desc.getDimensions() attrs = desc.getAttributes() iters = [] for i in range(attrs.size()): attrid = attrs[i].getId() iters.append(result.array.getConstIterator(attrid)) nc = 0 while not iters[0].end(): for i in range(attrs.size()): if (attrs[i].getName() == "EmptyTag"): continue currentchunk = iters[i].getChunk() chunkiter = currentchunk.getConstIterator( (scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunkiter.end(): if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() item = scidb.getTypedValue(dataitem, attrs[i].getType()) if (i == 0): xlist.append(item) if (i == 1): ylist.append(item) chunkiter.increment_to_next() nc += 1 for i in range(attrs.size()): iters[i].increment_to_next() #print len(xlist), "Points to fetch raw data" #print xlist,ylist db.completeQuery(result.queryID) #fetch the window from the raw data for i in range(len(xlist)): within(xlist[i], ylist[i]) #Disconnect from the SciDB server. db.disconnect() sys.exit(0) #success
def main(): idx=0 if (len(sys.argv) > 1): idx=int(sys.argv[1]) else: sys.stderr.write("Slab not provided\n") sys.exit(1) for z in range(20): x,y=slabs[idx] #print "processing polygons in", z, " for Slab :", x,y query="project(cross_join(cross_join(normal_groups_dim, store(redimension(project(apply(filter(between(normal_obs_%d,%d,%d,%d,%d),polygon is not null),check,true),oid,check), temp), temp) as b, normal_groups_dim.oid,b.oid) as A,normal_groups_centers as C,A.group,C.group),x_avg,y_avg)" %(z,x,y,x+U3,y+U3) #print query result=db.executeQuery(query,"afl") xlist=[] ylist=[] ### desc = result.array.getArrayDesc() dims = desc.getDimensions() attrs = desc.getAttributes() iters = [] for i in range (attrs.size()): attrid = attrs[i].getId() iters.append(result.array.getConstIterator(attrid)) nc = 0; while not iters[0].end(): for i in range (attrs.size()): if (attrs[i].getName() == "EmptyTag"): continue currentchunk = iters[i].getChunk() chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunkiter.end(): if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() item = scidb.getTypedValue(dataitem, attrs[i].getType()) if(i==0): xlist.append(item) if(i==1): ylist.append(item) chunkiter.increment_to_next() nc += 1; for i in range(attrs.size()): iters[i].increment_to_next(); #print len(xlist), "Points to fetch raw data" #print xlist,ylist db.completeQuery(result.queryID) #fetch the window from the raw data for i in range(len(xlist)): within(xlist[i],ylist[i]) #Disconnect from the SciDB server. db.disconnect() sys.exit(0) #success
def getMultiArrFromQueryForJSON(query_result,options): dimnames = options['dimnames'] desc = query_result.array.getArrayDesc() dims = desc.getDimensions() # list of DimensionDesc objects attrs = desc.getAttributes() # list of AttributeDesc objects if(dims.size() < 1 or dims.size() != len(dimnames)): return [] alldata = {} alldims = {} its = [] for i in range(attrs.size()-1): # find the right attrid its.append(query_result.array.getConstIterator(i)) for i in range(len(its)): # find the right attrid it = its[i] data = [] dims = [] while not it.end(): chunk = it.getChunk() chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) while not chunkiter.end(): temp = [] pos = chunkiter.getPosition() for dimi in range(len(pos)): temp.append(pos[dimi]) dims.append(temp) dataitem = chunkiter.getItem() typeddataitem = scidb.getTypedValue(dataitem, attrs[i].getType()) data.append(typeddataitem) chunkiter.increment_to_next() it.increment_to_next() alldata[attrs[i].getName()] = data alldims[attrs[i].getName()] = dims namesobj = {'attrs':[],'dims':[]} typesobj = {'attrs':{},'dims':{}} dimmap = {} for attri in range(attrs.size()-1): attrname = attrs[attri].getName() namesobj['attrs'].append(attrname) typesobj['attrs'][attrname] = attrs[attri].getType() for index in range(len(dimnames)): dimname = dimnames[index] namesobj['dims'].append(dimname) typesobj['dims'][dimname] = "int32" dimmap[dimname] = index return {'attrs':alldata,'dims':alldims, 'dimmap':dimmap, 'names': namesobj, 'types': typesobj}
def main(): query = "aggregate(cross_join(normal_groups,filter(sum(project(apply(cross_join(subarray ( Points, 0,3 ),join(subarray(normal_groups, NULL,NULL,NULL,18) AS Pi,subarray(normal_groups, NULL,1,NULL,NULL) AS Pj)),crosses,iif (((((Pi.y <= Points.y) AND (Pj.y > Points.y)) OR ((Pi.y > Points.y) AND (Pj.y <= Points.y))) AND (Points.x < Pi.x + ((Points.y - Pi.y) / (Pj.y - Pi.y)) * (Pj.x - Pi.x)) AND (Pi.x is not null and Pi.y is not null and Pj.x is not null and Pj.y is not null)), 1, 0)),crosses),crosses,Pj.group), crosses_sum > 0)),avg(normal_groups.x),avg(normal_groups.y),normal_groups.group)" result = db.executeQuery(query, "afl") xlist = [] ylist = [] ### desc = result.array.getArrayDesc() dims = desc.getDimensions() attrs = desc.getAttributes() iters = [] for i in range(attrs.size()): attrid = attrs[i].getId() iters.append(result.array.getConstIterator(attrid)) nc = 0 while not iters[0].end(): for i in range(attrs.size()): if attrs[i].getName() == "EmptyTag": continue currentchunk = iters[i].getChunk() chunkiter = currentchunk.getConstIterator( (scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS) ) while not chunkiter.end(): if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() item = scidb.getTypedValue(dataitem, attrs[i].getType()) if i == 0: xlist.append(item) if i == 1: ylist.append(item) chunkiter.increment_to_next() nc += 1 for i in range(attrs.size()): iters[i].increment_to_next() # print len(xlist), "Points to fetch raw data" db.completeQuery(result.queryID) # fetch the window from the raw data for i in range(len(xlist)): within(xlist[i], ylist[i]) # Disconnect from the SciDB server. db.disconnect() sys.exit(0) # success
def get_single_result(db, query): result = db.executeQuery(query, "afl") desc = result.array.getArrayDesc() attrs = desc.getAttributes() attrid = attrs[0].getId() attiter = result.array.getConstIterator(attrid) chunk = attiter.getChunk() chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() val = scidb.getTypedValue(dataitem, attrs[0].getType()) else: val = "#Error!" db.completeQuery(result.queryID) return val
def main(): query = "aggregate(cross_join(normal_groups,filter(sum(project(apply(cross_join(subarray ( Points, 0,3 ),join(subarray(normal_groups, NULL,NULL,NULL,18) AS Pi,subarray(normal_groups, NULL,1,NULL,NULL) AS Pj)),crosses,iif (((((Pi.y <= Points.y) AND (Pj.y > Points.y)) OR ((Pi.y > Points.y) AND (Pj.y <= Points.y))) AND (Points.x < Pi.x + ((Points.y - Pi.y) / (Pj.y - Pi.y)) * (Pj.x - Pi.x)) AND (Pi.x is not null and Pi.y is not null and Pj.x is not null and Pj.y is not null)), 1, 0)),crosses),crosses,Pj.group), crosses_sum > 0)),avg(normal_groups.x),avg(normal_groups.y),normal_groups.group)" result = db.executeQuery(query, "afl") xlist = [] ylist = [] ### desc = result.array.getArrayDesc() dims = desc.getDimensions() attrs = desc.getAttributes() iters = [] for i in range(attrs.size()): attrid = attrs[i].getId() iters.append(result.array.getConstIterator(attrid)) nc = 0 while not iters[0].end(): for i in range(attrs.size()): if (attrs[i].getName() == "EmptyTag"): continue currentchunk = iters[i].getChunk() chunkiter = currentchunk.getConstIterator( (scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunkiter.end(): if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() item = scidb.getTypedValue(dataitem, attrs[i].getType()) if (i == 0): xlist.append(item) if (i == 1): ylist.append(item) chunkiter.increment_to_next() nc += 1 for i in range(attrs.size()): iters[i].increment_to_next() #print len(xlist), "Points to fetch raw data" db.completeQuery(result.queryID) #fetch the window from the raw data for i in range(len(xlist)): within(xlist[i], ylist[i]) #Disconnect from the SciDB server. db.disconnect() sys.exit(0) #success
def next(self): """return the next row of data if it is available otherwise raise StopIteration""" chunk_iter = self.chunk_iterators[0][1] if chunk_iter.end(): # increment each attribute iterator to the next chunk for iterator in self.attr_iterators: iterator[1].increment_to_next() # attempt to update the chunk iterators based on the incremented # attribute iterators if self.__update(): raise StopIteration # get dimension values at this position coordinates = self.chunk_iterators[0][1].getPosition( ) # get the first tuple from the list, # then get the second item in the tuple pos_list = [] for i in range(len(coordinates)): pos_list.append(coordinates[i]) # get attribute values at this position attr_values = [] for chunk_iterator in self.chunk_iterators: attr = chunk_iterator[0] chunk_iter = chunk_iterator[1] if not chunk_iter.isEmpty(): attr_values.append( scidbapi.getTypedValue(chunk_iter.getItem(), attr.getType())) else: attr_values.append(None) chunk_iter.increment_to_next() return pos_list, attr_values
def get_column(db, query): values = [] result = db.executeQuery(query, "afl") desc = result.array.getArrayDesc() attrs = desc.getAttributes() attrid = attrs[0].getId() attiter = result.array.getConstIterator(attrid) while not attiter.end(): chunk = attiter.getChunk() chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunkiter.end(): if not chunkiter.isEmpty(): dataitem = chunkiter.getItem() val = scidb.getTypedValue(dataitem, attrs[0].getType()) values.append(val) else: val = "#Error!" chunkiter.increment_to_next() attiter.increment_to_next() db.completeQuery(result.queryID) return values
def load(self, att_list): assert len(att_list) > 0 with Scidb(self.host, self.port) as db: self.array = db._query(self.afl) data_list = [self.data[att] for att in att_list] ##TODO optimize here while not data_list[0].iterator.end(): while not data_list[0].chunk.iter.end(): pos_ = [dim.relative_position(data_list[0]. \ chunk.iter.getPosition()[i]) for i, dim in enumerate(self.dimensions)] for data in data_list: value = data.chunk.iter.getItem() if not value.isNull(): data.array[tuple(pos_)] = scidbapi.getTypedValue(value, data.att.Type) data.chunk.iter.increment_to_next() for data in data_list: data.iterator.increment_to_next() try: data.set_chunk() except: continue self.attsLoaded.extend(att_list)
def next(self): """return the next row of data if it is available otherwise raise StopIteration""" first_chunk_iter = self.__attr_chunk_iter_tuple_list[0][1] #if the chunk is finished, try to increment to the next chunk if (first_chunk_iter.end()): #increment each attribute iterator to the next chunk for attr_iter_tuple in self.__attr_iter_tuple_list: attr_iter_tuple[1].increment_to_next() #attempt to update the chunk iterators based on the incremented #attribute iterators if self.__update_attr_chunk_iter(): raise StopIteration #get dimension values at this position coordinates = self.__attr_chunk_iter_tuple_list[0][1].getPosition() #get the first tuple from the list, #then get the second item in the tuple pos_list = list() for i in range(len(coordinates)): pos_list.append(coordinates[i]) #get attribute values at this position attr_values = [] for attr_chunk_iter in self.__attr_chunk_iter_tuple_list: attr = attr_chunk_iter[0] chunk_iter = attr_chunk_iter[1] if (not chunk_iter.isEmpty()): attr_values.append(scidbapi.getTypedValue(chunk_iter.getItem(), attr.getType())) else: attr_values.append(None) chunk_iter.increment_to_next() return (pos_list, attr_values)
def print_dataitem(dtypeStr, dataitem, innerAttrList): value = scidb.getTypedValue(dataitem, dtypeStr) # print "Data: %s" % value innerAttrList.append(value)
def getAllAttrArrFromQueryForNP(query_result,attrnames=None,usenan=False): #if DEBUG_PRINT: print "parsing query result and building json dump",datetime.now() desc = query_result.array.getArrayDesc() dims = desc.getDimensions() # list of DimensionDesc objects attrs = desc.getAttributes() # list of AttributeDesc objects if(dims.size() < 1): return None dimtuple = [] for i,dim in enumerate(dims): dimtuple.append(dim.getCurrLength()) dimtuple = tuple(dimtuple) arrs = [] itemsets = [] its = [] getattrnames = False if attrnames is None: attrnames = [] getattrnames = True minobj = {} maxobj = {} for i in range(attrs.size()): # find the right attrid, "EmptyTag" should always be at the end if attrs[i].getName() != "EmptyTag": arrs.append(np.zeros(dimtuple)) # new array for each attribute itemsets.append(arrs[i].itemset) #used to insert items) if usenan: arrs[i].fill(np.nan) # fill with nulls if necessary its.append(query_result.array.getConstIterator(i)) if getattrnames: attrnames.append(attrs[i].getName()) #currtype = attrs[i].getType() while not its[0].end(): #get chunk iterators chunkiters = [] for itindex in range(len(its)): #if DEBUG_PRINT: print "itindex: ",itindex #mypos = its[itindex].getPosition() #if DEBUG_PRINT: print "position:" #if DEBUG_PRINT: print mypos[0],",",mypos[1] currentchunk =its[itindex].getChunk() chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) chunkiters.append(chunkiter) while not chunkiters[0].end(): currpos = chunkiters[0].getPosition() cpl = len(currpos) currpostuple = [0]*cpl for i in range(cpl): currpostuple[i] = currpos[i] currpostuple = tuple(currpostuple) #testcurrpos = tuple(currpos) #this is bad! causes memory leaks... for chunkiterindex in range(len(chunkiters)): #if DEBUG_PRINT: print "chunkiterindex: ",chunkiterindex dataitem = chunkiters[chunkiterindex].getItem() # look up the value according to its attribute's typestring currtype = attrs[chunkiterindex].getType() dataitem_val = scidb.getTypedValue(dataitem, currtype) itemsets[chunkiterindex](currpostuple,dataitem_val) # insert into every numpy array chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS for itindex in range(len(its)): its[itindex].increment_to_next() #if DEBUG_PRINT: print "done parsing results, returning dump-ready version",datetime.now() #if DEBUG_PRINT: print arrs if DEBUG_PRINT: print arrs[0].shape return {'data':arrs,'attrnames':attrnames,'stats':get_stats(arrs)}
result = db.executeQuery(query, "afl") db.completeQuery(result.queryID) f.write(query + '\n') else: print query #compute norm^2 for input tensor query = "select sum(pow(val,2)) from " + tensor if debug == 0: print query f.write(query + '\n') result = db.executeQuery(query, "aql") dataitem = result.array.getConstIterator( 0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query #compute norm^2 for core query = "select sum(pow(val,2)) from " + core if debug == 0: print query f.write(query + '\n') result = db.executeQuery(query, "aql") dataitem = result.array.getConstIterator( 0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normcore = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID)
def getOneAttrArrFromQuery(query_result,attrname): desc = query_result.array.getArrayDesc() dims = desc.getDimensions() # list of DimensionDesc objects attrs = desc.getAttributes() # list of AttributeDesc objects dimlengths= [] dimchunkintervals = [] dimoverlaps = [] dimindexes = [] dimindexesbase = [] if(dims.size() < 1): return []; for i in range(dims.size()): dimlengths.append(dims[i].getLength()) dimchunkintervals.append(dims[i].getChunkInterval()) dimoverlaps.append(dims[i].getChunkOverlap()) dimindexes.append(0) dimindexesbase.append(0) # get arr ready arr = createArray(dimlengths) #print "arr is initialized: ",str(arr) attrid = 0 for i in range(attrs.size()): # find the right attrid if(attrs[i].getName() == attrname): attrid = i #print "found attribute",attrname, ",id: %d" % attrid break # get the iterator for this attrid it = query_result.array.getConstIterator(attrid) start = True while not it.end(): #print "iterating over items..." currentchunk = it.getChunk() # TODO: will have to fix this at some point, can't just ignore empty cells or overlaps chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) if(not start): # don't update for the first chunk #update base indexes dimindexesbase = updateBaseIndex(dimindexesbase,dimlengths,dimchunkintervals) #printIndexes(dimindexesbase) verifyIndexes(dimindexesbase,dimlengths) #reset the indexes to new base indexes for i in range (dims.size()): dimindexes[i] = dimindexesbase[i] else: start = False while not chunkiter.end(): #printIndexes(dimindexes) verifyIndexes(dimindexes,dimlengths) dataitem = chunkiter.getItem() # look up the value according to its attribute's typestring item = scidb.getTypedValue(dataitem, attrs[attrid].getType()) # TBD: eliminate 2nd arg, make method on dataitem #print "Data: %s" % item #insert the item arr = insertItem(arr,item,dimindexes) #update the indexes dimindexes = updateIndexes(dimindexes,dimchunkintervals,dimindexesbase,dimlengths) lastpos = chunkiter.getPosition() #print lastpos[0],",",lastpos[1], ",",lastpos[2] chunkiter.increment_to_next() #print "current state of arr: ", str(arr) it.increment_to_next(); return arr
def getAllAttrArrFromQuery(query_result): desc = query_result.array.getArrayDesc() dims = desc.getDimensions() # list of DimensionDesc objects attrs = desc.getAttributes() # list of AttributeDesc objects dimlengths= [] dimchunkintervals = [] dimoverlaps = [] dimindexes = [] dimindexesbase = [] if(dims.size() < 1): return [] for i in range(dims.size()): dimlengths.append(dims[i].getLength()) dimchunkintervals.append(dims[i].getChunkInterval()) dimoverlaps.append(dims[i].getChunkOverlap()) dimindexes.append(0) dimindexesbase.append(0) # get arr ready arr = createArray(dimlengths) #print "arr is initialized: ",str(arr) its = [] attrnames = [] for i in range(attrs.size()): # find the right attrid its.append(query_result.array.getConstIterator(i)) attrnames.append(attrs[i].getName()) start = True while not its[0].end(): #get chunk iterators chunkiters = [] for itindex in range(len(its)): #print "itindex: ",itindex currentchunk =its[itindex].getChunk() chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) chunkiters.append(chunkiter) if(not start): # don't update for the first chunk #update base indexes dimindexesbase = updateBaseIndex(dimindexesbase,dimlengths,dimchunkintervals) #printIndexes(dimindexesbase) verifyIndexes(dimindexesbase,dimlengths) #reset the indexes to new base indexes for i in range (dims.size()): dimindexes[i] = dimindexesbase[i] else: start = False while not chunkiters[0].end(): #printIndexes(dimindexes) verifyIndexes(dimindexes,dimlengths) item = {} #empty dictionary for the attribute values for chunkiterindex in range(len(chunkiters)): #print "chunkiterindex: ",chunkiterindex dataitem = chunkiters[chunkiterindex].getItem() # look up the value according to its attribute's typestring item[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem #print "Data: %s" % item #chunkiters[i].increment_to_next() chunkiters[0].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS #lastpos = chunkiter.getPosition() #print lastpos[0],",",lastpos[1], ",",lastpos[2] #print item #insert the item arr = insertItem(arr,item,dimindexes) #update the indexes dimindexes = updateIndexes(dimindexes,dimchunkintervals,dimindexesbase,dimlengths) #print "current state of arr: ", str(arr) its[0].increment_to_next() return arr
print >> sys.stderr, " Exception Type: %s" % type( inst) # the exception instance print >> sys.stderr, " Exception Value: %r" % inst else: print query #compute norm for input query = "select sqrt(sum(pow(val,2))) from " + tensor if debug == 0: print query f.write(query + '\n') result = db.executeQuery(query, "aql") dataitem = result.array.getConstIterator( 0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query fit = 0 fitchangetol = 1e-4 # iterate until the fit converges or for max_iter for r in range(max_iter): fitold = fit for i in range(nmode): # matricized Khatri-Rao product query = "" for k in range(nmode): if k == i:
def print_dataitem(dtypeStr, dataitem): print "Data: %s" % scidb.getTypedValue(dataitem, dtypeStr)
import scidbapi as scidb db = scidb.connect("localhost", 1239) query = db.executeQuery("select sigma from sv", "aql") singular_values = [] iterator = query.array.getConstIterator(query.array.getArrayDesc().getAttributes()[0].getId()) while not iterator.end(): chunk_iterator = iterator.getChunk().getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunk_iterator.end(): singular_values.append(scidb.getTypedValue(chunk_iterator.getItem(), "double")) chunk_iterator.increment_to_next() iterator.increment_to_next() db.completeQuery(query.queryID) db.disconnect() import matplotlib.pyplot as pyplot pyplot.plot(singular_values) pyplot.xlabel("Rank") pyplot.ylabel("Singular Values") pyplot.show()
import scidbapi as scidb import scipy.cluster.vq db = scidb.connect("localhost", 1239) query = db.executeQuery("select v from rsv", "aql") dimensions = query.array.getArrayDesc().getDimensions() features = scipy.zeros((dimensions[0].getEndMax() + 1, dimensions[1].getEndMax() + 1)) iterator = query.array.getConstIterator(query.array.getArrayDesc().getAttributes()[0].getId()) while not iterator.end(): chunk_iterator = iterator.getChunk().getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)) while not chunk_iterator.end(): coordinates = chunk_iterator.getPosition() features[coordinates[0], coordinates[1]] = scidb.getTypedValue(chunk_iterator.getItem(), "double") chunk_iterator.increment_to_next() iterator.increment_to_next() db.completeQuery(query.queryID) db.disconnect() features = scipy.cluster.vq.whiten(features) centroids, distance = scipy.cluster.vq.kmeans(features, 5) clusters, distance = scipy.cluster.vq.vq(features, centroids) print clusters, distance
print query result=db.executeQuery(query,"afl") db.completeQuery(result.queryID) f.write(query+'\n'); else: print query #compute norm^2 for input tensor query="select sum(pow(val,2)) from " + tensor if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query #compute norm^2 for core query="select sum(pow(val,2)) from " + core if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normcore = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else:
def getAttrArrFromQueryForJSON(query_result,options): dimnames = options['dimnames'] attrnames = options['attrnames'] desc = query_result.array.getArrayDesc() dims = desc.getDimensions() # list of DimensionDesc objects attrs = desc.getAttributes() # list of AttributeDesc objects origarrnamelen = len(desc.getName()) - 2 #print "orig name length: ",origarrnamelen if(dims.size() < 1 or dims.size() != len(dimnames)): return [] arr = [] its = [] for i in range(attrs.size()): # find the right attrid for aname in attrnames: if aname == attrs[i].getName(): its.append(query_result.array.getConstIterator(i)) start = True while not its[0].end(): #get chunk iterators chunkiters = [] #print "start" for itindex in range(len(its)): #print "itindex: ",itindex #mypos = its[itindex].getPosition() #print "position:" #print mypos[0],",",mypos[1] currentchunk =its[itindex].getChunk() chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) chunkiters.append(chunkiter) while not chunkiters[0].end(): dataobj = {} dimobj= {} currpos = chunkiters[0].getPosition() for dimindex in range(len(currpos)): dname = dimnames[dimindex] dimobj[dname[:len(dname)-origarrnamelen]] = currpos[dimindex] # make sure you take off the array's name from each dimension dataobj["dims."+dname[:len(dname)-origarrnamelen]] = currpos[dimindex] attrobj = {} #empty dictionary for the attribute values #print "start" for chunkiterindex in range(len(chunkiters)): #print "chunkiterindex: ",chunkiterindex dataitem = chunkiters[chunkiterindex].getItem() # look up the value according to its attribute's typestring attrobj[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem dataobj["attrs."+attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) #print "Data: %s" % item #chunkiters[i].increment_to_next() #mypos = chunkiters[chunkiterindex].getPosition() #myposstring = "position: " #for myposi in range(len(mypos)): # myposstring += str(mypos[myposi])+", " #print myposstring chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS #lastpos = chunkiter.getPosition() #print lastpos[0],",",lastpos[1], ",",lastpos[2] #print attrobj #insert the item arr.append(dataobj) #arr.append({'dimensions':dimobj,'attributes':attrobj}) #print "current state of arr: ", str(arr) #its[1].increment_to_next() for itindex in range(len(its)): its[itindex].increment_to_next() namesobj = [] typesobj = {} for attri in range(len(attrnames)): attrname = attrnames[attri] namesobj.append({'name':"attrs."+attrname,'isattr':True}) typesobj["attrs."+attrname] = attrs[attri].getType() for dimname in dimnames: ndimname = "dims."+dimname[:len(dimname)-origarrnamelen] namesobj.append({'name':ndimname,'isattr':False}) typesobj[ndimname] = "int32" #print typesobj #print json.dumps({'data':arr, 'names': namesobj, 'types': typesobj}) return {'data':arr, 'names': namesobj, 'types': typesobj}
def getDataInJson(query_result): print "Parsing query result into JSON", datetime.now() desc = query_result.array.getArrayDesc() # Dimensions dims = desc.getDimensions() dimnames = [] for i in range(dims.size()): if dims[i].getBaseName() != "EmptyTag": dimnames.append(dims[i].getBaseName()) arr = [] its = [] # Attributes attrs = desc.getAttributes() attrnames = [] for i in range(attrs.size()): if attrs[i].getName() != "EmptyTag": its.append(query_result.array.getConstIterator(i)) attrnames.append(attrs[i].getName()) while not its[0].end(): chunkiters = [] for itindex in range(len(its)): currentchunk =its[itindex].getChunk() chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS | scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)) chunkiters.append(chunkiter) while not chunkiters[0].end(): dataobj = {} dimobj= {} currpos = chunkiters[0].getPosition() for dimindex in range(len(currpos)): #print dimindex dname = dimnames[dimindex] #print "dname" , dname dimobj[dname[:len(dname)]] = currpos[dimindex] # make sure you take off the array's name from each dimension dataobj["dims."+dname[:len(dname)]] = currpos[dimindex] attrobj = {} #empty dictionary for the attribute values #print "start" for chunkiterindex in range(len(chunkiters)): dataitem = chunkiters[chunkiterindex].getItem() attrobj[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem dataobj["attrs."+attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS arr.append(dataobj) for itindex in range(len(its)): its[itindex].increment_to_next() namesobj = [] typesobj = {} for attri in range(len(attrnames)): attrname = attrnames[attri] namesobj.append({'name':"attrs."+attrname,'isattr':True}) typesobj["attrs."+attrname] = attrs[attri].getType() for dimname in dimnames: ndimname = "dims."+dimname[:len(dimname)] namesobj.append({'name':ndimname,'isattr':False}) typesobj[ndimname] = "int32" print "Done parsing query results, returning dump-ready version", datetime.now() return {'data':arr, 'names': namesobj, 'types': typesobj}
db.completeQuery(result.queryID) except Exception, inst: print >> sys.stderr, " Exception Type: %s" % type(inst) # the exception instance print >> sys.stderr, " Exception Value: %r" % inst else: print query #compute norm for input query="select sqrt(sum(pow(val,2))) from " + tensor if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query fit = 0; fitchangetol = 1e-4; # iterate until the fit converges or for max_iter for r in range(max_iter): fitold = fit; for i in range(nmode): # matricized Khatri-Rao product query = "" for k in range(nmode): if k == i: