def main():
idx = 0
if (len(sys.argv) > 1):
idx = int(sys.argv[1])
else:
sys.stderr.write("Slab not provided\n")
sys.exit(1)
for z in range(20):
x, y = slabs[idx]
#print "processing polygons in", z, " for Slab :", x,y
query = "project(cross_join(cross_join(normal_groups_dim, store(redimension(project(apply(filter(between(normal_obs_%d,%d,%d,%d,%d),polygon is not null),check,true),oid,check), temp), temp) as b, normal_groups_dim.oid,b.oid) as A,normal_groups_centers as C,A.group,C.group),x_avg,y_avg)" % (
z, x, y, x + U3, y + U3)
#print query
result = db.executeQuery(query, "afl")
xlist = []
ylist = []
###
desc = result.array.getArrayDesc()
dims = desc.getDimensions()
attrs = desc.getAttributes()
iters = []
for i in range(attrs.size()):
attrid = attrs[i].getId()
iters.append(result.array.getConstIterator(attrid))
nc = 0
while not iters[0].end():
for i in range(attrs.size()):
if (attrs[i].getName() == "EmptyTag"):
continue
currentchunk = iters[i].getChunk()
chunkiter = currentchunk.getConstIterator(
(scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)
| (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunkiter.end():
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
item = scidb.getTypedValue(dataitem,
attrs[i].getType())
if (i == 0): xlist.append(item)
if (i == 1): ylist.append(item)
chunkiter.increment_to_next()
nc += 1
for i in range(attrs.size()):
iters[i].increment_to_next()
#print len(xlist), "Points to fetch raw data"
#print xlist,ylist
db.completeQuery(result.queryID)
#fetch the window from the raw data
for i in range(len(xlist)):
within(xlist[i], ylist[i])
#Disconnect from the SciDB server.
db.disconnect()
sys.exit(0) #success
def main():
idx=0
if (len(sys.argv) > 1):
idx=int(sys.argv[1])
else:
sys.stderr.write("Slab not provided\n")
sys.exit(1)
for z in range(20):
x,y=slabs[idx]
#print "processing polygons in", z, " for Slab :", x,y
query="project(cross_join(cross_join(normal_groups_dim, store(redimension(project(apply(filter(between(normal_obs_%d,%d,%d,%d,%d),polygon is not null),check,true),oid,check), temp), temp) as b, normal_groups_dim.oid,b.oid) as A,normal_groups_centers as C,A.group,C.group),x_avg,y_avg)" %(z,x,y,x+U3,y+U3)
#print query
result=db.executeQuery(query,"afl")
xlist=[]
ylist=[]
###
desc = result.array.getArrayDesc()
dims = desc.getDimensions()
attrs = desc.getAttributes()
iters = []
for i in range (attrs.size()):
attrid = attrs[i].getId()
iters.append(result.array.getConstIterator(attrid))
nc = 0;
while not iters[0].end():
for i in range (attrs.size()):
if (attrs[i].getName() == "EmptyTag"):
continue
currentchunk = iters[i].getChunk()
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunkiter.end():
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
item = scidb.getTypedValue(dataitem, attrs[i].getType())
if(i==0): xlist.append(item)
if(i==1): ylist.append(item)
chunkiter.increment_to_next()
nc += 1;
for i in range(attrs.size()):
iters[i].increment_to_next();
#print len(xlist), "Points to fetch raw data"
#print xlist,ylist
db.completeQuery(result.queryID)
#fetch the window from the raw data
for i in range(len(xlist)):
within(xlist[i],ylist[i])
#Disconnect from the SciDB server.
db.disconnect()
sys.exit(0) #success
def getMultiArrFromQueryForJSON(query_result,options):
dimnames = options['dimnames']
desc = query_result.array.getArrayDesc()
dims = desc.getDimensions() # list of DimensionDesc objects
attrs = desc.getAttributes() # list of AttributeDesc objects
if(dims.size() < 1 or dims.size() != len(dimnames)):
return []
alldata = {}
alldims = {}
its = []
for i in range(attrs.size()-1): # find the right attrid
its.append(query_result.array.getConstIterator(i))
for i in range(len(its)): # find the right attrid
it = its[i]
data = []
dims = []
while not it.end():
chunk = it.getChunk()
chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
while not chunkiter.end():
temp = []
pos = chunkiter.getPosition()
for dimi in range(len(pos)):
temp.append(pos[dimi])
dims.append(temp)
dataitem = chunkiter.getItem()
typeddataitem = scidb.getTypedValue(dataitem, attrs[i].getType())
data.append(typeddataitem)
chunkiter.increment_to_next()
it.increment_to_next()
alldata[attrs[i].getName()] = data
alldims[attrs[i].getName()] = dims
namesobj = {'attrs':[],'dims':[]}
typesobj = {'attrs':{},'dims':{}}
dimmap = {}
for attri in range(attrs.size()-1):
attrname = attrs[attri].getName()
namesobj['attrs'].append(attrname)
typesobj['attrs'][attrname] = attrs[attri].getType()
for index in range(len(dimnames)):
dimname = dimnames[index]
namesobj['dims'].append(dimname)
typesobj['dims'][dimname] = "int32"
dimmap[dimname] = index
return {'attrs':alldata,'dims':alldims, 'dimmap':dimmap, 'names': namesobj, 'types': typesobj}
def main():
query = "aggregate(cross_join(normal_groups,filter(sum(project(apply(cross_join(subarray ( Points, 0,3 ),join(subarray(normal_groups, NULL,NULL,NULL,18) AS Pi,subarray(normal_groups, NULL,1,NULL,NULL) AS Pj)),crosses,iif (((((Pi.y <= Points.y) AND (Pj.y > Points.y)) OR ((Pi.y > Points.y) AND (Pj.y <= Points.y))) AND (Points.x < Pi.x + ((Points.y - Pi.y) / (Pj.y - Pi.y)) * (Pj.x - Pi.x)) AND (Pi.x is not null and Pi.y is not null and Pj.x is not null and Pj.y is not null)), 1, 0)),crosses),crosses,Pj.group), crosses_sum > 0)),avg(normal_groups.x),avg(normal_groups.y),normal_groups.group)"
result = db.executeQuery(query, "afl")
xlist = []
ylist = []
###
desc = result.array.getArrayDesc()
dims = desc.getDimensions()
attrs = desc.getAttributes()
iters = []
for i in range(attrs.size()):
attrid = attrs[i].getId()
iters.append(result.array.getConstIterator(attrid))
nc = 0
while not iters[0].end():
for i in range(attrs.size()):
if attrs[i].getName() == "EmptyTag":
continue
currentchunk = iters[i].getChunk()
chunkiter = currentchunk.getConstIterator(
(scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS)
)
while not chunkiter.end():
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
item = scidb.getTypedValue(dataitem, attrs[i].getType())
if i == 0:
xlist.append(item)
if i == 1:
ylist.append(item)
chunkiter.increment_to_next()
nc += 1
for i in range(attrs.size()):
iters[i].increment_to_next()
# print len(xlist), "Points to fetch raw data"
db.completeQuery(result.queryID)
# fetch the window from the raw data
for i in range(len(xlist)):
within(xlist[i], ylist[i])
# Disconnect from the SciDB server.
db.disconnect()
sys.exit(0) # success
def get_single_result(db, query):
result = db.executeQuery(query, "afl")
desc = result.array.getArrayDesc()
attrs = desc.getAttributes()
attrid = attrs[0].getId()
attiter = result.array.getConstIterator(attrid)
chunk = attiter.getChunk()
chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
val = scidb.getTypedValue(dataitem, attrs[0].getType())
else:
val = "#Error!"
db.completeQuery(result.queryID)
return val
def main():
query = "aggregate(cross_join(normal_groups,filter(sum(project(apply(cross_join(subarray ( Points, 0,3 ),join(subarray(normal_groups, NULL,NULL,NULL,18) AS Pi,subarray(normal_groups, NULL,1,NULL,NULL) AS Pj)),crosses,iif (((((Pi.y <= Points.y) AND (Pj.y > Points.y)) OR ((Pi.y > Points.y) AND (Pj.y <= Points.y))) AND (Points.x < Pi.x + ((Points.y - Pi.y) / (Pj.y - Pi.y)) * (Pj.x - Pi.x)) AND (Pi.x is not null and Pi.y is not null and Pj.x is not null and Pj.y is not null)), 1, 0)),crosses),crosses,Pj.group), crosses_sum > 0)),avg(normal_groups.x),avg(normal_groups.y),normal_groups.group)"
result = db.executeQuery(query, "afl")
xlist = []
ylist = []
###
desc = result.array.getArrayDesc()
dims = desc.getDimensions()
attrs = desc.getAttributes()
iters = []
for i in range(attrs.size()):
attrid = attrs[i].getId()
iters.append(result.array.getConstIterator(attrid))
nc = 0
while not iters[0].end():
for i in range(attrs.size()):
if (attrs[i].getName() == "EmptyTag"):
continue
currentchunk = iters[i].getChunk()
chunkiter = currentchunk.getConstIterator(
(scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)
| (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunkiter.end():
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
item = scidb.getTypedValue(dataitem, attrs[i].getType())
if (i == 0): xlist.append(item)
if (i == 1): ylist.append(item)
chunkiter.increment_to_next()
nc += 1
for i in range(attrs.size()):
iters[i].increment_to_next()
#print len(xlist), "Points to fetch raw data"
db.completeQuery(result.queryID)
#fetch the window from the raw data
for i in range(len(xlist)):
within(xlist[i], ylist[i])
#Disconnect from the SciDB server.
db.disconnect()
sys.exit(0) #success
def next(self):
"""return the next row of data if it is available otherwise raise StopIteration"""
chunk_iter = self.chunk_iterators[0][1]
if chunk_iter.end():
# increment each attribute iterator to the next chunk
for iterator in self.attr_iterators:
iterator[1].increment_to_next()
# attempt to update the chunk iterators based on the incremented
# attribute iterators
if self.__update():
raise StopIteration
# get dimension values at this position
coordinates = self.chunk_iterators[0][1].getPosition(
) # get the first tuple from the list,
# then get the second item in the tuple
pos_list = []
for i in range(len(coordinates)):
pos_list.append(coordinates[i])
# get attribute values at this position
attr_values = []
for chunk_iterator in self.chunk_iterators:
attr = chunk_iterator[0]
chunk_iter = chunk_iterator[1]
if not chunk_iter.isEmpty():
attr_values.append(
scidbapi.getTypedValue(chunk_iter.getItem(),
attr.getType()))
else:
attr_values.append(None)
chunk_iter.increment_to_next()
return pos_list, attr_values
def get_column(db, query):
values = []
result = db.executeQuery(query, "afl")
desc = result.array.getArrayDesc()
attrs = desc.getAttributes()
attrid = attrs[0].getId()
attiter = result.array.getConstIterator(attrid)
while not attiter.end():
chunk = attiter.getChunk()
chunkiter = chunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS)|(scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunkiter.end():
if not chunkiter.isEmpty():
dataitem = chunkiter.getItem()
val = scidb.getTypedValue(dataitem, attrs[0].getType())
values.append(val)
else:
val = "#Error!"
chunkiter.increment_to_next()
attiter.increment_to_next()
db.completeQuery(result.queryID)
return values
def load(self, att_list):
assert len(att_list) > 0
with Scidb(self.host, self.port) as db:
self.array = db._query(self.afl)
data_list = [self.data[att] for att in att_list]
##TODO optimize here
while not data_list[0].iterator.end():
while not data_list[0].chunk.iter.end():
pos_ = [dim.relative_position(data_list[0]. \
chunk.iter.getPosition()[i]) for i, dim in
enumerate(self.dimensions)]
for data in data_list:
value = data.chunk.iter.getItem()
if not value.isNull():
data.array[tuple(pos_)] = scidbapi.getTypedValue(value, data.att.Type)
data.chunk.iter.increment_to_next()
for data in data_list:
data.iterator.increment_to_next()
try:
data.set_chunk()
except:
continue
self.attsLoaded.extend(att_list)
def next(self):
"""return the next row of data if it is available otherwise raise StopIteration"""
first_chunk_iter = self.__attr_chunk_iter_tuple_list[0][1]
#if the chunk is finished, try to increment to the next chunk
if (first_chunk_iter.end()):
#increment each attribute iterator to the next chunk
for attr_iter_tuple in self.__attr_iter_tuple_list:
attr_iter_tuple[1].increment_to_next()
#attempt to update the chunk iterators based on the incremented
#attribute iterators
if self.__update_attr_chunk_iter():
raise StopIteration
#get dimension values at this position
coordinates = self.__attr_chunk_iter_tuple_list[0][1].getPosition() #get the first tuple from the list,
#then get the second item in the tuple
pos_list = list()
for i in range(len(coordinates)):
pos_list.append(coordinates[i])
#get attribute values at this position
attr_values = []
for attr_chunk_iter in self.__attr_chunk_iter_tuple_list:
attr = attr_chunk_iter[0]
chunk_iter = attr_chunk_iter[1]
if (not chunk_iter.isEmpty()):
attr_values.append(scidbapi.getTypedValue(chunk_iter.getItem(), attr.getType()))
else:
attr_values.append(None)
chunk_iter.increment_to_next()
return (pos_list, attr_values)
def print_dataitem(dtypeStr, dataitem, innerAttrList):
value = scidb.getTypedValue(dataitem, dtypeStr)
# print "Data: %s" % value
innerAttrList.append(value)
def getAllAttrArrFromQueryForNP(query_result,attrnames=None,usenan=False):
#if DEBUG_PRINT: print "parsing query result and building json dump",datetime.now()
desc = query_result.array.getArrayDesc()
dims = desc.getDimensions() # list of DimensionDesc objects
attrs = desc.getAttributes() # list of AttributeDesc objects
if(dims.size() < 1):
return None
dimtuple = []
for i,dim in enumerate(dims):
dimtuple.append(dim.getCurrLength())
dimtuple = tuple(dimtuple)
arrs = []
itemsets = []
its = []
getattrnames = False
if attrnames is None:
attrnames = []
getattrnames = True
minobj = {}
maxobj = {}
for i in range(attrs.size()): # find the right attrid, "EmptyTag" should always be at the end
if attrs[i].getName() != "EmptyTag":
arrs.append(np.zeros(dimtuple)) # new array for each attribute
itemsets.append(arrs[i].itemset) #used to insert items)
if usenan:
arrs[i].fill(np.nan) # fill with nulls if necessary
its.append(query_result.array.getConstIterator(i))
if getattrnames:
attrnames.append(attrs[i].getName())
#currtype = attrs[i].getType()
while not its[0].end():
#get chunk iterators
chunkiters = []
for itindex in range(len(its)):
#if DEBUG_PRINT: print "itindex: ",itindex
#mypos = its[itindex].getPosition()
#if DEBUG_PRINT: print "position:"
#if DEBUG_PRINT: print mypos[0],",",mypos[1]
currentchunk =its[itindex].getChunk()
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
chunkiters.append(chunkiter)
while not chunkiters[0].end():
currpos = chunkiters[0].getPosition()
cpl = len(currpos)
currpostuple = [0]*cpl
for i in range(cpl):
currpostuple[i] = currpos[i]
currpostuple = tuple(currpostuple)
#testcurrpos = tuple(currpos) #this is bad! causes memory leaks...
for chunkiterindex in range(len(chunkiters)):
#if DEBUG_PRINT: print "chunkiterindex: ",chunkiterindex
dataitem = chunkiters[chunkiterindex].getItem()
# look up the value according to its attribute's typestring
currtype = attrs[chunkiterindex].getType()
dataitem_val = scidb.getTypedValue(dataitem, currtype)
itemsets[chunkiterindex](currpostuple,dataitem_val) # insert into every numpy array
chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS
for itindex in range(len(its)):
its[itindex].increment_to_next()
#if DEBUG_PRINT: print "done parsing results, returning dump-ready version",datetime.now()
#if DEBUG_PRINT: print arrs
if DEBUG_PRINT: print arrs[0].shape
return {'data':arrs,'attrnames':attrnames,'stats':get_stats(arrs)}
def print_dataitem(dtypeStr, dataitem, innerAttrList):
value = scidb.getTypedValue(dataitem, dtypeStr)
# print "Data: %s" % value
innerAttrList.append(value)
result = db.executeQuery(query, "afl")
db.completeQuery(result.queryID)
f.write(query + '\n')
else:
print query
#compute norm^2 for input tensor
query = "select sum(pow(val,2)) from " + tensor
if debug == 0:
print query
f.write(query + '\n')
result = db.executeQuery(query, "aql")
dataitem = result.array.getConstIterator(
0).getChunk().getConstIterator().getItem()
attrs = result.array.getArrayDesc().getAttributes()
normx = scidb.getTypedValue(dataitem, attrs[0].getType())
db.completeQuery(result.queryID)
else:
print query
#compute norm^2 for core
query = "select sum(pow(val,2)) from " + core
if debug == 0:
print query
f.write(query + '\n')
result = db.executeQuery(query, "aql")
dataitem = result.array.getConstIterator(
0).getChunk().getConstIterator().getItem()
attrs = result.array.getArrayDesc().getAttributes()
normcore = scidb.getTypedValue(dataitem, attrs[0].getType())
db.completeQuery(result.queryID)
def getOneAttrArrFromQuery(query_result,attrname):
desc = query_result.array.getArrayDesc()
dims = desc.getDimensions() # list of DimensionDesc objects
attrs = desc.getAttributes() # list of AttributeDesc objects
dimlengths= []
dimchunkintervals = []
dimoverlaps = []
dimindexes = []
dimindexesbase = []
if(dims.size() < 1):
return [];
for i in range(dims.size()):
dimlengths.append(dims[i].getLength())
dimchunkintervals.append(dims[i].getChunkInterval())
dimoverlaps.append(dims[i].getChunkOverlap())
dimindexes.append(0)
dimindexesbase.append(0)
# get arr ready
arr = createArray(dimlengths)
#print "arr is initialized: ",str(arr)
attrid = 0
for i in range(attrs.size()): # find the right attrid
if(attrs[i].getName() == attrname):
attrid = i
#print "found attribute",attrname, ",id: %d" % attrid
break
# get the iterator for this attrid
it = query_result.array.getConstIterator(attrid)
start = True
while not it.end():
#print "iterating over items..."
currentchunk = it.getChunk()
# TODO: will have to fix this at some point, can't just ignore empty cells or overlaps
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
if(not start): # don't update for the first chunk
#update base indexes
dimindexesbase = updateBaseIndex(dimindexesbase,dimlengths,dimchunkintervals)
#printIndexes(dimindexesbase)
verifyIndexes(dimindexesbase,dimlengths)
#reset the indexes to new base indexes
for i in range (dims.size()):
dimindexes[i] = dimindexesbase[i]
else:
start = False
while not chunkiter.end():
#printIndexes(dimindexes)
verifyIndexes(dimindexes,dimlengths)
dataitem = chunkiter.getItem()
# look up the value according to its attribute's typestring
item = scidb.getTypedValue(dataitem, attrs[attrid].getType()) # TBD: eliminate 2nd arg, make method on dataitem
#print "Data: %s" % item
#insert the item
arr = insertItem(arr,item,dimindexes)
#update the indexes
dimindexes = updateIndexes(dimindexes,dimchunkintervals,dimindexesbase,dimlengths)
lastpos = chunkiter.getPosition()
#print lastpos[0],",",lastpos[1], ",",lastpos[2]
chunkiter.increment_to_next()
#print "current state of arr: ", str(arr)
it.increment_to_next();
return arr
def getAllAttrArrFromQuery(query_result):
desc = query_result.array.getArrayDesc()
dims = desc.getDimensions() # list of DimensionDesc objects
attrs = desc.getAttributes() # list of AttributeDesc objects
dimlengths= []
dimchunkintervals = []
dimoverlaps = []
dimindexes = []
dimindexesbase = []
if(dims.size() < 1):
return []
for i in range(dims.size()):
dimlengths.append(dims[i].getLength())
dimchunkintervals.append(dims[i].getChunkInterval())
dimoverlaps.append(dims[i].getChunkOverlap())
dimindexes.append(0)
dimindexesbase.append(0)
# get arr ready
arr = createArray(dimlengths)
#print "arr is initialized: ",str(arr)
its = []
attrnames = []
for i in range(attrs.size()): # find the right attrid
its.append(query_result.array.getConstIterator(i))
attrnames.append(attrs[i].getName())
start = True
while not its[0].end():
#get chunk iterators
chunkiters = []
for itindex in range(len(its)):
#print "itindex: ",itindex
currentchunk =its[itindex].getChunk()
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
chunkiters.append(chunkiter)
if(not start): # don't update for the first chunk
#update base indexes
dimindexesbase = updateBaseIndex(dimindexesbase,dimlengths,dimchunkintervals)
#printIndexes(dimindexesbase)
verifyIndexes(dimindexesbase,dimlengths)
#reset the indexes to new base indexes
for i in range (dims.size()):
dimindexes[i] = dimindexesbase[i]
else:
start = False
while not chunkiters[0].end():
#printIndexes(dimindexes)
verifyIndexes(dimindexes,dimlengths)
item = {} #empty dictionary for the attribute values
for chunkiterindex in range(len(chunkiters)):
#print "chunkiterindex: ",chunkiterindex
dataitem = chunkiters[chunkiterindex].getItem()
# look up the value according to its attribute's typestring
item[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem
#print "Data: %s" % item
#chunkiters[i].increment_to_next()
chunkiters[0].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS
#lastpos = chunkiter.getPosition()
#print lastpos[0],",",lastpos[1], ",",lastpos[2]
#print item
#insert the item
arr = insertItem(arr,item,dimindexes)
#update the indexes
dimindexes = updateIndexes(dimindexes,dimchunkintervals,dimindexesbase,dimlengths)
#print "current state of arr: ", str(arr)
its[0].increment_to_next()
return arr
print >> sys.stderr, " Exception Type: %s" % type(
inst) # the exception instance
print >> sys.stderr, " Exception Value: %r" % inst
else:
print query
#compute norm for input
query = "select sqrt(sum(pow(val,2))) from " + tensor
if debug == 0:
print query
f.write(query + '\n')
result = db.executeQuery(query, "aql")
dataitem = result.array.getConstIterator(
0).getChunk().getConstIterator().getItem()
attrs = result.array.getArrayDesc().getAttributes()
normx = scidb.getTypedValue(dataitem, attrs[0].getType())
db.completeQuery(result.queryID)
else:
print query
fit = 0
fitchangetol = 1e-4
# iterate until the fit converges or for max_iter
for r in range(max_iter):
fitold = fit
for i in range(nmode):
# matricized Khatri-Rao product
query = ""
for k in range(nmode):
if k == i:
def print_dataitem(dtypeStr, dataitem): print "Data: %s" % scidb.getTypedValue(dataitem, dtypeStr)
import scidbapi as scidb
db = scidb.connect("localhost", 1239)
query = db.executeQuery("select sigma from sv", "aql")
singular_values = []
iterator = query.array.getConstIterator(query.array.getArrayDesc().getAttributes()[0].getId())
while not iterator.end():
chunk_iterator = iterator.getChunk().getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunk_iterator.end():
singular_values.append(scidb.getTypedValue(chunk_iterator.getItem(), "double"))
chunk_iterator.increment_to_next()
iterator.increment_to_next()
db.completeQuery(query.queryID)
db.disconnect()
import matplotlib.pyplot as pyplot
pyplot.plot(singular_values)
pyplot.xlabel("Rank")
pyplot.ylabel("Singular Values")
pyplot.show()
import scidbapi as scidb
import scipy.cluster.vq
db = scidb.connect("localhost", 1239)
query = db.executeQuery("select v from rsv", "aql")
dimensions = query.array.getArrayDesc().getDimensions()
features = scipy.zeros((dimensions[0].getEndMax() + 1, dimensions[1].getEndMax() + 1))
iterator = query.array.getConstIterator(query.array.getArrayDesc().getAttributes()[0].getId())
while not iterator.end():
chunk_iterator = iterator.getChunk().getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS) | (scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS))
while not chunk_iterator.end():
coordinates = chunk_iterator.getPosition()
features[coordinates[0], coordinates[1]] = scidb.getTypedValue(chunk_iterator.getItem(), "double")
chunk_iterator.increment_to_next()
iterator.increment_to_next()
db.completeQuery(query.queryID)
db.disconnect()
features = scipy.cluster.vq.whiten(features)
centroids, distance = scipy.cluster.vq.kmeans(features, 5)
clusters, distance = scipy.cluster.vq.vq(features, centroids)
print clusters, distance
print query result=db.executeQuery(query,"afl") db.completeQuery(result.queryID) f.write(query+'\n'); else: print query #compute norm^2 for input tensor query="select sum(pow(val,2)) from " + tensor if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query #compute norm^2 for core query="select sum(pow(val,2)) from " + core if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normcore = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else:
def getAttrArrFromQueryForJSON(query_result,options):
dimnames = options['dimnames']
attrnames = options['attrnames']
desc = query_result.array.getArrayDesc()
dims = desc.getDimensions() # list of DimensionDesc objects
attrs = desc.getAttributes() # list of AttributeDesc objects
origarrnamelen = len(desc.getName()) - 2
#print "orig name length: ",origarrnamelen
if(dims.size() < 1 or dims.size() != len(dimnames)):
return []
arr = []
its = []
for i in range(attrs.size()): # find the right attrid
for aname in attrnames:
if aname == attrs[i].getName():
its.append(query_result.array.getConstIterator(i))
start = True
while not its[0].end():
#get chunk iterators
chunkiters = []
#print "start"
for itindex in range(len(its)):
#print "itindex: ",itindex
#mypos = its[itindex].getPosition()
#print "position:"
#print mypos[0],",",mypos[1]
currentchunk =its[itindex].getChunk()
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
chunkiters.append(chunkiter)
while not chunkiters[0].end():
dataobj = {}
dimobj= {}
currpos = chunkiters[0].getPosition()
for dimindex in range(len(currpos)):
dname = dimnames[dimindex]
dimobj[dname[:len(dname)-origarrnamelen]] = currpos[dimindex] # make sure you take off the array's name from each dimension
dataobj["dims."+dname[:len(dname)-origarrnamelen]] = currpos[dimindex]
attrobj = {} #empty dictionary for the attribute values
#print "start"
for chunkiterindex in range(len(chunkiters)):
#print "chunkiterindex: ",chunkiterindex
dataitem = chunkiters[chunkiterindex].getItem()
# look up the value according to its attribute's typestring
attrobj[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem
dataobj["attrs."+attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType())
#print "Data: %s" % item
#chunkiters[i].increment_to_next()
#mypos = chunkiters[chunkiterindex].getPosition()
#myposstring = "position: "
#for myposi in range(len(mypos)):
# myposstring += str(mypos[myposi])+", "
#print myposstring
chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS
#lastpos = chunkiter.getPosition()
#print lastpos[0],",",lastpos[1], ",",lastpos[2]
#print attrobj
#insert the item
arr.append(dataobj)
#arr.append({'dimensions':dimobj,'attributes':attrobj})
#print "current state of arr: ", str(arr)
#its[1].increment_to_next()
for itindex in range(len(its)):
its[itindex].increment_to_next()
namesobj = []
typesobj = {}
for attri in range(len(attrnames)):
attrname = attrnames[attri]
namesobj.append({'name':"attrs."+attrname,'isattr':True})
typesobj["attrs."+attrname] = attrs[attri].getType()
for dimname in dimnames:
ndimname = "dims."+dimname[:len(dimname)-origarrnamelen]
namesobj.append({'name':ndimname,'isattr':False})
typesobj[ndimname] = "int32"
#print typesobj
#print json.dumps({'data':arr, 'names': namesobj, 'types': typesobj})
return {'data':arr, 'names': namesobj, 'types': typesobj}
def print_dataitem(dtypeStr, dataitem):
print "Data: %s" % scidb.getTypedValue(dataitem, dtypeStr)
def getDataInJson(query_result):
print "Parsing query result into JSON", datetime.now()
desc = query_result.array.getArrayDesc()
# Dimensions
dims = desc.getDimensions()
dimnames = []
for i in range(dims.size()):
if dims[i].getBaseName() != "EmptyTag":
dimnames.append(dims[i].getBaseName())
arr = []
its = []
# Attributes
attrs = desc.getAttributes()
attrnames = []
for i in range(attrs.size()):
if attrs[i].getName() != "EmptyTag":
its.append(query_result.array.getConstIterator(i))
attrnames.append(attrs[i].getName())
while not its[0].end():
chunkiters = []
for itindex in range(len(its)):
currentchunk =its[itindex].getChunk()
chunkiter = currentchunk.getConstIterator((scidb.swig.ConstChunkIterator.IGNORE_EMPTY_CELLS |
scidb.swig.ConstChunkIterator.IGNORE_OVERLAPS))
chunkiters.append(chunkiter)
while not chunkiters[0].end():
dataobj = {}
dimobj= {}
currpos = chunkiters[0].getPosition()
for dimindex in range(len(currpos)):
#print dimindex
dname = dimnames[dimindex]
#print "dname" , dname
dimobj[dname[:len(dname)]] = currpos[dimindex] # make sure you take off the array's name from each dimension
dataobj["dims."+dname[:len(dname)]] = currpos[dimindex]
attrobj = {} #empty dictionary for the attribute values
#print "start"
for chunkiterindex in range(len(chunkiters)):
dataitem = chunkiters[chunkiterindex].getItem()
attrobj[attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType()) # TBD: eliminate 2nd arg, make method on dataitem
dataobj["attrs."+attrnames[chunkiterindex]] = scidb.getTypedValue(dataitem, attrs[chunkiterindex].getType())
chunkiters[chunkiterindex].increment_to_next() # OMG THIS INCREMENTS ALL THE CHUNK ITERATOR OBJECTS
arr.append(dataobj)
for itindex in range(len(its)):
its[itindex].increment_to_next()
namesobj = []
typesobj = {}
for attri in range(len(attrnames)):
attrname = attrnames[attri]
namesobj.append({'name':"attrs."+attrname,'isattr':True})
typesobj["attrs."+attrname] = attrs[attri].getType()
for dimname in dimnames:
ndimname = "dims."+dimname[:len(dimname)]
namesobj.append({'name':ndimname,'isattr':False})
typesobj[ndimname] = "int32"
print "Done parsing query results, returning dump-ready version", datetime.now()
return {'data':arr, 'names': namesobj, 'types': typesobj}
db.completeQuery(result.queryID) except Exception, inst: print >> sys.stderr, " Exception Type: %s" % type(inst) # the exception instance print >> sys.stderr, " Exception Value: %r" % inst else: print query #compute norm for input query="select sqrt(sum(pow(val,2))) from " + tensor if debug == 0: print query f.write(query+'\n') result=db.executeQuery(query,"aql") dataitem = result.array.getConstIterator(0).getChunk().getConstIterator().getItem() attrs = result.array.getArrayDesc().getAttributes() normx = scidb.getTypedValue(dataitem, attrs[0].getType()) db.completeQuery(result.queryID) else: print query fit = 0; fitchangetol = 1e-4; # iterate until the fit converges or for max_iter for r in range(max_iter): fitold = fit; for i in range(nmode): # matricized Khatri-Rao product query = "" for k in range(nmode): if k == i:
