def generateQueries(DateFormat,timeQuery):
"""Converts nice DateFormat string and simple query format for time and generate mongo icky
DateFormat : String e.g. YYYYmmdd
timeQuery : Dict keys = format, begin, end, on
These are not necessary"""
timeQueryFormat = timeQuery['format'] if 'format' in timeQuery.keys() else DateFormat
tQFset = set(timeQueryFormat)
tFset = set(DateFormat)
if tQFset <= tFset:
tQHier = getHierarchy(tQFset)
Hier = getHierarchy(DateFormat)
mergedTimeFormat = ''.join(tFset.difference(tQFset)) + timeQueryFormat
timeFormatter = mongotimeformatter(mergedTimeFormat)
zeroLen = len(tFset.difference(tQFset))
tQHier0 = [x[0] for x in tQHier]
Hier0 = [x[0] for x in Hier]
basePathDict = dict([(m,getPathsTo(m,Hier)) for m in tQHier0])
belowPathDict = dict([(m,getPathsBelow(m,Hier)) for m in tQHier0])
Q = {}
for (k,op) in [('begin','$gte'),('end','$lt')]:
if k in timeQuery.keys():
timeObj = timeFormatter('X'*zeroLen + timeQuery[k])
for m in basePathDict.keys():
for p in basePathDict[m]:
if p in Q.keys():
Q[p][op] = rgetattr(timeObj,p)
else:
Q[p] = {op: rgetattr(timeObj,p)}
if 'on' in timeQuery.keys():
timeObj = timeFormatter('X'*zeroLen + timeQuery['on'])
paths = uniqify(ListUnion([getPathsTo(m,Hier) for m in tQFset]))
for p in paths:
p = p + ('',)
Q[p] = rgetattr(timeObj,p)
if not set(timeQuery.keys()).intersection(['begin','end','on']):
for m in set(tQHier0):
for p in basePathDict[m]:
p = p + ('',)
Q[p] = {'$exists':True}
for p in belowPathDict[m]:
p = p + ('',)
Q[p] = {'$exists':False}
return Q
def checkQuery(spaceQuery,ol):
"""compares spaceQuery to ol and if they match then returns nil
Otherwise it removes the unnessary spaceQuery properites to match to the correct remaining format"""
if isinstance(spaceQuery,list) or isinstance(spaceQuery,tuple):
for k in spaceQuery:
if rhasattr(ol,k.split('.')):
spaceQuery.pop(k)
return True
elif hasattr(spaceQuery,'keys'):
spaceQuery = convertSQ(spaceQuery)
OK = True
for k in spaceQuery.keys():
sq = spaceQuery[k]
if rhasattr(ol,k.split('.')):
spaceQuery.pop(k)
if rgetattr(ol,k.split('.')) != sq:
OK = False
return OK
def getQueryList(collection,keys,atVersion,toVersion,slicesCorrespondToIndexes):
totalVariables = collection.columns
VarMap = dict(zip(totalVariables,[str(x) for x in range(len(totalVariables))]))
origInd = VarMap['__originalVersion__'] ; retInd = VarMap['__retained__'] ; vNInd = VarMap['__versionNumber__']
keys = [str(x) for x in keys]
existence = [(k,{'$exists':True,'$ne':''}) for k in keys]
if keys:
Q1 = processArg(dict([(origInd,{'$gt':atVersion}),(vNInd,toVersion)] + existence),collection)
Q2 = processArg(dict([(retInd,{'$exists':False}),(vNInd,{'$lt':toVersion,'$gte':atVersion})] + existence),collection)
Q3 = processArg(dict([(retInd,True),(vNInd,{'$lt':toVersion,'$gte':atVersion}),(origInd,{'$lte':atVersion})] + existence),collection)
colnames = [k for k in keys if k.split('.')[0] in collection.columns]
colgroups = [k for k in keys if k in collection.columnGroups]
T= ListUnion([collection.columnGroups[k] for k in colgroups])
kInds = getStrs(collection,colnames + T)
R = list(collection.find(Q1,fields = kInds)) + list(collection.find(Q2,fields = kInds)) + (list(collection.find(Q3,fields = kInds)) if not slicesCorrespondToIndexes else [])
R = [son.SON([(collection.columns[int(k)],r[k]) for k in r.keys() if k.isdigit() and r[k]]) for r in R]
R = [[(k,rgetattr(r,k.split('.'))) for k in keys if rhasattr(r,k.split('.')) if k not in T] + [(g,[r[k] for k in collection.columnGroups[g] if k in r.keys() and r[k]]) for g in colgroups ] for r in R]
return uniqify(ListUnion([expand(r) for r in R]))
else:
return [()]
def MoreThanOne(collection,key):
v1 = rgetattr(list(collection.find({key:{'$exists':True}}).sort([(key,-1)]).limit(1))[0],key.split('.'))
v2 = rgetattr(list(collection.find({key:{'$exists':True}}).sort([(key,1)]).limit(1))[0],key.split('.'))
return v1 != v2
def smallAdd(d,query,collection,contentColNums, timeColInds ,timeColNames , timeColNameInds ,timeColNameDivisions ,timeColNamePhrases ,overallDate, overallDateFormat, timeFormatter ,reverseTimeFormatter ,dateDivisions ,datePhrases ,mindate ,maxdate ,overallLocation , spaceColNames , spaceColInds ,subColInd, valueProcessors,slicecount):
R = collection.find(query,timeout=False)
colnames = []
d['sliceContents'] = []
Subcollections = []
spaceVals = spaceColNames
commonLocation = overallLocation
for sv in spaceColNames:
commonLocation = loc.intersect(commonLocation,sv)
if not commonLocation:
break
for (i,r) in enumerate(R):
d['sliceContents'].append(' '.join([translate(valueProcessors.get(x,None),decode_obj(rgetattr(r,x.split('.')))) if rhasattr(r,x.split('.')) else '' for x in contentColNums]))
colnames = uniqify(colnames + r.keys())
if subColInd:
Subcollections += r[str(subColInd)]
if timeColInds:
for x in timeColInds:
if str(x) in r.keys():
time = r[str(x)]
if overallDate:
time = timeFormatter(overallDate + reverseTimeFormatter(time))
dateDivisions += td.getLowest(time)
datePhrases.append(td.phrase(time))
mindate = td.makemin(mindate,time)
maxdate = td.makemax(maxdate,time)
if spaceColInds:
for x in spaceColInds:
if str(x) in r.keys():
location = loc.integrate(overallLocation,r[str(x)])
commonLocation = loc.intersect(commonLocation,r[str(x)]) if commonLocation != None else None
spaceVals.append(location)
d['sliceContents'] = ' '.join(d['sliceContents'])
Subcollections = uniqify(Subcollections)
d['columnNames'] = [collection.columns[int(x)] for x in colnames if x.isdigit()]
d['dimension'] = len(d['columnNames'])
#time/date
if overallDateFormat:
d['dateFormat'] = overallDateFormat
if 'timeColNames' in collection.columnGroups.keys():
K = [k for (k,j) in enumerate(timeColNameInds) if str(j) in colnames]
dateDivisions += uniqify(ListUnion([timeColNameDivisions[k] for k in K]))
mindate = td.makemin(mindate,min([timeColNames[k] for k in K]))
maxdate = td.makemax(maxdate,max([timeColNames[k] for k in K]))
datePhrases += uniqify([timeColNamePhrases[k] for k in K])
d['beginDate'] = td.convertToDT(mindate)
d['endDate'] = td.convertToDT(maxdate,convertMode='High')
d['dateDivisions'] = uniqify(dateDivisions)
d['datePhrases'] = datePhrases if d['volume'] < 10000 else uniqify(datePhrases)
if spaceVals:
d['spatialDivisions'] = uniqify(ListUnion(map(loc.divisions,spaceVals)))
d['spatialDivisionsTight'] = uniqify(ListUnion(map(loc.divisions2,spaceVals)))
d['spatialPhrases'] = uniqify(map(loc.phrase,spaceVals))
d['spatialPhrasesTight'] = uniqify(map(loc.phrase2,spaceVals))
return d
