def getQueryList(collection,keys,atVersion,toVersion,slicesCorrespondToIndexes):
totalVariables = collection.columns
VarMap = dict(zip(totalVariables,[str(x) for x in range(len(totalVariables))]))
origInd = VarMap['__originalVersion__'] ; retInd = VarMap['__retained__'] ; vNInd = VarMap['__versionNumber__']
keys = [str(x) for x in keys]
existence = [(k,{'$exists':True,'$ne':''}) for k in keys]
if keys:
Q1 = processArg(dict([(origInd,{'$gt':atVersion}),(vNInd,toVersion)] + existence),collection)
Q2 = processArg(dict([(retInd,{'$exists':False}),(vNInd,{'$lt':toVersion,'$gte':atVersion})] + existence),collection)
Q3 = processArg(dict([(retInd,True),(vNInd,{'$lt':toVersion,'$gte':atVersion}),(origInd,{'$lte':atVersion})] + existence),collection)
colnames = [k for k in keys if k.split('.')[0] in collection.columns]
colgroups = [k for k in keys if k in collection.columnGroups]
T= ListUnion([collection.columnGroups[k] for k in colgroups])
kInds = getStrs(collection,colnames + T)
R = list(collection.find(Q1,fields = kInds)) + list(collection.find(Q2,fields = kInds)) + (list(collection.find(Q3,fields = kInds)) if not slicesCorrespondToIndexes else [])
R = [son.SON([(collection.columns[int(k)],r[k]) for k in r.keys() if k.isdigit() and r[k]]) for r in R]
R = [[(k,rgetattr(r,k.split('.'))) for k in keys if rhasattr(r,k.split('.')) if k not in T] + [(g,[r[k] for k in collection.columnGroups[g] if k in r.keys() and r[k]]) for g in colgroups ] for r in R]
return uniqify(ListUnion([expand(r) for r in R]))
else:
return [()]
def addToIndex(q,d,collection,solr_interface,slicecount,contentColNums = None, timeColInds=None,timeColNames=None, timeColNameInds = None,timeColNameDivisions = None,timeColNamePhrases=None,overallDate = '', overallDateFormat = '', timeFormatter = None,reverseTimeFormatter = None,dateDivisions=None,datePhrases=None,mindate = None,maxdate = None,overallLocation = None, spaceColNames = None, spaceColInds = None,subColInd = None,Return=False,valueProcessors=None,valueProcessorsKey=None):
q['__versionNumber__'] = collection.currentVersion
query = processArg(q,collection)
q.pop('__versionNumber__')
d['collectionName'] = collection.name
d['query'] = json.dumps(q,default=ju.default)
d['mongoID'] = mongoID(q,collection.name)
d['mongoText'] = queryToText(q,valueProcessorsKey)
d['sliceValues'] = queryValues(q,valueProcessorsKey)
d['sliceKeys'] = queryKeys(q,valueProcessorsKey)
d['versionNumber'] = collection.currentVersion
if dateDivisions == None:
dateDivisions = []
else:
dateDivisions = dateDivisions[:]
if datePhrases == None:
datePhrases = []
else:
datePhrases = datePhrases[:]
if spaceColNames == None:
spaceColNames = []
#stats
d['volume'] = collection.find(query).count()
contentColNums = [i for i in contentColNums if i not in query.keys()]
if d['volume'] > 0:
if d['volume'] < 5000:
smallAdd(d,query,collection,contentColNums, timeColInds ,timeColNames , timeColNameInds ,timeColNameDivisions ,timeColNamePhrases ,overallDate , overallDateFormat, timeFormatter ,reverseTimeFormatter ,dateDivisions ,datePhrases ,mindate ,maxdate ,overallLocation , spaceColNames , spaceColInds ,subColInd, valueProcessors,slicecount)
else:
largeAdd(d,query,collection,contentColNums, timeColInds ,timeColNames , timeColNameInds ,timeColNameDivisions ,timeColNamePhrases ,overallDate, overallDateFormat, timeFormatter ,reverseTimeFormatter ,dateDivisions ,datePhrases ,mindate ,maxdate ,overallLocation , spaceColNames , spaceColInds ,subColInd, valueProcessors)
Subcollections = uniqify(ListUnion(collection.find(query).distinct(str(subColInd))))
metadata = collection.metadata['']
for sc in Subcollections:
metadata.update(collection.metadata.get(sc,{}))
for k in metadata.keys():
if k in STANDARD_META:
if k in STANDARD_META_FORMATS.keys():
val = coerceToFormat(metadata[k],STANDARD_META_FORMATS[k])
if val:
d[str(k)] = val
else:
d[str(k)] = str(metadata[k])
if Return:
return d
else:
solr_interface.add(**d)
