def getQueryList(collection,keys,atVersion,toVersion,slicesCorrespondToIndexes): totalVariables = collection.columns VarMap = dict(zip(totalVariables,[str(x) for x in range(len(totalVariables))])) origInd = VarMap['__originalVersion__'] ; retInd = VarMap['__retained__'] ; vNInd = VarMap['__versionNumber__'] keys = [str(x) for x in keys] existence = [(k,{'$exists':True,'$ne':''}) for k in keys] if keys: Q1 = processArg(dict([(origInd,{'$gt':atVersion}),(vNInd,toVersion)] + existence),collection) Q2 = processArg(dict([(retInd,{'$exists':False}),(vNInd,{'$lt':toVersion,'$gte':atVersion})] + existence),collection) Q3 = processArg(dict([(retInd,True),(vNInd,{'$lt':toVersion,'$gte':atVersion}),(origInd,{'$lte':atVersion})] + existence),collection) colnames = [k for k in keys if k.split('.')[0] in collection.columns] colgroups = [k for k in keys if k in collection.columnGroups] T= ListUnion([collection.columnGroups[k] for k in colgroups]) kInds = getStrs(collection,colnames + T) R = list(collection.find(Q1,fields = kInds)) + list(collection.find(Q2,fields = kInds)) + (list(collection.find(Q3,fields = kInds)) if not slicesCorrespondToIndexes else []) R = [son.SON([(collection.columns[int(k)],r[k]) for k in r.keys() if k.isdigit() and r[k]]) for r in R] R = [[(k,rgetattr(r,k.split('.'))) for k in keys if rhasattr(r,k.split('.')) if k not in T] + [(g,[r[k] for k in collection.columnGroups[g] if k in r.keys() and r[k]]) for g in colgroups ] for r in R] return uniqify(ListUnion([expand(r) for r in R])) else: return [()]
def addToIndex(q,d,collection,solr_interface,slicecount,contentColNums = None, timeColInds=None,timeColNames=None, timeColNameInds = None,timeColNameDivisions = None,timeColNamePhrases=None,overallDate = '', overallDateFormat = '', timeFormatter = None,reverseTimeFormatter = None,dateDivisions=None,datePhrases=None,mindate = None,maxdate = None,overallLocation = None, spaceColNames = None, spaceColInds = None,subColInd = None,Return=False,valueProcessors=None,valueProcessorsKey=None): q['__versionNumber__'] = collection.currentVersion query = processArg(q,collection) q.pop('__versionNumber__') d['collectionName'] = collection.name d['query'] = json.dumps(q,default=ju.default) d['mongoID'] = mongoID(q,collection.name) d['mongoText'] = queryToText(q,valueProcessorsKey) d['sliceValues'] = queryValues(q,valueProcessorsKey) d['sliceKeys'] = queryKeys(q,valueProcessorsKey) d['versionNumber'] = collection.currentVersion if dateDivisions == None: dateDivisions = [] else: dateDivisions = dateDivisions[:] if datePhrases == None: datePhrases = [] else: datePhrases = datePhrases[:] if spaceColNames == None: spaceColNames = [] #stats d['volume'] = collection.find(query).count() contentColNums = [i for i in contentColNums if i not in query.keys()] if d['volume'] > 0: if d['volume'] < 5000: smallAdd(d,query,collection,contentColNums, timeColInds ,timeColNames , timeColNameInds ,timeColNameDivisions ,timeColNamePhrases ,overallDate , overallDateFormat, timeFormatter ,reverseTimeFormatter ,dateDivisions ,datePhrases ,mindate ,maxdate ,overallLocation , spaceColNames , spaceColInds ,subColInd, valueProcessors,slicecount) else: largeAdd(d,query,collection,contentColNums, timeColInds ,timeColNames , timeColNameInds ,timeColNameDivisions ,timeColNamePhrases ,overallDate, overallDateFormat, timeFormatter ,reverseTimeFormatter ,dateDivisions ,datePhrases ,mindate ,maxdate ,overallLocation , spaceColNames , spaceColInds ,subColInd, valueProcessors) Subcollections = uniqify(ListUnion(collection.find(query).distinct(str(subColInd)))) metadata = collection.metadata[''] for sc in Subcollections: metadata.update(collection.metadata.get(sc,{})) for k in metadata.keys(): if k in STANDARD_META: if k in STANDARD_META_FORMATS.keys(): val = coerceToFormat(metadata[k],STANDARD_META_FORMATS[k]) if val: d[str(k)] = val else: d[str(k)] = str(metadata[k]) if Return: return d else: solr_interface.add(**d)