def doPagination( page, totalFound, numPerPage, isZeroIndexed = True ):
"""creates the pagination object used by various Helios views"""
ret = []
if isZeroIndexed:
if page < 5:
startPage = 0
else:
startPage = page - 5
if totalFound % numPerPage:
lastPage = (totalFound // numPerPage) + 1
else:
lastPage = (totalFound // numPerPage)
endPage = min( lastPage, page + 5)
for i in range( startPage, endPage):
ret.append( { 'selected' : (i == page) , 'start' : ( i * numPerPage) + 1, 'end' : min( totalFound, ( i + 1) * numPerPage), 'page' : i, 'pageLabel' : i+1 } )
return {'pages' : ret , 'hasPrevious' : (page > 0), 'hasNext' : page < ( lastPage - 1 ), 'previousPage' : page-1, 'nextPage' : page+1 }
else: # 1 indexed (used by embedded HIP display)
page = page + 1 # TODO: deal with this ugly 0be1 wart
if page < 6:
startPage = 1
else:
startPage = page - 5
if totalFound % numPerPage:
lastPage = (totalFound // numPerPage) + 2
else:
lastPage = (totalFound // numPerPage) + 1
endPage = min( lastPage, page + 5)
logger.debug( "startPage is %s, endPage is %s, lastPage is %s, page is %s" % ( startPage, endPage, lastPage, page) )
for i in range( startPage, endPage):
ret.append( { 'selected' : (i == page) , 'start' : ( i * numPerPage) + 1, 'end' : min( totalFound, ( i + 1) * numPerPage), 'page' : i, 'pageLabel' : i } )
return {'pages' : ret , 'hasPrevious' : (page > 1), 'hasNext' : page < ( lastPage - 1 ), 'previousPage' : page-1, 'nextPage' : page+1 }
def doPagination(page, totalFound, numPerPage, isZeroIndexed=True):
"""creates the pagination object used by various Helios views"""
ret = []
if isZeroIndexed:
if page < 5:
startPage = 0
else:
startPage = page - 5
if totalFound % numPerPage:
lastPage = (totalFound // numPerPage) + 1
else:
lastPage = (totalFound // numPerPage)
endPage = min(lastPage, page + 5)
for i in range(startPage, endPage):
ret.append({
'selected': (i == page),
'start': (i * numPerPage) + 1,
'end': min(totalFound, (i + 1) * numPerPage),
'page': i,
'pageLabel': i + 1
})
return {
'pages': ret,
'hasPrevious': (page > 0),
'hasNext': page < (lastPage - 1),
'previousPage': page - 1,
'nextPage': page + 1
}
else: # 1 indexed (used by embedded HIP display)
page = page + 1 # TODO: deal with this ugly 0be1 wart
if page < 6:
startPage = 1
else:
startPage = page - 5
if totalFound % numPerPage:
lastPage = (totalFound // numPerPage) + 2
else:
lastPage = (totalFound // numPerPage) + 1
endPage = min(lastPage, page + 5)
logger.debug(
"startPage is %s, endPage is %s, lastPage is %s, page is %s" %
(startPage, endPage, lastPage, page))
for i in range(startPage, endPage):
ret.append({
'selected': (i == page),
'start': (i * numPerPage) + 1,
'end': min(totalFound, (i + 1) * numPerPage),
'page': i,
'pageLabel': i
})
return {
'pages': ret,
'hasPrevious': (page > 1),
'hasNext': page < (lastPage - 1),
'previousPage': page - 1,
'nextPage': page + 1
}
def makeSearchString(q, index, limits, handler=None): # sort,
"""translates search parameters into Solr query syntax."""
q = q.replace("_", " ")
# strip out stopwords here... this is necessary due to some
# unexpected side-effects of the general keyword search
# TODO: do NOT strip stopwords if a quoted search, eg. "the the" or "to be or not to be"
qWords = q.split(" ")
nonStopWords = []
for wordOn in qWords:
if wordOn not in STOPWORDS:
nonStopWords.append( wordOn )
q = " ".join(nonStopWords)
for orig,replacement in SEARCH_CHARACTER_REPLACEMENTS.iteritems():
q = q.replace(orig,replacement)
q = urllib.quote( q ).strip()
if not handler or (handler is "standard"):
if index in allFacetCodes:
# then treat as an exact match search; it is a facet, not free text entered by user
ret = '%s:"%s"' % (index, q)
else:
# then it is a search index, not a facet -- NOT exact match
ret = '%s:%s' % (index, q)
else:
# if you specify a handler you can't also specify an index...
ret = '%s' % q
for limitOn in limits:
# csdebug: how are these forbidden characters making it this far?
_limOn = urllib.unquote( limitOn )
limitSplit = _limOn.split(":")
logger.error("limitSplit is %s\n\n\n\n" % limitSplit) # csdebug
index = limitSplit[0] # todo error handling
term = ":".join( limitSplit[1:] )
for orig,replacement in SEARCH_CHARACTER_REPLACEMENTS.iteritems():
term = term.replace(orig,replacement)
logger.error("\n\nterm is now %s\n\n" % term)
logger.error("\n\nxxterm is now %s\n\n" % term)
term = term.replace("_", " ")
#.replace('"', "")
term = term.strip()
logger.error("\n\nyyterm is now %s\n\n" % term)
# we are going to put exact quotes around the whole thing, so we don't want to double-quote
term = urllib.quote(term)
logger.error("\n\n!!!term is now %s\n\n" % term)
ret = """%s AND %s:"%s\"""" % ( ret, index, term)
# get rid of any double spaces.
ret = ret.replace(" ", " ")
ret = ret.replace(" ", "%20")
logger.debug( "search string is %s" % ret )
return ret.strip()
def parseLocationIPAddressRanges(ranges):
addressLocationMap = {}
for k, v in ranges.iteritems():
if k in LOCATION_NAMES.keys():
if type(v) in types.StringTypes:
# individual IP address - - just put in key/value pair
asLong = IPAddressToInt(v)
addressLocationMap[asLong] = k
elif type(v) == types.TupleType:
logger.debug("on %s" % v)
startAsLong, endAsLong = IPAddressToInt(v[0]), IPAddressToInt(
v[1])
if startAsLong < endAsLong:
logger.debug("adding %s addresses..." %
(endAsLong - startAsLong))
for i in range(startAsLong, endAsLong + 1): # NB fencepost
addressLocationMap[i] = k
elif type(v) == types.ListType:
for x in v:
if type(x) == types.TupleType:
logger.debug("on\n%s" % v)
startAsLong, endAsLong = IPAddressToInt(
x[0]), IPAddressToInt(x[1])
if startAsLong < endAsLong:
logger.debug("adding %s addresses" %
(endAsLong - startAsLong))
for i in range(startAsLong,
endAsLong + 1): # NB fencepost
addressLocationMap[i] = k
return addressLocationMap
def parseLocationIPAddressRanges( ranges ):
addressLocationMap = {}
for k,v in ranges.iteritems():
if k in LOCATION_NAMES.keys():
if type(v) in types.StringTypes:
# individual IP address - - just put in key/value pair
asLong = IPAddressToInt( v )
addressLocationMap[asLong] = k
elif type(v) == types.TupleType:
logger.debug( "on %s" % v )
startAsLong, endAsLong = IPAddressToInt( v[0] ), IPAddressToInt( v[1] )
if startAsLong < endAsLong:
logger.debug( "adding %s addresses..." % ( endAsLong -startAsLong) )
for i in range(startAsLong, endAsLong+1): # NB fencepost
addressLocationMap[i] = k
elif type(v) == types.ListType:
for x in v:
if type(x) == types.TupleType:
logger.debug( "on\n%s" % v )
startAsLong, endAsLong = IPAddressToInt(x[0]), IPAddressToInt(x[1])
if startAsLong < endAsLong:
logger.debug( "adding %s addresses" % ( endAsLong -startAsLong) )
for i in range(startAsLong, endAsLong+1): # NB fencepost
addressLocationMap[i] = k
return addressLocationMap
def spellCheck( phraseToCheck ):
"""takes the phrase to check and returns a list of potential suggestions."""
ret = []
if not config.USE_YAHOO_SPELLING_WEB_SERVICE:
pass
else:
# TODO: will it handle utf8?
# check cache first.
cacheKey = "spellcheck~~%s~~" % phraseToCheck
suggestionsFromCache = cache.get( cacheKey )
if not suggestionsFromCache:
query = urllib.quote( phraseToCheck )
urlToGet = config.YAHOO_WEB_SERVICE_URL % dict( YAHOO_APPID = config.YAHOO_APPID, query = query )
logger.debug("fetching URL %s" % urlToGet )
data = urllib.urlopen( urlToGet ).read()
try:
respObject = simplejson.loads( data )
# note we do this to prevent caching bad data
cache.set( cacheKey, data, config.YAHOO_SPELLCHECK_CACHE_TIME)
# TODO: figure out if it will *ever* return multiple suggestions...
except:
logger.error( "exception doing spellCheck service" )
else:
logger.debug( "got spelling suggestion from cache")
respObject = simplejson.loads( suggestionsFromCache )
#print "respObject is %s, type is %s" % (respObject, type(respObject) ) # csdebug
if respObject.has_key("ResultSet") and type(respObject['ResultSet']) == types.DictType and respObject['ResultSet'].has_key("Result"):
ret.append( respObject['ResultSet']['Result'] )
logger.debug("returning %s" % ret)
return ret
#return ['pants', 'pantaloons'] # csdebug
def processFacets( facetCodes, facetCounts, q, limits, doExtendedTerms = True, doColors=False ):
# do postprocessing on facet data --put them in a format that is easier to work
# with in the templating language, split them up into the basic &extended
# for toggling, and translate codes where necessary
logger.debug( "limits is %s" % limits )
_facets = []
_bestBets = [] # best bets are facet terms which contain the search term
for facetCodeOn in facetCodes:
facetOn = {'type' : facetCodeOn['type'], 'terms' : [], 'extended_terms' : [], 'code' : facetCodeOn['code'],
'name' : facetCodeOn['name'], 'has_more' : False }
# check to see if this facet is a cloudy one. (This value may get set to False later on if there are not enough
# facets.)
if facetCodeOn['code'] in allRefineFacetCodes:
facetOn['show_cloud_link'] = True
else:
facetOn['show_cloud_link'] = False
# colors are used by the cloud view to differentiate facet types.
if doColors:
_color = allFacetsByCode[ facetCodeOn['code'] ].get( 'refine_color_class' , 'refine-default')
facetOn['color'] = _color
if facetCounts['facet_fields'].has_key( facetCodeOn['code'] ):
facetCountList = facetCounts['facet_fields'][facetCodeOn['code'] ]
# this is a list of alternating facets and counts
terms, counts = facetCountList[::2], facetCountList[1::2]
_facetOnTerms = []
if len(terms) < MIN_TERMS_TO_SHOW_CLOUD_LINK:
# not enough facets to justify offering a link to the cloud.
facetOn['show_cloud_link'] = False
for i in range(len(terms)):
# if there is a translate function associated with this facet, translate the code here.
if translateFunctionRefs.has_key( facetCodeOn['code'] ):
_label = translateFunctionRefs[ facetCodeOn['code'] ]( terms[i] ).strip()
else:
_label = terms[i]
_websafeTerm = terms[i]
#_websafeTerm = _websafeTerm.replace("'", "%27").replace("&", "%26" ).replace(" ", "_").replace('"', "%22")
for orig,replacement in FACET_TERM_REPLACEMENTS.iteritems():
_websafeTerm = _websafeTerm.replace(orig,replacement)
# TODO: see if this facet is already one of our limits; if so, remove it.
alreadyApplied = False
for limitOn in limits:
# using repr avoids unicode encoding errors.
_limitTerm = repr( u"%s:%s" % ( facetCodeOn['code'], _websafeTerm) )
_limitOn = repr(limitOn)
if _limitTerm == _limitOn:
alreadyApplied = True
if not alreadyApplied:
_facetOnTerms.append( dict( term=_websafeTerm, count=counts[i], label=_label) )
# check to see if this is a "best bet"
# nb it's not a best bet if the suggestion is already one of your limits!
# 'in <string>' comparison requires string as left operand; can't handle Unicode
# TODO: allow it to handle Unicode here without dying on "in"
qTerms = [x.strip() for x in q.replace(",", " ").lower().split()]
if (type(_label) is not types.UnicodeType):
numTermsMatched = 0
for termOn in qTerms:
# split apart all words and see if each one matches (so order doesn't matter)
if _label.lower().find( termOn ) > -1:
numTermsMatched += 1
if numTermsMatched == len(qTerms): # all words matched
isBestBet = True
_facetAsLimit = '%s:"%s"' %( facetCodeOn['code'] , terms[i] )
for limitOn in limits:
# check vs. each limit and see if they're a match -- don't want to suggest a limit already applied
if _facetAsLimit in limitOn:
isBestBet = False
if isBestBet:
_bestBets.append( dict( facetTerm=_websafeTerm, facetLabel = _label, facetIndexCode = facetCodeOn['code'], facetIndexLabel = facetCodeOn['name'] ) )
if doExtendedTerms and (len( _facetOnTerms ) > MAX_FACET_TERMS_BASIC):
facetOn['has_more'] = True
facetOn['terms'] , facetOn['extended_terms'] = _facetOnTerms[:MAX_FACET_TERMS_BASIC], _facetOnTerms[MAX_FACET_TERMS_BASIC: ]
else:
facetOn['terms'] = _facetOnTerms
_facets.append( facetOn )
return _facets, _bestBets
