def run_query(self, alert):
results = []
error = False
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if self.rate == 'rt' and len(self.valid_ids[cd['type']]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd['type'], results
cut_off_date = get_cut_off_date(self.rate)
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if self.rate == 'rt':
main_params['fq'].append(
'id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[cd['type']]]
),
)
results = self.connections[
cd['type']
].raw_query(
**main_params
).execute()
else:
logger.info(" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" %
(alert.query, search_form.errors))
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" %
alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd['type'], results
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
try:
main_query = build_main_query(kwargs['cd'], highlight='text')
except KeyError:
sf = SearchForm({'q': "*:*"})
if sf.is_valid():
main_query = build_main_query(sf.cleaned_data, highlight='text')
main_query['caller'] = 'api_search'
# Use a SolrList that has a couple of the normal functions built in.
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20)
)
return sl
def run_query(self, alert, cut_off_date):
results = None
error = False
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print " Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(
settings.SOLR_OPINION_URL, mode='r'
)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(
settings.SOLR_AUDIO_URL, mode='r'
)
results = conn.raw_query(**main_params).execute()
else:
print " Query for alert %s was invalid" % alert.alertText
print " Errors from the SearchForm: %s" % search_form.errors
error = True
except:
traceback.print_exc()
print " Search for this alert failed: %s" % alert.alertText
error = True
if self.verbosity >= 1:
if results:
print " There were %s results" % len(results)
else:
print " There were no results"
if self.verbosity >= 2:
print " The value of results is: %s" % results
return error, cd['type'], results,
def run_query(self, alert, cut_off_date):
results = None
error = False
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print " Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
results = conn.raw_query(**main_params).execute()
else:
print " Query for alert %s was invalid" % alert.alertText
print " Errors from the SearchForm: %s" % search_form.errors
error = True
except:
traceback.print_exc()
print " Search for this alert failed: %s" % alert.alertText
error = True
if self.verbosity >= 1:
if results:
print " There were %s results" % len(results)
else:
print " There were no results"
if self.verbosity >= 2:
print " The value of results is: %s" % results
return error, cd['type'], results,
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
main_query = {'caller': 'api_search'}
try:
main_query.update(build_main_query(kwargs['cd'], highlight='text'))
sl = SolrList(main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
type=kwargs['cd']['type'])
except KeyError:
sf = forms.SearchForm({'q': "*:*"})
if sf.is_valid():
main_query.update(
build_main_query(sf.cleaned_data, highlight='text'))
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
)
return sl
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
main_query = {'caller': 'api_search'}
try:
main_query.update(build_main_query(kwargs['cd'],
highlight='text'))
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
type=kwargs['cd']['type']
)
except KeyError:
sf = forms.SearchForm({'q': "*:*"})
if sf.is_valid():
main_query.update(build_main_query(sf.cleaned_data,
highlight='text'))
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
)
return sl
def items(self, obj):
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateArgued desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return conn.raw_query(**main_params).execute()
else:
return []
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateFiled desc',
'rows': '20',
'start': '0',
'caller': 'search_feed',
})
results_si = conn.raw_query(**main_params).execute()
return results_si
else:
return []
def do_search(request, rows=20, order_by=None):
conn = sunburnt.SolrInterface(settings.SOLR_URL, mode='r')
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if order_by:
# Allows an override by calling methods.
cd['order_by'] = order_by
search_form = _clean_form(request, cd)
try:
results_si = conn.raw_query(**search_utils.build_main_query(cd))
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(stat_facet_fields, search_form)
courts, court_count_human, court_count = search_utils.merge_form_with_courts(COURTS, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" % request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" %
request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(
settings.SOLR_OPINION_URL, mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(
settings.SOLR_AUDIO_URL, mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" % request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def emailer(self, cut_off_date):
"""Send out an email to every user whose alert has a new hit for a rate.
Look up all users that have alerts for a given period of time, and iterate
over them. For each of their alerts that has a hit, build up an email that
contains all the hits.
It's tempting to lookup alerts and iterate over those instead of over the
users. The problem with that is that it would send one email per *alert*,
not per *user*.
"""
# Query all users with alerts of the desired frequency
# Use the distinct method to only return one instance of each person.
if self.options.get('user_id'):
userProfiles = UserProfile.objects.filter(alert__alertFrequency=self.options['rate']).\
filter(user__pk=self.options['user_id']).distinct()
else:
userProfiles = UserProfile.objects.filter(alert__alertFrequency=self.options['rate']).distinct()
# for each user with a daily, weekly or monthly alert...
alerts_sent_count = 0
for userProfile in userProfiles:
#...get their alerts...
alerts = userProfile.alert.filter(alertFrequency=self.options['rate'])
if self.verbosity >= 1:
print "\n\nAlerts for user '%s': %s" % (userProfile.user, alerts)
print "*" * 40
hits = []
# ...and iterate over their alerts.
for alert in alerts:
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['filed_after'] = cut_off_date
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print "Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
results = self.conn.raw_query(**main_params).execute()
else:
print "Query for alert %s was invalid" % alert.alertText
print "Errors from the SearchForm: %s" % search_form.errors
continue
except:
traceback.print_exc()
print "Search for this alert failed: %s" % alert.alertText
continue
if self.verbosity >= 1:
print "There were %s results" % len(results)
if self.verbosity >= 2:
print "The value of results is: %s" % results
# hits is a multi-dimensional array. It consists of alerts,
# paired with a list of document dicts, of the form:
# [[alert1, [{hit1}, {hit2}, {hit3}]], [alert2, ...]]
try:
if len(results) > 0:
hits.append([alert, results])
alert.lastHitDate = now()
alert.save()
elif alert.sendNegativeAlert:
# if they want an alert even when no hits.
hits.append([alert, None])
if self.verbosity >= 1:
print "Sending results for negative alert '%s'" % alert.alertName
except Exception, e:
traceback.print_exc()
print "Search failed on this alert: %s" % alert.alertText
print e
if len(hits) > 0:
alerts_sent_count += 1
self.send_alert(userProfile, hits)
elif self.verbosity >= 1:
print "No hits, thus not sending mail for this alert."