def run_query(self, alert):
results = []
error = False
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if self.rate == 'rt' and len(self.valid_ids[cd['type']]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd['type'], results
cut_off_date = get_cut_off_date(self.rate)
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if self.rate == 'rt':
main_params['fq'].append(
'id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[cd['type']]]
),
)
results = self.connections[
cd['type']
].raw_query(
**main_params
).execute()
else:
logger.info(" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" %
(alert.query, search_form.errors))
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" %
alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd['type'], results
def obj_get(self, bundle, **kwargs):
search_form = SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
cd['q'] = 'id:%s' % kwargs['pk']
return self.get_object_list(bundle.request, cd=cd)[0]
else:
BadRequest("Invalid resource lookup data provided. Unable to complete your request.")
def run_query(self, alert, cut_off_date):
results = None
error = False
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print " Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(
settings.SOLR_OPINION_URL, mode='r'
)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(
settings.SOLR_AUDIO_URL, mode='r'
)
results = conn.raw_query(**main_params).execute()
else:
print " Query for alert %s was invalid" % alert.alertText
print " Errors from the SearchForm: %s" % search_form.errors
error = True
except:
traceback.print_exc()
print " Search for this alert failed: %s" % alert.alertText
error = True
if self.verbosity >= 1:
if results:
print " There were %s results" % len(results)
else:
print " There were no results"
if self.verbosity >= 2:
print " The value of results is: %s" % results
return error, cd['type'], results,
def obj_get_list(self, bundle, **kwargs):
search_form = SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['q'] == '':
cd['q'] = '*:*' # Get everything.
return self.get_object_list(bundle.request, cd=cd)
else:
BadRequest("Invalid resource lookup data provided. Unable to complete your query.")
def run_query(self, alert, cut_off_date):
results = None
error = False
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print " Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
results = conn.raw_query(**main_params).execute()
else:
print " Query for alert %s was invalid" % alert.alertText
print " Errors from the SearchForm: %s" % search_form.errors
error = True
except:
traceback.print_exc()
print " Search for this alert failed: %s" % alert.alertText
error = True
if self.verbosity >= 1:
if results:
print " There were %s results" % len(results)
else:
print " There were no results"
if self.verbosity >= 2:
print " The value of results is: %s" % results
return error, cd['type'], results,
def items(self, obj):
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateArgued desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return conn.raw_query(**main_params).execute()
else:
return []
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
try:
main_query = build_main_query(kwargs['cd'], highlight='text')
except KeyError:
sf = SearchForm({'q': "*:*"})
if sf.is_valid():
main_query = build_main_query(sf.cleaned_data, highlight='text')
main_query['caller'] = 'api_search'
# Use a SolrList that has a couple of the normal functions built in.
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20)
)
return sl
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateFiled desc',
'rows': '20',
'start': '0',
'caller': 'search_feed',
})
results_si = conn.raw_query(**main_params).execute()
return results_si
else:
return []
def do_search(request, rows=20, order_by=None):
conn = sunburnt.SolrInterface(settings.SOLR_URL, mode='r')
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if order_by:
# Allows an override by calling methods.
cd['order_by'] = order_by
search_form = _clean_form(request, cd)
try:
results_si = conn.raw_query(**search_utils.build_main_query(cd))
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(stat_facet_fields, search_form)
courts, court_count_human, court_count = search_utils.merge_form_with_courts(COURTS, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" % request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" %
request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(
settings.SOLR_OPINION_URL, mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(
settings.SOLR_AUDIO_URL, mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" % request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def coverage_graph(request):
courts = Court.objects.filter(in_use=True)
courts_json = json.dumps(build_court_dicts(courts))
search_form = SearchForm(request.GET)
precedential_statuses = [
field for field in search_form.fields.iterkeys()
if field.startswith('stat_')
]
# Build up the sourcing stats.
counts = Document.objects.values('source').annotate(Count('source'))
count_pro = 0
count_lawbox = 0
count_scraper = 0
for d in counts:
if 'R' in d['source']:
count_pro += d['source__count']
if 'C' in d['source']:
count_scraper += d['source__count']
if 'L' in d['source']:
count_lawbox += d['source__count']
opinion_courts = Court.objects.filter(in_use=True,
has_opinion_scraper=True)
oral_argument_courts = Court.objects.filter(in_use=True,
has_oral_argument_scraper=True)
return render_to_response(
'simple_pages/coverage_graph.html', {
'sorted_courts': courts_json,
'precedential_statuses': precedential_statuses,
'count_pro': count_pro,
'count_lawbox': count_lawbox,
'count_scraper': count_scraper,
'courts_with_opinion_scrapers': opinion_courts,
'courts_with_oral_argument_scrapers': oral_argument_courts,
'private': False
}, RequestContext(request))
def emailer(self, cut_off_date):
"""Send out an email to every user whose alert has a new hit for a rate.
Look up all users that have alerts for a given period of time, and iterate
over them. For each of their alerts that has a hit, build up an email that
contains all the hits.
It's tempting to lookup alerts and iterate over those instead of over the
users. The problem with that is that it would send one email per *alert*,
not per *user*.
"""
# Query all users with alerts of the desired frequency
# Use the distinct method to only return one instance of each person.
if self.options.get('user_id'):
userProfiles = UserProfile.objects.filter(alert__alertFrequency=self.options['rate']).\
filter(user__pk=self.options['user_id']).distinct()
else:
userProfiles = UserProfile.objects.filter(alert__alertFrequency=self.options['rate']).distinct()
# for each user with a daily, weekly or monthly alert...
alerts_sent_count = 0
for userProfile in userProfiles:
#...get their alerts...
alerts = userProfile.alert.filter(alertFrequency=self.options['rate'])
if self.verbosity >= 1:
print "\n\nAlerts for user '%s': %s" % (userProfile.user, alerts)
print "*" * 40
hits = []
# ...and iterate over their alerts.
for alert in alerts:
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['filed_after'] = cut_off_date
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print "Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
results = self.conn.raw_query(**main_params).execute()
else:
print "Query for alert %s was invalid" % alert.alertText
print "Errors from the SearchForm: %s" % search_form.errors
continue
except:
traceback.print_exc()
print "Search for this alert failed: %s" % alert.alertText
continue
if self.verbosity >= 1:
print "There were %s results" % len(results)
if self.verbosity >= 2:
print "The value of results is: %s" % results
# hits is a multi-dimensional array. It consists of alerts,
# paired with a list of document dicts, of the form:
# [[alert1, [{hit1}, {hit2}, {hit3}]], [alert2, ...]]
try:
if len(results) > 0:
hits.append([alert, results])
alert.lastHitDate = now()
alert.save()
elif alert.sendNegativeAlert:
# if they want an alert even when no hits.
hits.append([alert, None])
if self.verbosity >= 1:
print "Sending results for negative alert '%s'" % alert.alertName
except Exception, e:
traceback.print_exc()
print "Search failed on this alert: %s" % alert.alertText
print e
if len(hits) > 0:
alerts_sent_count += 1
self.send_alert(userProfile, hits)
elif self.verbosity >= 1:
print "No hits, thus not sending mail for this alert."
def show_results(request):
"""
This view can vary significantly, depending on how it is called:
- In its most simple form, it is called via GET and without any
parameters.
--> This loads the homepage.
- It might also be called with GET *with* parameters.
--> This loads search results.
- It might be called with a POST.
--> This attempts to save an alert.
It also has a few failure modes it needs to support:
- It must react properly to an invalid alert form.
- It must react properly to an invalid or failing search form.
All of these paths have tests.
"""
# Create a search string that does not contain the page numbers
get_string = search_utils.make_get_string(request)
get_string_sans_alert = search_utils.make_get_string(
request, ['page', 'edit_alert'])
render_dict = {
'private': True,
'get_string': get_string,
'get_string_sans_alert': get_string_sans_alert,
}
if request.method == 'POST':
# The user is trying to save an alert.
alert_form = CreateAlertForm(request.POST, user=request.user)
if alert_form.is_valid():
cd = alert_form.cleaned_data
# save the alert
if request.POST.get('edit_alert'):
# check if the user can edit this, or if they are url hacking
alert = get_object_or_404(Alert,
pk=request.POST.get('edit_alert'),
userprofile=request.user.profile)
alert_form = CreateAlertForm(cd,
instance=alert,
user=request.user)
alert_form.save()
action = "edited"
else:
alert_form = CreateAlertForm(cd, user=request.user)
alert = alert_form.save()
# associate the user with the alert
up = request.user.profile
up.alert.add(alert)
action = "created"
messages.add_message(request, messages.SUCCESS,
'Your alert was %s successfully.' % action)
# and redirect to the alerts page
return HttpResponseRedirect('/profile/alerts/')
else:
# Invalid form. Do the search again and show them the alert form
# with the errors
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search/search.html',
render_dict,
RequestContext(request),
)
else:
# Either a search or the homepage
if len(request.GET) == 0:
# No parameters --> Homepage.
if not is_bot(request):
tally_stat('search.homepage_loaded')
# Load the render_dict with good results that can be shown in the
# "Latest Cases" section
render_dict.update(
do_search(request, rows=5, order_by='dateFiled desc'))
# Get the results from the oral arguments as well
oa_dict = do_search(request,
rows=5,
order_by='dateArgued desc',
type='oa')
render_dict.update({'results_oa': oa_dict['results']})
# But give it a fresh form for the advanced search section
render_dict.update({'search_form': SearchForm(request.GET)})
ten_days_ago = make_aware(datetime.today() - timedelta(days=10),
utc)
alerts_in_last_ten = Stat.objects\
.filter(
name__contains='alerts.sent',
date_logged__gte=ten_days_ago)\
.aggregate(Sum('count'))['count__sum']
queries_in_last_ten = Stat.objects\
.filter(
name='search.results',
date_logged__gte=ten_days_ago) \
.aggregate(Sum('count'))['count__sum']
bulk_in_last_ten = Stat.objects\
.filter(
name__contains='bulk_data',
date_logged__gte=ten_days_ago)\
.aggregate(Sum('count'))['count__sum']
api_in_last_ten = Stat.objects \
.filter(
name__contains='api',
date_logged__gte=ten_days_ago) \
.aggregate(Sum('count'))['count__sum']
users_in_last_ten = User.objects\
.filter(date_joined__gte=ten_days_ago).count()
opinions_in_last_ten = Document.objects\
.filter(time_retrieved__gte=ten_days_ago).count()
oral_arguments_in_last_ten = Audio.objects\
.filter(time_retrieved__gte=ten_days_ago).count()
days_of_oa = naturalduration(
Audio.objects.aggregate(Sum('duration'))['duration__sum'],
as_dict=True,
)['d']
render_dict.update({
'alerts_in_last_ten': alerts_in_last_ten,
'queries_in_last_ten': queries_in_last_ten,
'opinions_in_last_ten': opinions_in_last_ten,
'oral_arguments_in_last_ten': oral_arguments_in_last_ten,
'bulk_in_last_ten': bulk_in_last_ten,
'api_in_last_ten': api_in_last_ten,
'users_in_last_ten': users_in_last_ten,
'days_of_oa': days_of_oa,
'private': False
})
return render_to_response('homepage.html', render_dict,
RequestContext(request))
else:
# User placed a search or is trying to edit an alert
if request.GET.get('edit_alert'):
# They're editing an alert
alert = get_object_or_404(Alert,
pk=request.GET.get('edit_alert'),
userprofile=request.user.profile)
alert_form = CreateAlertForm(
instance=alert,
initial={'query': get_string_sans_alert},
user=request.user,
)
else:
# Just a regular search
if not is_bot(request):
tally_stat('search.results')
# Create bare-bones alert form.
alert_form = CreateAlertForm(initial={
'query': get_string,
'rate': "dly"
},
user=request.user)
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search/search.html',
render_dict,
RequestContext(request),
)
