def run_query(self, alert):
results = []
error = False
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if self.rate == 'rt' and len(self.valid_ids[cd['type']]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd['type'], results
cut_off_date = get_cut_off_date(self.rate)
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if self.rate == 'rt':
main_params['fq'].append(
'id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[cd['type']]]
),
)
results = self.connections[
cd['type']
].raw_query(
**main_params
).execute()
else:
logger.info(" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" %
(alert.query, search_form.errors))
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" %
alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd['type'], results
def run_query(self, alert, cut_off_date):
results = None
error = False
try:
if self.verbosity >= 1:
print "Now running the query: %s" % alert.alertText
# Set up the data
data = search_utils.get_string_to_dict(alert.alertText)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
if self.verbosity >= 1:
print " Data sent to SearchForm is: %s" % data
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
results = conn.raw_query(**main_params).execute()
else:
print " Query for alert %s was invalid" % alert.alertText
print " Errors from the SearchForm: %s" % search_form.errors
error = True
except:
traceback.print_exc()
print " Search for this alert failed: %s" % alert.alertText
error = True
if self.verbosity >= 1:
if results:
print " There were %s results" % len(results)
else:
print " There were no results"
if self.verbosity >= 2:
print " The value of results is: %s" % results
return error, cd['type'], results,
def items(self, obj):
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateArgued desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return conn.raw_query(**main_params).execute()
else:
return []
def coverage_graph(request):
courts = Court.objects.filter(in_use=True)
courts_json = json.dumps(build_court_dicts(courts))
search_form = SearchForm(request.GET)
precedential_statuses = [
field for field in search_form.fields.iterkeys()
if field.startswith('stat_')
]
# Build up the sourcing stats.
counts = Document.objects.values('source').annotate(Count('source'))
count_pro = 0
count_lawbox = 0
count_scraper = 0
for d in counts:
if 'R' in d['source']:
count_pro += d['source__count']
if 'C' in d['source']:
count_scraper += d['source__count']
if 'L' in d['source']:
count_lawbox += d['source__count']
opinion_courts = Court.objects.filter(in_use=True,
has_opinion_scraper=True)
oral_argument_courts = Court.objects.filter(in_use=True,
has_oral_argument_scraper=True)
return render_to_response(
'simple_pages/coverage_graph.html', {
'sorted_courts': courts_json,
'precedential_statuses': precedential_statuses,
'count_pro': count_pro,
'count_lawbox': count_lawbox,
'count_scraper': count_scraper,
'courts_with_opinion_scrapers': opinion_courts,
'courts_with_oral_argument_scrapers': oral_argument_courts,
'private': False
}, RequestContext(request))
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search page with request: %s" %
request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def show_results(request):
"""
This view can vary significantly, depending on how it is called:
- In its most simple form, it is called via GET and without any
parameters.
--> This loads the homepage.
- It might also be called with GET *with* parameters.
--> This loads search results.
- It might be called with a POST.
--> This attempts to save an alert.
It also has a few failure modes it needs to support:
- It must react properly to an invalid alert form.
- It must react properly to an invalid or failing search form.
All of these paths have tests.
"""
# Create a search string that does not contain the page numbers
get_string = search_utils.make_get_string(request)
get_string_sans_alert = search_utils.make_get_string(
request, ['page', 'edit_alert'])
render_dict = {
'private': True,
'get_string': get_string,
'get_string_sans_alert': get_string_sans_alert,
}
if request.method == 'POST':
# The user is trying to save an alert.
alert_form = CreateAlertForm(request.POST, user=request.user)
if alert_form.is_valid():
cd = alert_form.cleaned_data
# save the alert
if request.POST.get('edit_alert'):
# check if the user can edit this, or if they are url hacking
alert = get_object_or_404(Alert,
pk=request.POST.get('edit_alert'),
userprofile=request.user.profile)
alert_form = CreateAlertForm(cd,
instance=alert,
user=request.user)
alert_form.save()
action = "edited"
else:
alert_form = CreateAlertForm(cd, user=request.user)
alert = alert_form.save()
# associate the user with the alert
up = request.user.profile
up.alert.add(alert)
action = "created"
messages.add_message(request, messages.SUCCESS,
'Your alert was %s successfully.' % action)
# and redirect to the alerts page
return HttpResponseRedirect('/profile/alerts/')
else:
# Invalid form. Do the search again and show them the alert form
# with the errors
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search/search.html',
render_dict,
RequestContext(request),
)
else:
# Either a search or the homepage
if len(request.GET) == 0:
# No parameters --> Homepage.
if not is_bot(request):
tally_stat('search.homepage_loaded')
# Load the render_dict with good results that can be shown in the
# "Latest Cases" section
render_dict.update(
do_search(request, rows=5, order_by='dateFiled desc'))
# Get the results from the oral arguments as well
oa_dict = do_search(request,
rows=5,
order_by='dateArgued desc',
type='oa')
render_dict.update({'results_oa': oa_dict['results']})
# But give it a fresh form for the advanced search section
render_dict.update({'search_form': SearchForm(request.GET)})
ten_days_ago = make_aware(datetime.today() - timedelta(days=10),
utc)
alerts_in_last_ten = Stat.objects\
.filter(
name__contains='alerts.sent',
date_logged__gte=ten_days_ago)\
.aggregate(Sum('count'))['count__sum']
queries_in_last_ten = Stat.objects\
.filter(
name='search.results',
date_logged__gte=ten_days_ago) \
.aggregate(Sum('count'))['count__sum']
bulk_in_last_ten = Stat.objects\
.filter(
name__contains='bulk_data',
date_logged__gte=ten_days_ago)\
.aggregate(Sum('count'))['count__sum']
api_in_last_ten = Stat.objects \
.filter(
name__contains='api',
date_logged__gte=ten_days_ago) \
.aggregate(Sum('count'))['count__sum']
users_in_last_ten = User.objects\
.filter(date_joined__gte=ten_days_ago).count()
opinions_in_last_ten = Document.objects\
.filter(time_retrieved__gte=ten_days_ago).count()
oral_arguments_in_last_ten = Audio.objects\
.filter(time_retrieved__gte=ten_days_ago).count()
days_of_oa = naturalduration(
Audio.objects.aggregate(Sum('duration'))['duration__sum'],
as_dict=True,
)['d']
render_dict.update({
'alerts_in_last_ten': alerts_in_last_ten,
'queries_in_last_ten': queries_in_last_ten,
'opinions_in_last_ten': opinions_in_last_ten,
'oral_arguments_in_last_ten': oral_arguments_in_last_ten,
'bulk_in_last_ten': bulk_in_last_ten,
'api_in_last_ten': api_in_last_ten,
'users_in_last_ten': users_in_last_ten,
'days_of_oa': days_of_oa,
'private': False
})
return render_to_response('homepage.html', render_dict,
RequestContext(request))
else:
# User placed a search or is trying to edit an alert
if request.GET.get('edit_alert'):
# They're editing an alert
alert = get_object_or_404(Alert,
pk=request.GET.get('edit_alert'),
userprofile=request.user.profile)
alert_form = CreateAlertForm(
instance=alert,
initial={'query': get_string_sans_alert},
user=request.user,
)
else:
# Just a regular search
if not is_bot(request):
tally_stat('search.results')
# Create bare-bones alert form.
alert_form = CreateAlertForm(initial={
'query': get_string,
'rate': "dly"
},
user=request.user)
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search/search.html',
render_dict,
RequestContext(request),
)
