def get_result_count(request, version, day_count):
"""Get the count of results for the past `day_count` number of days
GET parameters will be a complete search string
:param request: The Django request object
:param version: The API version number (ignored for now, but there for
later)
:param day_count: The number of days to average across. More is slower.
:return: A JSON object with the number of hits during the last day_range
period.
"""
search_form = SearchForm(request.GET.copy())
if not search_form.is_valid():
return JsonResponse(
{"error": "Invalid SearchForm"},
safe=True,
status=HTTP_400_BAD_REQUEST,
)
cd = search_form.cleaned_data
try:
si = get_solr_interface(cd)
except NotImplementedError:
logger.error(
"Tried getting solr connection for %s, but it's not "
"implemented yet",
cd["type"],
)
raise
response = (si.query().add_extra(
**build_alert_estimation_query(cd, int(day_count))).execute())
si.conn.http_connection.close()
return JsonResponse({"count": response.result.numFound}, safe=True)
def build_main_query_from_query_string(query_string,
updates=None,
kwargs=None):
"""Build a main query dict from a query string
:param query_string: A GET string to build from.
:param updates: A dict that can be added to the normal finished query
string to override any of its defaults.
:param kwargs: Kwargs to send to the build_main_query function
:return: A dict that can be sent to Solr for querying
"""
qd = QueryDict(query_string)
search_form = SearchForm(qd)
if not search_form.is_valid():
return None
cd = search_form.cleaned_data
if kwargs is None:
main_query = build_main_query(cd)
else:
main_query = build_main_query(cd, **kwargs)
if updates is not None:
main_query.update(updates)
return main_query
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
order_by = "dateFiled"
if cd["type"] == SEARCH_TYPES.OPINION:
solr = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode="r")
elif cd["type"] == SEARCH_TYPES.RECAP:
solr = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode="r")
else:
return []
main_params = search_utils.build_main_query(
cd, highlight=False, facet=False
)
main_params.update(
{
"sort": "%s desc" % order_by,
"rows": "20",
"start": "0",
"caller": "SearchFeed",
}
)
# Eliminate items that lack the ordering field.
main_params["fq"].append("%s:[* TO *]" % order_by)
items = solr.query().add_extra(**main_params).execute()
solr.conn.http_connection.close()
return items
else:
return []
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
order_by = 'dateFiled'
if cd['type'] == 'o':
solr = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
elif cd['type'] == 'r':
solr = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
else:
return []
main_params = search_utils.build_main_query(cd, highlight=False,
facet=False)
main_params.update({
'sort': '%s desc' % order_by,
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
# Eliminate items that lack the ordering field.
main_params['fq'].append('%s:[* TO *]' % order_by)
return solr.query().add_extra(**main_params).execute()
else:
return []
def advanced(request):
render_dict = {'private': False}
# I'm not thrilled about how this is repeating URLs in a view.
if request.path == reverse('advanced_o'):
obj_type = 'o'
# Needed b/c of facet values.
render_dict.update(do_search(request, rows=1, type=obj_type,
facet=True))
render_dict['search_form'] = SearchForm({'type': obj_type})
return render(request, 'advanced.html', render_dict)
else:
if request.path == reverse('advanced_r'):
obj_type = 'r'
elif request.path == reverse('advanced_oa'):
obj_type = 'oa'
elif request.path == reverse('advanced_p'):
obj_type = 'p'
else:
raise NotImplementedError("Unknown path: %s" % request.path)
courts = Court.objects.filter(in_use=True)
search_form = SearchForm({'type': obj_type})
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
render_dict.update({
'search_form': search_form,
'courts': courts,
'court_count_human': court_count_human,
'court_count': court_count,
})
return render(request, 'advanced.html', render_dict)
def run_query(self, alert, rate):
results = []
error = False
cd = {}
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if rate == 'rt' and len(self.valid_ids[cd['type']]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd['type'], results
cut_off_date = get_cut_off_date(rate)
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd, facet=False)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if rate == 'rt':
main_params['fq'].append('id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[cd['type']]]))
results = self.connections[cd['type']].query().add_extra(
**main_params).execute()
regroup_snippets(results)
else:
logger.info(" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" %
(alert.query, search_form.errors))
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" % alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd.get('type'), results
def do_search(request, rows=20, order_by=None, type=None, facet=True):
query_citation = None
error = False
paged_results = None
search_form = SearchForm(request.GET)
courts = Court.objects.filter(in_use=True)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
search_form = _clean_form(request, cd, courts)
if cd['type'] == 'o':
si = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
query_citation = get_query_citation(cd)
elif cd['type'] == 'r':
si = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
elif cd['type'] == 'oa':
si = ExtraSolrInterface(settings.SOLR_AUDIO_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
elif cd['type'] == 'p':
si = ExtraSolrInterface(settings.SOLR_PEOPLE_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
# Set up pagination
try:
if cd['type'] == 'r':
rows = 10
paginator = Paginator(results, rows)
page = request.GET.get('page', 1)
try:
paged_results = paginator.page(page)
except PageNotAnInteger:
paged_results = paginator.page(1)
except EmptyPage:
# Page is out of range (e.g. 9999), deliver last page.
paged_results = paginator.page(paginator.num_pages)
except Exception, e:
# Catches any Solr errors, and aborts.
logger.warning("Error loading pagination on search page with "
"request: %s" % request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
error = True
# Post processing of the results
regroup_snippets(paged_results)
def run_query(self, alert, rate):
results = []
error = False
cd = {}
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data["filed_before"]
except KeyError:
pass
data["order_by"] = "score desc"
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if rate == "rt" and len(self.valid_ids[cd["type"]]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd["type"], results
cut_off_date = get_cut_off_date(rate)
if cd["type"] == "o":
cd["filed_after"] = cut_off_date
elif cd["type"] == "oa":
cd["argued_after"] = cut_off_date
main_params = search_utils.build_main_query(cd)
main_params.update(
{
"rows": "20",
"start": "0",
"hl.tag.pre": "<em><strong>",
"hl.tag.post": "</strong></em>",
"caller": "cl_send_alerts",
}
)
if rate == "rt":
main_params["fq"].append("id:(%s)" % " OR ".join([str(i) for i in self.valid_ids[cd["type"]]]))
results = self.connections[cd["type"]].raw_query(**main_params).execute()
else:
logger.info(
" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" % (alert.query, search_form.errors)
)
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" % alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd.get("type"), results
def run_query(self, alert, rate):
results = []
cd = {}
logger.info("Now running the query: %s\n" % alert.query)
# Make a dict from the query string.
qd = QueryDict(alert.query.encode('utf-8'), mutable=True)
try:
del qd['filed_before']
except KeyError:
pass
qd['order_by'] = 'score desc'
cut_off_date = get_cut_off_date(rate)
# Default to 'o', if not available, according to the front end.
query_type = qd.get('type', 'o')
if query_type in ['o', 'r']:
qd['filed_after'] = cut_off_date
elif query_type == 'oa':
qd['argued_after'] = cut_off_date
logger.info("Data sent to SearchForm is: %s\n" % qd)
search_form = SearchForm(qd)
if search_form.is_valid():
cd = search_form.cleaned_data
if rate == Alert.REAL_TIME and \
len(self.valid_ids[query_type]) == 0:
# Bail out. No results will be found if no valid_ids.
return query_type, results
main_params = search_utils.build_main_query(cd, facet=False)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts:%s' % query_type,
})
if rate == Alert.REAL_TIME:
main_params['fq'].append(
'id:(%s)' %
' OR '.join([str(i) for i in self.valid_ids[query_type]]))
# Ignore warnings from this bit of code. Otherwise, it complains
# about the query URL being too long and having to POST it instead
# of being able to GET it.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = self.connections[query_type].query().add_extra(
**main_params).execute()
regroup_snippets(results)
logger.info("There were %s results." % len(results))
return qd, results
def run_query(self, alert, rate):
results = []
cd = {}
logger.info("Now running the query: %s\n" % alert.query)
# Make a dict from the query string.
qd = QueryDict(alert.query.encode("utf-8"), mutable=True)
try:
del qd["filed_before"]
except KeyError:
pass
qd["order_by"] = "score desc"
cut_off_date = get_cut_off_date(rate)
# Default to 'o', if not available, according to the front end.
query_type = qd.get("type", "o")
if query_type in ["o", "r"]:
qd["filed_after"] = cut_off_date
elif query_type == "oa":
qd["argued_after"] = cut_off_date
logger.info("Data sent to SearchForm is: %s\n" % qd)
search_form = SearchForm(qd)
if search_form.is_valid():
cd = search_form.cleaned_data
if (rate == Alert.REAL_TIME
and len(self.valid_ids[query_type]) == 0):
# Bail out. No results will be found if no valid_ids.
return query_type, results
main_params = search_utils.build_main_query(cd, facet=False)
main_params.update({
"rows": "20",
"start": "0",
"hl.tag.pre": "<em><strong>",
"hl.tag.post": "</strong></em>",
"caller": "cl_send_alerts:%s" % query_type,
})
if rate == Alert.REAL_TIME:
main_params["fq"].append(
"id:(%s)" %
" OR ".join([str(i) for i in self.valid_ids[query_type]]))
# Ignore warnings from this bit of code. Otherwise, it complains
# about the query URL being too long and having to POST it instead
# of being able to GET it.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = (self.connections[query_type].query().add_extra(
**main_params).execute())
regroup_snippets(results)
logger.info("There were %s results." % len(results))
return qd, results
def items(self, obj):
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateArgued desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return conn.raw_query(**main_params).execute()
else:
return []
def run_query(self, alert, rate):
results = []
cd = {}
logger.info("Now running the query: %s\n" % alert.query)
# Make a dict from the query string.
qd = QueryDict(alert.query.encode('utf-8'), mutable=True)
try:
del qd['filed_before']
except KeyError:
pass
qd['order_by'] = 'score desc'
cut_off_date = get_cut_off_date(rate)
# Default to 'o', if not available, according to the front end.
query_type = qd.get('type', 'o')
if query_type in ['o', 'r']:
qd['filed_after'] = cut_off_date
elif query_type == 'oa':
qd['argued_after'] = cut_off_date
logger.info("Data sent to SearchForm is: %s\n" % qd)
search_form = SearchForm(qd)
if search_form.is_valid():
cd = search_form.cleaned_data
if rate == Alert.REAL_TIME and \
len(self.valid_ids[query_type]) == 0:
# Bail out. No results will be found if no valid_ids.
return query_type, results
main_params = search_utils.build_main_query(cd, facet=False)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts:%s' % query_type,
})
if rate == Alert.REAL_TIME:
main_params['fq'].append('id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[query_type]]
))
results = self.connections[query_type].query().add_extra(
**main_params).execute()
regroup_snippets(results)
logger.info("There were %s results." % len(results))
return qd, results
def items(self, obj):
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
solr = ExtraSolrInterface(settings.SOLR_AUDIO_URL, mode="r")
main_params = search_utils.build_main_query(cd,
highlight=False,
facet=False)
main_params.update({
"sort": "dateArgued desc",
"rows": "20",
"start": "0",
"caller": "SearchFeed",
})
return solr.query().add_extra(**main_params).execute()
else:
return []
def advanced(request):
render_dict = {"private": False}
# I'm not thrilled about how this is repeating URLs in a view.
if request.path == reverse("advanced_o"):
obj_type = SEARCH_TYPES.OPINION
# Needed b/c of facet values.
o_results = do_search(
request.GET.copy(),
rows=1,
override_params={
"type": obj_type,
},
facet=True,
cache_key="opinion-homepage-results",
)
render_dict.update(o_results)
render_dict["search_form"] = SearchForm({"type": obj_type})
return render(request, "advanced.html", render_dict)
else:
courts = Court.objects.filter(in_use=True)
if request.path == reverse("advanced_r"):
obj_type = SEARCH_TYPES.RECAP
courts = courts.filter(
pacer_court_id__isnull=False,
end_date__isnull=True,
).exclude(jurisdiction=Court.FEDERAL_BANKRUPTCY_PANEL)
elif request.path == reverse("advanced_oa"):
obj_type = SEARCH_TYPES.ORAL_ARGUMENT
elif request.path == reverse("advanced_p"):
obj_type = SEARCH_TYPES.PEOPLE
else:
raise NotImplementedError("Unknown path: %s" % request.path)
search_form = SearchForm({"type": obj_type})
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
render_dict.update({
"search_form": search_form,
"courts": courts,
"court_count_human": court_count_human,
"court_count": court_count,
})
return render(request, "advanced.html", render_dict)
def list(self, request, *args, **kwargs):
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd["q"] == "":
cd["q"] = "*" # Get everything
paginator = pagination.PageNumberPagination()
sl = api_utils.get_object_list(request, cd=cd, paginator=paginator)
result_page = paginator.paginate_queryset(sl, request)
serializer = SearchResultSerializer(
result_page, many=True, context={"schema": sl.conn.schema})
return paginator.get_paginated_response(serializer.data)
# Invalid search.
return response.Response(search_form.errors,
status=status.HTTP_400_BAD_REQUEST)
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
solr = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
elif cd['type'] == 'r':
solr = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
main_params = search_utils.build_main_query(cd, highlight=False)
main_params.update({
'sort': 'dateFiled desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return solr.query().add_extra(**main_params).execute()
else:
return []
def coverage_graph(request: HttpRequest) -> HttpResponse:
coverage_cache_key = "coverage-data-v2"
coverage_data = cache.get(coverage_cache_key)
if coverage_data is None:
courts = Court.objects.filter(in_use=True)
courts_json = json.dumps(build_court_dicts(courts))
search_form = SearchForm(request.GET)
precedential_statuses = [
field
for field in search_form.fields.keys()
if field.startswith("stat_")
]
# Build up the sourcing stats.
counts = OpinionCluster.objects.values("source").annotate(
Count("source")
)
count_pro = 0
count_lawbox = 0
count_scraper = 0
for d in counts:
if "R" in d["source"]:
count_pro += d["source__count"]
if "C" in d["source"]:
count_scraper += d["source__count"]
if "L" in d["source"]:
count_lawbox += d["source__count"]
opinion_courts = Court.objects.filter(
in_use=True, has_opinion_scraper=True
)
oral_argument_courts = Court.objects.filter(
in_use=True, has_oral_argument_scraper=True
)
oa_duration = Audio.objects.aggregate(Sum("duration"))["duration__sum"]
if oa_duration:
oa_duration /= 60 # Avoids a "unsupported operand type" error
coverage_data = {
"sorted_courts": courts_json,
"precedential_statuses": precedential_statuses,
"oa_duration": oa_duration,
"count_pro": count_pro,
"count_lawbox": count_lawbox,
"count_scraper": count_scraper,
"courts_with_opinion_scrapers": opinion_courts,
"courts_with_oral_argument_scrapers": oral_argument_courts,
"private": False,
}
one_day = 60 * 60 * 24
cache.set(coverage_cache_key, coverage_data, one_day)
return render(request, "coverage.html", coverage_data)
def advanced(request):
render_dict = {'private': False}
# I'm not thrilled about how this is repeating URLs in a view.
if request.path == reverse('advanced_o'):
obj_type = 'o'
# Needed b/c of facet values.
o_results = do_search(request,
rows=1,
type=obj_type,
facet=True,
cache_key='opinion-homepage-results')
render_dict.update(o_results)
render_dict['search_form'] = SearchForm({'type': obj_type})
return render(request, 'advanced.html', render_dict)
else:
courts = Court.objects.filter(in_use=True)
if request.path == reverse('advanced_r'):
obj_type = 'r'
courts = courts.filter(
pacer_court_id__isnull=False,
end_date__isnull=True,
).exclude(jurisdiction=Court.FEDERAL_BANKRUPTCY_PANEL, )
elif request.path == reverse('advanced_oa'):
obj_type = 'oa'
elif request.path == reverse('advanced_p'):
obj_type = 'p'
else:
raise NotImplementedError("Unknown path: %s" % request.path)
search_form = SearchForm({'type': obj_type})
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
render_dict.update({
'search_form': search_form,
'courts': courts,
'court_count_human': court_count_human,
'court_count': court_count,
})
return render(request, 'advanced.html', render_dict)
def items(self, obj):
"""Do a Solr query here. Return the first 20 results"""
search_form = SearchForm(obj.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['type'] == 'o':
solr = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
elif cd['type'] == 'r':
solr = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
main_params = search_utils.build_main_query(cd,
highlight=False,
facet=False)
main_params.update({
'sort': 'dateFiled desc',
'rows': '20',
'start': '0',
'caller': 'SearchFeed',
})
return solr.query().add_extra(**main_params).execute()
else:
return []
def coverage_graph(request):
coverage_cache_key = 'coverage-data-v2'
coverage_data = cache.get(coverage_cache_key)
if coverage_data is None:
courts = Court.objects.filter(in_use=True)
courts_json = json.dumps(build_court_dicts(courts))
search_form = SearchForm(request.GET)
precedential_statuses = [
field for field in search_form.fields.keys()
if field.startswith('stat_')
]
# Build up the sourcing stats.
counts = OpinionCluster.objects.values('source').annotate(
Count('source'))
count_pro = 0
count_lawbox = 0
count_scraper = 0
for d in counts:
if 'R' in d['source']:
count_pro += d['source__count']
if 'C' in d['source']:
count_scraper += d['source__count']
if 'L' in d['source']:
count_lawbox += d['source__count']
opinion_courts = Court.objects.filter(in_use=True,
has_opinion_scraper=True)
oral_argument_courts = Court.objects.filter(
in_use=True, has_oral_argument_scraper=True)
oa_duration = Audio.objects.aggregate(Sum('duration'))['duration__sum']
if oa_duration:
oa_duration /= 60 # Avoids a "unsupported operand type" error
coverage_data = {
'sorted_courts': courts_json,
'precedential_statuses': precedential_statuses,
'oa_duration': oa_duration,
'count_pro': count_pro,
'count_lawbox': count_lawbox,
'count_scraper': count_scraper,
'courts_with_opinion_scrapers': opinion_courts,
'courts_with_oral_argument_scrapers': oral_argument_courts,
'private': False
}
one_day = 60 * 60 * 24
cache.set(coverage_cache_key, coverage_data, one_day)
return render(request, 'coverage.html', coverage_data)
def list(self, request, *args, **kwargs):
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['q'] == '':
cd['q'] = '*:*' # Get everything
paginator = pagination.PageNumberPagination()
sl = api_utils.get_object_list(request, cd=cd, paginator=paginator)
result_page = paginator.paginate_queryset(sl, request)
serializer = SearchResultSerializer(
result_page,
many=True,
context={'schema': sl.conn.schema}
)
return paginator.get_paginated_response(serializer.data)
# Invalid search.
return response.Response(
search_form.errors,
status=status.HTTP_400_BAD_REQUEST
)
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by:
cd['order_by'] = order_by
if type:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True).values(
'pk', 'short_name', 'jurisdiction',
'has_oral_argument_scraper')
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception, e:
logger.warning("Error loading search with request: %s" %
request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
def build_main_query_from_query_string(query_string, updates=None, kwargs=None):
"""Build a main query dict from a query string
:param query_string: A GET string to build from.
:param updates: A dict that can be added to the normal finished query
string to override any of its defaults.
:param kwargs: Kwargs to send to the build_main_query function
:return: A dict that can be sent to Solr for querying
"""
qd = QueryDict(query_string)
search_form = SearchForm(qd)
if not search_form.is_valid():
return None
cd = search_form.cleaned_data
if kwargs is None:
main_query = build_main_query(cd)
else:
main_query = build_main_query(cd, **kwargs)
if updates is not None:
main_query.update(updates)
return main_query
def coverage_graph(request):
courts = Court.objects.filter(in_use=True)
courts_json = json.dumps(build_court_dicts(courts))
search_form = SearchForm(request.GET)
precedential_statuses = [field for field in
search_form.fields.keys() if field.startswith('stat_')]
# Build up the sourcing stats.
counts = OpinionCluster.objects.values('source').annotate(Count('source'))
count_pro = 0
count_lawbox = 0
count_scraper = 0
for d in counts:
if 'R' in d['source']:
count_pro += d['source__count']
if 'C' in d['source']:
count_scraper += d['source__count']
if 'L' in d['source']:
count_lawbox += d['source__count']
opinion_courts = Court.objects.filter(
in_use=True,
has_opinion_scraper=True)
oral_argument_courts = Court.objects.filter(
in_use=True,
has_oral_argument_scraper=True)
return render_to_response(
'coverage_graph.html',
{
'sorted_courts': courts_json,
'precedential_statuses': precedential_statuses,
'count_pro': count_pro,
'count_lawbox': count_lawbox,
'count_scraper': count_scraper,
'courts_with_opinion_scrapers': opinion_courts,
'courts_with_oral_argument_scrapers': oral_argument_courts,
'private': False
},
RequestContext(request))
def show_results(request):
"""
This view can vary significantly, depending on how it is called:
- In its most simple form, it is called via GET and without any
parameters.
--> This loads the homepage.
- It might also be called with GET *with* parameters.
--> This loads search results.
- It might be called with a POST.
--> This attempts to save an alert.
It also has a few failure modes it needs to support:
- It must react properly to an invalid alert form.
- It must react properly to an invalid or failing search form.
All of these paths have tests.
"""
# Create a search string that does not contain the page numbers
get_string = make_get_string(request)
get_string_sans_alert = make_get_string(request, ['page', 'edit_alert'])
render_dict = {
'private': True,
'get_string': get_string,
'get_string_sans_alert': get_string_sans_alert,
}
if request.method == 'POST':
# The user is trying to save an alert.
alert_form = CreateAlertForm(request.POST, user=request.user)
if alert_form.is_valid():
cd = alert_form.cleaned_data
# save the alert
if request.POST.get('edit_alert'):
# check if the user can edit this, or if they are url hacking
alert = get_object_or_404(
Alert,
pk=request.POST.get('edit_alert'),
user=request.user,
)
alert_form = CreateAlertForm(cd,
instance=alert,
user=request.user)
alert_form.save()
action = "edited"
else:
alert_form = CreateAlertForm(cd, user=request.user)
alert = alert_form.save(commit=False)
alert.user = request.user
alert.save()
action = "created"
messages.add_message(request, messages.SUCCESS,
'Your alert was %s successfully.' % action)
# and redirect to the alerts page
return HttpResponseRedirect(reverse("profile_alerts"))
else:
# Invalid form. Do the search again and show them the alert form
# with the errors
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render(request, 'search.html', render_dict)
else:
# Either a search or the homepage
if len(request.GET) == 0:
# No parameters --> Homepage.
if not is_bot(request):
tally_stat('search.homepage_loaded')
# Ensure we get nothing from the future.
request.GET = request.GET.copy() # Makes it mutable
request.GET['filed_before'] = date.today()
homepage_cache_key = 'homepage-data'
homepage_dict = cache.get(homepage_cache_key)
if homepage_dict is not None:
return render(request, 'homepage.html', homepage_dict)
# Load the render_dict with good results that can be shown in the
# "Latest Cases" section
render_dict.update(
do_search(request,
rows=5,
order_by='dateFiled desc',
facet=False))
# Get the results from the oral arguments as well
oa_dict = do_search(request,
rows=5,
order_by='dateArgued desc',
type='oa',
facet=False)
render_dict.update({'results_oa': oa_dict['results']})
# But give it a fresh form for the advanced search section
render_dict.update({'search_form': SearchForm(request.GET)})
# Get a bunch of stats.
render_dict.update(get_homepage_stats())
six_hours = 60 * 60 * 6
cache.set(homepage_cache_key, render_dict, six_hours)
return render(request, 'homepage.html', render_dict)
else:
# User placed a search or is trying to edit an alert
if request.GET.get('edit_alert'):
# They're editing an alert
if request.user.is_anonymous():
return HttpResponseRedirect(
"{path}?next={next}{encoded_params}".format(
path=reverse('sign-in'),
next=request.path,
encoded_params=quote("?" +
request.GET.urlencode())))
else:
alert = get_object_or_404(Alert,
pk=request.GET.get('edit_alert'),
user=request.user)
alert_form = CreateAlertForm(
instance=alert,
initial={'query': get_string_sans_alert},
user=request.user,
)
else:
# Just a regular search
if not is_bot(request):
tally_stat('search.results')
# Create bare-bones alert form.
alert_form = CreateAlertForm(initial={
'query': get_string,
'rate': "dly"
},
user=request.user)
render_dict.update(do_search(request))
# Set the value to the query as a convenience
alert_form.fields['name'].widget.attrs['value'] = \
render_dict['search_summary_str']
render_dict.update({'alert_form': alert_form})
return render(request, 'search.html', render_dict)
def show_results(request):
"""
This view can vary significantly, depending on how it is called:
- In its most simple form, it is called via GET and without any
parameters.
--> This loads the homepage.
- It might also be called with GET *with* parameters.
--> This loads search results.
- It might be called with a POST.
--> This attempts to save an alert.
It also has a few failure modes it needs to support:
- It must react properly to an invalid alert form.
- It must react properly to an invalid or failing search form.
All of these paths have tests.
"""
# Create a search string that does not contain the page numbers
get_string = search_utils.make_get_string(request)
get_string_sans_alert = search_utils.make_get_string(
request, ['page', 'edit_alert'])
render_dict = {
'private': True,
'get_string': get_string,
'get_string_sans_alert': get_string_sans_alert,
}
if request.method == 'POST':
# The user is trying to save an alert.
alert_form = CreateAlertForm(request.POST, user=request.user)
if alert_form.is_valid():
cd = alert_form.cleaned_data
# save the alert
if request.POST.get('edit_alert'):
# check if the user can edit this, or if they are url hacking
alert = get_object_or_404(
Alert,
pk=request.POST.get('edit_alert'),
user=request.user,
)
alert_form = CreateAlertForm(cd,
instance=alert,
user=request.user)
alert_form.save()
action = "edited"
else:
alert_form = CreateAlertForm(cd, user=request.user)
alert = alert_form.save(commit=False)
alert.user = request.user
alert.save()
action = "created"
messages.add_message(request, messages.SUCCESS,
'Your alert was %s successfully.' % action)
# and redirect to the alerts page
return HttpResponseRedirect(reverse("profile_alerts"))
else:
# Invalid form. Do the search again and show them the alert form
# with the errors
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search.html',
render_dict,
RequestContext(request),
)
else:
# Either a search or the homepage
if len(request.GET) == 0:
# No parameters --> Homepage.
if not is_bot(request):
tally_stat('search.homepage_loaded')
# Load the render_dict with good results that can be shown in the
# "Latest Cases" section
render_dict.update(
do_search(request, rows=5, order_by='dateFiled desc'))
# Get the results from the oral arguments as well
oa_dict = do_search(request,
rows=5,
order_by='dateArgued desc',
type='oa')
render_dict.update({'results_oa': oa_dict['results']})
# But give it a fresh form for the advanced search section
render_dict.update({'search_form': SearchForm(request.GET)})
ten_days_ago = make_aware(datetime.today() - timedelta(days=10),
utc)
alerts_in_last_ten = Stat.objects.filter(
name__contains='alerts.sent',
date_logged__gte=ten_days_ago).aggregate(
Sum('count'))['count__sum']
queries_in_last_ten = Stat.objects.filter(
name='search.results',
date_logged__gte=ten_days_ago).aggregate(
Sum('count'))['count__sum']
bulk_in_last_ten = Stat.objects.filter(
name__contains='bulk_data',
date_logged__gte=ten_days_ago).aggregate(
Sum('count'))['count__sum']
r = redis.StrictRedis(
host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DATABASES['STATS'],
)
last_ten_days = [
'api:v3.d:%s.count' %
(date.today() - timedelta(days=x)).isoformat()
for x in range(0, 10)
]
api_in_last_ten = sum([
int(result) for result in r.mget(*last_ten_days)
if result is not None
])
users_in_last_ten = User.objects.filter(
date_joined__gte=ten_days_ago).count()
opinions_in_last_ten = Opinion.objects.filter(
date_created__gte=ten_days_ago).count()
oral_arguments_in_last_ten = Audio.objects.filter(
date_created__gte=ten_days_ago).count()
days_of_oa = naturalduration(
Audio.objects.aggregate(Sum('duration'))['duration__sum'],
as_dict=True,
)['d']
viz_in_last_ten = SCOTUSMap.objects.filter(
date_published__gte=ten_days_ago,
published=True,
).count()
visualizations = SCOTUSMap.objects.filter(
published=True,
deleted=False,
).annotate(Count('clusters'), ).filter(
# Ensures that we only show good stuff on homepage
clusters__count__gt=10, ).order_by(
'-date_published',
'-date_modified',
'-date_created',
)[:1]
render_dict.update({
'alerts_in_last_ten': alerts_in_last_ten,
'queries_in_last_ten': queries_in_last_ten,
'opinions_in_last_ten': opinions_in_last_ten,
'oral_arguments_in_last_ten': oral_arguments_in_last_ten,
'bulk_in_last_ten': bulk_in_last_ten,
'api_in_last_ten': api_in_last_ten,
'users_in_last_ten': users_in_last_ten,
'days_of_oa': days_of_oa,
'viz_in_last_ten': viz_in_last_ten,
'visualizations': visualizations,
'private': False, # VERY IMPORTANT!
})
return render_to_response('homepage.html', render_dict,
RequestContext(request))
else:
# User placed a search or is trying to edit an alert
if request.GET.get('edit_alert'):
# They're editing an alert
if request.user.is_anonymous():
return HttpResponseRedirect(
"{path}?next={next}{encoded_params}".format(
path=reverse('sign-in'),
next=request.path,
encoded_params=quote("?" +
request.GET.urlencode())))
else:
alert = get_object_or_404(Alert,
pk=request.GET.get('edit_alert'),
user=request.user)
alert_form = CreateAlertForm(
instance=alert,
initial={'query': get_string_sans_alert},
user=request.user,
)
else:
# Just a regular search
if not is_bot(request):
tally_stat('search.results')
# Create bare-bones alert form.
alert_form = CreateAlertForm(initial={
'query': get_string,
'rate': "dly"
},
user=request.user)
render_dict.update(do_search(request))
render_dict.update({'alert_form': alert_form})
return render_to_response(
'search.html',
render_dict,
RequestContext(request),
)
def do_search(
get_params,
rows=20,
override_params=None,
facet=True,
cache_key=None,
):
"""Do all the difficult solr work.
:param get_params: The request.GET parameters sent by the user. Note that
this cannot simply be request.GET since that is immutable and
override_params needs to be able to change this. Instead generally it's
best to send request.GET.copy().
:param rows: The number of solr results to request
:param override_params: A dict with additional or different GET params to
be sent to solr.
:param facet: Whether to complete faceting in the query
:param cache_key: A cache key with which to save the results. Note that it
does not do anything clever with the actual query, so if you use this, your
cache key should *already* have factored in the query. If None, no caching
is set or used. Results are saved for six hours.
:return A big dict of variables for use in the search results, homepage, or
other location.
"""
query_citation = None
error = False
paged_results = None
courts = Court.objects.filter(in_use=True)
# Add additional or overridden GET parameters
if override_params:
get_params.update(override_params)
search_form = SearchForm(get_params)
if search_form.is_valid():
cd = search_form.cleaned_data
# Do the query, hitting the cache if desired
try:
results = get_solr_result_objects(cd, facet)
paged_results = paginate_cached_solr_results(
get_params, cd, results, rows, cache_key)
except (NotImplementedError, RequestException, SolrError) as e:
error = True
logger.warning("Error loading search page with request: %s" %
get_params)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
# A couple special variables for particular search types
search_form = _clean_form(get_params, cd, courts)
if cd["type"] in [SEARCH_TYPES.OPINION, SEARCH_TYPES.RECAP]:
query_citation = get_query_citation(cd)
if cd["type"] == SEARCH_TYPES.RECAP:
panels = Court.FEDERAL_BANKRUPTCY_PANEL
courts = courts.filter(
pacer_court_id__isnull=False,
end_date__isnull=True).exclude(jurisdiction=panels)
else:
error = True
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
search_summary_str = search_form.as_text(court_count_human)
search_summary_dict = search_form.as_display_dict(court_count_human)
cited_cluster = add_depth_counts( # Also returns cited cluster if found
search_data=cd,
search_results=paged_results,
)
return {
"results": paged_results,
"facet_fields": make_stats_variable(search_form, paged_results),
"search_form": search_form,
"search_summary_str": search_summary_str,
"search_summary_dict": search_summary_dict,
"courts": courts,
"court_count_human": court_count_human,
"court_count": court_count,
"query_citation": query_citation,
"error": error,
"cited_cluster": cited_cluster,
}
def do_search(request, rows=20, order_by=None, type=None, facet=True,
cache_key=None):
"""Do all the difficult solr work.
:param request: The request made by the user
:param rows: The number of solr results to request
:param order_by: An opportunity to override the ordering of the search
results
:param type: An opportunity to override the type
:param facet: Whether to complete faceting in the query
:param cache_key: A cache key with which to save the results. Note that it
does not do anything clever with the actual query, so if you use this, your
cache key should *already* have factored in the query. If None, no caching
is set or used. Results are saved for six hours.
:return A big dict of variables for use in the search results, homepage, or
other location.
"""
query_citation = None
error = False
paged_results = None
search_form = SearchForm(request.GET)
courts = Court.objects.filter(in_use=True)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
# Do the query, hitting the cache if desired
# noinspection PyBroadException
try:
results = get_solr_result_objects(cd, facet)
paged_results = paginate_cached_solr_results(request, cd, results,
rows, cache_key)
except (NotImplementedError, RequestException, SolrError) as e:
error = True
logger.warning("Error loading search page with "
"request: %s" % request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
# A couple special variables for particular search types
search_form = _clean_form(request, cd, courts)
if cd['type'] == 'o':
query_citation = get_query_citation(cd)
elif cd['type'] == 'r':
panels = Court.FEDERAL_BANKRUPTCY_PANEL
courts = (courts.filter(pacer_court_id__isnull=False,
end_date__isnull=True)
.exclude(jurisdiction=panels))
else:
error = True
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
search_summary_str = search_form.as_text(court_count, court_count_human)
return {
'results': paged_results,
'facet_fields': make_stats_variable(search_form, paged_results),
'search_form': search_form,
'search_summary_str': search_summary_str,
'courts': courts,
'court_count_human': court_count_human,
'court_count': court_count,
'query_citation': query_citation,
'error': error,
}
def do_search(request, rows=20, order_by=None, type=None):
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
query_citation = None
status_facets = None
if cd['type'] == 'o':
si = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
stat_facet_fields = place_facet_queries(cd, si)
status_facets = make_stats_variable(stat_facet_fields,
search_form)
query_citation = get_query_citation(cd)
results = si.query().add_extra(**build_main_query(cd))
elif cd['type'] == 'r':
si = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd))
elif cd['type'] == 'oa':
si = ExtraSolrInterface(settings.SOLR_AUDIO_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd))
elif cd['type'] == 'p':
si = ExtraSolrInterface(settings.SOLR_PEOPLE_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd))
courts = Court.objects.filter(in_use=True)
courts, court_count_human, court_count = merge_form_with_courts(
courts,
search_form
)
except Exception as e:
if settings.DEBUG is True:
traceback.print_exc()
logger.warning("Error loading search with request: %s" % request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
else:
# Invalid form, send it back
logger.warning("Invalid form when loading search page with request: %s" % request.GET)
return {'error': True}
# Set up pagination
try:
paginator = Paginator(results, rows)
page = request.GET.get('page', 1)
try:
paged_results = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
paged_results = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
paged_results = paginator.page(paginator.num_pages)
except Exception, e:
# Catches any Solr errors, and aborts.
logger.warning("Error loading pagination on search page with request: %s" % request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
return {'error': True}
def do_search(
get_params,
rows=20,
override_params=None,
facet=True,
cache_key=None,
):
"""Do all the difficult solr work.
:param get_params: The request.GET parameters sent by the user. Note that
this cannot simply be request.GET since that is immutable and
override_params needs to be able to change this. Instead generally it's
best to send request.GET.copy().
:param rows: The number of solr results to request
:param override_params: A dict with additional or different GET params to
be sent to solr.
:param facet: Whether to complete faceting in the query
:param cache_key: A cache key with which to save the results. Note that it
does not do anything clever with the actual query, so if you use this, your
cache key should *already* have factored in the query. If None, no caching
is set or used. Results are saved for six hours.
:return A big dict of variables for use in the search results, homepage, or
other location.
"""
query_citation = None
error = False
paged_results = None
cited_cluster = None
courts = Court.objects.filter(in_use=True)
related_cluster_pks = None
# Add additional or overridden GET parameters
if override_params:
get_params.update(override_params)
search_form = SearchForm(get_params)
if search_form.is_valid():
cd = search_form.cleaned_data
# Do the query, hitting the cache if desired
try:
si = get_solr_interface(cd)
except NotImplementedError:
logger.error(
"Tried getting solr connection for %s, but it's not "
"implemented yet",
cd["type"],
)
raise
try:
# Is this a `related:<pks>` prefix query?
related_prefix_match = RELATED_PATTERN.search(cd["q"])
if related_prefix_match:
# Seed IDs
related_cluster_pks = related_prefix_match.group("pks").split(
",")
results = get_mlt_query(
si,
cd.copy(),
facet,
related_cluster_pks,
# Original query
cd["q"].replace(related_prefix_match.group("pfx"), ""),
)
else:
# Regular search queries
results = si.query().add_extra(
**build_main_query(cd, facet=facet))
paged_results = paginate_cached_solr_results(
get_params, cd, results, rows, cache_key)
cited_cluster = add_depth_counts(
# Also returns cited cluster if found
search_data=cd,
search_results=paged_results,
)
except (NotImplementedError, RequestException, SolrError) as e:
error = True
logger.warning("Error loading search page with request: %s" %
get_params)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
# A couple special variables for particular search types
search_form = _clean_form(get_params, cd, courts)
if cd["type"] in [
SEARCH_TYPES.OPINION,
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
]:
query_citation = get_query_citation(cd)
if cd["type"] in [
SEARCH_TYPES.RECAP,
SEARCH_TYPES.DOCKETS,
SEARCH_TYPES.PEOPLE,
]:
panels = Court.FEDERAL_BANKRUPTCY_PANEL
courts = courts.filter(
pacer_court_id__isnull=False,
end_date__isnull=True).exclude(jurisdiction=panels)
else:
error = True
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
search_summary_str = search_form.as_text(court_count_human)
search_summary_dict = search_form.as_display_dict(court_count_human)
related_cluster = (OpinionCluster.objects.get(
sub_opinions__pk__in=related_cluster_pks)
if related_cluster_pks else None)
return {
"results": paged_results,
"facet_fields": make_stats_variable(search_form, paged_results),
"search_form": search_form,
"search_summary_str": search_summary_str,
"search_summary_dict": search_summary_dict,
"courts": courts,
"court_count_human": court_count_human,
"court_count": court_count,
"query_citation": query_citation,
"error": error,
"cited_cluster": cited_cluster,
"related_cluster": related_cluster,
}
def show_results(request):
"""
This view can vary significantly, depending on how it is called:
- In its most simple form, it is called via GET and without any
parameters.
--> This loads the homepage.
- It might also be called with GET *with* parameters.
--> This loads search results.
- It might be called with a POST.
--> This attempts to save an alert.
It also has a few failure modes it needs to support:
- It must react properly to an invalid alert form.
- It must react properly to an invalid or failing search form.
All of these paths have tests.
"""
# Create a search string that does not contain the page numbers
get_string = make_get_string(request)
get_string_sans_alert = make_get_string(
request, ["page", "edit_alert", "show_alert_modal"])
render_dict = {
"private": True,
"get_string": get_string,
"get_string_sans_alert": get_string_sans_alert,
}
if request.method == "POST":
# The user is trying to save an alert.
alert_form = CreateAlertForm(request.POST, user=request.user)
if alert_form.is_valid():
cd = alert_form.cleaned_data
# save the alert
if request.POST.get("edit_alert"):
# check if the user can edit this, or if they are url hacking
alert = get_object_or_404(
Alert,
pk=request.POST.get("edit_alert"),
user=request.user,
)
alert_form = CreateAlertForm(cd,
instance=alert,
user=request.user)
alert_form.save()
action = "edited"
else:
alert_form = CreateAlertForm(cd, user=request.user)
alert = alert_form.save(commit=False)
alert.user = request.user
alert.save()
action = "created"
messages.add_message(
request,
messages.SUCCESS,
"Your alert was %s successfully." % action,
)
# and redirect to the alerts page
return HttpResponseRedirect(reverse("profile_alerts"))
else:
# Invalid form. Do the search again and show them the alert form
# with the errors
render_dict.update(do_search(request.GET.copy()))
render_dict.update({"alert_form": alert_form})
return render(request, "search.html", render_dict)
else:
# Either a search or the homepage
if len(request.GET) == 0:
# No parameters --> Homepage.
if not is_bot(request):
tally_stat("search.homepage_loaded")
# Ensure we get nothing from the future.
mutable_GET = request.GET.copy() # Makes it mutable
mutable_GET["filed_before"] = date.today()
# Load the render_dict with good results that can be shown in the
# "Latest Cases" section
render_dict.update(
do_search(
mutable_GET,
rows=5,
override_params={"order_by": "dateFiled desc"},
facet=False,
cache_key="homepage-data-o",
))
# Get the results from the oral arguments as well
render_dict.update({
"results_oa":
do_search(
mutable_GET,
rows=5,
override_params={
"order_by": "dateArgued desc",
"type": SEARCH_TYPES.ORAL_ARGUMENT,
},
facet=False,
cache_key="homepage-data-oa",
)["results"]
})
# But give it a fresh form for the advanced search section
render_dict.update({"search_form": SearchForm(request.GET)})
# Get a bunch of stats.
stats = get_homepage_stats()
render_dict.update(stats)
return render(request, "homepage.html", render_dict)
else:
# User placed a search or is trying to edit an alert
if request.GET.get("edit_alert"):
# They're editing an alert
if request.user.is_anonymous:
return HttpResponseRedirect(
"{path}?next={next}{encoded_params}".format(
path=reverse("sign-in"),
next=request.path,
encoded_params=quote("?" +
request.GET.urlencode()),
))
else:
alert = get_object_or_404(
Alert,
pk=request.GET.get("edit_alert"),
user=request.user,
)
alert_form = CreateAlertForm(
instance=alert,
initial={"query": get_string_sans_alert},
user=request.user,
)
else:
# Just a regular search
if not is_bot(request):
tally_stat("search.results")
# Create bare-bones alert form.
alert_form = CreateAlertForm(
initial={
"query": get_string,
"rate": "dly"
},
user=request.user,
)
render_dict.update(do_search(request.GET.copy()))
# Set the value to the query as a convenience
alert_form.fields["name"].widget.attrs["value"] = render_dict[
"search_summary_str"]
render_dict.update({"alert_form": alert_form})
return render(request, "search.html", render_dict)
def do_search(request, rows=20, order_by=None, type=None):
# Bind the search form.
search_form = SearchForm(request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
search_form = _clean_form(request, cd)
try:
if cd['type'] == 'o':
conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL,
mode='r')
stat_facet_fields = search_utils.place_facet_queries(cd, conn)
status_facets = search_utils.make_stats_variable(
stat_facet_fields, search_form)
elif cd['type'] == 'oa':
conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL,
mode='r')
status_facets = None
elif cd['type'] == 'p':
conn = sunburnt.SolrInterface(settings.SOLR_PEOPLE_URL,
mode='r')
status_facets = None
results_si = conn.raw_query(**search_utils.build_main_query(cd))
courts = Court.objects.filter(in_use=True)
courts, court_count_human, court_count = search_utils\
.merge_form_with_courts(courts, search_form)
except Exception as e:
if settings.DEBUG is True:
traceback.print_exc()
logger.warning("Error loading search with request: %s" %
request.GET)
logger.warning("Error was %s" % e)
return {'error': True}
else:
# Invalid form, send it back
logger.warning(
"Invalid form when loading search page with request: %s" %
request.GET)
return {'error': True}
# Set up pagination
try:
paginator = Paginator(results_si, rows)
page = request.GET.get('page', 1)
try:
paged_results = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
paged_results = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
paged_results = paginator.page(paginator.num_pages)
except Exception, e:
# Catches any Solr errors, and aborts.
logger.warning(
"Error loading pagination on search page with request: %s" %
request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
print e
return {'error': True}
def run_query(self, alert, rate):
results = []
error = False
cd = {}
try:
logger.info("Now running the query: %s\n" % alert.query)
# Set up the data
data = search_utils.get_string_to_dict(alert.query)
try:
del data['filed_before']
except KeyError:
pass
data['order_by'] = 'score desc'
logger.info(" Data sent to SearchForm is: %s\n" % data)
search_form = SearchForm(data)
if search_form.is_valid():
cd = search_form.cleaned_data
if rate == 'rt' and len(self.valid_ids[cd['type']]) == 0:
# Bail out. No results will be found if no valid_ids.
return error, cd['type'], results
cut_off_date = get_cut_off_date(rate)
if cd['type'] == 'o':
cd['filed_after'] = cut_off_date
elif cd['type'] == 'oa':
cd['argued_after'] = cut_off_date
main_params = search_utils.build_main_query(cd, facet=False)
main_params.update({
'rows': '20',
'start': '0',
'hl.tag.pre': '<em><strong>',
'hl.tag.post': '</strong></em>',
'caller': 'cl_send_alerts',
})
if rate == 'rt':
main_params['fq'].append('id:(%s)' % ' OR '.join(
[str(i) for i in self.valid_ids[cd['type']]]
))
results = self.connections[
cd['type']
].query().add_extra(
**main_params
).execute()
regroup_snippets(results)
else:
logger.info(" Query for alert %s was invalid\n"
" Errors from the SearchForm: %s\n" %
(alert.query, search_form.errors))
error = True
except:
traceback.print_exc()
logger.info(" Search for this alert failed: %s\n" %
alert.query)
error = True
logger.info(" There were %s results\n" % len(results))
return error, cd.get('type'), results
def do_search(request, rows=20, order_by=None, type=None, facet=True):
query_citation = None
error = False
paged_results = None
search_form = SearchForm(request.GET)
courts = Court.objects.filter(in_use=True)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
search_form = _clean_form(request, cd, courts)
if cd['type'] == 'o':
si = ExtraSolrInterface(settings.SOLR_OPINION_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
query_citation = get_query_citation(cd)
elif cd['type'] == 'r':
si = ExtraSolrInterface(settings.SOLR_RECAP_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
elif cd['type'] == 'oa':
si = ExtraSolrInterface(settings.SOLR_AUDIO_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
elif cd['type'] == 'p':
si = ExtraSolrInterface(settings.SOLR_PEOPLE_URL, mode='r')
results = si.query().add_extra(**build_main_query(cd, facet=facet))
# Set up pagination
try:
if cd['type'] == 'r':
rows = 10
paginator = Paginator(results, rows)
page = request.GET.get('page', 1)
try:
paged_results = paginator.page(page)
except PageNotAnInteger:
paged_results = paginator.page(1)
except EmptyPage:
# Page is out of range (e.g. 9999), deliver last page.
paged_results = paginator.page(paginator.num_pages)
except Exception as e:
# Catches any Solr errors, and aborts.
logger.warning("Error loading pagination on search page with "
"request: %s" % request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
error = True
# Post processing of the results
regroup_snippets(paged_results)
else:
error = True
courts, court_count_human, court_count = merge_form_with_courts(courts,
search_form)
return {
'results': paged_results,
'search_form': search_form,
'courts': courts,
'court_count_human': court_count_human,
'court_count': court_count,
'query_citation': query_citation,
'facet_fields': make_stats_variable(search_form, paged_results),
'error': error,
}
def do_search(request,
rows=20,
order_by=None,
type=None,
facet=True,
cache_key=None):
"""Do all the difficult solr work.
:param request: The request made by the user
:param rows: The number of solr results to request
:param order_by: An opportunity to override the ordering of the search
results
:param type: An opportunity to override the type
:param facet: Whether to complete faceting in the query
:param cache_key: A cache key with which to save the results. Note that it
does not do anything clever with the actual query, so if you use this, your
cache key should *already* have factored in the query. If None, no caching
is set or used. Results are saved for six hours.
:return A big dict of variables for use in the search results, homepage, or
other location.
"""
query_citation = None
error = False
paged_results = None
search_form = SearchForm(request.GET)
courts = Court.objects.filter(in_use=True)
if search_form.is_valid():
cd = search_form.cleaned_data
# Allows an override by calling methods.
if order_by is not None:
cd['order_by'] = order_by
if type is not None:
cd['type'] = type
# Do the query, hitting the cache if desired
try:
results = get_solr_result_objects(cd, facet)
paged_results = paginate_cached_solr_results(
request, cd, results, rows, cache_key)
except (NotImplementedError, RequestException, SolrError) as e:
error = True
logger.warning("Error loading search page with "
"request: %s" % request.GET)
logger.warning("Error was: %s" % e)
if settings.DEBUG is True:
traceback.print_exc()
# A couple special variables for particular search types
search_form = _clean_form(request, cd, courts)
if cd['type'] == 'o':
query_citation = get_query_citation(cd)
elif cd['type'] == 'r':
panels = Court.FEDERAL_BANKRUPTCY_PANEL
courts = (courts.filter(
pacer_court_id__isnull=False,
end_date__isnull=True).exclude(jurisdiction=panels))
else:
error = True
courts, court_count_human, court_count = merge_form_with_courts(
courts, search_form)
search_summary_str = search_form.as_text(court_count, court_count_human)
return {
'results': paged_results,
'facet_fields': make_stats_variable(search_form, paged_results),
'search_form': search_form,
'search_summary_str': search_summary_str,
'courts': courts,
'court_count_human': court_count_human,
'court_count': court_count,
'query_citation': query_citation,
'error': error,
}