def search_requests(query,
foil_id,
title,
agency_request_summary,
description,
requester_name,
date_rec_from,
date_rec_to,
date_due_from,
date_due_to,
date_closed_from,
date_closed_to,
agency_ein,
agency_user_guid,
open_,
closed,
in_progress,
due_soon,
overdue,
start,
sort_date_received,
sort_date_due,
sort_title,
tz_name,
size=None,
by_phrase=False,
highlight=False,
for_csv=False):
"""
The arguments of this function match the request parameters
of the '/search/requests' endpoints.
All date related params expect strings in the format "mm/dd/yyyy"
All sort related params expect "desc" or "asc"; other strings ignored
:param query: string to query for
:param foil_id: search by request id?
:param title: search by title?
:param agency_request_summary: search by agency request summary?
:param description: search by description?
:param requester_name: search by requester name?
:param date_rec_from: date created/submitted from
:param date_rec_to: date created/submitted to
:param date_due_from: date due from
:param date_due_to: date due to
:param date_closed_from: date closed from
:param date_closed_to: date closed to
:param agency_ein: agency ein to filter by
:param agency_user_guid: user (agency) guid to filter by
:param open_: filter by opened requests?
:param closed: filter by closed requests?
:param in_progress: filter by in-progress requests?
:param due_soon: filter by due-soon requests?
:param overdue: filter by overdue requests?
:param size: number of requests per page
:param start: starting index of request result set
:param sort_date_received: date created/submitted sort direction
:param sort_date_due: date due sort direction
:param sort_title: title sort direction
:param tz_name: timezone name (e.g. "America/New_York")
:param by_phrase: use phrase matching instead of full-text?
:param highlight: return highlights?
if True, will come at a slight performance cost (in order to
restrict highlights to public fields, iterating over elasticsearch
query results is required)
:param for_csv: search for a csv export
if True, will not check the maximum value of size against MAX_RESULT_SIZE
:return: elasticsearch json response with result information
"""
# clean query trailing/leading whitespace
if query is not None:
query = query.strip()
# return no results if there is nothing to query by
if query and not any(
(foil_id, title, agency_request_summary, description, requester_name)):
return MOCK_EMPTY_ELASTICSEARCH_RESULT
# if searching by foil-id, strip "FOIL-"
if foil_id:
query = query.lstrip("FOIL-").lstrip('foil-')
# set sort (list of "field:direction" pairs)
sort = [
':'.join((field, direction)) for field, direction in {
'date_received': sort_date_received,
'date_due': sort_date_due,
'title.keyword': sort_title
}.items() if direction in ("desc", "asc")
]
# if no sort options are selected use date_received desc by default
if len(sort) == 0:
sort = ['date_received:desc']
# set statuses (list of request statuses)
if current_user.is_agency:
statuses = {
request_status.OPEN: open_,
request_status.CLOSED: closed,
request_status.IN_PROGRESS: in_progress,
request_status.DUE_SOON: due_soon,
request_status.OVERDUE: overdue
}
statuses = [s for s, b in statuses.items() if b]
else:
statuses = []
if open_:
# Any request that isn't closed is considered open
statuses.extend([
request_status.OPEN, request_status.IN_PROGRESS,
request_status.DUE_SOON, request_status.OVERDUE
])
if closed:
statuses.append(request_status.CLOSED)
# set matching type (full-text or phrase matching)
match_type = 'match_phrase' if by_phrase else 'match'
# set date ranges
def datestr_local_to_utc(datestr):
return local_to_utc(datetime.strptime(datestr, DT_DATE_RANGE_FORMAT),
tz_name).strftime(DT_DATE_RANGE_FORMAT)
date_ranges = []
if any((date_rec_from, date_rec_to, date_due_from, date_due_to,
date_closed_from, date_closed_to)):
range_filters = {}
if date_rec_from or date_rec_to:
range_filters['date_received'] = {'format': ES_DATE_RANGE_FORMAT}
if date_due_from or date_due_to:
range_filters['date_due'] = {'format': ES_DATE_RANGE_FORMAT}
if date_closed_from or date_closed_to:
range_filters['date_closed'] = {'format': ES_DATE_RANGE_FORMAT}
if date_rec_from:
range_filters['date_received']['gte'] = datestr_local_to_utc(
date_rec_from)
if date_rec_to:
range_filters['date_received']['lt'] = datestr_local_to_utc(
date_rec_to)
if date_due_from:
range_filters['date_due']['gte'] = datestr_local_to_utc(
date_due_from)
if date_due_to:
range_filters['date_due']['lt'] = datestr_local_to_utc(date_due_to)
if date_closed_from:
range_filters['date_closed']['gte'] = datestr_local_to_utc(
date_closed_from)
if date_closed_to:
range_filters['date_closed']['lte'] = datestr_local_to_utc(
date_closed_to)
if date_rec_from or date_rec_to:
date_ranges.append(
{'range': {
'date_received': range_filters['date_received']
}})
if date_due_from or date_due_to:
date_ranges.append(
{'range': {
'date_due': range_filters['date_due']
}})
if date_closed_from or date_closed_to:
date_ranges.append(
{'range': {
'date_closed': range_filters['date_closed']
}})
# generate query dsl body
query_fields = {
'title': title,
'description': description,
'agency_request_summary': agency_request_summary,
'requester_name': requester_name
}
dsl_gen = RequestsDSLGenerator(query, query_fields, statuses, date_ranges,
agency_ein, agency_user_guid, match_type)
if foil_id:
dsl = dsl_gen.foil_id()
else:
if query:
if current_user.is_agency:
dsl = dsl_gen.agency_user()
elif current_user.is_anonymous:
dsl = dsl_gen.anonymous_user()
elif current_user.is_public:
dsl = dsl_gen.public_user()
else:
raise InvalidUserException(current_user)
else:
dsl = dsl_gen.queryless()
# add highlights to dsl
if highlight:
highlight_fields = {}
for name, add in query_fields.items():
if add:
highlight_fields[name] = {}
dsl.update({
'highlight': {
'pre_tags': ['<span class="highlight">'],
'post_tags': ['</span>'],
'fields': highlight_fields
}
})
# Calculate result set size
result_set_size = size if for_csv else min(size, MAX_RESULT_SIZE)
# search / run query
if not for_csv:
results = es.search(
index=current_app.config["ELASTICSEARCH_INDEX"],
doc_type='request',
body=dsl,
_source=[
'requester_id', 'date_submitted', 'date_due', 'date_received',
'date_created', 'date_closed', 'status', 'agency_ein',
'agency_name', 'agency_acronym', 'requester_name',
'title_private', 'agency_request_summary_private',
'public_title', 'title', 'agency_request_summary',
'description', 'assigned_users'
],
size=result_set_size,
from_=start,
sort=sort,
)
else:
results = es.search(
index=current_app.config["ELASTICSEARCH_INDEX"],
doc_type='request',
scroll='1m',
body=dsl,
_source=[
'requester_id', 'date_submitted', 'date_due', 'date_received',
'date_created', 'date_closed', 'status', 'agency_ein',
'agency_name', 'agency_acronym', 'requester_name',
'title_private', 'agency_request_summary_private',
'public_title', 'title', 'agency_request_summary',
'description', 'assigned_users'
],
size=result_set_size,
from_=start,
sort=sort,
)
sid = results['_scroll_id']
scroll_size = results['hits']['total']
scroll_results = results['hits']['hits']
while scroll_size > 0:
results = es.scroll(scroll='1m',
body={
"scroll": "1m",
"scroll_id": sid
})
scroll_size = len(results['hits']['hits'])
scroll_results += results['hits']['hits']
return scroll_results
# process highlights
if highlight and not foil_id:
_process_highlights(results, dsl_gen.requester_id)
return results
def new():
"""
Create a new FOIL request
sends a confirmation email after the Requests object is created.
title: request title
description: request description
agency: agency selected for the request
submission: submission method for the request
:return: redirect to homepage on successful form validation
if form fields are missing or has improper values, backend error messages (WTForms) will appear
"""
site_key = current_app.config["RECAPTCHA_SITE_KEY"]
kiosk_mode = eval_request_bool(
escape(flask_request.args.get("kiosk_mode", False)))
category = str(escape(flask_request.args.get("category", None)))
agency = str(escape(flask_request.args.get("agency", None)))
title = str(escape(flask_request.args.get("title", None)))
if current_user.is_public:
form = PublicUserRequestForm()
template_suffix = "user.html"
elif current_user.is_anonymous:
form = AnonymousRequestForm()
template_suffix = "anon.html"
elif current_user.is_agency:
form = AgencyUserRequestForm()
template_suffix = "agency.html"
else:
raise InvalidUserException(current_user)
new_request_template = "request/new_request_" + template_suffix
if flask_request.method == "POST":
# validate upload with no request id available
upload_path = None
if form.request_file.data:
form.request_file.validate(form)
upload_path = handle_upload_no_id(form.request_file)
if form.request_file.errors:
return render_template(new_request_template,
form=form,
site_key=site_key)
custom_metadata = json.loads(
flask_request.form.get("custom-request-forms-data", {}))
tz_name = (flask_request.form["tz-name"]
if flask_request.form["tz-name"] else
current_app.config["APP_TIMEZONE"])
if current_user.is_public:
request_id = create_request(
form.request_title.data,
form.request_description.data,
form.request_category.data,
agency_ein=form.request_agency.data,
upload_path=upload_path,
tz_name=tz_name,
custom_metadata=custom_metadata,
)
elif current_user.is_agency:
request_id = create_request(
form.request_title.data,
form.request_description.data,
category=None,
agency_ein=(form.request_agency.data
if form.request_agency.data != "None" else
current_user.default_agency_ein),
submission=form.method_received.data,
agency_date_submitted_local=form.request_date.data,
email=form.email.data,
first_name=form.first_name.data,
last_name=form.last_name.data,
user_title=form.user_title.data,
organization=form.user_organization.data,
phone=form.phone.data,
fax=form.fax.data,
address=get_address(form),
upload_path=upload_path,
tz_name=tz_name,
custom_metadata=custom_metadata,
)
else: # Anonymous User
request_id = create_request(
form.request_title.data,
form.request_description.data,
form.request_category.data,
agency_ein=form.request_agency.data,
email=form.email.data,
first_name=form.first_name.data,
last_name=form.last_name.data,
user_title=form.user_title.data,
organization=form.user_organization.data,
phone=form.phone.data,
fax=form.fax.data,
address=get_address(form),
upload_path=upload_path,
tz_name=tz_name,
custom_metadata=custom_metadata,
)
current_request = Requests.query.filter_by(id=request_id).first()
requester = current_request.requester
send_confirmation_email(request=current_request,
agency=current_request.agency,
user=requester)
if current_request.agency.is_active:
if requester.email:
flashed_message_html = render_template(
"request/confirmation_email.html")
flash(Markup(flashed_message_html), category="success")
else:
flashed_message_html = render_template(
"request/confirmation_non_email.html")
flash(Markup(flashed_message_html), category="warning")
return redirect(url_for("request.view", request_id=request_id))
else:
flashed_message_html = render_template(
"request/non_portal_agency_message.html",
agency=current_request.agency)
flash(Markup(flashed_message_html), category="warning")
return redirect(
url_for("request.non_portal_agency",
agency_name=current_request.agency.name))
return render_template(
new_request_template,
form=form,
site_key=site_key,
kiosk_mode=kiosk_mode,
category=category,
agency=agency,
title=title,
)
def new():
"""
Create a new FOIL request
sends a confirmation email after the Requests object is created.
title: request title
description: request description
agency: agency selected for the request
submission: submission method for the request
:return: redirect to homepage on successful form validation
if form fields are missing or has improper values, backend error messages (WTForms) will appear
"""
site_key = current_app.config['RECAPTCHA_SITE_KEY']
if current_user.is_public:
form = PublicUserRequestForm()
template_suffix = 'user.html'
elif current_user.is_anonymous:
form = AnonymousRequestForm()
template_suffix = 'anon.html'
elif current_user.is_agency:
form = AgencyUserRequestForm()
template_suffix = 'agency.html'
else:
raise InvalidUserException(current_user)
new_request_template = 'request/new_request_' + template_suffix
if flask_request.method == 'POST':
# validate upload with no request id available
upload_path = None
if form.request_file.data:
form.request_file.validate(form)
upload_path = handle_upload_no_id(form.request_file)
if form.request_file.errors:
return render_template(new_request_template, form=form, site_key=site_key)
# create request
if current_user.is_public:
request_id = create_request(form.request_title.data,
form.request_description.data,
form.request_category.data,
agency_ein=form.request_agency.data,
upload_path=upload_path,
tz_name=flask_request.form['tz-name'])
elif current_user.is_agency:
request_id = create_request(form.request_title.data,
form.request_description.data,
category=None,
agency_ein=(
form.request_agency.data
if form.request_agency.data != 'None'
else current_user.default_agency_ein),
submission=form.method_received.data,
agency_date_submitted=form.request_date.data,
email=form.email.data,
first_name=form.first_name.data,
last_name=form.last_name.data,
user_title=form.user_title.data,
organization=form.user_organization.data,
phone=form.phone.data,
fax=form.fax.data,
address=get_address(form),
upload_path=upload_path,
tz_name=flask_request.form['tz-name'])
else: # Anonymous User
request_id = create_request(form.request_title.data,
form.request_description.data,
form.request_category.data,
agency_ein=form.request_agency.data,
email=form.email.data,
first_name=form.first_name.data,
last_name=form.last_name.data,
user_title=form.user_title.data,
organization=form.user_organization.data,
phone=form.phone.data,
fax=form.fax.data,
address=get_address(form),
upload_path=upload_path,
tz_name=flask_request.form['tz-name'])
current_request = Requests.query.filter_by(id=request_id).first()
requester = current_request.requester
send_confirmation_email(request=current_request, agency=current_request.agency, user=requester)
if current_request.agency.is_active:
if requester.email:
flashed_message_html = render_template('request/confirmation_email.html')
flash(Markup(flashed_message_html), category='success')
else:
flashed_message_html = render_template('request/confirmation_non_email.html')
flash(Markup(flashed_message_html), category='warning')
return redirect(url_for('request.view', request_id=request_id))
else:
flashed_message_html = render_template('request/non_portal_agency_message.html',
agency=current_request.agency)
flash(Markup(flashed_message_html), category='warning')
return redirect(url_for('request.non_portal_agency', agency_name=current_request.agency.name))
return render_template(new_request_template, form=form, site_key=site_key)