def search_requests(query, foil_id, title, agency_request_summary, description, requester_name, date_rec_from, date_rec_to, date_due_from, date_due_to, date_closed_from, date_closed_to, agency_ein, agency_user_guid, open_, closed, in_progress, due_soon, overdue, start, sort_date_received, sort_date_due, sort_title, tz_name, size=None, by_phrase=False, highlight=False, for_csv=False): """ The arguments of this function match the request parameters of the '/search/requests' endpoints. All date related params expect strings in the format "mm/dd/yyyy" All sort related params expect "desc" or "asc"; other strings ignored :param query: string to query for :param foil_id: search by request id? :param title: search by title? :param agency_request_summary: search by agency request summary? :param description: search by description? :param requester_name: search by requester name? :param date_rec_from: date created/submitted from :param date_rec_to: date created/submitted to :param date_due_from: date due from :param date_due_to: date due to :param date_closed_from: date closed from :param date_closed_to: date closed to :param agency_ein: agency ein to filter by :param agency_user_guid: user (agency) guid to filter by :param open_: filter by opened requests? :param closed: filter by closed requests? :param in_progress: filter by in-progress requests? :param due_soon: filter by due-soon requests? :param overdue: filter by overdue requests? :param size: number of requests per page :param start: starting index of request result set :param sort_date_received: date created/submitted sort direction :param sort_date_due: date due sort direction :param sort_title: title sort direction :param tz_name: timezone name (e.g. "America/New_York") :param by_phrase: use phrase matching instead of full-text? :param highlight: return highlights? if True, will come at a slight performance cost (in order to restrict highlights to public fields, iterating over elasticsearch query results is required) :param for_csv: search for a csv export if True, will not check the maximum value of size against MAX_RESULT_SIZE :return: elasticsearch json response with result information """ # clean query trailing/leading whitespace if query is not None: query = query.strip() # return no results if there is nothing to query by if query and not any( (foil_id, title, agency_request_summary, description, requester_name)): return MOCK_EMPTY_ELASTICSEARCH_RESULT # if searching by foil-id, strip "FOIL-" if foil_id: query = query.lstrip("FOIL-").lstrip('foil-') # set sort (list of "field:direction" pairs) sort = [ ':'.join((field, direction)) for field, direction in { 'date_received': sort_date_received, 'date_due': sort_date_due, 'title.keyword': sort_title }.items() if direction in ("desc", "asc") ] # if no sort options are selected use date_received desc by default if len(sort) == 0: sort = ['date_received:desc'] # set statuses (list of request statuses) if current_user.is_agency: statuses = { request_status.OPEN: open_, request_status.CLOSED: closed, request_status.IN_PROGRESS: in_progress, request_status.DUE_SOON: due_soon, request_status.OVERDUE: overdue } statuses = [s for s, b in statuses.items() if b] else: statuses = [] if open_: # Any request that isn't closed is considered open statuses.extend([ request_status.OPEN, request_status.IN_PROGRESS, request_status.DUE_SOON, request_status.OVERDUE ]) if closed: statuses.append(request_status.CLOSED) # set matching type (full-text or phrase matching) match_type = 'match_phrase' if by_phrase else 'match' # set date ranges def datestr_local_to_utc(datestr): return local_to_utc(datetime.strptime(datestr, DT_DATE_RANGE_FORMAT), tz_name).strftime(DT_DATE_RANGE_FORMAT) date_ranges = [] if any((date_rec_from, date_rec_to, date_due_from, date_due_to, date_closed_from, date_closed_to)): range_filters = {} if date_rec_from or date_rec_to: range_filters['date_received'] = {'format': ES_DATE_RANGE_FORMAT} if date_due_from or date_due_to: range_filters['date_due'] = {'format': ES_DATE_RANGE_FORMAT} if date_closed_from or date_closed_to: range_filters['date_closed'] = {'format': ES_DATE_RANGE_FORMAT} if date_rec_from: range_filters['date_received']['gte'] = datestr_local_to_utc( date_rec_from) if date_rec_to: range_filters['date_received']['lt'] = datestr_local_to_utc( date_rec_to) if date_due_from: range_filters['date_due']['gte'] = datestr_local_to_utc( date_due_from) if date_due_to: range_filters['date_due']['lt'] = datestr_local_to_utc(date_due_to) if date_closed_from: range_filters['date_closed']['gte'] = datestr_local_to_utc( date_closed_from) if date_closed_to: range_filters['date_closed']['lte'] = datestr_local_to_utc( date_closed_to) if date_rec_from or date_rec_to: date_ranges.append( {'range': { 'date_received': range_filters['date_received'] }}) if date_due_from or date_due_to: date_ranges.append( {'range': { 'date_due': range_filters['date_due'] }}) if date_closed_from or date_closed_to: date_ranges.append( {'range': { 'date_closed': range_filters['date_closed'] }}) # generate query dsl body query_fields = { 'title': title, 'description': description, 'agency_request_summary': agency_request_summary, 'requester_name': requester_name } dsl_gen = RequestsDSLGenerator(query, query_fields, statuses, date_ranges, agency_ein, agency_user_guid, match_type) if foil_id: dsl = dsl_gen.foil_id() else: if query: if current_user.is_agency: dsl = dsl_gen.agency_user() elif current_user.is_anonymous: dsl = dsl_gen.anonymous_user() elif current_user.is_public: dsl = dsl_gen.public_user() else: raise InvalidUserException(current_user) else: dsl = dsl_gen.queryless() # add highlights to dsl if highlight: highlight_fields = {} for name, add in query_fields.items(): if add: highlight_fields[name] = {} dsl.update({ 'highlight': { 'pre_tags': ['<span class="highlight">'], 'post_tags': ['</span>'], 'fields': highlight_fields } }) # Calculate result set size result_set_size = size if for_csv else min(size, MAX_RESULT_SIZE) # search / run query if not for_csv: results = es.search( index=current_app.config["ELASTICSEARCH_INDEX"], doc_type='request', body=dsl, _source=[ 'requester_id', 'date_submitted', 'date_due', 'date_received', 'date_created', 'date_closed', 'status', 'agency_ein', 'agency_name', 'agency_acronym', 'requester_name', 'title_private', 'agency_request_summary_private', 'public_title', 'title', 'agency_request_summary', 'description', 'assigned_users' ], size=result_set_size, from_=start, sort=sort, ) else: results = es.search( index=current_app.config["ELASTICSEARCH_INDEX"], doc_type='request', scroll='1m', body=dsl, _source=[ 'requester_id', 'date_submitted', 'date_due', 'date_received', 'date_created', 'date_closed', 'status', 'agency_ein', 'agency_name', 'agency_acronym', 'requester_name', 'title_private', 'agency_request_summary_private', 'public_title', 'title', 'agency_request_summary', 'description', 'assigned_users' ], size=result_set_size, from_=start, sort=sort, ) sid = results['_scroll_id'] scroll_size = results['hits']['total'] scroll_results = results['hits']['hits'] while scroll_size > 0: results = es.scroll(scroll='1m', body={ "scroll": "1m", "scroll_id": sid }) scroll_size = len(results['hits']['hits']) scroll_results += results['hits']['hits'] return scroll_results # process highlights if highlight and not foil_id: _process_highlights(results, dsl_gen.requester_id) return results
def new(): """ Create a new FOIL request sends a confirmation email after the Requests object is created. title: request title description: request description agency: agency selected for the request submission: submission method for the request :return: redirect to homepage on successful form validation if form fields are missing or has improper values, backend error messages (WTForms) will appear """ site_key = current_app.config["RECAPTCHA_SITE_KEY"] kiosk_mode = eval_request_bool( escape(flask_request.args.get("kiosk_mode", False))) category = str(escape(flask_request.args.get("category", None))) agency = str(escape(flask_request.args.get("agency", None))) title = str(escape(flask_request.args.get("title", None))) if current_user.is_public: form = PublicUserRequestForm() template_suffix = "user.html" elif current_user.is_anonymous: form = AnonymousRequestForm() template_suffix = "anon.html" elif current_user.is_agency: form = AgencyUserRequestForm() template_suffix = "agency.html" else: raise InvalidUserException(current_user) new_request_template = "request/new_request_" + template_suffix if flask_request.method == "POST": # validate upload with no request id available upload_path = None if form.request_file.data: form.request_file.validate(form) upload_path = handle_upload_no_id(form.request_file) if form.request_file.errors: return render_template(new_request_template, form=form, site_key=site_key) custom_metadata = json.loads( flask_request.form.get("custom-request-forms-data", {})) tz_name = (flask_request.form["tz-name"] if flask_request.form["tz-name"] else current_app.config["APP_TIMEZONE"]) if current_user.is_public: request_id = create_request( form.request_title.data, form.request_description.data, form.request_category.data, agency_ein=form.request_agency.data, upload_path=upload_path, tz_name=tz_name, custom_metadata=custom_metadata, ) elif current_user.is_agency: request_id = create_request( form.request_title.data, form.request_description.data, category=None, agency_ein=(form.request_agency.data if form.request_agency.data != "None" else current_user.default_agency_ein), submission=form.method_received.data, agency_date_submitted_local=form.request_date.data, email=form.email.data, first_name=form.first_name.data, last_name=form.last_name.data, user_title=form.user_title.data, organization=form.user_organization.data, phone=form.phone.data, fax=form.fax.data, address=get_address(form), upload_path=upload_path, tz_name=tz_name, custom_metadata=custom_metadata, ) else: # Anonymous User request_id = create_request( form.request_title.data, form.request_description.data, form.request_category.data, agency_ein=form.request_agency.data, email=form.email.data, first_name=form.first_name.data, last_name=form.last_name.data, user_title=form.user_title.data, organization=form.user_organization.data, phone=form.phone.data, fax=form.fax.data, address=get_address(form), upload_path=upload_path, tz_name=tz_name, custom_metadata=custom_metadata, ) current_request = Requests.query.filter_by(id=request_id).first() requester = current_request.requester send_confirmation_email(request=current_request, agency=current_request.agency, user=requester) if current_request.agency.is_active: if requester.email: flashed_message_html = render_template( "request/confirmation_email.html") flash(Markup(flashed_message_html), category="success") else: flashed_message_html = render_template( "request/confirmation_non_email.html") flash(Markup(flashed_message_html), category="warning") return redirect(url_for("request.view", request_id=request_id)) else: flashed_message_html = render_template( "request/non_portal_agency_message.html", agency=current_request.agency) flash(Markup(flashed_message_html), category="warning") return redirect( url_for("request.non_portal_agency", agency_name=current_request.agency.name)) return render_template( new_request_template, form=form, site_key=site_key, kiosk_mode=kiosk_mode, category=category, agency=agency, title=title, )
def new(): """ Create a new FOIL request sends a confirmation email after the Requests object is created. title: request title description: request description agency: agency selected for the request submission: submission method for the request :return: redirect to homepage on successful form validation if form fields are missing or has improper values, backend error messages (WTForms) will appear """ site_key = current_app.config['RECAPTCHA_SITE_KEY'] if current_user.is_public: form = PublicUserRequestForm() template_suffix = 'user.html' elif current_user.is_anonymous: form = AnonymousRequestForm() template_suffix = 'anon.html' elif current_user.is_agency: form = AgencyUserRequestForm() template_suffix = 'agency.html' else: raise InvalidUserException(current_user) new_request_template = 'request/new_request_' + template_suffix if flask_request.method == 'POST': # validate upload with no request id available upload_path = None if form.request_file.data: form.request_file.validate(form) upload_path = handle_upload_no_id(form.request_file) if form.request_file.errors: return render_template(new_request_template, form=form, site_key=site_key) # create request if current_user.is_public: request_id = create_request(form.request_title.data, form.request_description.data, form.request_category.data, agency_ein=form.request_agency.data, upload_path=upload_path, tz_name=flask_request.form['tz-name']) elif current_user.is_agency: request_id = create_request(form.request_title.data, form.request_description.data, category=None, agency_ein=( form.request_agency.data if form.request_agency.data != 'None' else current_user.default_agency_ein), submission=form.method_received.data, agency_date_submitted=form.request_date.data, email=form.email.data, first_name=form.first_name.data, last_name=form.last_name.data, user_title=form.user_title.data, organization=form.user_organization.data, phone=form.phone.data, fax=form.fax.data, address=get_address(form), upload_path=upload_path, tz_name=flask_request.form['tz-name']) else: # Anonymous User request_id = create_request(form.request_title.data, form.request_description.data, form.request_category.data, agency_ein=form.request_agency.data, email=form.email.data, first_name=form.first_name.data, last_name=form.last_name.data, user_title=form.user_title.data, organization=form.user_organization.data, phone=form.phone.data, fax=form.fax.data, address=get_address(form), upload_path=upload_path, tz_name=flask_request.form['tz-name']) current_request = Requests.query.filter_by(id=request_id).first() requester = current_request.requester send_confirmation_email(request=current_request, agency=current_request.agency, user=requester) if current_request.agency.is_active: if requester.email: flashed_message_html = render_template('request/confirmation_email.html') flash(Markup(flashed_message_html), category='success') else: flashed_message_html = render_template('request/confirmation_non_email.html') flash(Markup(flashed_message_html), category='warning') return redirect(url_for('request.view', request_id=request_id)) else: flashed_message_html = render_template('request/non_portal_agency_message.html', agency=current_request.agency) flash(Markup(flashed_message_html), category='warning') return redirect(url_for('request.non_portal_agency', agency_name=current_request.agency.name)) return render_template(new_request_template, form=form, site_key=site_key)