def test_extend_date_due_soon(self):
request = self.rf.create_request_as_anonymous_user(due_date=datetime.utcnow())
date = calendar.addbusdays(utc_to_local(request.due_date, self.tz_name), 1)
due_date = process_due_date(local_to_utc(date, self.tz_name))
response = request.extend(date=date)
self.__test_extension(
response, determination_type.EXTENSION, str, due_date, request_status.DUE_SOON, request=request)
def __get_new_due_date(self, days=None, date=None):
assert days is not None or date is not None
if days is None:
new_due_date = process_due_date(local_to_utc(date, self.__tz_name))
else:
new_due_date = get_due_date(
utc_to_local(self.request.due_date, self.__tz_name), days,
self.__tz_name)
return new_due_date
def test_extend_days(self):
days = 20
reason = "Reasonable reasoning for the rational reasoner."
due_date = get_due_date(
utc_to_local(
self.request.due_date,
self.tz_name
),
days,
self.tz_name
)
response = self.request.extend(reason=reason, days=days)
self.__test_extension(response, determination_type.EXTENSION, reason, due_date)
def test_acknowledge_days(self):
days = 30
info = "Informative information that will inform you."
due_date = get_due_date(
utc_to_local(
self.request.due_date,
self.tz_name
),
days,
self.tz_name
)
response = self.request.acknowledge(info=info, days=days)
self.__test_extension(response, determination_type.ACKNOWLEDGMENT, info, due_date)
def convert_dates(results, dt_format=None, tz_name=None):
"""
Replace datetime values of requests search results with a
datetime object or a datetime string in the specified format.
Dates can also be offset according to the given time zone name.
:results: elasticsearch json results
:dt_format: datetime string format
:tz_name: time zone name
"""
for hit in results["hits"]["hits"]:
for field in ("date_submitted", "date_due", "date_created"):
dt = datetime.strptime(hit["_source"][field], ES_DATETIME_FORMAT)
if tz_name:
dt = utc_to_local(dt, tz_name)
hit["_source"][field] = dt.strftime(
dt_format) if dt_format is not None else dt
def convert_dates(results, dt_format=None, tz_name=None):
"""
Replace datetime values of requests search results with a
datetime object or a datetime string in the specified format.
Dates can also be offset according to the given time zone name.
:results: elasticsearch json results
:dt_format: datetime string format
:tz_name: time zone name
"""
for hit in results["hits"]["hits"]:
for field in ("date_submitted", "date_due", "date_received", "date_closed"):
dt_field = hit["_source"].get(field, None)
if dt_field is not None and dt_field:
dt = datetime.strptime(hit["_source"][field], ES_DATETIME_FORMAT)
else:
continue
if tz_name:
dt = utc_to_local(dt, tz_name)
hit["_source"][field] = dt.strftime(dt_format) if dt_format is not None else dt
def create_request(title,
description,
category,
tz_name,
agency_ein=None,
first_name=None,
last_name=None,
submission=DIRECT_INPUT,
agency_date_submitted_local=None,
email=None,
user_title=None,
organization=None,
phone=None,
fax=None,
address=None,
upload_path=None,
custom_metadata=None):
"""
Creates a new FOIL Request and associated Users, UserRequests, and Events.
:param title: request title
:param description: detailed description of the request
:param tz_name: client's timezone name
:param agency_ein: agency_ein selected for the request
:param first_name: first name of the requester
:param last_name: last name of the requester
:param submission: request submission method
:param agency_date_submitted_local: submission date chosen by agency
:param email: requester's email address
:param user_title: requester's organizational title
:param organization: requester's organization
:param phone: requester's phone number
:param fax: requester's fax number
:param address: requester's mailing address
:param upload_path: file path of the validated upload
:param custom_metadata: JSON containing all data from custom request forms
"""
# 1. Generate the request id
request_id = generate_request_id(agency_ein)
# 2a. Generate Email Notification Text for Agency
# agency_email = generate_email_template('agency_acknowledgment.html', request_id=request_id)
# 2b. Generate Email Notification Text for Requester
# 3a. Send Email Notification Text for Agency
# 3b. Send Email Notification Text for Requester
# 4a. Calculate Request Submitted Date (Round to next business day)
date_created_local = utc_to_local(datetime.utcnow(), tz_name)
if current_user.is_agency:
date_submitted_local = agency_date_submitted_local
else:
date_submitted_local = date_created_local
# 4b. Calculate Request Due Date (month day year but time is always 5PM, 5 Days after submitted date)
due_date = get_due_date(date_submitted_local, ACKNOWLEDGMENT_PERIOD_LENGTH,
tz_name)
date_created = local_to_utc(date_created_local, tz_name)
date_submitted = local_to_utc(date_submitted_local, tz_name)
# 5. Create Request
request = Requests(id=request_id,
title=title,
agency_ein=agency_ein,
category=category,
description=description,
date_created=date_created,
date_submitted=date_submitted,
due_date=due_date,
submission=submission,
custom_metadata=custom_metadata)
create_object(request)
guid_for_event = current_user.guid if not current_user.is_anonymous else None
# 6. Get or Create User
if current_user.is_public:
user = current_user
else:
user = Users(guid=generate_guid(),
email=email,
first_name=first_name,
last_name=last_name,
title=user_title or None,
organization=organization or None,
email_validated=False,
terms_of_use_accepted=False,
phone_number=phone,
fax_number=fax,
mailing_address=address,
is_anonymous_requester=True)
create_object(user)
# user created event
create_object(
Events(request_id,
guid_for_event,
event_type.USER_CREATED,
previous_value=None,
new_value=user.val_for_events,
response_id=None,
timestamp=datetime.utcnow()))
if upload_path is not None:
# 7. Move file to upload directory
upload_path = _move_validated_upload(request_id, upload_path)
# 8. Create response object
filename = os.path.basename(upload_path)
response = Files(request_id,
RELEASE_AND_PRIVATE,
filename,
filename,
fu.get_mime_type(upload_path),
fu.getsize(upload_path),
fu.get_hash(upload_path),
is_editable=False)
create_object(obj=response)
# 8. Create upload Event
upload_event = Events(user_guid=user.guid,
response_id=response.id,
request_id=request_id,
type_=event_type.FILE_ADDED,
timestamp=datetime.utcnow(),
new_value=response.val_for_events)
create_object(upload_event)
# Create response token if requester is anonymous
if current_user.is_anonymous or current_user.is_agency:
create_object(ResponseTokens(response.id))
role_to_user = {
role.PUBLIC_REQUESTER: user.is_public,
role.ANONYMOUS: user.is_anonymous_requester,
}
role_name = [k for (k, v) in role_to_user.items() if v][0]
# (key for "truthy" value)
# 9. Create Event
timestamp = datetime.utcnow()
event = Events(user_guid=user.guid
if current_user.is_anonymous else current_user.guid,
request_id=request_id,
type_=event_type.REQ_CREATED,
timestamp=timestamp,
new_value=request.val_for_events)
create_object(event)
if current_user.is_agency:
agency_event = Events(user_guid=current_user.guid,
request_id=request.id,
type_=event_type.AGENCY_REQ_CREATED,
timestamp=timestamp)
create_object(agency_event)
# 10. Create UserRequest for requester
user_request = UserRequests(
user_guid=user.guid,
request_user_type=user_type_request.REQUESTER,
request_id=request_id,
permissions=Roles.query.filter_by(name=role_name).first().permissions)
create_object(user_request)
create_object(
Events(request_id,
guid_for_event,
event_type.USER_ADDED,
previous_value=None,
new_value=user_request.val_for_events,
response_id=None,
timestamp=datetime.utcnow()))
# 11. Create the elasticsearch request doc only if agency has been onboarded
agency = Agencies.query.filter_by(ein=agency_ein).one()
# 12. Add all agency administrators to the request.
if agency.administrators:
# b. Store all agency users objects in the UserRequests table as Agency users with Agency Administrator
# privileges
_create_agency_user_requests(request_id=request_id,
agency_admins=agency.administrators,
guid_for_event=guid_for_event)
# 13. Add all parent agency administrators to the request.
if agency != agency.parent:
if (agency.parent.agency_features is not None
and agency_ein in agency.parent.agency_features.get(
'monitor_agency_requests', []) and agency.parent.is_active
and agency.parent.administrators):
_create_agency_user_requests(
request_id=request_id,
agency_admins=agency.parent.administrators,
guid_for_event=guid_for_event)
# (Now that we can associate the request with its requester AND agency users.)
if current_app.config['ELASTICSEARCH_ENABLED'] and agency.is_active:
request.es_create()
return request_id
def requests_doc(doc_type):
"""
Converts and sends the a search result-set as a
file of the specified document type.
- Filtering on set size is ignored; all results are returned.
- Currently only supports CSVs.
Document name format: "FOIL_requests_results_<timestamp:MM_DD_YYYY_at_HH_mm_pp>"
Request parameters are identical to those of /search/requests.
:param doc_type: document type ('csv' only)
"""
if current_user.is_agency and doc_type.lower() == 'csv':
try:
agency_ein = request.args.get('agency_ein', '')
except ValueError:
agency_ein = None
tz_name = request.args.get('tz_name')
start = 0
buffer = StringIO() # csvwriter cannot accept BytesIO
writer = csv.writer(buffer)
writer.writerow([
"FOIL ID", "Agency", "Title", "Description", "Agency Description",
"Current Status", "Date Created", "Date Received", "Date Due",
"Date Closed", "Requester Name", "Requester Email",
"Requester Title", "Requester Organization",
"Requester Phone Number", "Requester Fax Number",
"Requester Address 1", "Requester Address 2", "Requester City",
"Requester State", "Requester Zipcode", "Assigned User Emails"
])
while True:
results = search_requests(
request.args.get('query'),
eval_request_bool(request.args.get('foil_id')),
eval_request_bool(request.args.get('title')),
eval_request_bool(request.args.get('agency_request_summary')),
eval_request_bool(request.args.get('description')),
eval_request_bool(request.args.get('requester_name')),
request.args.get('date_rec_from'),
request.args.get('date_rec_to'),
request.args.get('date_due_from'),
request.args.get('date_due_to'),
request.args.get('date_closed_from'),
request.args.get('date_closed_to'),
agency_ein,
eval_request_bool(request.args.get('open')),
eval_request_bool(request.args.get('closed')),
eval_request_bool(request.args.get('in_progress')),
eval_request_bool(request.args.get('due_soon')),
eval_request_bool(request.args.get('overdue')),
ALL_RESULTS_CHUNKSIZE,
start,
request.args.get('sort_date_submitted'),
request.args.get('sort_date_due'),
request.args.get('sort_title'),
tz_name,
for_csv=True)
total = results["hits"]["total"]
if total != 0:
convert_dates(results, tz_name=tz_name)
for result in results["hits"]["hits"]:
r = Requests.query.filter_by(id=result["_id"]).one()
mailing_address = (r.requester.mailing_address
if r.requester.mailing_address
is not None else {})
date_closed = result['_source'].get('date_closed', '')
date_closed = date_closed if str(date_closed) != str(
list()) else ''
writer.writerow([
result["_id"], result["_source"]["agency_name"],
result["_source"]["title"],
result["_source"]["description"],
result["_source"]["agency_request_summary"], r.status,
result["_source"]["date_created"],
result["_source"]["date_submitted"],
result["_source"]["date_due"], date_closed,
result["_source"]["requester_name"], r.requester.email,
r.requester.title, r.requester.organization,
r.requester.phone_number, r.requester.fax_number,
mailing_address.get('address_one'),
mailing_address.get('address_two'),
mailing_address.get('city'),
mailing_address.get('state'),
mailing_address.get('zip'),
", ".join(u.email for u in r.agency_users)
])
start += ALL_RESULTS_CHUNKSIZE
if start > total:
break
if total != 0:
dt = datetime.utcnow()
timestamp = utc_to_local(dt,
tz_name) if tz_name is not None else dt
return send_file(
BytesIO(buffer.getvalue().encode('UTF-8')), # convert to bytes
attachment_filename="FOIL_requests_results_{}.csv".format(
timestamp.strftime("%m_%d_%Y_at_%I_%M_%p")),
as_attachment=True)
return '', 400
def generate_acknowledgment_report(self, current_user_guid: str,
date_from: datetime, date_to: datetime):
"""Celery task that generates the acknowledgment report for the user's agency with the specified date range.
Args:
current_user_guid: GUID of the current user
date_from: Date to filter report from
date_to: Date to filter report to
"""
current_user = Users.query.filter_by(guid=current_user_guid).one()
agency_ein = current_user.default_agency_ein
agency = Agencies.query.options(joinedload(Agencies.active_users)).options(
joinedload(
Agencies.inactive_users)).filter(Agencies.ein == agency_ein).one()
agency_users = agency.active_users + agency.inactive_users
request_list = Requests.query.join(
Events, Events.request_id == Requests.id).options(
joinedload(Requests.requester)).add_columns(
Events.user_guid, Events.type).filter(
Requests.agency_ein == agency_ein,
Requests.status != CLOSED,
Events.request_id == Requests.id,
Events.type.in_(
(REQ_ACKNOWLEDGED,
REQ_CREATED))).order_by(asc(Requests.id)).all()
headers = ('Request ID', 'Acknowledged', 'Acknowledged By', 'Date Created',
'Due Date', 'Status', 'Title', 'Description', 'Requester Name',
'Email', 'Phone Number', 'Address 1', 'Address 2', 'City',
'State', 'Zipcode')
data_from_dates = []
all_data = []
acknowledged_requests = {
res.Requests.id: {
"request": res.Requests,
"user_guid": res.user_guid
}
for res in filter(lambda x: x.type == REQ_ACKNOWLEDGED, request_list)
}
unacknowledged_requests = {
res.Requests.id: {
"request": res.Requests
}
for res in filter(
lambda x: x.type == REQ_CREATED and x.Requests.id not in
acknowledged_requests.keys(), request_list)
}
request_id_list = list(acknowledged_requests.keys()) + list(
unacknowledged_requests.keys())
for request_id in sorted(request_id_list):
ack_user = ''
was_acknowledged = False
if acknowledged_requests.get(request_id, None):
ack_user = [
user for user in agency_users
if user.guid == acknowledged_requests[request_id]["user_guid"]
]
ack_user = ack_user[0].name if ack_user else ''
was_acknowledged = True
request = acknowledged_requests.get(request_id)['request']
else:
request = unacknowledged_requests.get(request_id)['request']
req_date_created_local = utc_to_local(
request.date_created, current_app.config['APP_TIMEZONE'])
if date_from < req_date_created_local < date_to:
data_from_dates.append((
request.id,
was_acknowledged,
ack_user,
req_date_created_local.strftime('%m/%d/%Y'),
utc_to_local(
request.due_date,
current_app.config['APP_TIMEZONE']).strftime('%m/%d/%Y'),
request.status,
request.title,
request.description,
request.requester.name,
request.requester.email,
request.requester.phone_number,
request.requester.mailing_address.get('address_one'),
request.requester.mailing_address.get('address_two'),
request.requester.mailing_address.get('city'),
request.requester.mailing_address.get('state'),
request.requester.mailing_address.get('zip'),
))
all_data.append((
request.id,
was_acknowledged,
ack_user,
req_date_created_local.strftime('%m/%d/%Y'),
utc_to_local(
request.due_date,
current_app.config['APP_TIMEZONE']).strftime('%m/%d/%Y'),
request.status,
request.title,
request.description,
request.requester.name,
request.requester.email,
request.requester.phone_number,
request.requester.mailing_address.get('address_one'),
request.requester.mailing_address.get('address_two'),
request.requester.mailing_address.get('city'),
request.requester.mailing_address.get('state'),
request.requester.mailing_address.get('zip'),
))
date_from_string = date_from.strftime('%Y%m%d')
date_to_string = date_to.strftime('%Y%m%d')
dates_dataset = tablib.Dataset(*data_from_dates,
headers=headers,
title='{}_{}'.format(
date_from_string, date_to_string))
all_dataset = tablib.Dataset(*all_data, headers=headers, title='all')
excel_spreadsheet = tablib.Databook((dates_dataset, all_dataset))
send_email(subject='OpenRecords Acknowledgment Report',
to=[current_user.email],
template='email_templates/email_agency_report_generated',
agency_user=current_user.name,
attachment=excel_spreadsheet.export('xls'),
filename='FOIL_acknowledgments_{}_{}.xls'.format(
date_from_string, date_to_string),
mimetype='application/octect-stream')
def requests_doc(doc_type):
"""
Converts and sends the a search result-set as a
file of the specified document type.
- Filtering on set size is ignored; all results are returned.
- Currently only supports CSVs.
Document name format: "FOIL_requests_results_<timestamp:MM_DD_YYYY_at_HH_mm_pp>"
Request parameters are identical to those of /search/requests.
:param doc_type: document type ('csv' only)
"""
if current_user.is_agency and doc_type.lower() == 'csv':
try:
agency_ein = request.args.get('agency_ein', '')
except ValueError:
sentry.captureException()
agency_ein = None
tz_name = request.args.get('tz_name', current_app.config['APP_TIMEZONE'])
start = 0
buffer = StringIO() # csvwriter cannot accept BytesIO
writer = csv.writer(buffer)
writer.writerow(["FOIL ID",
"Agency",
"Title",
"Description",
"Agency Description",
"Current Status",
"Date Created",
"Date Received",
"Date Due",
"Date Closed",
"Requester Name",
"Requester Email",
"Requester Title",
"Requester Organization",
"Requester Phone Number",
"Requester Fax Number",
"Requester Address 1",
"Requester Address 2",
"Requester City",
"Requester State",
"Requester Zipcode",
"Assigned User Emails"])
results = search_requests(
query=request.args.get('query'),
foil_id=eval_request_bool(request.args.get('foil_id')),
title=eval_request_bool(request.args.get('title')),
agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')),
description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False,
requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False,
date_rec_from=request.args.get('date_rec_from'),
date_rec_to=request.args.get('date_rec_to'),
date_due_from=request.args.get('date_due_from'),
date_due_to=request.args.get('date_due_to'),
date_closed_from=request.args.get('date_closed_from'),
date_closed_to=request.args.get('date_closed_to'),
agency_ein=agency_ein,
agency_user_guid=request.args.get('agency_user'),
open_=eval_request_bool(request.args.get('open')),
closed=eval_request_bool(request.args.get('closed')),
in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False,
due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False,
overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False,
start=start,
sort_date_received=request.args.get('sort_date_submitted'),
sort_date_due=request.args.get('sort_date_due'),
sort_title=request.args.get('sort_title'),
tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE']),
for_csv=True
)
ids = [result["_id"] for result in results]
all_requests = Requests.query.filter(Requests.id.in_(ids)).options(
joinedload(Requests.agency_users)).options(joinedload(Requests.requester)).options(
joinedload(Requests.agency)).all()
for req in all_requests:
writer.writerow([
req.id,
req.agency.name,
req.title,
req.description,
req.agency_request_summary,
req.status,
req.date_created,
req.date_submitted,
req.due_date,
req.date_closed,
req.requester.name,
req.requester.email,
req.requester.title,
req.requester.organization,
req.requester.phone_number,
req.requester.fax_number,
req.requester.mailing_address.get('address_one'),
req.requester.mailing_address.get('address_two'),
req.requester.mailing_address.get('city'),
req.requester.mailing_address.get('state'),
req.requester.mailing_address.get('zip'),
", ".join(u.email for u in req.agency_users)])
dt = datetime.utcnow()
timestamp = utc_to_local(dt, tz_name) if tz_name is not None else dt
return send_file(
BytesIO(buffer.getvalue().encode('UTF-8')), # convert to bytes
attachment_filename="FOIL_requests_results_{}.csv".format(
timestamp.strftime("%m_%d_%Y_at_%I_%M_%p")),
as_attachment=True
)
return '', 400
def test_reopen(self):
date = calendar.addbusdays(utc_to_local(self.request.due_date, self.tz_name), 1)
response = self.request.reopen(date)
due_date = process_due_date(local_to_utc(date, self.tz_name))
self.__test_extension(response, determination_type.REOPENING, None, due_date)
self.assertEqual(self.request.agency_request_summary, None)
def test_extend_bad_due_date(self):
with self.assertRaises(AssertionError):
self.request.extend(date=calendar.addbusdays(
utc_to_local(self.request.due_date, self.tz_name), -1))
def __assert_request_data_correct(self,
user,
request,
agency_parent_ein,
default=False,
title=None,
description=None,
agency_request_summary=None,
agency_ein=None,
date_created=None,
due_date=None,
category='All',
title_privacy=True,
agency_request_summary_privacy=True,
submission=None,
status=request_status.OPEN):
request = Requests.query.get(request.id)
privacy = {"title": title_privacy, "agency_request_summary": agency_request_summary_privacy}
agency_ein = agency_ein or user.agency_ein or request.agency_ein
date_created_local = utc_to_local(date_created or request.date_created, self.tz_name)
date_submitted_local = get_following_date(date_created_local)
if default:
self.assertTrue(request.submission in submission_methods.ALL)
request_list = [
type(request.title),
type(request.description),
]
check_list = [
str,
str,
]
else:
request_list = [
request.title,
request.description,
request.agency_request_summary,
request.date_created,
request.submission,
]
check_list = [
title,
description,
agency_request_summary,
date_created,
submission,
]
request_list += [
request.id,
request.category,
request.privacy,
request.agency_ein,
request.status,
request.date_submitted,
request.due_date,
]
check_list += [
"FOIL-{}-{}-00001".format(datetime.today().year, agency_parent_ein),
category,
privacy,
agency_ein,
status,
local_to_utc(date_submitted_local, self.tz_name),
due_date or get_due_date(date_submitted_local, ACKNOWLEDGMENT_DAYS_DUE, self.tz_name)
]
self.assertEqual(request_list, check_list)
# check associated events
event_req_created = Events.query.filter_by(type=event_type.REQ_CREATED).one()
self.assertEqual(
[
event_req_created.user_guid,
event_req_created.auth_user_type,
event_req_created.request_id,
event_req_created.response_id,
event_req_created.previous_value,
event_req_created.new_value,
],
[
user.guid,
user.auth_user_type,
request.id,
None, # response_id
None, # previous_value
request.val_for_events # new_value
]
)
if user.is_agency:
event_agency_req_created = Events.query.filter_by(type=event_type.AGENCY_REQ_CREATED).one()
self.assertEqual(
[
event_agency_req_created.user_guid,
event_agency_req_created.auth_user_type,
event_agency_req_created.request_id,
event_agency_req_created.response_id,
event_agency_req_created.previous_value,
event_agency_req_created.new_value,
],
[
user.guid,
user.auth_user_type,
request.id,
None, # response_id
None, # previous_value
None, # new_value
]
)
def requests_doc(doc_type):
"""
Converts and sends the a search result-set as a
file of the specified document type.
- Filtering on set size is ignored; all results are returned.
- Currently only supports CSVs.
- CSV only includes requests belonging to that user's agency
Document name format: "FOIL_requests_results_<timestamp:MM_DD_YYYY_at_HH_mm_pp>"
Request parameters are identical to those of /search/requests.
:param doc_type: document type ('csv' only)
"""
if current_user.is_agency and doc_type.lower() == 'csv':
try:
agency_ein = request.args.get('agency_ein', '')
except ValueError:
sentry.captureException()
agency_ein = None
tz_name = request.args.get('tz_name', current_app.config['APP_TIMEZONE'])
start = 0
buffer = StringIO() # csvwriter cannot accept BytesIO
writer = csv.writer(buffer)
writer.writerow(["FOIL ID",
"Agency",
"Title",
"Description",
"Agency Request Summary",
"Current Status",
"Date Created",
"Date Received",
"Date Due",
"Date Closed",
"Requester Name",
"Requester Email",
"Requester Title",
"Requester Organization",
"Requester Phone Number",
"Requester Fax Number",
"Requester Address 1",
"Requester Address 2",
"Requester City",
"Requester State",
"Requester Zipcode",
"Assigned User Emails"])
results = search_requests(
query=request.args.get('query'),
foil_id=eval_request_bool(request.args.get('foil_id')),
title=eval_request_bool(request.args.get('title')),
agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')),
description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False,
requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False,
date_rec_from=request.args.get('date_rec_from'),
date_rec_to=request.args.get('date_rec_to'),
date_due_from=request.args.get('date_due_from'),
date_due_to=request.args.get('date_due_to'),
date_closed_from=request.args.get('date_closed_from'),
date_closed_to=request.args.get('date_closed_to'),
agency_ein=agency_ein,
agency_user_guid=request.args.get('agency_user'),
open_=eval_request_bool(request.args.get('open')),
closed=eval_request_bool(request.args.get('closed')),
in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False,
due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False,
overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False,
start=start,
sort_date_received=request.args.get('sort_date_submitted'),
sort_date_due=request.args.get('sort_date_due'),
sort_title=request.args.get('sort_title'),
tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE']),
for_csv=True
)
ids = [result["_id"] for result in results]
all_requests = Requests.query.filter(Requests.id.in_(ids)).options(
joinedload(Requests.agency_users)).options(joinedload(Requests.requester)).options(
joinedload(Requests.agency)).all()
user_agencies = current_user.get_agencies
for req in all_requests:
if req.agency_ein in user_agencies:
writer.writerow([
req.id,
req.agency.name,
req.title,
req.description,
req.agency_request_summary,
req.status,
req.date_created,
req.date_submitted,
req.due_date,
req.date_closed,
req.requester.name,
req.requester.email,
req.requester.title,
req.requester.organization,
req.requester.phone_number,
req.requester.fax_number,
req.requester.mailing_address.get('address_one'),
req.requester.mailing_address.get('address_two'),
req.requester.mailing_address.get('city'),
req.requester.mailing_address.get('state'),
req.requester.mailing_address.get('zip'),
", ".join(u.email for u in req.agency_users)])
dt = datetime.utcnow()
timestamp = utc_to_local(dt, tz_name) if tz_name is not None else dt
return send_file(
BytesIO(buffer.getvalue().encode('UTF-8')), # convert to bytes
attachment_filename="FOIL_requests_results_{}.csv".format(
timestamp.strftime("%m_%d_%Y_at_%I_%M_%p")),
as_attachment=True
)
return '', 400
def create_request(
self,
user,
title=None,
description=None,
agency_request_summary=None, # TODO: agency_request_summary_release_date
agency_ein=None,
date_created=None,
date_submitted=None,
due_date=None,
category=None,
title_privacy=True,
agency_request_summary_privacy=True,
submission=None,
status=request_status.OPEN,
tz_name=current_app.config["APP_TIMEZONE"]):
"""
Create a request as the supplied user. An anonymous requester
will be created if the supplied user is an agency user.
:rtype: RequestWrapper
"""
# check due date
if (date_created is not None
or date_submitted is not None) and due_date is not None:
def assert_date(date, date_var_str):
assert (
due_date - date
).days >= 1, "due_date must be at least 1 day after " + date_var_str
if date_created is not None:
assert_date(date_created, "date_created")
if date_submitted is not None:
assert_date(date_submitted, "date_submitted")
# check agency_ein
if (agency_ein is not None or self.agency_ein is not None) \
and user.auth_user_type == user_type_auth.AGENCY_USER:
assert (agency_ein or self.agency_ein) == user.agency_ein, \
"user's agency ein must match supplied agency ein"
agency_ein = agency_ein or self.agency_ein or user.agency_ein or get_random_agency(
).ein
# create dates
date_created_local = utc_to_local(date_created or datetime.utcnow(),
tz_name)
date_submitted_local = date_submitted or get_following_date(
date_created_local)
due_date = due_date or get_due_date(date_submitted_local,
ACKNOWLEDGMENT_DAYS_DUE, tz_name)
date_created = date_created or local_to_utc(date_created_local,
tz_name)
date_submitted = date_submitted or local_to_utc(
date_submitted_local, tz_name)
# create request
request = Requests(
generate_request_id(agency_ein),
title or fake.title(),
description or fake.description(),
agency_ein=agency_ein,
date_created=date_created or datetime.utcnow(),
date_submitted=date_submitted,
due_date=due_date,
category=category,
privacy={
"title": title_privacy,
"agency_request_summary": agency_request_summary_privacy
},
submission=submission or random.choice(submission_methods.ALL),
status=status,
)
if agency_request_summary is not None:
request.agency_request_summary = agency_request_summary
if agency_request_summary_privacy is not None:
request.agency_request_summary_release_date = calendar.addbusdays(
datetime.utcnow(), RELEASE_PUBLIC_DAYS
) if not agency_request_summary_privacy else None
create_object(request)
request = RequestWrapper(request, self.agency_user)
# create events
timestamp = datetime.utcnow()
create_object(
Events(user_guid=user.guid,
auth_user_type=user.auth_user_type,
request_id=request.id,
type_=event_type.REQ_CREATED,
timestamp=timestamp,
new_value=request.val_for_events))
if user.is_agency:
create_object(
Events(user_guid=user.guid,
auth_user_type=user.auth_user_type,
request_id=request.id,
type_=event_type.AGENCY_REQ_CREATED,
timestamp=timestamp))
# add users
if user.is_public or user.is_anonymous_requester:
request.add_user(user)
if user.is_agency: # then create and add anonymous requester
request.add_user(self.__uf.create_anonymous_user())
for admin in Agencies.query.filter_by(
ein=agency_ein).one().administrators:
request.add_user(admin)
# create request doc now that requester is set
request.es_create()
return request
def create_request(title,
description,
category,
tz_name,
agency_ein=None,
first_name=None,
last_name=None,
submission=DIRECT_INPUT,
agency_date_submitted_local=None,
email=None,
user_title=None,
organization=None,
phone=None,
fax=None,
address=None,
upload_path=None,
custom_metadata=None):
"""
Creates a new FOIL Request and associated Users, UserRequests, and Events.
:param title: request title
:param description: detailed description of the request
:param tz_name: client's timezone name
:param agency_ein: agency_ein selected for the request
:param first_name: first name of the requester
:param last_name: last name of the requester
:param submission: request submission method
:param agency_date_submitted_local: submission date chosen by agency
:param email: requester's email address
:param user_title: requester's organizational title
:param organization: requester's organization
:param phone: requester's phone number
:param fax: requester's fax number
:param address: requester's mailing address
:param upload_path: file path of the validated upload
:param custom_metadata: JSON containing all data from custom request forms
"""
# 1. Generate the request id
request_id = generate_request_id(agency_ein)
# 2a. Generate Email Notification Text for Agency
# agency_email = generate_email_template('agency_acknowledgment.html', request_id=request_id)
# 2b. Generate Email Notification Text for Requester
# 3a. Send Email Notification Text for Agency
# 3b. Send Email Notification Text for Requester
# 4a. Calculate Request Submitted Date (Round to next business day)
date_created_local = utc_to_local(datetime.utcnow(), tz_name)
if current_user.is_agency:
date_submitted_local = agency_date_submitted_local
else:
date_submitted_local = date_created_local
# 4b. Calculate Request Due Date (month day year but time is always 5PM, 5 Days after submitted date)
due_date = get_due_date(
date_submitted_local,
ACKNOWLEDGMENT_PERIOD_LENGTH,
tz_name)
date_created = local_to_utc(date_created_local, tz_name)
date_submitted = local_to_utc(date_submitted_local, tz_name)
# 5. Create Request
request = Requests(
id=request_id,
title=title,
agency_ein=agency_ein,
category=category,
description=description,
date_created=date_created,
date_submitted=date_submitted,
due_date=due_date,
submission=submission,
custom_metadata=custom_metadata
)
create_object(request)
guid_for_event = current_user.guid if not current_user.is_anonymous else None
# 6. Get or Create User
if current_user.is_public:
user = current_user
else:
user = Users(
guid=generate_guid(),
email=email,
first_name=first_name,
last_name=last_name,
title=user_title or None,
organization=organization or None,
email_validated=False,
terms_of_use_accepted=False,
phone_number=phone,
fax_number=fax,
mailing_address=address,
is_anonymous_requester=True
)
create_object(user)
# user created event
create_object(Events(
request_id,
guid_for_event,
event_type.USER_CREATED,
previous_value=None,
new_value=user.val_for_events,
response_id=None,
timestamp=datetime.utcnow()
))
if upload_path is not None:
# 7. Move file to upload directory
upload_path = _move_validated_upload(request_id, upload_path)
# 8. Create response object
filename = os.path.basename(upload_path)
response = Files(request_id,
RELEASE_AND_PRIVATE,
filename,
filename,
fu.get_mime_type(upload_path),
fu.getsize(upload_path),
fu.get_hash(upload_path),
is_editable=False)
create_object(obj=response)
# 8. Create upload Event
upload_event = Events(user_guid=user.guid,
response_id=response.id,
request_id=request_id,
type_=event_type.FILE_ADDED,
timestamp=datetime.utcnow(),
new_value=response.val_for_events)
create_object(upload_event)
# Create response token if requester is anonymous
if current_user.is_anonymous or current_user.is_agency:
create_object(ResponseTokens(response.id))
role_to_user = {
role.PUBLIC_REQUESTER: user.is_public,
role.ANONYMOUS: user.is_anonymous_requester,
}
role_name = [k for (k, v) in role_to_user.items() if v][0]
# (key for "truthy" value)
# 9. Create Event
timestamp = datetime.utcnow()
event = Events(user_guid=user.guid if current_user.is_anonymous else current_user.guid,
request_id=request_id,
type_=event_type.REQ_CREATED,
timestamp=timestamp,
new_value=request.val_for_events)
create_object(event)
if current_user.is_agency:
agency_event = Events(user_guid=current_user.guid,
request_id=request.id,
type_=event_type.AGENCY_REQ_CREATED,
timestamp=timestamp)
create_object(agency_event)
# 10. Create UserRequest for requester
user_request = UserRequests(user_guid=user.guid,
request_user_type=user_type_request.REQUESTER,
request_id=request_id,
permissions=Roles.query.filter_by(
name=role_name).first().permissions)
create_object(user_request)
create_object(Events(
request_id,
guid_for_event,
event_type.USER_ADDED,
previous_value=None,
new_value=user_request.val_for_events,
response_id=None,
timestamp=datetime.utcnow()
))
# 11. Create the elasticsearch request doc only if agency has been onboarded
agency = Agencies.query.filter_by(ein=agency_ein).one()
# 12. Add all agency administrators to the request.
if agency.administrators:
# b. Store all agency users objects in the UserRequests table as Agency users with Agency Administrator
# privileges
_create_agency_user_requests(request_id=request_id,
agency_admins=agency.administrators,
guid_for_event=guid_for_event)
# 13. Add all parent agency administrators to the request.
if agency != agency.parent:
if (
agency.parent.agency_features is not None and
agency_ein in agency.parent.agency_features.get('monitor_agency_requests', []) and
agency.parent.is_active and
agency.parent.administrators
):
_create_agency_user_requests(request_id=request_id,
agency_admins=agency.parent.administrators,
guid_for_event=guid_for_event)
# (Now that we can associate the request with its requester AND agency users.)
if current_app.config['ELASTICSEARCH_ENABLED'] and agency.is_active:
request.es_create()
return request_id
