def response_quick_closing(request_id):
"""Endpoint for quick closing a request that takes in form data from the front end.
Required form data include:
email-date: the number of days the acknowledgement will take. Defaults to 20 for quick closings
summary: string email body from the confirmation page
Args:
request_id: FOIL request ID
Returns:
Redirect to view request page
"""
required_fields = ['email-date',
'summary']
for field in required_fields:
if not flask_request.form.get(field, ''):
flash('Uh Oh, it looks like the acknowledgement/closing {} is missing! '
'This is probably NOT your fault.'.format(field), category='danger')
return redirect(url_for('request.view', request_id=request_id))
try:
add_quick_closing(request_id=request_id,
days=request_date.DEFAULT_QUICK_CLOSING_DAYS,
date=flask_request.form['email-date'],
tz_name=flask_request.form['tz-name'] if flask_request.form['tz-name'] else
current_app.config['APP_TIMEZONE'],
content=flask_request.form['summary'])
except UserRequestException as e:
sentry.captureException()
flash(str(e), category='danger')
return redirect(url_for('request.view', request_id=request_id))
def vpk_package(folder):
try:
check_call([os.path.abspath(current_app.config['VPK_BINARY_PATH']), folder])
except CalledProcessError:
sentry.captureException()
abort(500)
shutil.rmtree(folder)
def sftp_ctx():
"""
Context manager that provides an SFTP client object
(an SFTP session across an open SSH Transport)
"""
transport = paramiko.Transport((current_app.config['SFTP_HOSTNAME'],
int(current_app.config['SFTP_PORT'])))
authentication_kwarg = {}
if current_app.config['SFTP_PASSWORD']:
authentication_kwarg['password'] = current_app.config['SFTP_PASSWORD']
elif current_app.config['SFTP_RSA_KEY_FILE']:
authentication_kwarg['pkey'] = paramiko.RSAKey(filename=current_app.config['SFTP_RSA_KEY_FILE'])
else:
raise SFTPCredentialsException
transport.connect(username=current_app.config['SFTP_USERNAME'], **authentication_kwarg)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
yield sftp
except Exception as e:
sentry.captureException()
raise paramiko.SFTPError("Exception occurred with SFTP: {}".format(e))
finally:
sftp.close()
transport.close()
def _web_services_request(endpoint, params, method='GET'):
"""
Perform a request on an NYC.ID Web Services endpoint.
'userName' and 'signature' are added to the specified params.
:param endpoint: web services endpoint (e.g. "/account/validateEmail.htm")
:param params: request parameters excluding 'userName' and 'signature'
:param method: HTTP method
:return: request response
"""
current_app.logger.info("NYC.ID Web Services Requests: {} {}".format(method, endpoint))
params['userName'] = current_app.config['NYC_ID_USERNAME']
# don't refactor to use dict.update() - signature relies on userName param
params['signature'] = _generate_signature(
current_app.config['NYC_ID_PASSWORD'],
_generate_string_to_sign(method, endpoint, params)
)
req = None
# SSLError with 'bad signature' is sometimes thrown when sending the request which causes an nginx error and 502
# resending the request resolves the issue
for _ in range(5):
try:
req = requests.request(
method,
urljoin(current_app.config['WEB_SERVICES_URL'], endpoint),
verify=current_app.config['VERIFY_WEB_SERVICES'],
params=params # query string parameters always used
)
except SSLError:
sentry.captureException()
continue
break
return req
def status():
"""
Check the status of an upload.
Request Parameters:
- request_id
- filename
- for_update (bool, optional)
:returns: {
"status": upload status
}
"""
try:
status = redis.get(
get_upload_key(
request.args['request_id'],
secure_filename(request.args['filename']),
eval_request_bool(request.args.get('for_update'))
)
)
if status is not None:
response = {"status": status.decode("utf-8")}
else:
response = {"error": "Upload status not found."}
status_code = 200
except KeyError:
sentry.captureException()
response = {}
status_code = 422
return jsonify(response), status_code
def sftp_ctx():
"""
Context manager that provides an SFTP client object
(an SFTP session across an open SSH Transport)
"""
transport = paramiko.Transport((current_app.config['SFTP_HOSTNAME'],
int(current_app.config['SFTP_PORT'])))
authentication_kwarg = {}
if current_app.config['SFTP_PASSWORD']:
authentication_kwarg['password'] = current_app.config['SFTP_PASSWORD']
elif current_app.config['SFTP_RSA_KEY_FILE']:
authentication_kwarg['pkey'] = paramiko.RSAKey(
filename=current_app.config['SFTP_RSA_KEY_FILE'])
else:
raise SFTPCredentialsException
transport.connect(username=current_app.config['SFTP_USERNAME'],
**authentication_kwarg)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
yield sftp
except Exception as e:
sentry.captureException()
raise paramiko.SFTPError("Exception occurred with SFTP: {}".format(e))
finally:
sftp.close()
transport.close()
def response_closing(request_id):
"""
Endpoint for closing a request that takes in form data from the front end.
Required form data include:
-reasons: a list of closing reasons
-email-summary: string email body from the confirmation page
:param request_id: FOIL request ID
:return: redirect to view request page
"""
if flask_request.form.get('method') == EMAIL:
required_fields = ['reasons',
'method',
'summary']
else:
required_fields = ['letter_templates',
'method',
'summary']
for field in required_fields:
if not flask_request.form.get(field, ''):
flash('Uh Oh, it looks like the closing {} is missing! '
'This is probably NOT your fault.'.format(field), category='danger')
return redirect(url_for('request.view', request_id=request_id))
try:
add_closing(request_id,
flask_request.form.getlist('reasons'),
flask_request.form['summary'],
flask_request.form['method'],
flask_request.form.get('letter_templates'))
except UserRequestException as e:
sentry.captureException()
flash(str(e), category='danger')
return redirect(url_for('request.view', request_id=request_id))
def response_closing(request_id):
"""
Endpoint for closing a request that takes in form data from the front end.
Required form data include:
-reasons: a list of closing reasons
-email-summary: string email body from the confirmation page
:param request_id: FOIL request ID
:return: redirect to view request page
"""
if flask_request.form.get('method') == EMAIL:
required_fields = ['reasons',
'method',
'summary']
else:
required_fields = ['letter_templates',
'method',
'summary']
for field in required_fields:
if flask_request.form.get(field) is None:
flash('Uh Oh, it looks like the closing {} is missing! '
'This is probably NOT your fault.'.format(field), category='danger')
return redirect(url_for('request.view', request_id=request_id))
try:
add_closing(request_id,
flask_request.form.getlist('reasons'),
flask_request.form['summary'],
flask_request.form['method'],
flask_request.form.get('letter_templates'))
except UserRequestException as e:
sentry.captureException()
flash(str(e), category='danger')
return redirect(url_for('request.view', request_id=request_id))
def create_object(obj):
"""
Add a database record and its elasticsearch counterpart.
If 'obj' is a Requests object, nothing will be added to
the es index since a UserRequests record is created after
its associated request and the es doc requires a
requester id. 'es_create' is called explicitly for a
Requests object in app.request.utils.
:param obj: object (instance of sqlalchemy model) to create
:return: string representation of created object
or None if creation failed
"""
try:
db.session.add(obj)
db.session.commit()
except SQLAlchemyError:
sentry.captureException()
db.session.rollback()
current_app.logger.exception("Failed to CREATE {}".format(obj))
return None
else:
# create elasticsearch doc
if (
not isinstance(obj, Requests)
and hasattr(obj, 'es_create')
and current_app.config['ELASTICSEARCH_ENABLED']
):
obj.es_create()
return str(obj)
def index():
user = {'nickname': 'tatumn'}
posts = [{
'author': {
'nickname': 'John'
},
'body': 'Beautiful day in Portland!'
}, {
'author': {
'nickname': 'Susan'
},
'body': 'The Avengers movie was so cool!'
}]
try:
1 / 0
except Exception:
sentry.captureException()
sentry.captureMessage('hello, world!')
form = NameForm()
if form.validate_on_submit():
old_name = session.get('name')
if old_name is not None and old_name != form.name.data:
flash('Looks like you have changed your name!')
session['name'] = form.name.data
return redirect(url_for('home.index'))
return render_template('html/index.html',
title='home',
user=user,
current_time=datetime.utcnow(),
form=form,
name=session.get('name'),
posts=posts)
def _sftp_exists(sftp, path):
try:
sftp.stat(path)
return True
except IOError:
sentry.captureException()
return False
def send_async_email(msg):
try:
mail.send(msg)
except Exception as e:
sentry.captureException()
current_app.logger.exception("Failed to Send Email {} : {}".format(
msg, e))
def _sftp_exists(sftp, path):
try:
sftp.stat(path)
return True
except IOError:
sentry.captureException()
return False
def internal_error(error):
logger.error('ErrorHandler Exception %s', error)
message = self.find_message(error)
if sentry:
sentry.captureException()
return responses.status_403(message), 403
def vpk_package(folder):
try:
check_call(
[os.path.abspath(current_app.config['VPK_BINARY_PATH']), folder])
except CalledProcessError:
sentry.captureException()
abort(500)
shutil.rmtree(folder)
def update_request_statuses():
with scheduler.app.app_context():
try:
_update_request_statuses()
except Exception:
sentry.captureException()
send_email(subject="Update Request Statuses Failure",
to=[OPENRECORDS_DL_EMAIL],
email_content=traceback.format_exc().replace(
"\n", "<br/>").replace(" ", " "))
def test_query():
try:
rows = TestTable.query.all()
return rows
except DatabaseError:
sentry.captureException()
return None
except Exception as e:
sentry.captureMessage(e)
return None
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None):
"""
Scans an uploaded file (see scan_file) and moves
it to the data directory if it is clean. If is_update is set,
the file will also be placed under the 'updated' directory.
Updates redis accordingly.
:param request_id: id of request associated with the upload
:param filepath: path to uploaded and quarantined file
:param is_update: will the file replace an existing one?
:param response_id: id of response associated with the upload
"""
if is_update:
assert response_id is not None
else:
assert response_id is None
filename = os.path.basename(filepath)
key = get_upload_key(request_id, filename, is_update)
redis.set(key, upload_status.SCANNING)
try:
scan_file(filepath)
except VirusDetectedException:
sentry.captureException()
redis.delete(key)
else:
# complete upload
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id
)
if is_update:
dst_dir = os.path.join(
dst_dir,
UPDATED_FILE_DIRNAME
)
# store file metadata in redis
redis_set_file_metadata(response_id or request_id, filepath, is_update)
if not fu.exists(dst_dir):
try:
fu.makedirs(dst_dir)
except OSError as e:
sentry.captureException()
# in the time between the call to fu.exists
# and fu.makedirs, the directory was created
current_app.logger.error("OS Error: {}".format(e.args))
fu.move(
filepath,
os.path.join(dst_dir, filename)
)
redis.set(key, upload_status.READY)
def has_permission(request):
access_token = get_access_token(request)
if access_token is None:
return False
try:
return check_account(access_token)
except Exception:
sentry.captureException()
return False
def twitter_login_step1():
try:
return redirect(
libforget.twitter.get_login_url(
callback=url_for('twitter_login_step2', _external=True),
**app.config.get_namespace("TWITTER_")))
except (TwitterError, URLError):
if sentry:
sentry.captureException()
return redirect(
url_for('about', twitter_login_error='', _anchor='log_in'))
def test_es_update(self, es_update_patch):
role = Roles.query.first()
# check called for model with 'es_update' (Requests)
update_object({'title': 'new and improved TITLE X50 DELUXE'}, Requests,
self.request_id)
es_update_patch.assert_called_once_with()
# check not called for model without 'es_update' (Roles)
try:
update_object({'name': 'nombre'}, Roles, role.id)
except AttributeError:
sentry.captureException()
self.fail('es_update() called when it should not have been.')
def settings():
viewer = get_viewer()
try:
for attr in libforget.settings.attrs:
if attr in request.form:
setattr(viewer, attr, request.form[attr])
db.session.commit()
except ValueError:
if sentry:
sentry.captureException()
return 400
return redirect(url_for('index', settings_saved=''))
def _sftp_makedirs(sftp, path):
""" os.makedirs(path, exists_ok=True) """
dirs = []
while len(path) > 1:
dirs.append(path)
path, _ = os.path.split(path)
while len(dirs):
dir_ = dirs.pop()
try:
sftp.stat(dir_)
except IOError:
sentry.captureException()
sftp.mkdir(dir_)
def _sftp_makedirs(sftp, path):
""" os.makedirs(path, exists_ok=True) """
dirs = []
while len(path) > 1:
dirs.append(path)
path, _ = os.path.split(path)
while len(dirs):
dir_ = dirs.pop()
try:
sftp.stat(dir_)
except IOError:
sentry.captureException()
sftp.mkdir(dir_)
def assert_flashes(self, expected_message, expected_category):
"""
Assert flash messages are flashed properly with expected message and category.
:param expected_message: expected flash message
:param expected_category: expected flash category
"""
with self.client.session_transaction() as session:
try:
category, message = session['_flashes'][0]
except KeyError:
sentry.captureException()
raise AssertionError('Nothing was flashed')
assert expected_message in message
assert expected_category == category
def redis_get_user_session(session_id):
serialization_method = pickle
session_class = KVSession
try:
s = session_class(
serialization_method.loads(
current_app.kvsession_store.get(session_id)))
s.sid_s = session_id
return s
except KeyError:
sentry.captureException()
return None
def send_texts():
users = User.query.all()
for user in users:
if user.is_active and user.needs_message_now():
print "%s needs forecast" % user
try:
sent = user.send_forecast()
if sent:
print "sent forecast to %s" % user
else:
print "didn't send forecast to %s" % user
except Exception as e:
print e
sentry.captureException()
def redis_get_user_session(session_id):
serialization_method = pickle
session_class = KVSession
try:
s = session_class(serialization_method.loads(
current_app.kvsession_store.get(session_id)
))
s.sid_s = session_id
return s
except KeyError:
sentry.captureException()
return None
def get_object(obj_type, obj_id):
"""
Safely retrieve a database record by its id
and its sqlalchemy object type.
"""
if not obj_id:
return None
try:
return obj_type.query.get(obj_id)
except SQLAlchemyError:
sentry.captureException()
db.session.rollback()
current_app.logger.exception('Error searching "{}" table for id {}'.format(
obj_type.__tablename__, obj_id))
return None
def _sftp_get_mime_type(sftp, path):
with TemporaryFile() as tmp:
try:
sftp.getfo(path, tmp, _raise_if_too_big)
except MaxTransferSizeExceededException:
sentry.captureException()
tmp.seek(0)
if current_app.config['MAGIC_FILE']:
# Check using custom mime database file
m = magic.Magic(magic_file=current_app.config['MAGIC_FILE'],
mime=True)
mime_type = m.from_buffer(tmp.read())
else:
mime_type = magic.from_buffer(tmp.read(), mime=True)
return mime_type
def delete_object(obj):
"""
Delete a database record.
:param obj: object (instance of sqlalchemy model) to delete
:return: was the record deleted successfully?
"""
try:
db.session.delete(obj)
db.session.commit()
return True
except SQLAlchemyError:
sentry.captureException()
db.session.rollback()
current_app.logger.exception("Failed to DELETE {}".format(obj))
return False
def _sftp_get_mime_type(sftp, path):
with TemporaryFile() as tmp:
try:
sftp.getfo(path, tmp, _raise_if_too_big)
except MaxTransferSizeExceededException:
sentry.captureException()
tmp.seek(0)
if current_app.config['MAGIC_FILE']:
# Check using custom mime database file
m = magic.Magic(
magic_file=current_app.config['MAGIC_FILE'],
mime=True)
mime_type = m.from_buffer(tmp.read())
else:
mime_type = magic.from_buffer(tmp.read(), mime=True)
return mime_type
def resizer(img, remember):
save_to = image_save_to(app.config['IMAGES_CACHE_DIR'], img.id)
try:
image = Imagenator(img.path)
if img.strategy == 'fit':
image.resize_fitin(img.size)
else:
image.resize_crop(img.size, 2)
image.save(save_to, img.format.codec, img.quality)
except Exception as e:
sentry.captureException()
raise ResizerError(e)
return save_to
def edit(request_id):
"""
Updates a users permissions on a request and sends notification emails.
Expects a request body containing the user's guid and updated permissions.
Ex:
{
'user': '******',
1: true,
5: false
}
:return:
"""
current_user_request = current_user.user_requests.filter_by(
request_id=request_id).one()
current_request = current_user_request.request
if (current_user.is_agency
and (current_user.is_super
or current_user.is_agency_admin(current_request.agency.ein)
or current_user_request.has_permission(
permission.EDIT_USER_REQUEST_PERMISSIONS))):
user_data = flask_request.form
point_of_contact = True if role_name.POINT_OF_CONTACT in user_data else False
required_fields = ['user']
for field in required_fields:
if not user_data.get(field):
flash('Uh Oh, it looks like the {} is missing! '
'This is probably NOT your fault.'.format(field),
category='danger')
return redirect(url_for('request.view', request_id=request_id))
try:
permissions = [int(i) for i in user_data.getlist('permission')]
edit_user_request(request_id=request_id,
user_guid=user_data.get('user'),
permissions=permissions,
point_of_contact=point_of_contact)
except UserRequestException as e:
sentry.captureException()
flash(e, category='warning')
return redirect(url_for('request.view', request_id=request_id))
return 'OK', 200
return abort(403)
def twitter_login_step2():
try:
oauth_token = request.args.get('oauth_token', '')
oauth_verifier = request.args.get('oauth_verifier', '')
token = libforget.twitter.receive_verifier(
oauth_token, oauth_verifier,
**app.config.get_namespace("TWITTER_"))
session = login(token.account_id)
g.viewer = session
return redirect(url_for('index'))
except Exception:
if sentry:
sentry.captureException()
return redirect(
url_for('about', twitter_login_error='', _anchor='log_in'))
def validate_schema(data, schema_name):
"""
Validate the provided data against the provided JSON schema.
:param data: JSON data to be validated
:param schema_name: Name of the schema
:return: Boolean
"""
with open(os.path.join(current_app.config['JSON_SCHEMA_DIRECTORY'], schema_name + '.schema'), 'r') as fp:
schema = json.load(fp)
try:
validate(data, schema)
return True
except ValidationError as e:
sentry.captureException()
current_app.logger.info("Failed to validate {}\n{}".format(json.dumps(data), e))
return False
def bulk_delete(query):
"""
Delete multiple database records via a bulk delete query.
http://docs.sqlalchemy.org/en/latest/orm/query.html#sqlalchemy.orm.query.Query.delete
:param query: Query object
:return: the number of records deleted
"""
try:
num_deleted = query.delete()
db.session.commit()
return num_deleted
except SQLAlchemyError:
sentry.captureException()
db.session.rollback()
current_app.logger.exception("Failed to BULK DELETE {}".format(query))
return 0
def edit(request_id):
"""
Updates a users permissions on a request and sends notification emails.
Expects a request body containing the user's guid and updated permissions.
Ex:
{
'user': '******',
1: true,
5: false
}
:return:
"""
current_user_request = current_user.user_requests.filter_by(request_id=request_id).one()
current_request = current_user_request.request
if (
current_user.is_agency and (
current_user.is_super or
current_user.is_agency_admin(current_request.agency.ein) or
current_user_request.has_permission(permission.EDIT_USER_REQUEST_PERMISSIONS)
)
):
user_data = flask_request.form
point_of_contact = True if role_name.POINT_OF_CONTACT in user_data else False
required_fields = ['user']
for field in required_fields:
if not user_data.get(field):
flash('Uh Oh, it looks like the {} is missing! '
'This is probably NOT your fault.'.format(field), category='danger')
return redirect(url_for('request.view', request_id=request_id))
try:
permissions = [int(i) for i in user_data.getlist('permission')]
edit_user_request(request_id=request_id, user_guid=user_data.get('user'),
permissions=permissions, point_of_contact=point_of_contact)
except UserRequestException as e:
sentry.captureException()
flash(e, category='warning')
return redirect(url_for('request.view', request_id=request_id))
return 'OK', 200
return abort(403)
def is_allowed(user: Users, request_id: str, permission: int):
"""
:param user:
:param request_id:
:param permissions:
:return:
"""
try:
user_request = user.user_requests.filter_by(request_id=request_id).one()
return True if user_request.has_permission(permission) else False
except NoResultFound:
sentry.captureException()
return False
except AttributeError:
sentry.captureException()
return False
def send_texts():
# users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"])
users = User.query.filter_by(is_active=True)
for user in users:
if user.is_active:
current_offset = user.time_zone
suggested_offset = int(current_offset) + 1
message = "Hope you've found Rooster to be a helpful addition to your morning! The site costs $30/month to operate. I'd love your support: http://www.roosterapp.co/donate/"
try:
sent = user.send_message(message, "donation_request")
if sent:
print "sent request to %s" % user
else:
print "didn't send request to %s" % user
except Exception as e:
print e
sentry.captureException()
def send_texts():
users = User.query.all(is_active=True)
# users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"])
for user in users:
if user.is_active:
current_offset = user.time_zone
suggested_offset = int(current_offset) + 1
message = "Don't let daylight savings time mess up your alarm!\n\nYour current timezone offset is %s\n\nIf DST has started for you, reply with \"TZ: %s\" to update." % (current_offset, suggested_offset)
try:
sent = user.send_message(message, "dst_warning")
if sent:
print "sent warning to %s" % user
else:
print "didn't send warning to %s" % user
except Exception as e:
print e
sentry.captureException()
def _generate_signature(password, string):
"""
Generate an NYC.ID Web Services authentication signature using HMAC-SHA1
https://nyc4d.nycnet/nycid/web-services.shtml#signature
:param password: NYC.ID Service Account password
:param string: string to sign
:return: the authentication signature or None on failure
"""
signature = None
try:
hmac_sha1 = hmac.new(key=password.encode(),
msg=string.encode(),
digestmod=sha1)
signature = hmac_sha1.hexdigest()
except Exception as e:
sentry.captureException()
current_app.logger.error("Failed to generate NYC ID.Web Services "
"authentication signature: ", e)
return signature
def send_texts():
# users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"])
users = User.query.filter_by(is_active=True)
for user in users:
if user.is_active:
message = "We haven't heard from you in a while, so we're temporarily deactivating your Rooster App account.\n\nReply 'START' to reactivate."
try:
sent = user.send_message(message, "deactivation")
if sent:
print "sent request to %s" % user
else:
print "didn't send request to %s" % user
except Exception as e:
print e
sentry.captureException()
user.is_active = False
db.session.add(user)
db.session.commit()
def handle_upload_no_id(file_field):
"""
Try to store and scan an uploaded file when no request id
has been generated. Return the stored upload file path
on success, otherwise add errors to the file field.
:param file_field: form file field
:return: the file path to the stored upload
"""
path = None
valid_file_type, file_type = is_valid_file_type(file_field.data)
if not valid_file_type:
file_field.errors.append(
"File type '{}' is not allowed.".format(file_type))
else:
try:
path = _quarantine_upload_no_id(file_field.data)
except Exception as e:
sentry.captureException()
print("Error saving file {} : {}".format(
file_field.data.filename, e))
file_field.errors.append('Error saving file.')
else:
try:
scan_file(path)
except VirusDetectedException:
sentry.captureException()
file_field.errors.append('File is infected.')
except Exception:
sentry.captureException()
file_field.errors.append('Error scanning file.')
return path
def update_object(data, obj_type, obj_id, es_update=True):
"""
Update a database record and its elasticsearch counterpart.
:param data: a dictionary of attribute-value pairs
:param obj_type: sqlalchemy model
:param obj_id: id of record
:param es_update: update the elasticsearch index
:return: was the record updated successfully?
"""
obj = get_object(obj_type, obj_id)
if obj:
for attr, value in data.items():
if isinstance(value, dict):
# update json values
attr_json = getattr(obj, attr) or {}
for key, val in value.items():
attr_json[key] = val
setattr(obj, attr, attr_json)
flag_modified(obj, attr)
else:
setattr(obj, attr, value)
try:
db.session.commit()
except SQLAlchemyError:
sentry.captureException()
db.session.rollback()
current_app.logger.exception("Failed to UPDATE {}".format(obj))
else:
# update elasticsearch
if hasattr(obj, 'es_update') and current_app.config['ELASTICSEARCH_ENABLED'] and es_update:
obj.es_update()
return True
return False
def current_version():
try:
return check_output(['git', 'describe', '--always'])
except CalledProcessError:
sentry.captureException()
return ""
def requests_doc(doc_type):
"""
Converts and sends the a search result-set as a
file of the specified document type.
- Filtering on set size is ignored; all results are returned.
- Currently only supports CSVs.
- CSV only includes requests belonging to that user's agency
Document name format: "FOIL_requests_results_<timestamp:MM_DD_YYYY_at_HH_mm_pp>"
Request parameters are identical to those of /search/requests.
:param doc_type: document type ('csv' only)
"""
if current_user.is_agency and doc_type.lower() == 'csv':
try:
agency_ein = request.args.get('agency_ein', '')
except ValueError:
sentry.captureException()
agency_ein = None
tz_name = request.args.get('tz_name', current_app.config['APP_TIMEZONE'])
start = 0
buffer = StringIO() # csvwriter cannot accept BytesIO
writer = csv.writer(buffer)
writer.writerow(["FOIL ID",
"Agency",
"Title",
"Description",
"Agency Request Summary",
"Current Status",
"Date Created",
"Date Received",
"Date Due",
"Date Closed",
"Requester Name",
"Requester Email",
"Requester Title",
"Requester Organization",
"Requester Phone Number",
"Requester Fax Number",
"Requester Address 1",
"Requester Address 2",
"Requester City",
"Requester State",
"Requester Zipcode",
"Assigned User Emails"])
results = search_requests(
query=request.args.get('query'),
foil_id=eval_request_bool(request.args.get('foil_id')),
title=eval_request_bool(request.args.get('title')),
agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')),
description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False,
requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False,
date_rec_from=request.args.get('date_rec_from'),
date_rec_to=request.args.get('date_rec_to'),
date_due_from=request.args.get('date_due_from'),
date_due_to=request.args.get('date_due_to'),
date_closed_from=request.args.get('date_closed_from'),
date_closed_to=request.args.get('date_closed_to'),
agency_ein=agency_ein,
agency_user_guid=request.args.get('agency_user'),
open_=eval_request_bool(request.args.get('open')),
closed=eval_request_bool(request.args.get('closed')),
in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False,
due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False,
overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False,
start=start,
sort_date_received=request.args.get('sort_date_submitted'),
sort_date_due=request.args.get('sort_date_due'),
sort_title=request.args.get('sort_title'),
tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE']),
for_csv=True
)
ids = [result["_id"] for result in results]
all_requests = Requests.query.filter(Requests.id.in_(ids)).options(
joinedload(Requests.agency_users)).options(joinedload(Requests.requester)).options(
joinedload(Requests.agency)).all()
user_agencies = current_user.get_agencies
for req in all_requests:
if req.agency_ein in user_agencies:
writer.writerow([
req.id,
req.agency.name,
req.title,
req.description,
req.agency_request_summary,
req.status,
req.date_created,
req.date_submitted,
req.due_date,
req.date_closed,
req.requester.name,
req.requester.email,
req.requester.title,
req.requester.organization,
req.requester.phone_number,
req.requester.fax_number,
req.requester.mailing_address.get('address_one'),
req.requester.mailing_address.get('address_two'),
req.requester.mailing_address.get('city'),
req.requester.mailing_address.get('state'),
req.requester.mailing_address.get('zip'),
", ".join(u.email for u in req.agency_users)])
dt = datetime.utcnow()
timestamp = utc_to_local(dt, tz_name) if tz_name is not None else dt
return send_file(
BytesIO(buffer.getvalue().encode('UTF-8')), # convert to bytes
attachment_filename="FOIL_requests_results_{}.csv".format(
timestamp.strftime("%m_%d_%Y_at_%I_%M_%p")),
as_attachment=True
)
return '', 400
def post(request_id):
"""
Create a new upload.
Handles chunked files through the Content-Range header.
For filesize validation and more upload logic, see:
/static/js/upload/fileupload.js
Optional request body parameters:
- update (bool)
save the uploaded file to the 'updated' directory
(this indicates the file is meant to replace
a previously uploaded file)
- response_id (int)
the id of a response associated with the file
this upload is replacing
- REQUIRED if 'update' is 'true'
- ignored if 'update' is 'false'
:returns: {
"name": file name,
"size": file size
}
"""
files = request.files
file_ = files[next(files.keys())]
filename = secure_filename(file_.filename)
is_update = eval_request_bool(request.form.get('update'))
agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein
if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \
is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE):
response_id = request.form.get('response_id') if is_update else None
if upload_exists(request_id, filename, response_id):
response = {
"files": [{
"name": filename,
"error": "A file with this name has already "
"been uploaded for this request."
# TODO: "link": <link-to-existing-file> ? would be nice
}]
}
else:
upload_path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
request_id)
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filepath = os.path.join(upload_path, filename)
key = get_upload_key(request_id, filename, is_update)
try:
if CONTENT_RANGE_HEADER in request.headers:
start, size = parse_content_range(
request.headers[CONTENT_RANGE_HEADER])
# Only validate mime type on first chunk
valid_file_type = True
file_type = None
if start == 0:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if os.path.exists(filepath):
# remove existing file (upload 'restarted' for same file)
os.remove(filepath)
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
with open(filepath, 'ab') as fp:
fp.seek(start)
fp.write(file_.stream.read())
# scan if last chunk written
if os.path.getsize(filepath) == size:
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
else:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
file_.save(filepath)
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
if not valid_file_type:
response = {
"files": [{
"name": filename,
"error": "The file type '{}' is not allowed.".format(
file_type)
}]
}
else:
response = {
"files": [{
"name": filename,
"original_name": file_.filename,
"size": os.path.getsize(filepath),
}]
}
except Exception as e:
sentry.captureException()
redis.set(key, upload_status.ERROR)
current_app.logger.exception("Upload for file '{}' failed: {}".format(filename, e))
response = {
"files": [{
"name": filename,
"error": "There was a problem uploading this file."
}]
}
return jsonify(response), 200
def delete(r_id_type, r_id, filecode):
"""
Removes an uploaded file.
:param r_id_type: "response" or "request"
:param r_id: the Response or Request identifier
:param filecode: the encoded name of the uploaded file
(base64 without padding)
Optional request body parameters:
- quarantined_only (bool)
only delete the file if it is quarantined
(beware: takes precedence over 'updated_only')
- updated_only (bool)
only delete the file if it is in the 'updated' directory
:returns:
On success:
{ "deleted": filename }
On failure:
{ "error": error message }
"""
filename = secure_filename(b64decode_lenient(filecode))
if r_id_type not in ["request", "response"]:
response = {"error": "Invalid ID type."}
else:
try:
if r_id_type == "response":
response = Responses.query.filter_by(id=r_id, deleted=False)
r_id = response.request_id
path = ''
quarantined_only = eval_request_bool(request.form.get('quarantined_only'))
has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE))
if quarantined_only and has_add_edit:
path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
r_id
)
elif eval_request_bool(request.form.get('updated_only')) and \
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE):
path = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
r_id,
UPDATED_FILE_DIRNAME
)
else:
path_for_status = {
upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.READY: current_app.config['UPLOAD_DIRECTORY']
}
status = redis.get(get_upload_key(r_id, filename))
if status is not None:
dest_path = path_for_status[status.decode("utf-8")]
if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit) or (
dest_path == current_app.config['UPLOAD_DIRECTORY'] and
is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE)
):
path = os.path.join(
dest_path,
r_id
)
filepath = os.path.join(path, filename)
found = False
if path != '':
if quarantined_only:
if os.path.exists(filepath):
os.remove(filepath)
found = True
else:
if fu.exists(filepath):
fu.remove(filepath)
found = True
if found:
response = {"deleted": filename}
else:
response = {"error": "Upload not found."}
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error on DELETE /upload/: {}".format(e))
response = {"error": "Failed to delete '{}'".format(filename)}
return jsonify(response), 200
def view(request_id):
"""
This function is for testing purposes of the view a request back until backend functionality is implemented.
:return: redirect to view request page
"""
try:
current_request = Requests.query.filter_by(id=request_id).one()
assert current_request.agency.is_active
except NoResultFound:
print("Request with id '{}' does not exist.".format(request_id))
sentry.captureException()
return abort(404)
except AssertionError:
print("Request belongs to inactive agency.")
sentry.captureException()
return abort(404)
holidays = sorted(get_holidays_date_list(
datetime.utcnow().year,
(datetime.utcnow() + rd(years=DEFAULT_YEARS_HOLIDAY_LIST)).year)
)
active_users = []
assigned_users = []
if current_user.is_agency:
for agency_user in current_request.agency.active_users:
if not agency_user in current_request.agency.administrators and (agency_user != current_user):
# populate list of assigned users that can be removed from a request
if agency_user in current_request.agency_users:
assigned_users.append(agency_user)
# append to list of active users that can be added to a request
else:
active_users.append(agency_user)
permissions = {
'acknowledge': permission.ACKNOWLEDGE,
'deny': permission.DENY,
'extend': permission.EXTEND,
'close': permission.CLOSE,
're_open': permission.RE_OPEN,
'add_file': permission.ADD_FILE,
'edit_file_privacy': permission.EDIT_FILE_PRIVACY,
'delete_file': permission.DELETE_FILE,
'add_note': permission.ADD_NOTE,
'edit_note_privacy': permission.EDIT_NOTE_PRIVACY,
'delete_note': permission.DELETE_NOTE,
'add_link': permission.ADD_LINK,
'edit_link_privacy': permission.EDIT_LINK_PRIVACY,
'delete_link': permission.DELETE_LINK,
'add_instructions': permission.ADD_OFFLINE_INSTRUCTIONS,
'edit_instructions_privacy': permission.EDIT_OFFLINE_INSTRUCTIONS_PRIVACY,
'delete_instructions': permission.DELETE_OFFLINE_INSTRUCTIONS,
'generate_letter': permission.GENERATE_LETTER,
'add_user': permission.ADD_USER_TO_REQUEST,
'edit_user': permission.EDIT_USER_REQUEST_PERMISSIONS,
'remove_user': permission.REMOVE_USER_FROM_REQUEST,
'edit_title': permission.EDIT_TITLE,
'edit_title_privacy': permission.CHANGE_PRIVACY_TITLE,
'edit_agency_request_summary': permission.EDIT_AGENCY_REQUEST_SUMMARY,
'edit_agency_request_summary_privacy': permission.CHANGE_PRIVACY_AGENCY_REQUEST_SUMMARY,
'edit_requester_info': permission.EDIT_REQUESTER_INFO
}
# Build permissions dictionary for checking on the front-end.
for key, val in permissions.items():
if current_user.is_anonymous or not current_request.user_requests.filter_by(
user_guid=current_user.guid).first():
permissions[key] = False
else:
permissions[key] = is_allowed(current_user, request_id, val) if not current_user.is_anonymous else False
# Build dictionary of current permissions for all assigned users.
assigned_user_permissions = {}
for u in assigned_users:
assigned_user_permissions[u.guid] = UserRequests.query.filter_by(
request_id=request_id, user_guid=u.guid).one().get_permission_choice_indices()
point_of_contact = get_current_point_of_contact(request_id)
if point_of_contact:
current_point_of_contact = {'user_guid': point_of_contact.user_guid}
else:
current_point_of_contact = {'user_guid': ''}
# Determine if the Agency Request Summary should be shown.
show_agency_request_summary = False
if current_user in current_request.agency_users \
or current_request.agency_request_summary \
and (current_request.requester == current_user
and current_request.status == request_status.CLOSED
and not current_request.privacy['agency_request_summary']
or current_request.status == request_status.CLOSED
and current_request.agency_request_summary_release_date
and current_request.agency_request_summary_release_date
< datetime.utcnow()
and not current_request.privacy['agency_request_summary']):
show_agency_request_summary = True
# Determine if the title should be shown.
show_title = (current_user in current_request.agency_users or
current_request.requester == current_user or
not current_request.privacy['title'])
# Determine if "Generate Letter" functionality is enabled for the agency.
if 'letters' in current_request.agency.agency_features:
generate_letters_enabled = current_request.agency.agency_features['letters']['generate_letters']
else:
generate_letters_enabled = False
# Determine if custom request forms are enabled
if 'enabled' in current_request.agency.agency_features['custom_request_forms']:
custom_request_forms_enabled = current_request.agency.agency_features['custom_request_forms']['enabled']
else:
custom_request_forms_enabled = False
# Determine if custom request form panels should be expanded by default
if 'expand_by_default' in current_request.agency.agency_features['custom_request_forms']:
expand_by_default = current_request.agency.agency_features['custom_request_forms']['expand_by_default']
else:
expand_by_default = False
# Determine if request description should be hidden when custom forms are enabled
if 'description_hidden_by_default' in current_request.agency.agency_features['custom_request_forms']:
description_hidden_by_default = current_request.agency.agency_features['custom_request_forms']['description_hidden_by_default']
else:
description_hidden_by_default = False
return render_template(
'request/view_request.html',
request=current_request,
status=request_status,
agency_users=current_request.agency_users,
edit_requester_form=EditRequesterForm(current_request.requester),
contact_agency_form=ContactAgencyForm(current_request),
deny_request_form=DenyRequestForm(current_request.agency.ein),
close_request_form=CloseRequestForm(current_request.agency.ein),
reopen_request_form=ReopenRequestForm(current_request.agency.ein),
remove_user_request_form=RemoveUserRequestForm(assigned_users),
add_user_request_form=AddUserRequestForm(active_users),
edit_user_request_form=EditUserRequestForm(assigned_users),
generate_acknowledgment_letter_form=GenerateAcknowledgmentLetterForm(current_request.agency.ein),
generate_denial_letter_form=GenerateDenialLetterForm(current_request.agency.ein),
generate_closing_letter_form=GenerateClosingLetterForm(current_request.agency.ein),
generate_extension_letter_form=GenerateExtensionLetterForm(current_request.agency.ein),
generate_envelope_form=GenerateEnvelopeForm(current_request.agency_ein, current_request.requester),
generate_response_letter_form=GenerateResponseLetterForm(current_request.agency.ein),
assigned_user_permissions=assigned_user_permissions,
current_point_of_contact=current_point_of_contact,
holidays=holidays,
assigned_users=assigned_users,
active_users=active_users,
permissions=permissions,
show_agency_request_summary=show_agency_request_summary,
show_title=show_title,
is_requester=(current_request.requester == current_user),
permissions_length=len(permission.ALL),
generate_letters_enabled=generate_letters_enabled,
custom_request_forms_enabled = custom_request_forms_enabled,
expand_by_default=expand_by_default,
description_hidden_by_default=description_hidden_by_default
)
def requests():
"""
For request parameters, see app.search.utils.search_requests
All Users can search by:
- FOIL ID
Anonymous Users can search by:
- Title (public only)
- Agency Request Summary (public only)
Public Users can search by:
- Title (public only OR public and private if user is requester)
- Agency Request Summary (public only)
- Description (if user is requester)
Agency Users can search by:
- Title
- Agency Request Summary
- Description
- Requester Name
All Users can filter by:
- Status, Open (anything not Closed if not agency user)
- Status, Closed
- Date Submitted
- Agency
Only Agency Users can filter by:
- Status, In Progress
- Status, Due Soon
- Status, Overdue
- Date Due
"""
try:
agency_ein = request.args.get('agency_ein', '')
except ValueError:
sentry.captureException()
agency_ein = None
try:
size = int(request.args.get('size', DEFAULT_HITS_SIZE))
except ValueError:
sentry.captureException()
size = DEFAULT_HITS_SIZE
try:
start = int(request.args.get('start'), 0)
except ValueError:
sentry.captureException()
start = 0
query = request.args.get('query')
# Determine if searching for FOIL ID
foil_id = eval_request_bool(request.args.get('foil_id')) or re.match(r'^(FOIL-|foil-|)\d{4}-\d{3}-\d{5}$', query)
results = search_requests(
query=query,
foil_id=foil_id,
title=eval_request_bool(request.args.get('title')),
agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')),
description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False,
requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False,
date_rec_from=request.args.get('date_rec_from'),
date_rec_to=request.args.get('date_rec_to'),
date_due_from=request.args.get('date_due_from'),
date_due_to=request.args.get('date_due_to'),
date_closed_from=request.args.get('date_closed_from'),
date_closed_to=request.args.get('date_closed_to'),
agency_ein=agency_ein,
agency_user_guid=request.args.get('agency_user'),
open_=eval_request_bool(request.args.get('open')),
closed=eval_request_bool(request.args.get('closed')),
in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False,
due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False,
overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False,
size=size,
start=start,
sort_date_received=request.args.get('sort_date_submitted'),
sort_date_due=request.args.get('sort_date_due'),
sort_title=request.args.get('sort_title'),
tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE'])
)
# format results
total = results["hits"]["total"]
formatted_results = None
if total != 0:
convert_dates(results)
formatted_results = render_template("request/result_row.html",
requests=results["hits"]["hits"])
# query=query) # only for testing
return jsonify({
"count": len(results["hits"]["hits"]),
"total": total,
"results": formatted_results
}), 200
