def response_quick_closing(request_id): """Endpoint for quick closing a request that takes in form data from the front end. Required form data include: email-date: the number of days the acknowledgement will take. Defaults to 20 for quick closings summary: string email body from the confirmation page Args: request_id: FOIL request ID Returns: Redirect to view request page """ required_fields = ['email-date', 'summary'] for field in required_fields: if not flask_request.form.get(field, ''): flash('Uh Oh, it looks like the acknowledgement/closing {} is missing! ' 'This is probably NOT your fault.'.format(field), category='danger') return redirect(url_for('request.view', request_id=request_id)) try: add_quick_closing(request_id=request_id, days=request_date.DEFAULT_QUICK_CLOSING_DAYS, date=flask_request.form['email-date'], tz_name=flask_request.form['tz-name'] if flask_request.form['tz-name'] else current_app.config['APP_TIMEZONE'], content=flask_request.form['summary']) except UserRequestException as e: sentry.captureException() flash(str(e), category='danger') return redirect(url_for('request.view', request_id=request_id))
def vpk_package(folder): try: check_call([os.path.abspath(current_app.config['VPK_BINARY_PATH']), folder]) except CalledProcessError: sentry.captureException() abort(500) shutil.rmtree(folder)
def sftp_ctx(): """ Context manager that provides an SFTP client object (an SFTP session across an open SSH Transport) """ transport = paramiko.Transport((current_app.config['SFTP_HOSTNAME'], int(current_app.config['SFTP_PORT']))) authentication_kwarg = {} if current_app.config['SFTP_PASSWORD']: authentication_kwarg['password'] = current_app.config['SFTP_PASSWORD'] elif current_app.config['SFTP_RSA_KEY_FILE']: authentication_kwarg['pkey'] = paramiko.RSAKey(filename=current_app.config['SFTP_RSA_KEY_FILE']) else: raise SFTPCredentialsException transport.connect(username=current_app.config['SFTP_USERNAME'], **authentication_kwarg) sftp = paramiko.SFTPClient.from_transport(transport) try: yield sftp except Exception as e: sentry.captureException() raise paramiko.SFTPError("Exception occurred with SFTP: {}".format(e)) finally: sftp.close() transport.close()
def _web_services_request(endpoint, params, method='GET'): """ Perform a request on an NYC.ID Web Services endpoint. 'userName' and 'signature' are added to the specified params. :param endpoint: web services endpoint (e.g. "/account/validateEmail.htm") :param params: request parameters excluding 'userName' and 'signature' :param method: HTTP method :return: request response """ current_app.logger.info("NYC.ID Web Services Requests: {} {}".format(method, endpoint)) params['userName'] = current_app.config['NYC_ID_USERNAME'] # don't refactor to use dict.update() - signature relies on userName param params['signature'] = _generate_signature( current_app.config['NYC_ID_PASSWORD'], _generate_string_to_sign(method, endpoint, params) ) req = None # SSLError with 'bad signature' is sometimes thrown when sending the request which causes an nginx error and 502 # resending the request resolves the issue for _ in range(5): try: req = requests.request( method, urljoin(current_app.config['WEB_SERVICES_URL'], endpoint), verify=current_app.config['VERIFY_WEB_SERVICES'], params=params # query string parameters always used ) except SSLError: sentry.captureException() continue break return req
def status(): """ Check the status of an upload. Request Parameters: - request_id - filename - for_update (bool, optional) :returns: { "status": upload status } """ try: status = redis.get( get_upload_key( request.args['request_id'], secure_filename(request.args['filename']), eval_request_bool(request.args.get('for_update')) ) ) if status is not None: response = {"status": status.decode("utf-8")} else: response = {"error": "Upload status not found."} status_code = 200 except KeyError: sentry.captureException() response = {} status_code = 422 return jsonify(response), status_code
def sftp_ctx(): """ Context manager that provides an SFTP client object (an SFTP session across an open SSH Transport) """ transport = paramiko.Transport((current_app.config['SFTP_HOSTNAME'], int(current_app.config['SFTP_PORT']))) authentication_kwarg = {} if current_app.config['SFTP_PASSWORD']: authentication_kwarg['password'] = current_app.config['SFTP_PASSWORD'] elif current_app.config['SFTP_RSA_KEY_FILE']: authentication_kwarg['pkey'] = paramiko.RSAKey( filename=current_app.config['SFTP_RSA_KEY_FILE']) else: raise SFTPCredentialsException transport.connect(username=current_app.config['SFTP_USERNAME'], **authentication_kwarg) sftp = paramiko.SFTPClient.from_transport(transport) try: yield sftp except Exception as e: sentry.captureException() raise paramiko.SFTPError("Exception occurred with SFTP: {}".format(e)) finally: sftp.close() transport.close()
def response_closing(request_id): """ Endpoint for closing a request that takes in form data from the front end. Required form data include: -reasons: a list of closing reasons -email-summary: string email body from the confirmation page :param request_id: FOIL request ID :return: redirect to view request page """ if flask_request.form.get('method') == EMAIL: required_fields = ['reasons', 'method', 'summary'] else: required_fields = ['letter_templates', 'method', 'summary'] for field in required_fields: if not flask_request.form.get(field, ''): flash('Uh Oh, it looks like the closing {} is missing! ' 'This is probably NOT your fault.'.format(field), category='danger') return redirect(url_for('request.view', request_id=request_id)) try: add_closing(request_id, flask_request.form.getlist('reasons'), flask_request.form['summary'], flask_request.form['method'], flask_request.form.get('letter_templates')) except UserRequestException as e: sentry.captureException() flash(str(e), category='danger') return redirect(url_for('request.view', request_id=request_id))
def response_closing(request_id): """ Endpoint for closing a request that takes in form data from the front end. Required form data include: -reasons: a list of closing reasons -email-summary: string email body from the confirmation page :param request_id: FOIL request ID :return: redirect to view request page """ if flask_request.form.get('method') == EMAIL: required_fields = ['reasons', 'method', 'summary'] else: required_fields = ['letter_templates', 'method', 'summary'] for field in required_fields: if flask_request.form.get(field) is None: flash('Uh Oh, it looks like the closing {} is missing! ' 'This is probably NOT your fault.'.format(field), category='danger') return redirect(url_for('request.view', request_id=request_id)) try: add_closing(request_id, flask_request.form.getlist('reasons'), flask_request.form['summary'], flask_request.form['method'], flask_request.form.get('letter_templates')) except UserRequestException as e: sentry.captureException() flash(str(e), category='danger') return redirect(url_for('request.view', request_id=request_id))
def create_object(obj): """ Add a database record and its elasticsearch counterpart. If 'obj' is a Requests object, nothing will be added to the es index since a UserRequests record is created after its associated request and the es doc requires a requester id. 'es_create' is called explicitly for a Requests object in app.request.utils. :param obj: object (instance of sqlalchemy model) to create :return: string representation of created object or None if creation failed """ try: db.session.add(obj) db.session.commit() except SQLAlchemyError: sentry.captureException() db.session.rollback() current_app.logger.exception("Failed to CREATE {}".format(obj)) return None else: # create elasticsearch doc if ( not isinstance(obj, Requests) and hasattr(obj, 'es_create') and current_app.config['ELASTICSEARCH_ENABLED'] ): obj.es_create() return str(obj)
def index(): user = {'nickname': 'tatumn'} posts = [{ 'author': { 'nickname': 'John' }, 'body': 'Beautiful day in Portland!' }, { 'author': { 'nickname': 'Susan' }, 'body': 'The Avengers movie was so cool!' }] try: 1 / 0 except Exception: sentry.captureException() sentry.captureMessage('hello, world!') form = NameForm() if form.validate_on_submit(): old_name = session.get('name') if old_name is not None and old_name != form.name.data: flash('Looks like you have changed your name!') session['name'] = form.name.data return redirect(url_for('home.index')) return render_template('html/index.html', title='home', user=user, current_time=datetime.utcnow(), form=form, name=session.get('name'), posts=posts)
def _sftp_exists(sftp, path): try: sftp.stat(path) return True except IOError: sentry.captureException() return False
def send_async_email(msg): try: mail.send(msg) except Exception as e: sentry.captureException() current_app.logger.exception("Failed to Send Email {} : {}".format( msg, e))
def internal_error(error): logger.error('ErrorHandler Exception %s', error) message = self.find_message(error) if sentry: sentry.captureException() return responses.status_403(message), 403
def vpk_package(folder): try: check_call( [os.path.abspath(current_app.config['VPK_BINARY_PATH']), folder]) except CalledProcessError: sentry.captureException() abort(500) shutil.rmtree(folder)
def update_request_statuses(): with scheduler.app.app_context(): try: _update_request_statuses() except Exception: sentry.captureException() send_email(subject="Update Request Statuses Failure", to=[OPENRECORDS_DL_EMAIL], email_content=traceback.format_exc().replace( "\n", "<br/>").replace(" ", " "))
def test_query(): try: rows = TestTable.query.all() return rows except DatabaseError: sentry.captureException() return None except Exception as e: sentry.captureMessage(e) return None
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None): """ Scans an uploaded file (see scan_file) and moves it to the data directory if it is clean. If is_update is set, the file will also be placed under the 'updated' directory. Updates redis accordingly. :param request_id: id of request associated with the upload :param filepath: path to uploaded and quarantined file :param is_update: will the file replace an existing one? :param response_id: id of response associated with the upload """ if is_update: assert response_id is not None else: assert response_id is None filename = os.path.basename(filepath) key = get_upload_key(request_id, filename, is_update) redis.set(key, upload_status.SCANNING) try: scan_file(filepath) except VirusDetectedException: sentry.captureException() redis.delete(key) else: # complete upload dst_dir = os.path.join( current_app.config['UPLOAD_DIRECTORY'], request_id ) if is_update: dst_dir = os.path.join( dst_dir, UPDATED_FILE_DIRNAME ) # store file metadata in redis redis_set_file_metadata(response_id or request_id, filepath, is_update) if not fu.exists(dst_dir): try: fu.makedirs(dst_dir) except OSError as e: sentry.captureException() # in the time between the call to fu.exists # and fu.makedirs, the directory was created current_app.logger.error("OS Error: {}".format(e.args)) fu.move( filepath, os.path.join(dst_dir, filename) ) redis.set(key, upload_status.READY)
def has_permission(request): access_token = get_access_token(request) if access_token is None: return False try: return check_account(access_token) except Exception: sentry.captureException() return False
def twitter_login_step1(): try: return redirect( libforget.twitter.get_login_url( callback=url_for('twitter_login_step2', _external=True), **app.config.get_namespace("TWITTER_"))) except (TwitterError, URLError): if sentry: sentry.captureException() return redirect( url_for('about', twitter_login_error='', _anchor='log_in'))
def test_es_update(self, es_update_patch): role = Roles.query.first() # check called for model with 'es_update' (Requests) update_object({'title': 'new and improved TITLE X50 DELUXE'}, Requests, self.request_id) es_update_patch.assert_called_once_with() # check not called for model without 'es_update' (Roles) try: update_object({'name': 'nombre'}, Roles, role.id) except AttributeError: sentry.captureException() self.fail('es_update() called when it should not have been.')
def settings(): viewer = get_viewer() try: for attr in libforget.settings.attrs: if attr in request.form: setattr(viewer, attr, request.form[attr]) db.session.commit() except ValueError: if sentry: sentry.captureException() return 400 return redirect(url_for('index', settings_saved=''))
def _sftp_makedirs(sftp, path): """ os.makedirs(path, exists_ok=True) """ dirs = [] while len(path) > 1: dirs.append(path) path, _ = os.path.split(path) while len(dirs): dir_ = dirs.pop() try: sftp.stat(dir_) except IOError: sentry.captureException() sftp.mkdir(dir_)
def assert_flashes(self, expected_message, expected_category): """ Assert flash messages are flashed properly with expected message and category. :param expected_message: expected flash message :param expected_category: expected flash category """ with self.client.session_transaction() as session: try: category, message = session['_flashes'][0] except KeyError: sentry.captureException() raise AssertionError('Nothing was flashed') assert expected_message in message assert expected_category == category
def redis_get_user_session(session_id): serialization_method = pickle session_class = KVSession try: s = session_class( serialization_method.loads( current_app.kvsession_store.get(session_id))) s.sid_s = session_id return s except KeyError: sentry.captureException() return None
def send_texts(): users = User.query.all() for user in users: if user.is_active and user.needs_message_now(): print "%s needs forecast" % user try: sent = user.send_forecast() if sent: print "sent forecast to %s" % user else: print "didn't send forecast to %s" % user except Exception as e: print e sentry.captureException()
def redis_get_user_session(session_id): serialization_method = pickle session_class = KVSession try: s = session_class(serialization_method.loads( current_app.kvsession_store.get(session_id) )) s.sid_s = session_id return s except KeyError: sentry.captureException() return None
def get_object(obj_type, obj_id): """ Safely retrieve a database record by its id and its sqlalchemy object type. """ if not obj_id: return None try: return obj_type.query.get(obj_id) except SQLAlchemyError: sentry.captureException() db.session.rollback() current_app.logger.exception('Error searching "{}" table for id {}'.format( obj_type.__tablename__, obj_id)) return None
def _sftp_get_mime_type(sftp, path): with TemporaryFile() as tmp: try: sftp.getfo(path, tmp, _raise_if_too_big) except MaxTransferSizeExceededException: sentry.captureException() tmp.seek(0) if current_app.config['MAGIC_FILE']: # Check using custom mime database file m = magic.Magic(magic_file=current_app.config['MAGIC_FILE'], mime=True) mime_type = m.from_buffer(tmp.read()) else: mime_type = magic.from_buffer(tmp.read(), mime=True) return mime_type
def delete_object(obj): """ Delete a database record. :param obj: object (instance of sqlalchemy model) to delete :return: was the record deleted successfully? """ try: db.session.delete(obj) db.session.commit() return True except SQLAlchemyError: sentry.captureException() db.session.rollback() current_app.logger.exception("Failed to DELETE {}".format(obj)) return False
def _sftp_get_mime_type(sftp, path): with TemporaryFile() as tmp: try: sftp.getfo(path, tmp, _raise_if_too_big) except MaxTransferSizeExceededException: sentry.captureException() tmp.seek(0) if current_app.config['MAGIC_FILE']: # Check using custom mime database file m = magic.Magic( magic_file=current_app.config['MAGIC_FILE'], mime=True) mime_type = m.from_buffer(tmp.read()) else: mime_type = magic.from_buffer(tmp.read(), mime=True) return mime_type
def resizer(img, remember): save_to = image_save_to(app.config['IMAGES_CACHE_DIR'], img.id) try: image = Imagenator(img.path) if img.strategy == 'fit': image.resize_fitin(img.size) else: image.resize_crop(img.size, 2) image.save(save_to, img.format.codec, img.quality) except Exception as e: sentry.captureException() raise ResizerError(e) return save_to
def edit(request_id): """ Updates a users permissions on a request and sends notification emails. Expects a request body containing the user's guid and updated permissions. Ex: { 'user': '******', 1: true, 5: false } :return: """ current_user_request = current_user.user_requests.filter_by( request_id=request_id).one() current_request = current_user_request.request if (current_user.is_agency and (current_user.is_super or current_user.is_agency_admin(current_request.agency.ein) or current_user_request.has_permission( permission.EDIT_USER_REQUEST_PERMISSIONS))): user_data = flask_request.form point_of_contact = True if role_name.POINT_OF_CONTACT in user_data else False required_fields = ['user'] for field in required_fields: if not user_data.get(field): flash('Uh Oh, it looks like the {} is missing! ' 'This is probably NOT your fault.'.format(field), category='danger') return redirect(url_for('request.view', request_id=request_id)) try: permissions = [int(i) for i in user_data.getlist('permission')] edit_user_request(request_id=request_id, user_guid=user_data.get('user'), permissions=permissions, point_of_contact=point_of_contact) except UserRequestException as e: sentry.captureException() flash(e, category='warning') return redirect(url_for('request.view', request_id=request_id)) return 'OK', 200 return abort(403)
def twitter_login_step2(): try: oauth_token = request.args.get('oauth_token', '') oauth_verifier = request.args.get('oauth_verifier', '') token = libforget.twitter.receive_verifier( oauth_token, oauth_verifier, **app.config.get_namespace("TWITTER_")) session = login(token.account_id) g.viewer = session return redirect(url_for('index')) except Exception: if sentry: sentry.captureException() return redirect( url_for('about', twitter_login_error='', _anchor='log_in'))
def validate_schema(data, schema_name): """ Validate the provided data against the provided JSON schema. :param data: JSON data to be validated :param schema_name: Name of the schema :return: Boolean """ with open(os.path.join(current_app.config['JSON_SCHEMA_DIRECTORY'], schema_name + '.schema'), 'r') as fp: schema = json.load(fp) try: validate(data, schema) return True except ValidationError as e: sentry.captureException() current_app.logger.info("Failed to validate {}\n{}".format(json.dumps(data), e)) return False
def bulk_delete(query): """ Delete multiple database records via a bulk delete query. http://docs.sqlalchemy.org/en/latest/orm/query.html#sqlalchemy.orm.query.Query.delete :param query: Query object :return: the number of records deleted """ try: num_deleted = query.delete() db.session.commit() return num_deleted except SQLAlchemyError: sentry.captureException() db.session.rollback() current_app.logger.exception("Failed to BULK DELETE {}".format(query)) return 0
def edit(request_id): """ Updates a users permissions on a request and sends notification emails. Expects a request body containing the user's guid and updated permissions. Ex: { 'user': '******', 1: true, 5: false } :return: """ current_user_request = current_user.user_requests.filter_by(request_id=request_id).one() current_request = current_user_request.request if ( current_user.is_agency and ( current_user.is_super or current_user.is_agency_admin(current_request.agency.ein) or current_user_request.has_permission(permission.EDIT_USER_REQUEST_PERMISSIONS) ) ): user_data = flask_request.form point_of_contact = True if role_name.POINT_OF_CONTACT in user_data else False required_fields = ['user'] for field in required_fields: if not user_data.get(field): flash('Uh Oh, it looks like the {} is missing! ' 'This is probably NOT your fault.'.format(field), category='danger') return redirect(url_for('request.view', request_id=request_id)) try: permissions = [int(i) for i in user_data.getlist('permission')] edit_user_request(request_id=request_id, user_guid=user_data.get('user'), permissions=permissions, point_of_contact=point_of_contact) except UserRequestException as e: sentry.captureException() flash(e, category='warning') return redirect(url_for('request.view', request_id=request_id)) return 'OK', 200 return abort(403)
def is_allowed(user: Users, request_id: str, permission: int): """ :param user: :param request_id: :param permissions: :return: """ try: user_request = user.user_requests.filter_by(request_id=request_id).one() return True if user_request.has_permission(permission) else False except NoResultFound: sentry.captureException() return False except AttributeError: sentry.captureException() return False
def send_texts(): # users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"]) users = User.query.filter_by(is_active=True) for user in users: if user.is_active: current_offset = user.time_zone suggested_offset = int(current_offset) + 1 message = "Hope you've found Rooster to be a helpful addition to your morning! The site costs $30/month to operate. I'd love your support: http://www.roosterapp.co/donate/" try: sent = user.send_message(message, "donation_request") if sent: print "sent request to %s" % user else: print "didn't send request to %s" % user except Exception as e: print e sentry.captureException()
def send_texts(): users = User.query.all(is_active=True) # users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"]) for user in users: if user.is_active: current_offset = user.time_zone suggested_offset = int(current_offset) + 1 message = "Don't let daylight savings time mess up your alarm!\n\nYour current timezone offset is %s\n\nIf DST has started for you, reply with \"TZ: %s\" to update." % (current_offset, suggested_offset) try: sent = user.send_message(message, "dst_warning") if sent: print "sent warning to %s" % user else: print "didn't send warning to %s" % user except Exception as e: print e sentry.captureException()
def _generate_signature(password, string): """ Generate an NYC.ID Web Services authentication signature using HMAC-SHA1 https://nyc4d.nycnet/nycid/web-services.shtml#signature :param password: NYC.ID Service Account password :param string: string to sign :return: the authentication signature or None on failure """ signature = None try: hmac_sha1 = hmac.new(key=password.encode(), msg=string.encode(), digestmod=sha1) signature = hmac_sha1.hexdigest() except Exception as e: sentry.captureException() current_app.logger.error("Failed to generate NYC ID.Web Services " "authentication signature: ", e) return signature
def send_texts(): # users = User.query.filter_by(phone=os.environ["TEST_PHONE_NUM"]) users = User.query.filter_by(is_active=True) for user in users: if user.is_active: message = "We haven't heard from you in a while, so we're temporarily deactivating your Rooster App account.\n\nReply 'START' to reactivate." try: sent = user.send_message(message, "deactivation") if sent: print "sent request to %s" % user else: print "didn't send request to %s" % user except Exception as e: print e sentry.captureException() user.is_active = False db.session.add(user) db.session.commit()
def handle_upload_no_id(file_field): """ Try to store and scan an uploaded file when no request id has been generated. Return the stored upload file path on success, otherwise add errors to the file field. :param file_field: form file field :return: the file path to the stored upload """ path = None valid_file_type, file_type = is_valid_file_type(file_field.data) if not valid_file_type: file_field.errors.append( "File type '{}' is not allowed.".format(file_type)) else: try: path = _quarantine_upload_no_id(file_field.data) except Exception as e: sentry.captureException() print("Error saving file {} : {}".format( file_field.data.filename, e)) file_field.errors.append('Error saving file.') else: try: scan_file(path) except VirusDetectedException: sentry.captureException() file_field.errors.append('File is infected.') except Exception: sentry.captureException() file_field.errors.append('Error scanning file.') return path
def update_object(data, obj_type, obj_id, es_update=True): """ Update a database record and its elasticsearch counterpart. :param data: a dictionary of attribute-value pairs :param obj_type: sqlalchemy model :param obj_id: id of record :param es_update: update the elasticsearch index :return: was the record updated successfully? """ obj = get_object(obj_type, obj_id) if obj: for attr, value in data.items(): if isinstance(value, dict): # update json values attr_json = getattr(obj, attr) or {} for key, val in value.items(): attr_json[key] = val setattr(obj, attr, attr_json) flag_modified(obj, attr) else: setattr(obj, attr, value) try: db.session.commit() except SQLAlchemyError: sentry.captureException() db.session.rollback() current_app.logger.exception("Failed to UPDATE {}".format(obj)) else: # update elasticsearch if hasattr(obj, 'es_update') and current_app.config['ELASTICSEARCH_ENABLED'] and es_update: obj.es_update() return True return False
def current_version(): try: return check_output(['git', 'describe', '--always']) except CalledProcessError: sentry.captureException() return ""
def requests_doc(doc_type): """ Converts and sends the a search result-set as a file of the specified document type. - Filtering on set size is ignored; all results are returned. - Currently only supports CSVs. - CSV only includes requests belonging to that user's agency Document name format: "FOIL_requests_results_<timestamp:MM_DD_YYYY_at_HH_mm_pp>" Request parameters are identical to those of /search/requests. :param doc_type: document type ('csv' only) """ if current_user.is_agency and doc_type.lower() == 'csv': try: agency_ein = request.args.get('agency_ein', '') except ValueError: sentry.captureException() agency_ein = None tz_name = request.args.get('tz_name', current_app.config['APP_TIMEZONE']) start = 0 buffer = StringIO() # csvwriter cannot accept BytesIO writer = csv.writer(buffer) writer.writerow(["FOIL ID", "Agency", "Title", "Description", "Agency Request Summary", "Current Status", "Date Created", "Date Received", "Date Due", "Date Closed", "Requester Name", "Requester Email", "Requester Title", "Requester Organization", "Requester Phone Number", "Requester Fax Number", "Requester Address 1", "Requester Address 2", "Requester City", "Requester State", "Requester Zipcode", "Assigned User Emails"]) results = search_requests( query=request.args.get('query'), foil_id=eval_request_bool(request.args.get('foil_id')), title=eval_request_bool(request.args.get('title')), agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')), description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False, requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False, date_rec_from=request.args.get('date_rec_from'), date_rec_to=request.args.get('date_rec_to'), date_due_from=request.args.get('date_due_from'), date_due_to=request.args.get('date_due_to'), date_closed_from=request.args.get('date_closed_from'), date_closed_to=request.args.get('date_closed_to'), agency_ein=agency_ein, agency_user_guid=request.args.get('agency_user'), open_=eval_request_bool(request.args.get('open')), closed=eval_request_bool(request.args.get('closed')), in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False, due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False, overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False, start=start, sort_date_received=request.args.get('sort_date_submitted'), sort_date_due=request.args.get('sort_date_due'), sort_title=request.args.get('sort_title'), tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE']), for_csv=True ) ids = [result["_id"] for result in results] all_requests = Requests.query.filter(Requests.id.in_(ids)).options( joinedload(Requests.agency_users)).options(joinedload(Requests.requester)).options( joinedload(Requests.agency)).all() user_agencies = current_user.get_agencies for req in all_requests: if req.agency_ein in user_agencies: writer.writerow([ req.id, req.agency.name, req.title, req.description, req.agency_request_summary, req.status, req.date_created, req.date_submitted, req.due_date, req.date_closed, req.requester.name, req.requester.email, req.requester.title, req.requester.organization, req.requester.phone_number, req.requester.fax_number, req.requester.mailing_address.get('address_one'), req.requester.mailing_address.get('address_two'), req.requester.mailing_address.get('city'), req.requester.mailing_address.get('state'), req.requester.mailing_address.get('zip'), ", ".join(u.email for u in req.agency_users)]) dt = datetime.utcnow() timestamp = utc_to_local(dt, tz_name) if tz_name is not None else dt return send_file( BytesIO(buffer.getvalue().encode('UTF-8')), # convert to bytes attachment_filename="FOIL_requests_results_{}.csv".format( timestamp.strftime("%m_%d_%Y_at_%I_%M_%p")), as_attachment=True ) return '', 400
def post(request_id): """ Create a new upload. Handles chunked files through the Content-Range header. For filesize validation and more upload logic, see: /static/js/upload/fileupload.js Optional request body parameters: - update (bool) save the uploaded file to the 'updated' directory (this indicates the file is meant to replace a previously uploaded file) - response_id (int) the id of a response associated with the file this upload is replacing - REQUIRED if 'update' is 'true' - ignored if 'update' is 'false' :returns: { "name": file name, "size": file size } """ files = request.files file_ = files[next(files.keys())] filename = secure_filename(file_.filename) is_update = eval_request_bool(request.form.get('update')) agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \ is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE): response_id = request.form.get('response_id') if is_update else None if upload_exists(request_id, filename, response_id): response = { "files": [{ "name": filename, "error": "A file with this name has already " "been uploaded for this request." # TODO: "link": <link-to-existing-file> ? would be nice }] } else: upload_path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], request_id) if not os.path.exists(upload_path): os.mkdir(upload_path) filepath = os.path.join(upload_path, filename) key = get_upload_key(request_id, filename, is_update) try: if CONTENT_RANGE_HEADER in request.headers: start, size = parse_content_range( request.headers[CONTENT_RANGE_HEADER]) # Only validate mime type on first chunk valid_file_type = True file_type = None if start == 0: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if os.path.exists(filepath): # remove existing file (upload 'restarted' for same file) os.remove(filepath) if valid_file_type: redis.set(key, upload_status.PROCESSING) with open(filepath, 'ab') as fp: fp.seek(start) fp.write(file_.stream.read()) # scan if last chunk written if os.path.getsize(filepath) == size: scan_and_complete_upload.delay(request_id, filepath, is_update, response_id) else: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if valid_file_type: redis.set(key, upload_status.PROCESSING) file_.save(filepath) scan_and_complete_upload.delay(request_id, filepath, is_update, response_id) if not valid_file_type: response = { "files": [{ "name": filename, "error": "The file type '{}' is not allowed.".format( file_type) }] } else: response = { "files": [{ "name": filename, "original_name": file_.filename, "size": os.path.getsize(filepath), }] } except Exception as e: sentry.captureException() redis.set(key, upload_status.ERROR) current_app.logger.exception("Upload for file '{}' failed: {}".format(filename, e)) response = { "files": [{ "name": filename, "error": "There was a problem uploading this file." }] } return jsonify(response), 200
def delete(r_id_type, r_id, filecode): """ Removes an uploaded file. :param r_id_type: "response" or "request" :param r_id: the Response or Request identifier :param filecode: the encoded name of the uploaded file (base64 without padding) Optional request body parameters: - quarantined_only (bool) only delete the file if it is quarantined (beware: takes precedence over 'updated_only') - updated_only (bool) only delete the file if it is in the 'updated' directory :returns: On success: { "deleted": filename } On failure: { "error": error message } """ filename = secure_filename(b64decode_lenient(filecode)) if r_id_type not in ["request", "response"]: response = {"error": "Invalid ID type."} else: try: if r_id_type == "response": response = Responses.query.filter_by(id=r_id, deleted=False) r_id = response.request_id path = '' quarantined_only = eval_request_bool(request.form.get('quarantined_only')) has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE)) if quarantined_only and has_add_edit: path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], r_id ) elif eval_request_bool(request.form.get('updated_only')) and \ is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE): path = os.path.join( current_app.config['UPLOAD_DIRECTORY'], r_id, UPDATED_FILE_DIRNAME ) else: path_for_status = { upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.READY: current_app.config['UPLOAD_DIRECTORY'] } status = redis.get(get_upload_key(r_id, filename)) if status is not None: dest_path = path_for_status[status.decode("utf-8")] if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit) or ( dest_path == current_app.config['UPLOAD_DIRECTORY'] and is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) ): path = os.path.join( dest_path, r_id ) filepath = os.path.join(path, filename) found = False if path != '': if quarantined_only: if os.path.exists(filepath): os.remove(filepath) found = True else: if fu.exists(filepath): fu.remove(filepath) found = True if found: response = {"deleted": filename} else: response = {"error": "Upload not found."} except Exception as e: sentry.captureException() current_app.logger.exception("Error on DELETE /upload/: {}".format(e)) response = {"error": "Failed to delete '{}'".format(filename)} return jsonify(response), 200
def view(request_id): """ This function is for testing purposes of the view a request back until backend functionality is implemented. :return: redirect to view request page """ try: current_request = Requests.query.filter_by(id=request_id).one() assert current_request.agency.is_active except NoResultFound: print("Request with id '{}' does not exist.".format(request_id)) sentry.captureException() return abort(404) except AssertionError: print("Request belongs to inactive agency.") sentry.captureException() return abort(404) holidays = sorted(get_holidays_date_list( datetime.utcnow().year, (datetime.utcnow() + rd(years=DEFAULT_YEARS_HOLIDAY_LIST)).year) ) active_users = [] assigned_users = [] if current_user.is_agency: for agency_user in current_request.agency.active_users: if not agency_user in current_request.agency.administrators and (agency_user != current_user): # populate list of assigned users that can be removed from a request if agency_user in current_request.agency_users: assigned_users.append(agency_user) # append to list of active users that can be added to a request else: active_users.append(agency_user) permissions = { 'acknowledge': permission.ACKNOWLEDGE, 'deny': permission.DENY, 'extend': permission.EXTEND, 'close': permission.CLOSE, 're_open': permission.RE_OPEN, 'add_file': permission.ADD_FILE, 'edit_file_privacy': permission.EDIT_FILE_PRIVACY, 'delete_file': permission.DELETE_FILE, 'add_note': permission.ADD_NOTE, 'edit_note_privacy': permission.EDIT_NOTE_PRIVACY, 'delete_note': permission.DELETE_NOTE, 'add_link': permission.ADD_LINK, 'edit_link_privacy': permission.EDIT_LINK_PRIVACY, 'delete_link': permission.DELETE_LINK, 'add_instructions': permission.ADD_OFFLINE_INSTRUCTIONS, 'edit_instructions_privacy': permission.EDIT_OFFLINE_INSTRUCTIONS_PRIVACY, 'delete_instructions': permission.DELETE_OFFLINE_INSTRUCTIONS, 'generate_letter': permission.GENERATE_LETTER, 'add_user': permission.ADD_USER_TO_REQUEST, 'edit_user': permission.EDIT_USER_REQUEST_PERMISSIONS, 'remove_user': permission.REMOVE_USER_FROM_REQUEST, 'edit_title': permission.EDIT_TITLE, 'edit_title_privacy': permission.CHANGE_PRIVACY_TITLE, 'edit_agency_request_summary': permission.EDIT_AGENCY_REQUEST_SUMMARY, 'edit_agency_request_summary_privacy': permission.CHANGE_PRIVACY_AGENCY_REQUEST_SUMMARY, 'edit_requester_info': permission.EDIT_REQUESTER_INFO } # Build permissions dictionary for checking on the front-end. for key, val in permissions.items(): if current_user.is_anonymous or not current_request.user_requests.filter_by( user_guid=current_user.guid).first(): permissions[key] = False else: permissions[key] = is_allowed(current_user, request_id, val) if not current_user.is_anonymous else False # Build dictionary of current permissions for all assigned users. assigned_user_permissions = {} for u in assigned_users: assigned_user_permissions[u.guid] = UserRequests.query.filter_by( request_id=request_id, user_guid=u.guid).one().get_permission_choice_indices() point_of_contact = get_current_point_of_contact(request_id) if point_of_contact: current_point_of_contact = {'user_guid': point_of_contact.user_guid} else: current_point_of_contact = {'user_guid': ''} # Determine if the Agency Request Summary should be shown. show_agency_request_summary = False if current_user in current_request.agency_users \ or current_request.agency_request_summary \ and (current_request.requester == current_user and current_request.status == request_status.CLOSED and not current_request.privacy['agency_request_summary'] or current_request.status == request_status.CLOSED and current_request.agency_request_summary_release_date and current_request.agency_request_summary_release_date < datetime.utcnow() and not current_request.privacy['agency_request_summary']): show_agency_request_summary = True # Determine if the title should be shown. show_title = (current_user in current_request.agency_users or current_request.requester == current_user or not current_request.privacy['title']) # Determine if "Generate Letter" functionality is enabled for the agency. if 'letters' in current_request.agency.agency_features: generate_letters_enabled = current_request.agency.agency_features['letters']['generate_letters'] else: generate_letters_enabled = False # Determine if custom request forms are enabled if 'enabled' in current_request.agency.agency_features['custom_request_forms']: custom_request_forms_enabled = current_request.agency.agency_features['custom_request_forms']['enabled'] else: custom_request_forms_enabled = False # Determine if custom request form panels should be expanded by default if 'expand_by_default' in current_request.agency.agency_features['custom_request_forms']: expand_by_default = current_request.agency.agency_features['custom_request_forms']['expand_by_default'] else: expand_by_default = False # Determine if request description should be hidden when custom forms are enabled if 'description_hidden_by_default' in current_request.agency.agency_features['custom_request_forms']: description_hidden_by_default = current_request.agency.agency_features['custom_request_forms']['description_hidden_by_default'] else: description_hidden_by_default = False return render_template( 'request/view_request.html', request=current_request, status=request_status, agency_users=current_request.agency_users, edit_requester_form=EditRequesterForm(current_request.requester), contact_agency_form=ContactAgencyForm(current_request), deny_request_form=DenyRequestForm(current_request.agency.ein), close_request_form=CloseRequestForm(current_request.agency.ein), reopen_request_form=ReopenRequestForm(current_request.agency.ein), remove_user_request_form=RemoveUserRequestForm(assigned_users), add_user_request_form=AddUserRequestForm(active_users), edit_user_request_form=EditUserRequestForm(assigned_users), generate_acknowledgment_letter_form=GenerateAcknowledgmentLetterForm(current_request.agency.ein), generate_denial_letter_form=GenerateDenialLetterForm(current_request.agency.ein), generate_closing_letter_form=GenerateClosingLetterForm(current_request.agency.ein), generate_extension_letter_form=GenerateExtensionLetterForm(current_request.agency.ein), generate_envelope_form=GenerateEnvelopeForm(current_request.agency_ein, current_request.requester), generate_response_letter_form=GenerateResponseLetterForm(current_request.agency.ein), assigned_user_permissions=assigned_user_permissions, current_point_of_contact=current_point_of_contact, holidays=holidays, assigned_users=assigned_users, active_users=active_users, permissions=permissions, show_agency_request_summary=show_agency_request_summary, show_title=show_title, is_requester=(current_request.requester == current_user), permissions_length=len(permission.ALL), generate_letters_enabled=generate_letters_enabled, custom_request_forms_enabled = custom_request_forms_enabled, expand_by_default=expand_by_default, description_hidden_by_default=description_hidden_by_default )
def requests(): """ For request parameters, see app.search.utils.search_requests All Users can search by: - FOIL ID Anonymous Users can search by: - Title (public only) - Agency Request Summary (public only) Public Users can search by: - Title (public only OR public and private if user is requester) - Agency Request Summary (public only) - Description (if user is requester) Agency Users can search by: - Title - Agency Request Summary - Description - Requester Name All Users can filter by: - Status, Open (anything not Closed if not agency user) - Status, Closed - Date Submitted - Agency Only Agency Users can filter by: - Status, In Progress - Status, Due Soon - Status, Overdue - Date Due """ try: agency_ein = request.args.get('agency_ein', '') except ValueError: sentry.captureException() agency_ein = None try: size = int(request.args.get('size', DEFAULT_HITS_SIZE)) except ValueError: sentry.captureException() size = DEFAULT_HITS_SIZE try: start = int(request.args.get('start'), 0) except ValueError: sentry.captureException() start = 0 query = request.args.get('query') # Determine if searching for FOIL ID foil_id = eval_request_bool(request.args.get('foil_id')) or re.match(r'^(FOIL-|foil-|)\d{4}-\d{3}-\d{5}$', query) results = search_requests( query=query, foil_id=foil_id, title=eval_request_bool(request.args.get('title')), agency_request_summary=eval_request_bool(request.args.get('agency_request_summary')), description=eval_request_bool(request.args.get('description')) if not current_user.is_anonymous else False, requester_name=eval_request_bool(request.args.get('requester_name')) if current_user.is_agency else False, date_rec_from=request.args.get('date_rec_from'), date_rec_to=request.args.get('date_rec_to'), date_due_from=request.args.get('date_due_from'), date_due_to=request.args.get('date_due_to'), date_closed_from=request.args.get('date_closed_from'), date_closed_to=request.args.get('date_closed_to'), agency_ein=agency_ein, agency_user_guid=request.args.get('agency_user'), open_=eval_request_bool(request.args.get('open')), closed=eval_request_bool(request.args.get('closed')), in_progress=eval_request_bool(request.args.get('in_progress')) if current_user.is_agency else False, due_soon=eval_request_bool(request.args.get('due_soon')) if current_user.is_agency else False, overdue=eval_request_bool(request.args.get('overdue')) if current_user.is_agency else False, size=size, start=start, sort_date_received=request.args.get('sort_date_submitted'), sort_date_due=request.args.get('sort_date_due'), sort_title=request.args.get('sort_title'), tz_name=request.args.get('tz_name', current_app.config['APP_TIMEZONE']) ) # format results total = results["hits"]["total"] formatted_results = None if total != 0: convert_dates(results) formatted_results = render_template("request/result_row.html", requests=results["hits"]["hits"]) # query=query) # only for testing return jsonify({ "count": len(results["hits"]["hits"]), "total": total, "results": formatted_results }), 200