def errorhandler(e): """Handle error""" if not isinstance(e, HTTPException): e = InternalServerError() flash("Unexpected Error") return redirect(request.referrer)
def response_500(self, error=InternalServerError()): if isinstance(error, HTTPException): return self.error(error=error) return self.error(error=InternalServerError())
def run_wsgi(self): if self.headers.get('Expect', '').lower().strip() == '100-continue': self.wfile.write(b'HTTP/1.1 100 Continue\r\n\r\n') self.environ = environ = self.make_environ() headers_set = [] headers_sent = [] def write(data): assert headers_set, 'write() before start_response' if not headers_sent: status, response_headers = headers_sent[:] = headers_set try: code, msg = status.split(None, 1) except ValueError: code, msg = status, "" self.send_response(int(code), msg) header_keys = set() for key, value in response_headers: self.send_header(key, value) key = key.lower() header_keys.add(key) if not ('content-length' in header_keys or environ['REQUEST_METHOD'] == 'HEAD' or status < 200 or status in (204, 304)): self.close_connection = True self.send_header('Connection', 'close') if 'server' not in header_keys: self.send_header('Server', self.version_string()) if 'date' not in header_keys: self.send_header('Date', self.date_time_string()) self.end_headers() assert isinstance(data, bytes), 'applications must write bytes' self.wfile.write(data) self.wfile.flush() def start_response(status, response_headers, exc_info=None): if exc_info: try: if headers_sent: reraise(*exc_info) finally: exc_info = None elif headers_set: raise AssertionError('Headers already set') headers_set[:] = [status, response_headers] return write def execute(app): application_iter = app(environ, start_response) try: for data in application_iter: write(data) if not headers_sent: write(b'') finally: if hasattr(application_iter, 'close'): application_iter.close() application_iter = None try: execute(self.server.app) except (socket.error, socket.timeout) as e: self.connection_dropped(e, environ) except Exception: if self.server.passthrough_errors: raise from werkzeug.debug.tbtools import get_current_traceback traceback = get_current_traceback(ignore_system_exceptions=True) try: # if we haven't yet sent the headers but they are set # we roll back to be able to set them again. if not headers_sent: del headers_set[:] execute(InternalServerError()) except Exception: pass self.server.log('error', 'Error on request:\n%s', traceback.plaintext)
try: execute(app) except (socket.error, socket.timeout), e: self.connection_dropped(e, environ) except: if self.server.passthrough_errors: raise from werkzeug.debug.tbtools import get_current_traceback traceback = get_current_traceback(ignore_system_exceptions=True) try: # if we haven't yet sent the headers but they are set # we roll back to be able to set them again. if not headers_sent: del headers_set[:] execute(InternalServerError()) except: pass self.server.log('error', 'Error on request:\n%s', traceback.plaintext) def handle(self): """Handles a request ignoring dropped connections.""" try: return BaseHTTPRequestHandler.handle(self) except (socket.error, socket.timeout), e: self.connection_dropped(e) except: if self.server.ssl_context is None or not is_ssl_error(): raise
def why_me(): raise InternalServerError("Why me?")
def expect_content_type(value: str): content_type = request.headers["Content-Type"] content_type, _ = cgi.parse_header(content_type) if content_type != value: raise InternalServerError(f"Expected {value} payload")
def _get_playable_recommendations_list(mbids_and_ratings_list): """ Get artist, track etc info from recording mbid using labs.listenbrainz.api so that they can be played using BrainzPlayer. Refer to webserver/static/js/src/BrainzPlayer.tsx Args: mbids_and_ratings_list: Contains recording mbid and corresponding score. Returns: recommendations: list of recommendations of the format { 'listened_at' : 0, 'track_metadata' : { 'artist_name' : 'John Mayer', 'track_name' : 'Edge of desire', 'release_name' : "", 'additional_info' : { 'recording_mbid' : "181c4177-f33a-441d-b15d-910acaf18b07", 'artist_mbids' : "181c4177-f33a-441d-b15d-910acaf18b07" } } } """ data = [] for r in mbids_and_ratings_list: data.append({'[recording_mbid]': r['recording_mbid']}) r = requests.post(SERVER_URL, json=data) if r.status_code != 200: if r.status_code == 400: current_app.logger.error( 'Invalid data was sent to the labs API.\nData: {}'.format( data)) raise BadRequest else: current_app.logger.error( "API didn't send a valid response due to Internal Server Error.\nData: {}" .format(data)) raise InternalServerError try: rows = ujson.loads(r.text) except Exception as err: raise InternalServerError(str(err)) recommendations = [] for row in rows: recommendations.append({ 'listened_at': 0, 'track_metadata': { 'artist_name': row['artist_credit_name'], 'track_name': row['recording_name'], 'release_name': row.get('release_name', ""), 'additional_info': { 'recording_mbid': row['recording_mbid'], 'artist_mbids': row['[artist_credit_mbids]'] } } }) return recommendations
def patch_version(self): if self.sversion: return int(int(self.sversion / 100) / 100) raise InternalServerError(self._INFORMATION_MSG)
def errorhandler(e): # Handle error if not isinstance(e, HTTPException): e = InternalServerError() return error(e.name, e.code)
def generate_quiz_leaf(node, weight_depth, blacklisted_leaf_id=None): node_data = get_depth_popularity_info(node) weight_high_depth_query = "power(((l.popularity -:min_popularity + 1) / (:max_popularity - :min_popularity)) + 4 * ((q.depth - :min_depth) / (:max_depth - :min_depth)), 0.5)" weight_low_depth_query = "power(((l.popularity - :min_popularity + 1) / (:max_popularity - :min_popularity)) + 4 * (1 -((q.depth - :min_depth) / (:max_depth - :min_depth))), 0.5)" score_query = weight_high_depth_query if weight_depth else weight_low_depth_query leaf_response = db.session.execute( text( "select distinct l.id, l.ott, l.name, vernacular_by_ott.vernacular, iucn.status_code, images_by_ott.src, images_by_ott.src_id, " + score_query + " as score, q.depth, l.popularity, l.wikidata, l.eol" """ from ordered_leaves l join quiz_leaves_by_ott q on q.leaf_id = l.id join images_by_ott ON (l.ott = images_by_ott.ott AND images_by_ott.best_any = 1) left join vernacular_by_ott on (l.ott = vernacular_by_ott.ott and vernacular_by_ott.lang_primary = 'en' and vernacular_by_ott.preferred = 1) left join iucn on l.ott = iucn.ott where l.id between :leaf_left and :leaf_right and best_any = 1 and l.id != :blacklisted_leaf_id group by l.ott order by rand() * score desc limit 1 """), { "leaf_left": node["leaf_left"], "leaf_right": node["leaf_right"], "min_depth": node_data["min_depth"], "max_depth": node_data["max_depth"], "min_popularity": node_data["min_popularity"], "max_popularity": node_data["max_popularity"], "blacklisted_leaf_id": blacklisted_leaf_id or 0, }, ).fetchall() if not leaf_response: print("Leaf generation failed.") print(node, node_data, blacklisted_leaf_id) raise InternalServerError("Leaf generation failed.") leaf = dict( zip( [ "id", "ott", "name", "vernacular", "iucn", "img_src", "img_src_id", "score", "depth", "popularity", "wikidata", "eol", ], leaf_response[0], )) leaf["thumbnail"] = make_thumbnail_url(leaf["img_src"], leaf["img_src_id"]) return leaf
def major_version(self): if self.sversion is not None: return int(self.sversion / 10000) raise InternalServerError(self._INFORMATION_MSG)
def generate_quiz_question(quiz): node = (db.session.query( Node.id, Node.node_rgt).where(Node.ott == quiz["ott"]).first()) # Find a parent node which has at least two direct valid quiz nodes parent_node = (db.session.query(Node.id, Node.ott, Node.name).join( QuizNode, QuizNode.node_id == Node.id).where( between( Node.id, node.id, node.node_rgt)).where(QuizNode.can_branch == True).order_by( desc(func.pow(Node.popularity, 0.1) * func.rand())).first()) if not parent_node: raise InternalServerError( f"No suitable parent nodes for the ott: {quiz['ott']}") # Randomly pick two valid child nodes which will become node_left and node_right result = db.session.execute( text(""" SELECT n.id, n.leaf_lft, n.leaf_rgt FROM ordered_nodes n JOIN quiz_nodes q on q.node_id = n.id WHERE n.real_parent = :parent_node_id AND q.num_quiz_leaves > 1 ORDER BY rand() LIMIT 2 """), { "parent_node_id": parent_node.id }, ).fetchall() if len(result) < 2: print("Selected parent node has fewer than two valid node children.") print(parent_node._asdict()) raise InternalServerError() node_left_result, node_right_result = result node_left = dict(zip(["id", "leaf_left", "leaf_right"], node_left_result)) node_right = dict(zip(["id", "leaf_left", "leaf_right"], node_right_result)) leaf_left_1 = generate_quiz_leaf(node_left, False) leaf_left_2 = generate_quiz_leaf(node_left, True, leaf_left_1["id"]) leaf_right = generate_quiz_leaf(node_right, True) swap = random() > 0.5 question = { "compare": leaf_left_1, "option1": leaf_left_2 if swap else leaf_right, "option2": leaf_right if swap else leaf_left_2, } save_question_to_db( quiz_id=quiz["id"], leaf_compare=question["compare"], leaf_1=question["option1"], leaf_2=question["option2"], ) return question
def raiseInternalServerError(): raise InternalServerError('Hello world')
def post(self): if request.headers.get('Tus-Resumable') is None: raise BadRequest( 'Received file upload for unsupported file transfer protocol') file_size = request.headers.get('Upload-Length') max_file_size = current_app.config["MAX_CONTENT_LENGTH"] if not file_size: raise BadRequest('Received file upload of unspecified size') file_size = int(file_size) if file_size > max_file_size: raise RequestEntityTooLarge( f'The maximum file upload size is {max_file_size/1024/1024}MB.' ) data = self.parser.parse_args() filename = data.get('filename') if not filename: raise BadRequest('File name cannot be empty') if filename.endswith(FORBIDDEN_FILETYPES): raise BadRequest('File type is forbidden') document_guid = str(uuid.uuid4()) base_folder = current_app.config['UPLOADED_DOCUMENT_DEST'] folder = data.get('folder') folder = os.path.join(base_folder, folder) file_path = os.path.join(folder, document_guid) pretty_folder = data.get('pretty_folder') pretty_path = os.path.join(base_folder, pretty_folder, filename) try: if not os.path.exists(folder): os.makedirs(folder) with open(file_path, "wb") as f: f.seek(file_size - 1) f.write(b"\0") except IOError as e: raise InternalServerError('Unable to create file') cache.set(FILE_UPLOAD_SIZE(document_guid), file_size, TIMEOUT_24_HOURS) cache.set(FILE_UPLOAD_OFFSET(document_guid), 0, TIMEOUT_24_HOURS) cache.set(FILE_UPLOAD_PATH(document_guid), file_path, TIMEOUT_24_HOURS) document_info = DocumentManager( document_guid=document_guid, full_storage_path=file_path, upload_started_date=datetime.utcnow(), file_display_name=filename, path_display_name=pretty_path, ) document_info.save() response = make_response(jsonify(document_manager_guid=document_guid), 201) response.headers['Tus-Resumable'] = TUS_API_VERSION response.headers['Tus-Version'] = TUS_API_SUPPORTED_VERSIONS response.headers[ 'Location'] = f'{current_app.config["DOCUMENT_MANAGER_URL"]}/document-manager/{document_guid}' response.headers['Upload-Offset'] = 0 response.headers[ 'Access-Control-Expose-Headers'] = "Tus-Resumable,Tus-Version,Location,Upload-Offset" response.autocorrect_location_header = False return response
def errorhandler(e): """Handle error""" if not isinstance(e, HTTPException): e = InternalServerError()
def post(self): """ Create a new embedding """ entity = request.get_json() # looks weird but is the only reliable way to find out if a string value is a true boolean ;-) # see https://stackoverflow.com/questions/715417/converting-from-a-string-to-boolean-in-python handle_async = request.args.get('async', "", type=str).lower() in yes_list try: entity['created'] = datetime.datetime.now( datetime.timezone.utc).isoformat() b64_graph_str = entity.get('graph') try: graph_str = base64.b64decode(b64_graph_str) node_ids, edges = get_nodes_and_edges_from_graph( graph_str) # node_ids ==> List(str) # edges ==> List(Edge) except Exception as e: app.logger.exception(e) raise BadRequest( "The graph string has to be a base64 encoded graphml string! " "The exact error was: " + str(e)) len_nodes = len(node_ids) # Number of nodes len_edges = len(edges) # Number of edges if len_edges > 1900 or len_nodes > 600: raise BadRequest( "For fairness reasons this API will only handle graphs with less than 300 vertices and 900 " "edges. Your graph has {} vertices and {} edges which exceed the limit." "".format(len_nodes, len_edges)) # Check if self loops are present! We do not support self loops for e in edges: if e.source == e.target: raise BadRequest( "The Implementation only supports graphs where " "every edge has two distinct start and end nodes" ) # ignore double edges # # validate for no double edges # all_edge_endpoints = [{e.source, e.target} for e in edges] # duplicate_edges = get_duplicates(all_edge_endpoints) # if len(duplicate_edges) > 0: # abort(400, # "Multiedges are not allowed. " # "The following edges were recognized as duplicate {}".format(duplicate_edges)) # validate for unique edge ids duplicate_edge_ids = get_duplicates([e.id for e in edges]) if len(duplicate_edge_ids) > 0: abort( 400, "Edge ids have to be unique" "The following ids were recognized as duplicate {}" .format(duplicate_edge_ids)) # validate page id uniqueness page_ids = [p['id'] for p in entity.get('pages')] duplicate_page_ids = get_duplicates(page_ids) if len(duplicate_page_ids) > 0: abort( 400, "Duplicated page ids are not allowed. " "The following id were recognized as duplicate {}". format(duplicate_page_ids)) entity['status'] = 'IN_PROGRESS' entity = data_store.insert_new_element( entity) # entity id is returned here # validate graph not empty if len(page_ids) == 0 or len_edges == 0 or len_nodes == 0: abort( 400, "Please submit a graph with at least one node, edge and page" ) if handle_async: # abort(501, "Async handling is not enabled.") future_result: ProcessFuture = pool.schedule( SolverInterface.solve, (node_ids, edges, entity.get('pages'), entity.get('constraints'), entity['id'])) future_result.add_done_callback( processing_finished_callback) future_result.done() # remove old futures remove_old_jobs() jobs.append(QueueItem(entity.get('id'), future_result)) else: try: entity = handle_solver_result( SolverInterface.solve( node_ids, edges, entity.get('pages'), entity.get('constraints'), entity['id'])) except Exception as e1: error_callback(e1) entity = data_store.get_by_id(entity['id']) return jsonify(entity) except HTTPException as e: raise e except Exception as e: raise InternalServerError( "The error {} \noccured from this body \n{}".format( str(e), request.get_data(as_text=True))) from e
def handle_http_exception(error): """Return error description and details in response body This function is registered at init to handle HTTPException. When abort is called in the code, this triggers a HTTPException, and Flask calls this handler to generate a better response. Also, when an Exception is not caught in a view, Flask automatically calls the 500 error handler. flask_rest_api republishes webarg's abort override. This abort allows the caller to pass kwargs and stores those kwargs in exception.data. This handler uses this extra information to populate the response. Extra information considered by this handler: - message: a comment (string) - errors: a dict of errors, typically validation issues on a form - headers: a dict of additional headers When the error was triggered with 'abort', it is logged with INFO level. """ # TODO: manage case where messages/errors is a list? # TODO: use an error Schema # TODO: make log optional? log_info = True # Flask redirects unhandled exceptions to error 500 handler # If error is not a HTTPException, then it is an unhandled exception # Return a 500 (InternalServerError) if not isinstance(error, HTTPException): error = InternalServerError() # Flask logs uncaught exceptions as ERROR already, no need to log here log_info = False headers = {} payload = { 'status': str(error), } # Get additional info passed as kwargs when calling abort # data may not exist if HTTPException was raised not using webargs abort # or if not kwargs were passed (https://github.com/sloria/webargs/pull/184) data = getattr(error, 'data', None) if data: # If we passed a custom message if 'message' in data: payload['message'] = data['message'] # If we passed "errors" if 'errors' in data: payload['errors'] = data['errors'] # If webargs added validation errors as "messages" # (you should use 'errors' as it is more explicit) elif 'messages' in data: payload['errors'] = data['messages'] # If we passed additional headers if 'headers' in data: headers = data['headers'] # Log error as INFO, including payload content if log_info: log_string_content = [ str(error.code), ] for key in ('message', 'errors'): if key in payload: log_string_content.append(str(payload[key])) current_app.logger.info(' '.join(log_string_content)) return jsonify(payload), error.code, headers
def search(request_params: MultiDict) -> Response: """ Perform a search from the advanced search interface. This is intended to support ONLY form-based search, to replace the classic advanced search view. Parameters ---------- request_params : dict Returns ------- dict Response content. int HTTP status code. dict Extra headers to add to the response. Raises ------ InternalServerError Raised when there is an unrecoverable error while interacting with the search index. """ # We may need to intervene on the request parameters, so we'll # reinstantiate as a mutable MultiDict. if isinstance(request_params, ImmutableMultiDict): request_params = MultiDict(request_params.items(multi=True)) logger.debug("search request from advanced form") response_data: Dict[str, Any] = {} response_data["show_form"] = "advanced" not in request_params logger.debug("show_form: %s", str(response_data["show_form"])) # Here we intervene on the user's query to look for holdouts from # the classic search system's author indexing syntax (surname_f). We # rewrite with a comma, and show a warning to the user about the # change. has_classic = False for key, value in request_params.items(): if value is None: continue match = TERM_FIELD_PTN.search(key) if match is None: continue value = str(value) i = match.group(1) field = request_params.get(f"terms-{i}-field") # We are only looking for this syntax in the author search, or # in an all-fields search. if field not in ["all", "author"]: continue value, _has_classic = catch_underscore_syntax(value) has_classic = _has_classic if not has_classic else has_classic request_params.setlist(key, [value]) response_data["has_classic_format"] = has_classic form = forms.AdvancedSearchForm(request_params) q: Optional[Query] # We want to avoid attempting to validate if no query has been entered. # If a query was actually submitted via the form, 'advanced' will be # present in the request parameters. if "advanced" in request_params: if form.validate(): logger.debug("form is valid") q = _query_from_form(form) # Pagination is handled outside of the form. q = paginate(q, request_params) try: # Execute the search. We'll use the results directly in # template rendering, so they get added directly to the # response content. asdict( response_data.update(SearchSession.search(q)) # type: ignore except index.IndexConnectionError as ex: # There was a (hopefully transient) connection problem. Either # this will clear up relatively quickly (next request), or # there is a more serious outage. logger.error("IndexConnectionError: %s", ex) raise InternalServerError( "There was a problem connecting to the search index. This " "is quite likely a transient issue, so please try your " "search again. If this problem persists, please report it " "to [email protected].") from ex except index.QueryError as ex: # Base exception routers should pick this up and show bug page. logger.error("QueryError: %s", ex) raise InternalServerError( "There was a problem executing your query. Please try " "your search again. If this problem persists, please " "report it to [email protected].") from ex except index.OutsideAllowedRange as ex: raise BadRequest( "Hello clever friend. You can't get results in that range" " right now.") from ex response_data["query"] = q else: logger.debug("form is invalid: %s", str(form.errors)) if "order" in form.errors or "size" in form.errors: # It's likely that the user tried to set these parameters # manually, or that the search originated from somewhere else # (and was configured incorrectly). advanced_url = url_for("ui.advanced_search") raise BadRequest( f"It looks like there's something odd about your search" f" request. Please try <a href='{advanced_url}'>starting" f" over</a>.") # Force the form to be displayed, so that we can render errors. # This has most likely occurred due to someone manually crafting # a GET response, but it could be something else. response_data["show_form"] = True # We want the form handy even when it is not shown to the user. For # example, we can generate new form-friendly requests to update sort # order and page size by embedding the form (hidden). response_data["form"] = form return response_data, HTTPStatus.OK, {}
def xml(self, **kwargs): database = kwargs.get('database', None) if not database: database = db_monodb() req = openerp.http.request language = kwargs.get('language', None) if req.httprequest.method == 'GET': # Login database = kwargs.get('database', None) req.session.db = database try: self.authenticate(req, database, language) except Exception as e: logger.warning("Failed login attempt: %s" % e) return Response('Login with Odoo user name and password', 401, headers=[('WWW-Authenticate', 'Basic realm="odoo"')]) # As an optional extra security check we can validate a web token attached # to the request. It allows use to verify that the request is generated # from frePPLe and not from somebody else. # Generate data try: xp = exporter(req, database=database, company=kwargs.get('company', None), mode=int(kwargs.get('mode', 1))) # TODO Returning an iterator to stream the response back to the client and # to save memory on the server side return req.make_response( ''.join([i for i in xp.run()]), headers=[('Content-Type', 'application/xml;charset=utf8'), ('Cache-Control', 'no-cache, no-store, must-revalidate'), ('Pragma', 'no-cache'), ('Expires', '0')]) except Exception as e: logger.exception('Error generating frePPLe XML data') raise InternalServerError( description= 'Error generating frePPLe XML data: check the Odoo log file for more details' ) elif req.httprequest.method == 'POST': # Authenticate the user database = req.httprequest.form.get('database', None) req.session.db = database try: self.authenticate(req, database, language) except Exception as e: logger.warning("Failed login attempt %s" % e) return Response('Login with Odoo user name and password', 401, headers=[('WWW-Authenticate', 'Basic realm="odoo"')]) # Validate the company argument company_name = req.httprequest.form.get('company', None) company = None m = req.session.model('res.company') m_search = m.search([('name', '=', company_name)]) for i in m.browse(m_search): company = i if not company: return Response('Invalid company name argument', 401) # Verify that the data was posted from frePPLe and nobody else try: webtoken = req.httprequest.form.get('webtoken', None) decoded = jwt.decode(webtoken, company.webtoken_key, algorithms=['HS256']) if self.user != decoded.get('user', None): return Response('Incorrect or missing webtoken', 401) except: return Response('Incorrect or missing webtoken', 401) # Import the data try: ip = importer(req, database=database, company=company, mode=req.httprequest.form.get('mode', 1)) return req.make_response( ip.run(), [('Content-Type', 'text/plain'), ('Cache-Control', 'no-cache, no-store, must-revalidate'), ('Pragma', 'no-cache'), ('Expires', '0')]) except Exception as e: logger.exception('Error processing data posted by frePPLe') raise InternalServerError( description= 'Error processing data posted by frePPLe: check the Odoo log file for more details' ) else: raise MethodNotAllowed('Only GET and POST requests are accepted')
def cross_list(method: str, params: MultiDict, session: Session, submission_id: int, **kwargs) -> Response: submitter, client = user_and_client_from_session(session) submission, submission_events = load_submission(submission_id) form = ClassificationForm(params) form.operation._value = lambda: form.operation.data form.filter_choices(submission, session) # Create a formset to render removal option. # # We need forms for existing secondaries, to generate removal requests. # When the forms in the formset are submitted, they are handled as the # primary form in the POST request to this controller. formset = ClassificationForm.formset(submission) _primary_category = submission.primary_classification.category _primary = taxonomy.CATEGORIES[_primary_category] response_data = { 'submission_id': submission_id, 'submission': submission, 'submitter': submitter, 'client': client, 'form': form, 'formset': formset, 'primary': { 'id': submission.primary_classification.category, 'name': _primary['name'] }, } if method == 'POST': # Make sure that the user is not attempting to move to a different # step. # # Since the interface provides an "add" button to add cross-list # categories, we only want to handle the form data if the user is not # attempting to move to a different step. action = params.get('action') if not action: if not form.validate(): raise BadRequest(response_data) if form.operation.data == form.REMOVE: command_type = RemoveSecondaryClassification else: command_type = AddSecondaryClassification command = command_type(category=form.category.data, creator=submitter, client=client) if not validate_command(form, command, submission, 'category'): raise BadRequest(response_data) try: submission, _ = save(command, submission_id=submission_id) response_data['submission'] = submission except SaveError as e: raise InternalServerError(response_data) from e # Re-build the formset to reflect changes that we just made, and # generate a fresh form for adding another secondary. The POSTed # data should now be reflected in the formset. response_data['formset'] = ClassificationForm.formset(submission) form = ClassificationForm() form.operation._value = lambda: form.operation.data form.filter_choices(submission, session) response_data['form'] = form # Warn the user if they have too many secondaries. if len(submission.secondary_categories) > 3: alerts.flash_warning( Markup( 'Adding more than three cross-list classifications will' ' result in a delay in the acceptance of your submission.' )) if action in ['previous', 'save_exit', 'next']: return response_data, status.SEE_OTHER, {} return response_data, status.OK, {}
def run_wsgi(self): app = self.server.app environ = self.make_environ() headers_set = [] headers_sent = [] def write(data): assert headers_set, 'write() before start_response' if not headers_sent: status, response_headers = headers_sent[:] = headers_set try: code, msg = status.split(None, 1) except ValueError: code, msg = status, "" self.send_response(int(code), msg) header_keys = set() for key, value in response_headers: self.send_header(key, value) key = key.lower() header_keys.add(key) if 'content-length' not in header_keys: self.close_connection = True self.send_header('Connection', 'close') if 'server' not in header_keys: self.send_header('Server', self.version_string()) if 'date' not in header_keys: self.send_header('Date', self.date_time_string()) self.end_headers() assert type(data) is bytes, 'applications must write bytes' self.wfile.write(data) self.wfile.flush() def start_response(status, response_headers, exc_info=None): if exc_info: try: if headers_sent: reraise(*exc_info) finally: exc_info = None elif headers_set: raise AssertionError('Headers already set') headers_set[:] = [status, response_headers] return write def execute(app): application_iter = app(environ, start_response) try: # XXX: Is this actually correct? if not PY2 and isinstance(application_iter, bytes): # iterating over bytes' items would give us ints application_iter = (application_iter, ) for data in application_iter: write(data) # make sure the headers are sent if not headers_sent: write(b'') finally: if hasattr(application_iter, 'close'): application_iter.close() application_iter = None try: execute(app) except (socket.error, socket.timeout) as e: self.connection_dropped(e, environ) except Exception: if self.server.passthrough_errors: raise from werkzeug.debug.tbtools import get_current_traceback traceback = get_current_traceback(ignore_system_exceptions=True) try: # if we haven't yet sent the headers but they are set # we roll back to be able to set them again. if not headers_sent: del headers_set[:] execute(InternalServerError()) except Exception: pass self.server.log('error', 'Error on request:\n%s', traceback.plaintext)
def create_dataset_from_csv(self, df, name='__testtbl__'): current_app.logger.info('Dropping N/A records') current_app.logger.info('Before cleaning shape={0}'.format(df.shape)) df = df.dropna(axis=0, how='all') df = df.fillna(method='ffill') current_app.logger.info('After cleaning shape={0}'.format(df.shape)) columns = list(df.columns)[1:] current_app.logger.info('columns: {0}'.format(columns)) current_app.logger.info('create table: {0}'.format(name)) query = sql.SQL("""CREATE TABLE {schema}.{tbl} ( date TIMESTAMP WITHOUT TIME ZONE NOT NULL, {columns}, "mask" BOOLEAN DEFAULT FALSE, PRIMARY KEY (date) ) WITH ( OIDS = FALSE ); ALTER TABLE {schema}.{tbl} OWNER to {user}; """).format( schema=sql.Identifier(self.config['schema']), tbl=sql.Identifier(name), columns=sql.SQL(',').join( sql.SQL("{0} double precision").format(sql.Identifier(c)) for c in columns), user=sql.Identifier(self.config['user']), ) try: self.execute(query) except: raise InternalServerError( description="Fail to create create new table.") current_app.logger.info('uploading csv') cursor = self.conn.cursor() query = sql.SQL("""COPY {schema}.{tbl} ({columns}) FROM STDIN WITH CSV HEADER DELIMITER AS ','""").format( schema=sql.Identifier(self.config['schema']), tbl=sql.Identifier(name), columns=sql.SQL(',').join( sql.Identifier(c) for c in ['date'] + columns), ) try: csv = io.StringIO() df.to_csv(csv, index=False) # this is the tricky part # you have to seek back to the start before being read # otherwise it will continue to read from the end # which reads nothing csv.seek(0) cursor.copy_expert(sql=query, file=csv) self.conn.commit() except: self.conn.rollback() raise InternalServerError( description="Failed to write data into table") finally: cursor.close() current_app.logger.info('success')
def handle_unknown_error(error): return handle_http_error(InternalServerError())
def view500(): raise InternalServerError('error')
def on_imagen(self, request, nombre): try: normpath = posixpath.normpath(nombre) asset_data = self.img_mngr.get_item(normpath) except Exception, e: msg = u"Error interno al buscar imagen: %s" % e raise InternalServerError(msg) if asset_data is None: if self.verbose: print "WARNING: no pudimos encontrar", repr(nombre) try: width, _, height = request.args["s"].partition('-') width = int(width) height = int(height) except Exception, e: raise InternalServerError("Error al generar imagen") img = bmp.BogusBitMap(width, height) return Response(img.data, mimetype="img/bmp") type_ = guess_type(nombre)[0] return Response(asset_data, mimetype=type_) def on_institucional(self, request, path): path = os.path.join("institucional", path) asset_file = os.path.join(config.DIR_ASSETS, path) if os.path.isdir(asset_file): print "WARNING: ", repr(asset_file), "es un directorio" raise NotFound() if not os.path.exists(asset_file): print "WARNING: no pudimos encontrar", repr(asset_file) raise NotFound()
def view500(): # flask-login user is loaded during @login_required, so check that the db has been queried mock_user_get.assert_called_with(user['id']) raise InternalServerError('error')
def errorhandler(e): """Handle error""" if not isinstance(e, HTTPException): e = InternalServerError() return apology(e.name, e.code)
def send_async_email(app, msg): with app.app_context(): try: mail.send(msg) except ConnectionRefusedError: raise InternalServerError("[MAIL SERVER] not working")
def create_package(job_id: int, job_name: str, description: str, router_name: str, provider: str, bbox: List[float], result_path: str, in_pbf_path: str, compression: str, user_email: str, config_string='production', cleanup=True): """ Creates a routing package and puts it in a defined folder. """ # We need to create a new app for this, RQ runs async # and has no Flask context otherwise app = create_app(config_string) app_ctx = app.app_context() app_ctx.push() session: SessionBase = db.session # Set up the logger where we have access to the user email # and only if there hasn't been one before if not LOGGER.handlers: handler = AppSmtpHandler(**get_smtp_details(app.config, [user_email])) handler.setLevel(logging.INFO) LOGGER.addHandler(handler) bbox_geom: Polygon = bbox_to_geom(bbox) job = Job.query.get(job_id) succeeded = False # Huge try/except to make sure we only have to write a failure once. # Processing failures only raise HTTPErrors. If a job got deleted while it # was in the queue or even already running, any other exception will be thrown. # That way we can distinguish between the two scenarios and send emails accordingly. try: if not job: raise Exception( f"Job {job_id} doesn't exist anymore in the database.") job_status = job.status in_pbf_dir = app.config[provider.upper() + '_DIR'] router = get_router(router_name, provider, in_pbf_path) # Set Redis job ID and container ID rq_job: Job = get_current_job() # testing has no access to a real queue if not app.testing: # pragma: no cover job.set_rq_id(rq_job.id) job.set_container_id(router.container_id) # There's only 2 options how the PBF file is not available: # 1. The package just gets registered, so needs to be still extracted # 2. Smth went wrong in the pbf update or job deletion procedure if not os.path.isfile(in_pbf_path): # First take care of 2. if job_status == Statuses.COMPLETED.value: raise InternalServerError( f"Job {job_id} couldn't find its corresponding PBF file {in_pbf_path}" ) # Then take care of 1. try: best_pbf_path = get_pbfs_by_area(in_pbf_dir, bbox_geom)[0][0] # Raise a HTTP Error to not mess with error handling here except (AssertionError, FileNotFoundError) as e: raise InternalServerError(str(e)) # Cut the PBF to the bbox extent (osmium availability is checked in __init__.py) osmium_proc = extract_proc(bbox_geom, best_pbf_path, in_pbf_path) job.set_status(Statuses.EXTRACTING.value) session.commit() # Let osmium cut the local PBF _, osmium_stderr = osmium_proc.communicate() if osmium_stderr: # pragma: no cover raise InternalServerError( f"'osmium': {json.loads(osmium_stderr)}") # Check validity osmium_reader = Reader(in_pbf_path) if osmium_reader.header().box().bottom_left.lat == 0: raise InternalServerError( f"'osmium': Apparently bbox {bbox} is not within the extent of {in_pbf_path}" ) job.set_status(Statuses.TILING.value) session.commit() try: exit_code, output = router.build_graph() if exit_code: # pragma: no cover raise InternalServerError( f"'{router.name()}': {output.decode()}") except ImageNotFound: raise InternalServerError( f"Docker image {router.image} not found.'") job.set_status(Statuses.COMPLETED.value) session.commit() succeeded = True # catch all exceptions we're actually aware of except HTTPException as e: job.set_status(Statuses.FAILED.value) session.commit() LOGGER.error(e.description, extra=dict(user=user_email, job_id=job_id)) raise # any other exception is assumed to be a deleted job and will only be logged/email sent except Exception as e: # pragma: no cover msg = f"Job {job_id} by {user_email} was deleted." LOGGER.warning(msg, extra=dict(user=user_email, job_id=job_id)) raise finally: # always write the "last_ran" column job.set_last_finished(datetime.utcnow()) session.commit() if not succeeded: shutil.rmtree(os.path.dirname(result_path)) # Pop the context as we're done with app & db app_ctx.pop() # Write dataset to disk router.make_package(result_path, compression) # Create the meta JSON fname = os.path.basename(result_path) j = { "job_id": job_id, "filepath": fname, "name": job_name, "description": description, "extent": ",".join([str(f) for f in bbox]), "last_modified": str(datetime.utcnow()) } dirname = os.path.dirname(result_path) # Splits the first fname_sanitized = fname.split(os.extsep, 1)[0] with open(os.path.join(dirname, fname_sanitized + '.json'), 'w', encoding='utf8') as f: json.dump(j, f, indent=2, ensure_ascii=False) # only clean up if successful, otherwise retain the container for debugging if cleanup: router.cleanup() LOGGER.info( f"Job {job_id} by {user_email} finished successfully. Find the new dataset in {result_path}", extra={ "job_id": job_id, "user": user_email })
def bp_test(): raise InternalServerError()