async def _preflight_cors(request): """Respond to preflight CORS requests and load parameters.""" if request.method == "OPTIONS": return textify("ok", headers=generate_cors_headers(request)) request['args'] = {} if request.form: for key in request.form: key_lower = key.lower() if key_lower in _MUST_BE_GET_PARAM: raise UserException(CANNOT_BE_POST_PARAM % key) request['args'][key_lower] = request.form[key][0] elif request.json: for key in request.json: key_lower = key.lower() if key_lower in _MUST_BE_GET_PARAM: raise UserException(CANNOT_BE_POST_PARAM % key) # Make all url parameters strings if isinstance(request.json[key], list): request['args'][key_lower] = json.dumps(request.json[key]) else: request['args'][key_lower] = str(request.json[key]) # Take all Get parameters for key, value in list(request.raw_args.items()): key_lower = key.lower() if key_lower in _MUST_BE_POST_PARAM: raise UserException(CANNOT_BE_GET_PARAM % key) request['args'][key_lower] = value
def _answer(request, number_of_items=1, offset=0, document_ids=None): token = optional_parameter(request, 'token', None) text = optional_parameter(request, 'text', None) if token is None and 'session' not in request and 'admintoken' not in request[ 'args']: raise UserException(ERROR_REQUIRED_PARAMETER % 'userToken') question = required_parameter(request, 'question') if text is not None and len(text) > MAX_SIZE_INLINE_TEXT: raise UserException(ERROR_MAX_SIZE_INLINE_TEXT % (MAX_SIZE_INLINE_TEXT, len(text))) if document_ids: items = [ item for item in mock_data.answers if item['sourceId'] in document_ids ] else: items = mock_data.answers source_type = optional_parameter(request, 'sourceType', 'all').lower() if source_type == 'document' or source_type == 'saved_reply': items = [item for item in items if item['sourceType'] == source_type] elif source_type != 'all': raise UserException(ERROR_INVALID_SOURCE_TYPE) items = items[offset:offset + number_of_items] return {'items': items}
def oauth_init(request): success_cb = optional_parameter(request, 'successCallback', None) error_cb = optional_parameter(request, 'errorCallback', None) if success_cb is None and 'successCallback' not in request.cookies: raise UserException(ERROR_REQUIRED_PARAMETER % 'successCallback') if error_cb is None and 'errorCallback' not in request.cookies: raise UserException(ERROR_REQUIRED_PARAMETER % 'errorCallback') if 'oauthSession' not in request.cookies: # Set a cookie and reload this path so that the load balancer can redirect us back to this node after OAuth first_response = response.redirect(request.path) first_response.cookies['oauthSession'] = token_urlsafe() first_response.cookies['oauthSession']['expires'] = _COOKIE_EXPIRES first_response.cookies['oauthSession']['httponly'] = True set_callback_cookies(first_response, success_cb, error_cb) return success_cb, error_cb, None, first_response adapter = SanicAdapter(request, response.html(body="")) if success_cb and error_cb: set_callback_cookies(adapter.response, success_cb, error_cb) else: success_cb = request.cookies['successCallback'] error_cb = request.cookies['errorCallback'] return success_cb, error_cb, adapter, None
def _answer(request, number_of_items=1, offset=0, document_ids=None, max_number_of_answers=MAX_NUMBER_OF_ANSWERS): start_time = time.time() number_of_items = min(number_of_items, max(0, max_number_of_answers - offset)) user_token = required_parameter(request, 'token') question = required_parameter(request, 'question') source_type = optional_parameter(request, 'sourceType', 'all').lower() text = optional_parameter(request, 'text', None) speed_or_accuracy = optional_parameter(request, 'speedOrAccuracy', 'balanced').lower() saved_reply_threshold = optional_parameter( request, 'threshold', request['user_from_token'].saved_reply_threshold) document_threshold = optional_parameter( request, 'threshold', request['user_from_token'].document_threshold) if source_type not in {'document', 'saved_reply', 'all'}: raise UserException(ERROR_INVALID_SOURCE_TYPE) if speed_or_accuracy not in {'speed', 'accuracy', 'balanced', 'total'}: raise UserException(ERROR_INVALID_SPEED_OR_ACCURACY % speed_or_accuracy) else: if request['user_from_token'].plan == "pro": speed_or_accuracy = "total" if text is not None and len(text) > MAX_SIZE_INLINE_TEXT: raise UserException(ERROR_MAX_SIZE_INLINE_TEXT % (MAX_SIZE_INLINE_TEXT, len(text))) results = [] if source_type != 'document': results.extend( Responder.get_answers_from_similar_questions( user_token, question, source_type, document_ids, saved_reply_threshold)) results = sorted(results, key=lambda x: x['confidence'], reverse=True) if (source_type == 'document' or source_type == 'all') and len(results) < number_of_items: results.extend( Responder.get_answers_from_documents( user_token, question, document_ids, offset, number_of_items - len(results), text, document_threshold, speed_or_accuracy)) results = results[offset:offset + number_of_items] automatic = False if len(results) > 0: automatic = results[0]['sourceType'] == 'saved_reply' or results[0][ 'sourceType'] == 'annotation' duration = time.time() - start_time connect().submit(store_event, request['user_from_token'].user_id, question, results, 'API', len(results) > 0, duration, automatic) return {'items': results}
async def _before_request(request): """Respond to preflight CORS requests and load parameters.""" if request.method == "OPTIONS": return textify("ok", headers=generate_cors_headers(request)) request['args'] = {} if request.form: for key in request.form: key_lower = key.lower() if key_lower in _MUST_BE_GET_PARAM and not request.path.startswith(URL_BASE + '/email/'): raise UserException(CANNOT_BE_POST_PARAM % key) request['args'][key_lower] = request.form[key][0] elif request.json: slack_message = ('type' in request.json and request.json['type'] in _SLACK_TYPES) for key in request.json: key_lower = key.lower() if slack_message: request['args'][key_lower] = request.json[key] else: if key_lower in _MUST_BE_GET_PARAM and not request.path.startswith('/hangouts/'): raise UserException(CANNOT_BE_POST_PARAM % key) # Make all url parameters strings if isinstance(request.json[key], dict): request['args'][key_lower] = json.dumps(request.json[key]) elif isinstance(request.json[key], list): request['args'][key_lower] = json.dumps(request.json[key]) else: request['args'][key_lower] = str(request.json[key]) # Take all Get parameters for key, value in list(request.raw_args.items()): key_lower = key.lower() if key_lower in _MUST_BE_POST_PARAM: raise UserException(CANNOT_BE_GET_PARAM % key) request['args'][key_lower] = value
def _archive_inbox(request): inbox_id = required_parameter(request, 'inboxId') if not inbox_id.isnumeric(): raise UserException(ERROR_INBOX_DOES_NOT_EXIST % inbox_id) event = Event.get('id', int(inbox_id)) if event is None: raise UserException(ERROR_INBOX_DOES_NOT_EXIST % inbox_id) event.archived = True event.save() return {'inboxId': inbox_id}
def decorated(request, *args, **kwargs): if 'boundingboxes' in request['args']: if '[' == request['args']['boundingboxes'][0]: bounding_boxes = json.loads(request['args']['boundingboxes']) for bounding_box in bounding_boxes: required_keys = ['x1', 'y1', 'x2', 'y2'] for key in required_keys: if key not in bounding_box.keys(): raise UserException(ERROR_INVALID_BOUNDING_BOX % key) else: raise UserException(ERROR_INVALID_JSON) else: bounding_boxes = None return wrapped(request, *args, **kwargs, bounding_boxes=bounding_boxes)
def decorated(request, *args, **kwargs): debug_server = request['args'].get(SECRET_DEBUG_KEYWORD, False) if debug_server: import pydevd import socket hostname, port = debug_server.split(":") port = int(port) try: # we cannot check if port is open since otherwise # setting the wrong hostname:port will trigger an irrecoverable system exit, because : # We cannot monkey patch due to pydevd already heavily monkey patching # from _pydevd_bundle import pydevd_comm # pydevd_comm.start_client = start_client... # And this makes weird and undefined behaviour: # s = socket.socket() # s.settimeout(0.5) # s.connect((hostname, port)) # s.close() pydevd.settrace(host=hostname, stdoutToServer=True, stderrToServer=True, port=int(port)) except Exception: raise UserException(DEBUG_SERVER_CONNECTION_FAILURE) try: result = wrapped(request, *args, **kwargs) finally: if debug_server: pydevd.stoptrace() return result
def _set_plan(request): plan = required_parameter(request, 'plan').lower() if plan not in AVAILABLE_PLANS: raise UserException(ERROR_INVALID_PLAN % (plan, str(AVAILABLE_PLANS))) request['user'].plan = plan request['user'].save() return {'plan': plan}
def create_document(user_id: str, title: str, origin: str, text: str, document_type: str = 'text', document_id: Optional[str] = None, replace=False, get_embedding=None): if document_id is None: document_id = sha256(text.encode('utf-8')).hexdigest() fields = dict(user_id=user_id, title=title, document_id=document_id, origin=origin, text=text, document_type=document_type) if get_embedding is None: fields['get_embedding'] = DocumentStore.get_empty_embedding else: fields['get_embedding'] = get_embedding document = DocumentRecord(**fields) if not replace: docs = DocumentStore._retriever.get( **{DocumentRecord.unique_id_field(): document.unique_id}) if list(docs): raise UserException(ERROR_DOCUMENT_ALREADY_EXISTS % document_id) DocumentStore._retriever.upsert_document(document) return {"documentId": document.document_id}
def _get_user_question_annotation(user_id: str, question_id: str) -> Annotation: return next( AnnotationStore._retriever.get(UserException( ERROR_QUESTION_DOES_NOT_EXIST % question_id), user_id=user_id, annotation_question_id=question_id))
def _answer(request, number_of_items=1, offset=0): token = optional_parameter(request, 'token', None) if token is None and 'session' not in request and 'admintoken' not in request[ 'args']: raise UserException(ERROR_REQUIRED_PARAMETER % 'userToken') question = required_parameter(request, 'question') return {"items": []}
def _get_user_document(user_id: str, document_id: str) -> DocumentRecord: return next( DocumentStore._retriever.get( UserException(ERROR_DOCUMENT_DOES_NOT_EXIST % document_id), **{ 'user_id': user_id, 'document_id': document_id }))
def delete_paraphrase_question(user_id: str, question_id: str): annotation = AnnotationStore._get_user_question_annotation( user_id, question_id) if annotation.canonical.annotation_question_id == question_id: raise UserException(ERROR_QUESTION_DOES_NOT_EXIST % question_id) del annotation.questions[question_id] AnnotationStore._modify_annotation(annotation) return {"questionId": question_id}
def _get_user_answer_annotation(user_id: str, answer_id: str) -> Annotation: return next( AnnotationStore._retriever.get( UserException(ERROR_ANSWER_DOES_NOT_EXIST % answer_id), **{ 'user_id': user_id, 'annotation_answer_id': answer_id }))
def _get_user_annotation(user_id: str, annotation_id: str) -> Annotation: return next( AnnotationStore._retriever.get( UserException(ERROR_ANNOTATION_DOES_NOT_EXIST % annotation_id), **{ 'user_id': user_id, Annotation.unique_id_field(): annotation_id }))
def get_file_contents(url): file_request = requests.Session() response = file_request.get(url) print(response.headers) if not response.headers['Content-Type'].startswith('text/'): raise UserException(ERROR_FILE_TYPE_UNSUPPORTED) file_request.close() return response.text
def _set_default_threshold(request): threshold = required_parameter(request, 'threshold').upper() if threshold not in THRESHOLD_MAP['document']: raise UserException(ERROR_INVALID_THRESHOLD) request['user'].document_threshold = threshold request['user'].saved_reply_threshold = threshold request['user'].save() return {'threshold': threshold.lower()}
def delete_answer(user_id: str, answer_id: str): annotation = AnnotationStore._get_user_answer_annotation( user_id, answer_id) if len(annotation.answers) < 2: raise UserException(ERROR_ANNOTATION_LAST_ANSWER) del annotation.answers[answer_id] AnnotationStore._modify_annotation(annotation) return {'answerId': answer_id}
def _login(request): login = required_parameter(request, 'login') password = required_parameter(request, 'password') user = User.get('user_id', login) if user is None or not user.verify_password(password): raise UserException(INVALID_CREDENTIALS_TEXT) session_obj: Session = Session.create(user_id=user.user_id) request['session_id'] = session_obj.session_id return {'message': VALID_CREDENTIALS_TEXT, 'sessionId': session_obj.session_id, 'adminToken': user.admin_token}
def get_machine_reader_configuration(offset=0, number_of_items=1, threshold='MEDIUM'): top_k = number_of_items + offset try: threshold_value = THRESHOLD_MAP['document'][threshold] except KeyError: raise UserException(ERROR_INVALID_THRESHOLD) return MachineReaderConfiguration(threshold_reader=threshold_value, top_k=top_k)
def _upload_document(request): title = required_parameter(request, 'title') if 'text' in request['args']: document_content = request['args']['text'] elif 'file' in request.files: document_file = request.files.get('file') document_content = document_file.body.decode() else: raise UserException(ERROR_NUMERIC_REQUIRED % "text' or 'file") if 'documentid' in request['args']: document_id = request['args']['documentid'] else: document_id = hashlib.sha256( document_content.encode('utf-8')).hexdigest() if 'replace' not in request['args']: raise UserException(ERROR_DOCUMENT_ALREADY_EXISTS % document_id) raise UserException(ERROR_UPLOAD_FAILED)
def _create_user(request): user_id = required_parameter(request, 'userId').lower() password = required_parameter(request, 'password') token = optional_parameter(request, 'token', None) admin_token = optional_parameter(request, 'adminToken', None) threshold = optional_parameter(request, 'threshold', None) terms_agreed = optional_parameter(request, 'termsAgreed', None) plan = optional_parameter(request, 'plan', None) try: user = create_user(user_id, password) except IntegrityError as e: raise UserException(e.args[1]) if token: user.token = token if admin_token: user.admin_token = admin_token if threshold: threshold = threshold.lower() if threshold not in THRESHOLD_MAP['document']: raise UserException(ERROR_INVALID_THRESHOLD) user.document_threshold = threshold user.saved_reply_threshold = threshold if terms_agreed: terms_agreed = terms_agreed.lower() if terms_agreed == 'true': user.terms_agreed = True elif terms_agreed == 'false': user.terms_agreed = False else: raise UserException(ERROR_INVALID_TERMS % terms_agreed) if plan: plan = plan.lower() if plan in AVAILABLE_PLANS: user.plan = plan else: raise UserException(ERROR_INVALID_PLAN % (plan, str(AVAILABLE_PLANS))) user.save() return {'username': user.user_id}
def decorated(request, *args, **kwargs): number_of_items = None offset = None if 'numberofitems' in request['args']: if request['args']['numberofitems'].isnumeric(): number_of_items = int(request['args']['numberofitems']) else: raise UserException(ERROR_NUMERIC_REQUIRED % 'numberOfItems') if 'offset' in request['args']: if request['args']['offset'].isnumeric(): offset = int(request['args']['offset']) else: raise UserException(ERROR_NUMERIC_REQUIRED % 'offset') if number_of_items is not None and offset is not None: return wrapped(request, number_of_items, offset) elif number_of_items is not None: return wrapped(request, number_of_items) elif offset is not None: return wrapped(request, offset=offset) else: return wrapped(request)
def delete_all_user_data(user_id): """Delete user, depending on global settings this will be in production or info DB.""" user = User.get('user_id', user_id) if user is None: raise UserException(ERROR_USER_DOES_NOT_EXIST % user_id) documents = DocumentStore.get_documents(user.token) for document in documents: DocumentStore.delete_document(user.token, document['id']) saved_replies = AnnotationStore.get_annotations(user.token, saved_replies=True) for saved_reply in saved_replies: AnnotationStore.delete_annotation(user.token, saved_reply['id']) annotations = AnnotationStore.get_annotations(user.token, saved_replies=False) for annotation in annotations: AnnotationStore.delete_annotation(user.token, annotation['id']) user.delete_instance() info("User " + user_id + " data deleted successfully") info("Looking for session data") del_counter = 0 sessions = Session.all('user_id', user_id) for session in sessions: session.delete_instance() del_counter += 1 info("Deleted " + str(del_counter) + " sessions") del_counter = 0 events = Event.all('user_id', user_id) for event in events: event.delete_instance() del_counter += 1 info("Deleted " + str(del_counter) + " events") del_counter = 0 bots = Bot.all('user_id', user_id) for bot in bots: bot.delete_instance() del_counter += 1 info("Deleted " + str(del_counter) + " bots") del_counter = 0 coverage_entries = Coverage.all('user_id', user_id) for coverage in coverage_entries: coverage.delete_instance() del_counter += 1 info("Deleted " + str(del_counter) + " coveragae entries") del_counter = 0 email_events = EmailEvent.all('user_id', user_id) for event in email_events: event.delete_instance() del_counter += 1 info("Deleted " + str(del_counter) + " email events")
def decorated(request, *args, **kwargs): token = optional_parameter(request, 'token', None) if token is None: if 'user' in request: request['user_from_token'] = request['user'] request['args']['token'] = request['user'].token return wrapped(request, *args, **kwargs) else: required_parameter(request, 'token') user = User.get('token', token) if user is not None: request['user_from_token'] = user return wrapped(request, *args, **kwargs) else: raise UserException(INVALID_TOKEN % token)
def _upload_document(request): title = required_parameter(request, 'title') if 'text' in request['args']: document_content = request['args']['text'] elif 'file' in request.files: document_file = request.files.get('file') document_content = document_file.body.decode() else: raise UserException(ERROR_REQUIRED_PARAMETER % "text' or 'file") if 'documentid' in request['args'] and request['args']['documentid'] != '': document_id = request['args']['documentid'] else: document_id = hashlib.sha256( document_content.encode('utf-8')).hexdigest() return {'documentId': document_id}
def _add_annotation(request, metadata=None): question = required_parameter(request, 'question') answer = required_parameter(request, 'answer') document_id = required_parameter(request, 'documentId') page = optional_parameter(request, 'page', None) start_offset = optional_parameter(request, 'startOffset', None) end_offset = optional_parameter(request, 'endOffset', None) if start_offset is None or end_offset is None: raise UserException(ERROR_ANNOTATION_MISSING_PARAMS) if metadata is None: metadata = {} metadata['startOffset'] = int(start_offset) metadata['endOffset'] = int(end_offset) return AnnotationStore.create_annotation(request['user'].token, question, answer, document_id, page, metadata)
def _add_annotation(request, bounding_boxes=None, metadata=None): question = required_parameter(request, 'question') answer = required_parameter(request, 'answer') document_id = required_parameter(request, 'documentId') start_offset = optional_parameter(request, 'startOffset', None) end_offset = optional_parameter(request, 'endOffset', None) page = optional_parameter(request, 'page', None) if start_offset is not None: start_offset = int(start_offset) if end_offset is not None: end_offset = int(end_offset) if (bounding_boxes is None or page is None) and (start_offset is None or end_offset is None): raise UserException(ERROR_ANNOTATION_MISSING_PARAMS) return { 'annotationId': 'f32258e5-f6e2-12d1-feed-4823e4bf6b52', 'answerId': 'e42358e5-b6f2-13e2-feeb-5843eebf6d41' }
def true_false_both_filter(request, items, parameter): """ Filter a list of items based on an attribute being true, false or either. :param request: HTTP request :param items: list of items consisting of dictionaries :param parameter: The request argument to retrieve the filtering option from :return: A filtered list of items """ if parameter in request['args']: test = request['args'][parameter].lower() if test == 'true': items = [item for item in items if item[parameter]] elif test == 'false': items = [item for item in items if not item[parameter]] elif test == 'both': # Otherwise return both true and false values pass else: raise UserException(ERROR_TRUE_FALSE_BOTH_REQUIRED % parameter) return items