async def _preflight_cors(request):
"""Respond to preflight CORS requests and load parameters."""
if request.method == "OPTIONS":
return textify("ok", headers=generate_cors_headers(request))
request['args'] = {}
if request.form:
for key in request.form:
key_lower = key.lower()
if key_lower in _MUST_BE_GET_PARAM:
raise UserException(CANNOT_BE_POST_PARAM % key)
request['args'][key_lower] = request.form[key][0]
elif request.json:
for key in request.json:
key_lower = key.lower()
if key_lower in _MUST_BE_GET_PARAM:
raise UserException(CANNOT_BE_POST_PARAM % key)
# Make all url parameters strings
if isinstance(request.json[key], list):
request['args'][key_lower] = json.dumps(request.json[key])
else:
request['args'][key_lower] = str(request.json[key])
# Take all Get parameters
for key, value in list(request.raw_args.items()):
key_lower = key.lower()
if key_lower in _MUST_BE_POST_PARAM:
raise UserException(CANNOT_BE_GET_PARAM % key)
request['args'][key_lower] = value
def _answer(request, number_of_items=1, offset=0, document_ids=None):
token = optional_parameter(request, 'token', None)
text = optional_parameter(request, 'text', None)
if token is None and 'session' not in request and 'admintoken' not in request[
'args']:
raise UserException(ERROR_REQUIRED_PARAMETER % 'userToken')
question = required_parameter(request, 'question')
if text is not None and len(text) > MAX_SIZE_INLINE_TEXT:
raise UserException(ERROR_MAX_SIZE_INLINE_TEXT %
(MAX_SIZE_INLINE_TEXT, len(text)))
if document_ids:
items = [
item for item in mock_data.answers
if item['sourceId'] in document_ids
]
else:
items = mock_data.answers
source_type = optional_parameter(request, 'sourceType', 'all').lower()
if source_type == 'document' or source_type == 'saved_reply':
items = [item for item in items if item['sourceType'] == source_type]
elif source_type != 'all':
raise UserException(ERROR_INVALID_SOURCE_TYPE)
items = items[offset:offset + number_of_items]
return {'items': items}
def oauth_init(request):
success_cb = optional_parameter(request, 'successCallback', None)
error_cb = optional_parameter(request, 'errorCallback', None)
if success_cb is None and 'successCallback' not in request.cookies:
raise UserException(ERROR_REQUIRED_PARAMETER % 'successCallback')
if error_cb is None and 'errorCallback' not in request.cookies:
raise UserException(ERROR_REQUIRED_PARAMETER % 'errorCallback')
if 'oauthSession' not in request.cookies:
# Set a cookie and reload this path so that the load balancer can redirect us back to this node after OAuth
first_response = response.redirect(request.path)
first_response.cookies['oauthSession'] = token_urlsafe()
first_response.cookies['oauthSession']['expires'] = _COOKIE_EXPIRES
first_response.cookies['oauthSession']['httponly'] = True
set_callback_cookies(first_response, success_cb, error_cb)
return success_cb, error_cb, None, first_response
adapter = SanicAdapter(request, response.html(body=""))
if success_cb and error_cb:
set_callback_cookies(adapter.response, success_cb, error_cb)
else:
success_cb = request.cookies['successCallback']
error_cb = request.cookies['errorCallback']
return success_cb, error_cb, adapter, None
def _answer(request,
number_of_items=1,
offset=0,
document_ids=None,
max_number_of_answers=MAX_NUMBER_OF_ANSWERS):
start_time = time.time()
number_of_items = min(number_of_items,
max(0, max_number_of_answers - offset))
user_token = required_parameter(request, 'token')
question = required_parameter(request, 'question')
source_type = optional_parameter(request, 'sourceType', 'all').lower()
text = optional_parameter(request, 'text', None)
speed_or_accuracy = optional_parameter(request, 'speedOrAccuracy',
'balanced').lower()
saved_reply_threshold = optional_parameter(
request, 'threshold', request['user_from_token'].saved_reply_threshold)
document_threshold = optional_parameter(
request, 'threshold', request['user_from_token'].document_threshold)
if source_type not in {'document', 'saved_reply', 'all'}:
raise UserException(ERROR_INVALID_SOURCE_TYPE)
if speed_or_accuracy not in {'speed', 'accuracy', 'balanced', 'total'}:
raise UserException(ERROR_INVALID_SPEED_OR_ACCURACY %
speed_or_accuracy)
else:
if request['user_from_token'].plan == "pro":
speed_or_accuracy = "total"
if text is not None and len(text) > MAX_SIZE_INLINE_TEXT:
raise UserException(ERROR_MAX_SIZE_INLINE_TEXT %
(MAX_SIZE_INLINE_TEXT, len(text)))
results = []
if source_type != 'document':
results.extend(
Responder.get_answers_from_similar_questions(
user_token, question, source_type, document_ids,
saved_reply_threshold))
results = sorted(results, key=lambda x: x['confidence'], reverse=True)
if (source_type == 'document'
or source_type == 'all') and len(results) < number_of_items:
results.extend(
Responder.get_answers_from_documents(
user_token, question, document_ids, offset,
number_of_items - len(results), text, document_threshold,
speed_or_accuracy))
results = results[offset:offset + number_of_items]
automatic = False
if len(results) > 0:
automatic = results[0]['sourceType'] == 'saved_reply' or results[0][
'sourceType'] == 'annotation'
duration = time.time() - start_time
connect().submit(store_event, request['user_from_token'].user_id, question,
results, 'API',
len(results) > 0, duration, automatic)
return {'items': results}
async def _before_request(request):
"""Respond to preflight CORS requests and load parameters."""
if request.method == "OPTIONS":
return textify("ok", headers=generate_cors_headers(request))
request['args'] = {}
if request.form:
for key in request.form:
key_lower = key.lower()
if key_lower in _MUST_BE_GET_PARAM and not request.path.startswith(URL_BASE + '/email/'):
raise UserException(CANNOT_BE_POST_PARAM % key)
request['args'][key_lower] = request.form[key][0]
elif request.json:
slack_message = ('type' in request.json and request.json['type'] in _SLACK_TYPES)
for key in request.json:
key_lower = key.lower()
if slack_message:
request['args'][key_lower] = request.json[key]
else:
if key_lower in _MUST_BE_GET_PARAM and not request.path.startswith('/hangouts/'):
raise UserException(CANNOT_BE_POST_PARAM % key)
# Make all url parameters strings
if isinstance(request.json[key], dict):
request['args'][key_lower] = json.dumps(request.json[key])
elif isinstance(request.json[key], list):
request['args'][key_lower] = json.dumps(request.json[key])
else:
request['args'][key_lower] = str(request.json[key])
# Take all Get parameters
for key, value in list(request.raw_args.items()):
key_lower = key.lower()
if key_lower in _MUST_BE_POST_PARAM:
raise UserException(CANNOT_BE_GET_PARAM % key)
request['args'][key_lower] = value
def _archive_inbox(request):
inbox_id = required_parameter(request, 'inboxId')
if not inbox_id.isnumeric():
raise UserException(ERROR_INBOX_DOES_NOT_EXIST % inbox_id)
event = Event.get('id', int(inbox_id))
if event is None:
raise UserException(ERROR_INBOX_DOES_NOT_EXIST % inbox_id)
event.archived = True
event.save()
return {'inboxId': inbox_id}
def decorated(request, *args, **kwargs):
if 'boundingboxes' in request['args']:
if '[' == request['args']['boundingboxes'][0]:
bounding_boxes = json.loads(request['args']['boundingboxes'])
for bounding_box in bounding_boxes:
required_keys = ['x1', 'y1', 'x2', 'y2']
for key in required_keys:
if key not in bounding_box.keys():
raise UserException(ERROR_INVALID_BOUNDING_BOX % key)
else:
raise UserException(ERROR_INVALID_JSON)
else:
bounding_boxes = None
return wrapped(request, *args, **kwargs, bounding_boxes=bounding_boxes)
def decorated(request, *args, **kwargs):
debug_server = request['args'].get(SECRET_DEBUG_KEYWORD, False)
if debug_server:
import pydevd
import socket
hostname, port = debug_server.split(":")
port = int(port)
try:
# we cannot check if port is open since otherwise
# setting the wrong hostname:port will trigger an irrecoverable system exit, because :
# We cannot monkey patch due to pydevd already heavily monkey patching
# from _pydevd_bundle import pydevd_comm
# pydevd_comm.start_client = start_client...
# And this makes weird and undefined behaviour:
# s = socket.socket()
# s.settimeout(0.5)
# s.connect((hostname, port))
# s.close()
pydevd.settrace(host=hostname, stdoutToServer=True, stderrToServer=True, port=int(port))
except Exception:
raise UserException(DEBUG_SERVER_CONNECTION_FAILURE)
try:
result = wrapped(request, *args, **kwargs)
finally:
if debug_server:
pydevd.stoptrace()
return result
def _set_plan(request):
plan = required_parameter(request, 'plan').lower()
if plan not in AVAILABLE_PLANS:
raise UserException(ERROR_INVALID_PLAN % (plan, str(AVAILABLE_PLANS)))
request['user'].plan = plan
request['user'].save()
return {'plan': plan}
def create_document(user_id: str,
title: str,
origin: str,
text: str,
document_type: str = 'text',
document_id: Optional[str] = None,
replace=False,
get_embedding=None):
if document_id is None:
document_id = sha256(text.encode('utf-8')).hexdigest()
fields = dict(user_id=user_id,
title=title,
document_id=document_id,
origin=origin,
text=text,
document_type=document_type)
if get_embedding is None:
fields['get_embedding'] = DocumentStore.get_empty_embedding
else:
fields['get_embedding'] = get_embedding
document = DocumentRecord(**fields)
if not replace:
docs = DocumentStore._retriever.get(
**{DocumentRecord.unique_id_field(): document.unique_id})
if list(docs):
raise UserException(ERROR_DOCUMENT_ALREADY_EXISTS %
document_id)
DocumentStore._retriever.upsert_document(document)
return {"documentId": document.document_id}
def _get_user_question_annotation(user_id: str,
question_id: str) -> Annotation:
return next(
AnnotationStore._retriever.get(UserException(
ERROR_QUESTION_DOES_NOT_EXIST % question_id),
user_id=user_id,
annotation_question_id=question_id))
def _answer(request, number_of_items=1, offset=0):
token = optional_parameter(request, 'token', None)
if token is None and 'session' not in request and 'admintoken' not in request[
'args']:
raise UserException(ERROR_REQUIRED_PARAMETER % 'userToken')
question = required_parameter(request, 'question')
return {"items": []}
def _get_user_document(user_id: str, document_id: str) -> DocumentRecord:
return next(
DocumentStore._retriever.get(
UserException(ERROR_DOCUMENT_DOES_NOT_EXIST % document_id), **{
'user_id': user_id,
'document_id': document_id
}))
def delete_paraphrase_question(user_id: str, question_id: str):
annotation = AnnotationStore._get_user_question_annotation(
user_id, question_id)
if annotation.canonical.annotation_question_id == question_id:
raise UserException(ERROR_QUESTION_DOES_NOT_EXIST % question_id)
del annotation.questions[question_id]
AnnotationStore._modify_annotation(annotation)
return {"questionId": question_id}
def _get_user_answer_annotation(user_id: str,
answer_id: str) -> Annotation:
return next(
AnnotationStore._retriever.get(
UserException(ERROR_ANSWER_DOES_NOT_EXIST % answer_id), **{
'user_id': user_id,
'annotation_answer_id': answer_id
}))
def _get_user_annotation(user_id: str, annotation_id: str) -> Annotation:
return next(
AnnotationStore._retriever.get(
UserException(ERROR_ANNOTATION_DOES_NOT_EXIST % annotation_id),
**{
'user_id': user_id,
Annotation.unique_id_field(): annotation_id
}))
def get_file_contents(url):
file_request = requests.Session()
response = file_request.get(url)
print(response.headers)
if not response.headers['Content-Type'].startswith('text/'):
raise UserException(ERROR_FILE_TYPE_UNSUPPORTED)
file_request.close()
return response.text
def _set_default_threshold(request):
threshold = required_parameter(request, 'threshold').upper()
if threshold not in THRESHOLD_MAP['document']:
raise UserException(ERROR_INVALID_THRESHOLD)
request['user'].document_threshold = threshold
request['user'].saved_reply_threshold = threshold
request['user'].save()
return {'threshold': threshold.lower()}
def delete_answer(user_id: str, answer_id: str):
annotation = AnnotationStore._get_user_answer_annotation(
user_id, answer_id)
if len(annotation.answers) < 2:
raise UserException(ERROR_ANNOTATION_LAST_ANSWER)
del annotation.answers[answer_id]
AnnotationStore._modify_annotation(annotation)
return {'answerId': answer_id}
def _login(request):
login = required_parameter(request, 'login')
password = required_parameter(request, 'password')
user = User.get('user_id', login)
if user is None or not user.verify_password(password):
raise UserException(INVALID_CREDENTIALS_TEXT)
session_obj: Session = Session.create(user_id=user.user_id)
request['session_id'] = session_obj.session_id
return {'message': VALID_CREDENTIALS_TEXT, 'sessionId': session_obj.session_id, 'adminToken': user.admin_token}
def get_machine_reader_configuration(offset=0,
number_of_items=1,
threshold='MEDIUM'):
top_k = number_of_items + offset
try:
threshold_value = THRESHOLD_MAP['document'][threshold]
except KeyError:
raise UserException(ERROR_INVALID_THRESHOLD)
return MachineReaderConfiguration(threshold_reader=threshold_value,
top_k=top_k)
def _upload_document(request):
title = required_parameter(request, 'title')
if 'text' in request['args']:
document_content = request['args']['text']
elif 'file' in request.files:
document_file = request.files.get('file')
document_content = document_file.body.decode()
else:
raise UserException(ERROR_NUMERIC_REQUIRED % "text' or 'file")
if 'documentid' in request['args']:
document_id = request['args']['documentid']
else:
document_id = hashlib.sha256(
document_content.encode('utf-8')).hexdigest()
if 'replace' not in request['args']:
raise UserException(ERROR_DOCUMENT_ALREADY_EXISTS % document_id)
raise UserException(ERROR_UPLOAD_FAILED)
def _create_user(request):
user_id = required_parameter(request, 'userId').lower()
password = required_parameter(request, 'password')
token = optional_parameter(request, 'token', None)
admin_token = optional_parameter(request, 'adminToken', None)
threshold = optional_parameter(request, 'threshold', None)
terms_agreed = optional_parameter(request, 'termsAgreed', None)
plan = optional_parameter(request, 'plan', None)
try:
user = create_user(user_id, password)
except IntegrityError as e:
raise UserException(e.args[1])
if token:
user.token = token
if admin_token:
user.admin_token = admin_token
if threshold:
threshold = threshold.lower()
if threshold not in THRESHOLD_MAP['document']:
raise UserException(ERROR_INVALID_THRESHOLD)
user.document_threshold = threshold
user.saved_reply_threshold = threshold
if terms_agreed:
terms_agreed = terms_agreed.lower()
if terms_agreed == 'true':
user.terms_agreed = True
elif terms_agreed == 'false':
user.terms_agreed = False
else:
raise UserException(ERROR_INVALID_TERMS % terms_agreed)
if plan:
plan = plan.lower()
if plan in AVAILABLE_PLANS:
user.plan = plan
else:
raise UserException(ERROR_INVALID_PLAN % (plan, str(AVAILABLE_PLANS)))
user.save()
return {'username': user.user_id}
def decorated(request, *args, **kwargs):
number_of_items = None
offset = None
if 'numberofitems' in request['args']:
if request['args']['numberofitems'].isnumeric():
number_of_items = int(request['args']['numberofitems'])
else:
raise UserException(ERROR_NUMERIC_REQUIRED % 'numberOfItems')
if 'offset' in request['args']:
if request['args']['offset'].isnumeric():
offset = int(request['args']['offset'])
else:
raise UserException(ERROR_NUMERIC_REQUIRED % 'offset')
if number_of_items is not None and offset is not None:
return wrapped(request, number_of_items, offset)
elif number_of_items is not None:
return wrapped(request, number_of_items)
elif offset is not None:
return wrapped(request, offset=offset)
else:
return wrapped(request)
def delete_all_user_data(user_id):
"""Delete user, depending on global settings this will be in production or info DB."""
user = User.get('user_id', user_id)
if user is None:
raise UserException(ERROR_USER_DOES_NOT_EXIST % user_id)
documents = DocumentStore.get_documents(user.token)
for document in documents:
DocumentStore.delete_document(user.token, document['id'])
saved_replies = AnnotationStore.get_annotations(user.token,
saved_replies=True)
for saved_reply in saved_replies:
AnnotationStore.delete_annotation(user.token, saved_reply['id'])
annotations = AnnotationStore.get_annotations(user.token,
saved_replies=False)
for annotation in annotations:
AnnotationStore.delete_annotation(user.token, annotation['id'])
user.delete_instance()
info("User " + user_id + " data deleted successfully")
info("Looking for session data")
del_counter = 0
sessions = Session.all('user_id', user_id)
for session in sessions:
session.delete_instance()
del_counter += 1
info("Deleted " + str(del_counter) + " sessions")
del_counter = 0
events = Event.all('user_id', user_id)
for event in events:
event.delete_instance()
del_counter += 1
info("Deleted " + str(del_counter) + " events")
del_counter = 0
bots = Bot.all('user_id', user_id)
for bot in bots:
bot.delete_instance()
del_counter += 1
info("Deleted " + str(del_counter) + " bots")
del_counter = 0
coverage_entries = Coverage.all('user_id', user_id)
for coverage in coverage_entries:
coverage.delete_instance()
del_counter += 1
info("Deleted " + str(del_counter) + " coveragae entries")
del_counter = 0
email_events = EmailEvent.all('user_id', user_id)
for event in email_events:
event.delete_instance()
del_counter += 1
info("Deleted " + str(del_counter) + " email events")
def decorated(request, *args, **kwargs):
token = optional_parameter(request, 'token', None)
if token is None:
if 'user' in request:
request['user_from_token'] = request['user']
request['args']['token'] = request['user'].token
return wrapped(request, *args, **kwargs)
else:
required_parameter(request, 'token')
user = User.get('token', token)
if user is not None:
request['user_from_token'] = user
return wrapped(request, *args, **kwargs)
else:
raise UserException(INVALID_TOKEN % token)
def _upload_document(request):
title = required_parameter(request, 'title')
if 'text' in request['args']:
document_content = request['args']['text']
elif 'file' in request.files:
document_file = request.files.get('file')
document_content = document_file.body.decode()
else:
raise UserException(ERROR_REQUIRED_PARAMETER % "text' or 'file")
if 'documentid' in request['args'] and request['args']['documentid'] != '':
document_id = request['args']['documentid']
else:
document_id = hashlib.sha256(
document_content.encode('utf-8')).hexdigest()
return {'documentId': document_id}
def _add_annotation(request, metadata=None):
question = required_parameter(request, 'question')
answer = required_parameter(request, 'answer')
document_id = required_parameter(request, 'documentId')
page = optional_parameter(request, 'page', None)
start_offset = optional_parameter(request, 'startOffset', None)
end_offset = optional_parameter(request, 'endOffset', None)
if start_offset is None or end_offset is None:
raise UserException(ERROR_ANNOTATION_MISSING_PARAMS)
if metadata is None:
metadata = {}
metadata['startOffset'] = int(start_offset)
metadata['endOffset'] = int(end_offset)
return AnnotationStore.create_annotation(request['user'].token, question, answer,
document_id, page, metadata)
def _add_annotation(request, bounding_boxes=None, metadata=None):
question = required_parameter(request, 'question')
answer = required_parameter(request, 'answer')
document_id = required_parameter(request, 'documentId')
start_offset = optional_parameter(request, 'startOffset', None)
end_offset = optional_parameter(request, 'endOffset', None)
page = optional_parameter(request, 'page', None)
if start_offset is not None:
start_offset = int(start_offset)
if end_offset is not None:
end_offset = int(end_offset)
if (bounding_boxes is None or page is None) and (start_offset is None
or end_offset is None):
raise UserException(ERROR_ANNOTATION_MISSING_PARAMS)
return {
'annotationId': 'f32258e5-f6e2-12d1-feed-4823e4bf6b52',
'answerId': 'e42358e5-b6f2-13e2-feeb-5843eebf6d41'
}
def true_false_both_filter(request, items, parameter):
"""
Filter a list of items based on an attribute being true, false or either.
:param request: HTTP request
:param items: list of items consisting of dictionaries
:param parameter: The request argument to retrieve the filtering option from
:return: A filtered list of items
"""
if parameter in request['args']:
test = request['args'][parameter].lower()
if test == 'true':
items = [item for item in items if item[parameter]]
elif test == 'false':
items = [item for item in items if not item[parameter]]
elif test == 'both':
# Otherwise return both true and false values
pass
else:
raise UserException(ERROR_TRUE_FALSE_BOTH_REQUIRED % parameter)
return items
