def timeout_response(method: str, path: str) -> chalice.Response:
"""
Produce a chalice Response object that indicates a timeout. Stacktraces for all running threads, other than the
current thread, are provided in the response object.
"""
frames = sys._current_frames()
current_threadid = threading.get_ident()
trace_dump = {
thread_id: traceback.format_stack(frame)
for thread_id, frame in frames.items() if thread_id != current_threadid
}
problem = {
'status': requests.codes.gateway_timeout,
'code': 'timed_out',
'title': 'Timed out processing request.',
'traces': trace_dump
}
headers = {"Content-Type": "application/problem+json"}
if include_retry_after_header(return_code=requests.codes.gateway_timeout,
method=method,
uri=path):
headers['Retry-After'] = '10'
return chalice.Response(status_code=problem['status'],
headers=headers,
body=json.dumps(problem))
def dss_notification():
body = app.current_request.json_body
bundle_uuid = body['match']['bundle_uuid']
bundle_version = body['match']['bundle_version']
subscription_id = body['subscription_id']
event_type = body['event_type']
config = MatrixInfraConfig()
hmac_secret_key = config.dss_subscription_hmac_secret_key.encode()
HTTPSignatureAuth.verify(
requests.Request(url="http://host/dss/notification",
method=app.current_request.method,
headers=app.current_request.headers),
key_resolver=lambda key_id, algorithm: hmac_secret_key)
payload = {
'bundle_uuid': bundle_uuid,
'bundle_version': bundle_version,
'event_type': event_type,
}
queue_url = config.notification_q_url
SQSHandler().add_message_to_queue(queue_url, payload)
return chalice.Response(
status_code=requests.codes.ok,
body=f"Received notification from subscription {subscription_id}: "
f"{event_type} {bundle_uuid}.{bundle_version}")
def get_project_files(project_id):
files = get_downloadable_project_files(project_id, session)
return chalice.Response(status_code=200,
headers={
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*"
},
body=files)
def health_check(*args, **kwargs):
health_status = health.l2_health_checks()
health_res = {k: v for k, v in health_status.items() if k == "Healthy"}
return chalice.Response(status_code=200,
headers={"Content-Type": "application/json"},
body=json.dumps(health_res,
indent=4,
sort_keys=True,
default=str))
def dispatch(*args, **kwargs):
uri_params = app.current_request.uri_params or {}
path = app.current_request.context["resourcePath"].format(**uri_params)
req_body = app.current_request.raw_body if app.current_request._body is not None else None
app.log.info(
"[dispatch] path: %s query_string: %s method: %s",
path,
app.current_request.query_params,
app.current_request.method,
)
def maybe_fake_504() -> typing.Optional[chalice.Response]:
fake_504_probability_str = app.current_request.headers.get(
"DSS_FAKE_504_PROBABILITY", "0.0")
try:
fake_504_probability = float(fake_504_probability_str)
except ValueError:
return None
if random.random() > fake_504_probability:
return None
return timeout_response()
if not DeploymentStage.IS_PROD():
maybe_fake_504_result = maybe_fake_504()
if maybe_fake_504_result is not None:
return maybe_fake_504_result
with flask_app.test_request_context(
path=path,
base_url="https://{}".format(
app.current_request.headers["host"]),
query_string=app.current_request.query_params,
method=app.current_request.method,
headers=list(app.current_request.headers.items()),
data=req_body,
environ_base=app.current_request.stage_vars):
with nestedcontext.bind(time_left=lambda: (
(app.lambda_context.get_remaining_time_in_millis() / 1000
) - EXECUTION_TERMINATION_THRESHOLD_SECONDS),
skip_on_conflicts=True):
flask_res = flask_app.full_dispatch_request()
res_headers = dict(flask_res.headers)
# API Gateway/Cloudfront adds a duplicate Content-Length with a different value (not sure why)
res_headers.pop("Content-Length", None)
return chalice.Response(status_code=flask_res._status_code,
headers=res_headers,
body="".join([
c.decode() if isinstance(c, bytes) else c
for c in flask_res.response
]))
def get_application_secrets():
application_secret_file = os.environ["GOOGLE_APPLICATION_SECRETS"]
with open(application_secret_file, 'r') as fh:
data = json.loads(fh.read())
return chalice.Response(
status_code=requests.codes.ok,
headers={
'Content-Type': "application/json",
},
body=data,
)
def response_ccavenue_response_handler(responseId):
from ..main import app
response = Response.objects.get({"_id": ObjectId(responseId)})
form = Form.objects.only("formOptions").get({"_id": response.form.id})
formId = str(form.id)
merchant_id = form.formOptions.paymentMethods["ccavenue"]["merchant_id"]
config = CCAvenueConfig.objects.get({"merchant_id": merchant_id})
if not config:
mark_error_payment(
response,
f"CCAvenue config not found for merchant id: {merchant_id}.",
"ccavenue", paramDict)
res = dict(parse_qsl(app.current_request.raw_body.decode('utf-8')))
paramDict = decrypt(res['encResp'], config.SECRET_working_key)
if paramDict["merchant_param1"] != formId or paramDict[
"merchant_param2"] != responseId:
mark_error_payment(
response,
f"Form id / response id do not match. \nExpected: {formId}/{responseId}. \nReceived: {paramDict['merchant_param1']}/{paramDict['merchant_param1']}",
"ccavenue", paramDict)
if paramDict["order_status"] != "Success":
mark_error_payment(
response,
f"Order status does not mark success. Expected: Success. Received: {paramDict['order_status']}",
"ccavenue", paramDict)
# todo: redirect to another error page.
elif paramDict["order_status"] == "Success":
order_id = paramDict["order_id"]
if any(item.status == "SUCCESS" and item.id == order_id
and item.method == "ccavenue"
for item in response.payment_trail):
mark_error_payment(response,
f"Duplicate IPN transaction ID: {order_id}",
"ccavenue", paramDict)
mark_successful_payment(
form=form,
response=response,
full_value=paramDict,
method_name="ccavenue",
amount=paramDict["amount"], # TODO: or mer_amount?
currency=paramDict["currency"],
id=paramDict["order_id"])
response.save()
redirect_url = get(form.formOptions.paymentMethods,
"ccavenue.redirectUrl",
"http://www.chinmayamission.com")
return chalice.Response('',
status_code=302,
headers={'Location': redirect_url})
def get_project(project_id):
project = session.query(Project).get(project_id)
response_body = {
"id":
project.id,
"title":
project.title,
"assays":
get_project_assays(project.id),
"organs":
get_project_organs(project.id),
"species":
get_project_species(project.id),
"contributors":
get_project_contributors(project.id),
"description":
project.description,
"biosample_categories":
project.biosample_categories.split(","),
"development_stages":
project.development_stages.split(","),
"diseases":
project.diseases.split(","),
"cell_isolation_methods":
project.cell_isolation_methods.split(","),
"cell_types":
project.cell_types.split(","),
"cell_count":
project.cell_count,
"paired_end":
project.paired_end.split(","),
"nucleic_acid_sources":
project.nucleic_acid_sources.split(","),
"input_nucleic_acid_molecules":
project.input_nucleic_acid_molecules.split(","),
"publication_title":
project.publication_title,
"publication_doi":
project.publication_doi,
}
return chalice.Response(
status_code=200,
headers={
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*"
},
body=response_body,
)
def health():
# Level 2 healthcheck: Test connection can be made to redshift cluster but do not run any queries
redshift_handler.transaction([])
# Level 2 healthcheck checks that ecs query runner is active with expected number of tasks
service_name = f"matrix-service-query-runner-{os.environ['DEPLOYMENT_STAGE']}"
service = ecs_client.describe_services(cluster=service_name,
services=[service_name
])["services"][0]
status = service["status"]
running_task_count = service["runningCount"]
assert status == 'ACTIVE'
assert running_task_count > 0
return chalice.Response(status_code=200,
headers={'Content-Type': "text/html"},
body="OK")
def get_file(file_id):
file = session.query(File).get(file_id)
project = session.query(Project).get(file.project_id)
# file_prefix = f"{project.title}/{file.filename}"
file_prefix = f"{project.title}/matrix.loom"
download_url = generate_file_url(file_prefix)
response_body = {"url": download_url}
return chalice.Response(
status_code=200,
headers={
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*"
},
body=response_body,
)
def dispatch(*args, **kwargs):
uri_params = app.current_request.uri_params or {}
path = app.current_request.context["resourcePath"].format(**uri_params)
req_body = app.current_request.raw_body if app.current_request._body is not None else None
with flask_app.test_request_context(path=path,
base_url="https://{}".format(app.current_request.headers["host"]),
query_string=app.current_request.query_params,
method=app.current_request.method,
headers=list(app.current_request.headers.items()),
data=req_body,
environ_base=app.current_request.stage_vars):
flask_res = flask_app.full_dispatch_request()
res_headers = dict(flask_res.headers)
# API Gateway/Cloudfront adds a duplicate Content-Length with a different value (not sure why)
res_headers.pop("Content-Length", None)
return chalice.Response(status_code=flask_res._status_code,
headers=res_headers,
body="".join([c.decode() if isinstance(c, bytes) else c for c in flask_res.response]))
def get_projects():
projects = []
for project in session.query(Project).all():
projects.append({
"id": project.id,
"title": project.title,
"assays": get_project_assays(project.id),
"organs": get_project_organs(project.id),
"species": get_project_species(project.id),
"cell_count": project.cell_count,
})
return chalice.Response(status_code=200,
headers={
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*"
},
body=projects)
def timeout_response() -> chalice.Response:
"""
Produce a chalice Response object that indicates a timeout. Stacktraces for all running threads, other than the
current thread, are provided in the response object.
"""
frames = sys._current_frames()
current_threadid = threading.get_ident()
trace_dump = {
thread_id: traceback.format_stack(frame)
for thread_id, frame in frames.items() if thread_id != current_threadid
}
problem = {
'status': requests.codes.gateway_timeout,
'code': "timed_out",
'title': "Timed out processing request.",
'traces': trace_dump,
}
return chalice.Response(
status_code=problem['status'],
headers={"Content-Type": "application/problem+json"},
body=json.dumps(problem),
)
def synthesize_wav():
params = app.current_request.query_params or {}
sample_rate = int(params.get('rate', str(DEFAULT_SAMPLE_RATE)))
voice = params.get('voice', DEFAULT_VOICE)
app.log.debug('synthesize_wav({}, {}, <{} bytes>)'.format(
sample_rate, voice, len(app.current_request.raw_body)))
response = POLLY.synthesize_speech(
OutputFormat='pcm',
SampleRate=str(sample_rate),
Text=app.current_request.raw_body.decode('utf-8'),
VoiceId=voice)
with contextlib.closing(response['AudioStream']) as stream:
with tempfile.TemporaryFile() as temp:
output = wave.open(temp, 'wb')
output.setnchannels(1)
output.setsampwidth(2) # 16 bits per sample
output.setframerate(sample_rate)
output.writeframes(stream.read())
output.close()
temp.seek(0)
return chalice.Response(body=temp.read(),
headers={'Content-Type': 'audio/wav'},
status_code=200)
def dispatch(self, *args, **kwargs):
"""
This is the main entry point into the connexion application.
:param args:
:param kwargs:
:return:
"""
cr = self.current_request
context = cr.context
uri_params = cr.uri_params or {}
method = cr.method
query_params = cr.query_params or {}
path = context["resourcePath"].format(**uri_params)
if context["resourcePath"] in self.trailing_slash_routes:
if context["path"].endswith("/"):
path += "/"
else:
return chalice.Response(status_code=requests.codes.found,
headers={"Location": path + "/"},
body="")
req_body = cr.raw_body if cr._body is not None else None
# TODO figure out of host should be os.environ["API_DOMAIN_NAME"]
self.log.info({
"request": {
'method':
method,
'path':
path,
'sourceIp':
context['identity']['sourceIp'],
'content-length':
cr.headers.get('content-length', '-'),
'user-agent':
cr.headers.get('user-agent'),
'query-params':
str(query_params) if query_params is not None else ''
}
})
with self.connexion_request_context(
path=path,
base_url=os.environ["API_DOMAIN_NAME"],
query_string=list(query_params.items()),
method=method,
headers=list(cr.headers.items()),
data=req_body,
environ_base=cr.stage_vars):
try:
flask_res = self.connexion_full_dispatch_request()
status_code = flask_res._status_code
except Exception:
self.log.exception('The request failed!')
status_code = 500
finally:
self.log.info(
dict(dispatch=dict(method=method,
path=path,
status_code=status_code,
query_params=' ' + str(query_params)
if query_params is not None else '')))
res_headers = dict(flask_res.headers)
res_headers.update({
"X-AWS-REQUEST-ID":
self.lambda_context.aws_request_id,
"Strict-Transport-Security":
"max-age=31536000; includeSubDomains; preload"
})
res_headers.pop("Content-Length", None)
return chalice.Response(status_code=status_code,
headers=res_headers,
body=b"".join([c for c in flask_res.response
]).decode())
def slow_request():
time.sleep(40)
return chalice.Response(status_code=200,
headers={"Content-Type": "text/html"},
body="Slow request completed!")
def health():
return chalice.Response(status_code=200,
headers={"Content-Type": "text/html"},
body="OK")
def version():
data = {'version_info': {'version': os.getenv('MATRIX_VERSION')}}
return chalice.Response(status_code=200,
headers={'Content-Type': "application/json"},
body=data)
def dispatch(*args, **kwargs):
uri_params = app.current_request.uri_params or {}
path_pattern = app.current_request.context["resourcePath"]
path = path_pattern.format(**uri_params)
method = app.current_request.method
query_params = app.current_request.query_params
req_body = app.current_request.raw_body if app.current_request._body is not None else None
source_ip = app.current_request.context['identity']['sourceIp']
content_length = app.current_request.headers.get('content-length')
user_agent = app.current_request.headers.get('user-agent')
msg = {
"log-msg-type":
"analytics" if analytics_reply(method, path) else "info",
"system": "data-storage-service",
"request_info": {
"method":
method,
"path":
path,
"source_ip":
source_ip,
"content_length":
content_length if content_length else '-',
"user_agent":
user_agent,
"query_params":
' ' + str(query_params) if query_params is not None else ''
}
}
app.log.info(json.dumps(msg, indent=4))
def maybe_fake_504() -> bool:
fake_504_probability_str = app.current_request.headers.get(
"DSS_FAKE_504_PROBABILITY", "0.0")
try:
fake_504_probability = float(fake_504_probability_str)
except ValueError:
return None
if random.random() > fake_504_probability:
return None
return True
if not DeploymentStage.IS_PROD() and maybe_fake_504():
return timeout_response(method, path)
status_code = None
try:
with flask_app.test_request_context(
path=path,
base_url="https://{}".format(
app.current_request.headers["host"]),
query_string=list((app.current_request.query_params
or dict()).items()),
method=app.current_request.method,
headers=list(app.current_request.headers.items()),
data=req_body,
environ_base=app.current_request.stage_vars):
with nestedcontext.bind(
time_left=lambda: calculate_seconds_left(app),
skip_on_conflicts=True):
flask_res = flask_app.full_dispatch_request()
status_code = flask_res._status_code
except Exception:
app.log.exception('The request failed!')
finally:
res_headers = dict(flask_res.headers)
if query_params:
msg_query_params = str(query_params)
msg_started_at = query_params.get('started_at', '')
else:
msg_query_params = ''
msg_started_at = ''
msg = {
"log-msg-type":
"analytics" if analytics_reply(method, path) else "info",
"system": "data-storage-service",
"dispatch_info": {
"method": method,
"path": path,
"status_code": status_code,
"query_params": msg_query_params,
"started_at": msg_started_at,
"content-length": res_headers.get('Content-Length', ''),
"content-type": res_headers.get('Content-Type', '')
}
}
app.log.info(json.dumps(msg, indent=4))
# API Gateway/Cloudfront adds a duplicate Content-Length with a different value (not sure why)
res_headers.pop("Content-Length", None)
res_headers[
"Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload"
res_headers["X-AWS-REQUEST-ID"] = app.lambda_context.aws_request_id
if include_retry_after_header(return_code=status_code,
method=app.current_request.method,
uri=path):
res_headers['Retry-After'] = '10'
return chalice.Response(status_code=status_code,
headers=res_headers,
body="".join([
c.decode() if isinstance(c, bytes) else c
for c in flask_res.response
]))
def post_temperature():
data = app.current_request.json_body
data['temperature'] = Decimal(data['temperature'])
table.put_item(Item=data)
return chalice.Response(body='', status_code=204)
def version():
data = {'version_info': {'version': DSS_VERSION}}
return chalice.Response(status_code=requests.codes.ok,
headers={'Content-Type': "application/json"},
body=data)
def serve_swagger_ui():
return chalice.Response(status_code=200,
headers={"Content-Type": "text/html"},
body=swagger_ui_html)
def show_help():
app.log.debug('show_help')
return chalice.Response(
body=__doc__.format(stage=app.current_request.context['stage']),
headers={'Content-Type': 'text/plain'},
status_code=200)
