def reset_locked_items(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Reset locked items", event, logger) with Session() as session: try: reviews_in_progress = review_handler.get_old_reviews_in_progress( session) review_handler.delete_old_reviews_in_progress( reviews_in_progress, session) return { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": "{} Review(s) deleted".format(len(reviews_in_progress)) } except Exception: return { "statusCode": 400, "body": "Something went wrong. Check HTTP POST payload. Stacktrace: {}" .format(traceback.format_exc()) }
def confirm_submission(event, context): helper.log_method_initiated("Confirm submission", event, logger) submission_id = event['pathParameters']['submission_id'] stage = os.environ['STAGE'] if stage == 'prod': link = 'https://codetekt.org' else: link = 'https://{}.codetekt.org'.format(stage) body_html = io.open(os.path.join(os.path.dirname(__file__), 'resources', 'submission_confirmed_webpage.html'), mode='r', encoding='utf-8').read().format(link) with Session() as session: try: submission_handler.confirm_submission(submission_id, session) response = { 'statusCode': 200, 'headers': {"content-type": "text/html; charset=utf-8"}, 'body': body_html } except Exception: response = { "statusCode": 500, "body": "Could not confirm submission. Stacktrace: {}".format(traceback.format_exc()) } return helper.set_cors(response, event)
def get_open_items(event, context, is_test=False, session=None): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get open items for user", event, logger) if session == None: session = connection_handler.get_db_session(False, None) try: # get cognito user id id = helper.cognito_id_from_event(event) # get number of items from url path if ('queryStringParameters' in event and 'num_items' in event['queryStringParameters']): num_items = int(event['queryStringParameters']['num_items']) else: num_items = 5 user = user_handler.get_user_by_id(id, is_test, session) response = item_handler.get_open_items_for_user( user, num_items, is_test, session) items = response['items'] if len(items) < 1: response = { "statusCode": 404, "body": "There are currently no open items for this user." } else: # Transform each item into dict items_dict = [] for item in items: items_dict.append(item.to_dict()) body = { "items": items_dict, "is_open_review": response['is_open_review'] } response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps(body) } except Exception: response = { "statusCode": 400, "body": "Could not get user and/or num_items. Check URL path parameters. Stacktrace: {}" .format(traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_review(event, context): """Gets a review. Parameters ---------- - user_id is retrieved from the event - review_id is retrieved from query parameters Returns ------ - Status code 200 OK - The requested review """ logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get Review", event, logger) try: # get cognito id user_id = helper.cognito_id_from_event(event) except: return helper.get_text_response( 400, "Malformed request. Please provide a valid request.", event) with Session() as session: try: # get user from database user = user_handler.get_user_by_id(user_id, session) except: return helper.get_text_response(404, "No user found.", event) try: # Try to receive review review = review_handler.get_open_review(user_id, session) except: return helper.get_text_response(404, "No review found", event) if review is None: return responses.NoContent( event, 'No review in progress found for current user.' ).to_json_string() try: if review.user_id == user.id: return responses.Success( event, json.dumps( review.to_dict(with_questions_and_answers=True, with_tags=True))).to_json_string() else: return responses.Forbidden( event, 'User is not allowed to access review').to_json_string() except Exception as e: return responses.InternalError(event, "Internal error", e).to_json_string()
def get_items(event, context, is_test=False, session=None): """Returns the requested items. Parameters ---------- event: dict, required API Gateway Lambda Proxy Input Format #api-gateway-simple-proxy-for-lambda-input-format Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html context: object, required Lambda Context runtime methods and attributes Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ API Gateway Lambda Proxy Output Format: application/json Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html """ helper.log_method_initiated("Get items", event, logger) if session is None: session = get_db_session(is_test, session) if 'queryStringParameters' in event and event['queryStringParameters'] is not None: params = event['queryStringParameters'] for param in params: if hasattr(Item, param) is False: response = { "statusCode": 400, "body": "Could not get items. Query params do not match item model." } response_cors = helper.set_cors(response, event, is_test) return response_cors else: params = None items = item_handler.get_all_items(is_test, session, params=params) items_list = [] for item in items: items_list.append(item.to_dict()) response = { "statusCode": 200, 'headers': {"content-type": "application/json; charset=utf-8"}, "body": json.dumps(items_list) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_user(event, context, is_test=False, session=None): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get user", event, logger) if session == None: session = get_db_session(False, None) try: # get cognito id id = helper.cognito_id_from_event(event) try: user = user_handler.get_user_by_id(id, is_test, session) user_dict = user.to_dict() progress = user_handler.get_user_progress( user, is_test, session) total_rank = user_handler.get_user_rank( user, False, is_test, session) level_rank = user_handler.get_user_rank( user, True, is_test, session) solved_cases_total = user_handler.get_solved_cases( user, False, is_test, session) solved_cases_today = user_handler.get_solved_cases( user, True, is_test, session) exp_needed = user_handler.get_needed_exp(user, is_test, session) user_dict['progress'] = progress user_dict['total_rank'] = total_rank user_dict['level_rank'] = level_rank user_dict['solved_cases_total'] = solved_cases_total user_dict['solved_cases_today'] = solved_cases_today user_dict['exp_needed'] = exp_needed response = { "statusCode": 200, 'headers': {"content-type": "application/json; charset=utf-8"}, "body": json.dumps(user_dict) } except Exception: response = { "statusCode": 404, "body": "No user found with the specified id.{}".format(traceback.format_exc()) } except Exception: response = { "statusCode": 400, "body": "Could not get user. Check Cognito authentication. Stacktrace: {}".format(traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def handle_item_closed(event, context): try: helper.log_method_initiated("Send notification", event, logger) if "detail" not in event or "item_id" not in event["detail"]: return BadRequest(event, "Event contains no item_id.", add_cors_headers=False).to_json_string() with Session() as session: item_id = event["detail"]["item_id"] item = item_handler.get_item_by_id(item_id, session) if item is None: return BadRequest( event, f"No item was found with the given item_id [{item_id}].", add_cors_headers=False).to_json_string() item = item_handler.update_item_warning_tags(item, session) rating = int(item.result_score * 25) rating_text = get_rating_text(rating) parameters = dict(rating=rating, rating_text=rating_text, item_id=item.id, content=item.content) for submission in item.submissions: if submission.mail is not None: try: mail_sender.send_notification("item_closed", mail=submission.mail, replacements=parameters) except Exception as e: logger.exception(e) if submission.telegram_id is not None: try: mail_sender.send_notification( "item_closed", telegram_id=submission.telegram_id, replacements=parameters) except Exception as e: logger.exception(e) return Success(event, add_cors_headers=False).to_json_string() except Exception as e: return InternalError(event, "Error sending notification", e, add_cors_headers=False).to_json_string()
def submit_issue(event, context, is_test=False, session=None): helper.log_method_initiated("Submit issue", event, logger) if session == None: session = get_db_session(is_test, session) issue = Issue() #create object from issue_model.py (DB table: issues) issue = helper.body_to_object(event['body'], issue) # add ip address try: ip_address = event['requestContext']['identity']['sourceIp'] setattr(issue, 'ip_address', ip_address) except Exception: response = { "statusCode": 400, "body": "Could not read/add ip address. Check HTTP POST payload. Stacktrace: {}" .format(traceback.format_exc()) } issue = add_object(issue, is_test, session) if issue is None: response = { "statusCode": 400, "body": "Could not write issue to database. Check HTTP POST payload. Stacktrace: {}" .format(traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors response = send_issue_notification(issue) if response == False: response = { "statusCode": 500, "body": "Could not send issue mail. Stacktrace: {}".format( traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors else: response = {"statusCode": 201, "body": json.dumps(issue.to_dict())} response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_closed_items(event, context): helper.log_method_initiated("Get all closed items", event, logger) with Session() as session: try: # Get all closed items print("\n \n \n Getting items \n \n \n") allow_all_origins = False if 'queryStringParameters' in event and isinstance( event['queryStringParameters'], dict) and 'url' in event['queryStringParameters']: url_without_query_params = event['queryStringParameters'][ 'url'].split('?')[0] items = item_handler.get_closed_items_by_url( url_without_query_params, session) allow_all_origins = True else: items = item_handler.get_all_closed_items(session) if len(items) == 0: response = {"statusCode": 204, "body": "No closed items found"} else: items_list = [] for item in items: items_list.append( item.to_dict(with_tags=True, with_warnings=True)) response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps(items_list) } except Exception: response = { "statusCode": 400, "body": "Could not get closed items. Stacktrace: {}".format( traceback.format_exc()) } response_cors = helper.set_cors(response, event, allow_all_origins) return response_cors
def handle_item_rejected(event, context): try: helper.log_method_initiated("Handle item rejected", event, logger) if "detail" not in event or "item_id" not in event["detail"]: return BadRequest(event, "Event contains no item_id.", add_cors_headers=False).to_json_string() with Session() as session: item_id = event["detail"]["item_id"] item = item_handler.get_item_by_id(item_id, session) if item is None: return BadRequest( event, f"No item was found with the given item_id [{item_id}].", add_cors_headers=False).to_json_string() parameters = dict(content=item.content) for submission in item.submissions: if submission.mail is not None: try: mail_sender.send_notification("item_rejected", mail=submission.mail, replacements=parameters) except Exception as e: logger.exception(e) if submission.telegram_id is not None: try: mail_sender.send_notification( "item_rejected", telegram_id=submission.telegram_id, replacements=parameters) except Exception as e: logger.exception(e) return Success(event, add_cors_headers=False).to_json_string() except Exception as e: response = InternalError(event, "Error sending notification", e, add_cors_headers=False).to_json_string() logger.error(response) return response
def anonymize_unconfirmed_submissions(event, context): helper.log_method_initiated("Anonymize unconfirmed submissions", event, logger) with Session() as session: try: counter = submission_handler.anonymize_unconfirmed_submissions( session) logger.info("Anonymized {} submissions.".format(counter)) return except Exception: logger.error("Error while anonymizing submissions.") return
def attach_iot_policy(event, context): try: helper.log_method_initiated("Attach IoT policy.", event, logger) try: cognito_identity = helper.get_cognito_identity_from_event(event) except Exception: logger.error(ERROR_MSG_IDENTIY_ID_MISSING) return BadRequest(event, ERROR_MSG_IDENTIY_ID_MISSING).to_json_string() try: user_id = helper.cognito_id_from_event(event) except Exception: logger.error(ERROR_MSG_USER_ID_MISSING) return BadRequest(event, ERROR_MSG_USER_ID_MISSING).to_json_string() client = BotoClientProvider().get_client('iot') # Get the policy for the current user or create it try: policy = client.get_policy(policyName=user_id) except client.exceptions.ResourceNotFoundException as e: resource = POLICY_DOCUMENT_STRING_FORMAT.format(user_id) policy_document['Statement'][0]['Resource'] = resource policy = client.create_policy( policyName=user_id, policyDocument=json.dumps(policy_document)) attached_policies = client.list_attached_policies( target=cognito_identity) # If the policy is already attached to the current identity return if(policy['policyName'] in [attached_policy['policyName'] for attached_policy in attached_policies['policies']]): logger.info(INFO_MSG_POLICY_ALREADY_ATTACHED) return Success(event).to_json_string() client.attach_policy( policyName=policy['policyName'], target=cognito_identity) logger.info(INFO_MSG_POLICY_ATTACHED) return Success(event).to_json_string() except Exception as e: logger.exception(ERROR_MSG_EXCEPTION) return InternalError(event, ERROR_MSG_EXCEPTION, e).to_json_string()
def delete_user(event, context, is_test=False, session=None): """Deletes a user from DB and Cognito. Parameters ---------- event: dict, required API Gateway Lambda Proxy Input Format Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format context: object, required Lambda Context runtime methods and attributes Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ API Gateway Lambda Proxy Output Format: application/json Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html """ logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Delete user", event, logger) try: if session == None: session = get_db_session(is_test, session) user_handler.delete_user(event, is_test, session) response = {"statusCode": 200} except Exception: response = { "statusCode": 500, "body": "User could not be deleted. Exception: {}".format( traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def anonymize_unconfirmed_submissions(event, context, is_test=False, session=None): helper.log_method_initiated("Anonymize unconfirmed submissions", event, logger) if session == None: session = get_db_session(False, None) try: counter = submission_handler.anonymize_unconfirmed_submissions( is_test, session) logger.info("Anonymized {} submissions.".format(counter)) return except Exception: logger.error("Error while anonymizing submissions.") return
def get_tags_for_item(event, context, is_test=False, session=None): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get tags by item id", event, logger) if session is None: session = get_db_session(is_test, None) try: # get id (str) from path id = event['pathParameters']['item_id'] try: tag_objects = tag_handler.get_tags_by_itemid(id, is_test, session) tags = [] for obj in tag_objects: tags.append(obj.to_dict()['tag']) except Exception as e: response = { "statusCode": 404, "body": "No tags found. Exception: {}".format(e) } except Exception as e: response = { "statusCode": 400, "body": "Could not get item ID. Check HTTP GET payload. Exception: {}". format(e) } body_json = {"Tags": tags} response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps(body_json) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def topics_to_json(event, context, is_test=False, session=None): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Convert topics modelling output to json", event, logger) for record in event['Records']: bucket = record['s3']['bucket']['name'] key = unquote_plus(record['s3']['object']['key']) logger.info('bucket: {}'.format(bucket)) logger.info('key: {}'.format(key)) # download new topic and term list download_path = '/tmp/' os.chdir(download_path) output_file_name = "output.tar.gz" s3_client.download_file(bucket, key, download_path + output_file_name) # extract topics and terms with tarfile.open(download_path + output_file_name) as tar: tar.extractall(download_path) new_topics_json = {} csv_file = csv.DictReader(open("topic-terms.csv")) for row in csv_file: category = "c" + row["topic"] tag = "t" + row["topic"] term = row["term"] if category not in new_topics_json: new_topics_json[category] = {} if tag not in new_topics_json[category]: new_topics_json[category][tag] = {} if term not in new_topics_json[category][tag]: new_topics_json[category][tag][term] = row["weight"] # Write the tag-terms json file new_json_file_name = "tag-terms_new.json" with open(new_json_file_name, "w") as f: json.dump(new_topics_json, f, indent=4) # upload tag-terms json file destkey = 'topics/' + new_json_file_name s3_client.upload_file(new_json_file_name, bucket, destkey) # upload tag-terms json file destkey = 'topics/' + diff_json_file_name s3_client.upload_file(diff_json_file_name, bucket, destkey)
def get_item_types(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get Item Types", event, logger) with Session() as session: try: item_types = item_type_handler.get_all_item_types(session) if item_types == None: response = { "statusCode": 201, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": [] } else: response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps( [item_type.to_dict() for item_type in item_types]) } except Exception: response = { "statusCode": 500, "body": "Could not get item types. Stacktrace: {}".format( traceback.format_exc()) } return helper.set_cors(response, event)
def create_user(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Create user from cognito", event, logger) with Session() as session: if event['triggerSource'] == "PostConfirmation_ConfirmSignUp": user = User() user.name = event['userName'] user.id = event['request']['userAttributes']['sub'] if user.id == None or user.name == None: raise Exception("Something went wrong!") user = user_handler.create_user(user, session) client = boto3.client('cognito-idp') client.admin_add_user_to_group(UserPoolId=event['userPoolId'], Username=user.name, GroupName='Detective') return event
def get_item(event, context): helper.log_method_initiated("Get closed item", event, logger) with Session() as session: try: # Get item id from path params item_id = event['pathParameters']['item_id'] except KeyError: return helper.get_text_response(400, 'Could not read item id from path params', event) item = item_handler.get_item_by_id(item_id, session) if item is None: return helper.get_text_response(404, 'No item found with the specified id', event) if item.status != 'closed': return helper.get_text_response(403, 'You are not allowed to access an item, that has not been closed', event) response = { "statusCode": 200, 'headers': {"content-type": "application/json; charset=utf-8"}, "body": json.dumps(item.to_dict(True, True, True, True, True, True)) } return helper.set_cors(response, event)
def submit_issue(event, context, is_test=False, session=None): helper.log_method_initiated("Submit issue", event, logger) if session == None: session = get_db_session(is_test, session) issue = Issue() issue = helper.body_to_object(event['body'], issue) issue = add_object(issue, is_test, session) if issue is None: response = { "statusCode": 400, "body": "Could not write issue to database. Check HTTP POST payload. Stacktrace: {}" .format(traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors response = send_issue_notification(issue) if response == False: response = { "statusCode": 500, "body": "Could not send issue mail. Stacktrace: {}".format( traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors else: response = {"statusCode": 201, "body": json.dumps(issue.to_dict())} response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_closed_items(event, context, is_test=False, session=None): helper.log_method_initiated("Get all closed items", event, logger) if session == None: session = connection_handler.get_db_session(False, None) try: # Get all closed items items = item_handler.get_all_closed_items(is_test, session) if len(items) == 0: response = {"statusCode": 204, "body": "No closed items found"} else: items_list = [] for item in items: items_list.append(item.to_dict(with_tags=True)) response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps(items_list) } except Exception: response = { "statusCode": 400, "body": "Could not get closed items. Stacktrace: {}".format( traceback.format_exc()) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_review(event, context, is_test=False, session=None): """Gets a review. Parameters ---------- - user_id is retrieved from the event - review_id is retrieved from query parameters Returns ------ - Status code 200 OK - The requested review """ logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get Review", event, logger) if session == None: session = connection_handler.get_db_session(False, None) try: # get review id from url query params review_id = event['pathParameters']['review_id'] # get cognito id user_id = helper.cognito_id_from_event(event) except: return helper.get_text_response( 400, "Malformed request. Please provide a valid request.", event, is_test) try: # get user from database user = user_handler.get_user_by_id(user_id, is_test, session) except: return helper.get_text_response(404, "No user found.", event, is_test) # Try to receive item try: review = review_handler.get_review_by_id(review_id, is_test, session) except: return helper.get_text_response(404, "No review found", event, is_test) try: if review.user_id == user.id: response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps(review.to_dict_with_questions_and_answers()) } else: return helper.get_text_response(403, "Forbidden", event, is_test) except: return helper.get_text_response( 500, "Internal server error. Stacktrace: {}".format( traceback.format_exc()), event, is_test) response_cors = helper.set_cors(response, event, is_test) return response_cors
def get_user_ranking(event, context, is_test=False, session=None): """ returns a dictionary with list top_users: 10 top users list top_users_by_level: 10 top users on the user's level list top_users_by_period: 10 top users in a period p (1 week) """ n = 10 p = 1 descending = True attr = 'experience_points' logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Get User Ranking", event, logger) if session == None: session = get_db_session(False, None) user_ranking_dict = {} top_users_list = [] top_users_by_level_list = [] top_users_by_period_list = [] try: my_user_id = helper.cognito_id_from_event(event) except Exception: response = { "statusCode": 400, "body": "Could not get User. Check Cognito Authentication. Stacktrace: {}".format(traceback.format_exc()) } try: helper.log_method_initiated("Get Top n Users", event, logger) top_users = user_handler.get_top_users(n, attr, descending, is_test, session) for user in top_users: top_users_list.append(user.to_dict()) user_ranking_dict['top_users'] = top_users_list helper.log_method_initiated("Get Top n Users on the User's Level", event, logger) my_user = user_handler.get_user_by_id(my_user_id, is_test, session) user_level = my_user.level_id top_users_by_level = user_handler.get_top_users_by_level(user_level, n, attr, descending, is_test, session) for user in top_users_by_level: top_users_by_level_list.append(user.to_dict()) user_ranking_dict['top_users_by_level'] = top_users_by_level_list helper.log_method_initiated("Get Top n Users in a Period", event, logger) top_users_by_period = user_handler.get_top_users_by_period(n, p, attr, descending, is_test, session) for user in top_users_by_period: top_users_by_period_list.append(user.to_dict()) user_ranking_dict['top_users_by_period'] = top_users_by_period_list except Exception: response = { "statusCode": 500, "body": "Server Error. Stacktrace: {}".format(traceback.format_exc()) } response = { "statusCode": 200, 'headers': {"content-type": "application/json; charset=utf-8"}, "body": json.dumps(user_ranking_dict) } response_cors = helper.set_cors(response, event, is_test) return response_cors
def update_review(event, context, is_test=False, session=None): """Updates an existing review Args: event: AWS API Gateway event context ([type]): [description] is_test (bool, optional): Whether the function is executed in test mode. Defaults to False. session (optional): An sql alchemy session. Defaults to None. Returns ------ - Status code 200, 400, 404 or 500 (Created) - The updated review """ logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Create Review", event, logger) if session == None: session = connection_handler.get_db_session(False, None) try: user_id = helper.cognito_id_from_event(event) body = json.loads(event['body']) if isinstance(event['body'], str) else event['body'] except: return helper.get_text_response( 400, "Malformed request. Please provide a valid request.", event, is_test) if 'id' not in body: return helper.get_text_response( 400, "Malformed request. Please provide a review id.", event, is_test) try: review = review_handler.get_review_by_id(body['id'], is_test, session) except: return helper.get_text_response(404, "No review found", event, is_test) try: user = user_handler.get_user_by_id(user_id, is_test, session) except: return helper.get_text_response(404, "No user found.", event, is_test) if review.user_id != user.id: return helper.get_text_response(403, "Forbidden.", event, is_test) # If review is set closed if 'status' in body and body['status'] == 'closed': try: review = review_handler.close_review(review, is_test, session) if review.item.status == 'closed': notifications.notify_users(is_test, session, review.item) response = { "statusCode": 200, "body": json.dumps(review.to_dict_with_questions_and_answers()) } response_cors = helper.set_cors(response, event, is_test) return response_cors except: return helper.get_text_response( 500, "Internal server error. Stacktrace: {}".format( traceback.format_exc()), event, is_test) # If answers are appended elif 'questions' in body: if not isinstance(body['questions'], list): return helper.get_text_response( 400, "Malformed request. Please provide a valid request.", event, is_test) for question in body['questions']: if 'answer_value' in question: answer_value = question['answer_value'] else: return helper.get_text_response( 400, "Malformed request. Please provide a valid request.", event, is_test) if answer_value is not None: # Check if conditionality is met if question['parent_question_id'] is not None: for q in body['questions']: if q['question_id'] == question['parent_question_id']: parent_question = q parent_answer = review_answer_handler.get_parent_answer( question['answer_id'], is_test, session) if parent_question['answer_value'] > question[ 'upper_bound'] or parent_question[ 'answer_value'] < question['lower_bound']: return helper.get_text_response( 400, "Bad request. Please adhere to conditionality of questions.", event, is_test) # Update answer in db try: review_answer_handler.set_answer_value( question['answer_id'], question['answer_value'], is_test, session) except: return helper.get_text_response( 500, "Internal server error. Stacktrace: {}".format( traceback.format_exc()), event, is_test) response = { "statusCode": 200, "body": json.dumps(review.to_dict_with_questions_and_answers()) } response_cors = helper.set_cors(response, event, is_test) return response_cors else: return helper.get_text_response( 400, "Bad request. Please adhere to conditionality of questions.", event, is_test)
def update_review(event, context): """Updates an existing review Args: event: AWS API Gateway event context ([type]): [description]. Returns ------ - Status code 200, 400, 404 or 500 (Created) - The updated review """ logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Update Review", event, logger) try: user_id = helper.cognito_id_from_event(event) body = json.loads(event['body']) if isinstance(event['body'], str) else event['body'] except: return responses.BadRequest( event, "Malformed request. Please provide a valid request." ).to_json_string() if 'id' not in body: return responses.BadRequest( event, "Malformed request. Please provide a review id.").to_json_string() with Session() as session: try: review = review_handler.get_review_by_id(body['id'], session) except: return responses.NotFound(event, "No review found.").to_json_string() try: user = user_handler.get_user_by_id(user_id, session) except: return responses.NotFound(event, "No user found.").to_json_string() if review.user_id != user.id: return responses.Forbidden(event).to_json_string() # If review is set closed if 'status' in body and body['status'] == 'closed': try: review = review_handler.close_review(review, session) if review.item.status == 'closed': EventPublisher().publish_event('codetekt.review_service', 'item_closed', {'item_id': review.item.id}) return responses.Success( event, json.dumps( review.to_dict(with_questions_and_answers=True, with_tags=True))).to_json_string() except: return helper.get_text_response( 500, "Internal server error. Stacktrace: {}".format( traceback.format_exc()), event) # If answers are appended if 'questions' in body: if not isinstance(body['questions'], list): return responses.BadRequest( event, "Malformed request. Please provide a valid request." ).to_json_string() for question in body['questions']: if 'answer_value' in question: answer_value = question['answer_value'] else: return responses.BadRequest( event, "Malformed request. Please provide a valid request." ).to_json_string() if answer_value is not None: # Check if conditionality is met if question['parent_question_id'] is not None: for q in body['questions']: if q['question_id'] == question[ 'parent_question_id']: parent_question = q if parent_question['answer_value'] > question[ 'upper_bound'] or parent_question[ 'answer_value'] < question['lower_bound']: return responses.BadRequest( event, "Bad request. Please adhere to conditionality of questions." ).to_json_string() # Update answer in db try: review_answer_handler.set_answer_value( question['answer_id'], question['answer_value'], session) except Exception as e: return responses.InternalError( event, exception=e).to_json_string() # Save qualitative_comment if 'comment' in body: if body['comment']: if review.comment is None: try: comment_handler.create_comment(session, comment=body['comment'], user_id=user_id, parent_type='review', parent_id=review.id, is_review_comment=True) except Exception as e: return responses.InternalError( event, "Could not create comment for item", e).to_json_string() elif review.comment.comment != body['comment']: review.comment.comment = body['comment'] session.merge(review) if 'tags' in body: if body['tags'] is None: body['tags'] = [] if isinstance(body['tags'], list): try: db_tags = [ item_tag.tag.tag for item_tag in tag_handler.get_item_tags_by_review_id( review.id, session) ] body_tags_upper = [tag.upper() for tag in body['tags']] db_tags_upper = [tag.upper() for tag in db_tags] tags_to_add = [ tag for tag in body['tags'] if tag.upper() not in db_tags_upper ] tags_to_delete = [ tag for tag in db_tags if tag.upper() not in body_tags_upper ] for tag in tags_to_add: tag_handler.store_tag_for_item(review.item_id, tag, session, review.id) for tag in tags_to_delete: tag_handler.delete_itemtag_by_tag_and_review_id( tag, review.id, session) except Exception as e: return responses.InternalError( event, "Could not create tags for item", e).to_json_string() return responses.Success( event, json.dumps( review.to_dict(with_questions_and_answers=True, with_tags=True))).to_json_string()
def submit_item(event, context): client = boto3.client('stepfunctions', region_name="eu-central-1") helper.log_method_initiated("Item submission", event, logger) with Session() as session: try: body = event['body'] if isinstance(body, str): body_dict = json.loads(body) else: body_dict = body content = body_dict["content"] del body_dict["content"] if "type" in body_dict: type = body_dict["type"] del body_dict["type"] else: type = None if "item_type_id" in body_dict: item_type_id = body_dict["item_type_id"] del body_dict["item_type_id"] else: item_type_id = None if "item" in body_dict: del body_dict["item"] submission = Submission() helper.body_to_object(body_dict, submission) # add ip address ip_address = event['requestContext']['identity']['sourceIp'] setattr(submission, 'ip_address', ip_address) try: # Item already exists, item_id in submission is the id of the found item item = item_handler.get_item_by_content(content, session) submission.item_id = item.id new_item_created = False except Exception: # Item does not exist yet, item_id in submission is the id of the newly created item item = Item() item.content = content item.item_type_id = item_type_id item.type = type item = item_handler.create_item(item, session) new_item_created = True if content: str_urls = re.findall( 'http[s]?://(?:[a-zA-ZäöüÄÖÜ]|[0-9]|[$-_@.&+]|[!*(),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', content) url_handler.prepare_and_store_urls(item, str_urls, session) submission.item_id = item.id submission.status = item.status # Create submission submission_handler.create_submission_db(submission, session) if submission.mail: if item.status != 'Unsafe': send_confirmation_mail(submission) # Create response if item.status == 'Unsafe': response = { "statusCode": 403, "headers": { "content-type": "application/json; charset=utf-8", "new-item-created": str(new_item_created) }, "body": "Item not valid" } else: response = { "statusCode": 201, "headers": { "content-type": "application/json; charset=utf-8", "new-item-created": str(new_item_created) }, "body": json.dumps(item.to_dict()) } except Exception as e: logger.error("Couldn't submit item. Exception: %s", e) response = { "statusCode": 400, "body": "Could not create item and/or submission. Check HTTP POST payload. Stacktrace: {}" .format(traceback.format_exc()) } ## start SearchFactChecks only for safe items if (item.status != 'Unsafe') and (new_item_created == True): stage = os.environ['STAGE'] client.start_execution( stateMachineArn= 'arn:aws:states:eu-central-1:891514678401:stateMachine:SearchFactChecks_new-' + stage, name='SFC_' + item.id, input="{\"item\":{" "\"id\":\"" + item.id + "\"," "\"content\":\"" + remove_control_characters(item.content) + "\" } }") response_cors = helper.set_cors(response, event) return response_cors
def post_tags_for_item(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Post tags by item id", event, logger) with Session() as session: # get list of existing tags tags_existing = [] try: # get id (str) from path id = event['pathParameters']['item_id'] tag_objects = tag_handler.get_tags_by_itemid(id, session) for obj in tag_objects: tags_existing.append(obj.to_dict()['tag']) except Exception as e: response = { "statusCode": 400, "body": "Could not get item ID. Check HTTP POST payload. Exception: {}" .format(e) } # get list of posted tags body = event['body'] if isinstance(body, str): body_dict = json.loads(body) else: body_dict = body if 'tags' in body_dict: tags_posted = body_dict['tags'] else: tags_posted = [] # save tags if tags_posted != []: for str_tag in tags_posted: tag_handler.store_tag_for_item(id, str_tag, session) # create response tags_new = list(set(tags_posted) - set(tags_existing)) tags_counter_increased = list(set(tags_posted) - set(tags_new)) response = { "statusCode": 200, 'headers': { "content-type": "application/json; charset=utf-8" }, "body": json.dumps({ "added new tags": tags_new, "increased tag counter": tags_counter_increased }) } response_cors = helper.set_cors(response, event) return response_cors
def create_tagreport(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Report tags not considered in taxonomy", event, logger) # list with tags in the database but not covered by the taxonomy terms_new = [] with Session() as session: # read all tags from database tag_list = tag_handler.get_all_tags(session) for LanguageCode in UpdateFactChecks.model_languages: logger.info("LanguageCode: {}".format(LanguageCode)) # Read language specific taxonomy taxonomy_json = download_taxonomy(LanguageCode) term_list = [] term_test = "" # create list of terms already considered in taxonomy for category in taxonomy_json: if category == "similarity-threshold": continue for tag in taxonomy_json[category]: if tag.lower() not in term_list: term_list.append(tag.lower()) if category == "excluded-terms": continue for term in taxonomy_json[category][tag]: term = term.lower() if term_test == "": term_test = term if term not in term_list: term_list.append(term) tagreport_json = {"additional_tags": []} # Create list with tags not considered in taxonomy sim_input = [] for tag in tag_list: tag = str.lower(tag.tag) if (tag not in term_list) and (tag not in terms_new): # test if the tag is in the model vocabulary sim_input.append("\"" + term_test + "\"" + ",\"" + tag + "\"") terms_new.append(tag) # call sagemaker endpoint for similarity prediction to verify if tags are in vocabulary # limit number of model calls (10 calls with 100 predictions each last about 15s) if (len(sim_input)) > 1000: sim_input = sim_input[0:1000] while (len(sim_input)) > 0: # do similarity prediction for 100 tags at most payload_list = sim_input[0:100] payload = '\n'.join(payload_list) try: response = SearchFactChecks.post_DocSim(LanguageCode, payload) if not response.ok: raise Exception('Received status code {}.'.format( response.status_code)) result = response.text scores = json.loads(result) for score in scores: tag = terms_new.pop(0) sim_input.pop(0) # Is tag in vocabulary? if score != '0.00': tagreport_json["additional_tags"].append(tag) logger.info( "Taxonomy does not consider tag {}".format(tag)) except Exception as e: logger.error('DocSim error: {}.'.format(e)) continue upload_tagreport(tagreport_json, LanguageCode)
def predict_tags(event, context): logger = logging.getLogger() logger.setLevel(logging.INFO) helper.log_method_initiated("Predict tags for claim", event, logger) text = "" if 'Text' in event: text = str(event['Text']) if len(text) == 0: logger.error("There is no Text!") return [] # Check if LanguageCode is supported if 'LanguageCode' in event: LanguageCode = event['LanguageCode'] if LanguageCode not in UpdateFactChecks.model_languages: logger.error( "Language Code {} not supported!".format(LanguageCode)) return [] else: logger.error("There is no Language Code!") return [] taxonomy_json = download_taxonomy(LanguageCode) similarity_threshold = taxonomy_json["similarity-threshold"] for stopword in ["\"", ",", ".", "!", "?", "«", "»", "(", ")", "-"]: text = text.replace(stopword, " ") new_text = "" text_split = [] for substr in text.split(): if str.lower(substr) not in taxonomy_json["excluded-terms"]: new_text += substr + " " substr = str.lower(substr) text_split.append(substr) sim_input = [] term2tags = [] tags = [] for category in taxonomy_json: if category == "similarity-threshold": continue if category == "excluded-terms": continue for tag in taxonomy_json[category]: for term in taxonomy_json[category][tag]: sim_input.append("\"" + term + "\"" + ",\"" + new_text + "\"") term2tags.append(tag) if (term.lower() in text_split) and (tag not in tags): tags.append(tag) if tags != []: return tags # call sagemaker endpoint for similarity prediction try: if sim_input == []: raise Exception('Nothing to compare.') payload = '\n'.join(sim_input) response = SearchFactChecks.post_DocSim(LanguageCode, payload) if not response.ok: raise Exception('Received status code {}.'.format( response.status_code)) result = response.text scores = json.loads(result) ind = 0 for score in scores: if score == '': ind = ind + 1 continue sim = float(score) if sim > similarity_threshold: if term2tags[ind] not in tags: tags.append(term2tags[ind]) ind = ind + 1 except Exception as e: logger.error('DocSim error: {}.'.format(e)) return tags
def update_item(event, context, is_test=False, session=None): """Updates an item. Parameters ---------- event: dict, required API Gateway Lambda Proxy Input Format #api-gateway-simple-proxy-for-lambda-input-format Event doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html context: object, required Lambda Context runtime methods and attributes Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html Returns ------ API Gateway Lambda Proxy Output Format: application/json Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html """ helper.log_method_initiated("Update item", event, logger) if session is None: session = get_db_session(is_test, session) item_id = event['pathParameters']['item_id'] item = item_handler.get_item_by_id(item_id, is_test, session) if item is None: response = { "statusCode": 404, "body": "No item found with the specified id." } response_cors = helper.set_cors(response, event, is_test) return response_cors body = event['body'] body = json.loads(body) if isinstance(body, str) else body for key in body: if hasattr(item, key): if not isinstance(body[key], dict) and not isinstance( body[key], list): setattr(item, key, body[key]) else: response = { "statusCode": 400, "body": "Could not update item. Provided input does not match item model." } response_cors = helper.set_cors(response, event, is_test) return response_cors item = update_object(item, is_test, session) if item is None: response = { "statusCode": 500, "body": "Could not write changes to db. Event id: {}".format( event['requestContext']['requestId']) } response_cors = helper.set_cors(response, event, is_test) return response_cors response = {"statusCode": 200, "body": json.dumps(item.to_dict())} response_cors = helper.set_cors(response, event, is_test) return response_cors