def lambda_handler(event, context): """Collects data on a passed in References from eight tables and combines them into a single Json. Parameters: event (Dict):A series of key value pairs used in the search. Returns: out_json (Json):Nested Json responce of the eight tables data. """ logger.info("get_query_dashboard Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.QuerySearch(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} search_list = [ 'query_reference', 'survey_period', 'query_type', 'ru_reference', 'survey_code', 'query_status' ] try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: logger.info("Fetching Table Model: {}".format("query")) table_model = alchemy_functions.table_model(engine, metadata, "query") logger.info("Fetching Table Model: {}".format("contributor")) other_model = alchemy_functions.table_model(engine, metadata, "contributor") logger.info("Building SQL Statement: {}".format("Combined Tables")) all_query_sql = session.query(table_model, other_model)\ .join(other_model, table_model.columns.ru_reference == other_model.columns.ru_reference) added_query_sql = 0 for criteria in search_list: if criteria not in event.keys(): logger.info( "No parameters have been passed for {}.".format(criteria)) continue added_query_sql += 1 all_query_sql = all_query_sql.filter( getattr(table_model.columns, criteria) == event[criteria]) if added_query_sql == 0: all_query_sql = all_query_sql.filter( table_model.columns.query_status == 'Open') logger.info("Converting Data: {}".format("query")) query = alchemy_functions.to_df(all_query_sql) except db.exc.OperationalError as e: logger.error("Operational Error, When Retrieving Data: {} {}".format( "query", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Retrieve Data: {}".format( "query") } } except Exception as e: logger.error("General Error, Problem Retrieving " + "Data From The Table: {} {}".format("query", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Retrieve Data: {}".format("query") } } table_list = { 'step_exception': None, 'question_anomaly': None, 'failed_vet': None, 'query_task': None, 'query_task_update': None } logger.info("Initialising JSON.") out_json = '{"Queries":[ ' for index, query_row in query.iterrows(): curr_query = query[query['query_reference'] == query_row['query_reference']] if curr_query.empty: logger.info("No Queries Found.") continue ref = int(curr_query['query_reference'].iloc[0]) ru = curr_query['ru_reference'].iloc[0] period = str(curr_query['survey_period'].iloc[0]) survey = curr_query['survey_code'].iloc[0] try: for current_table in table_list: logger.info("Fetching Table Model: {}".format(current_table)) table_model = alchemy_functions.table_model( engine, metadata, current_table) logger.info("Fetching Table Data: {}".format(current_table)) if current_table in [ 'step_exception', 'query_task', 'query_task_update' ]: statement = session.query(table_model).filter( table_model.columns.query_reference == ref) elif current_table == "failed_vet": other_model = alchemy_functions.table_model( engine, metadata, "vet") statement = session.query(table_model, other_model)\ .filter( db.and_(table_model.columns.survey_period == period, table_model.columns.survey_code == survey, table_model.columns.ru_reference == ru)) else: statement = session.query(table_model).filter( db.and_(table_model.columns.survey_period == period, table_model.columns.survey_code == survey, table_model.columns.ru_reference == ru)) logger.info("Converting Data: {}".format(current_table)) table_list[current_table] = alchemy_functions.to_df(statement) except db.exc.OperationalError as e: logger.error( "Operational Error, When Retrieving Data: {} {}".format( current_table, e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To " + "Retrieve Data: {}".format(current_table) } } except Exception as e: logger.error("General Error, Problem Retrieving Data " + "From The Table: {} {}".format(current_table, e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed " + "To Retrieve Data: {}".format(current_table) } } logger.info("Creating JSON.") curr_query = json.dumps(curr_query.to_dict(orient='records'), sort_keys=True, default=str) curr_query = curr_query[1:-2] out_json += curr_query + ',"Exceptions":[ ' for index, step_row in table_list['step_exception'].iterrows(): row_step = step_row['step'] curr_step = table_list['step_exception'][ (table_list['step_exception']['step'] == row_step) & (table_list['step_exception']['run_id'] == step_row['run_id'])] if curr_step.empty: logger.info("No Exceptions Found.") continue curr_step = json.dumps(curr_step.to_dict(orient='records'), sort_keys=True, default=str) curr_step = curr_step[1:-2] out_json += (curr_step + ',"Anomalies":[ ') for index, ano_row in table_list['question_anomaly'].iterrows(): curr_ano = table_list['question_anomaly'][ (table_list['question_anomaly']['step'] == ano_row['step']) & (table_list['question_anomaly']['question_number'] == ano_row['question_number']) & (table_list['question_anomaly']['step'] == row_step)] if curr_ano.empty: logger.info("No Anomalies Found.") continue curr_ano = json.dumps(curr_ano.to_dict(orient='records'), sort_keys=True, default=str) curr_ano = curr_ano[1:-2] out_json += curr_ano + ',"FailedVETs":' curr_per = table_list['failed_vet'][ (table_list['failed_vet']['step'] == row_step) & (table_list['failed_vet']['question_number'] == ano_row['question_number'])] curr_per = json.dumps(curr_per.to_dict(orient='records'), sort_keys=True, default=str) out_json += (curr_per + '},') out_json = out_json[:-1] out_json += ']},' out_json = out_json[:-1] out_json += '],"QueryTasks":[ ' for index, tas_row in table_list['query_task'].iterrows(): curr_tas = table_list['query_task'][ (table_list['query_task']['query_reference'] == tas_row['query_reference']) & (table_list['query_task']['task_sequence_number'] == tas_row['task_sequence_number'])] if curr_tas.empty: logger.info("No Tasks Found.") continue curr_tas = json.dumps(curr_tas.to_dict(orient='records'), sort_keys=True, default=str) curr_tas = curr_tas[1:-2] out_json += curr_tas out_json = out_json + ',"QueryTaskUpdates":' if not table_list['query_task_update'].empty: curr_up = table_list['query_task_update'][ (table_list['query_task_update']['query_reference'] == tas_row['query_reference']) & (table_list['query_task_update']['task_sequence_number'] == tas_row['task_sequence_number'])] curr_up = json.dumps(curr_up.to_dict(orient='records'), sort_keys=True, default=str) else: curr_up = "[]" out_json += curr_up + '},' out_json = out_json[:-1] out_json += ']},' out_json = out_json[:-1] out_json += ']}' out_json = out_json.replace("NaN", "null") try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Database " + "Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, Database " + "Session Closed Badly." } } try: io_validation.Queries(strict=True).loads(out_json) except ValidationError as e: logger.error("Output: {}".format(out_json)) logger.error("Failed To Validate The Output: {}".format(e.messages)) return {"statusCode": 500, "body": {"Error": e.messages}} logger.info("get_query_dashboard Has Successfully Run.") return {"statusCode": 200, "body": json.loads(out_json)}
def lambda_handler(event, context): """Takes a contributor dictonary and updates the comments. Parameters: event (Dict):A series of key value pairs with the related comments to be added. Returns: Json message reporting the success of the update. """ logger.info("update_contributor Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.ContributorUpdate(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To " + "Connect To The Database." } } current_time = str(datetime.now()) try: logger.info( "Fetching Table Model: {}".format("contributor_survey_period")) table_model = alchemy_functions.table_model( engine, metadata, 'contributor_survey_period') logger.info("Updating Table: {}".format("contributor_survey_period")) session.query(table_model)\ .filter(db.and_( table_model.columns.survey_period == event['survey_period'], table_model.columns.survey_code == event['survey_code'], table_model.columns.ru_reference == event['ru_reference']))\ .update({table_model.columns.additional_comments: event['additional_comments'], table_model.columns.contributor_comments: event['contributor_comments'], table_model.columns.last_updated: current_time}, synchronize_session=False) except db.exc.OperationalError as e: logger.error("Operational Error, When Updating Data: {} {}".format( "contributor_survey_period", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format("contributor_survey_period") } } except Exception as e: logger.error("General Error, " + "Problem Updating Data From The Table: {} {}".format( "contributor_survey_period", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Update Data: {}".format( "contributor_survey_period") } } try: logger.info("Commit Session.") session.commit() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Commit Changes: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Commit Changes." } } except Exception as e: logger.error( "General Error, Failed To Commit Changes To Database: {}".format( e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To " + "Commit Changes To The Database." } } try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error("General Error, Failed To Close " + "The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Database Session Closed Badly." } } logger.info("update_contributor Has Successfully Run.") return { "statusCode": 200, "body": { "Success": "Successfully Updated The Tables." } }
def lambda_handler(event, context): """Takes a survey_period dictonary and updates the MI information based in the new run. Parameters: event (Dict):A series of key value pairs. Returns: Json message reporting the success of the update. """ logger.info("update_survey_period Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.SurveyPeriod(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: logger.info("Fetching Table Model: {}".format("survey_period")) table_model = alchemy_functions.table_model(engine, metadata, 'survey_period') logger.info("Updating Table: {}".format("survey_period")) session.query(table_model).filter(db.and_( table_model.columns.survey_period == event['survey_period'], table_model.columns.survey_code == event['survey_code']))\ .update({ table_model.columns.active_period: event['active_period'], table_model.columns.number_of_responses: event['number_of_responses'], table_model.columns.number_cleared: event['number_cleared'], table_model.columns.number_cleared_first_time: event['number_cleared_first_time'], table_model.columns.sample_size: event['sample_size']}, synchronize_session=False) except db.exc.OperationalError as e: logger.error("Operational Error, When Updating Data: {} {}".format( "survey_period", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format("survey_period") } } except Exception as e: logger.error("General Error, Problem Updating " + "Data From The Table: {} {}".format("survey_period", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Update Data: {}".format( "survey_period") } } try: logger.info("Commit Session.") session.commit() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Commit Changes: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Commit Changes." } } except Exception as e: logger.error( "General Error, Failed To Commit Changes To Database: {}".format( e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Commit Changes To The Database." } } try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error("General Error, " + "Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Database Session Closed Badly." } } logger.info("update_survey_period Has Successfully Run.") return { "statusCode": 200, "body": { "Success": "Successfully Updated The Tables." } }
def lambda_handler(event, context): """Collects data on a passed in Reference from six tables and combines them into a single Json. Parameters: event (Dict):A single key value pair of ru_reference and a string number. Returns: out_json (Json):Nested Json responce of the six tables data. """ logger.info("get_contributor Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.ContributorSearch(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} ref = event['ru_reference'] try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: table_list = { 'contributor': None, 'survey_enrolment': None, 'survey_contact': "contact", 'contributor_survey_period': "survey_period" } for current_table in table_list: logger.info("Fetching Table Model: {}".format(current_table)) table_model = alchemy_functions.table_model( engine, metadata, current_table) logger.info("Fetching Table Data: {}".format(current_table)) if current_table in [ "survey_contact", "contributor_survey_period" ]: other_model = alchemy_functions\ .table_model(engine, metadata, table_list[current_table]) statement = session.query(table_model, other_model)\ .join(other_model)\ .filter(table_model.columns.ru_reference == ref).all() else: statement = session.query(table_model)\ .filter(table_model.columns.ru_reference == ref).all() logger.info("Converting Data: {}".format(current_table)) table_list[current_table] = alchemy_functions.to_df(statement) except db.exc.OperationalError as e: logger.error("Operational Error, When Retrieving Data: {} {}".format( current_table, e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Retrieve Data: {}".format( current_table) } } except Exception as e: logger.error("General Error, Problem Retrieving " + "Data From The Table: {} {}".format(current_table, e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Retrieve Data: {}".format( current_table) } } logger.info("Creating JSON.") out_json = json.dumps(table_list["contributor"].to_dict(orient='records'), sort_keys=True, default=str) if not out_json == "[]": out_json = out_json[1:-2] out_json += ',"Surveys":[ ' for index, row in table_list['survey_enrolment'].iterrows(): curr_row = table_list['survey_enrolment'][( table_list['survey_enrolment']['survey_code'] == row['survey_code'])] curr_row = json.dumps(curr_row.to_dict(orient='records'), sort_keys=True, default=str) curr_row = curr_row[2:-2] out_json = out_json + "{" + curr_row + ',"Contacts":' curr_con = table_list['survey_contact'][( table_list['survey_contact']['survey_code'] == row['survey_code'])] curr_con = json.dumps(curr_con.to_dict(orient='records'), sort_keys=True, default=str) out_json += curr_con out_json = out_json + ',"Periods":' curr_per = table_list['contributor_survey_period'][( table_list['contributor_survey_period']['survey_code'] == row['survey_code'])] curr_per = json.dumps(curr_per.to_dict(orient='records'), sort_keys=True, default=str) out_json += curr_per + '},' out_json = out_json[:-1] out_json += ']}' try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Database Session Closed Badly." } } try: io_validation.Contributor(strict=True).loads(out_json) except ValidationError as e: logger.error("Output: {}".format(out_json)) logger.error("Failed To Validate The Output: {}".format(e.messages)) return {"statusCode": 500, "body": {"Error": e.messages}} logger.info("get_contributor Has Successfully Run.") return {"statusCode": 200, "body": json.loads(out_json)}
def lambda_handler(event, context): """Takes a query dictionary and inserts new data into the query tables with the run information. Parameters: event (Dict):A series of key value pairs nested to match table structure. Returns: Json message reporting the success of the insert alongside the new queries reference number. """ logger.info("create_query Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.Query(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: logger.info("Fetching Table Model: {}".format("query")) table_model = db_model.Query logger.info("Inserting Table Data: {}".format("query")) query_model = table_model( query_type=event['query_type'], ru_reference=event['ru_reference'], survey_code=event['survey_code'], survey_period=event['survey_period'], current_period=event['current_period'], date_raised=event['date_raised'], general_specific_flag=event['general_specific_flag'], industry_group=event['industry_group'], last_query_update=event['last_query_update'], query_active=event['query_active'], query_description=event['query_description'], query_status=event['query_status'], raised_by=event['raised_by'], results_state=event['results_state'], target_resolution_date=event['target_resolution_date']) session.add(query_model) session.flush() session.refresh(query_model) new_query = query_model.query_reference except db.exc.OperationalError as e: logger.error("Operational Error, When Inserting Data: {} {}".format( "query", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format("query") } } except Exception as e: logger.error("General Error, Problem Inserting " + "Data From The Table: {} {}".format("query", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Insert Data: {}".format("query") } } try: if "Exceptions" in event.keys(): logger.info("Inserting into step_exceptions table {}") exceptions = event["Exceptions"] for count, exception in enumerate(exceptions): logger.info("Inserting step_exception row {}".format(count)) logger.info( "Fetching Table Model: {}".format("step_exception")) table_model = db_model.StepException logger.info( "Inserting Table Data: {}".format("step_exception")) session.add( table_model( query_reference=new_query, survey_period=exception['survey_period'], run_id=exception['run_id'], ru_reference=exception['ru_reference'], step=exception['step'], survey_code=exception['survey_code'], error_code=exception['error_code'], error_description=exception['error_description'])) session.flush() try: if "Anomalies" in exception.keys(): anomalies = exception["Anomalies"] for count, anomaly in enumerate(anomalies): logger.info( "Inserting question_anomaly row {}".format( count)) logger.info("Fetching Table Model: {}".format( "question_anomaly")) table_model = db_model.QuestionAnomaly logger.info("Inserting Table Data: {}".format( "question_anomaly")) session.add( table_model( survey_period=anomaly['survey_period'], question_number=anomaly['question_number'], run_id=anomaly['run_id'], ru_reference=anomaly['ru_reference'], step=anomaly['step'], survey_code=anomaly['survey_code'], anomaly_description=anomaly[ 'anomaly_description'])) session.flush() try: if "FailedVETs" in anomaly.keys(): failed_vets = anomaly["FailedVETs"] for count, vets in enumerate(failed_vets): logger.info( "Inserting failed_vet row {}". format(count)) logger.info( "Fetching Table Model: {}".format( "failed_vet")) table_model = db_model.FailedVET logger.info( "Inserting Table Data: {}".format( "failed_vet")) session.add( table_model( failed_vet=vets['failed_vet'], survey_period=vets[ 'survey_period'], question_number=vets[ 'question_number'], run_id=vets['run_id'], ru_reference=vets[ 'ru_reference'], step=vets['step'], survey_code=vets['survey_code'] )) session.flush() except db.exc.OperationalError as e: logger.error("Operational Error, " + "When Inserting Data: {} {}". format("failed_vet", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format( "failed_vet") } } except Exception as e: logger.error( "General Error, Problem " + "Inserting Data From " + "The Table: {} {}".format("failed_vet", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Insert Data: {}".format( "failed_vet") } } except db.exc.OperationalError as e: logger.error( "Operational Error, When Inserting Data: {} {}".format( "question_anomaly", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format( "question_anomaly") } } except Exception as e: logger.error("General Error, Problem Inserting " + "Data From The Table: {} {}".format( "question_anomaly", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Insert Data: {}".format( "question_anomaly") } } except db.exc.OperationalError as e: logger.error("Operational Error, When Inserting Data: {} {}".format( "step_exception", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format("step_exception") } } except Exception as e: logger.error("General Error, Problem Inserting " + "Data From The Table: {} {}".format("step_exception", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Insert Data: {}".format( "step_exception") } } try: if "QueryTasks" in event.keys(): tasks = event["QueryTasks"] for count, task in enumerate(tasks): logger.info("Inserting query_task row {}".format(count)) logger.info("Fetching Table Model: {}".format("query_task")) table_model = db_model.QueryTask logger.info("Inserting Table Data: {}".format("query_task")) session.add( table_model( task_sequence_number=task['task_sequence_number'], query_reference=new_query, response_required_by=task['response_required_by'], task_description=task['task_description'], task_responsibility=task['task_responsibility'], task_status=task['task_status'], next_planned_action=task['next_planned_action'], when_action_required=task['when_action_required'])) session.flush() try: if "QueryTaskUpdates" in task.keys(): update_task = task["QueryTaskUpdates"] for count, query_task in enumerate(update_task): logger.info( "Inserting query_task_update row {}".format( count)) logger.info("Fetching Table Model: {}".format( "query_task_update")) table_model = db_model.QueryTaskUpdate logger.info("Inserting Table Data: {}".format( "query_task_update")) session.add( table_model( task_sequence_number=query_task[ 'task_sequence_number'], query_reference=new_query, last_updated=query_task['last_updated'], task_update_description=query_task[ 'task_update_description'], updated_by=query_task['updated_by'])) session.flush() except db.exc.OperationalError as e: logger.error( "Operational Error, When Inserting Data: {} {}".format( "query_task_update", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format( "query_task_update") } } except Exception as e: logger.error("General Error, Problem Inserting " + "Data From The Table: {} {}".format( "query_task_update", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Insert Data: {}".format( "query_task_update") } } except db.exc.OperationalError as e: logger.error("Operational Error, When Inserting Data: {} {}".format( "query_task", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Insert Data: {}".format("query_task") } } except Exception as e: logger.error("General Error, Problem Inserting " + "Data From The Table: {} {}".format("query_task", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Insert Data: {}".format("query_task") } } try: logger.info("Commit Session.") session.commit() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Commit Changes: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Commit Changes." } } except Exception as e: logger.error( "General Error, Failed To Commit Changes To Database: {}".format( e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Commit Changes To The Database." } } try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Database Session Closed Badly." } } logger.info("create_query Has Successfully Run.") return { "statusCode": 201, "body": { "Success": "Successfully Created Query: {}".format(new_query) } }
def lambda_handler(event, context): """Collects data on a passed in Reference from a table and returns a single Json. Parameters: event (Dict):Two key value pairs used in the search. Returns: out_json (Json):Json responce of the table data. """ logger.info("get_survey_periods Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.SurveySearch(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} search_list = ['survey_period', 'survey_code'] try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: logger.info("Fetching Table Model: {}".format("survey_period")) table_model = alchemy_functions.table_model(engine, metadata, "survey_period") logger.info("Fetching Table Data: {}".format("survey_period")) all_query_sql = session.query(table_model) added_query_sql = 0 for criteria in search_list: if criteria not in event.keys(): logger.info( "No parameters have been passed for {}.".format(criteria)) continue added_query_sql += 1 all_query_sql = all_query_sql.filter( getattr(table_model.columns, criteria) == event[criteria]) if added_query_sql == 0: all_query_sql = all_query_sql\ .filter(table_model.columns.survey_period == session.query( func.max(table_model.columns.survey_period))) logger.info("Converting Data: {}".format("survey_period")) query = alchemy_functions.to_df(all_query_sql) except db.exc.OperationalError as e: logger.error("Operational Error, When Retrieving Data: {} {}".format( "survey_period", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Retrieve Data: {} {}".format( "survey_period", e) } } except Exception as e: logger.error("General Error, Problem Retrieving " + "Data From The Table: {} {}".format("survey_period", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Retrieve Data: {}".format( "survey_period") } } logger.info("Creating JSON.") out_json = json.dumps(query.to_dict(orient='records'), sort_keys=True, default=str) try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, Database " + "Session Closed Badly." } } try: io_validation.SurveyPeriod(strict=True, many=True).loads(out_json) except ValidationError as e: logger.error("Output: {}".format(out_json)) logger.error("Failed To Validate The Output: {}".format(e.messages)) return {"statusCode": 500, "body": {"Error": e.messages}} logger.info("get_survey_periods Has Successfully Run.") return {"statusCode": 200, "body": json.loads(out_json)}
def lambda_handler(event, context): """Pull back from a database all reference data from five tables.""" logger.info("get_all_reference_data Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: table_list = { 'query_type': None, 'vet': None, 'survey': None, 'gor_regions': None, 'ssr_old_regions': None } for current_table in table_list: logger.info("Fetching Table Model: {}".format(current_table)) table_model = alchemy_functions.table_model( engine, metadata, current_table) logger.info("Fetching Table Data: {}".format(current_table)) statement = session.query(table_model).all() logger.info("Converting Data: {}".format(current_table)) table_data = alchemy_functions.to_df(statement) table_list[current_table] = table_data except db.exc.OperationalError as e: logger.error("Operational Error, When Retrieving Data: {} {}".format( current_table, e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Retrieve Data: {}".format( current_table) } } except Exception as e: logger.error( "General Error, Problem Retrieving Data From The Table: {} {}". format(current_table, e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Retrieve Data: {}".format( current_table) } } logger.info("Creating JSON.") out_json = '{"QueryTypes":' out_json += json.dumps(table_list["query_type"].to_dict(orient='records'), sort_keys=True, default=str) out_json += ',"VETsCodes":' out_json += json.dumps(table_list["vet"].to_dict(orient='records'), sort_keys=True, default=str) out_json += ',"Surveys":' out_json += json.dumps(table_list["survey"].to_dict(orient='records'), sort_keys=True, default=str) out_json += ',"GorRegions":' out_json += json.dumps(table_list["gor_regions"].to_dict(orient='records'), sort_keys=True, default=str) out_json += '}' try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Database Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Database Session Closed Badly." } } try: io_validation.AllReferenceData(strict=True).loads(out_json) except ValidationError as e: logger.error("Output: {}".format(out_json)) logger.error("Failed To Validate The Output: {}".format(e.messages)) return {"statusCode": 500, "body": {"Error": e.messages}} logger.info("get_all_reference_data Has Successfully Run.") return {"statusCode": 200, "body": json.loads(out_json)}
def lambda_handler(event, context): """Takes a query dictionary and updates the related query tables and inserts the new run information. Parameters: event (Dict):A series of key value pairs nested to match table structure. Returns: Json message reporting the success of the update. """ logger.info("update_query Has Started Running.") try: database, error = io_validation.Database(strict=True).load(os.environ) except ValidationError as e: logger.error("Database_Location Environment Variable" + "Has Not Been Set Correctly: {}".format(e.messages)) return { "statusCode": 500, "body": { "Error": "Configuration Error: {}".format(e) } } try: io_validation.QueryReference(strict=True).load(event) io_validation.Query(strict=True).load(event) except ValidationError as e: logger.error("Input: {}".format(event)) logger.error("Failed To Validate The Input: {}".format(e.messages)) return {"statusCode": 400, "body": {"Error": e.messages}} try: logger.info("Connecting To The Database.") engine = db.create_engine(database['Database_Location']) session = Session(engine, autoflush=False) metadata = db.MetaData() except db.exc.NoSuchModuleError as e: logger.error("Driver Error, Failed To Connect: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Driver Error, Failed To Connect." } } except db.exc.OperationalError as e: logger.error("Operational Error, Encountered: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To Connect." } } except Exception as e: logger.error( "General Error, Failed To Connect To The Database: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Connect To The Database." } } try: logger.info("Fetching Table Model: {}".format("query")) table_model = alchemy_functions.table_model(engine, metadata, 'query') logger.info("Updating Table Data: {}".format("query")) session.query(table_model)\ .filter(table_model.columns.query_reference == event['query_reference'])\ .update({table_model.columns.query_type: event['query_type'], table_model.columns.current_period: event['current_period'], table_model.columns.general_specific_flag: event['general_specific_flag'], table_model.columns.industry_group: event['industry_group'], table_model.columns.query_active: event['query_active'], table_model.columns.query_description: event['query_description'], table_model.columns.query_status: event['query_status'], table_model.columns.results_state: event['results_state'], table_model.columns.target_resolution_date: event['target_resolution_date']}, synchronize_session=False) except db.exc.OperationalError as e: logger.error("Operational Error, When Updating Data: {} {}".format( "query", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed " + "To Update Data: {}".format("query") } } except Exception as e: logger.error("General Error, Problem Updating Data " + "From The Table: {} {}".format("query", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Update Data: {}".format("query") } } try: if "Exceptions" in event.keys(): exceptions = event["Exceptions"] for count, exception in enumerate(exceptions): logger.info("Inserting step_exception row {}".format(count)) logger.info( "Fetching Table Model: {}".format("step_exception")) table_model = alchemy_functions.table_model( engine, metadata, 'step_exception') is_it_here = session.query(table_model)\ .filter(db.and_( table_model.columns.query_reference == exception['query_reference'], table_model.columns.run_id == exception['run_id'], table_model.columns.ru_reference == exception['ru_reference'], table_model.columns.survey_period == exception['survey_period'], table_model.columns.survey_code == exception['survey_code'], table_model.columns.step == exception['step']))\ .scalar() if is_it_here is not None: continue table_model = db_model.StepException logger.info("Updating Table Data: {}".format("step_exception")) session.add( table_model( query_reference=exception['query_reference'], survey_period=exception['survey_period'], run_id=exception['run_id'], ru_reference=exception['ru_reference'], step=exception['step'], survey_code=exception['survey_code'], error_code=exception['error_code'], error_description=exception['error_description'])) try: if "Anomalies" in exception.keys(): anomalies = exception["Anomalies"] for count, anomaly in enumerate(anomalies): logger.info( "Inserting question_anomaly row {}".format( count)) logger.info("Fetching Table Model: {}".format( "question_anomaly")) table_model = alchemy_functions\ .table_model(engine, metadata, 'question_anomaly') is_it_here = session.query(table_model) \ .filter(db.and_( table_model.columns.query_reference == anomaly['question_number'], table_model.columns.run_id == anomaly[ 'run_id'], table_model.columns.ru_reference == anomaly['ru_reference'], table_model.columns.survey_period == anomaly['survey_period'], table_model.columns.survey_code == anomaly['survey_code'], table_model.columns.step == anomaly[ 'step'])).scalar() if is_it_here is not None: continue table_model = db_model.QuestionAnomaly logger.info("Inserting Table Data: {}".format( "question_anomaly")) session.add( table_model( survey_period=anomaly['survey_period'], question_number=anomaly['question_number'], run_id=anomaly['run_id'], ru_reference=anomaly['ru_reference'], step=anomaly['step'], survey_code=anomaly['survey_code'], anomaly_description=anomaly[ 'anomaly_description'])) try: if "FailedVETs" in anomaly.keys(): failed_vets = anomaly["FailedVETs"] for count, vets in enumerate(failed_vets): logger.info( "Inserting failed_vet row {}". format(count)) logger.info( "Fetching Table Model: {}".format( "failed_vet")) table_model = alchemy_functions\ .table_model(engine, metadata, 'failed_vet') is_it_here = session\ .query(table_model).filter(db.and_( table_model.columns .query_reference == vets['question_number'], table_model.columns.run_id == vets['run_id'], table_model.columns .ru_reference == vets['ru_reference'], table_model.columns .survey_period == vets['survey_period'], table_model.columns .survey_code == vets['survey_code'], table_model.columns.step == vets['step'])).scalar() if is_it_here is not None: continue table_model = db_model.FailedVET logger.info( "Inserting Table Data: {}".format( "failed_vet")) session.add( table_model( failed_vet=vets['failed_vet'], survey_period=vets[ 'survey_period'], question_number=vets[ 'question_number'], run_id=vets['run_id'], ru_reference=vets[ 'ru_reference'], step=vets['step'], survey_code=vets['survey_code'] )) except db.exc.OperationalError as e: logger.error("Operational Error, " + "When Updating Data: {} {}". format("failed_vet", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format( "failed_vet") } } except Exception as e: logger.error( "General Error, Problem Updating Data " + "From The Table: {} {}".format( "failed_vet", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Update Data: {}".format( "failed_vet") } } except db.exc.OperationalError as e: logger.error( "Operational Error, When Updating Data: {} {}".format( "question_anomaly", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format( "question_anomaly") } } except Exception as e: logger.error("General Error, Problem Updating " + "Data From The Table: {} {}".format( "question_anomaly", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Update Data: {}".format( "question_anomaly") } } except db.exc.OperationalError as e: logger.error("Operational Error, When Updating Data: {} {}".format( "step_exception", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format("step_exception") } } except Exception as e: logger.error("General Error, Problem Updating " + "Data From The Table: {} {}".format("step_exception", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Update Data: {}".format( "step_exception") } } try: if "QueryTasks" in event.keys(): tasks = event["QueryTasks"] for count, task in enumerate(tasks): logger.info("Inserting query_task row {}".format(count)) logger.info("Fetching Table Model: {}".format("query_task")) table_model = alchemy_functions.table_model( engine, metadata, 'query_task') logger.info("Updating Table Data: {}".format("query_task")) session.query(table_model)\ .filter(db.and_(table_model.columns.task_sequence_number == task['task_sequence_number'], table_model.columns.query_reference == task['query_reference']))\ .update({table_model.columns.response_required_by: task['response_required_by'], table_model.columns.task_description: task['task_description'], table_model.columns.task_responsibility: task['task_responsibility'], table_model.columns.task_status: task['task_status'], table_model.columns.next_planned_action: task['next_planned_action'], table_model.columns.when_action_required: task['when_action_required']}, synchronize_session=False) try: if "QueryTaskUpdates" in task.keys(): update_task = task["QueryTaskUpdates"] for count, query_task in enumerate(update_task): logger.info( "Inserting query_task_update row {}".format( count)) logger.info("Fetching Table Model: {}".format( "query_task_update")) table_model = alchemy_functions\ .table_model(engine, metadata, 'query_task_update') is_it_here = session.query(table_model)\ .filter(db.and_( table_model.columns.query_reference == query_task['query_reference'], table_model.columns.task_sequence_number == query_task['task_sequence_number'], table_model.columns.last_updated == query_task['last_updated'])).scalar() if is_it_here is not None: continue table_model = db_model.QueryTaskUpdate logger.info("Inserting Table Data: {}".format( "query_task_update")) session.add( table_model( task_sequence_number=query_task[ 'task_sequence_number'], query_reference=query_task[ 'query_reference'], last_updated=query_task['last_updated'], task_update_description=query_task[ 'task_update_description'], updated_by=query_task['updated_by'])) except db.exc.OperationalError as e: logger.error( "Operational Error, When Updating Data: {} {}".format( "query_task_update", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, " + "Failed To Update Data: {}".format( "query_task_update") } } except Exception as e: logger.error( "General Error, Problem Updating Data " + "From The Table: {} {}".format("query_task_update", e)) return { "statusCode": 500, "body": { "Error": "General Error, " + "Failed To Update Data: {}".format( "query_task_update") } } except db.exc.OperationalError as e: logger.error("Operational Error, When Updating Data: {} {}".format( "query_task", e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed " + "To Update Data: {}".format("query_task") } } except Exception as e: logger.error("General Error, Problem Updating Data From " + "The Table: {} {}".format("query_task", e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Update Data: {}".format("query_task") } } try: logger.info("Commit Session.") session.commit() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Commit Changes: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Failed To " + "Commit Changes." } } except Exception as e: logger.error( "General Error, Failed To Commit Changes To Database: {}".format( e)) return { "statusCode": 500, "body": { "Error": "General Error, Failed To Commit " + "Changes To The Database." } } try: logger.info("Closing Session.") session.close() except db.exc.OperationalError as e: logger.error( "Operational Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "Operational Error, Database " + "Session Closed Badly." } } except Exception as e: logger.error( "General Error, Failed To Close The Session: {}".format(e)) return { "statusCode": 500, "body": { "Error": "General Error, Database " + "Session Closed Badly." } } logger.info("update_query Has Successfully Run.") return { "statusCode": 200, "body": { "Success": "Successfully Updated The Tables." } }