def test_data_enrichment(lookup_data, column_names, file_list, marine_check): """ Runs data_enrichment function. NOTE: This function calls do_merge and marine_mismatch_detector and doesn't mock them. If this test fails check the tests for these funtions have passed as they may be at fault. :param lookup_data: Name of bucket to create - Type: dict :param column_names: Name of bucket to create - Type: list :param file_list: Name of bucket to create - Type: list :param marine_check: Name of bucket to create - Type: boolean :return Test Pass/Fail """ with mock.patch.dict(lambda_method_function.os.environ, method_environment_variables): with open("tests/fixtures/test_method_input.json", "r") as file: test_data = file.read() test_data = pd.DataFrame(json.loads(test_data)) bucket_name = method_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) test_generic_library.upload_files(client, bucket_name, file_list) output, test_anomalies = lambda_method_function.data_enrichment( test_data, marine_check, "survey", "period", "test_bucket", lookup_data, "responder_id") for column_name in column_names: assert column_name in output.columns.values
def test_wrangler_success_returned(which_lambda, which_environment_variables, which_runtime_variables, lambda_name, file_list, method_data, prepared_data): """ Runs the wrangler function. :param which_lambda: Main function. :param which_environment_variables: Environment Variables. - Dict. :param which_runtime_variables: RuntimeVariables. - Dict. :param lambda_name: Name of the py file. - String. :param file_list: Files to be added to the fake S3. - List(String). :param method_data: File name/location of the data to be passed out by the method. - String. :param prepared_data: File name/location of the data to be used for comparison. - String. :return Test Pass/Fail """ bucket_name = which_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) test_generic_library.upload_files(client, bucket_name, file_list) with open(prepared_data, "r") as file_1: test_data_prepared = file_1.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with open(method_data, "r") as file_2: test_data_out = file_2.read() with mock.patch.dict(which_lambda.os.environ, which_environment_variables): with mock.patch( lambda_name + '.aws_functions.save_to_s3', side_effect=test_generic_library.replacement_save_to_s3): with mock.patch(lambda_name + ".boto3.client") as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object mock_client_object.invoke.return_value.get.return_value.read \ .return_value.decode.return_value = json.dumps({ "data": test_data_out, "success": True, "anomalies": [] }) output = which_lambda.lambda_handler( which_runtime_variables, test_generic_library.context_object) with open( "tests/fixtures/" + which_runtime_variables["RuntimeVariables"]["out_file_name"], "r") as file_3: test_data_produced = file_3.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) assert output assert_frame_equal(produced_data, prepared_data)
def test_wrangler_success_passed(mock_s3_get, which_lambda, input_file, wrangler_file, runtime_file, which_runtime_variables_wrangler, which_runtime_variables_method, which_environment_variables, which_file_list, wrangler_boto3): """ Runs the wrangler function. :param mock_s3_get - Replacement Function For The Data Retrieval AWS Functionality. :return Test Pass/Fail """ with open(input_file, "r") as file: wrangler_input = json.dumps(file.read()) mock_s3_get.return_value = wrangler_input bucket_name = which_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = which_file_list test_generic_library.upload_files(client, bucket_name, file_list) with mock.patch.dict(which_lambda.os.environ, which_environment_variables): with mock.patch(wrangler_boto3) as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object # Rather than mock the get/decode we tell the code that when the invoke is # called pass the variables to this replacement function instead. mock_client_object.invoke.side_effect =\ test_generic_library.replacement_invoke # This stops the Error caused by the replacement function from stopping # the test. with pytest.raises(exception_classes.LambdaFailure): which_lambda.lambda_handler( which_runtime_variables_wrangler, test_generic_library.context_object) with open(input_file, "r") as file_2: test_data_prepared = file_2.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with open(wrangler_file, "r") as file_3: test_data_produced = file_3.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) # Compares the data. assert_frame_equal(produced_data, prepared_data) with open(runtime_file, "r") as file_4: test_dict_prepared = file_4.read() produced_dict = json.loads(test_dict_prepared) # Ensures data is not in the RuntimeVariables and then compares. which_runtime_variables_method["RuntimeVariables"]["data"] = None assert produced_dict == which_runtime_variables_method["RuntimeVariables"]
def test_wrangler_success_passed(): """ Runs the wrangler function up to the method invoke. :param None :return Test Pass/Fail """ bucket_name = wrangler_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = ["test_wrangler_input.json"] test_generic_library.upload_files(client, bucket_name, file_list) with mock.patch.dict(lambda_wrangler_function.os.environ, wrangler_environment_variables): with mock.patch("enrichment_wrangler.boto3.client") as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object # Rather than mock the get/decode we tell the code that when the invoke is # called pass the variables to this replacement function instead. mock_client_object.invoke.side_effect =\ test_generic_library.replacement_invoke # This stops the Error caused by the replacement function from stopping # the test. with pytest.raises(exception_classes.LambdaFailure): lambda_wrangler_function.lambda_handler( wrangler_runtime_variables, test_generic_library.context_object) with open("tests/fixtures/test_method_input.json", "r") as file_2: test_data_prepared = file_2.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with open("tests/fixtures/test_wrangler_to_method_input.json", "r") as file_3: test_data_produced = file_3.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) # Compares the data. assert_frame_equal(produced_data, prepared_data) with open("tests/fixtures/test_wrangler_to_method_runtime.json", "r") as file_4: test_dict_prepared = file_4.read() produced_dict = json.loads(test_dict_prepared) # Ensures data is not in the RuntimeVariables and then compares. method_runtime_variables["RuntimeVariables"]["data"] = None assert produced_dict == method_runtime_variables["RuntimeVariables"]
def test_wrangler_success_returned(mock_s3_put, which_lambda, input_file, prepared_method_file, prepared_wrangler_file, which_environment_variables, which_runtime_variables_wrangler, wrangler_boto3): """ Runs the wrangler function after the method invoke. :param None :return Test Pass/Fail """ bucket_name = wrangler_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = [input_file] test_generic_library.upload_files(client, bucket_name, file_list) with open(prepared_method_file, "r") as file_2: test_data_out = file_2.read() with mock.patch.dict(which_lambda.os.environ, which_environment_variables): with mock.patch(wrangler_boto3) as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object mock_client_object.invoke.return_value.get.return_value.read \ .return_value.decode.return_value = json.dumps({ "data": test_data_out, "success": True, "anomalies": [] }) output = which_lambda.lambda_handler( which_runtime_variables_wrangler, test_generic_library.context_object) with open(prepared_wrangler_file, "r") as file_3: test_data_prepared = file_3.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with open( "tests/fixtures/" + which_runtime_variables_wrangler["RuntimeVariables"] ["out_file_name"], "r") as file_4: test_data_produced = file_4.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) assert output assert_frame_equal(produced_data, prepared_data)
def test_client_error(mock_bpm_status, which_lambda, which_runtime_variables, which_environment_variables, which_data, expected_message, assertion): bucket_name = which_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = [ "test_wrangler_agg_input.json", "test_wrangler_splitter_input.json" ] test_generic_library.upload_files(client, bucket_name, file_list) test_generic_library.client_error(which_lambda, which_runtime_variables, which_environment_variables, which_data, expected_message, assertion)
def test_wrangler_success_returned(mock_s3_put): """ Runs the wrangler function. :param mock_s3_put - Replacement Function For The Data Saveing AWS Functionality. :return Test Pass/Fail """ bucket_name = wrangler_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = ["test_wrangler_input.json"] test_generic_library.upload_files(client, bucket_name, file_list) with open("tests/fixtures/test_method_prepared_output.json", "r") as file_2: test_data_out = file_2.read() with mock.patch.dict(lambda_wrangler_function.os.environ, wrangler_environment_variables): with mock.patch("strata_period_wrangler.boto3.client") as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object mock_client_object.invoke.return_value.get.return_value.read \ .return_value.decode.return_value = json.dumps({ "data": test_data_out, "success": True, "anomalies": "[]" }) output = lambda_wrangler_function.lambda_handler( wrangler_runtime_variables, test_generic_library.context_object) with open("tests/fixtures/test_wrangler_prepared_output.json", "r") as file_3: test_data_prepared = file_3.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with open( "tests/fixtures/" + wrangler_runtime_variables["RuntimeVariables"]["out_file_name"], "r") as file_4: test_data_produced = file_4.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) assert output assert_frame_equal(produced_data, prepared_data)
def test_method_success(): """ Runs the method function. :param None :return Test Pass/Fail """ with mock.patch.dict(lambda_method_function.os.environ, method_environment_variables): with open("tests/fixtures/test_method_prepared_output.json", "r") as file_1: file_data = file_1.read() prepared_data_main = pd.DataFrame(json.loads(file_data)) with open("tests/fixtures/test_method_anomalies_prepared_output.json", "r")\ as file_2: file_data = file_2.read() prepared_data_anomalies = pd.DataFrame(json.loads(file_data)) with open("tests/fixtures/test_method_input.json", "r") as file_3: test_data = file_3.read() method_runtime_variables["RuntimeVariables"]["data"] = test_data bucket_name = method_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = [ "responder_county_lookup.json", "county_marine_lookup.json" ] test_generic_library.upload_files(client, bucket_name, file_list) output = lambda_method_function.lambda_handler( method_runtime_variables, test_generic_library.context_object) produced_data_main = pd.DataFrame(json.loads( output["data"])).sort_index(axis=1) produced_data_anomalies = pd.DataFrame(json.loads(output["anomalies"])) assert output["success"] assert_frame_equal(produced_data_main, prepared_data_main) assert_frame_equal(produced_data_anomalies, prepared_data_anomalies)
def test_combiner_success(mock_bpm_status, mock_sns, mock_s3_put): """ Runs the wrangler function. :param mock_s3_put: Replacement Function For The Data Saving AWS Functionality. - Mock. :param mock_sns: Replacement function mocking SNS sends. :param mock_bpm_status: Replacement function mocking bpm status calls. :return Test Pass/Fail """ bucket_name = generic_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) file_list = [ "test_wrangler_agg_input.json", "test_wrangler_cell_prepared_output.json", "test_wrangler_ent_prepared_output.json", "test_wrangler_top2_prepared_output.json" ] test_generic_library.upload_files(client, bucket_name, file_list) with open("tests/fixtures/test_wrangler_combiner_prepared_output.json", "r")\ as file_1: test_data_prepared = file_1.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared)) with mock.patch.dict(lambda_combiner_function.os.environ, generic_environment_variables): output = lambda_combiner_function.lambda_handler( combiner_runtime_variables, test_generic_library.context_object) with open( "tests/fixtures/" + combiner_runtime_variables["RuntimeVariables"]["out_file_name"], "r") as file_4: test_data_produced = file_4.read() produced_data = pd.DataFrame(json.loads(test_data_produced)) assert output assert_frame_equal(produced_data, prepared_data)
def test_do_merge(file_name, column_names, join_column): """ Runs do_merge function. :param file_name: Name of bucket to create - Type: String :param column_names: Name of bucket to create - Type: list :param join_column: Name of bucket to create - Type: String :return Test Pass/Fail """ with open("tests/fixtures/test_method_input.json", "r") as file: test_data = file.read() test_data = pd.DataFrame(json.loads(test_data)) bucket_name = "test_bucket" client = test_generic_library.create_bucket(bucket_name) test_generic_library.upload_files(client, bucket_name, [(file_name + ".json")]) output = lambda_method_function.do_merge(test_data, file_name, column_names, join_column, bucket_name) for column_name in column_names: assert column_name in output.columns.values
def test_wrangler_success_passed(which_lambda, which_environment_variables, which_runtime_variables, lambda_name, file_list, method_data, which_method_variables): """ Runs the wrangler function. :param which_lambda: Main function. :param which_environment_variables: Environment Variables. - Dict. :param which_runtime_variables: RuntimeVariables. - Dict. :param lambda_name: Name of the py file. - String. :param file_list: Files to be added to the fake S3. - List(String). :param method_data: File name/location of the data to be passed out by the method. - String. :param which_method_variables: Variables to compare against. - Dict. :return Test Pass/Fail """ bucket_name = which_environment_variables["bucket_name"] client = test_generic_library.create_bucket(bucket_name) test_generic_library.upload_files(client, bucket_name, file_list) with mock.patch.dict(which_lambda.os.environ, which_environment_variables): with mock.patch( lambda_name + '.aws_functions.save_to_s3', side_effect=test_generic_library.replacement_save_to_s3): with mock.patch(lambda_name + ".boto3.client") as mock_client: mock_client_object = mock.Mock() mock_client.return_value = mock_client_object # Rather than mock the get/decode we tell the code that when the invoke is # called pass the variables to this replacement function instead. mock_client_object.invoke.side_effect = \ test_generic_library.replacement_invoke # This stops the Error caused by the replacement function from stopping # the test. with pytest.raises(exception_classes.LambdaFailure): which_lambda.lambda_handler( which_runtime_variables, test_generic_library.context_object) with open(method_data, "r") as file_1: test_data_prepared = file_1.read() prepared_data = pd.DataFrame(json.loads(test_data_prepared), dtype=float) with open("tests/fixtures/test_wrangler_to_method_input.json", "r") as file_2: test_data_produced = file_2.read() produced_data = pd.DataFrame(json.loads(test_data_produced), dtype=float) # Compares the data. assert_frame_equal(produced_data, prepared_data) with open("tests/fixtures/test_wrangler_to_method_runtime.json", "r") as file_3: test_dict_prepared = file_3.read() produced_dict = json.loads(test_dict_prepared) # Ensures data is not in the RuntimeVariables and then compares. which_method_variables["RuntimeVariables"]["data"] = None assert produced_dict == which_method_variables["RuntimeVariables"]