def test_data_enrichment(lookup_data, column_names, file_list, marine_check):
"""
Runs data_enrichment function.
NOTE: This function calls do_merge and marine_mismatch_detector and doesn't
mock them. If this test fails check the tests for these funtions have passed
as they may be at fault.
:param lookup_data: Name of bucket to create - Type: dict
:param column_names: Name of bucket to create - Type: list
:param file_list: Name of bucket to create - Type: list
:param marine_check: Name of bucket to create - Type: boolean
:return Test Pass/Fail
"""
with mock.patch.dict(lambda_method_function.os.environ,
method_environment_variables):
with open("tests/fixtures/test_method_input.json", "r") as file:
test_data = file.read()
test_data = pd.DataFrame(json.loads(test_data))
bucket_name = method_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
test_generic_library.upload_files(client, bucket_name, file_list)
output, test_anomalies = lambda_method_function.data_enrichment(
test_data, marine_check, "survey", "period", "test_bucket",
lookup_data, "responder_id")
for column_name in column_names:
assert column_name in output.columns.values
def test_wrangler_success_returned(which_lambda, which_environment_variables,
which_runtime_variables, lambda_name,
file_list, method_data, prepared_data):
"""
Runs the wrangler function.
:param which_lambda: Main function.
:param which_environment_variables: Environment Variables. - Dict.
:param which_runtime_variables: RuntimeVariables. - Dict.
:param lambda_name: Name of the py file. - String.
:param file_list: Files to be added to the fake S3. - List(String).
:param method_data: File name/location of the data
to be passed out by the method. - String.
:param prepared_data: File name/location of the data
to be used for comparison. - String.
:return Test Pass/Fail
"""
bucket_name = which_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
test_generic_library.upload_files(client, bucket_name, file_list)
with open(prepared_data, "r") as file_1:
test_data_prepared = file_1.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with open(method_data, "r") as file_2:
test_data_out = file_2.read()
with mock.patch.dict(which_lambda.os.environ, which_environment_variables):
with mock.patch(
lambda_name + '.aws_functions.save_to_s3',
side_effect=test_generic_library.replacement_save_to_s3):
with mock.patch(lambda_name + ".boto3.client") as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
mock_client_object.invoke.return_value.get.return_value.read \
.return_value.decode.return_value = json.dumps({
"data": test_data_out,
"success": True,
"anomalies": []
})
output = which_lambda.lambda_handler(
which_runtime_variables,
test_generic_library.context_object)
with open(
"tests/fixtures/" +
which_runtime_variables["RuntimeVariables"]["out_file_name"],
"r") as file_3:
test_data_produced = file_3.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
assert output
assert_frame_equal(produced_data, prepared_data)
def test_wrangler_success_passed(mock_s3_get, which_lambda, input_file,
wrangler_file, runtime_file,
which_runtime_variables_wrangler,
which_runtime_variables_method,
which_environment_variables, which_file_list,
wrangler_boto3):
"""
Runs the wrangler function.
:param mock_s3_get - Replacement Function For The Data Retrieval AWS Functionality.
:return Test Pass/Fail
"""
with open(input_file, "r") as file:
wrangler_input = json.dumps(file.read())
mock_s3_get.return_value = wrangler_input
bucket_name = which_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = which_file_list
test_generic_library.upload_files(client, bucket_name, file_list)
with mock.patch.dict(which_lambda.os.environ, which_environment_variables):
with mock.patch(wrangler_boto3) as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
# Rather than mock the get/decode we tell the code that when the invoke is
# called pass the variables to this replacement function instead.
mock_client_object.invoke.side_effect =\
test_generic_library.replacement_invoke
# This stops the Error caused by the replacement function from stopping
# the test.
with pytest.raises(exception_classes.LambdaFailure):
which_lambda.lambda_handler(
which_runtime_variables_wrangler,
test_generic_library.context_object)
with open(input_file, "r") as file_2:
test_data_prepared = file_2.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with open(wrangler_file, "r") as file_3:
test_data_produced = file_3.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
# Compares the data.
assert_frame_equal(produced_data, prepared_data)
with open(runtime_file, "r") as file_4:
test_dict_prepared = file_4.read()
produced_dict = json.loads(test_dict_prepared)
# Ensures data is not in the RuntimeVariables and then compares.
which_runtime_variables_method["RuntimeVariables"]["data"] = None
assert produced_dict == which_runtime_variables_method["RuntimeVariables"]
def test_wrangler_success_passed():
"""
Runs the wrangler function up to the method invoke.
:param None
:return Test Pass/Fail
"""
bucket_name = wrangler_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = ["test_wrangler_input.json"]
test_generic_library.upload_files(client, bucket_name, file_list)
with mock.patch.dict(lambda_wrangler_function.os.environ,
wrangler_environment_variables):
with mock.patch("enrichment_wrangler.boto3.client") as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
# Rather than mock the get/decode we tell the code that when the invoke is
# called pass the variables to this replacement function instead.
mock_client_object.invoke.side_effect =\
test_generic_library.replacement_invoke
# This stops the Error caused by the replacement function from stopping
# the test.
with pytest.raises(exception_classes.LambdaFailure):
lambda_wrangler_function.lambda_handler(
wrangler_runtime_variables,
test_generic_library.context_object)
with open("tests/fixtures/test_method_input.json", "r") as file_2:
test_data_prepared = file_2.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with open("tests/fixtures/test_wrangler_to_method_input.json",
"r") as file_3:
test_data_produced = file_3.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
# Compares the data.
assert_frame_equal(produced_data, prepared_data)
with open("tests/fixtures/test_wrangler_to_method_runtime.json",
"r") as file_4:
test_dict_prepared = file_4.read()
produced_dict = json.loads(test_dict_prepared)
# Ensures data is not in the RuntimeVariables and then compares.
method_runtime_variables["RuntimeVariables"]["data"] = None
assert produced_dict == method_runtime_variables["RuntimeVariables"]
def test_wrangler_success_returned(mock_s3_put, which_lambda, input_file,
prepared_method_file,
prepared_wrangler_file,
which_environment_variables,
which_runtime_variables_wrangler,
wrangler_boto3):
"""
Runs the wrangler function after the method invoke.
:param None
:return Test Pass/Fail
"""
bucket_name = wrangler_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = [input_file]
test_generic_library.upload_files(client, bucket_name, file_list)
with open(prepared_method_file, "r") as file_2:
test_data_out = file_2.read()
with mock.patch.dict(which_lambda.os.environ, which_environment_variables):
with mock.patch(wrangler_boto3) as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
mock_client_object.invoke.return_value.get.return_value.read \
.return_value.decode.return_value = json.dumps({
"data": test_data_out,
"success": True,
"anomalies": []
})
output = which_lambda.lambda_handler(
which_runtime_variables_wrangler,
test_generic_library.context_object)
with open(prepared_wrangler_file, "r") as file_3:
test_data_prepared = file_3.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with open(
"tests/fixtures/" +
which_runtime_variables_wrangler["RuntimeVariables"]
["out_file_name"], "r") as file_4:
test_data_produced = file_4.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
assert output
assert_frame_equal(produced_data, prepared_data)
def test_client_error(mock_bpm_status, which_lambda, which_runtime_variables,
which_environment_variables, which_data,
expected_message, assertion):
bucket_name = which_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = [
"test_wrangler_agg_input.json", "test_wrangler_splitter_input.json"
]
test_generic_library.upload_files(client, bucket_name, file_list)
test_generic_library.client_error(which_lambda, which_runtime_variables,
which_environment_variables, which_data,
expected_message, assertion)
def test_wrangler_success_returned(mock_s3_put):
"""
Runs the wrangler function.
:param mock_s3_put - Replacement Function For The Data Saveing AWS Functionality.
:return Test Pass/Fail
"""
bucket_name = wrangler_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = ["test_wrangler_input.json"]
test_generic_library.upload_files(client, bucket_name, file_list)
with open("tests/fixtures/test_method_prepared_output.json",
"r") as file_2:
test_data_out = file_2.read()
with mock.patch.dict(lambda_wrangler_function.os.environ,
wrangler_environment_variables):
with mock.patch("strata_period_wrangler.boto3.client") as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
mock_client_object.invoke.return_value.get.return_value.read \
.return_value.decode.return_value = json.dumps({
"data": test_data_out,
"success": True,
"anomalies": "[]"
})
output = lambda_wrangler_function.lambda_handler(
wrangler_runtime_variables,
test_generic_library.context_object)
with open("tests/fixtures/test_wrangler_prepared_output.json",
"r") as file_3:
test_data_prepared = file_3.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with open(
"tests/fixtures/" +
wrangler_runtime_variables["RuntimeVariables"]["out_file_name"],
"r") as file_4:
test_data_produced = file_4.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
assert output
assert_frame_equal(produced_data, prepared_data)
def test_method_success():
"""
Runs the method function.
:param None
:return Test Pass/Fail
"""
with mock.patch.dict(lambda_method_function.os.environ,
method_environment_variables):
with open("tests/fixtures/test_method_prepared_output.json",
"r") as file_1:
file_data = file_1.read()
prepared_data_main = pd.DataFrame(json.loads(file_data))
with open("tests/fixtures/test_method_anomalies_prepared_output.json", "r")\
as file_2:
file_data = file_2.read()
prepared_data_anomalies = pd.DataFrame(json.loads(file_data))
with open("tests/fixtures/test_method_input.json", "r") as file_3:
test_data = file_3.read()
method_runtime_variables["RuntimeVariables"]["data"] = test_data
bucket_name = method_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = [
"responder_county_lookup.json", "county_marine_lookup.json"
]
test_generic_library.upload_files(client, bucket_name, file_list)
output = lambda_method_function.lambda_handler(
method_runtime_variables, test_generic_library.context_object)
produced_data_main = pd.DataFrame(json.loads(
output["data"])).sort_index(axis=1)
produced_data_anomalies = pd.DataFrame(json.loads(output["anomalies"]))
assert output["success"]
assert_frame_equal(produced_data_main, prepared_data_main)
assert_frame_equal(produced_data_anomalies, prepared_data_anomalies)
def test_combiner_success(mock_bpm_status, mock_sns, mock_s3_put):
"""
Runs the wrangler function.
:param mock_s3_put: Replacement Function
For The Data Saving AWS Functionality. - Mock.
:param mock_sns: Replacement function mocking SNS sends.
:param mock_bpm_status: Replacement function mocking bpm status calls.
:return Test Pass/Fail
"""
bucket_name = generic_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
file_list = [
"test_wrangler_agg_input.json",
"test_wrangler_cell_prepared_output.json",
"test_wrangler_ent_prepared_output.json",
"test_wrangler_top2_prepared_output.json"
]
test_generic_library.upload_files(client, bucket_name, file_list)
with open("tests/fixtures/test_wrangler_combiner_prepared_output.json", "r")\
as file_1:
test_data_prepared = file_1.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared))
with mock.patch.dict(lambda_combiner_function.os.environ,
generic_environment_variables):
output = lambda_combiner_function.lambda_handler(
combiner_runtime_variables, test_generic_library.context_object)
with open(
"tests/fixtures/" +
combiner_runtime_variables["RuntimeVariables"]["out_file_name"],
"r") as file_4:
test_data_produced = file_4.read()
produced_data = pd.DataFrame(json.loads(test_data_produced))
assert output
assert_frame_equal(produced_data, prepared_data)
def test_do_merge(file_name, column_names, join_column):
"""
Runs do_merge function.
:param file_name: Name of bucket to create - Type: String
:param column_names: Name of bucket to create - Type: list
:param join_column: Name of bucket to create - Type: String
:return Test Pass/Fail
"""
with open("tests/fixtures/test_method_input.json", "r") as file:
test_data = file.read()
test_data = pd.DataFrame(json.loads(test_data))
bucket_name = "test_bucket"
client = test_generic_library.create_bucket(bucket_name)
test_generic_library.upload_files(client, bucket_name,
[(file_name + ".json")])
output = lambda_method_function.do_merge(test_data, file_name,
column_names, join_column,
bucket_name)
for column_name in column_names:
assert column_name in output.columns.values
def test_wrangler_success_passed(which_lambda, which_environment_variables,
which_runtime_variables, lambda_name,
file_list, method_data,
which_method_variables):
"""
Runs the wrangler function.
:param which_lambda: Main function.
:param which_environment_variables: Environment Variables. - Dict.
:param which_runtime_variables: RuntimeVariables. - Dict.
:param lambda_name: Name of the py file. - String.
:param file_list: Files to be added to the fake S3. - List(String).
:param method_data: File name/location of the data
to be passed out by the method. - String.
:param which_method_variables: Variables to compare against. - Dict.
:return Test Pass/Fail
"""
bucket_name = which_environment_variables["bucket_name"]
client = test_generic_library.create_bucket(bucket_name)
test_generic_library.upload_files(client, bucket_name, file_list)
with mock.patch.dict(which_lambda.os.environ, which_environment_variables):
with mock.patch(
lambda_name + '.aws_functions.save_to_s3',
side_effect=test_generic_library.replacement_save_to_s3):
with mock.patch(lambda_name + ".boto3.client") as mock_client:
mock_client_object = mock.Mock()
mock_client.return_value = mock_client_object
# Rather than mock the get/decode we tell the code that when the invoke is
# called pass the variables to this replacement function instead.
mock_client_object.invoke.side_effect = \
test_generic_library.replacement_invoke
# This stops the Error caused by the replacement function from stopping
# the test.
with pytest.raises(exception_classes.LambdaFailure):
which_lambda.lambda_handler(
which_runtime_variables,
test_generic_library.context_object)
with open(method_data, "r") as file_1:
test_data_prepared = file_1.read()
prepared_data = pd.DataFrame(json.loads(test_data_prepared),
dtype=float)
with open("tests/fixtures/test_wrangler_to_method_input.json",
"r") as file_2:
test_data_produced = file_2.read()
produced_data = pd.DataFrame(json.loads(test_data_produced),
dtype=float)
# Compares the data.
assert_frame_equal(produced_data, prepared_data)
with open("tests/fixtures/test_wrangler_to_method_runtime.json",
"r") as file_3:
test_dict_prepared = file_3.read()
produced_dict = json.loads(test_dict_prepared)
# Ensures data is not in the RuntimeVariables and then compares.
which_method_variables["RuntimeVariables"]["data"] = None
assert produced_dict == which_method_variables["RuntimeVariables"]