def test_push_metrics_to_cloudwatch_zero_byte_submissions():
metadata_file_name = "tests/data/metadata_output_zero_byte.json"
bucket_name = "bucket_name"
data = open(metadata_file_name, 'r').read()
metadata = json.loads(data)
cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1')
cloudwatch_client.put_metric_data(
Namespace=os.environ["WAZE_ZERO_BYTE_SUBMISSIONS_COUNT_METRIC"],
MetricData=[
{
'MetricName':
'Zero Byte Submissions by State and traffic type',
'Dimensions': [{
'Name': 'State',
'Value': metadata["State"]
}, {
'Name': 'TrafficType',
'Value': metadata["TrafficType"]
}],
'Value':
1,
'Unit':
'Count'
},
])
metadata_obj = HandleBucketEvent()
metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata)
assert True
def test_push_metrics_to_cloudwatch_waze_curated():
metadata_file_name = "tests/data/metadata_output_curated.json"
bucket_name = "bucket_name"
data = open(metadata_file_name, 'r').read()
metadata = json.loads(data)
cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1')
cloudwatch_client.put_metric_data(
Namespace=os.environ["WAZE_CURATED_COUNTS_METRIC"],
MetricData=[
{
'MetricName':
'Counts by state and traffic type',
'Dimensions': [{
'Name': 'State',
'Value': metadata["State"]
}, {
'Name': 'TableName',
'Value': metadata["TableName"]
}],
'Value':
1,
'Unit':
'Count'
},
])
metadata_obj = HandleBucketEvent()
metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata)
assert True
def test_push_metrics_to_cloudwatch_cv():
metadata_file_name = "tests/data/metadata_output_cv.json"
bucket_name = "bucket_name"
data = open(metadata_file_name, 'r').read()
metadata = json.loads(data)
cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1')
cloudwatch_client.put_metric_data(
Namespace=os.environ["CV_SUBMISSIONS_COUNTS_METRIC"],
MetricData=[
{
'MetricName':
'Counts by provider and datatype',
'Dimensions': [{
'Name': 'DataProvider',
'Value': metadata["DataProvider"]
}, {
'Name': 'DataType',
'Value': metadata["DataType"]
}],
'Value':
1,
'Unit':
'Count'
},
])
metadata_obj = HandleBucketEvent()
metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata)
assert True
def test_get_s3_head_object_exception():
with pytest.raises(Exception):
bucket_name = "bucket_name"
s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json"
s3_client = boto3.client('s3', region_name='us-east-1')
s3_client.create_bucket(Bucket=bucket_name)
metadata_obj = HandleBucketEvent()
metadata_obj.get_s3_head_object(bucket_name, s3_key)
def test_get_s3_head_object():
file_name = "tests/data/event_input.json"
bucket_name = "bucket_name"
s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json"
s3_client = boto3.client('s3', region_name='us-east-1')
s3_client.create_bucket(Bucket=bucket_name)
s3_client.put_object(Bucket=bucket_name, Body=file_name, Key=s3_key)
metadata_obj = HandleBucketEvent()
metadata_obj.get_s3_head_object(bucket_name, s3_key)
assert True
def test_fetch_s3_details_from_event():
file_name = "tests/data/event_input.json"
bucket_name = "bucket_name" # "bucket_name"
s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json"
data = open(file_name, 'r').read()
data_obj = json.loads(data)
metadata_obj = HandleBucketEvent()
bucket, key = metadata_obj.fetch_s3_details_from_event(data_obj)
assert bucket == bucket_name
assert key == s3_key
def test_push_metadata_to_elasticsearch_index_exception():
with pytest.raises(Exception):
metadata_file_name = "tests/data/metadata_output_raw.json"
bucket_name = "bucket_name"
data = open(metadata_file_name, 'r').read()
metadata = json.loads(data)
elasticsearch_client = Mock()
metadata_obj = HandleBucketEvent()
metadata_obj.es_client = elasticsearch_client
metadata_obj.push_metadata_to_elasticsearch(bucket_name, metadata)
def test_push_metadata_to_elasticsearch():
elasticsearch_endpoint = None
if os.environ.get("SUBMISSIONS_BUCKET_NAME", None):
elasticsearch_endpoint = os.environ.pop("ELASTICSEARCH_ENDPOINT")
metadata_file_name = "tests/data/metadata_output_curated.json"
bucket_name = "bucket_name"
data = open(metadata_file_name, 'r').read()
metadata = json.loads(data)
metadata_obj = HandleBucketEvent()
try:
with pytest.raises(KeyError):
metadata_obj.push_metadata_to_elasticsearch(bucket_name, metadata)
pass
finally:
os.environ["ELASTICSEARCH_ENDPOINT"] = elasticsearch_endpoint
def test_create_metadata_object_waze_raw():
input_file_name = "tests/data/head_object_input_raw.json"
metadata_output_file_name = "tests/data/metadata_output_raw.json"
s3_key = "waze/state=ME/type=jam/year=2010/month=06/day=28/hour=07/minute=54/foo5.json"
data = open(input_file_name, 'r').read()
head_object = json.loads(data)
head_object["LastModified"] = datetime.datetime(2010,
6,
28,
7,
55,
52,
tzinfo=tzutc())
metadata = open(metadata_output_file_name, 'r').read()
metadata_output = json.loads(metadata)
metadata_obj = HandleBucketEvent()
metadata_response = metadata_obj.create_metadata_object(
head_object, s3_key)
assert metadata_response == metadata_output
def test_create_metadata_object_cv():
input_file_name = "tests/data/head_object_input_cv.json"
metadata_output_file_name = "tests/data/metadata_output_cv.json"
s3_key = "cv/thea/Bluetooth/2010/06/28/tmp-Bluetooth_RAW_11_26_18.csv_0"
data = open(input_file_name, 'r').read()
head_object = json.loads(data)
head_object["LastModified"] = datetime.datetime(2010,
6,
28,
5,
00,
52,
tzinfo=tzutc())
metadata = open(metadata_output_file_name, 'r').read()
metadata_output = json.loads(metadata)
metadata_obj = HandleBucketEvent()
metadata_response = metadata_obj.create_metadata_object(
head_object, s3_key)
assert metadata_response == metadata_output
def test_create_metadata_object_waze_curated():
input_file_name = "tests/data/head_object_input_curated.json"
metadata_output_file_name = "tests/data/metadata_output_curated.json"
s3_key = "waze/version=20180720/content/state=IA/table=alert/projection=redshift/year=2010/month=06/day=28/hour=07/minute=55/foo4.csv.gz"
data = open(input_file_name, 'r').read()
head_object = json.loads(data)
head_object["LastModified"] = datetime.datetime(2010,
6,
28,
7,
55,
50,
tzinfo=tzutc())
metadata = open(metadata_output_file_name, 'r').read()
metadata_output = json.loads(metadata)
metadata_obj = HandleBucketEvent()
metadata_response = metadata_obj.create_metadata_object(
head_object, s3_key)
assert metadata_response == metadata_output
def test_fetch_s3_details_from_event_exception():
with pytest.raises(Exception):
file_name = "tests/data/event_input.json"
data = open(file_name, 'r').read()
metadata_obj = HandleBucketEvent()
metadata_obj.fetch_s3_details_from_event(data)
def test_handle_bucket_event():
with pytest.raises(Exception):
assert HandleBucketEvent().handle_bucket_event(None) is None