def test_push_metrics_to_cloudwatch_zero_byte_submissions(): metadata_file_name = "tests/data/metadata_output_zero_byte.json" bucket_name = "bucket_name" data = open(metadata_file_name, 'r').read() metadata = json.loads(data) cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1') cloudwatch_client.put_metric_data( Namespace=os.environ["WAZE_ZERO_BYTE_SUBMISSIONS_COUNT_METRIC"], MetricData=[ { 'MetricName': 'Zero Byte Submissions by State and traffic type', 'Dimensions': [{ 'Name': 'State', 'Value': metadata["State"] }, { 'Name': 'TrafficType', 'Value': metadata["TrafficType"] }], 'Value': 1, 'Unit': 'Count' }, ]) metadata_obj = HandleBucketEvent() metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata) assert True
def test_push_metrics_to_cloudwatch_waze_curated(): metadata_file_name = "tests/data/metadata_output_curated.json" bucket_name = "bucket_name" data = open(metadata_file_name, 'r').read() metadata = json.loads(data) cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1') cloudwatch_client.put_metric_data( Namespace=os.environ["WAZE_CURATED_COUNTS_METRIC"], MetricData=[ { 'MetricName': 'Counts by state and traffic type', 'Dimensions': [{ 'Name': 'State', 'Value': metadata["State"] }, { 'Name': 'TableName', 'Value': metadata["TableName"] }], 'Value': 1, 'Unit': 'Count' }, ]) metadata_obj = HandleBucketEvent() metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata) assert True
def test_push_metrics_to_cloudwatch_cv(): metadata_file_name = "tests/data/metadata_output_cv.json" bucket_name = "bucket_name" data = open(metadata_file_name, 'r').read() metadata = json.loads(data) cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1') cloudwatch_client.put_metric_data( Namespace=os.environ["CV_SUBMISSIONS_COUNTS_METRIC"], MetricData=[ { 'MetricName': 'Counts by provider and datatype', 'Dimensions': [{ 'Name': 'DataProvider', 'Value': metadata["DataProvider"] }, { 'Name': 'DataType', 'Value': metadata["DataType"] }], 'Value': 1, 'Unit': 'Count' }, ]) metadata_obj = HandleBucketEvent() metadata_obj.publish_custom_metrics_to_cloudwatch(bucket_name, metadata) assert True
def test_get_s3_head_object_exception(): with pytest.raises(Exception): bucket_name = "bucket_name" s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json" s3_client = boto3.client('s3', region_name='us-east-1') s3_client.create_bucket(Bucket=bucket_name) metadata_obj = HandleBucketEvent() metadata_obj.get_s3_head_object(bucket_name, s3_key)
def test_get_s3_head_object(): file_name = "tests/data/event_input.json" bucket_name = "bucket_name" s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json" s3_client = boto3.client('s3', region_name='us-east-1') s3_client.create_bucket(Bucket=bucket_name) s3_client.put_object(Bucket=bucket_name, Body=file_name, Key=s3_key) metadata_obj = HandleBucketEvent() metadata_obj.get_s3_head_object(bucket_name, s3_key) assert True
def test_fetch_s3_details_from_event(): file_name = "tests/data/event_input.json" bucket_name = "bucket_name" # "bucket_name" s3_key = "waze/state=LA/type=alert/year=2010/month=05/day=30/hour=16/minute=42/blahblah3.json" data = open(file_name, 'r').read() data_obj = json.loads(data) metadata_obj = HandleBucketEvent() bucket, key = metadata_obj.fetch_s3_details_from_event(data_obj) assert bucket == bucket_name assert key == s3_key
def test_push_metadata_to_elasticsearch_index_exception(): with pytest.raises(Exception): metadata_file_name = "tests/data/metadata_output_raw.json" bucket_name = "bucket_name" data = open(metadata_file_name, 'r').read() metadata = json.loads(data) elasticsearch_client = Mock() metadata_obj = HandleBucketEvent() metadata_obj.es_client = elasticsearch_client metadata_obj.push_metadata_to_elasticsearch(bucket_name, metadata)
def test_push_metadata_to_elasticsearch(): elasticsearch_endpoint = None if os.environ.get("SUBMISSIONS_BUCKET_NAME", None): elasticsearch_endpoint = os.environ.pop("ELASTICSEARCH_ENDPOINT") metadata_file_name = "tests/data/metadata_output_curated.json" bucket_name = "bucket_name" data = open(metadata_file_name, 'r').read() metadata = json.loads(data) metadata_obj = HandleBucketEvent() try: with pytest.raises(KeyError): metadata_obj.push_metadata_to_elasticsearch(bucket_name, metadata) pass finally: os.environ["ELASTICSEARCH_ENDPOINT"] = elasticsearch_endpoint
def test_create_metadata_object_waze_raw(): input_file_name = "tests/data/head_object_input_raw.json" metadata_output_file_name = "tests/data/metadata_output_raw.json" s3_key = "waze/state=ME/type=jam/year=2010/month=06/day=28/hour=07/minute=54/foo5.json" data = open(input_file_name, 'r').read() head_object = json.loads(data) head_object["LastModified"] = datetime.datetime(2010, 6, 28, 7, 55, 52, tzinfo=tzutc()) metadata = open(metadata_output_file_name, 'r').read() metadata_output = json.loads(metadata) metadata_obj = HandleBucketEvent() metadata_response = metadata_obj.create_metadata_object( head_object, s3_key) assert metadata_response == metadata_output
def test_create_metadata_object_cv(): input_file_name = "tests/data/head_object_input_cv.json" metadata_output_file_name = "tests/data/metadata_output_cv.json" s3_key = "cv/thea/Bluetooth/2010/06/28/tmp-Bluetooth_RAW_11_26_18.csv_0" data = open(input_file_name, 'r').read() head_object = json.loads(data) head_object["LastModified"] = datetime.datetime(2010, 6, 28, 5, 00, 52, tzinfo=tzutc()) metadata = open(metadata_output_file_name, 'r').read() metadata_output = json.loads(metadata) metadata_obj = HandleBucketEvent() metadata_response = metadata_obj.create_metadata_object( head_object, s3_key) assert metadata_response == metadata_output
def test_create_metadata_object_waze_curated(): input_file_name = "tests/data/head_object_input_curated.json" metadata_output_file_name = "tests/data/metadata_output_curated.json" s3_key = "waze/version=20180720/content/state=IA/table=alert/projection=redshift/year=2010/month=06/day=28/hour=07/minute=55/foo4.csv.gz" data = open(input_file_name, 'r').read() head_object = json.loads(data) head_object["LastModified"] = datetime.datetime(2010, 6, 28, 7, 55, 50, tzinfo=tzutc()) metadata = open(metadata_output_file_name, 'r').read() metadata_output = json.loads(metadata) metadata_obj = HandleBucketEvent() metadata_response = metadata_obj.create_metadata_object( head_object, s3_key) assert metadata_response == metadata_output
def test_fetch_s3_details_from_event_exception(): with pytest.raises(Exception): file_name = "tests/data/event_input.json" data = open(file_name, 'r').read() metadata_obj = HandleBucketEvent() metadata_obj.fetch_s3_details_from_event(data)
def test_handle_bucket_event(): with pytest.raises(Exception): assert HandleBucketEvent().handle_bucket_event(None) is None