def main():
"""
Run the application.
"""
# Get the list of valid locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
location_filter = LocationFilter(locations)
deduplicator = Deduplicator(int(config['deduplicator']['cache_time_to_live_seconds']))
pipeline = fixed_duration_source.compose(location_filter).compose(deduplicator)
# Create the sink that will handle the events that come through the pipeline
averager = Averager(int(config['averager']['average_period_seconds']), int(config['averager']['expiry_time_seconds']))
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
pipeline.sink(averager).handle(receiver.get_events())
# Show final stats
print(f'Processed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds')
print(f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}')
print(f'Invalid locations skipped: {location_filter.invalid_events_filtered}')
print(f'Duplicated events skipped: {deduplicator.duplicate_events_ignored}')
def main():
"""
Run the application.
"""
# Get the list of valid locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
#print(f'\n{locations}\n')
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds, locations)
pipeline = fixed_duration_source
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
pipeline.sink(Printer()).handle(receiver.get_events())
# Show final stats
print(
f'\nProcessed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds'
)
print(
f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}\n'
)
def main():
"""
Run the application.
"""
# Get the list of valid locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
# print(f'\n{locations}\n')
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
# HandleEvents(locations, sns_topic_arn)
HandleEventsThreading(locations, sns_topic_arn)
def main():
"""
Run the application.
"""
# Get the list of valid sensor locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
for location in locations:
print(f'{location}')
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
remove_duplicates = RemoveDuplicates()
average_values = AverageValuesPerMinute()
values_per_location = ValuesPerLocation(locations)
pipeline = values_per_location.compose(
average_values.compose(
remove_duplicates.compose(fixed_duration_source)))
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
# print all the data
pipeline.sink(Printer(locations, values_per_location)).handle(
receiver.get_events())
values_per_location.plot_values.plot(
values_per_location.event_location_container, False)
SaveAveragePerMinToCSV(average_values)
# Show final stats
print(
f'\nProcessed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds'
)
print(
f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}\n'
)
print(f'Duplicated events skipped: {remove_duplicates.counter}')
average_value_index = 0
with open("average_values.csv", "w") as file:
file.write("date,hour,minute,hours decimal,average value\n")
for average_value in average_values.average_values:
local_time_struct = time.localtime(
average_values.
average_values_timestamp[average_value_index])
local_time_string_csv = time.strftime("%d/%m/%Y,%H,%M",
local_time_struct)
local_time_string_hour = time.strftime("%H", local_time_struct)
local_time_string_min = time.strftime("%M", local_time_struct)
local_time_string_hour_decimal = int(
local_time_string_min) / 60
hours_decimal = f"{int(local_time_string_hour) + local_time_string_hour_decimal}"
local_time_string_print = time.strftime(
"%d/%m/%Y %H:%M:%S", local_time_struct)
file.write(
f"{local_time_string_csv},{hours_decimal},{average_value}\n"
)
print(
f'Average Value at {local_time_string_print}: {average_value}'
)
average_value_index += 1
def test_parse_invalid_locations(patched_download_file):
with raises(Exception, match='.*Malformed locations file.*'):
get_locations('bucket', 'key')
def test_parse_valid_locations(patched_download_file):
locations = get_locations('bucket', 'key')
assert len(locations) == 2
assert locations[0] == Location(x=1.1, y=1.2, id='abc')
assert locations[1] == Location(x=2.1, y=2.2, id='def')
def test_parse_empty_locations(patched_download_file):
assert len(get_locations('bucket', 'key')) == 0