def main():
"""
Run the application.
"""
# Get the list of valid locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
location_filter = LocationFilter(locations)
deduplicator = Deduplicator(int(config['deduplicator']['cache_time_to_live_seconds']))
pipeline = fixed_duration_source.compose(location_filter).compose(deduplicator)
# Create the sink that will handle the events that come through the pipeline
averager = Averager(int(config['averager']['average_period_seconds']), int(config['averager']['expiry_time_seconds']))
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
pipeline.sink(averager).handle(receiver.get_events())
# Show final stats
print(f'Processed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds')
print(f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}')
print(f'Invalid locations skipped: {location_filter.invalid_events_filtered}')
print(f'Duplicated events skipped: {deduplicator.duplicate_events_ignored}')
def main():
"""
Run the application.
"""
# Get the list of valid locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
#print(f'\n{locations}\n')
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds, locations)
pipeline = fixed_duration_source
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
pipeline.sink(Printer()).handle(receiver.get_events())
# Show final stats
print(
f'\nProcessed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds'
)
print(
f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}\n'
)
def threadFunction(event_queue, sns_topic_arn, index):
"""
Function performed by different threads. Gets data from sqs
"""
print('thread: ' + str(index))
print('............')
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
pipeline = fixed_duration_source # pipeline is fixed duration source object, which inherits from pipeline object
receiver = Receiver(queue_subscription)
print(next(receiver.get_events))
def HandleEvents(locations, sns_topic_arn):
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
pipeline = fixed_duration_source # pipeline is fixed duration source object, which inherits from pipeline object
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
duplicateTracker = DuplicateTracker()
rollingAverage = RollingAverage()
# todo add compose method to make below clearer
pipeline.sink(LocationProcessor(locations)).handle(
rollingAverage.handle(
duplicateTracker.handle(receiver.get_events())))
# Show final stats
print(
f'\nProcessed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds'
)
print(
f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}\n'
)
def main():
"""
Run the application.
"""
# Get the list of valid sensor locations from S3
s3_bucket = config['locations']['s3_bucket']
locations_key = config['locations']['s3_key']
locations = get_locations(s3_bucket, locations_key)
for location in locations:
print(f'{location}')
# Create the pipeline steps that events will pass through when being processed
run_time_seconds = int(config['run_time_seconds'])
fixed_duration_source = FixedDurationSource(run_time_seconds)
remove_duplicates = RemoveDuplicates()
average_values = AverageValuesPerMinute()
values_per_location = ValuesPerLocation(locations)
pipeline = values_per_location.compose(
average_values.compose(
remove_duplicates.compose(fixed_duration_source)))
# Create an SQS queue that will be subscribed to the SNS topic
sns_topic_arn = config['receiver']['sns_topic_arn']
with QueueSubscription(sns_topic_arn) as queue_subscription:
# Process events as they come in from the queue
receiver = Receiver(queue_subscription)
# print all the data
pipeline.sink(Printer(locations, values_per_location)).handle(
receiver.get_events())
values_per_location.plot_values.plot(
values_per_location.event_location_container, False)
SaveAveragePerMinToCSV(average_values)
# Show final stats
print(
f'\nProcessed {fixed_duration_source.events_processed} events in {run_time_seconds} seconds'
)
print(
f'Events/s: {fixed_duration_source.events_processed / run_time_seconds:.2f}\n'
)
print(f'Duplicated events skipped: {remove_duplicates.counter}')
average_value_index = 0
with open("average_values.csv", "w") as file:
file.write("date,hour,minute,hours decimal,average value\n")
for average_value in average_values.average_values:
local_time_struct = time.localtime(
average_values.
average_values_timestamp[average_value_index])
local_time_string_csv = time.strftime("%d/%m/%Y,%H,%M",
local_time_struct)
local_time_string_hour = time.strftime("%H", local_time_struct)
local_time_string_min = time.strftime("%M", local_time_struct)
local_time_string_hour_decimal = int(
local_time_string_min) / 60
hours_decimal = f"{int(local_time_string_hour) + local_time_string_hour_decimal}"
local_time_string_print = time.strftime(
"%d/%m/%Y %H:%M:%S", local_time_struct)
file.write(
f"{local_time_string_csv},{hours_decimal},{average_value}\n"
)
print(
f'Average Value at {local_time_string_print}: {average_value}'
)
average_value_index += 1
def receiver_for_tests():
queue_subscription = MagicMock()
type(queue_subscription).queue_url = PropertyMock(return_value=QUEUE_URL)
return Receiver(queue_subscription)