def aggregate_flowd():
""" aggregate collected flowd data
:return: None
"""
# init metadata (progress maintenance)
metadata = AggMetadata()
# register aggregate classes to stream data to
stream_agg_objects = list()
resolutions = [60, 60*5]
for agg_class in lib.aggregates.get_aggregators():
for resolution in agg_class.resolutions():
stream_agg_objects.append(agg_class(resolution))
# parse flow data and stream to registered consumers
prev_recv=metadata.last_sync()
for flow_record in parse_flow(prev_recv):
if flow_record is None or prev_recv != flow_record['recv']:
# commit data on receive timestamp change or last record
for stream_agg_object in stream_agg_objects:
stream_agg_object.commit()
metadata.update_sync_time(prev_recv)
if flow_record is not None:
# send to aggregator
for stream_agg_object in stream_agg_objects:
stream_agg_object.add(flow_record)
prev_recv = flow_record['recv']
# expire old data
for stream_agg_object in stream_agg_objects:
stream_agg_object.cleanup()
del stream_agg_object
del metadata
def aggregate_flowd(config, do_vacuum=False):
""" aggregate collected flowd data
:param config: script configuration
:param do_vacuum: vacuum database after cleanup
:return: None
"""
# init metadata (progress maintenance)
metadata = AggMetadata(config.database_dir)
# register aggregate classes to stream data to
stream_agg_objects = list()
for agg_class in lib.aggregates.get_aggregators():
for resolution in agg_class.resolutions():
stream_agg_objects.append(
agg_class(resolution, config.database_dir))
# parse flow data and stream to registered consumers
prev_recv = metadata.last_sync()
commit_record_count = 0
for flow_record in parse_flow(prev_recv, config.flowd_source):
if flow_record is None or (prev_recv != flow_record['recv']
and commit_record_count > 100000):
# commit data on receive timestamp change or last record
for stream_agg_object in stream_agg_objects:
stream_agg_object.commit()
commit_record_count = 0
metadata.update_sync_time(prev_recv)
if flow_record is not None:
# send to aggregator
for stream_agg_object in stream_agg_objects:
# class add() may change the flow contents for processing, its better to isolate
# parameters here.
stream_agg_object.add(copy.copy(flow_record))
commit_record_count += 1
prev_recv = flow_record['recv']
# expire old data
for stream_agg_object in stream_agg_objects:
stream_agg_object.cleanup(do_vacuum)
del stream_agg_object
del metadata
def aggregate_flowd(do_vacuum=False):
""" aggregate collected flowd data
:param do_vacuum: vacuum database after cleanup
:return: None
"""
# init metadata (progress maintenance)
metadata = AggMetadata()
# register aggregate classes to stream data to
stream_agg_objects = list()
for agg_class in lib.aggregates.get_aggregators():
for resolution in agg_class.resolutions():
stream_agg_objects.append(agg_class(resolution))
# parse flow data and stream to registered consumers
prev_recv = metadata.last_sync()
commit_record_count = 0
for flow_record in parse_flow(prev_recv):
if flow_record is None or (prev_recv != flow_record['recv'] and commit_record_count > 100000):
# commit data on receive timestamp change or last record
for stream_agg_object in stream_agg_objects:
stream_agg_object.commit()
metadata.update_sync_time(prev_recv)
if flow_record is not None:
# send to aggregator
for stream_agg_object in stream_agg_objects:
# class add() may change the flow contents for processing, its better to isolate
# paremeters here.
flow_record_cpy = copy.copy(flow_record)
stream_agg_object.add(flow_record_cpy)
commit_record_count += 1
prev_recv = flow_record['recv']
# expire old data
for stream_agg_object in stream_agg_objects:
stream_agg_object.cleanup(do_vacuum)
del stream_agg_object
del metadata
