def create_data(self, seed=None, how_much=60):
seed = seed or self.seed
cur_time = seed
for i in range(1, how_much + 1):
for provider in registry.get_provider_instances():
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=cur_time,
)
raw_panel_data = provider.get_data()
for key, value in raw_panel_data.items():
data_set_obj = PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=value,
)
if i % 60 == 0:
Panel.objects.generate_hourly_aggregates(cur_time)
if i % 1440 == 0:
Panel.objects.generate_daily_aggregates(cur_time)
cur_time += datetime.timedelta(seconds=60)
def update_panels(self):
data = []
shared_now = datetime.datetime.now()
# function to sort the panels by priority
key = lambda obj: obj.get_priority()
for provider in sorted(registry.get_provider_instances(), key=key):
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=shared_now,
aggregate_type=PANEL_AGGREGATE_MINUTE,
)
raw_panel_data = provider.get_data()
for key, value in raw_panel_data.items():
data_set_obj = PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=value,
)
data.append(panel_data_obj)
return data
def update_panels(self):
data = []
shared_now = datetime.datetime.now()
# function to sort the panels by priority
key = lambda obj: obj.get_priority()
for provider in sorted(registry.get_provider_instances(), key=key):
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=shared_now,
aggregate_type=PANEL_AGGREGATE_MINUTE,
)
raw_panel_data = provider.get_data()
for key, value in raw_panel_data.items():
data_set_obj = PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=value,
)
data.append(panel_data_obj)
return data
def create_data(self, seed=None, how_much=60):
seed = seed or self.seed
cur_time = seed
for i in range(1, how_much + 1):
for provider in registry.get_provider_instances():
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=cur_time,
)
raw_panel_data = provider.get_data()
for key, value in raw_panel_data.items():
data_set_obj = PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=value,
)
if i % 60 == 0:
Panel.objects.generate_hourly_aggregates(cur_time)
if i % 1440 == 0:
Panel.objects.generate_daily_aggregates(cur_time)
cur_time += datetime.timedelta(seconds=60)
def _generate_aggregate(self, timedelta, aggregate_type, seed):
data = []
shared_now = seed or datetime.datetime.now()
low_date = shared_now - timedelta
# function to sort the panels by priority
key = lambda obj: obj.get_priority()
for provider in sorted(registry.get_provider_instances(), key=key):
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
# create a new panel data object to store the aggregates
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=shared_now,
aggregate_type=aggregate_type,
)
# get the panel data queryset for the previous `timedelta`
data_qs = panel_obj.data.minute_data().filter(created_date__range=(
low_date, shared_now
))
# get what we really want, which are the k/v pairs
data_sets = PanelDataSet.objects.filter(panel_data__in=data_qs)
# find the unique keys and aggregate over them
distinct_keys = data_sets.values_list('key', flat=True).distinct()
for key in distinct_keys:
# calculate the average and store it
average = data_sets.filter(key=key).aggregate(
average=Avg('value')
)['average']
PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=average,
)
data.append(panel_data_obj)
return data
def _generate_aggregate(self, timedelta, aggregate_type, seed):
data = []
shared_now = seed or datetime.datetime.now()
low_date = shared_now - timedelta
# function to sort the panels by priority
key = lambda obj: obj.get_priority()
for provider in sorted(registry.get_provider_instances(), key=key):
# pull the data off the panel and store
panel_obj = provider.get_panel_instance()
# create a new panel data object to store the aggregates
panel_data_obj = PanelData.objects.create(
panel=panel_obj,
created_date=shared_now,
aggregate_type=aggregate_type,
)
# get the panel data queryset for the previous `timedelta`
data_qs = panel_obj.data.minute_data().filter(
created_date__range=(low_date, shared_now))
# get what we really want, which are the k/v pairs
data_sets = PanelDataSet.objects.filter(panel_data__in=data_qs)
# find the unique keys and aggregate over them
distinct_keys = data_sets.values_list('key', flat=True).distinct()
for key in distinct_keys:
# calculate the average and store it
average = data_sets.filter(key=key).aggregate(
average=Avg('value'))['average']
PanelDataSet.objects.create(
panel_data=panel_data_obj,
key=key,
value=average,
)
data.append(panel_data_obj)
return data
