def create_data(self, seed=None, how_much=60): seed = seed or self.seed cur_time = seed for i in range(1, how_much + 1): for provider in registry.get_provider_instances(): # pull the data off the panel and store panel_obj = provider.get_panel_instance() panel_data_obj = PanelData.objects.create( panel=panel_obj, created_date=cur_time, ) raw_panel_data = provider.get_data() for key, value in raw_panel_data.items(): data_set_obj = PanelDataSet.objects.create( panel_data=panel_data_obj, key=key, value=value, ) if i % 60 == 0: Panel.objects.generate_hourly_aggregates(cur_time) if i % 1440 == 0: Panel.objects.generate_daily_aggregates(cur_time) cur_time += datetime.timedelta(seconds=60)
def update_panels(self): data = [] shared_now = datetime.datetime.now() # function to sort the panels by priority key = lambda obj: obj.get_priority() for provider in sorted(registry.get_provider_instances(), key=key): # pull the data off the panel and store panel_obj = provider.get_panel_instance() panel_data_obj = PanelData.objects.create( panel=panel_obj, created_date=shared_now, aggregate_type=PANEL_AGGREGATE_MINUTE, ) raw_panel_data = provider.get_data() for key, value in raw_panel_data.items(): data_set_obj = PanelDataSet.objects.create( panel_data=panel_data_obj, key=key, value=value, ) data.append(panel_data_obj) return data
def _generate_aggregate(self, timedelta, aggregate_type, seed): data = [] shared_now = seed or datetime.datetime.now() low_date = shared_now - timedelta # function to sort the panels by priority key = lambda obj: obj.get_priority() for provider in sorted(registry.get_provider_instances(), key=key): # pull the data off the panel and store panel_obj = provider.get_panel_instance() # create a new panel data object to store the aggregates panel_data_obj = PanelData.objects.create( panel=panel_obj, created_date=shared_now, aggregate_type=aggregate_type, ) # get the panel data queryset for the previous `timedelta` data_qs = panel_obj.data.minute_data().filter(created_date__range=( low_date, shared_now )) # get what we really want, which are the k/v pairs data_sets = PanelDataSet.objects.filter(panel_data__in=data_qs) # find the unique keys and aggregate over them distinct_keys = data_sets.values_list('key', flat=True).distinct() for key in distinct_keys: # calculate the average and store it average = data_sets.filter(key=key).aggregate( average=Avg('value') )['average'] PanelDataSet.objects.create( panel_data=panel_data_obj, key=key, value=average, ) data.append(panel_data_obj) return data
def _generate_aggregate(self, timedelta, aggregate_type, seed): data = [] shared_now = seed or datetime.datetime.now() low_date = shared_now - timedelta # function to sort the panels by priority key = lambda obj: obj.get_priority() for provider in sorted(registry.get_provider_instances(), key=key): # pull the data off the panel and store panel_obj = provider.get_panel_instance() # create a new panel data object to store the aggregates panel_data_obj = PanelData.objects.create( panel=panel_obj, created_date=shared_now, aggregate_type=aggregate_type, ) # get the panel data queryset for the previous `timedelta` data_qs = panel_obj.data.minute_data().filter( created_date__range=(low_date, shared_now)) # get what we really want, which are the k/v pairs data_sets = PanelDataSet.objects.filter(panel_data__in=data_qs) # find the unique keys and aggregate over them distinct_keys = data_sets.values_list('key', flat=True).distinct() for key in distinct_keys: # calculate the average and store it average = data_sets.filter(key=key).aggregate( average=Avg('value'))['average'] PanelDataSet.objects.create( panel_data=panel_data_obj, key=key, value=average, ) data.append(panel_data_obj) return data