def handle(self, *args, **options): force_clear = options.get('force-clear') verbose = options.get('verbose') recalculate = options.get('recalculate-last') if recalculate: for MetricClass in METRICS: RecordPrototype._db_filter(date=MetricClass._last_datetime().date(), type=MetricClass.TYPE).delete() for MetricClass in METRICS: if force_clear or MetricClass.FULL_CLEAR_RECUIRED: if verbose: print('clear %s' % MetricClass.TYPE) MetricClass.clear() for i, MetricClass in enumerate(METRICS): metric = MetricClass() if verbose: print('[%3d] calculate %s' % (i, metric.TYPE)) metric.initialize() metric.complete_values() models.FullStatistics.objects.all().delete() models.FullStatistics.objects.create(data=RecordPrototype.get_js_data())
def test_store_value__float(self): self.metric.store_value(datetime.datetime(year=6, month=6, day=6, hour=6), 666.6) self.assertEqual(RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 1) record = RecordPrototype._db_get_object(0) self.assertEqual(record.date, datetime.datetime(year=6, month=6, day=6, hour=6)) self.assertEqual(record.value_int, 666) self.assertEqual(record.value_float, 666.6) self.assertTrue(record.type.is_TEST_FLOAT)
def test_select__int(self): dates = [self.date + self.timedelta * i for i in range(4)] RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=dates[1], value_int=417) RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=dates[2], value_int=666) self.assertEqual( RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[0], date_to=dates[3]), [(dates[1], 417), (dates[2], 666)], ) self.assertEqual( RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[1], date_to=dates[2]), [(dates[1], 417), (dates[2], 666)], ) self.assertEqual( RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[1], date_to=dates[1]), [(dates[1], 417)], ) self.assertEqual( RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[2], date_to=dates[2]), [(dates[2], 666)], ) self.assertEqual( RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[3], date_to=dates[3]), [] )
def test_store_value__float(self): self.metric.store_value( datetime.datetime(year=6, month=6, day=6, hour=6), 666.6) self.assertEqual( RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 1) record = RecordPrototype._db_get_object(0) self.assertEqual(record.date, datetime.datetime(year=6, month=6, day=6, hour=6)) self.assertEqual(record.value_int, 666) self.assertEqual(record.value_float, 666.6) self.assertTrue(record.type.is_TEST_FLOAT)
def test_no_second_record(self): self.metric.complete_values() self.assertTrue(RecordPrototype._db_count() > 0) self.metric.initialize() with self.check_not_changed(RecordPrototype._db_count): self.metric.complete_values()
def test_select__float(self): date_1 = datetime.datetime.now() date_2 = datetime.datetime.now() date_3 = datetime.datetime.now() date_4 = datetime.datetime.now() RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=date_2, value_float=41.7) RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=date_3, value_float=66.6) self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_1, date_to=date_4), [(date_2, 41.7), (date_3, 66.6)]) self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_2, date_to=date_3), [(date_2, 41.7), (date_3, 66.6)]) self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_2, date_to=date_2), [(date_2, 41.7)]) self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_3, date_to=date_3), [(date_3, 66.6)]) self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_4, date_to=date_4), [])
def handle(self, *args, **options): for record in relations.RECORD_TYPE.records: data = RecordPrototype.select_for_js(record, date_from=datetime.datetime(year=2013, month=8, day=1), date_to=datetime.datetime(year=2014, month=6, day=4)) with open('/tmp/stats_%s.csv' % record.name.lower(), 'w') as f: for time, value in data: f.write('%s;%s\n' % (time, value))
def test_remove_by_type(self): RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=self.date, value_int=41.7) record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=self.date, value_float=66.6) RecordPrototype.remove_by_type(relations.RECORD_TYPE.TEST_INT) self.assertEqual(list(RecordPrototype._db_all().values_list('id', flat=True)), [record.id])
def test_create(self): with self.check_delta(RecordPrototype._db_count, 1): record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=self.date, value_int=666, value_float=666.6) record.reload() self.assertTrue(record.type.is_TEST_INT) self.assertEqual(record.date, self.date) self.assertEqual(record.value_int, 666) self.assertEqual(record.value_float, 666.6)
def handle(self, *args, **options): force_clear = options.get('force-clear') verbose = options.get('verbose') recalculate = options.get('recalculate-last') if recalculate: for MetricClass in METRICS: RecordPrototype._db_filter(date=MetricClass._last_datetime().date(), type=MetricClass.TYPE).delete() for MetricClass in METRICS: if force_clear or MetricClass.FULL_CLEAR_RECUIRED: if verbose: print 'clear %s' % MetricClass.TYPE MetricClass.clear() for i, MetricClass in enumerate(METRICS): metric = MetricClass() if verbose: print '[%3d] calculate %s' % (i, metric.TYPE) metric.initialize() metric.complete_values() data_version = int(settings.get(statistics_settings.JS_DATA_FILE_VERSION_KEY, 0)) data_version += 1 output_file = statistics_settings.JS_DATA_FILE_LOCATION % data_version output_dir_name = os.path.dirname(output_file) if not os.path.exists(output_dir_name): os.makedirs(output_dir_name, 0755) with open(output_file, 'w') as f: f.write(jinja2.render('statistics/js_data.js', context={'data': s11n.to_json(RecordPrototype.get_js_data())}).encode('utf-8')) settings[statistics_settings.JS_DATA_FILE_VERSION_KEY] = str(data_version)
def complete_values(self): sources = [] for source in self.SOURCES: data = RecordPrototype._db_filter(type=source, date__gt=self.last_date).order_by('date').values_list('date', 'value_int') sources.append(data) for source_record in zip(*sources): dates, values = zip(*source_record) if list(dates) != [dates[0]]*len(dates): raise exceptions.UnequalDatesError() self.store_value(dates[0], self.get_combined_value(*values))
def clear(cls): RecordPrototype.remove_by_type(cls.TYPE)
def print_funnel(self, year, month, new_users): DATE_FROM = datetime.datetime(year, month, 1) if month == 12: year += 1 month = 0 DATE_TO = datetime.datetime(year, month+1, 1) fast_registrations = sum(RecordPrototype.select_values(relations.RECORD_TYPE.REGISTRATIONS_TRIES, date_from=DATE_FROM, date_to=DATE_TO)) registrations = sum(RecordPrototype.select_values(relations.RECORD_TYPE.REGISTRATIONS_COMPLETED, date_from=DATE_FROM, date_to=DATE_TO)) new_accounts_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO).values_list('id', flat=True)) all_payers_ids = set(InvoicePrototype._db_filter(models.Q(state=INVOICE_STATE.CONFIRMED)|models.Q(state=INVOICE_STATE.FORCED), sender_type=ENTITY_TYPE.XSOLLA, currency=CURRENCY_TYPE.PREMIUM).values_list('recipient_id', flat=True)) payers = len(new_accounts_ids & all_payers_ids) alive_after_week_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=7)).values_list('id', flat=True)) alive_after_week = len(alive_after_week_ids & new_accounts_ids) alive_after_1_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=30)).values_list('id', flat=True)) alive_after_2_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=60)).values_list('id', flat=True)) alive_after_3_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=90)).values_list('id', flat=True)) alive_after_4_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=120)).values_list('id', flat=True)) alive_after_5_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=150)).values_list('id', flat=True)) alive_after_6_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at')+datetime.timedelta(days=180)).values_list('id', flat=True)) alive_after_1_month = len(alive_after_1_month_ids & new_accounts_ids) alive_after_2_month = len(alive_after_2_month_ids & new_accounts_ids) alive_after_3_month = len(alive_after_3_month_ids & new_accounts_ids) alive_after_4_month = len(alive_after_4_month_ids & new_accounts_ids) alive_after_5_month = len(alive_after_5_month_ids & new_accounts_ids) alive_after_6_month = len(alive_after_6_month_ids & new_accounts_ids) print('--------------------------------------') print('from %s to %s' % (DATE_FROM.date(), DATE_TO.date())) print('visitors: %d' % new_users) print('registration tries %d (%.3f from visitors)' % (fast_registrations, float(fast_registrations)/new_users)) print('registration completed %d (%.3f from visitors)' % (registrations, float(registrations)/new_users)) print('payers %d (%.3f from registrations)' % (payers, float(payers)/registrations)) print('alive after week %d (%.3f from registrations)' % (alive_after_week, float(alive_after_week)/registrations)) print('alive after 1_month %d (%.3f from registrations)' % (alive_after_1_month, float(alive_after_1_month)/registrations)) print('alive after 2 month %d (%.3f from registrations)' % (alive_after_2_month, float(alive_after_2_month)/registrations)) print('alive after 3 month %d (%.3f from registrations)' % (alive_after_3_month, float(alive_after_3_month)/registrations)) print('alive after 4 month %d (%.4f from registrations)' % (alive_after_4_month, float(alive_after_4_month)/registrations)) print('alive after 5 month %d (%.5f from registrations)' % (alive_after_5_month, float(alive_after_5_month)/registrations)) print('alive after 6 month %d (%.6f from registrations)' % (alive_after_6_month, float(alive_after_6_month)/registrations))
def _last_datetime(cls): try: return RecordPrototype._db_filter(type=cls.TYPE).order_by('-date')[0].date except IndexError: return statistics_settings.START_DATE - datetime.timedelta(days=1)
def store_value(self, date, value): return RecordPrototype.create(type=self.TYPE, date=date, value_int=value if self.TYPE.value_type.is_INT else None, value_float=value if self.TYPE.value_type.is_FLOAT else None)
def test_create__float_specified(self): record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=self.date, value_int=666) record.reload() self.assertEqual(record.value_float, 666.0)
def _last_datetime(cls): try: return RecordPrototype._db_filter( type=cls.TYPE).order_by('-date')[0].date except IndexError: return statistics_settings.START_DATE - datetime.timedelta(days=1)
def print_funnel(self, year, month, new_users): DATE_FROM = datetime.datetime(year, month, 1) if month == 12: year += 1 month = 0 DATE_TO = datetime.datetime(year, month + 1, 1) fast_registrations = sum( RecordPrototype.select_values( relations.RECORD_TYPE.REGISTRATIONS_TRIES, date_from=DATE_FROM, date_to=DATE_TO)) registrations = sum( RecordPrototype.select_values( relations.RECORD_TYPE.REGISTRATIONS_COMPLETED, date_from=DATE_FROM, date_to=DATE_TO)) new_accounts_ids = set( AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO).values_list( 'id', flat=True)) all_payers_ids = set( InvoicePrototype._db_filter( models.Q(state=INVOICE_STATE.CONFIRMED) | models.Q(state=INVOICE_STATE.FORCED), sender_type=ENTITY_TYPE.XSOLLA, currency=CURRENCY_TYPE.PREMIUM).values_list('recipient_id', flat=True)) payers = len(new_accounts_ids & all_payers_ids) alive_after_week_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=7)).values_list('id', flat=True)) alive_after_week = len(alive_after_week_ids & new_accounts_ids) alive_after_1_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=30)).values_list('id', flat=True)) alive_after_2_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=60)).values_list('id', flat=True)) alive_after_3_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=90)).values_list('id', flat=True)) alive_after_4_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=120)).values_list('id', flat=True)) alive_after_5_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=150)).values_list('id', flat=True)) alive_after_6_month_ids = set( AccountPrototype._db_filter( created_at__gte=DATE_FROM, created_at__lte=DATE_TO, active_end_at__gte=models.F('created_at') + datetime.timedelta(days=180)).values_list('id', flat=True)) alive_after_1_month = len(alive_after_1_month_ids & new_accounts_ids) alive_after_2_month = len(alive_after_2_month_ids & new_accounts_ids) alive_after_3_month = len(alive_after_3_month_ids & new_accounts_ids) alive_after_4_month = len(alive_after_4_month_ids & new_accounts_ids) alive_after_5_month = len(alive_after_5_month_ids & new_accounts_ids) alive_after_6_month = len(alive_after_6_month_ids & new_accounts_ids) print('--------------------------------------') print('from %s to %s' % (DATE_FROM.date(), DATE_TO.date())) print('visitors: %d' % new_users) print('registration tries %d (%.3f from visitors)' % (fast_registrations, float(fast_registrations) / new_users)) print('registration completed %d (%.3f from visitors)' % (registrations, float(registrations) / new_users)) print('payers %d (%.3f from registrations)' % (payers, float(payers) / registrations)) print('alive after week %d (%.3f from registrations)' % (alive_after_week, float(alive_after_week) / registrations)) print( 'alive after 1_month %d (%.3f from registrations)' % (alive_after_1_month, float(alive_after_1_month) / registrations)) print( 'alive after 2 month %d (%.3f from registrations)' % (alive_after_2_month, float(alive_after_2_month) / registrations)) print( 'alive after 3 month %d (%.3f from registrations)' % (alive_after_3_month, float(alive_after_3_month) / registrations)) print( 'alive after 4 month %d (%.4f from registrations)' % (alive_after_4_month, float(alive_after_4_month) / registrations)) print( 'alive after 5 month %d (%.5f from registrations)' % (alive_after_5_month, float(alive_after_5_month) / registrations)) print( 'alive after 6 month %d (%.6f from registrations)' % (alive_after_6_month, float(alive_after_6_month) / registrations))
def test_last_datetime__no_records(self): self.assertEqual(RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 0) self.assertEqual(self.metric._last_datetime(), datetime.datetime(year=2012, month=6, day=26))
def store_value(self, date, value): return RecordPrototype.create( type=self.TYPE, date=date, value_int=value if self.TYPE.value_type.is_INT else None, value_float=value if self.TYPE.value_type.is_FLOAT else None)
def test_last_datetime__no_records(self): self.assertEqual( RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 0) self.assertEqual(self.metric._last_datetime(), datetime.datetime(year=2012, month=6, day=26))