def handle(self, *args, **options):
force_clear = options.get('force-clear')
verbose = options.get('verbose')
recalculate = options.get('recalculate-last')
if recalculate:
for MetricClass in METRICS:
RecordPrototype._db_filter(date=MetricClass._last_datetime().date(),
type=MetricClass.TYPE).delete()
for MetricClass in METRICS:
if force_clear or MetricClass.FULL_CLEAR_RECUIRED:
if verbose:
print('clear %s' % MetricClass.TYPE)
MetricClass.clear()
for i, MetricClass in enumerate(METRICS):
metric = MetricClass()
if verbose:
print('[%3d] calculate %s' % (i, metric.TYPE))
metric.initialize()
metric.complete_values()
models.FullStatistics.objects.all().delete()
models.FullStatistics.objects.create(data=RecordPrototype.get_js_data())
def handle(self, *args, **options):
force_clear = options.get('force-clear')
verbose = options.get('verbose')
recalculate = options.get('recalculate-last')
if recalculate:
for MetricClass in METRICS:
RecordPrototype._db_filter(date=MetricClass._last_datetime().date(),
type=MetricClass.TYPE).delete()
for MetricClass in METRICS:
if force_clear or MetricClass.FULL_CLEAR_RECUIRED:
if verbose:
print('clear %s' % MetricClass.TYPE)
MetricClass.clear()
for i, MetricClass in enumerate(METRICS):
metric = MetricClass()
if verbose:
print('[%3d] calculate %s' % (i, metric.TYPE))
metric.initialize()
metric.complete_values()
models.FullStatistics.objects.all().delete()
models.FullStatistics.objects.create(data=RecordPrototype.get_js_data())
def test_store_value__float(self):
self.metric.store_value(datetime.datetime(year=6, month=6, day=6, hour=6), 666.6)
self.assertEqual(RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 1)
record = RecordPrototype._db_get_object(0)
self.assertEqual(record.date, datetime.datetime(year=6, month=6, day=6, hour=6))
self.assertEqual(record.value_int, 666)
self.assertEqual(record.value_float, 666.6)
self.assertTrue(record.type.is_TEST_FLOAT)
def test_select__int(self):
dates = [self.date + self.timedelta * i for i in range(4)]
RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=dates[1], value_int=417)
RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=dates[2], value_int=666)
self.assertEqual(
RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[0], date_to=dates[3]),
[(dates[1], 417), (dates[2], 666)],
)
self.assertEqual(
RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[1], date_to=dates[2]),
[(dates[1], 417), (dates[2], 666)],
)
self.assertEqual(
RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[1], date_to=dates[1]),
[(dates[1], 417)],
)
self.assertEqual(
RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[2], date_to=dates[2]),
[(dates[2], 666)],
)
self.assertEqual(
RecordPrototype.select(type=relations.RECORD_TYPE.TEST_INT, date_from=dates[3], date_to=dates[3]), []
)
def test_store_value__float(self):
self.metric.store_value(
datetime.datetime(year=6, month=6, day=6, hour=6), 666.6)
self.assertEqual(
RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 1)
record = RecordPrototype._db_get_object(0)
self.assertEqual(record.date,
datetime.datetime(year=6, month=6, day=6, hour=6))
self.assertEqual(record.value_int, 666)
self.assertEqual(record.value_float, 666.6)
self.assertTrue(record.type.is_TEST_FLOAT)
def test_no_second_record(self):
self.metric.complete_values()
self.assertTrue(RecordPrototype._db_count() > 0)
self.metric.initialize()
with self.check_not_changed(RecordPrototype._db_count):
self.metric.complete_values()
def test_no_second_record(self):
self.metric.complete_values()
self.assertTrue(RecordPrototype._db_count() > 0)
self.metric.initialize()
with self.check_not_changed(RecordPrototype._db_count):
self.metric.complete_values()
def test_select__float(self):
date_1 = datetime.datetime.now()
date_2 = datetime.datetime.now()
date_3 = datetime.datetime.now()
date_4 = datetime.datetime.now()
RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=date_2, value_float=41.7)
RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=date_3, value_float=66.6)
self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_1, date_to=date_4), [(date_2, 41.7), (date_3, 66.6)])
self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_2, date_to=date_3), [(date_2, 41.7), (date_3, 66.6)])
self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_2, date_to=date_2), [(date_2, 41.7)])
self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_3, date_to=date_3), [(date_3, 66.6)])
self.assertEqual(RecordPrototype.select(type=relations.RECORD_TYPE.TEST_FLOAT, date_from=date_4, date_to=date_4), [])
def handle(self, *args, **options):
for record in relations.RECORD_TYPE.records:
data = RecordPrototype.select_for_js(record,
date_from=datetime.datetime(year=2013, month=8, day=1),
date_to=datetime.datetime(year=2014, month=6, day=4))
with open('/tmp/stats_%s.csv' % record.name.lower(), 'w') as f:
for time, value in data:
f.write('%s;%s\n' % (time, value))
def handle(self, *args, **options):
for record in relations.RECORD_TYPE.records:
data = RecordPrototype.select_for_js(record,
date_from=datetime.datetime(year=2013, month=8, day=1),
date_to=datetime.datetime(year=2014, month=6, day=4))
with open('/tmp/stats_%s.csv' % record.name.lower(), 'w') as f:
for time, value in data:
f.write('%s;%s\n' % (time, value))
def test_remove_by_type(self):
RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=self.date, value_int=41.7)
record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_FLOAT, date=self.date, value_float=66.6)
RecordPrototype.remove_by_type(relations.RECORD_TYPE.TEST_INT)
self.assertEqual(list(RecordPrototype._db_all().values_list('id', flat=True)), [record.id])
def test_create(self):
with self.check_delta(RecordPrototype._db_count, 1):
record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT,
date=self.date,
value_int=666, value_float=666.6)
record.reload()
self.assertTrue(record.type.is_TEST_INT)
self.assertEqual(record.date, self.date)
self.assertEqual(record.value_int, 666)
self.assertEqual(record.value_float, 666.6)
def handle(self, *args, **options):
force_clear = options.get('force-clear')
verbose = options.get('verbose')
recalculate = options.get('recalculate-last')
if recalculate:
for MetricClass in METRICS:
RecordPrototype._db_filter(date=MetricClass._last_datetime().date(),
type=MetricClass.TYPE).delete()
for MetricClass in METRICS:
if force_clear or MetricClass.FULL_CLEAR_RECUIRED:
if verbose:
print 'clear %s' % MetricClass.TYPE
MetricClass.clear()
for i, MetricClass in enumerate(METRICS):
metric = MetricClass()
if verbose:
print '[%3d] calculate %s' % (i, metric.TYPE)
metric.initialize()
metric.complete_values()
data_version = int(settings.get(statistics_settings.JS_DATA_FILE_VERSION_KEY, 0))
data_version += 1
output_file = statistics_settings.JS_DATA_FILE_LOCATION % data_version
output_dir_name = os.path.dirname(output_file)
if not os.path.exists(output_dir_name):
os.makedirs(output_dir_name, 0755)
with open(output_file, 'w') as f:
f.write(jinja2.render('statistics/js_data.js',
context={'data': s11n.to_json(RecordPrototype.get_js_data())}).encode('utf-8'))
settings[statistics_settings.JS_DATA_FILE_VERSION_KEY] = str(data_version)
def complete_values(self):
sources = []
for source in self.SOURCES:
data = RecordPrototype._db_filter(type=source,
date__gt=self.last_date).order_by('date').values_list('date', 'value_int')
sources.append(data)
for source_record in zip(*sources):
dates, values = zip(*source_record)
if list(dates) != [dates[0]]*len(dates):
raise exceptions.UnequalDatesError()
self.store_value(dates[0], self.get_combined_value(*values))
def complete_values(self):
sources = []
for source in self.SOURCES:
data = RecordPrototype._db_filter(type=source,
date__gt=self.last_date).order_by('date').values_list('date', 'value_int')
sources.append(data)
for source_record in zip(*sources):
dates, values = zip(*source_record)
if list(dates) != [dates[0]]*len(dates):
raise exceptions.UnequalDatesError()
self.store_value(dates[0], self.get_combined_value(*values))
def clear(cls):
RecordPrototype.remove_by_type(cls.TYPE)
def print_funnel(self, year, month, new_users):
DATE_FROM = datetime.datetime(year, month, 1)
if month == 12:
year += 1
month = 0
DATE_TO = datetime.datetime(year, month+1, 1)
fast_registrations = sum(RecordPrototype.select_values(relations.RECORD_TYPE.REGISTRATIONS_TRIES, date_from=DATE_FROM, date_to=DATE_TO))
registrations = sum(RecordPrototype.select_values(relations.RECORD_TYPE.REGISTRATIONS_COMPLETED, date_from=DATE_FROM, date_to=DATE_TO))
new_accounts_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM, created_at__lte=DATE_TO).values_list('id', flat=True))
all_payers_ids = set(InvoicePrototype._db_filter(models.Q(state=INVOICE_STATE.CONFIRMED)|models.Q(state=INVOICE_STATE.FORCED),
sender_type=ENTITY_TYPE.XSOLLA,
currency=CURRENCY_TYPE.PREMIUM).values_list('recipient_id', flat=True))
payers = len(new_accounts_ids & all_payers_ids)
alive_after_week_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=7)).values_list('id', flat=True))
alive_after_week = len(alive_after_week_ids & new_accounts_ids)
alive_after_1_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=30)).values_list('id', flat=True))
alive_after_2_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=60)).values_list('id', flat=True))
alive_after_3_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=90)).values_list('id', flat=True))
alive_after_4_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=120)).values_list('id', flat=True))
alive_after_5_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=150)).values_list('id', flat=True))
alive_after_6_month_ids = set(AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at')+datetime.timedelta(days=180)).values_list('id', flat=True))
alive_after_1_month = len(alive_after_1_month_ids & new_accounts_ids)
alive_after_2_month = len(alive_after_2_month_ids & new_accounts_ids)
alive_after_3_month = len(alive_after_3_month_ids & new_accounts_ids)
alive_after_4_month = len(alive_after_4_month_ids & new_accounts_ids)
alive_after_5_month = len(alive_after_5_month_ids & new_accounts_ids)
alive_after_6_month = len(alive_after_6_month_ids & new_accounts_ids)
print('--------------------------------------')
print('from %s to %s' % (DATE_FROM.date(), DATE_TO.date()))
print('visitors: %d' % new_users)
print('registration tries %d (%.3f from visitors)' % (fast_registrations, float(fast_registrations)/new_users))
print('registration completed %d (%.3f from visitors)' % (registrations, float(registrations)/new_users))
print('payers %d (%.3f from registrations)' % (payers, float(payers)/registrations))
print('alive after week %d (%.3f from registrations)' % (alive_after_week, float(alive_after_week)/registrations))
print('alive after 1_month %d (%.3f from registrations)' % (alive_after_1_month, float(alive_after_1_month)/registrations))
print('alive after 2 month %d (%.3f from registrations)' % (alive_after_2_month, float(alive_after_2_month)/registrations))
print('alive after 3 month %d (%.3f from registrations)' % (alive_after_3_month, float(alive_after_3_month)/registrations))
print('alive after 4 month %d (%.4f from registrations)' % (alive_after_4_month, float(alive_after_4_month)/registrations))
print('alive after 5 month %d (%.5f from registrations)' % (alive_after_5_month, float(alive_after_5_month)/registrations))
print('alive after 6 month %d (%.6f from registrations)' % (alive_after_6_month, float(alive_after_6_month)/registrations))
def clear(cls):
RecordPrototype.remove_by_type(cls.TYPE)
def _last_datetime(cls):
try:
return RecordPrototype._db_filter(type=cls.TYPE).order_by('-date')[0].date
except IndexError:
return statistics_settings.START_DATE - datetime.timedelta(days=1)
def store_value(self, date, value):
return RecordPrototype.create(type=self.TYPE,
date=date,
value_int=value if self.TYPE.value_type.is_INT else None,
value_float=value if self.TYPE.value_type.is_FLOAT else None)
def test_create__float_specified(self):
record = RecordPrototype.create(type=relations.RECORD_TYPE.TEST_INT, date=self.date, value_int=666)
record.reload()
self.assertEqual(record.value_float, 666.0)
def _last_datetime(cls):
try:
return RecordPrototype._db_filter(
type=cls.TYPE).order_by('-date')[0].date
except IndexError:
return statistics_settings.START_DATE - datetime.timedelta(days=1)
def print_funnel(self, year, month, new_users):
DATE_FROM = datetime.datetime(year, month, 1)
if month == 12:
year += 1
month = 0
DATE_TO = datetime.datetime(year, month + 1, 1)
fast_registrations = sum(
RecordPrototype.select_values(
relations.RECORD_TYPE.REGISTRATIONS_TRIES,
date_from=DATE_FROM,
date_to=DATE_TO))
registrations = sum(
RecordPrototype.select_values(
relations.RECORD_TYPE.REGISTRATIONS_COMPLETED,
date_from=DATE_FROM,
date_to=DATE_TO))
new_accounts_ids = set(
AccountPrototype._db_filter(created_at__gte=DATE_FROM,
created_at__lte=DATE_TO).values_list(
'id', flat=True))
all_payers_ids = set(
InvoicePrototype._db_filter(
models.Q(state=INVOICE_STATE.CONFIRMED)
| models.Q(state=INVOICE_STATE.FORCED),
sender_type=ENTITY_TYPE.XSOLLA,
currency=CURRENCY_TYPE.PREMIUM).values_list('recipient_id',
flat=True))
payers = len(new_accounts_ids & all_payers_ids)
alive_after_week_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=7)).values_list('id', flat=True))
alive_after_week = len(alive_after_week_ids & new_accounts_ids)
alive_after_1_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=30)).values_list('id', flat=True))
alive_after_2_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=60)).values_list('id', flat=True))
alive_after_3_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=90)).values_list('id', flat=True))
alive_after_4_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=120)).values_list('id', flat=True))
alive_after_5_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=150)).values_list('id', flat=True))
alive_after_6_month_ids = set(
AccountPrototype._db_filter(
created_at__gte=DATE_FROM,
created_at__lte=DATE_TO,
active_end_at__gte=models.F('created_at') +
datetime.timedelta(days=180)).values_list('id', flat=True))
alive_after_1_month = len(alive_after_1_month_ids & new_accounts_ids)
alive_after_2_month = len(alive_after_2_month_ids & new_accounts_ids)
alive_after_3_month = len(alive_after_3_month_ids & new_accounts_ids)
alive_after_4_month = len(alive_after_4_month_ids & new_accounts_ids)
alive_after_5_month = len(alive_after_5_month_ids & new_accounts_ids)
alive_after_6_month = len(alive_after_6_month_ids & new_accounts_ids)
print('--------------------------------------')
print('from %s to %s' % (DATE_FROM.date(), DATE_TO.date()))
print('visitors: %d' % new_users)
print('registration tries %d (%.3f from visitors)' %
(fast_registrations, float(fast_registrations) / new_users))
print('registration completed %d (%.3f from visitors)' %
(registrations, float(registrations) / new_users))
print('payers %d (%.3f from registrations)' %
(payers, float(payers) / registrations))
print('alive after week %d (%.3f from registrations)' %
(alive_after_week, float(alive_after_week) / registrations))
print(
'alive after 1_month %d (%.3f from registrations)' %
(alive_after_1_month, float(alive_after_1_month) / registrations))
print(
'alive after 2 month %d (%.3f from registrations)' %
(alive_after_2_month, float(alive_after_2_month) / registrations))
print(
'alive after 3 month %d (%.3f from registrations)' %
(alive_after_3_month, float(alive_after_3_month) / registrations))
print(
'alive after 4 month %d (%.4f from registrations)' %
(alive_after_4_month, float(alive_after_4_month) / registrations))
print(
'alive after 5 month %d (%.5f from registrations)' %
(alive_after_5_month, float(alive_after_5_month) / registrations))
print(
'alive after 6 month %d (%.6f from registrations)' %
(alive_after_6_month, float(alive_after_6_month) / registrations))
def test_last_datetime__no_records(self):
self.assertEqual(RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 0)
self.assertEqual(self.metric._last_datetime(), datetime.datetime(year=2012, month=6, day=26))
def store_value(self, date, value):
return RecordPrototype.create(
type=self.TYPE,
date=date,
value_int=value if self.TYPE.value_type.is_INT else None,
value_float=value if self.TYPE.value_type.is_FLOAT else None)
def test_last_datetime__no_records(self):
self.assertEqual(
RecordPrototype._db_filter(type=TestMetric.TYPE).count(), 0)
self.assertEqual(self.metric._last_datetime(),
datetime.datetime(year=2012, month=6, day=26))
