def index_installed_daily(ids, **kw):
"""
Takes a list of Installed ids and uses its addon and date fields to index
stats for that day.
ids -- ids of mkt.webapps.Installed objects
"""
from mkt.webapps.models import Installed
es = elasticutils.get_es()
# Get Installed's
qs = (Installed.objects.filter(
id__in=set(ids)).order_by('-created').values('addon', 'created'))
log.info('[%s] Indexing %s installed counts for daily stats.' %
(qs[0]['created'], len(qs)))
addons_dates = defaultdict(lambda: defaultdict(dict))
for installed in qs:
addon = installed['addon']
date = installed['created'].strftime('%Y%m%d')
try:
if not date in addons_dates[addon]:
key = ord_word('ins' + str(addon) + str(date))
data = search.get_installed_daily(installed)
if not already_indexed(Installed, data):
Installed.index(data, bulk=True, id=key)
addons_dates[addon][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_installed_daily.retry(args=[ids], exc=exc)
raise
def index_installed_daily(ids, **kw):
"""
Takes a list of Installed ids and uses its addon and date fields to index
stats for that day.
ids -- ids of mkt.webapps.Installed objects
"""
from mkt.webapps.models import Installed
index = kw.get('index', Installed._get_index())
es = amo.search.get_es()
# Get Installed's
qs = (Installed.objects.filter(id__in=set(ids)).
order_by('-created').values('addon', 'created'))
log.info('[%s] Indexing %s installed counts for daily stats.' %
(qs[0]['created'], len(qs)))
addons_dates = defaultdict(lambda: defaultdict(dict))
for installed in qs:
addon = installed['addon']
date = installed['created'].strftime('%Y%m%d')
try:
if not date in addons_dates[addon]:
key = ord_word('ins' + str(addon) + str(date))
data = search.get_installed_daily(installed)
for index in get_indices(index):
if not already_indexed(Installed, data, index):
Installed.index(data, bulk=True, id=key,
index=index)
addons_dates[addon][date] = 0
es.flush_bulk(forced=True)
except Exception, exc:
index_installed_daily.retry(args=[ids], exc=exc, **kw)
raise
def setUp(self):
self.today = datetime.date.today()
self.webapp = Addon.objects.get(pk=337141)
self.user = UserProfile.objects.get(pk=999)
self.client.login(username="******", password="******")
self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
Installed.index(search.extract_installed_count(self.in_), id=self.in_.pk)
self.refresh("users_install")
def setUp(self):
self.today = datetime.date.today()
self.webapp = Addon.objects.get(pk=337141)
self.user = UserProfile.objects.get(pk=999)
self.client.login(username='******', password='******')
self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
installed = {'addon': self.in_.addon.id, 'created': self.in_.created}
Installed.index(search.get_installed_daily(installed), id=self.in_.pk)
self.refresh('users_install')
def setUp(self):
self.today = datetime.date.today()
self.webapp = Addon.objects.get(pk=337141)
self.user = UserProfile.objects.get(pk=999)
self.client.login(username='******', password='******')
self.in_ = Installed.objects.create(addon=self.webapp, user=self.user)
installed = {'addon': self.in_.addon.id, 'created': self.in_.created}
Installed.index(search.get_installed_daily(installed),
id=self.in_.pk)
self.refresh('users_install')
def index_latest_mkt_stats(index=None, aliased=True):
raise_if_reindex_in_progress()
yesterday = datetime.date.today() - datetime.timedelta(days=1)
try:
latest = Contribution.search(index).order_by('-date').values_dict()
latest_contribution = latest and latest[0]['date'] or yesterday
except pyes.exceptions.SearchPhaseExecutionException:
latest_contribution = yesterday
try:
latest = Installed.search(index).order_by('-date').values_dict()
latest_install = latest and latest[0]['date'] or yesterday
except pyes.exceptions.SearchPhaseExecutionException:
latest_install = yesterday
latest = min(latest_contribution, latest_install)
fmt = lambda d: d.strftime('%Y-%m-%d')
date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
cron_log.info('index_mkt_stats --date=%s' % date_range)
call_command('index_mkt_stats',
addons=None,
date=date_range,
index=index,
aliased=True)
def index_installed_counts(ids, **kw):
es = elasticutils.get_es()
qs = Installed.objects.filter(id__in=set(ids))
if qs:
log.info('Indexing %s installed counts: %s'
% (len(qs), qs[0].created))
try:
for installed in qs:
addon_id = installed.addon_id
key = '%s-%s' % (addon_id, installed.created)
data = search.extract_installed_count(installed)
Installed.index(data, bulk=True, id=key)
es.flush_bulk(forced=True)
except Exception, exc:
index_installed_counts.retry(args=[ids], exc=exc)
raise
def index_installed_counts(ids, **kw):
from mkt.webapps.models import Installed
es = elasticutils.get_es()
qs = Installed.objects.filter(id__in=set(ids))
if qs.exists():
log.info('Indexing %s installed counts: %s' % (len(qs), qs[0].created))
try:
for installed in qs:
addon_id = installed.addon_id
key = '%s-%s' % (addon_id, installed.created)
data = search.extract_installed_count(installed)
Installed.index(data, bulk=True, id=key)
es.flush_bulk(forced=True)
except Exception, exc:
index_installed_counts.retry(args=[ids], exc=exc)
raise
def index_latest_mkt_stats():
latest_contribution = Contribution.search().order_by('-date'
).values_dict()[0]['date']
latest_install = Installed.search().order_by('-date'
).values_dict()[0]['date']
latest = min(latest_contribution, latest_install)
fmt = lambda d: d.strftime('%Y-%m-%d')
date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
cron_log.info('index_mkt_stats --date=%s' % date_range)
call_command('index_mkt_stats', addons=None, date=date_range)
def index_latest_mkt_stats():
latest_contribution = Contribution.search().order_by(
'-date').values_dict()[0]['date']
latest_install = Installed.search().order_by(
'-date').values_dict()[0]['date']
latest = min(latest_contribution, latest_install)
fmt = lambda d: d.strftime('%Y-%m-%d')
date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
cron_log.info('index_mkt_stats --date=%s' % date_range)
call_command('index_mkt_stats', addons=None, date=date_range)
def index_latest_mkt_stats(index=None, aliased=True):
raise_if_reindex_in_progress()
yesterday = datetime.date.today() - datetime.timedelta(days=1)
try:
latest = Contribution.search(index).order_by('-date').values_dict()
latest_contribution = latest and latest[0]['date'] or yesterday
except pyes.exceptions.SearchPhaseExecutionException:
latest_contribution = yesterday
try:
latest = Installed.search(index).order_by('-date').values_dict()
latest_install = latest and latest[0]['date'] or yesterday
except pyes.exceptions.SearchPhaseExecutionException:
latest_install = yesterday
latest = min(latest_contribution, latest_install)
fmt = lambda d: d.strftime('%Y-%m-%d')
date_range = '%s:%s' % (fmt(latest), fmt(datetime.date.today()))
cron_log.info('index_mkt_stats --date=%s' % date_range)
call_command('index_mkt_stats', addons=None, date=date_range, index=index,
aliased=True)