def update_cached_items() -> None:
tasks = []
items = (DashboardItem.objects.filter(
Q(
Q(dashboard__is_shared=True)
| Q(dashboard__last_accessed_at__gt=timezone.now() -
relativedelta(days=7)))).exclude(refreshing=True).exclude(
deleted=True))
for item in items.filter(filters__isnull=False).exclude(
filters={}).distinct("filters"):
filter = Filter(data=item.filters)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(),
item.team_id))
payload = {"filter": filter.toJSON(), "team_id": item.team_id}
tasks.append(
update_cache_item_task.s(cache_key, TRENDS_ENDPOINT, payload))
for item in items.filter(funnel_id__isnull=False).distinct("funnel_id"):
cache_key = generate_cache_key("funnel_{}_{}".format(
item.funnel_id, item.team_id))
payload = {"funnel_id": item.funnel_id, "team_id": item.team_id}
tasks.append(
update_cache_item_task.s(cache_key, FUNNEL_ENDPOINT, payload))
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
def update_cached_items() -> None:
tasks = []
items = (
DashboardItem.objects.filter(
Q(Q(dashboard__is_shared=True) | Q(dashboard__last_accessed_at__gt=timezone.now() - relativedelta(days=7)))
)
.exclude(dashboard__deleted=True)
.exclude(refreshing=True)
.exclude(deleted=True)
.distinct("filters_hash")
)
for item in DashboardItem.objects.filter(
pk__in=Subquery(items.filter(filters__isnull=False).exclude(filters={}).distinct("filters").values("pk"))
).order_by(F("last_refresh").asc(nulls_first=True))[0:PARALLEL_DASHBOARD_ITEM_CACHE]:
filter = get_filter(data=item.dashboard_filters(), team=item.team)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), item.team_id))
cache_type = get_cache_type(filter)
payload = {"filter": filter.toJSON(), "team_id": item.team_id}
tasks.append(update_cache_item_task.s(cache_key, cache_type, payload))
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
def update_cached_items() -> None:
tasks = []
items = (DashboardItem.objects.filter(
Q(
Q(dashboard__is_shared=True)
| Q(dashboard__last_accessed_at__gt=timezone.now() -
relativedelta(days=7)))).exclude(
dashboard__deleted=True).exclude(refreshing=True).exclude(
deleted=True))
for item in items.filter(filters__isnull=False).exclude(
filters={}).distinct("filters"):
filter = Filter(data=item.filters)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(),
item.team_id))
curr_data = cache.get(cache_key)
# if task is logged and loading leave it alone
if curr_data and curr_data.get("task_id", None):
continue
cache_type = CacheType.FUNNEL if filter.insight == "FUNNELS" else CacheType.TRENDS
payload = {"filter": filter.toJSON(), "team_id": item.team_id}
tasks.append(update_cache_item_task.s(cache_key, cache_type, payload))
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
def update_cached_items() -> None:
tasks = []
items = (Insight.objects.filter(
Q(
Q(dashboard__is_shared=True)
| Q(dashboard__last_accessed_at__gt=timezone.now() -
relativedelta(days=7)))).exclude(
dashboard__deleted=True).exclude(refreshing=True).exclude(
deleted=True).exclude(refresh_attempt__gt=2).exclude(
filters={}).order_by(
F("last_refresh").asc(nulls_first=True)))
for item in items[0:PARALLEL_INSIGHT_CACHE]:
try:
cache_key, cache_type, payload = dashboard_item_update_task_params(
item)
if item.filters_hash != cache_key:
item.save(
) # force update if the saved key is different from the cache key
tasks.append(
update_cache_item_task.s(cache_key, cache_type, payload))
except Exception as e:
item.refresh_attempt = (item.refresh_attempt or 0) + 1
item.save()
capture_exception(e)
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
statsd.gauge("update_cache_queue_depth", items.count())
def update_cached_items() -> None:
tasks = []
items = (Insight.objects.filter(
Q(
Q(dashboard__is_shared=True)
| Q(dashboard__last_accessed_at__gt=timezone.now() -
relativedelta(days=7)))).exclude(
dashboard__deleted=True).exclude(refreshing=True).exclude(
deleted=True).distinct("filters_hash"))
for item in Insight.objects.filter(pk__in=Subquery(
items.filter(filters__isnull=False).exclude(
filters={}).distinct("filters").values("pk"))).order_by(
F("last_refresh").asc(
nulls_first=True))[0:PARALLEL_DASHBOARD_ITEM_CACHE]:
cache_key, cache_type, payload = dashboard_item_update_task_params(
item)
tasks.append(update_cache_item_task.s(cache_key, cache_type, payload))
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
