def update_cached_items() -> None:
tasks = []
items = (
DashboardItem.objects.filter(
Q(Q(dashboard__is_shared=True) | Q(dashboard__last_accessed_at__gt=timezone.now() - relativedelta(days=7)))
)
.exclude(dashboard__deleted=True)
.exclude(refreshing=True)
.exclude(deleted=True)
.distinct("filters_hash")
)
for item in DashboardItem.objects.filter(
pk__in=Subquery(items.filter(filters__isnull=False).exclude(filters={}).distinct("filters").values("pk"))
).order_by(F("last_refresh").asc(nulls_first=True))[0:PARALLEL_DASHBOARD_ITEM_CACHE]:
filter = get_filter(data=item.dashboard_filters(), team=item.team)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), item.team_id))
cache_type = get_cache_type(filter)
payload = {"filter": filter.toJSON(), "team_id": item.team_id}
tasks.append(update_cache_item_task.s(cache_key, cache_type, payload))
logger.info("Found {} items to refresh".format(len(tasks)))
taskset = group(tasks)
taskset.apply_async()
def wrapper(self, request) -> T:
# prepare caching params
team = cast(User, request.user).team
if not team:
return f(self, request)
filter = get_filter(request=request, team=team)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(),
team.pk))
# return cached result if possible
if not should_refresh(request):
cached_result_package = get_safe_cache(cache_key)
if cached_result_package and cached_result_package.get("result"):
cached_result_package["is_cached"] = True
return cached_result_package
# call function being wrapped
fresh_result_package = cast(T, f(self, request))
# cache new data
if isinstance(fresh_result_package, dict):
result = fresh_result_package.get("result")
if not isinstance(result, dict) or not result.get("loading"):
fresh_result_package["last_refresh"] = now()
fresh_result_package["is_cached"] = False
cache.set(
cache_key,
fresh_result_package,
TEMP_CACHE_RESULTS_TTL,
)
if filter:
dashboard_items = Insight.objects.filter(
team_id=team.pk, filters_hash=cache_key)
dashboard_items.update(last_refresh=now())
return fresh_result_package
def dashboard_item_saved(sender, instance: DashboardItem, **kwargs):
if instance.filters and instance.filters != {}:
filter = get_filter(data=instance.filters, team=instance.team)
instance.filters = filter.to_dict()
instance.filters_hash = generate_cache_key("{}_{}".format(
filter.toJSON(), instance.team_id))
def dashboard_item_saved(sender, instance: Insight, dashboard=None, **kwargs):
if instance.filters and instance.filters != {}:
filter = get_filter(
data=instance.dashboard_filters(dashboard=dashboard),
team=instance.team)
instance.filters_hash = generate_cache_key("{}_{}".format(
filter.toJSON(), instance.team_id))
def dashboard_item_update_task_params(
item: DashboardItem, dashboard: Optional[Dashboard] = None
) -> Tuple[str, CacheType, Dict]:
filter = get_filter(data=item.dashboard_filters(dashboard), team=item.team)
cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), item.team_id))
cache_type = get_cache_type(filter)
payload = {"filter": filter.toJSON(), "team_id": item.team_id}
return cache_key, cache_type, payload
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> None:
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
if result:
cache.set(key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, CACHED_RESULTS_TTL)
def update_cache_item(key: str, cache_type: CacheType,
payload: dict) -> List[Dict[str, Any]]:
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
cache.set(key, {
"result": result,
"type": cache_type,
"last_refresh": timezone.now()
}, CACHED_RESULTS_TTL)
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def wrapper(*args,
**kwargs) -> Dict[str, Union[List, datetime, bool, str]]:
# prepare caching params
request: HttpRequest = args[1]
team = cast(User, request.user).team
filter = None
if not team:
return f(*args, **kwargs)
filter = get_filter(request=request, team=team)
cache_key = generate_cache_key("{}_{}".format(
filter.toJSON(), team.pk))
# return cached result if possible
if not request.GET.get("refresh", False):
cached_result = get_safe_cache(cache_key)
if cached_result and cached_result.get("result"):
return {**cached_result, "is_cached": True}
# call function being wrapped
result = f(*args, **kwargs)
# cache new data
if result is not None and not (
isinstance(result.get("result"), dict)
and result["result"].get("loading")):
cache.set(
cache_key,
{
"result": result["result"],
"last_refresh": now()
},
TEMP_CACHE_RESULTS_TTL,
)
if filter:
dashboard_items = DashboardItem.objects.filter(
team_id=team.pk, filters_hash=cache_key)
dashboard_items.update(last_refresh=now())
return result
def wrapper(*args, **kwargs):
# prepare caching params
request: HttpRequest = args[1]
team = cast(User, request.user).team
filter = None
if not team:
return f(*args, **kwargs)
filter = get_filter(request=request, team=team)
cache_key = generate_cache_key("{}_{}".format(
filter.toJSON(), team.pk))
payload = {"filter": filter.toJSON(), "team_id": team.pk}
# return cached result if possible
if not request.GET.get("refresh", False):
cached_result = cache.get(cache_key)
if cached_result and cached_result.get("result"):
return cached_result["result"]
# call function being wrapped
result = f(*args, **kwargs)
# cache new data
if result is not None and (not isinstance(result, dict)
or not result.get("loading")):
cache.set(
cache_key,
{
"result": result,
"details": payload,
},
TEMP_CACHE_RESULTS_TTL,
)
if filter:
dashboard_items = DashboardItem.objects.filter(
team_id=team.pk, filters_hash=cache_key)
dashboard_items.update(last_refresh=now())
return result
def update_cache_item(key: str, cache_type: CacheType,
payload: dict) -> List[Dict[str, Any]]:
timer = statsd.timer("update_cache_item_timer").start()
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
# Doing the filtering like this means we'll update _all_ Insights with the same filters hash
dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
try:
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
cache.set(key, {
"result": result,
"type": cache_type,
"last_refresh": timezone.now()
}, settings.CACHED_RESULTS_TTL)
except Exception as e:
timer.stop()
statsd.incr("update_cache_item_error")
dashboard_items.filter(refresh_attempt=None).update(refresh_attempt=0)
dashboard_items.update(refreshing=False,
refresh_attempt=F("refresh_attempt") + 1)
raise e
timer.stop()
statsd.incr("update_cache_item_success")
dashboard_items.update(last_refresh=timezone.now(),
refreshing=False,
refresh_attempt=0)
return result
