def update_cached_items() -> None: tasks = [] items = ( DashboardItem.objects.filter( Q(Q(dashboard__is_shared=True) | Q(dashboard__last_accessed_at__gt=timezone.now() - relativedelta(days=7))) ) .exclude(dashboard__deleted=True) .exclude(refreshing=True) .exclude(deleted=True) .distinct("filters_hash") ) for item in DashboardItem.objects.filter( pk__in=Subquery(items.filter(filters__isnull=False).exclude(filters={}).distinct("filters").values("pk")) ).order_by(F("last_refresh").asc(nulls_first=True))[0:PARALLEL_DASHBOARD_ITEM_CACHE]: filter = get_filter(data=item.dashboard_filters(), team=item.team) cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), item.team_id)) cache_type = get_cache_type(filter) payload = {"filter": filter.toJSON(), "team_id": item.team_id} tasks.append(update_cache_item_task.s(cache_key, cache_type, payload)) logger.info("Found {} items to refresh".format(len(tasks))) taskset = group(tasks) taskset.apply_async()
def wrapper(self, request) -> T: # prepare caching params team = cast(User, request.user).team if not team: return f(self, request) filter = get_filter(request=request, team=team) cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), team.pk)) # return cached result if possible if not should_refresh(request): cached_result_package = get_safe_cache(cache_key) if cached_result_package and cached_result_package.get("result"): cached_result_package["is_cached"] = True return cached_result_package # call function being wrapped fresh_result_package = cast(T, f(self, request)) # cache new data if isinstance(fresh_result_package, dict): result = fresh_result_package.get("result") if not isinstance(result, dict) or not result.get("loading"): fresh_result_package["last_refresh"] = now() fresh_result_package["is_cached"] = False cache.set( cache_key, fresh_result_package, TEMP_CACHE_RESULTS_TTL, ) if filter: dashboard_items = Insight.objects.filter( team_id=team.pk, filters_hash=cache_key) dashboard_items.update(last_refresh=now()) return fresh_result_package
def dashboard_item_saved(sender, instance: DashboardItem, **kwargs): if instance.filters and instance.filters != {}: filter = get_filter(data=instance.filters, team=instance.team) instance.filters = filter.to_dict() instance.filters_hash = generate_cache_key("{}_{}".format( filter.toJSON(), instance.team_id))
def dashboard_item_saved(sender, instance: Insight, dashboard=None, **kwargs): if instance.filters and instance.filters != {}: filter = get_filter( data=instance.dashboard_filters(dashboard=dashboard), team=instance.team) instance.filters_hash = generate_cache_key("{}_{}".format( filter.toJSON(), instance.team_id))
def dashboard_item_update_task_params( item: DashboardItem, dashboard: Optional[Dashboard] = None ) -> Tuple[str, CacheType, Dict]: filter = get_filter(data=item.dashboard_filters(dashboard), team=item.team) cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), item.team_id)) cache_type = get_cache_type(filter) payload = {"filter": filter.toJSON(), "team_id": item.team_id} return cache_key, cache_type, payload
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> None: result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) if result: cache.set(key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, CACHED_RESULTS_TTL)
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Dict[str, Any]]: result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) cache.set(key, { "result": result, "type": cache_type, "last_refresh": timezone.now() }, CACHED_RESULTS_TTL) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def wrapper(*args, **kwargs) -> Dict[str, Union[List, datetime, bool, str]]: # prepare caching params request: HttpRequest = args[1] team = cast(User, request.user).team filter = None if not team: return f(*args, **kwargs) filter = get_filter(request=request, team=team) cache_key = generate_cache_key("{}_{}".format( filter.toJSON(), team.pk)) # return cached result if possible if not request.GET.get("refresh", False): cached_result = get_safe_cache(cache_key) if cached_result and cached_result.get("result"): return {**cached_result, "is_cached": True} # call function being wrapped result = f(*args, **kwargs) # cache new data if result is not None and not ( isinstance(result.get("result"), dict) and result["result"].get("loading")): cache.set( cache_key, { "result": result["result"], "last_refresh": now() }, TEMP_CACHE_RESULTS_TTL, ) if filter: dashboard_items = DashboardItem.objects.filter( team_id=team.pk, filters_hash=cache_key) dashboard_items.update(last_refresh=now()) return result
def wrapper(*args, **kwargs): # prepare caching params request: HttpRequest = args[1] team = cast(User, request.user).team filter = None if not team: return f(*args, **kwargs) filter = get_filter(request=request, team=team) cache_key = generate_cache_key("{}_{}".format( filter.toJSON(), team.pk)) payload = {"filter": filter.toJSON(), "team_id": team.pk} # return cached result if possible if not request.GET.get("refresh", False): cached_result = cache.get(cache_key) if cached_result and cached_result.get("result"): return cached_result["result"] # call function being wrapped result = f(*args, **kwargs) # cache new data if result is not None and (not isinstance(result, dict) or not result.get("loading")): cache.set( cache_key, { "result": result, "details": payload, }, TEMP_CACHE_RESULTS_TTL, ) if filter: dashboard_items = DashboardItem.objects.filter( team_id=team.pk, filters_hash=cache_key) dashboard_items.update(last_refresh=now()) return result
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Dict[str, Any]]: timer = statsd.timer("update_cache_item_timer").start() result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) # Doing the filtering like this means we'll update _all_ Insights with the same filters hash dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) try: if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) cache.set(key, { "result": result, "type": cache_type, "last_refresh": timezone.now() }, settings.CACHED_RESULTS_TTL) except Exception as e: timer.stop() statsd.incr("update_cache_item_error") dashboard_items.filter(refresh_attempt=None).update(refresh_attempt=0) dashboard_items.update(refreshing=False, refresh_attempt=F("refresh_attempt") + 1) raise e timer.stop() statsd.incr("update_cache_item_success") dashboard_items.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) return result