def query_recent_by_user(user_id, feed_unionids=None, days=14, limit=300, detail=False): if feed_unionids: feed_ids = [x.feed_id for x in feed_unionids] q = UserFeed.objects.only('id', 'feed_id')\ .filter(user_id=user_id, feed_id__in=feed_ids) user_feeds = list(q.all()) else: q = UserFeed.objects.only('id', 'feed_id')\ .filter(user_id=user_id) user_feeds = list(q.all()) feed_ids = [x.feed_id for x in q.all()] dt_begin = timezone.now() - timezone.timedelta(days=days) q = Story.objects.filter(feed_id__in=feed_ids)\ .filter(dt_published__gte=dt_begin) detail = Detail.from_schema(detail, StoryDetailSchema) q = q.defer(*detail.exclude_fields) q = q.order_by('-dt_published')[:limit] storys = list(q.all()) story_ids = [x.id for x in storys] q = UserStory.objects.filter(user_id=user_id, feed_id__in=feed_ids, story_id__in=story_ids) q = q.exclude(is_favorited=False, is_watched=False) user_storys = list(q.all()) union_storys = UnionStory._merge_storys(storys, user_storys, user_feeds=user_feeds, user_id=user_id, detail=detail) return union_storys
def query_by_user(user_id, hints=None, detail=False): """获取用户所有的订阅,支持增量查询 hints: T.list(T.dict(id=T.unionid, dt_updated=T.datetime)) """ detail = Detail.from_schema(detail, FeedDetailSchema) exclude_fields = [f'feed__{x}' for x in detail.exclude_fields] if not hints: q = UserFeed.objects.select_related('feed').filter(user_id=user_id) q = q.defer(*exclude_fields) union_feeds = UnionFeed._merge_user_feeds(list(q.all()), detail=detail) return len(union_feeds), union_feeds, [] hints = {x['id'].feed_id: x['dt_updated'] for x in hints} q = UserFeed.objects.filter(user_id=user_id).select_related('feed') q = q.only("id", 'feed_id', 'feed__dt_updated') user_feeds = list(q.all()) total = len(user_feeds) feed_ids = {user_feed.feed_id for user_feed in user_feeds} deteted_ids = [] for feed_id in set(hints) - feed_ids: deteted_ids.append(FeedUnionId(user_id, feed_id)) updates = [] for user_feed in user_feeds: feed_id = user_feed.feed_id dt_updated = user_feed.feed.dt_updated if feed_id not in hints or not dt_updated: updates.append(feed_id) elif dt_updated > hints[feed_id]: updates.append(feed_id) q = UserFeed.objects.select_related('feed')\ .filter(user_id=user_id, feed_id__in=updates) q = q.defer(*exclude_fields) union_feeds = UnionFeed._merge_user_feeds(list(q.all()), detail=detail) return total, union_feeds, deteted_ids
def _query_storys_by_feed(cls, feed_id, offset, size, detail): q = Story.objects.filter(feed_id=feed_id, offset__gte=offset) detail = Detail.from_schema(detail, StoryDetailSchema) q = q.defer(*detail.exclude_fields) q = q.order_by('offset')[:size] storys = list(q.all()) return storys
def batch_get_by_feed_offset(cls, user_id, story_keys, detail=False): """ story_keys: List[Tuple[feed_id, offset]] """ story_keys = cls._validate_story_keys(user_id, story_keys) if not story_keys: return [] detail = Detail.from_schema(detail, StoryDetailSchema) select_fields = set(cls._story_field_names()) - set( detail.exclude_fields) select_fields_quoted = ','.join( ['"{}"'.format(x) for x in select_fields]) # Note: below query can not use index, it's very slow # WHERE ("feed_id","offset")=Any(%s) # WHERE ("feed_id","offset")=Any(ARRAY[(XX, YY), ...]) where_items = [] for feed_id, offset in story_keys: # ensure integer, avoid sql inject attack feed_id, offset = int(feed_id), int(offset) where_items.append(f'("feed_id"={feed_id} AND "offset"={offset})') where_clause = ' OR '.join(where_items) sql = f""" SELECT {select_fields_quoted} FROM rssant_api_story WHERE {where_clause} """ storys = list(Story.objects.raw(sql)) union_storys = cls._query_union_storys(user_id=user_id, storys=storys, detail=detail) return union_storys
def query_recent_by_user(cls, user_id, feed_unionids=None, days=14, limit=300, detail=False): """ Deprecated since 1.4.2, use batch_get_by_feed_offset instead """ if (not feed_unionids) and feed_unionids is not None: return [] # when feed_unionids is empty list, return empty list if feed_unionids: feed_ids = [x.feed_id for x in feed_unionids] feed_ids = cls._query_user_feed_ids(user_id, feed_ids) else: feed_ids = cls._query_user_feed_ids(user_id) dt_begin = timezone.now() - timezone.timedelta(days=days) q = Story.objects.filter(feed_id__in=feed_ids)\ .filter(dt_published__gte=dt_begin) detail = Detail.from_schema(detail, StoryDetailSchema) q = q.defer(*detail.exclude_fields) q = q.order_by('-dt_published')[:limit] storys = list(q.all()) union_storys = cls._query_union_storys(user_id=user_id, storys=storys, detail=detail) return union_storys
def query_by_feed(feed_unionid, offset=None, size=10, detail=False): user_id, feed_id = feed_unionid q = UserFeed.objects.select_related('feed')\ .filter(user_id=user_id, feed_id=feed_id)\ .only('id', 'story_offset', 'feed_id', 'feed__id', 'feed__total_storys') try: user_feed = q.get() except UserFeed.DoesNotExist as ex: raise FeedNotFoundError() from ex total = user_feed.feed.total_storys if offset is None: offset = user_feed.story_offset q = Story.objects.filter(feed_id=feed_id, offset__gte=offset) detail = Detail.from_schema(detail, StoryDetailSchema) q = q.defer(*detail.exclude_fields) q = q.order_by('offset')[:size] storys = list(q.all()) story_ids = [x.id for x in storys] q = UserStory.objects.filter(user_id=user_id, feed_id=feed_id, story_id__in=story_ids) q = q.exclude(is_favorited=False, is_watched=False) user_storys = list(q.all()) ret = UnionStory._merge_storys(storys, user_storys, user_feeds=[user_feed], user_id=user_id, detail=detail) return total, offset, ret
def _query_by_tag(user_id, is_favorited=None, is_watched=None, detail=False): q = UserStory.objects.select_related('story').filter(user_id=user_id) detail = Detail.from_schema(detail, StoryDetailSchema) exclude_fields = [f'story__{x}' for x in detail.exclude_fields] q = q.defer(*exclude_fields) if is_favorited is not None: q = q.filter(is_favorited=is_favorited) if is_watched is not None: q = q.filter(is_watched=is_watched) user_storys = list(q.all()) storys = [x.story for x in user_storys] union_storys = UnionStory._merge_storys(storys, user_storys, user_id=user_id, detail=detail) return union_storys
def to_dict(self): ret = dict( id=self.id, user=dict(id=self.user_id), feed=dict(id=self.feed_id), offset=self.offset, title=self.title, link=self.link, has_mathjax=self.has_mathjax, is_watched=self.is_watched, is_favorited=self.is_favorited, ) detail = Detail.from_schema(self._detail, StoryDetailSchema) for k in detail.include_fields: ret[k] = getattr(self, k) return ret
def to_dict(self): ret = dict( id=self.id, user=dict(id=self.user_id), is_ready=self.is_ready, status=self.status, url=self.url, total_storys=self.total_storys, story_offset=self.story_offset, num_unread_storys=self.num_unread_storys, dt_updated=self.dt_updated, dt_created=self.dt_created, ) detail = Detail.from_schema(self._detail, FeedDetailSchema) for k in detail.include_fields: ret[k] = getattr(self, k) return ret
offset_early_story dt_early_story_published dt_latest_story_published """).extra_fields(""" description encoding etag last_modified content_length content_hash_base64 dt_checked dt_synced """).default(False) FEED_DETAIL_FIELDS = [ f'feed__{x}' for x in Detail.from_schema(False, FeedDetailSchema).exclude_fields ] class Feed(Model, ContentHashMixin): """订阅的最新数据""" class Meta: indexes = [ models.Index(fields=["url"]), ] class Admin: display_fields = ['status', 'title', 'url'] url = models.TextField(unique=True, help_text="供稿地址") status = models.CharField(
def _is_include_content(self, detail): detail = Detail.from_schema(detail, StoryDetailSchema) return 'content' in detail.include_fields
def get_by_offset(feed_id, offset, detail=False) -> 'Story': q = Story.objects.filter(feed_id=feed_id, offset=offset) detail = Detail.from_schema(detail, StoryDetailSchema) q = q.defer(*detail.exclude_fields) return q.seal().get()
dt_published dt_updated dt_created dt_watched dt_favorited """).extra_fields(""" author audio_url iframe_url dt_synced summary content content_hash_base64 """).default(False) STORY_DETAIL_FEILDS = Detail.from_schema(False, StoryDetailSchema).exclude_fields USER_STORY_DETAIL_FEILDS = [f'story__{x}' for x in STORY_DETAIL_FEILDS] class Story(Model, ContentHashMixin): """故事""" class Meta: unique_together = ( ('feed', 'offset'), ('feed', 'unique_id'), ) indexes = [ models.Index(fields=["feed", "offset"]), models.Index(fields=["feed", "unique_id"]), ]
def _get_exclude_fields(cls, detail): detail = Detail.from_schema(detail, StoryDetailSchema) exclude_fields = set(detail.exclude_fields) exclude_fields.discard('content') return exclude_fields
def get_first_by_url(url, detail=True) -> 'Feed': detail = Detail.from_schema(detail, FeedDetailSchema) q = Feed.objects.seal().defer(*detail.exclude_fields) return q.filter(url=url).first()
def get_by_pk(feed_id, detail=True) -> 'Feed': detail = Detail.from_schema(detail, FeedDetailSchema) q = Feed.objects.seal().defer(*detail.exclude_fields) return q.get(pk=feed_id)
icon author version link description warnings encoding etag last_modified content_length content_hash_base64 dt_checked dt_synced """).default(False) FEED_DETAIL_FIELDS = Detail.from_schema(False, FeedDetailSchema).exclude_fields USER_FEED_DETAIL_FIELDS = [f'feed__{x}' for x in FEED_DETAIL_FIELDS] class Feed(Model, ContentHashMixin): """订阅的最新数据""" class Meta: indexes = [ models.Index(fields=["url"]), models.Index(fields=["reverse_url"]), ] class Admin: display_fields = ['status', 'title', 'url'] # TODO: deprecate url, use reverse_url instead