def query_recent_by_user(user_id,
feed_unionids=None,
days=14,
limit=300,
detail=False):
if feed_unionids:
feed_ids = [x.feed_id for x in feed_unionids]
q = UserFeed.objects.only('id', 'feed_id')\
.filter(user_id=user_id, feed_id__in=feed_ids)
user_feeds = list(q.all())
else:
q = UserFeed.objects.only('id', 'feed_id')\
.filter(user_id=user_id)
user_feeds = list(q.all())
feed_ids = [x.feed_id for x in q.all()]
dt_begin = timezone.now() - timezone.timedelta(days=days)
q = Story.objects.filter(feed_id__in=feed_ids)\
.filter(dt_published__gte=dt_begin)
detail = Detail.from_schema(detail, StoryDetailSchema)
q = q.defer(*detail.exclude_fields)
q = q.order_by('-dt_published')[:limit]
storys = list(q.all())
story_ids = [x.id for x in storys]
q = UserStory.objects.filter(user_id=user_id,
feed_id__in=feed_ids,
story_id__in=story_ids)
q = q.exclude(is_favorited=False, is_watched=False)
user_storys = list(q.all())
union_storys = UnionStory._merge_storys(storys,
user_storys,
user_feeds=user_feeds,
user_id=user_id,
detail=detail)
return union_storys
def query_by_user(user_id, hints=None, detail=False):
"""获取用户所有的订阅，支持增量查询
hints: T.list(T.dict(id=T.unionid, dt_updated=T.datetime))
"""
detail = Detail.from_schema(detail, FeedDetailSchema)
exclude_fields = [f'feed__{x}' for x in detail.exclude_fields]
if not hints:
q = UserFeed.objects.select_related('feed').filter(user_id=user_id)
q = q.defer(*exclude_fields)
union_feeds = UnionFeed._merge_user_feeds(list(q.all()), detail=detail)
return len(union_feeds), union_feeds, []
hints = {x['id'].feed_id: x['dt_updated'] for x in hints}
q = UserFeed.objects.filter(user_id=user_id).select_related('feed')
q = q.only("id", 'feed_id', 'feed__dt_updated')
user_feeds = list(q.all())
total = len(user_feeds)
feed_ids = {user_feed.feed_id for user_feed in user_feeds}
deteted_ids = []
for feed_id in set(hints) - feed_ids:
deteted_ids.append(FeedUnionId(user_id, feed_id))
updates = []
for user_feed in user_feeds:
feed_id = user_feed.feed_id
dt_updated = user_feed.feed.dt_updated
if feed_id not in hints or not dt_updated:
updates.append(feed_id)
elif dt_updated > hints[feed_id]:
updates.append(feed_id)
q = UserFeed.objects.select_related('feed')\
.filter(user_id=user_id, feed_id__in=updates)
q = q.defer(*exclude_fields)
union_feeds = UnionFeed._merge_user_feeds(list(q.all()), detail=detail)
return total, union_feeds, deteted_ids
def _query_storys_by_feed(cls, feed_id, offset, size, detail):
q = Story.objects.filter(feed_id=feed_id, offset__gte=offset)
detail = Detail.from_schema(detail, StoryDetailSchema)
q = q.defer(*detail.exclude_fields)
q = q.order_by('offset')[:size]
storys = list(q.all())
return storys
def batch_get_by_feed_offset(cls, user_id, story_keys, detail=False):
"""
story_keys: List[Tuple[feed_id, offset]]
"""
story_keys = cls._validate_story_keys(user_id, story_keys)
if not story_keys:
return []
detail = Detail.from_schema(detail, StoryDetailSchema)
select_fields = set(cls._story_field_names()) - set(
detail.exclude_fields)
select_fields_quoted = ','.join(
['"{}"'.format(x) for x in select_fields])
# Note: below query can not use index, it's very slow
# WHERE ("feed_id","offset")=Any(%s)
# WHERE ("feed_id","offset")=Any(ARRAY[(XX, YY), ...])
where_items = []
for feed_id, offset in story_keys:
# ensure integer, avoid sql inject attack
feed_id, offset = int(feed_id), int(offset)
where_items.append(f'("feed_id"={feed_id} AND "offset"={offset})')
where_clause = ' OR '.join(where_items)
sql = f"""
SELECT {select_fields_quoted}
FROM rssant_api_story
WHERE {where_clause}
"""
storys = list(Story.objects.raw(sql))
union_storys = cls._query_union_storys(user_id=user_id,
storys=storys,
detail=detail)
return union_storys
def query_recent_by_user(cls,
user_id,
feed_unionids=None,
days=14,
limit=300,
detail=False):
"""
Deprecated since 1.4.2, use batch_get_by_feed_offset instead
"""
if (not feed_unionids) and feed_unionids is not None:
return [] # when feed_unionids is empty list, return empty list
if feed_unionids:
feed_ids = [x.feed_id for x in feed_unionids]
feed_ids = cls._query_user_feed_ids(user_id, feed_ids)
else:
feed_ids = cls._query_user_feed_ids(user_id)
dt_begin = timezone.now() - timezone.timedelta(days=days)
q = Story.objects.filter(feed_id__in=feed_ids)\
.filter(dt_published__gte=dt_begin)
detail = Detail.from_schema(detail, StoryDetailSchema)
q = q.defer(*detail.exclude_fields)
q = q.order_by('-dt_published')[:limit]
storys = list(q.all())
union_storys = cls._query_union_storys(user_id=user_id,
storys=storys,
detail=detail)
return union_storys
def query_by_feed(feed_unionid, offset=None, size=10, detail=False):
user_id, feed_id = feed_unionid
q = UserFeed.objects.select_related('feed')\
.filter(user_id=user_id, feed_id=feed_id)\
.only('id', 'story_offset', 'feed_id', 'feed__id', 'feed__total_storys')
try:
user_feed = q.get()
except UserFeed.DoesNotExist as ex:
raise FeedNotFoundError() from ex
total = user_feed.feed.total_storys
if offset is None:
offset = user_feed.story_offset
q = Story.objects.filter(feed_id=feed_id, offset__gte=offset)
detail = Detail.from_schema(detail, StoryDetailSchema)
q = q.defer(*detail.exclude_fields)
q = q.order_by('offset')[:size]
storys = list(q.all())
story_ids = [x.id for x in storys]
q = UserStory.objects.filter(user_id=user_id,
feed_id=feed_id,
story_id__in=story_ids)
q = q.exclude(is_favorited=False, is_watched=False)
user_storys = list(q.all())
ret = UnionStory._merge_storys(storys,
user_storys,
user_feeds=[user_feed],
user_id=user_id,
detail=detail)
return total, offset, ret
def _query_by_tag(user_id, is_favorited=None, is_watched=None, detail=False):
q = UserStory.objects.select_related('story').filter(user_id=user_id)
detail = Detail.from_schema(detail, StoryDetailSchema)
exclude_fields = [f'story__{x}' for x in detail.exclude_fields]
q = q.defer(*exclude_fields)
if is_favorited is not None:
q = q.filter(is_favorited=is_favorited)
if is_watched is not None:
q = q.filter(is_watched=is_watched)
user_storys = list(q.all())
storys = [x.story for x in user_storys]
union_storys = UnionStory._merge_storys(storys, user_storys, user_id=user_id, detail=detail)
return union_storys
def to_dict(self):
ret = dict(
id=self.id,
user=dict(id=self.user_id),
feed=dict(id=self.feed_id),
offset=self.offset,
title=self.title,
link=self.link,
has_mathjax=self.has_mathjax,
is_watched=self.is_watched,
is_favorited=self.is_favorited,
)
detail = Detail.from_schema(self._detail, StoryDetailSchema)
for k in detail.include_fields:
ret[k] = getattr(self, k)
return ret
def to_dict(self):
ret = dict(
id=self.id,
user=dict(id=self.user_id),
is_ready=self.is_ready,
status=self.status,
url=self.url,
total_storys=self.total_storys,
story_offset=self.story_offset,
num_unread_storys=self.num_unread_storys,
dt_updated=self.dt_updated,
dt_created=self.dt_created,
)
detail = Detail.from_schema(self._detail, FeedDetailSchema)
for k in detail.include_fields:
ret[k] = getattr(self, k)
return ret
offset_early_story
dt_early_story_published
dt_latest_story_published
""").extra_fields("""
description
encoding
etag
last_modified
content_length
content_hash_base64
dt_checked
dt_synced
""").default(False)
FEED_DETAIL_FIELDS = [
f'feed__{x}' for x in Detail.from_schema(False, FeedDetailSchema).exclude_fields
]
class Feed(Model, ContentHashMixin):
"""订阅的最新数据"""
class Meta:
indexes = [
models.Index(fields=["url"]),
]
class Admin:
display_fields = ['status', 'title', 'url']
url = models.TextField(unique=True, help_text="供稿地址")
status = models.CharField(
def _is_include_content(self, detail):
detail = Detail.from_schema(detail, StoryDetailSchema)
return 'content' in detail.include_fields
def get_by_offset(feed_id, offset, detail=False) -> 'Story':
q = Story.objects.filter(feed_id=feed_id, offset=offset)
detail = Detail.from_schema(detail, StoryDetailSchema)
q = q.defer(*detail.exclude_fields)
return q.seal().get()
dt_published
dt_updated
dt_created
dt_watched
dt_favorited
""").extra_fields("""
author
audio_url
iframe_url
dt_synced
summary
content
content_hash_base64
""").default(False)
STORY_DETAIL_FEILDS = Detail.from_schema(False,
StoryDetailSchema).exclude_fields
USER_STORY_DETAIL_FEILDS = [f'story__{x}' for x in STORY_DETAIL_FEILDS]
class Story(Model, ContentHashMixin):
"""故事"""
class Meta:
unique_together = (
('feed', 'offset'),
('feed', 'unique_id'),
)
indexes = [
models.Index(fields=["feed", "offset"]),
models.Index(fields=["feed", "unique_id"]),
]
def _get_exclude_fields(cls, detail):
detail = Detail.from_schema(detail, StoryDetailSchema)
exclude_fields = set(detail.exclude_fields)
exclude_fields.discard('content')
return exclude_fields
def get_first_by_url(url, detail=True) -> 'Feed':
detail = Detail.from_schema(detail, FeedDetailSchema)
q = Feed.objects.seal().defer(*detail.exclude_fields)
return q.filter(url=url).first()
def get_by_pk(feed_id, detail=True) -> 'Feed':
detail = Detail.from_schema(detail, FeedDetailSchema)
q = Feed.objects.seal().defer(*detail.exclude_fields)
return q.get(pk=feed_id)
icon
author
version
link
description
warnings
encoding
etag
last_modified
content_length
content_hash_base64
dt_checked
dt_synced
""").default(False)
FEED_DETAIL_FIELDS = Detail.from_schema(False, FeedDetailSchema).exclude_fields
USER_FEED_DETAIL_FIELDS = [f'feed__{x}' for x in FEED_DETAIL_FIELDS]
class Feed(Model, ContentHashMixin):
"""订阅的最新数据"""
class Meta:
indexes = [
models.Index(fields=["url"]),
models.Index(fields=["reverse_url"]),
]
class Admin:
display_fields = ['status', 'title', 'url']
# TODO: deprecate url, use reverse_url instead