def _retry_feed_creations(ctx: ActorContext, feed_creation_id_urls): feed_creation_ids = [id for (id, url) in feed_creation_id_urls] FeedCreation.bulk_set_pending(feed_creation_ids) expire_at = time.time() + 60 * 60 for feed_creation_id, url in feed_creation_id_urls: ctx.hope('worker_rss.find_feed', dict( feed_creation_id=feed_creation_id, url=url, ), expire_at=expire_at)
def do_save_feed_creation_result( ctx: ActorContext, feed_creation_id: T.int, messages: T.list(T.str), feed: FeedSchema.optional, ): with transaction.atomic(): feed_dict = feed try: feed_creation = FeedCreation.get_by_pk(feed_creation_id) except FeedCreation.DoesNotExist: LOG.warning(f'feed creation {feed_creation_id} not exists') return if feed_creation.status == FeedStatus.READY: LOG.info(f'feed creation {feed_creation_id} is ready') return feed_creation.message = '\n\n'.join(messages) feed_creation.dt_updated = timezone.now() if not feed_dict: feed_creation.status = FeedStatus.ERROR feed_creation.save() FeedUrlMap(source=feed_creation.url, target=FeedUrlMap.NOT_FOUND).save() return url = feed_dict['url'] feed = Feed.get_first_by_url(url) if not feed: now = timezone.now() feed = Feed(url=url, status=FeedStatus.READY, reverse_url=reverse_url(url), dt_updated=now, dt_checked=now, dt_synced=now) feed.save() feed_creation.status = FeedStatus.READY feed_creation.feed_id = feed.id feed_creation.save() user_feed = UserFeed.objects.filter(user_id=feed_creation.user_id, feed_id=feed.id).first() if user_feed: LOG.info('UserFeed#{} user_id={} feed_id={} already exists'.format( user_feed.id, feed_creation.user_id, feed.id)) else: user_feed = UserFeed( user_id=feed_creation.user_id, feed_id=feed.id, is_from_bookmark=feed_creation.is_from_bookmark, ) user_feed.save() FeedUrlMap(source=feed_creation.url, target=feed.url).save() if feed.url != feed_creation.url: FeedUrlMap(source=feed.url, target=feed.url).save() ctx.hope('harbor_rss.update_feed', dict( feed_id=feed.id, feed=validate_feed_output(feed_dict), ))
def feed_get_creation(request, pk: T.int, detail: FeedDetailSchema) -> FeedCreationSchema: try: feed_creation = FeedCreation.get_by_pk(pk, user_id=request.user.id, detail=detail) except FeedCreation.DoesNotExist: return Response({'message': 'feed creation does not exist'}, status=400) return feed_creation.to_dict(detail=detail)
def do_clean_feed_creation(ctx: ActorContext): # 删除所有入库时间超过24小时的订阅创建信息 num_deleted = FeedCreation.delete_by_status(survival_seconds=24 * 60 * 60) LOG.info('delete {} old feed creations'.format(num_deleted)) # 重试 status=UPDATING 超过4小时的订阅 feed_creation_id_urls = FeedCreation.query_id_urls_by_status( FeedStatus.UPDATING, survival_seconds=4 * 60 * 60) num_retry_updating = len(feed_creation_id_urls) LOG.info('retry {} status=UPDATING feed creations'.format(num_retry_updating)) _retry_feed_creations(ctx, feed_creation_id_urls) # 重试 status=PENDING 超过4小时的订阅 feed_creation_id_urls = FeedCreation.query_id_urls_by_status( FeedStatus.PENDING, survival_seconds=4 * 60 * 60) num_retry_pending = len(feed_creation_id_urls) LOG.info('retry {} status=PENDING feed creations'.format(num_retry_pending)) _retry_feed_creations(ctx, feed_creation_id_urls) return dict( num_deleted=num_deleted, num_retry_updating=num_retry_updating, num_retry_pending=num_retry_pending, )
def feed_query_creation( request, limit: T.int.min(10).max(2000).default(500), detail: FeedDetailSchema ) -> T.dict( total=T.int.min(0), size=T.int.min(0), feed_creations=T.list(FeedCreationSchema).maxlen(2000), ): feed_creations = FeedCreation.query_by_user(request.user.id, limit=limit, detail=detail) feed_creations = [x.to_dict() for x in feed_creations] return dict( total=len(feed_creations), size=len(feed_creations), feed_creations=feed_creations, )