def _retry_feed_creations(ctx: ActorContext, feed_creation_id_urls):
feed_creation_ids = [id for (id, url) in feed_creation_id_urls]
FeedCreation.bulk_set_pending(feed_creation_ids)
expire_at = time.time() + 60 * 60
for feed_creation_id, url in feed_creation_id_urls:
ctx.hope('worker_rss.find_feed', dict(
feed_creation_id=feed_creation_id,
url=url,
), expire_at=expire_at)
def do_save_feed_creation_result(
ctx: ActorContext,
feed_creation_id: T.int,
messages: T.list(T.str),
feed: FeedSchema.optional,
):
with transaction.atomic():
feed_dict = feed
try:
feed_creation = FeedCreation.get_by_pk(feed_creation_id)
except FeedCreation.DoesNotExist:
LOG.warning(f'feed creation {feed_creation_id} not exists')
return
if feed_creation.status == FeedStatus.READY:
LOG.info(f'feed creation {feed_creation_id} is ready')
return
feed_creation.message = '\n\n'.join(messages)
feed_creation.dt_updated = timezone.now()
if not feed_dict:
feed_creation.status = FeedStatus.ERROR
feed_creation.save()
FeedUrlMap(source=feed_creation.url,
target=FeedUrlMap.NOT_FOUND).save()
return
url = feed_dict['url']
feed = Feed.get_first_by_url(url)
if not feed:
now = timezone.now()
feed = Feed(url=url,
status=FeedStatus.READY,
reverse_url=reverse_url(url),
dt_updated=now,
dt_checked=now,
dt_synced=now)
feed.save()
feed_creation.status = FeedStatus.READY
feed_creation.feed_id = feed.id
feed_creation.save()
user_feed = UserFeed.objects.filter(user_id=feed_creation.user_id,
feed_id=feed.id).first()
if user_feed:
LOG.info('UserFeed#{} user_id={} feed_id={} already exists'.format(
user_feed.id, feed_creation.user_id, feed.id))
else:
user_feed = UserFeed(
user_id=feed_creation.user_id,
feed_id=feed.id,
is_from_bookmark=feed_creation.is_from_bookmark,
)
user_feed.save()
FeedUrlMap(source=feed_creation.url, target=feed.url).save()
if feed.url != feed_creation.url:
FeedUrlMap(source=feed.url, target=feed.url).save()
ctx.hope('harbor_rss.update_feed',
dict(
feed_id=feed.id,
feed=validate_feed_output(feed_dict),
))
def feed_get_creation(request, pk: T.int,
detail: FeedDetailSchema) -> FeedCreationSchema:
try:
feed_creation = FeedCreation.get_by_pk(pk,
user_id=request.user.id,
detail=detail)
except FeedCreation.DoesNotExist:
return Response({'message': 'feed creation does not exist'},
status=400)
return feed_creation.to_dict(detail=detail)
def do_clean_feed_creation(ctx: ActorContext):
# 删除所有入库时间超过24小时的订阅创建信息
num_deleted = FeedCreation.delete_by_status(survival_seconds=24 * 60 * 60)
LOG.info('delete {} old feed creations'.format(num_deleted))
# 重试 status=UPDATING 超过4小时的订阅
feed_creation_id_urls = FeedCreation.query_id_urls_by_status(
FeedStatus.UPDATING, survival_seconds=4 * 60 * 60)
num_retry_updating = len(feed_creation_id_urls)
LOG.info('retry {} status=UPDATING feed creations'.format(num_retry_updating))
_retry_feed_creations(ctx, feed_creation_id_urls)
# 重试 status=PENDING 超过4小时的订阅
feed_creation_id_urls = FeedCreation.query_id_urls_by_status(
FeedStatus.PENDING, survival_seconds=4 * 60 * 60)
num_retry_pending = len(feed_creation_id_urls)
LOG.info('retry {} status=PENDING feed creations'.format(num_retry_pending))
_retry_feed_creations(ctx, feed_creation_id_urls)
return dict(
num_deleted=num_deleted,
num_retry_updating=num_retry_updating,
num_retry_pending=num_retry_pending,
)
def feed_query_creation(
request,
limit: T.int.min(10).max(2000).default(500),
detail: FeedDetailSchema
) -> T.dict(
total=T.int.min(0),
size=T.int.min(0),
feed_creations=T.list(FeedCreationSchema).maxlen(2000),
):
feed_creations = FeedCreation.query_by_user(request.user.id, limit=limit, detail=detail)
feed_creations = [x.to_dict() for x in feed_creations]
return dict(
total=len(feed_creations),
size=len(feed_creations),
feed_creations=feed_creations,
)
