def do_find_feed(
ctx: ActorContext,
feed_creation_id: T.int,
url: T.url,
):
# immediately send message to update status
ctx.ask(
'harbor_rss.update_feed_creation_status',
dict(
feed_creation_id=feed_creation_id,
status=FeedStatus.UPDATING,
))
messages = []
def message_handler(msg):
LOG.info(msg)
messages.append(msg)
options = dict(message_handler=message_handler, **_get_proxy_options())
with FeedFinder(url, **options) as finder:
found = finder.find()
try:
feed = _parse_found(found) if found else None
except Invalid as ex:
message_handler(f'invalid feed: {ex}')
feed = None
ctx.tell(
'harbor_rss.save_feed_creation_result',
dict(
feed_creation_id=feed_creation_id,
messages=messages,
feed=feed,
))
def do_find_feed(
ctx: ActorContext,
feed_creation_id: T.int,
url: T.url,
):
# immediately send message to update status
ctx.ask('harbor_rss.update_feed_creation_status', dict(
feed_creation_id=feed_creation_id,
status=FeedStatus.UPDATING,
))
messages = []
def message_handler(msg):
LOG.info(msg)
messages.append(msg)
options = dict(message_handler=message_handler, **_get_proxy_options())
options.update(allow_private_address=CONFIG.allow_private_address)
with FeedFinder(url, **options) as finder:
found = finder.find()
try:
feed = _parse_found(found) if found else None
except (Invalid, FeedParserError) as ex:
LOG.error('invalid feed url=%r: %s', unquote(url), ex, exc_info=ex)
message_handler(f'invalid feed: {ex}')
feed = None
ctx.tell('harbor_rss.save_feed_creation_result', dict(
feed_creation_id=feed_creation_id,
messages=messages,
feed=feed,
))
def do_init(ctx: ActorContext):
while True:
try:
ctx.ask('registery.register',
dict(node=ctx.registery.current_node.to_spec()))
except Exception as ex:
LOG.warning(f'ask registery.register failed: {ex}')
else:
break
time.sleep(3)
def do_ping(ctx: ActorContext, message: T.str) -> T.dict(message=T.str):
LOG.info(ctx.message)
r = ctx.ask('registery.query')
LOG.info(r)
ctx.tell('worker.pong', dict(message=message))
if message == 'error':
raise ValueError(message)
return dict(message=message)
def do_sync_story_fulltext(
ctx: ActorContext,
feed_id: T.int,
offset: T.int,
) -> T.dict(
feed_id=T.int,
offset=T.int.min(0),
use_proxy=T.bool,
url=T.url,
response_status=T.int,
accept=T_ACCEPT,
):
with log_django_context_metric('harbor_rss.sync_story_fulltext:read'):
feed = Feed.get_by_pk(feed_id, detail='+use_proxy')
story = STORY_SERVICE.get_by_offset(feed_id, offset, detail=True)
assert story, f'story#{feed_id},{offset} not found'
story_content_info = StoryContentInfo(story.content)
num_sub_sentences = len(split_sentences(story_content_info.text))
ret = dict(
feed_id=feed_id,
offset=offset,
url=story.link,
use_proxy=feed.use_proxy,
accept=FulltextAcceptStrategy.REJECT.value,
)
try:
result = ctx.ask(
'worker_rss.fetch_story',
dict(
url=story.link,
use_proxy=feed.use_proxy,
feed_id=feed_id,
offset=offset,
num_sub_sentences=num_sub_sentences,
))
except _TIMEOUT_ERRORS as ex:
LOG.error(f'Ask worker_rss.fetch_story timeout: {ex}')
ret.update(response_status=FeedResponseStatus.CONNECTION_TIMEOUT)
return ret
else:
ret.update(
response_status=result['response_status'],
use_proxy=result['use_proxy'],
)
if not result['content']:
return ret
with log_django_context_metric('harbor_rss.sync_story_fulltext:write'):
accept = _update_story(
story=story,
story_content_info=story_content_info,
content=result['content'],
summary=None, # not need update summary
url=result['url'],
sentence_count=result['sentence_count'],
)
ret.update(accept=accept.value)
return ret
def do_local_ask(ctx: ActorContext) -> T.dict(message=T.str):
LOG.info(ctx.message)
r = ctx.ask('worker.async_local_ask')
LOG.info(r)
return r