def oauth_api_request(method, url, **kwargs): """ when network error, fallback to use rss proxy """ options = _proxy_helper.get_proxy_options() client = RSSProxyClient(**options, proxy_strategy=_proxy_strategy) return client.request(method, url, **kwargs)
def _setup(): _proxy_options = _proxy_helper.get_proxy_options() service = DNSService.create( **_proxy_options, allow_private_address=CONFIG.allow_private_address, ) return service
def test_find_real(start_url: str): finder, messages = _create_finder( start_url, **_proxy_helper.get_proxy_options(), ) with finder: found = finder.find() if found: response, result = found print(f"Got: response={response} result={result}") assert found, messages
async def do_fetch_story( ctx: ActorContext, feed_id: T.int, offset: T.int.min(0), url: T.url, use_proxy: T.bool.default(False), num_sub_sentences: T.int.optional, ) -> SCHEMA_FETCH_STORY_RESULT: LOG.info(f'fetch story#{feed_id},{offset} url={unquote(url)} begin') options = _proxy_helper.get_proxy_options() if DNS_SERVICE.is_resolved_url(url): use_proxy = False # make timeout less than actor default 30s to avoid ask timeout options.update(request_timeout=25) async with AsyncFeedReader(**options) as reader: use_proxy = use_proxy and reader.has_proxy url, content, response = await _fetch_story(reader, feed_id, offset, url, use_proxy=use_proxy) DEFAULT_RESULT = dict(feed_id=feed_id, offset=offset, url=url, response_status=response.status, use_proxy=response.use_proxy) if not content: return DEFAULT_RESULT if len(content) >= _MAX_STORY_HTML_LENGTH: content = story_html_clean(content) if len(content) >= _MAX_STORY_HTML_LENGTH: msg = 'too large story#%s,%s size=%s url=%r' LOG.warning(msg, feed_id, offset, len(content), url) content = story_html_to_text(content)[:_MAX_STORY_HTML_LENGTH] msg_func = ctx.ask if ctx.message.is_ask else ctx.hope result = await msg_func( 'worker_rss.process_story_webpage', dict( feed_id=feed_id, offset=offset, url=url, text=content, num_sub_sentences=num_sub_sentences, )) if not ctx.message.is_ask: return DEFAULT_RESULT result.update(DEFAULT_RESULT) return result
def update_feed_use_proxy(): if not CONFIG.rss_proxy_enable: click.echo('rss proxy not enable!') return blacklist = [ '%博客园%', '%微信%', '%新浪%', '%的评论%', '%Comments on%', ] sql = """ select * from rssant_api_feed where (NOT title LIKE ANY(%s)) AND ( dt_created >= '2020-04-01' or (total_storys <= 5 and dt_updated <= '2019-12-01') ) """ feeds = list(Feed.objects.raw(sql, [blacklist])) click.echo(f'{len(feeds)} feeds need check') reader = FeedReader(**_proxy_helper.get_proxy_options()) proxy_feeds = [] with reader: for i, feed in enumerate(feeds): click.echo(f'#{i} {feed}') status = reader.read(feed.url).status click.echo(f' #{i} status={FeedResponseStatus.name_of(status)}') if FeedResponseStatus.is_need_proxy(status): proxy_status = reader.read(feed.url, use_proxy=True).status click.echo( f' #{i} proxy_status={FeedResponseStatus.name_of(proxy_status)}' ) if proxy_status == 200: proxy_feeds.append(feed) click.echo(f'{len(proxy_feeds)} feeds need use proxy') if proxy_feeds: with transaction.atomic(): for feed in tqdm.tqdm(proxy_feeds, ncols=80, ascii=True): feed.refresh_from_db() feed.use_proxy = True feed.save()
def do_find_feed( ctx: ActorContext, feed_creation_id: T.int, url: T.url, ): # immediately send message to update status ctx.ask( 'harbor_rss.update_feed_creation_status', dict( feed_creation_id=feed_creation_id, status=FeedStatus.UPDATING, )) messages = [] def message_handler(msg): LOG.info(msg) messages.append(msg) options = _proxy_helper.get_proxy_options() options.update(message_handler=message_handler) options.update(request_timeout=CONFIG.feed_reader_request_timeout) options.update(dns_service=DNS_SERVICE) with FeedFinder(url, **options) as finder: found = finder.find() try: feed = _parse_found(found) if found else None except (Invalid, FeedParserError) as ex: LOG.error('invalid feed url=%r: %s', unquote(url), ex, exc_info=ex) message_handler(f'invalid feed: {ex}') feed = None ctx.tell( 'harbor_rss.save_feed_creation_result', dict( feed_creation_id=feed_creation_id, messages=messages, feed=feed, ))
async def do_fetch_story( ctx: ActorContext, feed_id: T.int, offset: T.int, url: T.url, use_proxy: T.bool.default(False), num_sub_sentences: T.int.optional, ): LOG.info(f'fetch story#{feed_id},{offset} url={unquote(url)} begin') options = _proxy_helper.get_proxy_options() if DNS_SERVICE.is_resolved_url(url): use_proxy = False async with AsyncFeedReader(**options) as reader: use_proxy = use_proxy and reader.has_proxy url_content = await _fetch_story(reader, feed_id, offset, url, use_proxy=use_proxy) if not url_content: return url, content = url_content if len(content) >= _MAX_STORY_HTML_LENGTH: content = story_html_clean(content) if len(content) >= _MAX_STORY_HTML_LENGTH: msg = 'too large story#%s,%s size=%s url=%r' LOG.warning(msg, feed_id, offset, len(content), url) content = story_html_to_text(content)[:_MAX_STORY_HTML_LENGTH] await ctx.hope( 'worker_rss.process_story_webpage', dict( feed_id=feed_id, offset=offset, url=url, text=content, num_sub_sentences=num_sub_sentences, ))
def do_sync_feed( ctx: ActorContext, feed_id: T.int, url: T.url, use_proxy: T.bool.default(False), checksum_data: T.bytes.maxlen(4096).optional, content_hash_base64: T.str.optional, etag: T.str.optional, last_modified: T.str.optional, is_refresh: T.bool.default(False), ): params = {} if not is_refresh: params = dict(etag=etag, last_modified=last_modified) options = _proxy_helper.get_proxy_options() if DNS_SERVICE.is_resolved_url(url): use_proxy = False switch_prob = 0.25 # the prob of switch from use proxy to not use proxy with FeedReader(**options) as reader: use_proxy = reader.has_proxy and use_proxy if use_proxy and random.random() < switch_prob: use_proxy = False response = reader.read(url, **params, use_proxy=use_proxy) LOG.info( f'read feed#{feed_id} url={unquote(url)} status={response.status}') need_proxy = FeedResponseStatus.is_need_proxy(response.status) if (not use_proxy) and reader.has_proxy and need_proxy: LOG.info(f'try use proxy read feed#{feed_id} url={unquote(url)}') proxy_response = reader.read(url, **params, use_proxy=True) LOG.info( f'proxy read feed#{feed_id} url={unquote(url)} status={proxy_response.status}' ) if proxy_response.ok: response = proxy_response if (not response.ok) or (not response.content): status = FeedStatus.READY if response.status == 304 else FeedStatus.ERROR _update_feed_info(ctx, feed_id, status=status, response=response) return new_hash = compute_hash_base64(response.content) if (not is_refresh) and (new_hash == content_hash_base64): LOG.info( f'feed#{feed_id} url={unquote(url)} not modified by compare content hash!' ) _update_feed_info(ctx, feed_id, response=response) return LOG.info(f'parse feed#{feed_id} url={unquote(url)}') try: raw_result = RawFeedParser().parse(response) except FeedParserError as ex: LOG.warning('failed parse feed#%s url=%r: %s', feed_id, unquote(url), ex) _update_feed_info(ctx, feed_id, status=FeedStatus.ERROR, response=response, warnings=str(ex)) return if raw_result.warnings: warnings = '; '.join(raw_result.warnings) LOG.warning('warning parse feed#%s url=%r: %s', feed_id, unquote(url), warnings) try: feed = _parse_found((response, raw_result), checksum_data=checksum_data, is_refresh=is_refresh) except (Invalid, FeedParserError) as ex: LOG.error('invalid feed#%s url=%r: %s', feed_id, unquote(url), ex, exc_info=ex) _update_feed_info(ctx, feed_id, status=FeedStatus.ERROR, response=response, warnings=str(ex)) return ctx.tell('harbor_rss.update_feed', dict(feed_id=feed_id, feed=feed, is_refresh=is_refresh))