def test_parse_story_no_id_no_summary_no_url():
# total 3 storys
# skip the no id story
# story#0 no content no summary, no url
# story#1 has content no summary, no url but id is valid url
filename = 'well/v2ex-no-id-no-summary-no-url.xml'
response = _read_response(_data_dir, filename)
raw_result = RawFeedParser().parse(response)
assert raw_result.storys
# assert skip the no id story
assert len(raw_result.storys) == 2
# assert no summary
assert not raw_result.storys[0]['summary']
assert not raw_result.storys[1]['summary']
# assert content
assert not raw_result.storys[0]['content']
assert raw_result.storys[1]['content']
# assert pick id as url, discard the invalid one
assert not raw_result.storys[0]['url']
assert raw_result.storys[1]['url']
result = FeedParser().parse(raw_result)
assert result.storys
assert len(raw_result.storys) == len(result.storys)
# assert content
assert not result.storys[0]['content']
assert result.storys[1]['content']
# assert extract summary from content
assert not result.storys[0]['summary']
assert result.storys[1]['summary']
# assert pick id as url, discard the invalid one
assert not result.storys[0]['url']
assert result.storys[1]['url']
def test_raw_parse_bad_encoding():
content = os.urandom(16 * 1024)
response = _create_builder(content=content).build()
parser = RawFeedParser()
with pytest.raises(FeedParserError) as ex:
parser.parse(response)
assert ex
def test_parse_too_many_storys():
items = []
num_storys = 2000
base = datetime.datetime.now()
for i in range(num_storys):
if i < num_storys // 2:
date_published = None
else:
date_published = (base + datetime.timedelta(seconds=i)).isoformat()
items.append({
"id": f"{i}",
"content_html": f"content_{i}",
"summary": f"summary_{i}",
"url": f"https://example.org/post/{i}",
"date_published": date_published,
})
feed = {
"version": "https://jsonfeed.org/version/1",
"title": "Too many storys",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": items
}
data = json.dumps(feed).encode('utf-8')
response = _create_builder(data).build()
raw_result = RawFeedParser().parse(response)
assert len(raw_result.storys) == num_storys
result = FeedParser().parse(raw_result)
assert len(result.storys) == _MAX_STORYS
expected = set(range(num_storys - _MAX_STORYS, num_storys))
story_ids = {int(x['ident']) for x in result.storys}
assert story_ids == expected
def test_raw_parse_warn(filename):
response = _read_response(_data_dir / 'warn', filename)
parser = RawFeedParser()
result = parser.parse(response)
assert result
assert result.warnings and isinstance(result.warnings, list)
assert result.storys
assert result.feed['version']
assert result.feed['title']
def _parse_well_feed(filename) -> FeedResult:
response = _read_response(_data_dir / 'well', filename)
raw_result = RawFeedParser().parse(response)
assert raw_result.feed
assert raw_result.storys
assert not raw_result.warnings
result = FeedParser().parse(raw_result)
assert len(result.storys) == len(raw_result.storys)
return result
def test_raw_parse_bad_encoding():
content = os.urandom(16 * 1024)
builder = FeedResponseBuilder()
builder.url('https://blog.example.com/feed')
builder.content(content)
response = builder.build()
parser = RawFeedParser()
with pytest.raises(FeedParserError) as ex:
parser.parse(response)
assert ex
def test_parser_and_checksum(filepath):
response = _read_response(_data_dir, filepath)
raw_parser = RawFeedParser()
raw_result = raw_parser.parse(response)
assert raw_result.feed
assert raw_result.storys
parser = FeedParser()
result = parser.parse(raw_result)
assert result.feed
assert result.storys
assert result.checksum.size() == len(result.storys)
def test_raw_parser_incomplete_content():
response = _read_response(_data_dir / 'warn', 'https-tmioe-com-feed.xml')
parser = RawFeedParser()
result = parser.parse(response)
assert len(result.storys) == 5
assert result.feed['version'] == 'rss20'
assert result.feed['title'] == 'ZAPRO · 杂铺'
expect_title = "TikTok 抖音国际版 v18.6.2 解锁全部国家任意切换"
expect_url = "https://tmioe.com/1463.html"
got_storys = [x for x in result.storys if x['url'] == expect_url]
assert got_storys
assert got_storys[0]['title'] == expect_title
def do_sync_feed(
ctx: ActorContext,
feed_id: T.int,
url: T.url,
use_proxy: T.bool.default(False),
content_hash_base64: T.str.optional,
etag: T.str.optional,
last_modified: T.str.optional,
):
params = dict(etag=etag, last_modified=last_modified, use_proxy=use_proxy)
options = _get_proxy_options()
options.update(allow_private_address=CONFIG.allow_private_address)
with FeedReader(**options) as reader:
response = reader.read(url, **params)
LOG.info(
f'read feed#{feed_id} url={unquote(url)} response.status={response.status}'
)
if response.status != 200 or not response.content:
return
new_hash = compute_hash_base64(response.content)
if new_hash == content_hash_base64:
LOG.info(
f'feed#{feed_id} url={unquote(url)} not modified by compare content hash!'
)
return
LOG.info(f'parse feed#{feed_id} url={unquote(url)}')
try:
raw_result = RawFeedParser().parse(response)
except FeedParserError as ex:
LOG.warning('failed parse feed#%s url=%r: %s', feed_id, unquote(url),
ex)
return
if raw_result.warnings:
warnings = '; '.join(raw_result.warnings)
LOG.warning('warning parse feed#%s url=%r: %s', feed_id, unquote(url),
warnings)
return
try:
feed = _parse_found((response, raw_result))
except (Invalid, FeedParserError) as ex:
LOG.error('invalid feed#%s url=%r: %s',
feed_id,
unquote(url),
ex,
exc_info=ex)
return
ctx.tell('harbor_rss.update_feed', dict(feed_id=feed_id, feed=feed))
def test_raw_parse_date_timestamp():
content = '''
<rss version="2.0">
<channel>
<title>博客中国</title>
<generator>http://www.blogchina.com</generator>
<item>
<title><![CDATA[ 新能源车崛起，传统汽车如何避免诺基亚式危机？ ]]></title>
<link>http://jianghulaoliu.blogchina.com/956463775.html</link>
<description><![CDATA[ 近日，特斯拉降价16万使得特斯拉消费激增 ]]></description>
<source></source>
<pubDate>1611146768</pubDate>
</item>
'''.encode('utf-8')
response = _create_builder(content=content).build()
result = RawFeedParser().parse(response)
assert len(result.storys) == 1
dt: datetime.datetime = result.storys[0]['dt_published']
assert dt == datetime.datetime.fromtimestamp(1611146768, tz=UTC)
def test_parse_large_content(template_name, content_length, summary_length):
content_snip = "<span>12345678</span>"
summary_snip = '<span>123</span>'
content_repeat = (content_length // len(content_snip)) + 1
content = content_snip * content_repeat
summary_repeat = (summary_length // len(summary_snip)) + 1
summary = summary_snip * summary_repeat
template = large_feed_templates[template_name]
# use replace instead format to avoid KeyError for json string
data = template\
.replace('${content}', content)\
.replace('${summary}', summary)\
.encode('utf-8')
response = _create_builder(content=data).build()
raw_result = RawFeedParser().parse(response)
assert raw_result and len(raw_result.storys) == 1
assert len(raw_result.storys[0]['content']) <= _RAW_MAX_CONTENT_LENGTH
assert len(raw_result.storys[0]['summary']) <= _RAW_MAX_SUMMARY_LENGTH
result = FeedParser().parse(raw_result)
assert result and len(result.storys) == 1
assert len(result.storys[0]['content']) <= _MAX_CONTENT_LENGTH
assert len(result.storys[0]['summary']) <= _MAX_SUMMARY_LENGTH
def do_sync_feed(
ctx: ActorContext,
feed_id: T.int,
url: T.url,
use_proxy: T.bool.default(False),
checksum_data: T.bytes.maxlen(4096).optional,
content_hash_base64: T.str.optional,
etag: T.str.optional,
last_modified: T.str.optional,
is_refresh: T.bool.default(False),
):
params = {}
if not is_refresh:
params = dict(etag=etag, last_modified=last_modified)
options = _proxy_helper.get_proxy_options()
if DNS_SERVICE.is_resolved_url(url):
use_proxy = False
switch_prob = 0.25 # the prob of switch from use proxy to not use proxy
with FeedReader(**options) as reader:
use_proxy = reader.has_proxy and use_proxy
if use_proxy and random.random() < switch_prob:
use_proxy = False
response = reader.read(url, **params, use_proxy=use_proxy)
LOG.info(
f'read feed#{feed_id} url={unquote(url)} status={response.status}')
need_proxy = FeedResponseStatus.is_need_proxy(response.status)
if (not use_proxy) and reader.has_proxy and need_proxy:
LOG.info(f'try use proxy read feed#{feed_id} url={unquote(url)}')
proxy_response = reader.read(url, **params, use_proxy=True)
LOG.info(
f'proxy read feed#{feed_id} url={unquote(url)} status={proxy_response.status}'
)
if proxy_response.ok:
response = proxy_response
if (not response.ok) or (not response.content):
status = FeedStatus.READY if response.status == 304 else FeedStatus.ERROR
_update_feed_info(ctx, feed_id, status=status, response=response)
return
new_hash = compute_hash_base64(response.content)
if (not is_refresh) and (new_hash == content_hash_base64):
LOG.info(
f'feed#{feed_id} url={unquote(url)} not modified by compare content hash!'
)
_update_feed_info(ctx, feed_id, response=response)
return
LOG.info(f'parse feed#{feed_id} url={unquote(url)}')
try:
raw_result = RawFeedParser().parse(response)
except FeedParserError as ex:
LOG.warning('failed parse feed#%s url=%r: %s', feed_id, unquote(url),
ex)
_update_feed_info(ctx,
feed_id,
status=FeedStatus.ERROR,
response=response,
warnings=str(ex))
return
if raw_result.warnings:
warnings = '; '.join(raw_result.warnings)
LOG.warning('warning parse feed#%s url=%r: %s', feed_id, unquote(url),
warnings)
try:
feed = _parse_found((response, raw_result),
checksum_data=checksum_data,
is_refresh=is_refresh)
except (Invalid, FeedParserError) as ex:
LOG.error('invalid feed#%s url=%r: %s',
feed_id,
unquote(url),
ex,
exc_info=ex)
_update_feed_info(ctx,
feed_id,
status=FeedStatus.ERROR,
response=response,
warnings=str(ex))
return
ctx.tell('harbor_rss.update_feed',
dict(feed_id=feed_id, feed=feed, is_refresh=is_refresh))
def test_raw_parse_failed(filename):
response = _read_response(_data_dir / 'failed', filename)
parser = RawFeedParser()
with pytest.raises(FeedParserError) as ex:
parser.parse(response)
assert ex