feedparser.RESOLVE_RELATIVE_URIS = False
feedparser.SANITIZE_HTML = False
UTC = datetime.timezone.utc
# TODO: maybe remove in the future
# On the date story ident change to v2 format
STORY_INDENT_V2_DATE = datetime.datetime(2020, 9, 1, 0, 0, 0, tzinfo=UTC)
RawFeedSchema = T.dict(
version=T.str,
title=T.str,
url=T.str,
home_url=T.str.optional,
icon_url=T.str.optional,
description=T.str.optional,
dt_updated=T.datetime.object.optional,
author_name=T.str.optional,
author_url=T.str.optional,
author_avatar_url=T.str.optional,
)
_MAX_CONTENT_LENGTH = 1000 * 1024
_MAX_SUMMARY_LENGTH = 10 * 1024
RawStorySchema = T.dict(
ident=T.str,
title=T.str,
url=T.str.optional,
content=T.str.maxlen(_MAX_CONTENT_LENGTH).optional,
summary=T.str.maxlen(_MAX_SUMMARY_LENGTH).optional,
from rssant_common.validator import compiler
from .response import FeedResponse
LOG = logging.getLogger(__name__)
feedparser.RESOLVE_RELATIVE_URIS = False
feedparser.SANITIZE_HTML = False
UTC = datetime.timezone.utc
RawFeedSchema = T.dict(
version=T.str,
title=T.str,
url=T.str,
home_url=T.str.optional,
icon_url=T.str.optional,
description=T.str.optional,
dt_updated=T.datetime.object.optional,
author_name=T.str.optional,
author_url=T.str.optional,
author_avatar_url=T.str.optional,
)
RawStorySchema = T.dict(
ident=T.str,
title=T.str,
url=T.str.optional,
content=T.str.optional,
summary=T.str.optional,
image_url=T.str.optional,
dt_published=T.datetime.object.optional,
dt_updated=T.datetime.object.optional,
import time
import base64
import json
import hmac
import brotli
from validr import T, Invalid
from rssant_common.validator import compiler
validate_image_token = compiler.compile(
T.dict(
timestamp=T.int,
referrer=T.url.optional,
))
class ImageTokenEncodeError(Exception):
"""ImageTokenEncodeError"""
class ImageTokenDecodeError(Exception):
"""ImageTokenDecodeError"""
class ImageTokenExpiredError(ImageTokenDecodeError):
"""ImageTokenExpiredError"""
class ImageToken:
def __init__(self, *, referrer: str = None, timestamp: int = None):
self.referrer = (referrer or '')[:255]
self.timestamp = timestamp or int(time.time())
def do_sync_local_ask(ctx: ActorContext) -> T.dict(message=T.str):
LOG.info(ctx.message)
r = dict(message='local_ask OK')
LOG.info(r)
return r
from rssant_api.helper import shorten
from .processor import (
story_html_to_text, story_html_clean, story_extract_attach,
story_has_mathjax, process_story_links, normalize_url, validate_url,
)
LOG = logging.getLogger(__name__)
FeedSchema = T.dict(
version=T.str.maxlen(200),
title=T.str.maxlen(200),
url=T.url,
home_url=T.url.invalid_to_default.optional,
icon_url=T.url.invalid_to_default.optional,
description=T.str.maxlen(300).optional,
dt_updated=T.datetime.object.optional,
author_name=T.str.maxlen(100).optional,
author_url=T.url.invalid_to_default.optional,
author_avatar_url=T.url.invalid_to_default.optional,
)
_MAX_CONTENT_LENGTH = 300 * 1024
_MAX_SUMMARY_LENGTH = 300
_MAX_STORYS = 300
StorySchema = T.dict(
ident=T.str.maxlen(200),
title=T.str.maxlen(200),
url=T.url.optional,
content=T.str.maxlen(_MAX_CONTENT_LENGTH).optional,
from validr import Compiler, T
from ..helper import expect_position
from . import case
class User:
def __init__(self, userid):
self.userid = userid
@case({
T.dict(userid=T.int): [
({
'userid': 1
}, {
'userid': 1
}),
(User(1), {
'userid': 1
}),
({
'userid': 1,
'extra': 'xxx'
}, {
'userid': 1
}),
],
T.dict(userid=T.int).optional: {
'valid': [
None,
def test_schema():
assert T(MyModel) == T.dict(id=T.int.min(0))
assert T(User) == T.dict(id=T.int.min(100).default(100), name=T.str)
None,
[],
],
'invalid': [
123,
]
},
T.list(T.int).unique: {
'valid': [
[1, 2, 3],
],
'invalid': [
[1, 2, '2'],
]
},
T.list(T.dict(key=T.int)).unique: {
'valid': [
[{
'key': 1
}, {
'key': 2
}],
],
'invalid': [
[{
'key': 1
}, {
'key': 1
}],
]
},
>>> encoded = encode_image_url(url, referer)
>>> decoded = decode_image_url(encoded)
>>> decoded['url'] == url
True
>>> decoded['referer'] == referer
True
"""
import base64
import json
import brotli
from validr import T, Invalid
from rssant_common.validator import compiler
validate_image_url = compiler.compile(
T.dict(
url=T.url,
referer=T.url.optional,
))
class ImageUrlEncodeError(Exception):
"""ImageUrlEncodeError"""
class ImageUrlDecodeError(Exception):
"""ImageUrlDecodeError"""
def encode_image_url(url, referer=None):
try:
text = json.dumps(validate_image_url(dict(url=url, referer=referer)))
data = brotli.compress(text.encode('utf-8'))
from validr import T
from cached_property import cached_property
from .actor import Actor
from .network_helper import LOCAL_NODE_NAME
from .helper import generate_message_id
from .message import ActorMessage
LOG = logging.getLogger(__name__)
NodeSpecSchema = T.dict(
name=T.str,
modules=T.list(T.str),
networks=T.list(T.dict(
name=T.str,
url=T.str.optional,
))
)
class NodeInfo:
def __init__(self, name: str, modules: set, networks: list):
self.name = name
self.modules = modules
self._networks = networks
def __repr__(self):
return '<{} #{} {}>'.format(type(self).__name__, self.id, self.name)
@cached_property
import os.path
import re
from dotenv import load_dotenv
from validr import T, modelclass, fields, Invalid
from rssant_common.validator import compiler
from actorlib.network_helper import LOCAL_NODE_NAME
validate_extra_networks = compiler.compile(
T.list(T.dict(
name=T.str,
url=T.url.relaxed,
)))
@modelclass(compiler=compiler)
class ConfigModel:
pass
class EnvConfig(ConfigModel):
debug: bool = T.bool.default(False).desc('debug')
profiler_enable: bool = T.bool.default(False).desc(
'enable profiler or not')
debug_toolbar_enable: bool = T.bool.default(False).desc(
'enable debug toolbar or not')
log_level: str = T.enum('DEBUG,INFO,WARNING,ERROR').default('INFO')
root_url: str = T.url.relaxed.default('http://localhost:6789')
scheduler_network: str = T.str.default('localhost')
scheduler_url: str = T.url.relaxed.default(
content_hash_base64=T.str,
link=T.str.optional,
author=T.str.optional,
icon=T.str.optional,
description=T.str.optional,
version=T.str.optional,
dt_updated=T.datetime.object.optional,
encoding=T.str.optional,
etag=T.str.optional,
last_modified=T.str.optional,
)
FeedOutputSchemaFields = FeedSchemaFields.copy()
FeedOutputSchemaFields.update(dt_updated=T.datetime.optional, )
StorySchema = T.dict(**StorySchemaFields)
FeedSchema = T.dict(
**FeedSchemaFields,
storys=T.list(StorySchema),
)
StoryOutputSchema = T.dict(**StoryOutputSchemaFields)
FeedOutputSchema = T.dict(
**FeedOutputSchemaFields,
storys=T.list(StoryOutputSchema),
)
validate_feed_output = compiler.compile(FeedOutputSchema)
@actor('harbor_rss.update_feed_creation_status')
async def do_echo(self, text: T.str) -> T.dict(text=T.str):
return dict(text=text)
with pytest.raises(Invalid):
assert _(T.int.optional)('')
with pytest.raises(Invalid):
assert _(T.dict(key=T.int).optional)('')
with pytest.raises(Invalid):
assert _(T.int)(None)
with pytest.raises(Invalid):
assert _(T.str)(None)
with pytest.raises(Invalid):
assert _(T.dict(key=T.int))(None)
with pytest.raises(Invalid):
assert _(T.list(T.int))(None)
def test_default():
assert _(T.int.default(0))(None) == 0
assert _(T.str.default('x'))(None) == 'x'
assert _(T.int.optional.default(0))(None) == 0
assert _(T.str.optional.default('x'))(None) == 'x'
@schema_error_position(
(T.unknown, ''),
(T.str.unknown, ''),
(T.dict(key=T.list(T.dict(key=T.unknown))), 'key[].key'),
)
def test_schema_error_position():
pass
REFERER_DENY_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_deny_url = compile_url_blacklist(REFERER_DENY_LIST)
StorySchema = T.dict(
unique_id=T.str,
title=T.str,
content_hash_base64=T.str,
author=T.str.optional,
link=T.url.optional,
image_url=T.url.optional,
iframe_url=T.url.optional,
audio_url=T.url.optional,
has_mathjax=T.bool.optional,
dt_published=T.datetime.optional,
dt_updated=T.datetime.optional,
summary=T.str.optional,
content=T.str.optional,
)
FeedSchema = T.dict(
url=T.url,
use_proxy=T.bool.default(False),
title=T.str,
content_length=T.int,
content_hash_base64=T.str,
link=T.url.optional,
def test_compiled_items():
compiler = Compiler()
value = compiler.compile(T.int.min(0))
assert repr(T.dict(key=value)) == 'T.dict({key})'
assert repr(T.list(value)) == 'T.list(int)'
REFERER_DENY_LIST = """
qpic.cn
qlogo.cn
qq.com
"""
is_referer_deny_url = compile_url_blacklist(REFERER_DENY_LIST)
StorySchema = T.dict(
unique_id=T.str,
title=T.str,
content_hash_base64=T.str,
author=T.str.optional,
link=T.url.optional,
has_mathjax=T.bool.optional,
dt_published=T.datetime.optional,
dt_updated=T.datetime.optional,
summary=T.str.optional,
content=T.str.optional,
)
FeedSchema = T.dict(
url=T.url,
title=T.str,
content_length=T.int,
content_hash_base64=T.str,
link=T.url.optional,
author=T.str.optional,
icon=T.str.optional,
description=T.str.optional,
async def do_echo(self, text: T.str.default("")) -> T.dict(text=T.str):
"""A echo method"""
if text == "error":
raise EchoError("I echo an error")
return {"text": text * self.echo_times}
def test_dict_error_position():
validate = compiler.compile(T.dict(key=T.list(T.dict(key=T.int))))
validate({'key': [{'key': 'x'}]})
async def get_echo(self) -> T.dict(text=T.str):
text = self.request.query.get("text") or ""
return {"text": text * self.echo_times}
icon=T.str.optional,
description=T.str.optional,
version=T.str.optional,
dt_updated=T.datetime.object.optional,
encoding=T.str.optional,
etag=T.str.optional,
last_modified=T.str.optional,
response_status=T.int.optional,
checksum_data=T.bytes.maxlen(4096).optional,
warnings=T.str.optional,
)
FeedOutputSchemaFields = FeedSchemaFields.copy()
FeedOutputSchemaFields.update(dt_updated=T.datetime.optional, )
StorySchema = T.dict(**StorySchemaFields)
FeedSchema = T.dict(
**FeedSchemaFields,
storys=T.list(StorySchema),
)
FeedInfoSchemaFieldNames = [
'response_status',
'warnings',
]
FeedInfoSchemaFields = {
k: FeedSchemaFields[k]
for k in FeedInfoSchemaFieldNames
}
FeedInfoSchema = T.dict(
**FeedInfoSchemaFields,
async def do_async_local_ask(ctx: ActorContext) -> T.dict(message=T.str):
LOG.info(ctx.message)
r = await ctx.ask('worker.sync_local_ask')
LOG.info(r)
return r