def __init__(self, name=None, permission_classes=None):
self.name = name
if permission_classes:
permission_classes = tuple(permission_classes)
self.permission_classes = permission_classes
self._schema_compiler = Compiler(validators=VALIDATORS)
self._routes = []
def send_recall_timed(receiver_file, duration):
receiver_file = os.path.abspath(os.path.expanduser(receiver_file))
done_file = receiver_file + '.done'
with open(receiver_file) as f:
receivers = _parse_receivers(f.read())
try:
with open(done_file) as f:
done_receivers = set(_parse_receivers(f.read()))
except FileNotFoundError:
done_receivers = set()
todo_receivers = [x for x in receivers if x not in done_receivers]
parse_duration = Compiler().compile(T.timedelta.min(0))
duration = parse_duration(duration)
status_msg = 'total={} done={} todo={}'.format(
len(receivers), len(done_receivers), len(todo_receivers))
click.confirm('Send recall email to {} receivers?'.format(status_msg), abort=True)
if not todo_receivers:
return
subject = '好久不见，甚是想念，欢迎回蚁阅看看~'
sender = CONFIG.smtp_username
email = EmailTemplate(subject, filename='recall.html')
sleep_time = duration / len(todo_receivers)
for receiver in todo_receivers:
click.echo('> {}'.format(receiver))
email.send(sender, receiver, context=dict(username=receiver))
with open(done_file, 'a+') as f:
f.write(receiver + '\n')
time.sleep(sleep_time)
def generate_fields_schema(layout_type):
"""
"""
SUBFIELD_VALIDATE = Contact.SUBFIELD_VALIDATE
INSTANCE_FIELDS = layout_type.fields
dict_schema = {
"$self": "dict",
}
fields_schema = {
key: {
subkey:
f"{SUBFIELD_VALIDATE[subkey]}.minlen({1 if INSTANCE_FIELDS[key]['requirement'] else 0})"
if SUBFIELD_VALIDATE[subkey] == "str" else
SUBFIELD_VALIDATE[subkey]
for subkey in SUBFIELD_VALIDATE.keys()
}
for key in INSTANCE_FIELDS.keys()
}
dict_schema.update(fields_schema)
schema = T(dict_schema)
compiled_schema = Compiler().compile(schema)
return compiled_schema
def url_validator(compiler, scheme='http https', default_schema=None, maxlen=1024, relaxed=False):
"""
Args:
default_schema: 接受没有scheme的url并尝试修正
relaxed: accept not strict url
"""
if relaxed:
return Compiler().compile(T.url.maxlen(maxlen).scheme(scheme))
schemes = set(scheme.replace(',', ' ').split(' '))
if default_schema and default_schema not in schemes:
raise SchemaError('invalid default_schema {}'.format(default_schema))
_django_validate_url = URLValidator(schemes=schemes)
def validate(value):
if default_schema:
value = coerce_url(value, default_schema=default_schema)
try:
_django_validate_url(value)
except ValidationError:
# TODO: access ValidationError.messages will cause error when
# django/i18n not setup, maybe use validators package instead
# raise Invalid(','.join(ex.messages).rstrip('.'))
raise Invalid('invalid or incorrect url format')
if len(value) > maxlen:
raise Invalid(f'url length must <= {maxlen}')
return value
return validate
def test_custom_validator():
@validator(string=True)
def choice_validator(compiler, items):
choices = set(items.split())
def validate(value):
if value in choices:
return value
raise Invalid('invalid choice')
return validate
compiler = Compiler(validators={'choice': choice_validator})
schema = T.list(T.choice('A B C D').default('A'))
assert T(schema) == schema # test copy custom validator
validate = compiler.compile(schema)
assert validate(['A', 'B', 'C', 'D', None]) == ['A', 'B', 'C', 'D', 'A']
def test_wrapped_validator():
str_validator = builtin_validators['str']
assert str_validator.is_string
logs = []
@validator(string=True)
def wrapped_str_validator(*args, **kwargs):
_validate = str_validator.validator(*args, **kwargs)
def validate(value):
logs.append(value)
return _validate(value)
return validate
compiler = Compiler(validators={'str': wrapped_str_validator})
validate = compiler.compile(T.str.optional)
assert validate('abc') == 'abc'
assert logs == ['abc']
def test_validr(benchmark):
from validr import T, Compiler
schema = Compiler().compile(
T.dict(
location=T.dict(lat=T.float.min(-90).max(90),
lng=T.float.min(-180).max(180)),
name=T.str,
alt_names=T.list(T.str),
population=T.dict(city=T.int.min(0), metro=T.int.min(0)),
))
assert benchmark(schema, data) == data
return validate
@validator(accept=str, output=str)
def enum_validator(compiler, items):
items = set(items.replace(',', ' ').split())
def validate(value):
if value in items:
return value
raise Invalid('value must be one of {}'.format(items))
return validate
VALIDATORS = {
'cursor': cursor_validator,
'url': url_validator,
'datetime': datetime_validator,
'feed_unionid': create_unionid_validator(FeedUnionId),
'story_unionid': create_unionid_validator(StoryUnionId),
'detail': detail_validator,
'dict': dict_validator,
'str': str_validator,
'interval': interval_validator,
'enum': enum_validator,
}
compiler = Compiler(validators=VALIDATORS)
def test_illegal_version():
with pytest.raises(SchemaError):
Compiler().compile(T.uuid.version(10))
yield schema, value, value
for value in items.get('invalid', []):
yield schema, value, Invalid
for value, expect in items.get('expect', []):
yield schema, value, expect
else:
for item in items:
if type(item) is tuple:
value, expect = item
yield schema, value, expect
else:
for value in item:
yield schema, value, Invalid
compiler = Compiler()
def case(cases):
"""Genereate test from cases data"""
def decorator(f):
@pytest.mark.parametrize('schema,value,expect', expend(cases))
def wrapped(schema, value, expect):
f = compiler.compile(schema)
if expect is Invalid:
with pytest.raises(Invalid):
f(value)
else:
assert f(value) == expect
return wrapped
return decorator
@validator(accept=bytes, output=bytes)
def bytes_validator(compiler, maxlen=None):
def validate(value):
if not isinstance(value, bytes):
raise Invalid('invalid bytes type')
if maxlen is not None:
if len(value) > maxlen:
raise Invalid('value must <= {}'.format(maxlen))
return value
return validate
VALIDATORS = {
'cursor': cursor_validator,
'url': url_validator,
'datetime': datetime_validator,
'feed_unionid': create_unionid_validator(FeedUnionId),
'story_unionid': create_unionid_validator(StoryUnionId),
'detail': detail_validator,
'str': str_validator,
'bytes': bytes_validator,
}
compiler = Compiler(validators=VALIDATORS)
# warming up django url validator
compiler.compile(T.url)('https://example.com/')
from timeit import Timer
from validr import T, Compiler, Invalid
from validators import url as _validators_url
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
def validators_url(x):
return _validators_url(x) is True
_validr_url = Compiler().compile(T.url.scheme('http https'))
def validr_url(x):
try:
_validr_url(x)
except Invalid:
return False
else:
return True
_django_url = URLValidator({'http', 'https'})
def django_url(x):
try:
_django_url(x)
except ValidationError:
return False
import pytest
from validr import Invalid, Compiler, T
from .helper import schema_error_position
_ = Compiler().compile
def test_optional():
assert _(T.int.optional)(None) is None
assert _(T.str.optional)(None) == ''
assert _(T.str.optional)('') == ''
assert _(T.list(T.int).optional)(None) is None
assert _(T.dict(key=T.int).optional)(None) is None
with pytest.raises(Invalid):
assert _(T.int.optional)('')
with pytest.raises(Invalid):
assert _(T.dict(key=T.int).optional)('')
with pytest.raises(Invalid):
assert _(T.int)(None)
with pytest.raises(Invalid):
assert _(T.str)(None)
with pytest.raises(Invalid):
assert _(T.dict(key=T.int))(None)
with pytest.raises(Invalid):
assert _(T.list(T.int))(None)
def test_default():
def _load_schema_compiler(self):
self.validators = INTERNAL_VALIDATORS.copy()
if hasattr(self.config_module, "validators"):
self.validators.update(self.config_module.validators)
self.schema_compiler = Compiler(self.validators)
def test_create_enum_validator():
abcd_validator = create_enum_validator('abcd', ['A', 'B', 'C', 'D'])
compiler = Compiler(validators={'abcd': abcd_validator})
schema = T.list(T.abcd.default('A'))
validate = compiler.compile(schema)
assert validate(['A', 'B', 'C', 'D', None]) == ['A', 'B', 'C', 'D', 'A']
def test_load_schema():
compiler = Compiler()
schema = T.list(T.int.min(0))
assert T(schema) == schema
assert T(compiler.compile(schema)) == schema
assert T(['int.min(0)']) == schema
def test_compiled_items():
compiler = Compiler()
value = compiler.compile(T.int.min(0))
assert repr(T.dict(key=value)) == 'T.dict({key})'
assert repr(T.list(value)) == 'T.list(int)'
def __init__(self):
super().__init__()
self._schema_compiler = Compiler()
import os.path
import re
from functools import cached_property
from urllib.parse import urlparse
from dotenv import load_dotenv
from validr import T, Compiler, modelclass, fields, Invalid
from rssant_common.network_helper import LOCAL_NODE_NAME
MAX_FEED_COUNT = 5000
compiler = Compiler()
validate_extra_networks = compiler.compile(
T.list(T.dict(
name=T.str,
url=T.url,
)))
@modelclass(compiler=compiler)
class ConfigModel:
pass
class GitHubConfigModel(ConfigModel):
domain: str = T.str
client_id: str = T.str
secret: str = T.str
id = T.int.min(100).default(100)
name = T.str
def __post_init__(self):
self.id_x3 = self.id * 3
class CustomModel(MyModel):
def __eq__(self, other):
return id(self) == id(other)
def get_id(self):
return self.id
@modelclass(compiler=Compiler(), immutable=True)
class ImmutableModel:
id = T.int.min(0)
def __init__(self, id=None):
self.id = id
def test_model():
with pytest.raises(Invalid):
User(name="test", xxx=123)
user = User(name="test")
assert user.id == 100
assert user.name == "test"
with pytest.raises(Invalid):
user.id = -1
import time
import base64
import json
import hmac
import brotli
from validr import T, Invalid, Compiler
validate_image_token = Compiler().compile(
T.dict(
timestamp=T.int.min(0),
referrer=T.url.optional,
feed=T.int.min(0).optional.desc('feed id'),
offset=T.int.min(0).optional.desc('story offset'),
))
class ImageTokenEncodeError(Exception):
"""ImageTokenEncodeError"""
class ImageTokenDecodeError(Exception):
"""ImageTokenDecodeError"""
class ImageTokenExpiredError(ImageTokenDecodeError):
"""ImageTokenExpiredError"""
class ImageToken:
def __init__(
self,
class ValidrRouteTableDef(RouteTableDef):
def __init__(self):
super().__init__()
self._schema_compiler = Compiler(validators=VALIDATORS)
@staticmethod
def _response_from_invalid(ex):
return json_response(
{
'description': str(ex),
'position': ex.position,
'message': ex.message,
'field': ex.field,
'value': ex.value,
},
status=400)
def decorate(self, f):
assert inspect.iscoroutinefunction(f), f'{f} is not coroutine function'
params = get_params(f)
if params is not None:
params = self._schema_compiler.compile(params)
returns = get_returns(f)
if returns is not None:
returns = self._schema_compiler.compile(returns)
async def wrapped(request, **kwargs):
ret = None
if params is not None:
maps = [kwargs, request.match_info]
if request.method in ['GET', 'DELETE']:
maps.append(request.query)
else:
try:
maps.append(await request.json())
except json.JSONDecodeError:
return json_response({"message": 'Invalid JSON'},
status=400)
try:
kwargs = params(ChainMap(*maps))
except Invalid as ex:
ret = self._response_from_invalid(ex)
if ret is None:
ret = await f(request, **kwargs)
if returns is not None:
if not isinstance(ret, StreamResponse):
ret = returns(ret)
ret = json_response(ret)
elif ret is None:
ret = Response(status=204)
return ret
wrapped.__name__ = f.__name__
wrapped.__qualname__ = f.__qualname__
wrapped.__doc__ = f.__doc__
return wrapped
def route(self, *args, **kwargs):
routes_decorate = super().route(*args, **kwargs)
def wrapper(f):
return routes_decorate(self.decorate(f))
return wrapper
class RestRouter:
def __init__(self, name=None, permission_classes=None):
self.name = name
if permission_classes:
permission_classes = tuple(permission_classes)
self.permission_classes = permission_classes
self._schema_compiler = Compiler(validators=VALIDATORS)
self._routes = []
@property
def urls(self):
def key_func(r):
f, url, methods, params, returns = r
return url
urls_map = {}
routes = sorted(self._routes, key=key_func)
groups = itertools.groupby(routes, key=key_func)
for url, group in groups:
view = self._make_view(list(group))
urls_map[url] = path(url, view)
# keep urls same order with self._routes
# and constant url should priority then path argument
urls = []
urls_priority = []
urls_added = set()
for f, url, methods, params, returns in self._routes:
if url not in urls_added:
urls_added.add(url)
if '<' in url and ':' in url and '>' in url:
urls.append(urls_map[url])
else:
urls_priority.append(urls_map[url])
return urls_priority + urls
@staticmethod
def _response_from_invalid(ex):
return Response(
{
'description': str(ex),
'position': ex.position,
'message': ex.message,
'field': ex.field,
'value': ex.value,
},
status=400)
@staticmethod
def _make_method(method, f, params, returns):
def rest_method(self, request, format=None, **kwargs):
ret = None
validr_cost = 0
if params is not None:
maps = [kwargs]
if request.method in ['GET', 'DELETE']:
maps.append(request.query_params)
else:
maps.append(request.data)
t_begin = time.time()
try:
kwargs = params(ChainMap(*maps))
except Invalid as ex:
ret = RestRouter._response_from_invalid(ex)
validr_cost += time.time() - t_begin
t_begin = time.time()
if ret is None:
ret = f(request, **kwargs)
api_cost = time.time() - t_begin
if returns is not None:
if not isinstance(ret, (Response, HttpResponse)):
t_begin = time.time()
ret = returns(ret)
validr_cost += time.time() - t_begin
ret = Response(ret)
elif ret is None:
ret = Response(status=204)
if validr_cost > 0:
ret['X-Validr-Time'] = '{:.0f}ms'.format(validr_cost * 1000)
if api_cost > 0:
ret['X-API-Time'] = '{:.0f}ms'.format(api_cost * 1000)
return ret
rest_method.__name__ = method.lower()
rest_method.__qualname__ = method.lower()
rest_method.__doc__ = f.__doc__
return rest_method
def _make_view(self, group):
method_maps = {}
method_meta = {}
for f, url, methods, params, returns in group:
for method in methods:
if method in method_maps:
raise ValueError(f'duplicated method {method} of {url}')
m = self._make_method(method, f, params, returns)
method_maps[method] = m
method_meta[method] = f, url, params, returns
class RestApiView(APIView):
if self.permission_classes:
permission_classes = self.permission_classes
schema = RestViewSchema(method_meta)
if 'GET' in method_maps:
get = method_maps['GET']
if 'POST' in method_maps:
post = method_maps['POST']
if 'PUT' in method_maps:
put = method_maps['PUT']
if 'DELETE' in method_maps:
delete = method_maps['DELETE']
if 'PATCH' in method_maps:
patch = method_maps['PATCH']
return RestApiView.as_view()
def _route(self, url, methods):
if isinstance(methods, str):
methods = set(methods.strip().replace(',', ' ').split())
else:
methods = set(methods)
methods = set(x.upper() for x in methods)
def wrapper(f):
params = get_params(f)
if params is not None:
params = self._schema_compiler.compile(params)
returns = get_returns(f)
if returns is not None:
returns = self._schema_compiler.compile(returns)
self._routes.append((f, url, methods, params, returns))
return f
return wrapper
def get(self, url=''):
return self._route(url, methods='GET')
def post(self, url=''):
return self._route(url, methods='POST')
def put(self, url=''):
return self._route(url, methods='PUT')
def delete(self, url=''):
return self._route(url, methods='DELETE')
def patch(self, url=''):
return self._route(url, methods='PATCH')
def route(self, url='', methods='GET'):
return self._route(url, methods=methods)
__call__ = route
def __init__(self):
super().__init__()
self._schema_compiler = Compiler(validators=VALIDATORS)
def __init__(
self,
actors,
host='0.0.0.0',
port=8000,
concurrency=100,
name=None,
subpath=None,
networks=None,
registery_node_spec=None,
storage_dir_path=None,
storage_compact_wal_delta=1000,
queue_max_complete_size=1000,
max_retry_time=10 * 60,
max_retry_count=3,
token=None,
schema_compiler=None,
on_startup=None,
on_shutdown=None,
):
if schema_compiler is None:
schema_compiler = Compiler()
self.schema_compiler = schema_compiler
self.actors = {}
self.timers = {}
for handler in chain(actors, BUILTIN_ACTORS):
if inspect.isclass(handler):
handler = handler(self)
x = Actor(handler, schema_compiler=schema_compiler)
if x.timer is not None:
self.timers[x.name] = x
self.actors[x.name] = x
actor_modules = {x.module for x in self.actors.values()}
if not name:
name = f'actor-{port}'
self.name = name
if not networks:
networks = []
networks.append(get_localhost_network(port=port, subpath=subpath))
current_node_spec = dict(
name=self.name,
modules=actor_modules,
networks=networks,
)
self.token = token
self.registery = ActorRegistery(
current_node_spec=current_node_spec,
registery_node_spec=registery_node_spec)
if storage_dir_path:
storage_dir_path = os.path.abspath(os.path.expanduser(storage_dir_path))
storage_path = os.path.join(storage_dir_path, self.name)
self.storage = ActorLocalStorage(
dirpath=storage_path, compact_wal_delta=storage_compact_wal_delta)
else:
self.storage = None
self.queue = ActorMessageQueue(
registery=self.registery,
actors=self.actors,
storage=self.storage,
max_complete_size=queue_max_complete_size,
concurrency=concurrency,
max_retry_count=max_retry_count,
max_retry_time=max_retry_time,
)
self.concurrency = concurrency
self.executor = ActorExecutor(
self.actors,
queue=self.queue,
registery=self.registery,
concurrency=concurrency,
token=self.token,
)
self.host = host
self.port = port
Info("actor_node", "actor node info").info({'name': self.name})
self.subpath = subpath or ''
self.receiver = MessageReceiver(
host=self.host, port=self.port, subpath=self.subpath,
queue=self.queue, registery=self.registery, token=self.token)
self._client = None
self._on_startup_handlers = []
self._on_shutdown_handlers = []
if on_startup:
self._on_startup_handlers.extend(on_startup)
if on_shutdown:
self._on_shutdown_handlers.extend(on_shutdown)
except ImportError:
return
try:
import rlcompleter
except ImportError:
return
readline.set_completer(rlcompleter.Completer(context).complete)
readline.parse_and_bind("tab:complete")
# command history
if os.path.exists(HISTORY_PATH):
readline.read_history_file(HISTORY_PATH)
atexit.register(_save_history)
HeadersSchema = T.list(T.dict(name=T.str, value=T.str))
validate_headers = Compiler().compile(HeadersSchema)
class Shell:
def __init__(self, app):
self.app = app
headers = self._load_headers()
self._client = Client(app, headers=headers)
def _load_headers(self):
if not os.path.exists(HEADERS_PATH):
return None
LOG.info(f"Load headers from {HEADERS_PATH!r}")
try:
with open(HEADERS_PATH) as f:
data = json.load(f)
def main():
service = Service(_DocumentSampleService, {}, [], Compiler())
print(format_service_doc(service))
for method in service.methods:
print('-' * 60)
print(format_method_doc(method))
from validr import T, Compiler, modelclass, asdict
@modelclass
class Model:
user = T.dict(userid=T.int.min(0).max(9).desc("UserID"))
tags = T.list(T.int.min(0))
style = T.dict(
width=T.int.desc("width"),
height=T.int.desc("height"),
border_width=T.int.desc("border_width"),
border_style=T.str.desc("border_style"),
border_color=T.str.desc("border_color"),
color=T.str.desc("color"),
)
optional = T.str.optional.desc("unknown value")
compiler = Compiler()
default = compiler.compile(T(Model))
def model(value):
return asdict(Model(value))
CASES = {"default": default, "model": model}
def test_invalid_default():
with pytest.raises(SchemaError):
Compiler().compile(T.int.default('abc'))
def decorator(predictor_cls):
s = T(schema)
predictor_cls._schema_func = Compiler().compile(s)
return predictor_cls
