def __init__(self, name=None, permission_classes=None): self.name = name if permission_classes: permission_classes = tuple(permission_classes) self.permission_classes = permission_classes self._schema_compiler = Compiler(validators=VALIDATORS) self._routes = []
def send_recall_timed(receiver_file, duration): receiver_file = os.path.abspath(os.path.expanduser(receiver_file)) done_file = receiver_file + '.done' with open(receiver_file) as f: receivers = _parse_receivers(f.read()) try: with open(done_file) as f: done_receivers = set(_parse_receivers(f.read())) except FileNotFoundError: done_receivers = set() todo_receivers = [x for x in receivers if x not in done_receivers] parse_duration = Compiler().compile(T.timedelta.min(0)) duration = parse_duration(duration) status_msg = 'total={} done={} todo={}'.format( len(receivers), len(done_receivers), len(todo_receivers)) click.confirm('Send recall email to {} receivers?'.format(status_msg), abort=True) if not todo_receivers: return subject = '好久不见,甚是想念,欢迎回蚁阅看看~' sender = CONFIG.smtp_username email = EmailTemplate(subject, filename='recall.html') sleep_time = duration / len(todo_receivers) for receiver in todo_receivers: click.echo('> {}'.format(receiver)) email.send(sender, receiver, context=dict(username=receiver)) with open(done_file, 'a+') as f: f.write(receiver + '\n') time.sleep(sleep_time)
def generate_fields_schema(layout_type): """ """ SUBFIELD_VALIDATE = Contact.SUBFIELD_VALIDATE INSTANCE_FIELDS = layout_type.fields dict_schema = { "$self": "dict", } fields_schema = { key: { subkey: f"{SUBFIELD_VALIDATE[subkey]}.minlen({1 if INSTANCE_FIELDS[key]['requirement'] else 0})" if SUBFIELD_VALIDATE[subkey] == "str" else SUBFIELD_VALIDATE[subkey] for subkey in SUBFIELD_VALIDATE.keys() } for key in INSTANCE_FIELDS.keys() } dict_schema.update(fields_schema) schema = T(dict_schema) compiled_schema = Compiler().compile(schema) return compiled_schema
def url_validator(compiler, scheme='http https', default_schema=None, maxlen=1024, relaxed=False): """ Args: default_schema: 接受没有scheme的url并尝试修正 relaxed: accept not strict url """ if relaxed: return Compiler().compile(T.url.maxlen(maxlen).scheme(scheme)) schemes = set(scheme.replace(',', ' ').split(' ')) if default_schema and default_schema not in schemes: raise SchemaError('invalid default_schema {}'.format(default_schema)) _django_validate_url = URLValidator(schemes=schemes) def validate(value): if default_schema: value = coerce_url(value, default_schema=default_schema) try: _django_validate_url(value) except ValidationError: # TODO: access ValidationError.messages will cause error when # django/i18n not setup, maybe use validators package instead # raise Invalid(','.join(ex.messages).rstrip('.')) raise Invalid('invalid or incorrect url format') if len(value) > maxlen: raise Invalid(f'url length must <= {maxlen}') return value return validate
def test_custom_validator(): @validator(string=True) def choice_validator(compiler, items): choices = set(items.split()) def validate(value): if value in choices: return value raise Invalid('invalid choice') return validate compiler = Compiler(validators={'choice': choice_validator}) schema = T.list(T.choice('A B C D').default('A')) assert T(schema) == schema # test copy custom validator validate = compiler.compile(schema) assert validate(['A', 'B', 'C', 'D', None]) == ['A', 'B', 'C', 'D', 'A']
def test_wrapped_validator(): str_validator = builtin_validators['str'] assert str_validator.is_string logs = [] @validator(string=True) def wrapped_str_validator(*args, **kwargs): _validate = str_validator.validator(*args, **kwargs) def validate(value): logs.append(value) return _validate(value) return validate compiler = Compiler(validators={'str': wrapped_str_validator}) validate = compiler.compile(T.str.optional) assert validate('abc') == 'abc' assert logs == ['abc']
def test_validr(benchmark): from validr import T, Compiler schema = Compiler().compile( T.dict( location=T.dict(lat=T.float.min(-90).max(90), lng=T.float.min(-180).max(180)), name=T.str, alt_names=T.list(T.str), population=T.dict(city=T.int.min(0), metro=T.int.min(0)), )) assert benchmark(schema, data) == data
return validate @validator(accept=str, output=str) def enum_validator(compiler, items): items = set(items.replace(',', ' ').split()) def validate(value): if value in items: return value raise Invalid('value must be one of {}'.format(items)) return validate VALIDATORS = { 'cursor': cursor_validator, 'url': url_validator, 'datetime': datetime_validator, 'feed_unionid': create_unionid_validator(FeedUnionId), 'story_unionid': create_unionid_validator(StoryUnionId), 'detail': detail_validator, 'dict': dict_validator, 'str': str_validator, 'interval': interval_validator, 'enum': enum_validator, } compiler = Compiler(validators=VALIDATORS)
def test_illegal_version(): with pytest.raises(SchemaError): Compiler().compile(T.uuid.version(10))
yield schema, value, value for value in items.get('invalid', []): yield schema, value, Invalid for value, expect in items.get('expect', []): yield schema, value, expect else: for item in items: if type(item) is tuple: value, expect = item yield schema, value, expect else: for value in item: yield schema, value, Invalid compiler = Compiler() def case(cases): """Genereate test from cases data""" def decorator(f): @pytest.mark.parametrize('schema,value,expect', expend(cases)) def wrapped(schema, value, expect): f = compiler.compile(schema) if expect is Invalid: with pytest.raises(Invalid): f(value) else: assert f(value) == expect return wrapped return decorator
@validator(accept=bytes, output=bytes) def bytes_validator(compiler, maxlen=None): def validate(value): if not isinstance(value, bytes): raise Invalid('invalid bytes type') if maxlen is not None: if len(value) > maxlen: raise Invalid('value must <= {}'.format(maxlen)) return value return validate VALIDATORS = { 'cursor': cursor_validator, 'url': url_validator, 'datetime': datetime_validator, 'feed_unionid': create_unionid_validator(FeedUnionId), 'story_unionid': create_unionid_validator(StoryUnionId), 'detail': detail_validator, 'str': str_validator, 'bytes': bytes_validator, } compiler = Compiler(validators=VALIDATORS) # warming up django url validator compiler.compile(T.url)('https://example.com/')
from timeit import Timer from validr import T, Compiler, Invalid from validators import url as _validators_url from django.core.validators import URLValidator from django.core.exceptions import ValidationError def validators_url(x): return _validators_url(x) is True _validr_url = Compiler().compile(T.url.scheme('http https')) def validr_url(x): try: _validr_url(x) except Invalid: return False else: return True _django_url = URLValidator({'http', 'https'}) def django_url(x): try: _django_url(x) except ValidationError: return False
import pytest from validr import Invalid, Compiler, T from .helper import schema_error_position _ = Compiler().compile def test_optional(): assert _(T.int.optional)(None) is None assert _(T.str.optional)(None) == '' assert _(T.str.optional)('') == '' assert _(T.list(T.int).optional)(None) is None assert _(T.dict(key=T.int).optional)(None) is None with pytest.raises(Invalid): assert _(T.int.optional)('') with pytest.raises(Invalid): assert _(T.dict(key=T.int).optional)('') with pytest.raises(Invalid): assert _(T.int)(None) with pytest.raises(Invalid): assert _(T.str)(None) with pytest.raises(Invalid): assert _(T.dict(key=T.int))(None) with pytest.raises(Invalid): assert _(T.list(T.int))(None) def test_default():
def _load_schema_compiler(self): self.validators = INTERNAL_VALIDATORS.copy() if hasattr(self.config_module, "validators"): self.validators.update(self.config_module.validators) self.schema_compiler = Compiler(self.validators)
def test_create_enum_validator(): abcd_validator = create_enum_validator('abcd', ['A', 'B', 'C', 'D']) compiler = Compiler(validators={'abcd': abcd_validator}) schema = T.list(T.abcd.default('A')) validate = compiler.compile(schema) assert validate(['A', 'B', 'C', 'D', None]) == ['A', 'B', 'C', 'D', 'A']
def test_load_schema(): compiler = Compiler() schema = T.list(T.int.min(0)) assert T(schema) == schema assert T(compiler.compile(schema)) == schema assert T(['int.min(0)']) == schema
def test_compiled_items(): compiler = Compiler() value = compiler.compile(T.int.min(0)) assert repr(T.dict(key=value)) == 'T.dict({key})' assert repr(T.list(value)) == 'T.list(int)'
def __init__(self): super().__init__() self._schema_compiler = Compiler()
import os.path import re from functools import cached_property from urllib.parse import urlparse from dotenv import load_dotenv from validr import T, Compiler, modelclass, fields, Invalid from rssant_common.network_helper import LOCAL_NODE_NAME MAX_FEED_COUNT = 5000 compiler = Compiler() validate_extra_networks = compiler.compile( T.list(T.dict( name=T.str, url=T.url, ))) @modelclass(compiler=compiler) class ConfigModel: pass class GitHubConfigModel(ConfigModel): domain: str = T.str client_id: str = T.str secret: str = T.str
id = T.int.min(100).default(100) name = T.str def __post_init__(self): self.id_x3 = self.id * 3 class CustomModel(MyModel): def __eq__(self, other): return id(self) == id(other) def get_id(self): return self.id @modelclass(compiler=Compiler(), immutable=True) class ImmutableModel: id = T.int.min(0) def __init__(self, id=None): self.id = id def test_model(): with pytest.raises(Invalid): User(name="test", xxx=123) user = User(name="test") assert user.id == 100 assert user.name == "test" with pytest.raises(Invalid): user.id = -1
import time import base64 import json import hmac import brotli from validr import T, Invalid, Compiler validate_image_token = Compiler().compile( T.dict( timestamp=T.int.min(0), referrer=T.url.optional, feed=T.int.min(0).optional.desc('feed id'), offset=T.int.min(0).optional.desc('story offset'), )) class ImageTokenEncodeError(Exception): """ImageTokenEncodeError""" class ImageTokenDecodeError(Exception): """ImageTokenDecodeError""" class ImageTokenExpiredError(ImageTokenDecodeError): """ImageTokenExpiredError""" class ImageToken: def __init__( self,
class ValidrRouteTableDef(RouteTableDef): def __init__(self): super().__init__() self._schema_compiler = Compiler(validators=VALIDATORS) @staticmethod def _response_from_invalid(ex): return json_response( { 'description': str(ex), 'position': ex.position, 'message': ex.message, 'field': ex.field, 'value': ex.value, }, status=400) def decorate(self, f): assert inspect.iscoroutinefunction(f), f'{f} is not coroutine function' params = get_params(f) if params is not None: params = self._schema_compiler.compile(params) returns = get_returns(f) if returns is not None: returns = self._schema_compiler.compile(returns) async def wrapped(request, **kwargs): ret = None if params is not None: maps = [kwargs, request.match_info] if request.method in ['GET', 'DELETE']: maps.append(request.query) else: try: maps.append(await request.json()) except json.JSONDecodeError: return json_response({"message": 'Invalid JSON'}, status=400) try: kwargs = params(ChainMap(*maps)) except Invalid as ex: ret = self._response_from_invalid(ex) if ret is None: ret = await f(request, **kwargs) if returns is not None: if not isinstance(ret, StreamResponse): ret = returns(ret) ret = json_response(ret) elif ret is None: ret = Response(status=204) return ret wrapped.__name__ = f.__name__ wrapped.__qualname__ = f.__qualname__ wrapped.__doc__ = f.__doc__ return wrapped def route(self, *args, **kwargs): routes_decorate = super().route(*args, **kwargs) def wrapper(f): return routes_decorate(self.decorate(f)) return wrapper
class RestRouter: def __init__(self, name=None, permission_classes=None): self.name = name if permission_classes: permission_classes = tuple(permission_classes) self.permission_classes = permission_classes self._schema_compiler = Compiler(validators=VALIDATORS) self._routes = [] @property def urls(self): def key_func(r): f, url, methods, params, returns = r return url urls_map = {} routes = sorted(self._routes, key=key_func) groups = itertools.groupby(routes, key=key_func) for url, group in groups: view = self._make_view(list(group)) urls_map[url] = path(url, view) # keep urls same order with self._routes # and constant url should priority then path argument urls = [] urls_priority = [] urls_added = set() for f, url, methods, params, returns in self._routes: if url not in urls_added: urls_added.add(url) if '<' in url and ':' in url and '>' in url: urls.append(urls_map[url]) else: urls_priority.append(urls_map[url]) return urls_priority + urls @staticmethod def _response_from_invalid(ex): return Response( { 'description': str(ex), 'position': ex.position, 'message': ex.message, 'field': ex.field, 'value': ex.value, }, status=400) @staticmethod def _make_method(method, f, params, returns): def rest_method(self, request, format=None, **kwargs): ret = None validr_cost = 0 if params is not None: maps = [kwargs] if request.method in ['GET', 'DELETE']: maps.append(request.query_params) else: maps.append(request.data) t_begin = time.time() try: kwargs = params(ChainMap(*maps)) except Invalid as ex: ret = RestRouter._response_from_invalid(ex) validr_cost += time.time() - t_begin t_begin = time.time() if ret is None: ret = f(request, **kwargs) api_cost = time.time() - t_begin if returns is not None: if not isinstance(ret, (Response, HttpResponse)): t_begin = time.time() ret = returns(ret) validr_cost += time.time() - t_begin ret = Response(ret) elif ret is None: ret = Response(status=204) if validr_cost > 0: ret['X-Validr-Time'] = '{:.0f}ms'.format(validr_cost * 1000) if api_cost > 0: ret['X-API-Time'] = '{:.0f}ms'.format(api_cost * 1000) return ret rest_method.__name__ = method.lower() rest_method.__qualname__ = method.lower() rest_method.__doc__ = f.__doc__ return rest_method def _make_view(self, group): method_maps = {} method_meta = {} for f, url, methods, params, returns in group: for method in methods: if method in method_maps: raise ValueError(f'duplicated method {method} of {url}') m = self._make_method(method, f, params, returns) method_maps[method] = m method_meta[method] = f, url, params, returns class RestApiView(APIView): if self.permission_classes: permission_classes = self.permission_classes schema = RestViewSchema(method_meta) if 'GET' in method_maps: get = method_maps['GET'] if 'POST' in method_maps: post = method_maps['POST'] if 'PUT' in method_maps: put = method_maps['PUT'] if 'DELETE' in method_maps: delete = method_maps['DELETE'] if 'PATCH' in method_maps: patch = method_maps['PATCH'] return RestApiView.as_view() def _route(self, url, methods): if isinstance(methods, str): methods = set(methods.strip().replace(',', ' ').split()) else: methods = set(methods) methods = set(x.upper() for x in methods) def wrapper(f): params = get_params(f) if params is not None: params = self._schema_compiler.compile(params) returns = get_returns(f) if returns is not None: returns = self._schema_compiler.compile(returns) self._routes.append((f, url, methods, params, returns)) return f return wrapper def get(self, url=''): return self._route(url, methods='GET') def post(self, url=''): return self._route(url, methods='POST') def put(self, url=''): return self._route(url, methods='PUT') def delete(self, url=''): return self._route(url, methods='DELETE') def patch(self, url=''): return self._route(url, methods='PATCH') def route(self, url='', methods='GET'): return self._route(url, methods=methods) __call__ = route
def __init__(self): super().__init__() self._schema_compiler = Compiler(validators=VALIDATORS)
def __init__( self, actors, host='0.0.0.0', port=8000, concurrency=100, name=None, subpath=None, networks=None, registery_node_spec=None, storage_dir_path=None, storage_compact_wal_delta=1000, queue_max_complete_size=1000, max_retry_time=10 * 60, max_retry_count=3, token=None, schema_compiler=None, on_startup=None, on_shutdown=None, ): if schema_compiler is None: schema_compiler = Compiler() self.schema_compiler = schema_compiler self.actors = {} self.timers = {} for handler in chain(actors, BUILTIN_ACTORS): if inspect.isclass(handler): handler = handler(self) x = Actor(handler, schema_compiler=schema_compiler) if x.timer is not None: self.timers[x.name] = x self.actors[x.name] = x actor_modules = {x.module for x in self.actors.values()} if not name: name = f'actor-{port}' self.name = name if not networks: networks = [] networks.append(get_localhost_network(port=port, subpath=subpath)) current_node_spec = dict( name=self.name, modules=actor_modules, networks=networks, ) self.token = token self.registery = ActorRegistery( current_node_spec=current_node_spec, registery_node_spec=registery_node_spec) if storage_dir_path: storage_dir_path = os.path.abspath(os.path.expanduser(storage_dir_path)) storage_path = os.path.join(storage_dir_path, self.name) self.storage = ActorLocalStorage( dirpath=storage_path, compact_wal_delta=storage_compact_wal_delta) else: self.storage = None self.queue = ActorMessageQueue( registery=self.registery, actors=self.actors, storage=self.storage, max_complete_size=queue_max_complete_size, concurrency=concurrency, max_retry_count=max_retry_count, max_retry_time=max_retry_time, ) self.concurrency = concurrency self.executor = ActorExecutor( self.actors, queue=self.queue, registery=self.registery, concurrency=concurrency, token=self.token, ) self.host = host self.port = port Info("actor_node", "actor node info").info({'name': self.name}) self.subpath = subpath or '' self.receiver = MessageReceiver( host=self.host, port=self.port, subpath=self.subpath, queue=self.queue, registery=self.registery, token=self.token) self._client = None self._on_startup_handlers = [] self._on_shutdown_handlers = [] if on_startup: self._on_startup_handlers.extend(on_startup) if on_shutdown: self._on_shutdown_handlers.extend(on_shutdown)
except ImportError: return try: import rlcompleter except ImportError: return readline.set_completer(rlcompleter.Completer(context).complete) readline.parse_and_bind("tab:complete") # command history if os.path.exists(HISTORY_PATH): readline.read_history_file(HISTORY_PATH) atexit.register(_save_history) HeadersSchema = T.list(T.dict(name=T.str, value=T.str)) validate_headers = Compiler().compile(HeadersSchema) class Shell: def __init__(self, app): self.app = app headers = self._load_headers() self._client = Client(app, headers=headers) def _load_headers(self): if not os.path.exists(HEADERS_PATH): return None LOG.info(f"Load headers from {HEADERS_PATH!r}") try: with open(HEADERS_PATH) as f: data = json.load(f)
def main(): service = Service(_DocumentSampleService, {}, [], Compiler()) print(format_service_doc(service)) for method in service.methods: print('-' * 60) print(format_method_doc(method))
from validr import T, Compiler, modelclass, asdict @modelclass class Model: user = T.dict(userid=T.int.min(0).max(9).desc("UserID")) tags = T.list(T.int.min(0)) style = T.dict( width=T.int.desc("width"), height=T.int.desc("height"), border_width=T.int.desc("border_width"), border_style=T.str.desc("border_style"), border_color=T.str.desc("border_color"), color=T.str.desc("color"), ) optional = T.str.optional.desc("unknown value") compiler = Compiler() default = compiler.compile(T(Model)) def model(value): return asdict(Model(value)) CASES = {"default": default, "model": model}
def test_invalid_default(): with pytest.raises(SchemaError): Compiler().compile(T.int.default('abc'))
def decorator(predictor_cls): s = T(schema) predictor_cls._schema_func = Compiler().compile(s) return predictor_cls