def setUp(self): V.Object.REQUIRED_PROPERTIES = True V.base.reset_type_names() self.complex_validator = self.parse({ "n": "number", "?i": V.Nullable("integer", 0), "?b": bool, "?e": V.Enum(["r", "g", "b"]), "?d": V.AnyOf("date", "datetime"), "?s": V.String(min_length=1, max_length=8), "?p": V.Nullable(re.compile(r"\d{1,4}$")), "?l": [{ "+s2": "string" }], "?t": (unicode, "number"), "?h": V.Mapping(int, ["string"]), "?o": V.NonNullable({"+i2": "integer"}), })
def test_parsing_required_properties(self): get_schema = lambda: { "foo": V.Nullable("number"), "?nested": [V.Nullable({"baz": "string"})] } valid = [{"foo": 3, "nested": [None]}] missing_properties = [{}, {"foo": 3, "nested": [{}]}] for _ in xrange(3): with V.parsing(required_properties=False): self._testValidation(get_schema(), valid=valid + missing_properties) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=valid, invalid=missing_properties) # gotcha: calling parse() with required_properties=True is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. if V.Object.REQUIRED_PROPERTIES: self._testValidation(V.parse(get_schema(), required_properties=True), invalid=[missing_properties[1]]) else: self._testValidation(V.parse(get_schema(), required_properties=True), valid=[missing_properties[1]])
def __init__(self, allow_extra): schema = { '+id': int, '+client_name': V.String(max_length=255), '+sort_index': float, 'client_phone': V.Nullable(V.String(max_length=255)), 'location': {'latitude': float, 'longitude': float}, 'contractor': V.Range(V.AdaptTo(int), min_value=1), 'upstream_http_referrer': V.Nullable(V.String(max_length=1023)), '+grecaptcha_response': V.String(min_length=20, max_length=1000), 'last_updated': V.AdaptBy(dateutil.parser.parse), 'skills': V.Nullable( [ { '+subject': str, '+subject_id': int, '+category': str, '+qual_level': str, '+qual_level_id': int, 'qual_level_ranking': V.Nullable(float, default=0), } ], default=[], ), } self.validator = V.parse(schema, additional_properties=allow_extra)
def setUp(self): super(OptionalPropertiesTestValidator, self).setUp() V.Object.REQUIRED_PROPERTIES = False self.complex_validator = self.parse({ "+n": "+number", "i": V.Nullable("integer", 0), "b": bool, "e": V.Enum(["r", "g", "b"]), "d": V.AnyOf("date", "datetime"), "s": V.String(min_length=1, max_length=8), "p": V.Nullable(re.compile(r"\d{1,4}$")), "l": [{ "+s2": "string" }], "t": (unicode, "number"), "h": V.Mapping(int, ["string"]), "o": V.NonNullable({"+i2": "integer"}), })
def test_nullable_with_default(self): self._testValidation(V.Nullable("integer", -1), adapted=[(None, -1), (0, 0)], invalid=[1.1, True, False]) self._testValidation(V.Nullable("integer", lambda: -1), adapted=[(None, -1), (0, 0)], invalid=[1.1, True, False])
def test_nullable(self): for obj in "?integer", V.Nullable(V.Integer()), V.Nullable("+integer"): self._testValidation(obj, valid=[None, 0], invalid=[1.1, True, False]) self._testValidation(V.Nullable(["?string"]), valid=[None, [], ["foo"], [None], ["foo", None]], invalid=["", [None, "foo", 1]])
def inner(env, *a, **kw): schema = v.parse({ 'page': v.Nullable(v.AdaptTo(int), 1), 'last': v.Nullable(str) }) data = schema.validate(env.request.args) page, last = data['page'], data.get('last') page = { 'limit': env('ui_per_page'), 'offset': env('ui_per_page') * (page - 1), 'last': last, 'count': env('ui_per_page') * page, 'current': page, 'next': page + 1, } return wrapper.func(env, page, *a, **kw)
def test_parsing_additional_properties(self): get_schema = lambda: { "?bar": "boolean", "?nested": [V.Nullable({"?baz": "integer"})] } values = [{"x1": "yes"}, {"bar": True, "nested": [{"x1": "yes"}]}] for _ in xrange(3): with V.parsing(additional_properties=True): self._testValidation(get_schema(), valid=values) with V.parsing(additional_properties=False): self._testValidation(get_schema(), invalid=values) # gotcha: calling parse() with additional_properties=False is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. # The 'additional_properties' parameter effectively is applied at # the top level dict only self._testValidation(V.parse(get_schema(), additional_properties=False), invalid=values[:1], valid=values[1:]) with V.parsing(additional_properties=V.Object.REMOVE): self._testValidation(get_schema(), adapted=[(values[0], {}), (values[1], { "bar": True, "nested": [{}] })]) # same gotcha as above self._testValidation(V.parse( get_schema(), additional_properties=V.Object.REMOVE), adapted=[(values[0], {}), (values[1], values[1])]) with V.parsing(additional_properties="string"): self._testValidation(get_schema(), valid=values, invalid=[{ "x1": 42 }, { "bar": True, "nested": [{ "x1": 42 }] }]) # same gotcha as above self._testValidation(V.parse(get_schema(), additional_properties="string"), invalid=[{ "x1": 42 }], valid=[{ "bar": True, "nested": [{ "x1": 42 }] }])
def test_nested_parsing(self): get_schema = lambda: { "bar": "integer", "?nested": [V.Nullable({"baz": "number"})] } values = [ { "bar": 1 }, { "bar": 1, "nested": [{ "baz": 0 }, None] }, { "bar": 1, "xx": 2 }, { "bar": 1, "nested": [{ "baz": 2.1, "xx": 1 }] }, {}, { "bar": 1, "nested": [{}] }, ] if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) with V.parsing(additional_properties=False): self._testValidation(get_schema(), valid=values[:2], invalid=values[2:]) self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values)
class BuildMessage(BaseMessage): project = attr.ib() delay = attr.ib() incremental = attr.ib(default=None) _validator = V.parse({ "project": "string", "delay": "number", "?incremental": V.Nullable("boolean") })
def test_adapt_missing_property(self): self._testValidation( { "foo": "number", "?bar": V.Nullable("boolean", False) }, adapted=[({ "foo": -12 }, { "foo": -12, "bar": False })])
def mark(env): def name(value): if isinstance(value, str): value = [value] return [v for v in value if v] schema = v.parse({ '+action': v.Enum(('+', '-', '=')), '+name': v.AdaptBy(name), '+ids': [int], 'old_name': v.AdaptBy(name), 'thread': v.Nullable(bool, False), 'last': v.Nullable(str) }) data = schema.validate(env.request.json) if not data['ids']: return 'OK' ids = tuple(data['ids']) if data['thread']: i = env.sql(''' SELECT id FROM emails WHERE thrid IN %s AND created <= %s ''', [ids, data['last']]) ids = tuple(r[0] for r in i) mark = ft.partial(syncer.mark, env, ids=ids, new=True) if data['action'] == '=': if data.get('old_name') is None: raise ValueError('Missing parameter "old_name" for %r' % data) if data['old_name'] == data['name']: return [] mark('-', set(data['old_name']) - set(data['name'])) mark('+', set(data['name']) - set(data['old_name'])) return 'OK' mark(data['action'], data['name']) return 'OK'
def test_nullable_with_default_object_property(self): class ObjectNullable(V.Nullable): default_object_property = property(lambda self: self.default) regular_nullables = [ "?integer", V.Nullable("integer"), V.Nullable("integer", None), V.Nullable("integer", default=None), V.Nullable("integer", lambda: None), V.Nullable("integer", default=lambda: None) ] for obj in regular_nullables: self._testValidation({"?foo": obj}, adapted=[({}, {})]) object_nullables = [ ObjectNullable("integer"), ObjectNullable("integer", None), ObjectNullable("integer", default=None), ObjectNullable("integer", lambda: None), ObjectNullable("integer", default=lambda: None), ] for obj in object_nullables: self._testValidation({"?foo": obj}, adapted=[({}, {"foo": None})])
class ArtifactMessage(BaseMessage): project = attr.ib() artifact_type = attr.ib() artifact = attr.ib() success = attr.ib() filename = attr.ib(default=None) last_hash = attr.ib(default=None) _validator = V.parse({ "project": "string", "artifact_type": V.Enum({"tool", "package", "file"}), "artifact": "string", # TODO(arsen): architecture "success": "boolean", "?filename": "?string", "?last_hash": V.Nullable(_is_blake2b_digest) })
def emails(env, page): schema = v.parse({'q': v.Nullable(str, '')}) q = schema.validate(env.request.args)['q'] ctx = {'labels': ['\\All'], 'by_thread': False} if q.startswith('g! '): # Gmail search ids = syncer.search(env, env.email, q[3:]) select_ids = env.mogrify(''' (SELECT * FROM unnest(%s::bigint[][])) AS ids(id) ''', [ids]) else: select_ids, ctx = parse_query(env, q, page) select_ids = '(%s) AS ids' % select_ids if ctx['by_thread']: res = threads(env, select_ids, ctx, page) else: i = env.sql(''' SELECT e.id, thrid, subj, labels, time, fr, "to", text, cc, created, html, attachments, parent FROM emails e JOIN {select_ids} ON e.id = ids.id ORDER BY {order_by} LIMIT {page[limit]} OFFSET {page[offset]} '''.format( select_ids=select_ids, page=page, order_by=ctx.get('order_by', 'id DESC') )) def emails(): for msg in i: msg = dict(msg) yield msg res = ctx_emails(env, emails()) res['labels'] = ctx_labels(env, ctx['labels']) from . import log log.info('keywords=%(keywords)s', ctx) if ctx['keywords'] != {'in'} and 'thr' not in ctx['keywords']: q += ' thr:0' res['search_query'] = q return res
def test_humanized_names(self): class DummyValidator(V.Validator): name = "dummy" def validate(self, value, adapt=True): return value self.assertEqual(DummyValidator().humanized_name, "dummy") self.assertEqual( V.Nullable(DummyValidator()).humanized_name, "dummy or null") self.assertEqual( V.AnyOf("boolean", DummyValidator()).humanized_name, "boolean or dummy") self.assertEqual( V.AllOf("boolean", DummyValidator()).humanized_name, "boolean and dummy") self.assertEqual( V.ChainOf("boolean", DummyValidator()).humanized_name, "boolean chained to dummy") self.assertEqual(Date().humanized_name, "date or datetime")
def get_conf(conf=None): if not conf: with open('conf.json', 'br') as f: conf = json.loads(f.read().decode()) exists = v.Condition(lambda v: Path(v).exists()) strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/')) app_dir = Path(__file__).parent.resolve() base_dir = app_dir.parent log_handlers = ['console_simple', 'console_detail', 'file'] with v.parsing(additional_properties=False): schema = v.parse({ 'debug': v.Nullable(bool, False), '+pg_username': str, '+pg_password': str, '+cookie_secret': str, 'google_id': str, 'google_secret': str, 'readonly': v.Nullable(bool, True), 'enabled': v.Nullable(bool, True), 'log_handlers': (v.Nullable([v.Enum(log_handlers)], log_handlers[:1])), 'log_level': v.Nullable(str, 'DEBUG'), 'log_file': v.Nullable(str, ''), 'path_attachments': v.Nullable(str, str(base_dir / 'attachments')), 'path_theme': v.Nullable(exists, str(base_dir / 'front')), 'imap_body_maxsize': v.Nullable(int, 50 * 1024 * 1024), 'imap_batch_size': v.Nullable(int, 2000), 'imap_debug': v.Nullable(int, 0), 'smtp_debug': v.Nullable(bool, False), 'async_pool': v.Nullable(int, 0), 'ui_ga_id': v.Nullable(str, ''), 'ui_is_public': v.Nullable(bool, False), 'ui_use_names': v.Nullable(bool, True), 'ui_per_page': v.Nullable(int, 100), 'ui_greeting': v.Nullable(str, ''), 'ui_ws_proxy': v.Nullable(bool, False), 'ui_ws_enabled': v.Nullable(bool, True), 'ui_ws_timeout': v.Nullable(int, 1000), 'ui_firebug': v.Nullable(bool, False), 'ui_tiny_thread': v.Nullable(int, 5), 'ui_by_thread': v.Nullable(bool, False), 'from_emails': v.Nullable([str], []), 'host_ws': v.Nullable(str, 'ws://localhost/async/'), 'host_web': v.Nullable(strip_slash, 'http://localhost:8000'), 'search_lang': v.Nullable([str], ['simple', 'english']), }) conf = schema.validate(conf) path = Path(conf['path_attachments']) if not path.exists(): path.mkdir() return conf
class Results(object): __slots__ = ("navigator", "_query", "period", "_results", "_speed") def __init__(self, navigator, query, period): self.navigator = navigator self._query = query self.period = period self._results = False self._speed = None @property def results(self): if self._results is False: start = time() self._results = self.navigator.query( self._query.replace('__into__', '')) self._speed = (time() - start) * 1000 return self._results def refresh(self): self._results = False self.results return self def json(self, debug=False): results = self.results result = dict(results=results, meta={ "status": 200, "total": len(results), "speed": "%.fms" % self._speed }) if self.period: result["meta"]["time"] = { "start": str(self.period.start), "end": str(self.period.end) } if debug: result['meta']['query'] = self.pg() return dumps(result, default=json_defaults) @valideer.accepts(into=valideer.Nullable( valideer.Pattern(r"^[a-zA-Z\_]{1,25}$"))) def pg(self, into=None): return self._query.replace("__into__", (" INTO " + (into or '')) if into else '') @property def value(self): results = self.results if results and len(results) == 1: return list(self)[0] else: return results def __str__(self): return str(self.value) def __nonzero__(self): return bool(self.results) @valideer.accepts(index="integer") def __getitem__(self, index): return self.results[index] def __getattr__(self, index): if len(self.results) == 1: return self.navigator.format(index, self.results[0][index]) else: raise ValueError("Cannot get attr from list") def __iter__(self): """Returns the results with python objects inserted """ results = self.results if results: for row in iter(results): yield dict([(key, self.navigator.format(key, value)) for key, value in row.iteritems()]) def __len__(self): results = self.results return len(results) if results else 0 def __cmp__(self, other): results = self.results if len(results) == 1: x = results[0][results[0].keys()[0]] return 0 if other == x else -1 if other > x else 1 else: return False
import requests import toml import valideer as V import yaml import zmq.green as zmq from logbook import Logger, StderrHandler, StreamHandler import xbbs.messages as msgs import xbbs.util as xutils with V.parsing(required_properties=True, additional_properties=None): CONFIG_VALIDATOR = V.parse({ "job_endpoint": xutils.Endpoint(xutils.Endpoint.Side.BIND), "capabilities": V.Nullable(V.AdaptBy(xutils.list_to_set), set()), }) @attr.s class XbbsWorker: current_project = attr.ib(default=None) current_job = attr.ib(default=None) zmq = attr.ib(default=zmq.Context.instance()) def download(url, to): src = urlparse(url, scheme='file') if src.scheme == 'file': shutil.copy(src.path, to) else:
def draft(env, thrid, action): saved = env.storage('compose', thrid=thrid) saved_path = env.files.subpath('compose', thrid=thrid) if action == 'preview': schema = v.parse({ '+fr': str, '+to': str, '+subj': str, '+body': str, '+quoted': bool, '+forward': bool, '+id': v.Nullable(str), 'quote': v.Nullable(str) }) data = schema.validate(env.request.json) if env.request.args.get('save', False): saved.set(data) return get_html(data['body'], data.get('quote', '')) elif action == 'upload': count = env.request.form.get('count', type=int) files = [] for n, i in enumerate(env.request.files.getlist('files'), count): path = '/'.join([saved_path, str(n), f.slugify(i.filename)]) env.files.write(path, i.stream.read()) files.append(env.files.to_dict(path, i.mimetype, i.filename)) return files elif action == 'send': import dns.resolver import dns.exception class Email(v.Validator): def validate(self, value, adapt=True): if not value: raise v.ValidationError('No email') addr = parseaddr(value)[1] hostname = addr[addr.find('@') + 1:] try: dns.resolver.query(hostname, 'MX') except dns.exception.DNSException: raise v.ValidationError('No MX record for %s' % hostname) return value schema = v.parse({ '+to': v.ChainOf( v.AdaptBy(lambda v: [i.strip() for i in v.split(',')]), [Email] ), '+fr': Email, '+subj': str, '+body': str, 'id': v.Nullable(str), 'quote': v.Nullable(str, ''), }) msg = schema.validate(env.request.json) if msg.get('id'): parent = env.sql(''' SELECT thrid, msgid, refs FROM emails WHERE id=%s LIMIT 1 ''', [msg['id']]).fetchone() msg['in_reply_to'] = parent.get('msgid') msg['refs'] = parent.get('refs', [])[-10:] else: parent = {} sendmail(env, msg) if saved.get(): draft(env, thrid, 'rm') syncer.sync_gmail(env, env.email, only=['\\All'], fast=1, force=1) url = url_query(env, 'in', '\\Sent') if parent.get('thrid'): url = env.url_for('thread', {'id': parent['thrid']}) return {'url': url} elif action == 'rm': if saved.get({}).get('files'): env.files.rm(saved_path) saved.rm() return 'OK' env.abort(400)
def compose(env, id=None): if not env.storage.get('gmail_info'): return env.abort(400) schema = v.parse({ 'target': v.Nullable(v.Enum(('all', 'forward'))) }) args = schema.validate(env.request.args) fr = env.from_emails[0] ctx = { 'fr': fr, 'to': '', 'subj': '', 'body': '', 'files': [], 'quoted': False, 'forward': False, 'id': id, 'draft': False, 'from_emails': env.from_emails } parent = {} if id: parent = env.sql(''' SELECT thrid, "to", fr, cc, bcc, subj, reply_to, html, time, attachments, embedded FROM emails WHERE id=%s LIMIT 1 ''', [id]).fetchone() to_all = parent['to'][:] + parent['cc'][:] fr = env.from_email(parent['fr']) if fr: to = to_all else: fr_ = env.from_email(to_all) if fr_: fr = fr_ to_all = [ a for a in to_all if parseaddr(a)[1] != parseaddr(fr)[1] ] to = (parent['reply_to'] or parent['fr'])[:] forward = args.get('target') == 'forward' if forward: to = [] elif args.get('target') == 'all': to += to_all ctx.update({ 'fr': fr, 'to': ', '.join(to), 'subj': 'Re: %s' % f.humanize_subj(parent['subj'], empty=''), 'quote': ctx_quote(env, parent, forward), 'quoted': forward, 'forward': forward, }) thrid = parent.get('thrid') saved = env.storage('compose', thrid=thrid) saved_path = env.files.subpath('compose', thrid=thrid) if saved.get(): ctx.update(saved.get()) ctx['draft'] = saved.get() is not None ctx['title'] = ctx.get('subj') or 'New message' if ctx['forward'] and not ctx['draft']: env.files.copy(f.slugify(id), saved_path) files = list(parent['attachments']) + list(parent['embedded'].values()) for i in files: path = i['path'].replace(id, saved_path) asset = env.files.to_dict(**dict(i, path=path)) ctx['files'].append(asset) quote = ctx.get('quote') if quote: parent_url = re.escape(env.files.url(i['path'])) ctx['quote'] = re.sub(parent_url, asset['url'], quote) ctx['links'] = { a: env.url_for('draft', {'thrid': str(thrid or 'new'), 'action': a}) for a in ('preview', 'rm', 'send', 'upload') } return ctx
"string", "build_root": V.AllOf("string", path.isabs), "intake": V.AdaptBy(_receive_adaptor), "worker_endpoint": xutils.Endpoint(xutils.Endpoint.Side.BIND), # use something like a C identifier, except disallow underscore as a # first character too. this is so that we have a namespace for xbbs # internal directories, such as collection directories "projects": V.Mapping( xutils.PROJECT_REGEX, { "git": "string", "?description": "string", "?classes": V.Nullable(["string"], []), "packages": "string", "?fingerprint": "string", "tools": "string", "?incremental": "boolean", "?distfile_path": "string", "?mirror_root": "string", "?default_branch": "string", }) }) PUBKEY_VALIDATOR = V.parse({ # I'm only validating the keys that xbbs uses "signature-by": "string" }) with V.parsing(required_properties=True, additional_properties=None):
def test_adapts(self): @V.adapts( body={ "+field_ids": ["integer"], "?scores": V.Mapping("string", float), "?users": [{ "+name": ("+string", "+string"), "?sex": "gender", "?active": V.Nullable("boolean", True), }] }) def f(body): return body adapted = f({ "field_ids": [1, 5], "scores": { "foo": 23.1, "bar": 2.0 }, "users": [ { "name": ("Nick", "C"), "sex": "male" }, { "name": ("Kim", "B"), "active": False }, { "name": ("Joe", "M"), "active": None }, ] }) self.assertEqual(adapted["field_ids"], [1, 5]) self.assertEqual(adapted["scores"]["foo"], 23.1) self.assertEqual(adapted["scores"]["bar"], 2.0) self.assertEqual(adapted["users"][0]["name"], ("Nick", "C")) self.assertEqual(adapted["users"][0]["sex"], "male") self.assertEqual(adapted["users"][0]["active"], True) self.assertEqual(adapted["users"][1]["name"], ("Kim", "B")) self.assertEqual(adapted["users"][1].get("sex"), None) self.assertEqual(adapted["users"][1]["active"], False) self.assertEqual(adapted["users"][2]["name"], ("Joe", "M")) self.assertEqual(adapted["users"][2].get("sex"), None) self.assertEqual(adapted["users"][2].get("active"), True) invalid = [ # missing 'field_ids' from body partial(f, {}), # score value is not float partial(f, { "field_ids": [], "scores": { "a": "2.3" } }), # 'name' is not a length-2 tuple partial(f, { "field_ids": [], "users": [{ "name": ("Bob", "R", "Junior") }] }), # name[1] is not a string partial(f, { "field_ids": [], "users": [{ "name": ("Bob", 12) }] }), # name[1] is required partial(f, { "field_ids": [], "users": [{ "name": ("Bob", None) }] }), ] for fcall in invalid: self.assertRaises(V.ValidationError, fcall)