def test_parsing_required_properties(self): get_schema = lambda: { "foo": V.Nullable("number"), "?nested": [V.Nullable({"baz": "string"})] } valid = [{"foo": 3, "nested": [None]}] missing_properties = [{}, {"foo": 3, "nested": [{}]}] for _ in xrange(3): with V.parsing(required_properties=False): self._testValidation(get_schema(), valid=valid + missing_properties) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=valid, invalid=missing_properties) # gotcha: calling parse() with required_properties=True is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. if V.Object.REQUIRED_PROPERTIES: self._testValidation(V.parse(get_schema(), required_properties=True), invalid=[missing_properties[1]]) else: self._testValidation(V.parse(get_schema(), required_properties=True), valid=[missing_properties[1]])
def test_parsing_required_properties(self): get_schema = lambda: { "foo": V.Nullable("number"), "?nested": [V.Nullable({ "baz": "string" })] } valid = [{"foo": 3, "nested": [None]}] missing_properties = [{}, {"foo": 3, "nested": [{}]}] for _ in xrange(3): with V.parsing(required_properties=False): self._testValidation(get_schema(), valid=valid + missing_properties) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=valid, invalid=missing_properties) # gotcha: calling parse() with required_properties=True is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. if V.Object.REQUIRED_PROPERTIES: self._testValidation(V.parse(get_schema(), required_properties=True), invalid=[missing_properties[1]]) else: self._testValidation(V.parse(get_schema(), required_properties=True), valid=[missing_properties[1]])
def test_nested_parsing(self): get_schema = lambda: { "bar": "integer", "?nested": [V.Nullable({"baz": "number"})] } values = [ { "bar": 1 }, { "bar": 1, "nested": [{ "baz": 0 }, None] }, { "bar": 1, "xx": 2 }, { "bar": 1, "nested": [{ "baz": 2.1, "xx": 1 }] }, {}, { "bar": 1, "nested": [{}] }, ] if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) with V.parsing(additional_properties=False): self._testValidation(get_schema(), valid=values[:2], invalid=values[2:]) self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values)
def test_parsing_additional_properties(self): get_schema = lambda: { "?bar": "boolean", "?nested": [V.Nullable({"?baz": "integer"})] } values = [{"x1": "yes"}, {"bar": True, "nested": [{"x1": "yes"}]}] for _ in xrange(3): with V.parsing(additional_properties=True): self._testValidation(get_schema(), valid=values) with V.parsing(additional_properties=False): self._testValidation(get_schema(), invalid=values) # gotcha: calling parse() with additional_properties=False is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. # The 'additional_properties' parameter effectively is applied at # the top level dict only self._testValidation(V.parse(get_schema(), additional_properties=False), invalid=values[:1], valid=values[1:]) with V.parsing(additional_properties=V.Object.REMOVE): self._testValidation(get_schema(), adapted=[(values[0], {}), (values[1], { "bar": True, "nested": [{}] })]) # same gotcha as above self._testValidation(V.parse( get_schema(), additional_properties=V.Object.REMOVE), adapted=[(values[0], {}), (values[1], values[1])]) with V.parsing(additional_properties="string"): self._testValidation(get_schema(), valid=values, invalid=[{ "x1": 42 }, { "bar": True, "nested": [{ "x1": 42 }] }]) # same gotcha as above self._testValidation(V.parse(get_schema(), additional_properties="string"), invalid=[{ "x1": 42 }], valid=[{ "bar": True, "nested": [{ "x1": 42 }] }])
def get_conf(conf=None): if not conf: with open('conf.json', 'br') as f: conf = json.loads(f.read().decode()) exists = v.Condition(lambda v: Path(v).exists()) strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/')) app_dir = Path(__file__).parent.resolve() base_dir = app_dir.parent log_handlers = ['console_simple', 'console_detail', 'file'] with v.parsing(additional_properties=False): schema = v.parse({ 'debug': v.Nullable(bool, False), '+pg_username': str, '+pg_password': str, '+cookie_secret': str, 'google_id': str, 'google_secret': str, 'readonly': v.Nullable(bool, True), 'enabled': v.Nullable(bool, True), 'log_handlers': ( v.Nullable([v.Enum(log_handlers)], log_handlers[:1]) ), 'log_level': v.Nullable(str, 'DEBUG'), 'log_file': v.Nullable(str, ''), 'path_attachments': v.Nullable(str, str(base_dir / 'attachments')), 'path_theme': v.Nullable(exists, str(base_dir / 'front')), 'imap_body_maxsize': v.Nullable(int, 50 * 1024 * 1024), 'imap_batch_size': v.Nullable(int, 2000), 'imap_debug': v.Nullable(int, 0), 'smtp_debug': v.Nullable(bool, False), 'async_pool': v.Nullable(int, 0), 'ui_ga_id': v.Nullable(str, ''), 'ui_is_public': v.Nullable(bool, False), 'ui_use_names': v.Nullable(bool, True), 'ui_per_page': v.Nullable(int, 100), 'ui_greeting': v.Nullable(str, ''), 'ui_ws_proxy': v.Nullable(bool, False), 'ui_ws_enabled': v.Nullable(bool, True), 'ui_ws_timeout': v.Nullable(int, 1000), 'ui_firebug': v.Nullable(bool, False), 'ui_tiny_thread': v.Nullable(int, 5), 'ui_by_thread': v.Nullable(bool, False), 'from_emails': v.Nullable([str], []), 'host_ws': v.Nullable(str, 'ws://localhost/async/'), 'host_web': v.Nullable(strip_slash, 'http://localhost:8000'), 'search_lang': v.Nullable([str], ['simple', 'english']), }) conf = schema.validate(conf) path = Path(conf['path_attachments']) if not path.exists(): path.mkdir() return conf
def test_parsing_additional_properties(self): get_schema = lambda: { "?bar": "boolean", "?nested": [V.Nullable({ "?baz": "integer" })] } values = [{"x1": "yes"}, {"bar": True, "nested": [{"x1": "yes"}]}] for _ in xrange(3): with V.parsing(additional_properties=True): self._testValidation(get_schema(), valid=values) with V.parsing(additional_properties=False): self._testValidation(get_schema(), invalid=values) # gotcha: calling parse() with additional_properties=False is not # equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. # The 'additional_properties' parameter effectively is applied at # the top level dict only self._testValidation(V.parse(get_schema(), additional_properties=False), invalid=values[:1], valid=values[1:]) with V.parsing(additional_properties=V.Object.REMOVE): self._testValidation(get_schema(), adapted=[(values[0], {}), (values[1], {"bar": True, "nested": [{}]})]) # same gotcha as above self._testValidation(V.parse(get_schema(), additional_properties=V.Object.REMOVE), adapted=[(values[0], {}), (values[1], values[1])]) with V.parsing(additional_properties="string"): self._testValidation(get_schema(), valid=values, invalid=[{"x1": 42}, {"bar": True, "nested": [{"x1": 42}]}]) # same gotcha as above self._testValidation(V.parse(get_schema(), additional_properties="string"), invalid=[{"x1": 42}], valid=[{"bar": True, "nested": [{"x1": 42}]}])
def test_parsing_ignore_optional_property_errors(self): get_schema = lambda: V.Nullable({ "+foo": "number", "?bar": "boolean", "?nested": [{ "+baz": "string", "?zoo": "number", }] }) invalid_required = [ {"foo": "2", "bar": True}, ] invalid_optional = [ {"foo": 3, "bar": "nan"}, {"foo": 3.1, "nested": [{"baz": "x", "zoo": "12"}]}, {"foo": 0, "nested": [{"baz": 1, "zoo": 2}]}, ] adapted = [ {"foo": 3}, {"foo": 3.1, "nested": [{"baz": "x"}]}, {"foo": 0}, ] for _ in xrange(3): with V.parsing(ignore_optional_property_errors=False): self._testValidation(get_schema(), invalid=invalid_required + invalid_optional) with V.parsing(ignore_optional_property_errors=True): self._testValidation(get_schema(), invalid=invalid_required, adapted=zip(invalid_optional, adapted)) # gotcha: calling parse() with ignore_optional_property_errors=True # is not equivalent to the above call because the V.Nullable() calls in # get_schema have already called implicitly parse() without parameters. self._testValidation(V.parse(get_schema(), ignore_optional_property_errors=False), invalid=invalid_required + invalid_optional) self._testValidation(V.parse(get_schema(), ignore_optional_property_errors=True), invalid=invalid_required + invalid_optional)
def testGetNotes(self, apikey): repl = self.api.getNotes(66, apikey) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_NOTES) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e)))
def testGetRequests(self): print("Testing GET requests"), repl = self.cache_requests expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTS) replFields = self.xmlToDict(repl.content) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e)))
def testGetServices(self): print("Testing GET services") repl = self.cache_services expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_SERVICE) replFields = self.xmlToDict(repl.content) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs %s " % (str(e)))
def testRequestsExtKS(self): print("Testing GET extended requests"), repl = self.api.getRequests() expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTSEXT) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e)))
def test_nested_parsing(self): get_schema = lambda: { "bar": "integer", "?nested": [V.Nullable({ "baz": "number" })] } values = [ {"bar": 1}, {"bar": 1, "nested": [{"baz": 0}, None]}, {"bar": 1, "xx": 2}, {"bar": 1, "nested": [{"baz": 2.1, "xx": 1}]}, {}, {"bar": 1, "nested": [{}]}, ] if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values) with V.parsing(required_properties=True): self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) with V.parsing(additional_properties=False): self._testValidation(get_schema(), valid=values[:2], invalid=values[2:]) self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) if V.Object.REQUIRED_PROPERTIES: self._testValidation(get_schema(), valid=values[:4], invalid=values[4:]) else: self._testValidation(get_schema(), valid=values)
def testGetServicsDef(self): #TODO: walk trough all definitions print("Testing GET service definition"), firstCode = self.__getFirstServiceCode() if firstCode is not None: repl = self.api.getServiceDef(firstCode) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_DEF) replFields = self.xmlToDict(repl) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) else: print "(No service definitions available for testing)"
def testGetServicsDef(self): #TODO: walk trough all definitions print("Testing GET service definition"), firstCode = self.__getFirstServiceCode() if firstCode is not None: repl = self.api.getServiceDef(firstCode) expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [ repl.status_code, repl.encoding, repl.headers["content-type"] ] self.showDifferences(expect, resp, "http skelleton") print("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_DEF) replFields = self.xmlToDict(repl) try: validator.validate(replFields) print("OK") except ValidationError as e: print("differs at %s " % (str(e))) else: print "(No service definitions available for testing)"
def testRequestsExt(self): print ("Testing GET extended requests"), # repl=self.cache_requests repl = self.api.getRequests() expect = [200, "UTF-8", "text/xml; charset=utf-8"] resp = [repl.status_code, repl.encoding, repl.headers["content-type"]] self.showDifferences(expect, resp, "http skelleton") replFields = self.xmlToDict(repl.content) print ("- limits"), l = len(replFields["service_requests"]) if l > 200: print ("to much (%d)" % l) # 90 days or first 200 else: print ("ok") print ("- XML structure"), with valideer.parsing(additional_properties=True): validator = valideer.parse(self.SCHEMA_REQUESTSEXT) try: validator.validate(replFields) print ("OK") except ValidationError as e: print ("differs at %s " % (str(e)))
import attr import gevent import gevent.fileobject as gfobj import gevent.util import logbook.concurrency import requests import toml import valideer as V import yaml import zmq.green as zmq from logbook import Logger, StderrHandler, StreamHandler import xbbs.messages as msgs import xbbs.util as xutils with V.parsing(required_properties=True, additional_properties=None): CONFIG_VALIDATOR = V.parse({ "job_endpoint": xutils.Endpoint(xutils.Endpoint.Side.BIND), "capabilities": V.Nullable(V.AdaptBy(xutils.list_to_set), set()), }) @attr.s class XbbsWorker: current_project = attr.ib(default=None) current_job = attr.ib(default=None) zmq = attr.ib(default=zmq.Context.instance())
import xbbs.messages as msgs import xbbs.util as xutils app = Flask(__name__) zctx = zmq.Context.instance() @V.accepts(x=V.AnyOf("string", ["string"])) def _list_wrap(x): if not isinstance(x, str): return x return [x] with V.parsing(required_properties=True, additional_properties=V.Object.REMOVE): CONFIG_VALIDATOR = V.parse({ "coordinator_endpoint": xutils.Endpoint(xutils.Endpoint.Side.CONNECT), "?coordinator_timeout": "integer", "?start_delay": "integer", "?github_secret": "string", "?github": V.Mapping("string", V.AdaptBy(_list_wrap)) }) XBBS_CFG_DIR = os.getenv("XBBS_CFG_DIR", "/etc/xbbs") with open(path.join(XBBS_CFG_DIR, "webhooks.toml"), "r") as fcfg:
def get_conf(conf=None): if not conf: with open('conf.json', 'br') as f: conf = json.loads(f.read().decode()) exists = v.Condition(lambda v: Path(v).exists()) strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/')) app_dir = Path(__file__).parent.resolve() base_dir = app_dir.parent log_handlers = ['console_simple', 'console_detail', 'file'] with v.parsing(additional_properties=False): schema = v.parse({ 'debug': v.Nullable(bool, False), '+pg_username': str, '+pg_password': str, '+cookie_secret': str, 'google_id': str, 'google_secret': str, 'readonly': v.Nullable(bool, True), 'enabled': v.Nullable(bool, True), 'log_handlers': (v.Nullable([v.Enum(log_handlers)], log_handlers[:1])), 'log_level': v.Nullable(str, 'DEBUG'), 'log_file': v.Nullable(str, ''), 'path_attachments': v.Nullable(str, str(base_dir / 'attachments')), 'path_theme': v.Nullable(exists, str(base_dir / 'front')), 'imap_body_maxsize': v.Nullable(int, 50 * 1024 * 1024), 'imap_batch_size': v.Nullable(int, 2000), 'imap_debug': v.Nullable(int, 0), 'smtp_debug': v.Nullable(bool, False), 'async_pool': v.Nullable(int, 0), 'ui_ga_id': v.Nullable(str, ''), 'ui_is_public': v.Nullable(bool, False), 'ui_use_names': v.Nullable(bool, True), 'ui_per_page': v.Nullable(int, 100), 'ui_greeting': v.Nullable(str, ''), 'ui_ws_proxy': v.Nullable(bool, False), 'ui_ws_enabled': v.Nullable(bool, True), 'ui_ws_timeout': v.Nullable(int, 1000), 'ui_firebug': v.Nullable(bool, False), 'ui_tiny_thread': v.Nullable(int, 5), 'ui_by_thread': v.Nullable(bool, False), 'from_emails': v.Nullable([str], []), 'host_ws': v.Nullable(str, 'ws://localhost/async/'), 'host_web': v.Nullable(strip_slash, 'http://localhost:8000'), 'search_lang': v.Nullable([str], ['simple', 'english']), }) conf = schema.validate(conf) path = Path(conf['path_attachments']) if not path.exists(): path.mkdir() return conf