def test_parsing_required_properties(self):
get_schema = lambda: {
"foo": V.Nullable("number"),
"?nested": [V.Nullable({"baz": "string"})]
}
valid = [{"foo": 3, "nested": [None]}]
missing_properties = [{}, {"foo": 3, "nested": [{}]}]
for _ in xrange(3):
with V.parsing(required_properties=False):
self._testValidation(get_schema(),
valid=valid + missing_properties)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=valid,
invalid=missing_properties)
# gotcha: calling parse() with required_properties=True is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(V.parse(get_schema(),
required_properties=True),
invalid=[missing_properties[1]])
else:
self._testValidation(V.parse(get_schema(),
required_properties=True),
valid=[missing_properties[1]])
def test_parsing_required_properties(self):
get_schema = lambda: {
"foo": V.Nullable("number"),
"?nested": [V.Nullable({
"baz": "string"
})]
}
valid = [{"foo": 3, "nested": [None]}]
missing_properties = [{}, {"foo": 3, "nested": [{}]}]
for _ in xrange(3):
with V.parsing(required_properties=False):
self._testValidation(get_schema(),
valid=valid + missing_properties)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=valid, invalid=missing_properties)
# gotcha: calling parse() with required_properties=True is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(V.parse(get_schema(), required_properties=True),
invalid=[missing_properties[1]])
else:
self._testValidation(V.parse(get_schema(), required_properties=True),
valid=[missing_properties[1]])
def test_nested_parsing(self):
get_schema = lambda: {
"bar": "integer",
"?nested": [V.Nullable({"baz": "number"})]
}
values = [
{
"bar": 1
},
{
"bar": 1,
"nested": [{
"baz": 0
}, None]
},
{
"bar": 1,
"xx": 2
},
{
"bar": 1,
"nested": [{
"baz": 2.1,
"xx": 1
}]
},
{},
{
"bar": 1,
"nested": [{}]
},
]
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
with V.parsing(additional_properties=False):
self._testValidation(get_schema(),
valid=values[:2],
invalid=values[2:])
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
def test_parsing_additional_properties(self):
get_schema = lambda: {
"?bar": "boolean",
"?nested": [V.Nullable({"?baz": "integer"})]
}
values = [{"x1": "yes"}, {"bar": True, "nested": [{"x1": "yes"}]}]
for _ in xrange(3):
with V.parsing(additional_properties=True):
self._testValidation(get_schema(), valid=values)
with V.parsing(additional_properties=False):
self._testValidation(get_schema(), invalid=values)
# gotcha: calling parse() with additional_properties=False is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
# The 'additional_properties' parameter effectively is applied at
# the top level dict only
self._testValidation(V.parse(get_schema(),
additional_properties=False),
invalid=values[:1],
valid=values[1:])
with V.parsing(additional_properties=V.Object.REMOVE):
self._testValidation(get_schema(),
adapted=[(values[0], {}),
(values[1], {
"bar": True,
"nested": [{}]
})])
# same gotcha as above
self._testValidation(V.parse(
get_schema(), additional_properties=V.Object.REMOVE),
adapted=[(values[0], {}),
(values[1], values[1])])
with V.parsing(additional_properties="string"):
self._testValidation(get_schema(),
valid=values,
invalid=[{
"x1": 42
}, {
"bar": True,
"nested": [{
"x1": 42
}]
}])
# same gotcha as above
self._testValidation(V.parse(get_schema(),
additional_properties="string"),
invalid=[{
"x1": 42
}],
valid=[{
"bar": True,
"nested": [{
"x1": 42
}]
}])
def get_conf(conf=None):
if not conf:
with open('conf.json', 'br') as f:
conf = json.loads(f.read().decode())
exists = v.Condition(lambda v: Path(v).exists())
strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/'))
app_dir = Path(__file__).parent.resolve()
base_dir = app_dir.parent
log_handlers = ['console_simple', 'console_detail', 'file']
with v.parsing(additional_properties=False):
schema = v.parse({
'debug': v.Nullable(bool, False),
'+pg_username': str,
'+pg_password': str,
'+cookie_secret': str,
'google_id': str,
'google_secret': str,
'readonly': v.Nullable(bool, True),
'enabled': v.Nullable(bool, True),
'log_handlers': (
v.Nullable([v.Enum(log_handlers)], log_handlers[:1])
),
'log_level': v.Nullable(str, 'DEBUG'),
'log_file': v.Nullable(str, ''),
'path_attachments': v.Nullable(str, str(base_dir / 'attachments')),
'path_theme': v.Nullable(exists, str(base_dir / 'front')),
'imap_body_maxsize': v.Nullable(int, 50 * 1024 * 1024),
'imap_batch_size': v.Nullable(int, 2000),
'imap_debug': v.Nullable(int, 0),
'smtp_debug': v.Nullable(bool, False),
'async_pool': v.Nullable(int, 0),
'ui_ga_id': v.Nullable(str, ''),
'ui_is_public': v.Nullable(bool, False),
'ui_use_names': v.Nullable(bool, True),
'ui_per_page': v.Nullable(int, 100),
'ui_greeting': v.Nullable(str, ''),
'ui_ws_proxy': v.Nullable(bool, False),
'ui_ws_enabled': v.Nullable(bool, True),
'ui_ws_timeout': v.Nullable(int, 1000),
'ui_firebug': v.Nullable(bool, False),
'ui_tiny_thread': v.Nullable(int, 5),
'ui_by_thread': v.Nullable(bool, False),
'from_emails': v.Nullable([str], []),
'host_ws': v.Nullable(str, 'ws://localhost/async/'),
'host_web': v.Nullable(strip_slash, 'http://localhost:8000'),
'search_lang': v.Nullable([str], ['simple', 'english']),
})
conf = schema.validate(conf)
path = Path(conf['path_attachments'])
if not path.exists():
path.mkdir()
return conf
def test_parsing_additional_properties(self):
get_schema = lambda: {
"?bar": "boolean",
"?nested": [V.Nullable({
"?baz": "integer"
})]
}
values = [{"x1": "yes"},
{"bar": True, "nested": [{"x1": "yes"}]}]
for _ in xrange(3):
with V.parsing(additional_properties=True):
self._testValidation(get_schema(), valid=values)
with V.parsing(additional_properties=False):
self._testValidation(get_schema(), invalid=values)
# gotcha: calling parse() with additional_properties=False is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
# The 'additional_properties' parameter effectively is applied at
# the top level dict only
self._testValidation(V.parse(get_schema(), additional_properties=False),
invalid=values[:1], valid=values[1:])
with V.parsing(additional_properties=V.Object.REMOVE):
self._testValidation(get_schema(),
adapted=[(values[0], {}),
(values[1], {"bar": True, "nested": [{}]})])
# same gotcha as above
self._testValidation(V.parse(get_schema(), additional_properties=V.Object.REMOVE),
adapted=[(values[0], {}),
(values[1], values[1])])
with V.parsing(additional_properties="string"):
self._testValidation(get_schema(),
valid=values,
invalid=[{"x1": 42},
{"bar": True, "nested": [{"x1": 42}]}])
# same gotcha as above
self._testValidation(V.parse(get_schema(), additional_properties="string"),
invalid=[{"x1": 42}],
valid=[{"bar": True, "nested": [{"x1": 42}]}])
def test_parsing_ignore_optional_property_errors(self):
get_schema = lambda: V.Nullable({
"+foo": "number",
"?bar": "boolean",
"?nested": [{
"+baz": "string",
"?zoo": "number",
}]
})
invalid_required = [
{"foo": "2", "bar": True},
]
invalid_optional = [
{"foo": 3, "bar": "nan"},
{"foo": 3.1, "nested": [{"baz": "x", "zoo": "12"}]},
{"foo": 0, "nested": [{"baz": 1, "zoo": 2}]},
]
adapted = [
{"foo": 3},
{"foo": 3.1, "nested": [{"baz": "x"}]},
{"foo": 0},
]
for _ in xrange(3):
with V.parsing(ignore_optional_property_errors=False):
self._testValidation(get_schema(),
invalid=invalid_required + invalid_optional)
with V.parsing(ignore_optional_property_errors=True):
self._testValidation(get_schema(),
invalid=invalid_required,
adapted=zip(invalid_optional, adapted))
# gotcha: calling parse() with ignore_optional_property_errors=True
# is not equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
self._testValidation(V.parse(get_schema(), ignore_optional_property_errors=False),
invalid=invalid_required + invalid_optional)
self._testValidation(V.parse(get_schema(), ignore_optional_property_errors=True),
invalid=invalid_required + invalid_optional)
def testGetNotes(self, apikey):
repl = self.api.getNotes(66, apikey)
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
replFields = self.xmlToDict(repl.content)
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_NOTES)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs at %s " % (str(e)))
def testGetRequests(self):
print("Testing GET requests"),
repl = self.cache_requests
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_REQUESTS)
replFields = self.xmlToDict(repl.content)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs at %s " % (str(e)))
def testGetServices(self):
print("Testing GET services")
repl = self.cache_services
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_SERVICE)
replFields = self.xmlToDict(repl.content)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs %s " % (str(e)))
def testRequestsExtKS(self):
print("Testing GET extended requests"),
repl = self.api.getRequests()
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
replFields = self.xmlToDict(repl.content)
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_REQUESTSEXT)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs at %s " % (str(e)))
def test_nested_parsing(self):
get_schema = lambda: {
"bar": "integer",
"?nested": [V.Nullable({
"baz": "number"
})]
}
values = [
{"bar": 1},
{"bar": 1, "nested": [{"baz": 0}, None]},
{"bar": 1, "xx": 2},
{"bar": 1, "nested": [{"baz": 2.1, "xx": 1}]},
{},
{"bar": 1, "nested": [{}]},
]
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4], invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=values[:4], invalid=values[4:])
with V.parsing(additional_properties=False):
self._testValidation(get_schema(),
valid=values[:2], invalid=values[2:])
self._testValidation(get_schema(),
valid=values[:4], invalid=values[4:])
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4], invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
def testGetServicsDef(self):
#TODO: walk trough all definitions
print("Testing GET service definition"),
firstCode = self.__getFirstServiceCode()
if firstCode is not None:
repl = self.api.getServiceDef(firstCode)
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_DEF)
replFields = self.xmlToDict(repl)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs at %s " % (str(e)))
else:
print "(No service definitions available for testing)"
def testGetServicsDef(self):
#TODO: walk trough all definitions
print("Testing GET service definition"),
firstCode = self.__getFirstServiceCode()
if firstCode is not None:
repl = self.api.getServiceDef(firstCode)
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [
repl.status_code, repl.encoding, repl.headers["content-type"]
]
self.showDifferences(expect, resp, "http skelleton")
print("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_DEF)
replFields = self.xmlToDict(repl)
try:
validator.validate(replFields)
print("OK")
except ValidationError as e:
print("differs at %s " % (str(e)))
else:
print "(No service definitions available for testing)"
def testRequestsExt(self):
print ("Testing GET extended requests"),
# repl=self.cache_requests
repl = self.api.getRequests()
expect = [200, "UTF-8", "text/xml; charset=utf-8"]
resp = [repl.status_code, repl.encoding, repl.headers["content-type"]]
self.showDifferences(expect, resp, "http skelleton")
replFields = self.xmlToDict(repl.content)
print ("- limits"),
l = len(replFields["service_requests"])
if l > 200:
print ("to much (%d)" % l)
# 90 days or first 200
else:
print ("ok")
print ("- XML structure"),
with valideer.parsing(additional_properties=True):
validator = valideer.parse(self.SCHEMA_REQUESTSEXT)
try:
validator.validate(replFields)
print ("OK")
except ValidationError as e:
print ("differs at %s " % (str(e)))
import attr
import gevent
import gevent.fileobject as gfobj
import gevent.util
import logbook.concurrency
import requests
import toml
import valideer as V
import yaml
import zmq.green as zmq
from logbook import Logger, StderrHandler, StreamHandler
import xbbs.messages as msgs
import xbbs.util as xutils
with V.parsing(required_properties=True, additional_properties=None):
CONFIG_VALIDATOR = V.parse({
"job_endpoint":
xutils.Endpoint(xutils.Endpoint.Side.BIND),
"capabilities":
V.Nullable(V.AdaptBy(xutils.list_to_set), set()),
})
@attr.s
class XbbsWorker:
current_project = attr.ib(default=None)
current_job = attr.ib(default=None)
zmq = attr.ib(default=zmq.Context.instance())
import xbbs.messages as msgs
import xbbs.util as xutils
app = Flask(__name__)
zctx = zmq.Context.instance()
@V.accepts(x=V.AnyOf("string", ["string"]))
def _list_wrap(x):
if not isinstance(x, str):
return x
return [x]
with V.parsing(required_properties=True,
additional_properties=V.Object.REMOVE):
CONFIG_VALIDATOR = V.parse({
"coordinator_endpoint":
xutils.Endpoint(xutils.Endpoint.Side.CONNECT),
"?coordinator_timeout":
"integer",
"?start_delay":
"integer",
"?github_secret":
"string",
"?github":
V.Mapping("string", V.AdaptBy(_list_wrap))
})
XBBS_CFG_DIR = os.getenv("XBBS_CFG_DIR", "/etc/xbbs")
with open(path.join(XBBS_CFG_DIR, "webhooks.toml"), "r") as fcfg:
def get_conf(conf=None):
if not conf:
with open('conf.json', 'br') as f:
conf = json.loads(f.read().decode())
exists = v.Condition(lambda v: Path(v).exists())
strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/'))
app_dir = Path(__file__).parent.resolve()
base_dir = app_dir.parent
log_handlers = ['console_simple', 'console_detail', 'file']
with v.parsing(additional_properties=False):
schema = v.parse({
'debug':
v.Nullable(bool, False),
'+pg_username':
str,
'+pg_password':
str,
'+cookie_secret':
str,
'google_id':
str,
'google_secret':
str,
'readonly':
v.Nullable(bool, True),
'enabled':
v.Nullable(bool, True),
'log_handlers': (v.Nullable([v.Enum(log_handlers)],
log_handlers[:1])),
'log_level':
v.Nullable(str, 'DEBUG'),
'log_file':
v.Nullable(str, ''),
'path_attachments':
v.Nullable(str, str(base_dir / 'attachments')),
'path_theme':
v.Nullable(exists, str(base_dir / 'front')),
'imap_body_maxsize':
v.Nullable(int, 50 * 1024 * 1024),
'imap_batch_size':
v.Nullable(int, 2000),
'imap_debug':
v.Nullable(int, 0),
'smtp_debug':
v.Nullable(bool, False),
'async_pool':
v.Nullable(int, 0),
'ui_ga_id':
v.Nullable(str, ''),
'ui_is_public':
v.Nullable(bool, False),
'ui_use_names':
v.Nullable(bool, True),
'ui_per_page':
v.Nullable(int, 100),
'ui_greeting':
v.Nullable(str, ''),
'ui_ws_proxy':
v.Nullable(bool, False),
'ui_ws_enabled':
v.Nullable(bool, True),
'ui_ws_timeout':
v.Nullable(int, 1000),
'ui_firebug':
v.Nullable(bool, False),
'ui_tiny_thread':
v.Nullable(int, 5),
'ui_by_thread':
v.Nullable(bool, False),
'from_emails':
v.Nullable([str], []),
'host_ws':
v.Nullable(str, 'ws://localhost/async/'),
'host_web':
v.Nullable(strip_slash, 'http://localhost:8000'),
'search_lang':
v.Nullable([str], ['simple', 'english']),
})
conf = schema.validate(conf)
path = Path(conf['path_attachments'])
if not path.exists():
path.mkdir()
return conf
