def setUp(self):
V.Object.REQUIRED_PROPERTIES = True
V.base.reset_type_names()
self.complex_validator = self.parse({
"n":
"number",
"?i":
V.Nullable("integer", 0),
"?b":
bool,
"?e":
V.Enum(["r", "g", "b"]),
"?d":
V.AnyOf("date", "datetime"),
"?s":
V.String(min_length=1, max_length=8),
"?p":
V.Nullable(re.compile(r"\d{1,4}$")),
"?l": [{
"+s2": "string"
}],
"?t": (unicode, "number"),
"?h":
V.Mapping(int, ["string"]),
"?o":
V.NonNullable({"+i2": "integer"}),
})
def test_parsing_required_properties(self):
get_schema = lambda: {
"foo": V.Nullable("number"),
"?nested": [V.Nullable({"baz": "string"})]
}
valid = [{"foo": 3, "nested": [None]}]
missing_properties = [{}, {"foo": 3, "nested": [{}]}]
for _ in xrange(3):
with V.parsing(required_properties=False):
self._testValidation(get_schema(),
valid=valid + missing_properties)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=valid,
invalid=missing_properties)
# gotcha: calling parse() with required_properties=True is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(V.parse(get_schema(),
required_properties=True),
invalid=[missing_properties[1]])
else:
self._testValidation(V.parse(get_schema(),
required_properties=True),
valid=[missing_properties[1]])
def __init__(self, allow_extra):
schema = {
'+id': int,
'+client_name': V.String(max_length=255),
'+sort_index': float,
'client_phone': V.Nullable(V.String(max_length=255)),
'location': {'latitude': float, 'longitude': float},
'contractor': V.Range(V.AdaptTo(int), min_value=1),
'upstream_http_referrer': V.Nullable(V.String(max_length=1023)),
'+grecaptcha_response': V.String(min_length=20, max_length=1000),
'last_updated': V.AdaptBy(dateutil.parser.parse),
'skills': V.Nullable(
[
{
'+subject': str,
'+subject_id': int,
'+category': str,
'+qual_level': str,
'+qual_level_id': int,
'qual_level_ranking': V.Nullable(float, default=0),
}
],
default=[],
),
}
self.validator = V.parse(schema, additional_properties=allow_extra)
def setUp(self):
super(OptionalPropertiesTestValidator, self).setUp()
V.Object.REQUIRED_PROPERTIES = False
self.complex_validator = self.parse({
"+n":
"+number",
"i":
V.Nullable("integer", 0),
"b":
bool,
"e":
V.Enum(["r", "g", "b"]),
"d":
V.AnyOf("date", "datetime"),
"s":
V.String(min_length=1, max_length=8),
"p":
V.Nullable(re.compile(r"\d{1,4}$")),
"l": [{
"+s2": "string"
}],
"t": (unicode, "number"),
"h":
V.Mapping(int, ["string"]),
"o":
V.NonNullable({"+i2": "integer"}),
})
def test_nullable_with_default(self):
self._testValidation(V.Nullable("integer", -1),
adapted=[(None, -1), (0, 0)],
invalid=[1.1, True, False])
self._testValidation(V.Nullable("integer", lambda: -1),
adapted=[(None, -1), (0, 0)],
invalid=[1.1, True, False])
def test_nullable(self):
for obj in "?integer", V.Nullable(V.Integer()), V.Nullable("+integer"):
self._testValidation(obj,
valid=[None, 0],
invalid=[1.1, True, False])
self._testValidation(V.Nullable(["?string"]),
valid=[None, [], ["foo"], [None], ["foo", None]],
invalid=["", [None, "foo", 1]])
def inner(env, *a, **kw):
schema = v.parse({
'page': v.Nullable(v.AdaptTo(int), 1),
'last': v.Nullable(str)
})
data = schema.validate(env.request.args)
page, last = data['page'], data.get('last')
page = {
'limit': env('ui_per_page'),
'offset': env('ui_per_page') * (page - 1),
'last': last,
'count': env('ui_per_page') * page,
'current': page,
'next': page + 1,
}
return wrapper.func(env, page, *a, **kw)
def test_parsing_additional_properties(self):
get_schema = lambda: {
"?bar": "boolean",
"?nested": [V.Nullable({"?baz": "integer"})]
}
values = [{"x1": "yes"}, {"bar": True, "nested": [{"x1": "yes"}]}]
for _ in xrange(3):
with V.parsing(additional_properties=True):
self._testValidation(get_schema(), valid=values)
with V.parsing(additional_properties=False):
self._testValidation(get_schema(), invalid=values)
# gotcha: calling parse() with additional_properties=False is not
# equivalent to the above call because the V.Nullable() calls in
# get_schema have already called implicitly parse() without parameters.
# The 'additional_properties' parameter effectively is applied at
# the top level dict only
self._testValidation(V.parse(get_schema(),
additional_properties=False),
invalid=values[:1],
valid=values[1:])
with V.parsing(additional_properties=V.Object.REMOVE):
self._testValidation(get_schema(),
adapted=[(values[0], {}),
(values[1], {
"bar": True,
"nested": [{}]
})])
# same gotcha as above
self._testValidation(V.parse(
get_schema(), additional_properties=V.Object.REMOVE),
adapted=[(values[0], {}),
(values[1], values[1])])
with V.parsing(additional_properties="string"):
self._testValidation(get_schema(),
valid=values,
invalid=[{
"x1": 42
}, {
"bar": True,
"nested": [{
"x1": 42
}]
}])
# same gotcha as above
self._testValidation(V.parse(get_schema(),
additional_properties="string"),
invalid=[{
"x1": 42
}],
valid=[{
"bar": True,
"nested": [{
"x1": 42
}]
}])
def test_nested_parsing(self):
get_schema = lambda: {
"bar": "integer",
"?nested": [V.Nullable({"baz": "number"})]
}
values = [
{
"bar": 1
},
{
"bar": 1,
"nested": [{
"baz": 0
}, None]
},
{
"bar": 1,
"xx": 2
},
{
"bar": 1,
"nested": [{
"baz": 2.1,
"xx": 1
}]
},
{},
{
"bar": 1,
"nested": [{}]
},
]
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
with V.parsing(required_properties=True):
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
with V.parsing(additional_properties=False):
self._testValidation(get_schema(),
valid=values[:2],
invalid=values[2:])
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
if V.Object.REQUIRED_PROPERTIES:
self._testValidation(get_schema(),
valid=values[:4],
invalid=values[4:])
else:
self._testValidation(get_schema(), valid=values)
class BuildMessage(BaseMessage):
project = attr.ib()
delay = attr.ib()
incremental = attr.ib(default=None)
_validator = V.parse({
"project": "string",
"delay": "number",
"?incremental": V.Nullable("boolean")
})
def test_adapt_missing_property(self):
self._testValidation(
{
"foo": "number",
"?bar": V.Nullable("boolean", False)
},
adapted=[({
"foo": -12
}, {
"foo": -12,
"bar": False
})])
def mark(env):
def name(value):
if isinstance(value, str):
value = [value]
return [v for v in value if v]
schema = v.parse({
'+action': v.Enum(('+', '-', '=')),
'+name': v.AdaptBy(name),
'+ids': [int],
'old_name': v.AdaptBy(name),
'thread': v.Nullable(bool, False),
'last': v.Nullable(str)
})
data = schema.validate(env.request.json)
if not data['ids']:
return 'OK'
ids = tuple(data['ids'])
if data['thread']:
i = env.sql('''
SELECT id FROM emails WHERE thrid IN %s AND created <= %s
''', [ids, data['last']])
ids = tuple(r[0] for r in i)
mark = ft.partial(syncer.mark, env, ids=ids, new=True)
if data['action'] == '=':
if data.get('old_name') is None:
raise ValueError('Missing parameter "old_name" for %r' % data)
if data['old_name'] == data['name']:
return []
mark('-', set(data['old_name']) - set(data['name']))
mark('+', set(data['name']) - set(data['old_name']))
return 'OK'
mark(data['action'], data['name'])
return 'OK'
def test_nullable_with_default_object_property(self):
class ObjectNullable(V.Nullable):
default_object_property = property(lambda self: self.default)
regular_nullables = [
"?integer",
V.Nullable("integer"),
V.Nullable("integer", None),
V.Nullable("integer", default=None),
V.Nullable("integer", lambda: None),
V.Nullable("integer", default=lambda: None)
]
for obj in regular_nullables:
self._testValidation({"?foo": obj}, adapted=[({}, {})])
object_nullables = [
ObjectNullable("integer"),
ObjectNullable("integer", None),
ObjectNullable("integer", default=None),
ObjectNullable("integer", lambda: None),
ObjectNullable("integer", default=lambda: None),
]
for obj in object_nullables:
self._testValidation({"?foo": obj}, adapted=[({}, {"foo": None})])
class ArtifactMessage(BaseMessage):
project = attr.ib()
artifact_type = attr.ib()
artifact = attr.ib()
success = attr.ib()
filename = attr.ib(default=None)
last_hash = attr.ib(default=None)
_validator = V.parse({
"project": "string",
"artifact_type": V.Enum({"tool", "package", "file"}),
"artifact": "string",
# TODO(arsen): architecture
"success": "boolean",
"?filename": "?string",
"?last_hash": V.Nullable(_is_blake2b_digest)
})
def emails(env, page):
schema = v.parse({'q': v.Nullable(str, '')})
q = schema.validate(env.request.args)['q']
ctx = {'labels': ['\\All'], 'by_thread': False}
if q.startswith('g! '):
# Gmail search
ids = syncer.search(env, env.email, q[3:])
select_ids = env.mogrify('''
(SELECT * FROM unnest(%s::bigint[][])) AS ids(id)
''', [ids])
else:
select_ids, ctx = parse_query(env, q, page)
select_ids = '(%s) AS ids' % select_ids
if ctx['by_thread']:
res = threads(env, select_ids, ctx, page)
else:
i = env.sql('''
SELECT
e.id, thrid, subj, labels, time, fr, "to", text, cc, created,
html, attachments, parent
FROM emails e
JOIN {select_ids} ON e.id = ids.id
ORDER BY {order_by}
LIMIT {page[limit]} OFFSET {page[offset]}
'''.format(
select_ids=select_ids,
page=page,
order_by=ctx.get('order_by', 'id DESC')
))
def emails():
for msg in i:
msg = dict(msg)
yield msg
res = ctx_emails(env, emails())
res['labels'] = ctx_labels(env, ctx['labels'])
from . import log
log.info('keywords=%(keywords)s', ctx)
if ctx['keywords'] != {'in'} and 'thr' not in ctx['keywords']:
q += ' thr:0'
res['search_query'] = q
return res
def test_humanized_names(self):
class DummyValidator(V.Validator):
name = "dummy"
def validate(self, value, adapt=True):
return value
self.assertEqual(DummyValidator().humanized_name, "dummy")
self.assertEqual(
V.Nullable(DummyValidator()).humanized_name, "dummy or null")
self.assertEqual(
V.AnyOf("boolean", DummyValidator()).humanized_name,
"boolean or dummy")
self.assertEqual(
V.AllOf("boolean", DummyValidator()).humanized_name,
"boolean and dummy")
self.assertEqual(
V.ChainOf("boolean", DummyValidator()).humanized_name,
"boolean chained to dummy")
self.assertEqual(Date().humanized_name, "date or datetime")
def get_conf(conf=None):
if not conf:
with open('conf.json', 'br') as f:
conf = json.loads(f.read().decode())
exists = v.Condition(lambda v: Path(v).exists())
strip_slash = v.AdaptBy(lambda v: str(v).rstrip('/'))
app_dir = Path(__file__).parent.resolve()
base_dir = app_dir.parent
log_handlers = ['console_simple', 'console_detail', 'file']
with v.parsing(additional_properties=False):
schema = v.parse({
'debug':
v.Nullable(bool, False),
'+pg_username':
str,
'+pg_password':
str,
'+cookie_secret':
str,
'google_id':
str,
'google_secret':
str,
'readonly':
v.Nullable(bool, True),
'enabled':
v.Nullable(bool, True),
'log_handlers': (v.Nullable([v.Enum(log_handlers)],
log_handlers[:1])),
'log_level':
v.Nullable(str, 'DEBUG'),
'log_file':
v.Nullable(str, ''),
'path_attachments':
v.Nullable(str, str(base_dir / 'attachments')),
'path_theme':
v.Nullable(exists, str(base_dir / 'front')),
'imap_body_maxsize':
v.Nullable(int, 50 * 1024 * 1024),
'imap_batch_size':
v.Nullable(int, 2000),
'imap_debug':
v.Nullable(int, 0),
'smtp_debug':
v.Nullable(bool, False),
'async_pool':
v.Nullable(int, 0),
'ui_ga_id':
v.Nullable(str, ''),
'ui_is_public':
v.Nullable(bool, False),
'ui_use_names':
v.Nullable(bool, True),
'ui_per_page':
v.Nullable(int, 100),
'ui_greeting':
v.Nullable(str, ''),
'ui_ws_proxy':
v.Nullable(bool, False),
'ui_ws_enabled':
v.Nullable(bool, True),
'ui_ws_timeout':
v.Nullable(int, 1000),
'ui_firebug':
v.Nullable(bool, False),
'ui_tiny_thread':
v.Nullable(int, 5),
'ui_by_thread':
v.Nullable(bool, False),
'from_emails':
v.Nullable([str], []),
'host_ws':
v.Nullable(str, 'ws://localhost/async/'),
'host_web':
v.Nullable(strip_slash, 'http://localhost:8000'),
'search_lang':
v.Nullable([str], ['simple', 'english']),
})
conf = schema.validate(conf)
path = Path(conf['path_attachments'])
if not path.exists():
path.mkdir()
return conf
class Results(object):
__slots__ = ("navigator", "_query", "period", "_results", "_speed")
def __init__(self, navigator, query, period):
self.navigator = navigator
self._query = query
self.period = period
self._results = False
self._speed = None
@property
def results(self):
if self._results is False:
start = time()
self._results = self.navigator.query(
self._query.replace('__into__', ''))
self._speed = (time() - start) * 1000
return self._results
def refresh(self):
self._results = False
self.results
return self
def json(self, debug=False):
results = self.results
result = dict(results=results,
meta={
"status": 200,
"total": len(results),
"speed": "%.fms" % self._speed
})
if self.period:
result["meta"]["time"] = {
"start": str(self.period.start),
"end": str(self.period.end)
}
if debug:
result['meta']['query'] = self.pg()
return dumps(result, default=json_defaults)
@valideer.accepts(into=valideer.Nullable(
valideer.Pattern(r"^[a-zA-Z\_]{1,25}$")))
def pg(self, into=None):
return self._query.replace("__into__",
(" INTO " + (into or '')) if into else '')
@property
def value(self):
results = self.results
if results and len(results) == 1:
return list(self)[0]
else:
return results
def __str__(self):
return str(self.value)
def __nonzero__(self):
return bool(self.results)
@valideer.accepts(index="integer")
def __getitem__(self, index):
return self.results[index]
def __getattr__(self, index):
if len(self.results) == 1:
return self.navigator.format(index, self.results[0][index])
else:
raise ValueError("Cannot get attr from list")
def __iter__(self):
"""Returns the results with python objects inserted
"""
results = self.results
if results:
for row in iter(results):
yield dict([(key, self.navigator.format(key, value))
for key, value in row.iteritems()])
def __len__(self):
results = self.results
return len(results) if results else 0
def __cmp__(self, other):
results = self.results
if len(results) == 1:
x = results[0][results[0].keys()[0]]
return 0 if other == x else -1 if other > x else 1
else:
return False
import requests
import toml
import valideer as V
import yaml
import zmq.green as zmq
from logbook import Logger, StderrHandler, StreamHandler
import xbbs.messages as msgs
import xbbs.util as xutils
with V.parsing(required_properties=True, additional_properties=None):
CONFIG_VALIDATOR = V.parse({
"job_endpoint":
xutils.Endpoint(xutils.Endpoint.Side.BIND),
"capabilities":
V.Nullable(V.AdaptBy(xutils.list_to_set), set()),
})
@attr.s
class XbbsWorker:
current_project = attr.ib(default=None)
current_job = attr.ib(default=None)
zmq = attr.ib(default=zmq.Context.instance())
def download(url, to):
src = urlparse(url, scheme='file')
if src.scheme == 'file':
shutil.copy(src.path, to)
else:
def draft(env, thrid, action):
saved = env.storage('compose', thrid=thrid)
saved_path = env.files.subpath('compose', thrid=thrid)
if action == 'preview':
schema = v.parse({
'+fr': str,
'+to': str,
'+subj': str,
'+body': str,
'+quoted': bool,
'+forward': bool,
'+id': v.Nullable(str),
'quote': v.Nullable(str)
})
data = schema.validate(env.request.json)
if env.request.args.get('save', False):
saved.set(data)
return get_html(data['body'], data.get('quote', ''))
elif action == 'upload':
count = env.request.form.get('count', type=int)
files = []
for n, i in enumerate(env.request.files.getlist('files'), count):
path = '/'.join([saved_path, str(n), f.slugify(i.filename)])
env.files.write(path, i.stream.read())
files.append(env.files.to_dict(path, i.mimetype, i.filename))
return files
elif action == 'send':
import dns.resolver
import dns.exception
class Email(v.Validator):
def validate(self, value, adapt=True):
if not value:
raise v.ValidationError('No email')
addr = parseaddr(value)[1]
hostname = addr[addr.find('@') + 1:]
try:
dns.resolver.query(hostname, 'MX')
except dns.exception.DNSException:
raise v.ValidationError('No MX record for %s' % hostname)
return value
schema = v.parse({
'+to': v.ChainOf(
v.AdaptBy(lambda v: [i.strip() for i in v.split(',')]),
[Email]
),
'+fr': Email,
'+subj': str,
'+body': str,
'id': v.Nullable(str),
'quote': v.Nullable(str, ''),
})
msg = schema.validate(env.request.json)
if msg.get('id'):
parent = env.sql('''
SELECT thrid, msgid, refs
FROM emails WHERE id=%s LIMIT 1
''', [msg['id']]).fetchone()
msg['in_reply_to'] = parent.get('msgid')
msg['refs'] = parent.get('refs', [])[-10:]
else:
parent = {}
sendmail(env, msg)
if saved.get():
draft(env, thrid, 'rm')
syncer.sync_gmail(env, env.email, only=['\\All'], fast=1, force=1)
url = url_query(env, 'in', '\\Sent')
if parent.get('thrid'):
url = env.url_for('thread', {'id': parent['thrid']})
return {'url': url}
elif action == 'rm':
if saved.get({}).get('files'):
env.files.rm(saved_path)
saved.rm()
return 'OK'
env.abort(400)
def compose(env, id=None):
if not env.storage.get('gmail_info'):
return env.abort(400)
schema = v.parse({
'target': v.Nullable(v.Enum(('all', 'forward')))
})
args = schema.validate(env.request.args)
fr = env.from_emails[0]
ctx = {
'fr': fr, 'to': '', 'subj': '', 'body': '', 'files': [],
'quoted': False, 'forward': False, 'id': id, 'draft': False,
'from_emails': env.from_emails
}
parent = {}
if id:
parent = env.sql('''
SELECT
thrid, "to", fr, cc, bcc, subj, reply_to, html, time,
attachments, embedded
FROM emails WHERE id=%s LIMIT 1
''', [id]).fetchone()
to_all = parent['to'][:] + parent['cc'][:]
fr = env.from_email(parent['fr'])
if fr:
to = to_all
else:
fr_ = env.from_email(to_all)
if fr_:
fr = fr_
to_all = [
a for a in to_all
if parseaddr(a)[1] != parseaddr(fr)[1]
]
to = (parent['reply_to'] or parent['fr'])[:]
forward = args.get('target') == 'forward'
if forward:
to = []
elif args.get('target') == 'all':
to += to_all
ctx.update({
'fr': fr,
'to': ', '.join(to),
'subj': 'Re: %s' % f.humanize_subj(parent['subj'], empty=''),
'quote': ctx_quote(env, parent, forward),
'quoted': forward,
'forward': forward,
})
thrid = parent.get('thrid')
saved = env.storage('compose', thrid=thrid)
saved_path = env.files.subpath('compose', thrid=thrid)
if saved.get():
ctx.update(saved.get())
ctx['draft'] = saved.get() is not None
ctx['title'] = ctx.get('subj') or 'New message'
if ctx['forward'] and not ctx['draft']:
env.files.copy(f.slugify(id), saved_path)
files = list(parent['attachments']) + list(parent['embedded'].values())
for i in files:
path = i['path'].replace(id, saved_path)
asset = env.files.to_dict(**dict(i, path=path))
ctx['files'].append(asset)
quote = ctx.get('quote')
if quote:
parent_url = re.escape(env.files.url(i['path']))
ctx['quote'] = re.sub(parent_url, asset['url'], quote)
ctx['links'] = {
a: env.url_for('draft', {'thrid': str(thrid or 'new'), 'action': a})
for a in ('preview', 'rm', 'send', 'upload')
}
return ctx
"string",
"build_root":
V.AllOf("string", path.isabs),
"intake":
V.AdaptBy(_receive_adaptor),
"worker_endpoint":
xutils.Endpoint(xutils.Endpoint.Side.BIND),
# use something like a C identifier, except disallow underscore as a
# first character too. this is so that we have a namespace for xbbs
# internal directories, such as collection directories
"projects":
V.Mapping(
xutils.PROJECT_REGEX, {
"git": "string",
"?description": "string",
"?classes": V.Nullable(["string"], []),
"packages": "string",
"?fingerprint": "string",
"tools": "string",
"?incremental": "boolean",
"?distfile_path": "string",
"?mirror_root": "string",
"?default_branch": "string",
})
})
PUBKEY_VALIDATOR = V.parse({
# I'm only validating the keys that xbbs uses
"signature-by": "string"
})
with V.parsing(required_properties=True, additional_properties=None):
def test_adapts(self):
@V.adapts(
body={
"+field_ids": ["integer"],
"?scores":
V.Mapping("string", float),
"?users": [{
"+name": ("+string", "+string"),
"?sex": "gender",
"?active": V.Nullable("boolean", True),
}]
})
def f(body):
return body
adapted = f({
"field_ids": [1, 5],
"scores": {
"foo": 23.1,
"bar": 2.0
},
"users": [
{
"name": ("Nick", "C"),
"sex": "male"
},
{
"name": ("Kim", "B"),
"active": False
},
{
"name": ("Joe", "M"),
"active": None
},
]
})
self.assertEqual(adapted["field_ids"], [1, 5])
self.assertEqual(adapted["scores"]["foo"], 23.1)
self.assertEqual(adapted["scores"]["bar"], 2.0)
self.assertEqual(adapted["users"][0]["name"], ("Nick", "C"))
self.assertEqual(adapted["users"][0]["sex"], "male")
self.assertEqual(adapted["users"][0]["active"], True)
self.assertEqual(adapted["users"][1]["name"], ("Kim", "B"))
self.assertEqual(adapted["users"][1].get("sex"), None)
self.assertEqual(adapted["users"][1]["active"], False)
self.assertEqual(adapted["users"][2]["name"], ("Joe", "M"))
self.assertEqual(adapted["users"][2].get("sex"), None)
self.assertEqual(adapted["users"][2].get("active"), True)
invalid = [
# missing 'field_ids' from body
partial(f, {}),
# score value is not float
partial(f, {
"field_ids": [],
"scores": {
"a": "2.3"
}
}),
# 'name' is not a length-2 tuple
partial(f, {
"field_ids": [],
"users": [{
"name": ("Bob", "R", "Junior")
}]
}),
# name[1] is not a string
partial(f, {
"field_ids": [],
"users": [{
"name": ("Bob", 12)
}]
}),
# name[1] is required
partial(f, {
"field_ids": [],
"users": [{
"name": ("Bob", None)
}]
}),
]
for fcall in invalid:
self.assertRaises(V.ValidationError, fcall)
