def query_model_view(self,
model,
view_name,
key=None,
keys=None,
group=None,
limit=None,
include_docs=None,
reduce=True,
method=None):
'''Query an existing view
All view parameters here:
http://wiki.apache.org/couchdb/HTTP_view_API
'''
meta = model._meta
kwargs = {}
if key:
kwargs['key'] = json.dumps(key)
elif keys:
kwargs['keys'] = json.dumps(keys)
if group:
kwargs['group'] = 'true'
if limit:
kwargs['limit'] = int(limit)
if include_docs:
kwargs['include_docs'] = 'true'
if not reduce:
kwargs['reduce'] = 'false'
return self.request(method or 'get', self._database, '_design',
meta.table_name, '_view', view_name, **kwargs)
def json_params(self):
for name, value in self.cfg.items():
try:
json.dumps(value)
except Exception:
continue
yield name, value
def query_model_view(self, model, view_name, key=None, keys=None,
group=None, limit=None, include_docs=None,
reduce=True, method=None):
'''Query an existing view
All view parameters here:
http://wiki.apache.org/couchdb/HTTP_view_API
'''
meta = model._meta
kwargs = {}
if key:
kwargs['key'] = json.dumps(key)
elif keys:
kwargs['keys'] = json.dumps(keys)
if group:
kwargs['group'] = 'true'
if limit:
kwargs['limit'] = int(limit)
if include_docs:
kwargs['include_docs'] = 'true'
if not reduce:
kwargs['reduce'] = 'false'
return self.request(method or 'get', self._database, '_design',
meta.table_name, '_view', view_name, **kwargs)
def test_create_instance(self):
models = self.mapper(Task)
yield models.create_tables()
task = yield models.task.create(id='bjbhjscbhj', name='foo')
self.assertEqual(task['id'], 'bjbhjscbhj')
data = task.to_json()
self.assertEqual(len(data), 2)
self.assertFalse(task._modified)
# make sure it is serializable
json.dumps(data)
task = yield models.task(task.id)
yield models.drop_tables()
def _encode_body(self, data, files, json):
body = None
if isinstance(data, (str, bytes)):
if files:
raise ValueError('data cannot be a string or bytes when '
'files are present')
body = to_bytes(data, self.charset)
elif data and is_streamed(data):
if files:
raise ValueError('data cannot be an iterator when '
'files are present')
if 'content-length' not in self.headers:
self.headers['transfer-encoding'] = 'chunked'
return data
elif data or files:
if files:
body, content_type = self._encode_files(data, files)
else:
body, content_type = self._encode_params(data)
self.headers['Content-Type'] = content_type
elif json:
body = _json.dumps(json).encode(self.charset)
self.headers['Content-Type'] = 'application/json'
if body:
self.headers['content-length'] = str(len(body))
return body
def do_stream(self, request):
if self.require:
yield ('<script type="text/javascript">\n'
'var require = %s;\n'
'</script>\n') % json.dumps(self.require_script())
for child in self.children:
yield child
def test_json_with_async_string(self):
astr = wsgi.String('ciao')
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
self.assertEqual(response.content_type,
'application/json; charset=utf-8')
self.assertEqual(response.render(), json.dumps({'bla': 'ciao'}))
def do_stream(self, request):
if self.require:
yield ('<script type="text/javascript">\n'
'var require = %s;\n'
'</script>\n') % json.dumps(self.require_script())
for child in self.children:
yield child
def __call__(self, server, data=None):
headers = {}
for hv in bytes(data).decode('utf-8').split('\r\n'):
hv = hv.split(':')
if len(hv) >= 2:
headers[hv[0].strip()] = (':'.join(hv[1:])).strip()
self.headers = json.dumps(headers).encode('utf-8')
def _encode_body(self, data, files, json):
body = None
if isinstance(data, (str, bytes)):
if files:
raise ValueError('data cannot be a string or bytes when '
'files are present')
body = to_bytes(data, self.charset)
elif data and is_streamed(data):
if files:
raise ValueError('data cannot be an iterator when '
'files are present')
if 'content-length' not in self.headers:
self.headers['transfer-encoding'] = 'chunked'
return data
elif data or files:
if files:
body, content_type = self._encode_files(data, files)
else:
body, content_type = self._encode_params(data)
self.headers['Content-Type'] = content_type
elif json:
body = _json.dumps(json).encode(self.charset)
self.headers['Content-Type'] = 'application/json'
if body:
self.headers['content-length'] = str(len(body))
return body
def test_simple_json_as_list(self):
response = wsgi.Json({'bla': 'foo'}, as_list=True)
self.assertEqual(len(response.children), 1)
self.assertEqual(response.content_type,
'application/json; charset=utf-8')
self.assertTrue(response.as_list)
self.assertEqual(response.render(), json.dumps([{'bla': 'foo'}]))
def test_simple_json_as_list(self):
response = wsgi.Json({'bla': 'foo'}, as_list=True)
self.assertEqual(len(response.children), 1)
self.assertEqual(response.content_type,
'application/json; charset=utf-8')
self.assertTrue(response.as_list)
self.assertEqual(response.render(), json.dumps([{'bla': 'foo'}]))
def _consume_in_subprocess(self, job, kwargs):
params = dict(self.json_params())
loop = job._loop
transport, protocol = yield from loop.subprocess_exec(
lambda: StreamProtocol(job),
sys.executable,
PROCESS_FILE,
json.dumps(sys.path),
json.dumps(params),
job.task.serialise())
process = Process(transport, protocol, loop)
yield from process.communicate()
if job.task.stacktrace:
raise RemoteStackTrace
return job.task.result
def test_json_with_async_string(self):
astr = wsgi.String('ciao')
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
self.assertEqual(response.content_type,
'application/json; charset=utf-8')
self.assertEqual(response.render(), json.dumps({'bla': 'ciao'}))
def to_json(self):
self.body._tag = None
body = yield multi_async(self.body.stream(request))
self.head._tag = None
data = {'body': body}
data.extend(self.head.to_json())
coroutine_return(json.dumps(data))
def on_message(self, websocket, msg):
request = websocket.request
client = request.cache.mailclient
if msg and client:
msg = json.loads(msg)
if 'mailbox' in msg:
mailbox = yield client.examine(msg['mailbox'])
self.write(request, json.dumps({'mailbox': mailbox}))
def on_message(self, websocket, msg):
request = websocket.request
client = request.cache.mailclient
if msg and client:
msg = json.loads(msg)
if 'mailbox' in msg:
mailbox = yield client.examine(msg['mailbox'])
self.write(request, json.dumps({'mailbox': mailbox}))
def _call(self, name, *args, **kwargs):
data = self._get_data(name, *args, **kwargs)
is_ascii = self._encoding == 'ascii'
body = json.dumps(data, ensure_ascii=is_ascii).encode(self._encoding)
if not self._batch:
self._batch = []
self._batch.append(body)
return data['id']
def publish(self, websocket, channel, message=''):
user, authenticated = self.user(websocket)
msg = {'message': message,
'user': user,
'authenticated': authenticated,
'channel': channel}
channel = '%s:%s' % (websocket.cfg.exc_id, channel)
return self._pubsub.publish(channel, json.dumps(msg))
def publish(self, websocket, channel, message=''):
user, authenticated = self.user(websocket)
msg = {'message': message,
'user': user,
'authenticated': authenticated,
'channel': channel}
channel = '%s:%s' % (websocket.cfg.exc_id, channel)
return self._pubsub.publish(channel, json.dumps(msg))
def serialise(self, method=None):
'''Serialise this task using the serialisation ``method``
'''
method = method or 'json'
if method == 'json':
return json.dumps(self.__dict__)
else:
raise ImproperlyConfigured('Unknown serialisation "%s"' % method)
def _call(self, name, *args, **kwargs):
data = self._get_data(name, *args, **kwargs)
body = json.dumps(data).encode('utf-8')
resp = self.http.post(self.url, data=body)
if self._full_response:
return resp
res = resp.on_finished.add_callback(self._end_call)
return res.result if self.http.force_sync else res
def coveralls(http=None, url=None, data_file=None, repo_token=None,
git=None, service_name=None, service_job_id=None,
strip_dirs=None, ignore_errors=False, stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
http = HttpClient(loop=new_event_loop())
if not git:
try:
git = gitrepo()
except Exception: # pragma nocover
pass
data = {'source_files': coverage.coveralls(strip_dirs=strip_dirs,
ignore_errors=ignore_errors)}
if git:
data['git'] = git
if os.environ.get('TRAVIS'):
data['service_name'] = service_name or 'travis-ci'
data['service_job_id'] = os.environ.get('TRAVIS_JOB_ID')
else:
assert repo_token, 'Requires repo_token if not submitting from travis'
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
try:
info = response.json()
code = 0
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
code = 1
stream.write('\n%s\n' % info['message'])
except Exception:
code = 1
stream.write('Critical error %s\n' % response.status_code)
return code
def test_json_with_async_string2(self):
d = Future()
astr = wsgi.AsyncString(d)
response = wsgi.Json({"bla": astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Future)
d.set_result("ciao")
result = yield from result
self.assertEqual(result, json.dumps({"bla": "ciao"}))
def test_json_with_async_string2(self):
d = Deferred()
astr = wsgi.AsyncString(d)
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Deferred)
d.callback('ciao')
result = yield result
self.assertEqual(result, json.dumps({'bla': 'ciao'}))
def test_json_with_async_string2(self):
d = Deferred()
astr = wsgi.AsyncString(d)
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Deferred)
d.callback('ciao')
result = yield result
self.assertEqual(result, json.dumps({'bla': 'ciao'}))
async def test_json_with_async_string2(self):
d = Future()
astr = wsgi.String(d)
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Future)
d.set_result('ciao')
result = await result
self.assertEqual(result, json.dumps({'bla': 'ciao'}))
async def test_json_with_async_string2(self):
d = Future()
astr = wsgi.String(d)
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Future)
d.set_result('ciao')
result = await result
self.assertEqual(result, json.dumps({'bla': 'ciao'}))
def test_create_instance(self):
models = self.mapper(Task)
yield models.create_tables()
task = yield models.task.create(id='bjbhjscbhj', name='foo',
kwargs={'bal': 'foo'})
self.assertEqual(task.id, 'bjbhjscbhj')
self.assertEqual(task.kwargs, {'bal': 'foo'})
data = task.to_json()
self.assertEqual(len(data), 3)
self.assertFalse(task._modified)
# make sure it is serializable
json.dumps(data)
task = yield models.task.get(task.id)
self.assertEqual(task.id, 'bjbhjscbhj')
self.assertEqual(task.kwargs, {'bal': 'foo'})
data = json.dumps(task.to_json())
#
task = yield models.task.get(task.id)
data = json.dumps(task.to_json())
yield models.drop_tables()
def json_response(self, data, status_code=None):
ct = 'application/json'
content_types = self.content_types
if not content_types or ct in content_types:
response = self.response
response.content_type = ct
response.content = json.dumps(data)
if status_code:
response.status_code = status_code
return response
else:
raise HttpException(status=415, msg=content_types)
def _call(self, name, *args, **kwargs):
data = self._get_data(name, *args, **kwargs)
body = json.dumps(data).encode('utf-8')
resp = yield self._http.post(self._url, data=body)
if self._full_response:
coroutine_return(resp)
else:
content = resp.decode_content()
if resp.is_error:
if 'error' not in content:
resp.raise_for_status()
coroutine_return(self.loads(content))
def info_get(self, request):
response = request.response
response.content_type = APPJSON
data = {'websocket': self.websockets_enabled,
'entropy': randint(0, MAXSIZE),
'cookie_needed': self.cookie_needed,
'origins': ['*:*']}
response.content = json.dumps(data)
nocache(response)
cors(request)
response._can_store_cookies = False
return response
def info_get(self, request):
response = request.response
response.content_type = APPJSON
data = {
'websocket': self.websockets_enabled,
'entropy': randint(0, MAXSIZE),
'cookie_needed': self.cookie_needed,
'origins': ['*:*']
}
response.content = json.dumps(data)
nocache(response)
cors(request)
response._can_store_cookies = False
return response
def _encode_params(self, data):
content_type = self.headers.get("content-type")
# No content type given
if not content_type:
if self.encode_multipart:
return encode_multipart_formdata(data, boundary=self.multipart_boundary, charset=self.charset)
else:
content_type = "application/x-www-form-urlencoded"
body = urlencode(data).encode(self.charset)
elif content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
else:
raise ValueError("Don't know how to encode body for %s" % content_type)
return body, content_type
def _encode_params(self):
content_type = self.headers.get('content-type')
# No content type given
if not content_type:
if self.encode_multipart:
return encode_multipart_formdata(
self.data, boundary=self.multipart_boundary,
charset=self.charset)
else:
content_type = 'application/x-www-form-urlencoded'
body = urlencode(self.data).encode(self.charset)
elif content_type in JSON_CONTENT_TYPES:
body = json.dumps(self.data).encode(self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
def on_message(self, websocket, msg):
'''When a new message arrives, it publishes to all listening clients.
'''
if msg:
lines = []
for l in msg.split('\n'):
l = l.strip()
if l:
lines.append(l)
msg = ' '.join(lines)
if msg:
user = websocket.handshake.get('django.user')
if user.is_authenticated():
user = user.username
else:
user = '******'
msg = {'message': msg, 'user': user, 'time': time.time()}
self.pubsub(websocket).publish('webchat', json.dumps(msg))
def _encode_params(self, params):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
content_type = FORM_URL_ENCODED
if hasattr(params, 'read'):
params = params.read()
if content_type in JSON_CONTENT_TYPES:
body = _json.dumps(params)
elif content_type == FORM_URL_ENCODED:
body = urlencode(tuple(split_url_params(params)))
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
params, charset=self.charset)
else:
body = params
return to_bytes(body, self.charset), content_type
def _encode_params(self, params):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
content_type = FORM_URL_ENCODED
if hasattr(params, 'read'):
params = params.read()
if content_type in JSON_CONTENT_TYPES:
body = _json.dumps(params)
elif content_type == FORM_URL_ENCODED:
body = urlencode(tuple(split_url_params(params)))
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
params, charset=self.charset)
else:
body = params
return to_bytes(body, self.charset), content_type
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
def _encode_params(self, data):
content_type = self.headers.get('content-type')
# No content type given, chose one
if not content_type:
if self.encode_multipart:
content_type = MULTIPART_FORM_DATA
else:
content_type = FORM_URL_ENCODED
if content_type in JSON_CONTENT_TYPES:
body = json.dumps(data).encode(self.charset)
elif content_type == FORM_URL_ENCODED:
body = urlencode(data).encode(self.charset)
elif content_type == MULTIPART_FORM_DATA:
body, content_type = encode_multipart_formdata(
data, boundary=self.multipart_boundary, charset=self.charset)
else:
raise ValueError("Don't know how to encode body for %s" %
content_type)
return body, content_type
def render_error(request, exc):
'''Default renderer for errors.'''
cfg = request.get('pulsar.cfg')
debug = cfg.debug if cfg else False
response = request.response
if not response.content_type:
content_type = request.get('default.content_type')
response.content_type = request.content_types.best_match(
content_type or DEFAULT_RESPONSE_CONTENT_TYPES)
content_type = None
if response.content_type:
content_type = response.content_type.split(';')[0]
is_html = content_type == 'text/html'
if debug:
msg = render_error_debug(request, exc, is_html)
else:
msg = escape(error_messages.get(response.status_code) or exc)
if is_html:
msg = textwrap.dedent("""
<h1>{0[reason]}</h1>
{0[msg]}
<h3>{0[version]}</h3>
""").format({
"reason": response.status,
"msg": msg,
"version": request.environ['SERVER_SOFTWARE']
})
#
if content_type == 'text/html':
doc = HtmlDocument(title=response.status)
doc.head.embedded_css.append(error_css)
doc.body.append(Html('div', msg, cn='pulsar-error'))
return doc.render(request)
elif content_type in JSON_CONTENT_TYPES:
return json.dumps({'status': response.status_code, 'message': msg})
else:
return '\n'.join(msg) if isinstance(msg, (list, tuple)) else msg
def render_error(request, exc):
'''Default renderer for errors.'''
cfg = request.get('pulsar.cfg')
debug = cfg.debug if cfg else False
response = request.response
if not response.content_type:
response.content_type = request.content_types.best_match(
DEFAULT_RESPONSE_CONTENT_TYPES)
content_type = None
if response.content_type:
content_type = response.content_type.split(';')[0]
if content_type == 'text/html':
request.html_document.head.title = response.status
if debug:
msg = render_error_debug(request, exc, content_type)
else:
msg = error_messages.get(response.status_code) or ''
if content_type == 'text/html':
msg = textwrap.dedent("""
<h1>{0[reason]}</h1>
{0[msg]}
<h3>{0[version]}</h3>
""").format({"reason": response.status, "msg": msg,
"version": request.environ['SERVER_SOFTWARE']})
#
if content_type == 'text/html':
doc = request.html_document
doc.head.embedded_css.append(error_css)
doc.body.append(Html('div', msg, cn='pulsar-error'))
return doc.render(request)
elif content_type in JSON_CONTENT_TYPES:
return json.dumps({'status': response.status_code,
'message': msg})
else:
return '\n'.join(msg) if isinstance(msg, (list, tuple)) else msg
def to_string(self, stream):
if len(stream) == 1 and not self.as_list:
return json.dumps(stream[0])
else:
return json.dumps(stream)
def to_string(self, stream):
if len(stream) == 1 and not self.as_list:
return json.dumps(stream[0])
else:
return json.dumps(stream)
def __call__(self, twitter, messages):
if not self.store:
self.store = create_store(twitter.cfg.data_store)
self.pubsub = self.store.pubsub()
for message in messages:
self.pubsub.publish(self.channel, json.dumps(message))
def encode(self, message):
'''Encode a message when publishing.'''
if not isinstance(message, dict):
message = {'message': message}
message['time'] = time.time()
return json.dumps(message)
def coveralls(http=None,
url=None,
data_file=None,
repo_token=None,
git=None,
service_name=None,
service_job_id=None,
strip_dirs=None,
ignore_errors=False,
stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
http = HttpClient(loop=new_event_loop())
if not git:
try:
git = gitrepo()
except Exception: # pragma nocover
pass
data = {
'source_files':
coverage.coveralls(strip_dirs=strip_dirs, ignore_errors=ignore_errors)
}
if git:
data['git'] = git
if os.environ.get('TRAVIS'):
data['service_name'] = service_name or 'travis-ci'
data['service_job_id'] = os.environ.get('TRAVIS_JOB_ID')
else:
assert repo_token, 'Requires repo_token if not submitting from travis'
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
try:
info = response.json()
code = 0
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
code = 1
stream.write('\n%s\n' % info['message'])
except Exception:
code = 1
stream.write('Critical error %s\n' % response.status_code)
return code
def _send_mailboxes(self, websocket):
request = websocket.handshake
result = yield request.cache.mailclient.list("", "*")
result = sorted([e[2] for e in result])
websocket.write(json.dumps({'list': result}))
def to_string(self, stream):
stream = stream
if len(stream) == 1 and not self.as_list:
stream = stream[0]
return json.dumps(stream, ensure_ascii=self.charset == 'ascii')
def on_message(self, websocket, msg):
websocket.write(json.dumps([(i, random()) for i in range(100)]))
def generate(self):
# yield a byte so that headers are sent
yield b''
# we must have the headers now
yield json.dumps(dict(self.headers))
def _send_mailboxes(self, websocket):
request = websocket.handshake
result = yield request.cache.mailclient.list("", "*")
result = sorted([e[2] for e in result])
websocket.write(json.dumps({'list': result}))
def json_encoder(message):
v = {'message': message, 'time': time.time()}
return json.dumps(v)
