def setUpClass(cls):
name = 'calc_' + cls.concurrency
s = server(bind='127.0.0.1:0', name=name, concurrency=cls.concurrency)
cls.app_cfg = yield send('arbiter', 'run', s)
cls.uri = 'http://{0}:{1}'.format(*cls.app_cfg.addresses[0])
cls.p = rpc.JsonProxy(cls.uri, timeout=cls.rpc_timeout)
cls.sync = rpc.JsonProxy(cls.uri, loop=new_event_loop())
def __init__(self,
url,
version=None,
data=None,
full_response=False,
http=None,
timeout=None,
sync=False,
loop=None,
encoding='ascii',
**kw):
self.sync = sync
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
if sync and not loop:
loop = new_event_loop()
http = HttpClient(timeout=timeout, loop=loop, **kw)
http.headers['accept'] = 'application/json, text/*; q=0.5'
http.headers['content-type'] = 'application/json'
self._http = http
self._encoding = encoding
def setUpClass(cls):
name = 'calc_' + cls.concurrency
s = server(bind='127.0.0.1:0', name=name, concurrency=cls.concurrency)
cls.app_cfg = yield send('arbiter', 'run', s)
cls.uri = 'http://{0}:{1}'.format(*cls.app_cfg.addresses[0])
cls.p = rpc.JsonProxy(cls.uri, timeout=cls.rpc_timeout)
cls.sync = rpc.JsonProxy(cls.uri, loop=new_event_loop())
def create_store(url, loop=None, **kw):
'''Create a new client :class:`Store` for a valid ``url``.
A valid ``url`` taks the following forms::
pulsar://user:[email protected]:6410
redis://user:[email protected]:6500/11?namespace=testdb.
postgresql://user:[email protected]:6500/testdb
couchdb://user:[email protected]:6500/testdb
:param loop: optional event loop, if not provided it is obtained
via the ``get_event_loop`` method. If not loop is installed a bright
new event loop is created via the :func:`.new_event_loop`.
In the latter case the event loop is employed only for synchronous type
requests via the :meth:`~.EventLoop.run_until_complete` method.
:param kw: additional key-valued parameters to pass to the :class:`Store`
initialisation method.
:return: a :class:`Store`.
'''
if isinstance(url, Store):
return url
scheme, address, params = parse_store_url(url)
dotted_path = data_stores.get(scheme)
if not dotted_path:
raise ImproperlyConfigured('%s store not available' % scheme)
loop = loop or get_event_loop()
if not loop:
loop = new_event_loop(logger=logging.getLogger(dotted_path))
store_class = module_attribute(dotted_path)
params.update(kw)
return store_class(scheme, address, loop, **params)
def __init__(
self,
url,
version=None,
data=None,
full_response=False,
http=None,
timeout=None,
sync=False,
loop=None,
encoding="ascii",
**kw
):
self.sync = sync
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
if sync and not loop:
loop = new_event_loop()
http = HttpClient(timeout=timeout, loop=loop, **kw)
http.headers["accept"] = "application/json, text/*; q=0.5"
http.headers["content-type"] = "application/json"
self._http = http
self._encoding = encoding
def test_sync_store(self):
loop = new_event_loop()
store = self.create_store(self.cfg.couchdb_server, loop=loop)
self.assertEqual(store._loop, loop)
self.assertEqual(store._http._loop, loop)
result = store.ping()
self.assertTrue('version' in result)
self.assertEqual(result['couchdb'], 'Welcome')
def test_sync_store(self):
loop = new_event_loop()
store = self.create_store(self.cfg.couchdb_server, loop=loop)
self.assertEqual(store._loop, loop)
self.assertEqual(store._http._loop, loop)
result = store.ping()
self.assertTrue("version" in result)
self.assertEqual(result["couchdb"], "Welcome")
def http_sessions(self, model=None):
"""Return an HTTP session handler for a given concurrency model
"""
if model == concurrency.THREAD_IO:
return HttpClient(loop=new_event_loop())
elif model == concurrency.ASYNC_IO:
return self.http
else:
return GreenHttp(self.http)
def coveralls(http=None, url=None, data_file=None, repo_token=None,
git=None, service_name=None, service_job_id=None,
strip_dirs=None, ignore_errors=False, stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
http = HttpClient(loop=new_event_loop())
if not git:
try:
git = gitrepo()
except Exception: # pragma nocover
pass
data = {'source_files': coverage.coveralls(strip_dirs=strip_dirs,
ignore_errors=ignore_errors)}
if git:
data['git'] = git
if os.environ.get('TRAVIS'):
data['service_name'] = service_name or 'travis-ci'
data['service_job_id'] = os.environ.get('TRAVIS_JOB_ID')
else:
assert repo_token, 'Requires repo_token if not submitting from travis'
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
try:
info = response.json()
code = 0
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
code = 1
stream.write('\n%s\n' % info['message'])
except Exception:
code = 1
stream.write('Critical error %s\n' % response.status_code)
return code
def minify(self, options, data):
b = convert_bytes(len(data))
self.write('Minimise %s css file via http://cssminifier.com' % b)
http = HttpClient(loop=new_event_loop())
response = http.post('http://cssminifier.com/raw',
data={'input': data})
if response.status_code == 200:
return native_str(response.get_content())
else:
response.raise_for_status()
def minify(self, options, data):
b = convert_bytes(len(data))
self.write('Minimise %s css file via http://cssminifier.com' % b)
http = HttpClient(loop=new_event_loop())
response = http.post('http://cssminifier.com/raw',
data={'input': data})
if response.status_code == 200:
return native_str(response.get_content())
else:
response.raise_for_status()
def run_async(self, argv, **params):
'''Run a command using pulsar asynchronous engine.'''
loop = get_event_loop()
run_until_complete = False
if loop is None:
run_until_complete = True
loop = new_event_loop()
future = async (self.run(argv, **params), loop)
if run_until_complete:
return loop.run_until_complete(future)
else:
return future
def run_async(self, argv, **params):
'''Run a command using pulsar asynchronous engine.'''
loop = get_event_loop()
run_until_complete = False
if loop is None:
run_until_complete = True
loop = new_event_loop()
future = async(self.run(argv, **params), loop)
if run_until_complete:
return loop.run_until_complete(future)
else:
return future
def __test_close_sync(self):
loop = new_event_loop()
c = self.http(loop=loop)
handler = Echo(loop)
ws = c.get(self.ws_echo, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
self.assertEqual(ws._loop, loop)
self.assertFalse(ws._loop.is_running())
ws.write('send close 1001')
message = ws._loop.run_until_complete(handler.get())
self.assertEqual(message, 'CLOSE')
self.assertTrue(ws.close_reason)
self.assertEqual(ws.close_reason[0], 1001)
self.assertTrue(ws._connection.closed)
def create_store(url, loop=None, **kw):
'''Create a new :class:`Store` for a valid ``url``.
:param url: a valid ``url`` takes the following forms:
:ref:`Pulsar datastore <store_pulsar>`::
pulsar://user:[email protected]:6410
:ref:`Redis <store_redis>`::
redis://user:[email protected]:6500/11?namespace=testdb
:ref:`CouchDb <store_couchdb>`::
couchdb://user:[email protected]:6500/testdb
https+couchdb://user:[email protected]:6500/testdb
:param loop: optional event loop, obtained by
:func:`~asyncio.get_event_loop` if not provided.
To create a synchronous client pass a new event loop created via
the :func:`~asyncio.new_event_loop`.
In the latter case the event loop is employed only for synchronous
type requests via the :meth:`~asyncio.BaseEventLoop.run_until_complete`
method.
:param kw: additional key-valued parameters to pass to the :class:`.Store`
initialisation method. It can contains parameters such as
``database``, ``user`` and ``password`` to override the
``url`` values. Additional parameters are processed by the
:meth:`.Store._init` method.
:return: a :class:`Store`.
'''
if isinstance(url, Store):
return url
scheme, address, params = parse_store_url(url)
dotted_path = data_stores.get(scheme)
if not dotted_path:
raise ImproperlyConfigured('%s store not available' % scheme)
loop = loop or get_event_loop()
if not loop:
loop = new_event_loop(logger=logging.getLogger(dotted_path))
store_class = module_attribute(dotted_path)
if not store_class.registered:
store_class.registered = True
store_class.register()
params.update(kw)
return store_class(scheme, address, loop, **params)
def search_engine(url, loop=None, **kw):
if isinstance(url, SearchEngine):
return url
if isinstance(url, dict):
extra = url.copy()
url = extra.pop('url', None)
extra.update(kw)
kw = extra
scheme, address, params = parse_store_url(url)
dotted_path = search_engines.get(scheme)
if not dotted_path:
raise ImproperlyConfigured('%s search engine not available' % scheme)
loop = loop or get_event_loop()
if not loop:
loop = new_event_loop(logger=logging.getLogger(dotted_path))
engine_class = module_attribute(dotted_path)
params.update(kw)
return engine_class(scheme, address, loop, **params)
def media_libraries():
global _media_libraries
if _media_libraries is None:
if os.path.isfile('libs.json'): # pragma nocover
with open('libs.json') as f:
data = f.read()
_media_libraries = json.loads(data)
else:
from pulsar import new_event_loop
from pulsar.apps.http import HttpClient
http = HttpClient(loop=new_event_loop())
try:
response = http.get(_libs_url)
_media_libraries = response.json()
except Exception: # pragma nocover
http.logger.error('Could not import media library',
exc_info=True)
_media_libraries = {'libs': {}, 'deps': {}}
return _media_libraries
def media_libraries():
global _media_libraries
if _media_libraries is None:
if os.path.isfile('libs.json'): # pragma nocover
with open('libs.json') as f:
data = f.read()
_media_libraries = json.loads(data)
else:
from pulsar import new_event_loop
from pulsar.apps.http import HttpClient
http = HttpClient(loop=new_event_loop())
try:
response = http.get(_libs_url)
_media_libraries = response.json()
except Exception: # pragma nocover
http.logger.error('Could not import media library',
exc_info=True)
_media_libraries = {'libs': {}, 'deps': {}}
return _media_libraries
def http(self):
'''Get the HTTP client
'''
if self._http is None:
api_url = self.app.config['API_URL']
headers = [('content-type', 'application/json')]
# Remote API
if is_absolute_uri(api_url):
if self.app.green_pool:
self._http = HttpClient(headers=headers)
else:
self._http = HttpClient(loop=new_event_loop())
# Local API
else:
self._http = LocalClient(self.app, headers)
token = self.app.config['API_AUTHENTICATION_TOKEN']
if token:
self._http.headers['Athentication'] = 'Bearer %s' % token
return self._http
def http(self):
'''Get the HTTP client
'''
if self._http is None:
api_url = self.app.config['API_URL']
headers = [('content-type', 'application/json')]
# Remote API
if is_absolute_uri(api_url):
if self.app.green_pool:
self._http = HttpClient(headers=headers)
else:
self._http = HttpClient(loop=new_event_loop())
# Local API
else:
self._http = LocalClient(self.app, headers)
token = self.app.config['API_AUTHENTICATION_TOKEN']
if token:
self._http.headers['Athentication'] = 'Bearer %s' % token
return self._http
def http(self):
if self._http is None:
if self.app.green_pool:
from pulsar.apps.greenio import wait
self._wait = wait
api_url = self.app.config['API_URL']
# Remote API
if is_absolute_uri(api_url):
if self._wait:
self._http = HttpClient()
else:
self._http = HttpClient(loop=new_event_loop())
# Local API
else:
self._http = LocalClient(self.app)
token = self.app.config['API_AUTHENTICATION_TOKEN']
if token:
self._http.headers['Athentication'] = 'Bearer %s' % token
return self._http
def check_server(name):
'''Check if server ``name`` is available at the address specified
``<name>_server`` config value.
:rtype: boolean
'''
cfg = get_actor().cfg
addr = cfg.get('%s_server' % name)
if addr:
if ('%s://' % name) not in addr:
addr = '%s://%s' % (name, addr)
try:
sync_store = create_store(addr, loop=new_event_loop())
except ImproperlyConfigured:
return False
try:
sync_store._loop.run_until_complete(sync_store.ping())
return True
except Exception:
return False
else:
return False
def check_server(name):
'''Check if server ``name`` is available at the address specified
``<name>_server`` config value.
:rtype: boolean
'''
cfg = get_actor().cfg
addr = cfg.get('%s_server' % name)
if addr:
if ('%s://' % name) not in addr:
addr = '%s://%s' % (name, addr)
try:
sync_store = create_store(addr, loop=new_event_loop())
except ImproperlyConfigured:
return False
try:
sync_store._loop.run_until_complete(sync_store.ping())
return True
except Exception:
return False
else:
return False
def setUpClass(cls):
cls.store = cls.create_store(loop=new_event_loop())
def testLoop(self, loop=None):
self._loop = loop or get_event_loop() or new_event_loop()
def setUpClass(cls):
yield cls.create_pulsar_store()
cls.sync_store = cls.create_store('%s/10' % cls.pulsards_uri,
loop=new_event_loop())
cls.client = cls.store.client()
def setUp(self):
self.loop = new_event_loop()
def setUpClass(cls):
addr = "redis://%s" % cls.cfg.redis_server
cls.store = cls.create_store(addr)
cls.sync_store = create_store(addr, loop=new_event_loop())
cls.client = cls.store.client()
def coveralls(http=None,
url=None,
data_file=None,
repo_token=None,
git=None,
service_name=None,
service_job_id=None,
strip_dirs=None,
ignore_errors=False,
stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
http = HttpClient(loop=new_event_loop())
if not git:
try:
git = gitrepo()
except Exception: # pragma nocover
pass
data = {
'source_files':
coverage.coveralls(strip_dirs=strip_dirs, ignore_errors=ignore_errors)
}
if git:
data['git'] = git
if os.environ.get('TRAVIS'):
data['service_name'] = service_name or 'travis-ci'
data['service_job_id'] = os.environ.get('TRAVIS_JOB_ID')
else:
assert repo_token, 'Requires repo_token if not submitting from travis'
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
try:
info = response.json()
code = 0
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
code = 1
stream.write('\n%s\n' % info['message'])
except Exception:
code = 1
stream.write('Critical error %s\n' % response.status_code)
return code
def sync_client(self):
return Echo(self.server_cfg.addresses[0], loop=new_event_loop())
def sync_client(self):
return Echo(self.server_cfg.addresses[0], loop=new_event_loop())
def setUpClass(cls):
addr = 'redis://%s' % cls.cfg.redis_server
cls.store = cls.create_store(addr)
cls.sync_store = create_store(addr, loop=new_event_loop())
cls.client = cls.store.client()
def setUpClass(cls):
addr = 'redis://%s' % cls.cfg.mongodb_server
cls.store = cls.create_store(addr)
cls.sync_store = cls.create_store(addr, loop=new_event_loop())
def setUpClass(cls):
yield cls.create_pulsar_store()
cls.sync_store = cls.create_store('%s/10' % cls.pulsards_uri,
loop=new_event_loop())
cls.client = cls.store.client()
