def execute_transaction(self, transaction): '''Execute a ``transaction`` ''' updates = [] models = [] for command in transaction.commands: action = command.action if not action: raise NotImplementedError else: model = command.args updates.append(dict(self.model_data(model, action))) models.append(model) # if updates: executed = yield self.update_documents(self._database, updates) errors = [] for doc, model in zip(executed, models): if doc.get('ok'): model['id'] = doc['id'] model['_rev'] = doc['rev'] model._modified.clear() elif doc.get('error'): errors.append(CouchDbError(doc['error'], doc['reason'])) if errors: raise errors[0] coroutine_return(models)
def run(self, consumer, runner, testcls, all_tests): '''Run all test functions from the :attr:`testcls`. It uses the following algorithm: * Run the class method ``setUpClass`` of :attr:`testcls` if defined, unless the test class should be skipped * Call :meth:`run_test` for each test functions in :attr:`testcls` * Run the class method ``tearDownClass`` of :attr:`testcls` if defined, unless the test class should be skipped. ''' cfg = testcls.cfg loop = consumer._loop runner.startTestClass(testcls) error = None sequential = getattr(testcls, '_sequential_execution', cfg.sequential) skip_tests = getattr(testcls, '__unittest_skip__', False) if not skip_tests: error = yield self._run(runner, testcls, 'setUpClass', add_err=False) # run the tests if sequential: # Loop over all test cases in class for test in all_tests: yield self.run_test(test, runner, error) else: all = (self.run_test(test, runner, error) for test in all_tests) yield multi_async(all, loop=loop) if not skip_tests: yield self._run(runner, testcls, 'tearDownClass', add_err=False) runner.stopTestClass(testcls) coroutine_return(runner.result)
def execute(self, *args, **options): connection = yield self._pool.connect() with connection: result = yield connection.execute(*args, **options) if isinstance(result, ResponseError): raise result.exception coroutine_return(result)
def get_page(self, request, site_id=None, url=None, id=None, **kw): '''Retrieve a page instance if not already available.''' if 'pages' not in request.cache: site_id = request.app.config['SITE_ID'] page = None page_list = yield request.models.page.filter( site=site_id).load_related('site').all() pages = dict(((p.url, p) for p in page_list)) pages_id = dict(((str(p.id), p) for p in page_list)) if id: assert url == None, 'url and id should provided together!' page = pages_id.get(id) elif isinstance(url, (list, tuple)): for u in url: page = pages.get(u) if page is not None: break else: page = pages.get(url) if page is None: site = yield request.models.site.filter(id=site_id).all() if site: site = site[0] else: site = yield request.models.site.new(id=site_id) else: site = page.site request.cache.site = site request.cache.pages = pages request.cache.pages_id = pages request.cache.page = page coroutine_return(request.cache.page)
def _query_view(self, callback=None, reduce=True, query_type=None, **kw): view = self._store.query_model_view keys = None if self.aggregated: view_name = None for field, lookups in self.aggregated.items(): keys = [] for lookup in lookups: if lookup.type == 'value': keys.append(lookup.value) else: raise NotImplementedError if view_name: raise NotImplementedError else: view_name = field else: view_name = 'id' if query_type: view_name = '%s_%s' % (view_name, query_type) kw['keys'] = keys try: result = yield view(self._meta, view_name, reduce=reduce, **kw) except CouchDbNoViewError: raise odm.QueryError('Couchdb view for %s not available for %s' % (view_name, self._meta)) if callback: query = yield callback(result) elif reduce: query = sum((q['value'] for q in result['rows'])) else: query = self.models((q['doc'] for q in result['rows'])) coroutine_return(query)
def may_pool_task(self, worker): '''Called in the ``worker`` event loop. It pools a new task if possible, and add it to the queue of tasks consumed by the ``worker`` CPU-bound thread.''' next_time = 0 if worker.is_running(): thread_pool = worker.thread_pool if not thread_pool: worker.logger.warning('No thread pool, cannot poll tasks.') elif self.num_concurrent_tasks < self.backlog: if self.max_tasks and self.processed >= self.max_tasks: if not self.num_concurrent_tasks: worker.logger.warning( 'Processed %s tasks. Restarting.') worker.stop() coroutine_return() else: task = yield self.get_task() if task: # Got a new task self.processed += 1 self.concurrent_tasks.add(task.id) thread_pool.apply(self._execute_task, worker, task) else: worker.logger.info('%s concurrent requests. Cannot poll.', self.num_concurrent_tasks) next_time = 1 worker.event_loop.call_later(next_time, self.may_pool_task, worker)
def request(self, method, *bits, **kwargs): '''Execute the HTTP request''' if self._password: try: lock = self._lock if not lock: self._lock = lock = Lock(loop=self._loop) yield lock.acquire() url = '%s/_session' % self._address response = yield self._http.post( url, data={'name': self._user, 'password': self._password}, encode_multipart=False) self.fire_event('request', response) if response.status_code != 200: response.raise_for_status() self._lock = None else: yield lock.acquire() finally: lock.release() url = '%s/%s' % (self._address, '/'.join(bits)) response = yield self._http.request(method, url, data=kwargs, headers=self.headers) self.fire_event('request', response) if response.request.method == 'HEAD': coroutine_return(response) else: data = response.decode_content() if 'error' in data: raise couch_db_error(**data) else: coroutine_return(data)
def execute_pipeline(self, commands, raise_on_error=True): conn = yield self._pool.connect() with conn: result = yield conn.execute_pipeline(commands, raise_on_error) if isinstance(result, ResponseError): raise result.exception coroutine_return(result)
def get_model(self, manager, pkvalue): try: data = yield self.request('get', self._database, pkvalue) except CouchDbError: raise odm.ModelNotFound(pkvalue) else: coroutine_return(self.build_model(manager, data))
def _subscribe(self, *args): if not self._connection: protocol_factory = partial(PubsubProtocol, self, producer=self.store) self._connection = yield self.store.connect(protocol_factory) self._connection.execute(*args) coroutine_return()
def get_task(self, task_id, timeout): if not task_id: try: task_id = yield self.queue.get(timeout) except Empty: coroutine_return() yield self._tasks.get(task_id)
def run_testcls(self, consumer, runner, testcls, all_tests): '''Run all test functions from the :attr:`testcls`. It uses the following algorithm: * Run the class method ``setUpClass`` of :attr:`testcls` if defined, unless the test class should be skipped * Call :meth:`run_test` for each test functions in :attr:`testcls` * Run the class method ``tearDownClass`` of :attr:`testcls` if defined, unless the test class should be skipped. ''' cfg = testcls.cfg runner.startTestClass(testcls) error = None sequential = getattr(testcls, '_sequential_execution', cfg.sequential) skip_tests = getattr(testcls, '__unittest_skip__', False) if not skip_tests: error = yield self._run(runner, testcls, 'setUpClass', add_err=False) # run the tests if sequential: # Loop over all test cases in class for test in all_tests: yield self.run_test(test, runner, error) else: all = (self.run_test(test, runner, error) for test in all_tests) yield multi_async(all) if not skip_tests: yield self._run(runner, testcls, 'tearDownClass', add_err=False) runner.stopTestClass(testcls) coroutine_return(runner.result)
def create_superuser(self, request, *args, **kwargs): for b in self.auth_backends: try: user = yield b.create_superuser(request, *args, **kwargs) if user is not None: coroutine_return(user) except Exception: continue
def queue_task(self, request, jobname, meta_data=None, **kw): if not jobname: raise rpc.InvalidParams('"jobname" is not specified!') meta_data = meta_data or {} meta_data.update(self.task_request_parameters(request)) task_backend = yield self.task_backend() result = yield task_backend.queue_task(jobname, meta_data, **kw) coroutine_return(result)
def _get_app(arbiter, name, safe=True): monitor = arbiter.get_actor(name) if monitor: cfg = yield From(monitor.start_event) if safe: coroutine_return(cfg) else: coroutine_return(monitor.app)
def _get_app(arbiter, name, safe=True): monitor = arbiter.get_actor(name) if monitor: cfg = yield monitor.start_event if safe: coroutine_return(cfg) else: coroutine_return(monitor.app)
def __call__(self, message): '''Send a ``message`` to the server and wait for a response. :return: a :class:`.Future` ''' protocol = yield self.pool.connect() with protocol: result = yield protocol.send(message) coroutine_return(result)
def get_router_page(self, request): handler = request.app_handler rule = handler.rule path = request.path[1:] page = yield self.get_page(request, url=(rule, path)) if isinstance(page, list): pages = dict(((p.url, p) for p in page)) page = pages.get(path, pages.get(rule)) coroutine_return(page)
def authenticate_and_login(self, request, user, **params): ''':meth:`authenticate` and :meth:`login` a ``user``. Raises an :class:`AuthenticationError` if the ``user`` could not be authenticated.''' if user.is_active: user = self.authenticate(request, user, **params) user = yield self.login(request, user) coroutine_return(user) else: raise AuthenticationError('%s is not active' % user)
def _create_connection(self, pair=None): '''Default method for connecting to remote datastore. ''' protocol_factory = self.store.create_protocol host, port = pair or self.store._host _, protocol = yield self._loop.create_connection( protocol_factory, host, port) socket_info = SocketInfo(protocol, self.pool_id) coroutine_return(socket_info)
def rpc_job_list(self, request, jobnames=None): '''Return the list of Jobs registered with task queue with meta information. If a list of ``jobnames`` is given, it returns only jobs included in the list. ''' task_backend = yield self.task_backend() coroutine_return(task_backend.job_list(jobnames=jobnames))
def coro1(): done = yield 3 fut = Future() fut._loop.call_soon(fut.set_exception, ValueError('test')) try: yield fut except ValueError: done += 1 coroutine_return(done)
def _call(self, request): response = request.response data = {} exc_info = None callable = None try: try: data = yield request.body_data() except ValueError: raise InvalidRequest( status=415, msg='Content-Type must be application/json') if data.get('jsonrpc') != self.version: raise InvalidRequest( 'jsonrpc must be supplied and equal to "%s"' % self.version) params = data.get('params') if isinstance(params, dict): args, kwargs = (), params else: args, kwargs = tuple(params or ()), {} # callable = self.get_handler(data.get('method')) result = yield callable(request, *args, **kwargs) except Exception as exc: result = exc exc_info = sys.exc_info() else: try: json.dumps(result) except Exception as exc: result = exc exc_info = sys.exc_info() # res = {'id': data.get('id'), "jsonrpc": self.version} if exc_info: msg = None code = getattr(result, 'fault_code', None) if not code: if isinstance(result, TypeError) and callable: msg = checkarity(callable, args, kwargs, discount=1) code = -32602 if msg else -32603 msg = msg or str(result) or 'JSON RPC exception' code = getattr(result, 'fault_code', code) if code == -32603: logger.error(msg, exc_info=exc_info) else: logger.warning(msg) error = { 'code': code, 'message': msg, 'data': getattr(result, 'data', '') } response.status_code = getattr(result, 'status', 400) res['error'] = error else: res['result'] = result coroutine_return(res)
def rpc_server_info(self, request): '''Return a dictionary of information regarding the server and workers. It invokes the :meth:`extra_server_info` for adding custom information. ''' info = yield pulsar.send('arbiter', 'info') info = yield self.extra_server_info(request, info) coroutine_return(info)
def get(self, request): """Get the form""" user = request.cache.session.user if user.is_authenticated(): yield smart_redirect(request) form = self.LoginForm(request) html = yield form.layout(request, action=request.full_path()) response = yield html.http_response(request) coroutine_return(response)
def get(self, request): '''Get the form''' user = request.cache.session.user if user.is_authenticated(): yield smart_redirect(request) form = self.LoginForm(request) html = yield form.layout(request, action=request.full_path()) response = yield html.http_response(request) coroutine_return(response)
def get(self, request): user = request.cache.session.user if user.is_authenticated(): yield smart_redirect(request) html = Html('div') for api in apis.available(request.app.config): a = api.html_login_link(request) html.append(a) response = yield html.http_response(request) coroutine_return(response)
def get(self, request): user = request.cache.session.user if user.is_authenticated(): yield smart_redirect(request) html = Html("div") for api in apis.available(request.app.config): a = api.html_login_link(request) html.append(a) response = yield html.http_response(request) coroutine_return(response)
def __call__(self, actor): '''Starts an echo server on a newly spawn actor''' address = self.address server = TcpServer(partial(Connection, EchoServerProtocol), actor._loop, self.address) yield server.start_serving() actor.servers['echo'] = server actor.extra['echo-address'] = server.address actor.bind_event('stopping', self._stop_server) coroutine_return(actor)
def logout(self, request): '''Logout user''' for b in self.auth_backends: try: u = yield b.logout(request) except Exception: continue if u is not None: coroutine_return(u) raise LogoutError('Could not logout')
def create_page_form(self, request, path): '''Called when an authorised user lands on a non-existing page.''' form = self.form_factory(request, initial={'url': path}) action = request.full_path('/%s' % request.app.config['PAGE_EDIT_URL'], redirect=request.full_path()) formhtml = yield form.layout(request, action=action) dia = Dialog(request, {'title': 'This page does not exist, Create one ?', 'body': formhtml}) coroutine_return(dia)
def _call(self, request): response = request.response data = {} exc_info = None callable = None try: try: data = yield request.body_data() except ValueError: raise InvalidRequest( status=415, msg='Content-Type must be application/json') if data.get('jsonrpc') != self.version: raise InvalidRequest( 'jsonrpc must be supplied and equal to "%s"' % self.version) params = data.get('params') if isinstance(params, dict): args, kwargs = (), params else: args, kwargs = tuple(params or ()), {} # callable = self.get_handler(data.get('method')) result = yield callable(request, *args, **kwargs) except Exception as exc: result = exc exc_info = sys.exc_info() else: try: json.dumps(result) except Exception as exc: result = exc exc_info = sys.exc_info() # res = {'id': data.get('id'), "jsonrpc": self.version} if exc_info: msg = None code = getattr(result, 'fault_code', None) if not code: if isinstance(result, TypeError) and callable: msg = checkarity(callable, args, kwargs, discount=1) code = -32602 if msg else -32603 msg = msg or str(result) or 'JSON RPC exception' code = getattr(result, 'fault_code', code) if code == -32603: logger.error(msg, exc_info=exc_info) else: logger.warning(msg) error = {'code': code, 'message': msg, 'data': getattr(result, 'data', '')} response.status_code = getattr(result, 'status', 400) res['error'] = error else: res['result'] = result coroutine_return(res)
def rpc_queue_task(self, request, jobname=None, **kw): '''Queue a new ``jobname`` in the task queue. The task can be of any type as long as it is registered in the task queue registry. To check the available tasks call the :meth:`rpc_job_list` function. It returns the task :attr:`~Task.id`. ''' result = yield self.queue_task(request, jobname, **kw) coroutine_return(task_to_json(result))
def rpc_wait_for_task(self, request, id=None, timeout=None): '''Wait for a task to have finished. :param id: the id of the task to wait for. :param timeout: optional timeout in seconds. :return: the json representation of the task once it has finished. ''' if id: task_backend = yield self.task_backend() result = yield task_backend.wait_for_task(id, timeout=timeout) coroutine_return(task_to_json(result))
def post(self, request): app = request.app data, files = yield request.data_and_files() form = self.LoginForm(request, data=data, files=files) valid = yield form.is_valid() if valid: url = request.url_data.get('redirect', '/') response = yield form.redirect(request, url) else: response = yield form.http_response(request) coroutine_return(response)
def post(self, request): app = request.app data, files = yield request.data_and_files() form = self.LoginForm(request, data=data, files=files) valid = yield form.is_valid() if valid: url = request.url_data.get("redirect", "/") response = yield form.redirect(request, url) else: response = yield form.http_response(request) coroutine_return(response)
def create_page_form(self, request, path): '''Called when an authorised user lands on a non-existing page.''' form = self.form_factory(request, initial={'url': path}) action = request.full_path('/%s' % request.app.config['PAGE_EDIT_URL'], redirect=request.full_path()) formhtml = yield form.layout(request, action=action) dia = Dialog(request, { 'title': 'This page does not exist, Create one ?', 'body': formhtml }) coroutine_return(dia)
def _async(self, callable, future, safe=False): while isfuture(future): kw = {} try: resp = yield future except Exception as exc: if not safe: raise future = callable(exc=exc) else: future = callable(resp) coroutine_return(future)
def __call__(self, message): '''Send a ``message`` to the server and wait for a response. :return: a :class:`.Future` ''' connection = yield self.pool.connect() with connection: consumer = connection.current_consumer() consumer.start(message) result = yield consumer.on_finished result = consumer if self.full_response else consumer.buffer coroutine_return(result)