def render_all_docs(self, request): # /database/_all_docs?stuff => database, _all_docs?stuff database, rest = request.path[1:].split('/', 1) deferred = defer.Deferred() deferred.addCallback(make_success_callback(request)) deferred.addErrback(make_errback(request)) # build the query string to send to shard requests strip_params = ['skip'] # handled by smartproxy, do not pass args = dict([(k,v) for k,v in request.args.iteritems() if k.lower() not in strip_params]) qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '') if qs: qs = '?' + qs # this is exactly like a view with no reduce shards = self.conf_data.shards(database) reducer = AllDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue) #hash keys numShards = len(shards) shardContent = [[] for x in shards] body = get_body(request, {}) if 'keys' in body: for key in body['keys']: where = lounge_hash(key)%numShards shardContent[where].append(key) for i,shard in enumerate(shards): nodes = self.conf_data.nodes(shard) urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes] shardBody =cjson.encode(dict(keys=shardContent[i])) fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue, body=shardBody) fetcher.fetch(request) return server.NOT_DONE_YET
def render_temp_view(self, request): """Farm out a view query to all nodes and combine the results.""" deferred = defer.Deferred() deferred.addCallback(make_success_callback(request)) deferred.addErrback(make_errback(request)) raw_body = request.content.read() body = cjson.decode(raw_body) reduce_fn = body.get("reduce", None) if reduce_fn is not None: reduce_fn = reduce_fn.replace("\n", " ") # TODO do we need this? uri = request.uri[1:] db, req = uri.split("/", 1) shards = self.conf_data.shards(db) reducer = Reducer(reduce_fn, len(shards), {}, deferred, self.reduce_queue) failed = False for shard in shards: def succeed(data): log.err("This should not get called?") pass def fail(data): if not failed: failed = True deferred.errback(data) shard_deferred = defer.Deferred() shard_deferred.addCallback(succeed) shard_deferred.addErrback(fail) nodes = self.conf_data.nodes(shard) urls = ["/".join([node, req]) for node in nodes] fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue) fetcher.fetch(request) return server.NOT_DONE_YET
def render_all_docs(self, request): # /database/_all_docs?stuff => database, _all_docs?stuff database, rest = request.path[1:].split('/', 1) deferred = defer.Deferred() deferred.addCallback(make_success_callback(request)) deferred.addErrback(make_errback(request)) # build the query string to send to shard requests strip_params = ['skip'] # handled by smartproxy, do not pass args = dict([(k,v) for k,v in request.args.iteritems() if k.lower() not in strip_params]) qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '') if qs: qs = '?' + qs # this is exactly like a view with no reduce shards = self.conf_data.shards(database) reducer = AllDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue) for shard in shards: nodes = self.conf_data.nodes(shard) urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes] fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue) fetcher.fetch(request) return server.NOT_DONE_YET
def do_bulk_docs(self, request): """Split and farm out bulk docs""" # /database/_all_docs?stuff => database, _all_docs?stuff database, rest = request.path[1:].split('/', 1) deferred = defer.Deferred() deferred.addCallback(make_success_callback(request)) deferred.addErrback(make_errback(request)) # build the query string to send to shard requests args = dict([(k,v) for k,v in request.args.iteritems()]) qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '') if qs: qs = '?' + qs # this is exactly like a view with no reduce shards = self.conf_data.shards(database) reducer = BulkDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue) #hash keys numShards = len(shards) shardContent = [[] for x in shards] body = get_body(request, {}) if 'docs' in body: for doc in body['docs']: doc_id = doc['_id'] where = lounge_hash(doc_id) % numShards shardContent[where].append(doc) for i,shard in enumerate(shards): nodes = self.conf_data.nodes(shard) urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes] shardBody = cjson.encode(dict(docs=shardContent[i])) fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue, body=shardBody) fetcher.fetch(request) return server.NOT_DONE_YET