Ejemplo n.º 1
0
	def render_all_docs(self, request):
		# /database/_all_docs?stuff => database, _all_docs?stuff
		database, rest = request.path[1:].split('/', 1)
		deferred = defer.Deferred()
		deferred.addCallback(make_success_callback(request))
		deferred.addErrback(make_errback(request))

		# build the query string to send to shard requests
		strip_params = ['skip'] # handled by smartproxy, do not pass
		args = dict([(k,v) for k,v in request.args.iteritems() if k.lower() not in strip_params])
		qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '')
		if qs: qs = '?' + qs

		# this is exactly like a view with no reduce
		shards = self.conf_data.shards(database)
		reducer = AllDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue)

		#hash keys
		numShards = len(shards)
		shardContent = [[] for x in shards]
		body = get_body(request, {})
		if 'keys' in body:
			for key in body['keys']:
				where = lounge_hash(key)%numShards
				shardContent[where].append(key)

		for i,shard in enumerate(shards):
			nodes = self.conf_data.nodes(shard)
			urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes]
			shardBody =cjson.encode(dict(keys=shardContent[i]))
			fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue, body=shardBody)

			fetcher.fetch(request)

		return server.NOT_DONE_YET
Ejemplo n.º 2
0
    def render_temp_view(self, request):
        """Farm out a view query to all nodes and combine the results."""
        deferred = defer.Deferred()
        deferred.addCallback(make_success_callback(request))
        deferred.addErrback(make_errback(request))

        raw_body = request.content.read()
        body = cjson.decode(raw_body)
        reduce_fn = body.get("reduce", None)
        if reduce_fn is not None:
            reduce_fn = reduce_fn.replace("\n", " ")  # TODO do we need this?

        uri = request.uri[1:]
        db, req = uri.split("/", 1)
        shards = self.conf_data.shards(db)
        reducer = Reducer(reduce_fn, len(shards), {}, deferred, self.reduce_queue)

        failed = False
        for shard in shards:

            def succeed(data):
                log.err("This should not get called?")
                pass

            def fail(data):
                if not failed:
                    failed = True
                    deferred.errback(data)

            shard_deferred = defer.Deferred()
            shard_deferred.addCallback(succeed)
            shard_deferred.addErrback(fail)

            nodes = self.conf_data.nodes(shard)
            urls = ["/".join([node, req]) for node in nodes]
            fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue)
            fetcher.fetch(request)

        return server.NOT_DONE_YET
Ejemplo n.º 3
0
	def render_all_docs(self, request):
		# /database/_all_docs?stuff => database, _all_docs?stuff
		database, rest = request.path[1:].split('/', 1)
		deferred = defer.Deferred()
		deferred.addCallback(make_success_callback(request))
		deferred.addErrback(make_errback(request))

		# build the query string to send to shard requests
		strip_params = ['skip'] # handled by smartproxy, do not pass
		args = dict([(k,v) for k,v in request.args.iteritems() if k.lower() not in strip_params])
		qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '')
		if qs: qs = '?' + qs

		# this is exactly like a view with no reduce
		shards = self.conf_data.shards(database)
		reducer = AllDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue)
		for shard in shards:
			nodes = self.conf_data.nodes(shard)
			urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes]
			fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue)
			fetcher.fetch(request)

		return server.NOT_DONE_YET
Ejemplo n.º 4
0
	def do_bulk_docs(self, request):
		"""Split and farm out bulk docs"""
		# /database/_all_docs?stuff => database, _all_docs?stuff
		database, rest = request.path[1:].split('/', 1)
		deferred = defer.Deferred()
		deferred.addCallback(make_success_callback(request))
		deferred.addErrback(make_errback(request))

		# build the query string to send to shard requests
		args = dict([(k,v) for k,v in request.args.iteritems()])
		qs = urllib.urlencode([(k,v) for k in args for v in args[k]] or '')
		if qs: qs = '?' + qs

		# this is exactly like a view with no reduce
		shards = self.conf_data.shards(database)
		reducer = BulkDocsReducer(None, len(shards), request.args, deferred, self.reduce_queue)

		#hash keys
		numShards = len(shards)
		shardContent = [[] for x in shards]
		body = get_body(request, {})
		if 'docs' in body:
			for doc in body['docs']:
				doc_id = doc['_id']
				where = lounge_hash(doc_id) % numShards
				shardContent[where].append(doc)

		for i,shard in enumerate(shards):
			nodes = self.conf_data.nodes(shard)
			urls = [self._rewrite_url("/".join([node, rest])) + qs for node in nodes]
			shardBody = cjson.encode(dict(docs=shardContent[i]))

			fetcher = MapResultFetcher(shard, urls, reducer, deferred, self.client_queue, body=shardBody)
			fetcher.fetch(request)

		return server.NOT_DONE_YET