Example #1
0
    def _init_futures(self):
        """Build futures for results and output; hook up callbacks"""
        if not self._children:
            for msg_id in self.msg_ids:
                future = self._client._futures.get(msg_id, None)
                if not future:
                    result = self._client.results.get(msg_id, _default)
                    # result resides in local cache, construct already-resolved Future
                    if result is not _default:
                        future = MessageFuture(msg_id)
                        future.output = Future()
                        future.output.metadata = self.client.metadata[msg_id]
                        future.set_result(result)
                        future.output.set_result(None)
                if not future:
                    raise KeyError("No Future or result for msg_id: %s" %
                                   msg_id)
                self._children.append(future)

        self._result_future = multi_future(self._children)

        self._sent_future = multi_future([f.tracker for f in self._children])
        self._sent_future.add_done_callback(self._handle_sent)

        self._output_future = multi_future([self._result_future] +
                                           [f.output for f in self._children])
        # on completion of my constituents, trigger my own resolution
        self._result_future.add_done_callback(self._resolve_result)
        self._output_future.add_done_callback(self._resolve_output)
        self.add_done_callback(self._finalize_result)
Example #2
0
    def _init_futures(self):
        """Build futures for results and output; hook up callbacks"""
        if not self._children:
            for msg_id in self.msg_ids:
                future = self._client._futures.get(msg_id, None)
                if not future:
                    result = self._client.results.get(msg_id, _default)
                    # result resides in local cache, construct already-resolved Future
                    if result is not _default:
                        future = MessageFuture(msg_id)
                        future.output = Future()
                        future.output.metadata = self.client.metadata[msg_id]
                        future.set_result(result)
                        future.output.set_result(None)
                if not future:
                    raise KeyError("No Future or result for msg_id: %s" % msg_id)
                self._children.append(future)

        self._result_future = multi_future(self._children)

        self._sent_future = multi_future([f.tracker for f in self._children])
        self._sent_future.add_done_callback(self._handle_sent)

        self._output_future = multi_future([self._result_future] + [f.output for f in self._children])
        # on completion of my constituents, trigger my own resolution
        self._result_future.add_done_callback(self._resolve_result)
        self._output_future.add_done_callback(self._resolve_output)
Example #3
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future([
         self.delay(3, "v1"),
         self.delay(1, "v2"),
     ])
     self.assertEqual(responses, ["v1", "v2"])
Example #4
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future(
         [gen.Task(self.delay_callback, 3, arg="v1"), gen.Task(self.delay_callback, 1, arg="v2")]
     )
     self.assertEqual(responses, ["v1", "v2"])
     self.stop()
Example #5
0
    def test_multi_future_exceptions(self):
        with ExpectLog(app_log, "Multiple exceptions in yield list"):
            with self.assertRaises(RuntimeError) as cm:
                yield [
                    self.async_exception(RuntimeError("error 1")),
                    self.async_exception(RuntimeError("error 2")),
                ]
        self.assertEqual(str(cm.exception), "error 1")

        # With only one exception, no error is logged.
        with self.assertRaises(RuntimeError):
            yield [
                self.async_exception(RuntimeError("error 1")),
                self.async_future(2)
            ]

        # Exception logging may be explicitly quieted.
        with self.assertRaises(RuntimeError):
            yield gen.multi_future(
                [
                    self.async_exception(RuntimeError("error 1")),
                    self.async_exception(RuntimeError("error 2")),
                ],
                quiet_exceptions=RuntimeError,
            )
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future([
         self.delay(3, "v1"),
         self.delay(1, "v2"),
     ])
     self.assertEqual(responses, ["v1", "v2"])
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future(dict(
         foo=self.delay(3, "v1"),
         bar=self.delay(1, "v2"),
     ))
     self.assertEqual(responses, dict(foo="v1", bar="v2"))
Example #8
0
 def wait_until_finished(self):
     # if there are futures to wait for
     while self.futures:
         # wait for the futures to finish
         futures = self.futures
         self.futures = []
         self.loop.run_sync(lambda: multi_future(futures))
Example #9
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future(dict(
         foo=self.delay(3, "v1"),
         bar=self.delay(1, "v2"),
     ))
     self.assertEqual(responses, dict(foo="v1", bar="v2"))
Example #10
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future(
         dict(foo=gen.Task(self.delay_callback, 3, arg="v1"), bar=gen.Task(self.delay_callback, 1, arg="v2"))
     )
     self.assertEqual(responses, dict(foo="v1", bar="v2"))
     self.stop()
Example #11
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future([
         gen.Task(self.delay_callback, 3, arg="v1"),
         gen.Task(self.delay_callback, 1, arg="v2"),
     ])
     self.assertEqual(responses, ["v1", "v2"])
     self.stop()
Example #12
0
 def f():
     # callbacks run at different times
     responses = yield gen.multi_future(dict(
         foo=gen.Task(self.delay_callback, 3, arg="v1"),
         bar=gen.Task(self.delay_callback, 1, arg="v2"),
     ))
     self.assertEqual(responses, dict(foo="v1", bar="v2"))
     self.stop()
Example #13
0
 def put(self, *args, **kwargs):
     futures = []
     for i, region in enumerate(self.regions_list):
         db_instance = getattr(self, 'ddb-%s' % region)
         futures.append(db_instance.put(*args, **kwargs))
     try:
         yield gen.multi_future(futures)
     except Exception, e:
         self._process_multiple_exceptions(futures)
Example #14
0
	def search_movie(self, search):
		try:
			print('search movie', search)
			if not self.movie:
				self._get_movie_obj(all=True)
			futures = []
			for m in self.movie:
				result = m.search(search)
				futures.append(result)
			result = yield gen.multi_future(futures)
			return result
		except Exception as e:
			traceback.print_exc(file=sys.stdout)
Example #15
0
    def test_multi_future_exceptions(self):
        with ExpectLog(app_log, "Multiple exceptions in yield list"):
            with self.assertRaises(RuntimeError) as cm:
                yield [self.async_exception(RuntimeError("error 1")), self.async_exception(RuntimeError("error 2"))]
        self.assertEqual(str(cm.exception), "error 1")

        # With only one exception, no error is logged.
        with self.assertRaises(RuntimeError):
            yield [self.async_exception(RuntimeError("error 1")), self.async_future(2)]

        # Exception logging may be explicitly quieted.
        with self.assertRaises(RuntimeError):
            yield gen.multi_future(
                [self.async_exception(RuntimeError("error 1")), self.async_exception(RuntimeError("error 2"))],
                quiet_exceptions=RuntimeError,
            )
Example #16
0
    def test_api_call_queue_raises_exceptions(self):
        """
        Test that the api call queue raises exceptions and proceeds to execute
        other queued api calls.
        """
        @gen.coroutine
        def _call_without_exception():
            result = yield self.api_call_queue.call(
                self._mock_api_function_sync, delay=0.05)
            self.assertEqual(result, 'OK')
            raise gen.Return('no exception')

        @gen.coroutine
        def _call_with_exception():
            err = ValueError('test exception')
            try:
                yield self.api_call_queue.call(self._mock_api_function_sync,
                                               exception=err,
                                               delay=0.05)
            except Exception as e:
                self.assertEqual(err, e)
            raise gen.Return('exception')

        @gen.coroutine
        def _call_with_exception_after_boto2_rate_limit():
            """
            First rate limit, then raise an exception.
            This should take:
                call delay * 2 + min rate limiting delay * 1
            """
            try:
                yield self.api_call_queue.call(
                    self._mock_api_function_sync,
                    exception=[
                        self.boto2_throttle_exception_1, self.boto2_exception
                    ],
                    delay=0.05)
            except Exception as e:
                self.assertEqual(self.boto2_exception, e)

            raise gen.Return('exception')

        @gen.coroutine
        def _call_with_exception_after_boto3_rate_limit():
            """
            First rate limit, then raise an exception.
            This should take:
                call delay * 2 + min rate limiting delay * 1
            """
            try:
                yield self.api_call_queue.call(
                    self._mock_api_function_sync,
                    exception=[
                        self.boto3_throttle_exception, self.boto3_exception
                    ],
                    delay=0.05)
            except Exception as e:
                self.assertEqual(self.boto3_exception, e)

            raise gen.Return('exception')

        call_wrappers = [
            # Should take 0.05s.
            _call_without_exception(),
            # Should take 0.05s.
            _call_with_exception(),
            # Should take 0.05s.
            _call_without_exception(),
            # Should take 0.05s + 0.05s + 0.05s.
            _call_with_exception_after_boto2_rate_limit(),
            # Should take 0.05s + 0.05s + 0.05s.
            _call_with_exception_after_boto3_rate_limit(),
        ]

        start = time.time()
        results = yield gen.multi_future(call_wrappers)
        stop = time.time()
        run_time = stop - start

        self.assertTrue(0.45 <= run_time < 0.55)
        self.assertEqual(results, [
            'no exception', 'exception', 'no exception', 'exception',
            'exception'
        ])
Example #17
0
    def _searcher(self,
                  string,
                  find_at,
                  query={},
                  database=None,
                  count=50,
                  cache=1800):
        if not self.site_id:
            self.site_id = self.site.site_db['_id']

        if self.site_id:
            seo_string = function.seo_encode(string)
            #  default database
            if not database:
                database = self.site.db.post

            # in cache
            if cache > 0:
                result = yield self.site.cache.get(source="%s%s" %
                                                   (seo_string, find_at))
                if result:
                    return result

            words = []
            word = seo_string

            # generic keywork
            while True:
                if not word in words:
                    words.append(word)
                if '-' in word:
                    word = word.rsplit('-', 1)[0]
                else:
                    break

            word = seo_string
            while True:
                if word and not word in words:
                    words.append(word)
                if '-' in word:
                    word = word.split('-', 1)[1]
                else:
                    break

            # start search
            if not 'site_id' in query:
                query['site_id'] = self.site_id

            search_dict = {}
            search_future = []
            for word in words:
                word = word.strip()
                if len(word) > 0:
                    # change search query
                    query[find_at] = {
                        '$regex': re.compile(".*" + word + ".*", re.IGNORECASE)
                    }
                    search_future.append(
                        database.find(query, {
                            "_id": 1
                        }).to_list(length=count))

            search_result = yield gen.multi_future(search_future)
            for result in search_result:
                if result:
                    for mv in result:
                        if mv['_id'] not in search_dict:
                            search_dict[mv['_id']] = 1
                        else:
                            search_dict[mv['_id']] += 1

            # sort result
            sort_dict = {}
            for k, v in search_dict.items():
                if v not in sort_dict:
                    sort_dict[v] = [k]
                else:
                    sort_dict[v].append(k)
            result = [x for s in sort_dict.values() for x in s][::-1][:count]

            # cache store
            if cache > 0:
                yield self.site.cache.set(seo_string, result, cache)
            return result
        return []
Example #18
0
	def _searcher(self, string, find_at, query ={}, database=None, count=50, cache=1800):
		if not self.site_id:
			self.site_id 	= self.site.site_db['_id']

		if self.site_id:
			seo_string 	= function.seo_encode(string)
			#  default database
			if not database:
				database = self.site.db.post

			# in cache
			if cache > 0:
				result 	= yield self.site.cache.get(source="%s%s" % (seo_string, find_at))
				if result:
					return result

			words = []
			word = seo_string
			
			# generic keywork
			while True:
				if not word in words:
					words.append(word)
				if '-' in word:
					word = word.rsplit('-',1)[0]
				else:
					break

			word = seo_string
			while True:
				if word and not word in words:
					words.append(word)
				if '-' in word:
					word = word.split('-',1)[1]
				else:
					break

			# start search
			if not 'site_id' in query:
				query['site_id'] = self.site_id

			search_dict 	= {}
			search_future 	= []
			for word in words:
				word 	= word.strip()
				if len(word) > 0:
					# change search query
					query[find_at] = {'$regex': re.compile(".*" + word + ".*", re.IGNORECASE)}
					search_future.append(database.find(query, {"_id": 1}).to_list(length=count))
			
			search_result = yield gen.multi_future(search_future)
			for result in search_result:
				if result:
					for mv in result:
						if mv['_id'] not in search_dict:
							search_dict[mv['_id']] = 1
						else:
							search_dict[mv['_id']] += 1
			
			# sort result
			sort_dict 	= {}
			for k, v in search_dict.items():
				if v not in sort_dict:
					sort_dict[v] 	= [k]
				else:
					sort_dict[v].append(k)
			result 	= [x for s in sort_dict.values() for x in s][::-1][:count]
			
			# cache store
			if cache > 0:
				yield self.site.cache.set(seo_string, result, cache)
			return result
		return []