Exemple #1
0
def async_queries(queries, limit=100000):

    task_runner = AsyncMultiTask()
    for query in queries:
        task_runner.append(QueryTask(query, limit=limit))
    task_runner.run()

    return task_runner
Exemple #2
0
def async_queries(queries, limit=100000):

    task_runner = AsyncMultiTask()
    for query in queries:
        task_runner.append(QueryTask(query, limit=limit))
    task_runner.run()

    return task_runner
Exemple #3
0
    def _fetch(cls, requests):
        """
        Performs the memcache and data store fetches for requests
        whose data isn't in the local cache.
        """

        if not requests:
            return

        cls.listener.on_fetch(requests)

        mkeys = requests.keys()

        # first, try to fetch the keys from memcache
        results = memcache.get_multi(mkeys)

        # if any of the keys couldn't be fetched from
        # memcache, prepare async tasks to fetch
        # their queries from the data store
        runner = AsyncMultiTask()
        has_tasks = False

        # if there are any pending async tasks,
        # execute them using the AsyncMultiTask runner.
        db_fetches = {}

        for mkey, request in requests.items():
            if not results.get(mkey):
                task = QueryTask(request.query,
                                 limit = request.limit,
                                 client_state = mkey)
                runner.append(task)
                has_tasks = True

                cls.listener.on_db_request(request)

        if has_tasks:
            runner.run()
            cls.listener.on_db_fetch()
            for task in runner:
                val = task.get_result()
                mkey = task.client_state

                # if the task only tried to fetch a single object,
                # return the first element of the result list
                # or none
                if task.limit == 1:
                    if val:
                        val = val[0]
                    else:
                        val = None

                results[mkey] = val                
                db_fetches[mkey] = val

        # if any results were fetched from the data store,
        # store them in memcache
        if db_fetches:
            memcache.set_multi(db_fetches)

        return results
Exemple #4
0
    def _fetch(cls, requests):
        """
        Performs the memcache and data store fetches for requests
        whose data isn't in the local cache.
        """

        if not requests:
            return

        cls.listener.on_fetch(requests)

        mkeys = requests.keys()

        # first, try to fetch the keys from memcache
        results = memcache.get_multi(mkeys)

        # if any of the keys couldn't be fetched from
        # memcache, prepare async tasks to fetch
        # their queries from the data store
        runner = AsyncMultiTask()
        has_tasks = False

        # if there are any pending async tasks,
        # execute them using the AsyncMultiTask runner.
        db_fetches = {}

        for mkey, request in requests.items():
            if not results.get(mkey):
                task = QueryTask(request.query,
                                 limit=request.limit,
                                 client_state=mkey)
                runner.append(task)
                has_tasks = True

                cls.listener.on_db_request(request)

        if has_tasks:
            runner.run()
            cls.listener.on_db_fetch()
            for task in runner:
                val = task.get_result()
                mkey = task.client_state

                # if the task only tried to fetch a single object,
                # return the first element of the result list
                # or none
                if task.limit == 1:
                    if val:
                        val = val[0]
                    else:
                        val = None

                results[mkey] = val
                db_fetches[mkey] = val

        # if any results were fetched from the data store,
        # store them in memcache
        if db_fetches:
            memcache.set_multi(db_fetches)

        return results