def pull_from_data_pipes(page):
    """
    Receives a page of data-pipes.
    Invokes the Twitter API to fetch the home-timeline per each data-pipe.
    Send the received results to the buildings-creator task
    :param page: batch of data-pipe objects read from the database.
    """
    # TODO send to an async web-fetcher service (Tornado)
    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    count = 0
    for dp in page:
        if dp.get("connectedBldg") is None:
            logging.warning("Data-pipe {} still not connected to anything"
                            .format(dp["_id"]))
            continue
        auth.set_access_token(dp["tokens"]["accessToken"],
                              dp["tokens"]["accessTokenSecret"])
        t = tweepy.API(auth)
        args = {"count": TWITTER_POSTS_LIMIT}
        latest_id = dp.get("latestId")
        new_latest_id = None
        max_id = None
        done = False
        batch_count = 0
        while not done:
            keys = []
            payloads = []
            batch_count += 1
            if latest_id is not None:
                args["since_id"] = latest_id
            if max_id is not None:
                args["max_id"] = max_id - 1
            logging.info("Invoking twitter API with args: {}".format(args))
            results = t.home_timeline(**args)

            for post in results:
                if latest_id is not None and post.id <= latest_id:
                    done = True
                elif new_latest_id is None or post.id > new_latest_id:
                    new_latest_id = post.id
                if not done:
                    max_id = post.id
                    keys.append(post.id)
                    payloads.append(extract_payload_from_post(post))
                    count += 1
            else:
                done = True

            if latest_id is None and new_latest_id is not None:
                done = True

            if payloads:
                # TODO check whether the connected bldg is a flr or a bldg
                target_flr = dp["connectedBldg"] + "-l0"
                logging.info("Sending {} buildings to {}.."
                             .format(len(payloads), target_flr))
                create_buildings.s(TWITTER_SOCIAL_POST,
                                   keys, payloads, target_flr) \
                    .apply_async()
                if new_latest_id is not None:
                    update_data_pipe(dp["_id"],
                                     {"latestId": new_latest_id})
    return count
Esempio n. 2
0
def _update_data_pipe(address, data_pipe):
    update_data_pipe(data_pipe["_id"], {
        "connectedBldg": address
    })
def test_update_data_pipe(get_db):
    db = get_db.return_value
    change = {"field1": "value 1"}
    dp_id = 1
    update_data_pipe(dp_id, change)
    db.data_pipes.update.assert_called_once_with({'_id': dp_id}, {'$set': change})