def pull_from_data_pipes(page):
"""
Receives a page of data-pipes.
Invokes the Twitter API to fetch the home-timeline per each data-pipe.
Send the received results to the buildings-creator task
:param page: batch of data-pipe objects read from the database.
"""
# TODO send to an async web-fetcher service (Tornado)
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
count = 0
for dp in page:
if dp.get("connectedBldg") is None:
logging.warning("Data-pipe {} still not connected to anything"
.format(dp["_id"]))
continue
auth.set_access_token(dp["tokens"]["accessToken"],
dp["tokens"]["accessTokenSecret"])
t = tweepy.API(auth)
args = {"count": TWITTER_POSTS_LIMIT}
latest_id = dp.get("latestId")
new_latest_id = None
max_id = None
done = False
batch_count = 0
while not done:
keys = []
payloads = []
batch_count += 1
if latest_id is not None:
args["since_id"] = latest_id
if max_id is not None:
args["max_id"] = max_id - 1
logging.info("Invoking twitter API with args: {}".format(args))
results = t.home_timeline(**args)
for post in results:
if latest_id is not None and post.id <= latest_id:
done = True
elif new_latest_id is None or post.id > new_latest_id:
new_latest_id = post.id
if not done:
max_id = post.id
keys.append(post.id)
payloads.append(extract_payload_from_post(post))
count += 1
else:
done = True
if latest_id is None and new_latest_id is not None:
done = True
if payloads:
# TODO check whether the connected bldg is a flr or a bldg
target_flr = dp["connectedBldg"] + "-l0"
logging.info("Sending {} buildings to {}.."
.format(len(payloads), target_flr))
create_buildings.s(TWITTER_SOCIAL_POST,
keys, payloads, target_flr) \
.apply_async()
if new_latest_id is not None:
update_data_pipe(dp["_id"],
{"latestId": new_latest_id})
return count
def _update_data_pipe(address, data_pipe):
update_data_pipe(data_pipe["_id"], {
"connectedBldg": address
})
def test_update_data_pipe(get_db):
db = get_db.return_value
change = {"field1": "value 1"}
dp_id = 1
update_data_pipe(dp_id, change)
db.data_pipes.update.assert_called_once_with({'_id': dp_id}, {'$set': change})