def idMapper(doc):
yield (None, doc['id'])
def maxFindingReducer(keys, values, rereduce):
return max(values)
view = ViewDefinition('index',
'max_tweet_id',
idMapper,
maxFindingReducer,
language='python')
view.sync(db)
KW['since_id'] = int([_id
for _id in db.view('index/max_tweet_id')][0].value)
# Harvest tweets for the given timeline.
# For friend and home timelines, the unofficial limitation is about 800 statuses although
# other documentation may state otherwise. The public timeline only returns 20 statuses
# and gets updated every 60 seconds.
# See http://groups.google.com/group/twitter-development-talk/browse_thread/thread/4678df70c301be43
# Note that the count and since_id params have no effect for the public timeline
page_num = 1
while page_num <= MAX_PAGES:
KW['page'] = page_num
api_call = getattr(t.statuses, TIMELINE_NAME + '_timeline')
tweets = makeTwitterRequest(t, api_call, **KW)
db.update(tweets, all_or_nothing=True)
print 'Fetched %i tweets' % len(tweets)
page_num += 1
SEARCH_TERM = sys.argv[1]
MAX_PAGES = 15
KW = {
'domain': 'search.twitter.com',
'count': 200,
'rpp': 100,
'q': SEARCH_TERM,
}
server = couchdb.Server('http://*****:*****@', ''), )
try:
db = server.create(DB)
except couchdb.http.PreconditionFailed, e:
# already exists, so append to it, and be mindful of duplicates
db = server[DB]
t = twitter.Twitter(domain='search.twitter.com')
for page in range(1, 16):
KW['page'] = page
tweets = makeTwitterRequest(t, t.search, **KW)
db.update(tweets['results'], all_or_nothing=True)
if len(tweets['results']) == 0:
break
print 'Fetched %i tweets' % len(tweets['results'])
min_conversation_id = min([int(i[1]) for i in conversation if i[1] is not None])
max_conversation_id = max([int(i[1]) for i in conversation if i[1] is not None])
# Pull tweets from other user using user timeline API to minimize API expenses...
t = login()
reply_tweets = []
results = []
page = 1
while True:
results = makeTwitterRequest(t,
t.statuses.user_timeline,
count=200,
# Per <http://dev.twitter.com/doc/get/statuses/user_timeline>, some
# caveats apply with the oldest id you can fetch using "since_id"
since_id=min_conversation_id,
max_id=max_conversation_id,
skip_users='true',
screen_name=USER,
page=page)
reply_tweets += results
page += 1
if len(results) == 0:
break
# During testing, it was observed that some tweets may not resolve or possibly
# even come back with null id values -- possibly a temporary fluke. Workaround.
missing_tweets = []
for (doc_id, in_reply_to_id) in conversation:
try:
print [rt for rt in reply_tweets if rt['id'] == in_reply_to_id][0]['text']
[int(i[1]) for i in conversation if i[1] is not None])
# Pull tweets from other user using user timeline API to minimize API expenses...
t = login()
reply_tweets = []
results = []
page = 1
while True:
results = makeTwitterRequest(
t,
t.statuses.user_timeline,
count=200,
# Per <http://dev.twitter.com/doc/get/statuses/user_timeline>, some
# caveats apply with the oldest id you can fetch using "since_id"
since_id=min_conversation_id,
max_id=max_conversation_id,
skip_users='true',
screen_name=USER,
page=page)
reply_tweets += results
page += 1
if len(results) == 0:
break
# During testing, it was observed that some tweets may not resolve or possibly
# even come back with null id values -- possibly a temporary fluke. Workaround.
missing_tweets = []
for (doc_id, in_reply_to_id) in conversation:
try:
def idMapper(doc):
yield (None, doc['id'])
def maxFindingReducer(keys, values, rereduce):
return max(values)
view = ViewDefinition('index', 'max_tweet_id', idMapper, maxFindingReducer,
language='python')
view.sync(db)
KW['since_id'] = int([_id for _id in db.view('index/max_tweet_id')][0].value)
# Harvest tweets for the given timeline.
# For friend and home timelines, the unofficial limitation is about 800 statuses although
# other documentation may state otherwise. The public timeline only returns 20 statuses
# and gets updated every 60 seconds.
# See http://groups.google.com/group/twitter-development-talk/browse_thread/thread/4678df70c301be43
# Note that the count and since_id params have no effect for the public timeline
page_num = 1
while page_num <= MAX_PAGES:
KW['page'] = page_num
api_call = getattr(t.statuses, TIMELINE_NAME + '_timeline')
tweets = makeTwitterRequest(t, api_call, **KW)
db.update(tweets, all_or_nothing=True)
print 'Fetched %i tweets' % len(tweets)
page_num += 1
# See http://code.google.com/p/twitter-api/issues/detail?id=214
#############################################
t = login()
r = redis.Redis()
count = 0
for screen_name in screen_names:
ids = []
cursor = -1
while cursor != 0:
try:
response = makeTwitterRequest(t,
t.friends.ids,
screen_name=screen_name,
cursor=cursor)
ids += response['ids']
cursor = response['next_cursor']
print >> sys.stderr, 'Fetched %i ids for %s' % (len(ids), screen_name)
except twitter.api.TwitterHTTPError, e:
if e.e.code == 404:
print >> sys.stderr, "404 Error with screen_name '%s'. Continuing." % screen_name
break
# Store the ids into Redis
[r.sadd(getRedisIdByScreenName(screen_name, 'friend_ids'), _id) for _id in
ids]
count += 1
SEARCH_TERM = sys.argv[1]
MAX_PAGES = 15
KW = {
'domain': 'search.twitter.com',
'count': 200,
'rpp': 100,
'q': SEARCH_TERM,
}
server = couchdb.Server('http://*****:*****@', ''), )
try:
db = server.create(DB)
except couchdb.http.PreconditionFailed, e:
# already exists, so append to it, and be mindful of duplicates
db = server[DB]
t = twitter.Twitter(domain='search.twitter.com')
for page in range(1, 16):
KW['page'] = page
tweets = makeTwitterRequest(t, t.search, **KW)
db.update(tweets['results'], all_or_nothing=True)
if len(tweets['results']) == 0:
break
print 'Fetched %i tweets' % len(tweets['results'])