class TweetDumper(object):
"""
This class is able to retrieve tweets from the user. If you need to update
a preexisting database just run with page=0 and interrupt the script as
soon you see Skipping warning.
"""
ARGS = ('user', 'page')
DESC = "Retrieve tweets of <user> starting from <page>"
METHOD = 'save_tweet'
URL = "http://api.twitter.com/1/statuses/user_timeline.json?" \
"&count=200&page={:d}"
def __init__(self):
self.url = self.URL
self.collector = Collector()
self.invoker = Requester('proxylist')
def dump(self, politician, page=1):
try:
page = int(page)
url = self.url
if politician.isdigit():
url += '&user_id={:s}'
else:
url += '&screen_name={:s}'
while True:
print("Retrieving tweets at page {:d}".format(page))
response, content = self.invoker.request(
url.format(page, politician)
)
collection = json.loads(content)
if len(collection) == 0:
break
meth = getattr(self.collector, self.METHOD)
meth.__call__(collection)
page += 1
finally:
print("Committing changes to the database")
self.collector.save()
self.invoker.save('proxylist')
def __init__(self):
self.url = self.URL
self.collector = Collector()
self.invoker = Requester('proxylist')
