def test_parse_host_config():
assert set() == parse_host_list("")
assert set("h") == parse_host_list("h")
assert set(["1", "2"]) == parse_host_list("1,2")
assert set(["1", "2"]) == parse_host_list(" 1 , 2 ")
def test_parse_host_config():
assert set() == parse_host_list("")
assert set("h") == parse_host_list("h")
assert set(["1", "2"]) == parse_host_list("1,2")
assert set(["1", "2"]) == parse_host_list(" 1 , 2 ")
def main(args=sys.argv[1:]):
options = {
'oauth': False,
'save-dir': ".",
'api-rate': False,
'timeline': "",
'mentions': "",
'dms': "",
'favorites': False,
'follow-redirects': False,
'redirect-sites': None,
'isoformat': False,
}
try:
parse_args(args, options)
except GetoptError as e:
err("I can't do that, %s." % e)
raise SystemExit(1)
# exit if no user given
# except if asking for API rate, or archive of timeline or mentions
if not options['extra_args'] and not (
options['api-rate'] or options['timeline'] or options['mentions']
or options['dms']):
print(__doc__)
return
# authenticate using OAuth, asking for token if necessary
if options['oauth']:
# oauth_filename = (os.getenv("HOME", "") + os.sep
# + ".twitter-archiver_oauth")
# if not os.path.exists(oauth_filename):
# oauth_dance("Twitter-Archiver", CONSUMER_KEY, CONSUMER_SECRET,
# oauth_filename)
# oauth_token, oauth_token_secret = read_token_file(oauth_filename)
auths = get_auths_data()
random_index = random.randint(0, len(auths) - 1)
print('Using the number %d oauth user' % random_index)
auth = OAuth(auths[random_index][0], auths[random_index][1],
CONSUMER_KEY, CONSUMER_SECRET)
else:
auth = NoAuth()
twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com')
if options['api-rate']:
rate_limit_status(twitter)
return
global format_text
if options['follow-redirects'] or options['redirect-sites']:
if options['redirect-sites']:
hosts = parse_host_list(options['redirect-sites'])
else:
hosts = None
format_text = functools.partial(expand_format_text, hosts)
else:
format_text = direct_format_text
# read users from command-line or stdin
users = options['extra_args']
if len(users) == 1 and users[0] == "-":
users = [line.strip() for line in sys.stdin.readlines()]
# save tweets for every user
total, total_new = 0, 0
for user in users:
filename = options['save-dir'] + os.sep + user
user_id = twitter.users.lookup(screen_name=user)[0]['id']
# print('%d\n' % user_id)
if options['favorites']:
filename = filename + "-favorites"
print("* Archiving %s tweets in %s" % (user, filename))
tweets = {}
try:
tweets = load_tweets(filename)
except Exception as e:
err("Error when loading saved tweets: %s - continuing without" %
str(e))
new = 0
before = len(tweets)
try:
statuses(twitter,
user,
tweets,
options['mentions'],
options['favorites'],
isoformat=options['isoformat'])
except KeyboardInterrupt:
err()
err("Interrupted")
raise SystemExit(1)
save_tweets(filename, tweets, user_id)
total += len(tweets)
new = len(tweets) - before
total_new += new
print("Total tweets for %s: %i (%i new)" % (user, len(tweets), new))
print("Total: %i tweets (%i new) for %i users" %
(total, total_new, len(users)))
def main(args=sys.argv[1:]):
options = {
'oauth': False,
'save-dir': ".",
'api-rate': False,
'timeline': "",
'mentions': "",
'dms': "",
'favorites': False,
'follow-redirects': False,
'redirect-sites': None,
'isoformat': False,
}
try:
parse_args(args, options)
except GetoptError as e:
err("I can't do that, %s." % e)
raise SystemExit(1)
# exit if no user given
# except if asking for API rate, or archive of timeline or mentions
if not options['extra_args'] and not (options['api-rate'] or
options['timeline'] or
options['mentions'] or
options['dms']):
print(__doc__)
return
# authenticate using OAuth, asking for token if necessary
if options['oauth']:
# oauth_filename = (os.getenv("HOME", "") + os.sep
# + ".twitter-archiver_oauth")
# if not os.path.exists(oauth_filename):
# oauth_dance("Twitter-Archiver", CONSUMER_KEY, CONSUMER_SECRET,
# oauth_filename)
# oauth_token, oauth_token_secret = read_token_file(oauth_filename)
auths = get_auths_data()
random_index = random.randint(0, len(auths) - 1)
print('Using the number %d oauth user' % random_index)
auth = OAuth(auths[random_index][0], auths[random_index][1], CONSUMER_KEY,
CONSUMER_SECRET)
else:
auth = NoAuth()
twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com')
if options['api-rate']:
rate_limit_status(twitter)
return
global format_text
if options['follow-redirects'] or options['redirect-sites'] :
if options['redirect-sites']:
hosts = parse_host_list(options['redirect-sites'])
else:
hosts = None
format_text = functools.partial(expand_format_text, hosts)
else:
format_text = direct_format_text
# read users from command-line or stdin
users = options['extra_args']
if len(users) == 1 and users[0] == "-":
users = [line.strip() for line in sys.stdin.readlines()]
# save tweets for every user
total, total_new = 0, 0
for user in users:
filename = options['save-dir'] + os.sep + user
user_id = twitter.users.lookup(screen_name=user)[0]['id']
# print('%d\n' % user_id)
if options['favorites']:
filename = filename + "-favorites"
print("* Archiving %s tweets in %s" % (user, filename))
tweets = {}
try:
tweets = load_tweets(filename)
except Exception as e:
err("Error when loading saved tweets: %s - continuing without"
% str(e))
new = 0
before = len(tweets)
try:
statuses(twitter, user, tweets, options['mentions'], options['favorites'], isoformat=options['isoformat'])
except KeyboardInterrupt:
err()
err("Interrupted")
raise SystemExit(1)
save_tweets(filename, tweets, user_id)
total += len(tweets)
new = len(tweets) - before
total_new += new
print("Total tweets for %s: %i (%i new)" % (user, len(tweets), new))
print("Total: %i tweets (%i new) for %i users"
% (total, total_new, len(users)))