def test_parse_host_config(): assert set() == parse_host_list("") assert set("h") == parse_host_list("h") assert set(["1", "2"]) == parse_host_list("1,2") assert set(["1", "2"]) == parse_host_list(" 1 , 2 ")
def main(args=sys.argv[1:]): options = { 'oauth': False, 'save-dir': ".", 'api-rate': False, 'timeline': "", 'mentions': "", 'dms': "", 'favorites': False, 'follow-redirects': False, 'redirect-sites': None, 'isoformat': False, } try: parse_args(args, options) except GetoptError as e: err("I can't do that, %s." % e) raise SystemExit(1) # exit if no user given # except if asking for API rate, or archive of timeline or mentions if not options['extra_args'] and not ( options['api-rate'] or options['timeline'] or options['mentions'] or options['dms']): print(__doc__) return # authenticate using OAuth, asking for token if necessary if options['oauth']: # oauth_filename = (os.getenv("HOME", "") + os.sep # + ".twitter-archiver_oauth") # if not os.path.exists(oauth_filename): # oauth_dance("Twitter-Archiver", CONSUMER_KEY, CONSUMER_SECRET, # oauth_filename) # oauth_token, oauth_token_secret = read_token_file(oauth_filename) auths = get_auths_data() random_index = random.randint(0, len(auths) - 1) print('Using the number %d oauth user' % random_index) auth = OAuth(auths[random_index][0], auths[random_index][1], CONSUMER_KEY, CONSUMER_SECRET) else: auth = NoAuth() twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com') if options['api-rate']: rate_limit_status(twitter) return global format_text if options['follow-redirects'] or options['redirect-sites']: if options['redirect-sites']: hosts = parse_host_list(options['redirect-sites']) else: hosts = None format_text = functools.partial(expand_format_text, hosts) else: format_text = direct_format_text # read users from command-line or stdin users = options['extra_args'] if len(users) == 1 and users[0] == "-": users = [line.strip() for line in sys.stdin.readlines()] # save tweets for every user total, total_new = 0, 0 for user in users: filename = options['save-dir'] + os.sep + user user_id = twitter.users.lookup(screen_name=user)[0]['id'] # print('%d\n' % user_id) if options['favorites']: filename = filename + "-favorites" print("* Archiving %s tweets in %s" % (user, filename)) tweets = {} try: tweets = load_tweets(filename) except Exception as e: err("Error when loading saved tweets: %s - continuing without" % str(e)) new = 0 before = len(tweets) try: statuses(twitter, user, tweets, options['mentions'], options['favorites'], isoformat=options['isoformat']) except KeyboardInterrupt: err() err("Interrupted") raise SystemExit(1) save_tweets(filename, tweets, user_id) total += len(tweets) new = len(tweets) - before total_new += new print("Total tweets for %s: %i (%i new)" % (user, len(tweets), new)) print("Total: %i tweets (%i new) for %i users" % (total, total_new, len(users)))
def main(args=sys.argv[1:]): options = { 'oauth': False, 'save-dir': ".", 'api-rate': False, 'timeline': "", 'mentions': "", 'dms': "", 'favorites': False, 'follow-redirects': False, 'redirect-sites': None, 'isoformat': False, } try: parse_args(args, options) except GetoptError as e: err("I can't do that, %s." % e) raise SystemExit(1) # exit if no user given # except if asking for API rate, or archive of timeline or mentions if not options['extra_args'] and not (options['api-rate'] or options['timeline'] or options['mentions'] or options['dms']): print(__doc__) return # authenticate using OAuth, asking for token if necessary if options['oauth']: # oauth_filename = (os.getenv("HOME", "") + os.sep # + ".twitter-archiver_oauth") # if not os.path.exists(oauth_filename): # oauth_dance("Twitter-Archiver", CONSUMER_KEY, CONSUMER_SECRET, # oauth_filename) # oauth_token, oauth_token_secret = read_token_file(oauth_filename) auths = get_auths_data() random_index = random.randint(0, len(auths) - 1) print('Using the number %d oauth user' % random_index) auth = OAuth(auths[random_index][0], auths[random_index][1], CONSUMER_KEY, CONSUMER_SECRET) else: auth = NoAuth() twitter = Twitter(auth=auth, api_version='1.1', domain='api.twitter.com') if options['api-rate']: rate_limit_status(twitter) return global format_text if options['follow-redirects'] or options['redirect-sites'] : if options['redirect-sites']: hosts = parse_host_list(options['redirect-sites']) else: hosts = None format_text = functools.partial(expand_format_text, hosts) else: format_text = direct_format_text # read users from command-line or stdin users = options['extra_args'] if len(users) == 1 and users[0] == "-": users = [line.strip() for line in sys.stdin.readlines()] # save tweets for every user total, total_new = 0, 0 for user in users: filename = options['save-dir'] + os.sep + user user_id = twitter.users.lookup(screen_name=user)[0]['id'] # print('%d\n' % user_id) if options['favorites']: filename = filename + "-favorites" print("* Archiving %s tweets in %s" % (user, filename)) tweets = {} try: tweets = load_tweets(filename) except Exception as e: err("Error when loading saved tweets: %s - continuing without" % str(e)) new = 0 before = len(tweets) try: statuses(twitter, user, tweets, options['mentions'], options['favorites'], isoformat=options['isoformat']) except KeyboardInterrupt: err() err("Interrupted") raise SystemExit(1) save_tweets(filename, tweets, user_id) total += len(tweets) new = len(tweets) - before total_new += new print("Total tweets for %s: %i (%i new)" % (user, len(tweets), new)) print("Total: %i tweets (%i new) for %i users" % (total, total_new, len(users)))