The type field is not used in dynamic monitors. @author: wt """ import ohsn.util.twitter_util as twutil import pymongo from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import math import profiles_check import lookup app_id_follower, twitter_follower = twutil.twitter_auth() follower_remain, follower_lock = 0, 1 def handle_follower_rate_limiting(): global app_id_follower, twitter_follower while True: print '---------handle_follower_rate_limiting------------------' try: rate_limit_status = twitter_follower.get_application_rate_limit_status(resources=['followers']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_follower) app_id_follower, twitter_follower = twutil.twitter_change_auth(app_id_follower) continue
Created on 10:58, 15/02/16 @author: wt Get users' profiles Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the user_id and/or screen_name parameters. """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import pymongo import time import profiles_check app_id_look, twitter_look = twutil.twitter_auth() lookup_remain, lookup_lock = 0, 1 def handle_lookup_rate_limiting(): global app_id_look, twitter_look while True: print "---------handle_lookup_rate_limiting------------------" try: rate_limit_status = twitter_look.get_application_rate_limit_status(resources=["users"]) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S" ) + "\t" + "Cannot test due to last incorrect connection, change Twitter APP ID", str(detail) twutil.release_app(app_id_look) app_id_look, twitter_look = twutil.twitter_change_auth(app_id_look)
4. Target users are those have timelines more than 3000 since_id: Returns results with an ID greater than (that is, more recent than) the specified ID. max_id: Returns results with an ID less than (that is, older than) or equal to the specified ID. Each crawl is the most recent 200 timelines. Set the oldest one in each crawl as max_id. SEE: https://unsupervisedlearning.wordpress.com/2014/07/06/scraping-your-twitter-homepage-with-python-and-mongodb/ """ import ohsn.util.twitter_util as twutil import datetime from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import time import pymongo timeline_app_id, timeline_twitter = twutil.twitter_auth() timeline_remain, timeline_lock = 0, 1 def store_tweets(tweets_to_save, collection): """ Simple wrapper to facilitate persisting tweets. Right now, the only pre-processing accomplished is coercing date values to datetime. """ print datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S") + "\t" + 'Size of Stored Timelines: ' + str( len(tweets_to_save)) for tweet in tweets_to_save: tweet['created_at'] = datetime.datetime.strptime( tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y') try:
Created on 10:58, 15/02/16 @author: wt Get users' profiles Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the user_id and/or screen_name parameters. """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import pymongo import time import profiles_check app_id_look, twitter_look = twutil.twitter_auth() lookup_remain, lookup_lock = 0, 1 def handle_lookup_rate_limiting(): global app_id_look, twitter_look while True: print '---------handle_lookup_rate_limiting------------------' try: rate_limit_status = twitter_look.get_application_rate_limit_status( resources=['users']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_look) app_id_look, twitter_look = twutil.twitter_change_auth(app_id_look)
1: follower. The type field is not used in dynamic monitors. @author: wt """ import ohsn.util.twitter_util as twutil import pymongo from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import math import profiles_check import lookup app_id_follower, twitter_follower = twutil.twitter_auth() follower_remain, follower_lock = 0, 1 def handle_follower_rate_limiting(): global app_id_follower, twitter_follower while True: print '---------handle_follower_rate_limiting------------------' try: rate_limit_status = twitter_follower.get_application_rate_limit_status( resources=['followers']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_follower) app_id_follower, twitter_follower = twutil.twitter_change_auth(
https://dev.twitter.com/docs/api/1.1/get/search/tweets https://dev.twitter.com/rest/reference/get/search/tweets https://api.twitter.com/1.1/search/tweets.json?q=%23freebandnames&since_id=24012619984051000&max_id=250126199840518145&result_type=mixed&count=4 """ import ohsn.util.twitter_util as twutil import ohsn.util.db_util as dbt from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import pymongo search_app_id, search_twitter = twutil.twitter_auth() search_remain, search_lock = 0, 1 def store_tweets(tweets_to_save, collection): """ Simple wrapper to facilitate persisting tweets. Right now, the only pre-processing accomplished is coercing date values to datetime. """ print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + 'Size of Stored Timelines: ' + str(len(tweets_to_save)) for tweet in tweets_to_save: tweet['created_at'] = datetime.datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y') try: collection.insert(tweet) except pymongo.errors.DuplicateKeyError: pass
0: following 1: follower. @author: wt """ import ohsn.util.twitter_util as twutil import pymongo from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import math import profiles_check import lookup app_id_friend, twitter_friend = twutil.twitter_auth() following_remain, following_lock = 0, 1 def handle_following_rate_limiting(): global app_id_friend, twitter_friend while True: print '---------handle_following_rate_limiting------------------' try: rate_limit_status = twitter_friend.get_application_rate_limit_status(resources=['friends']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_friend) app_id_friend, twitter_friend = twutil.twitter_change_auth(app_id_friend) continue
Created on 10:58, 15/02/16 @author: wt Get users' profiles Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the user_id and/or screen_name parameters. """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import pymongo import time from itertools import islice app_id_look_tweet, twitter_look_tweet = twutil.twitter_auth() lookup_remain_tweet, lookup_lock_tweet = 0, 1 def handle_lookup_tweet_rate_limiting(): global app_id_look_tweet, twitter_look_tweet while True: print '---------handle_lookup_rate_limiting------------------' try: rate_limit_status = twitter_look_tweet.get_application_rate_limit_status( resources=['statuses']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_look_tweet) app_id_look_tweet, twitter_look_tweet = twutil.twitter_change_auth(
# -*- coding: utf-8 -*- """ Created on 10:58, 15/02/16 @author: wt """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time app_id_friendship, twitter_friendship = twutil.twitter_auth(56) friendships_remain, friendships_lock = 0, 1 def handle_friendship_rate_limiting(): global app_id_friendship, twitter_friendship while True: print '---------handle_friendship_rate_limiting------------------' try: rate_limit_status = twitter_friendship.get_application_rate_limit_status(resources=['friendships']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_friendship) app_id_friendship, twitter_friendship = twutil.twitter_change_auth(app_id_friendship) continue except TwythonAuthError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Author Error, change Twitter APP ID', str(detail)
1: follower. @author: wt """ import ohsn.util.twitter_util as twutil import pymongo from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import math import profiles_check import lookup import random app_id_friend, twitter_friend = twutil.twitter_auth() following_remain, following_lock = 0, 1 def handle_following_rate_limiting(): global app_id_friend, twitter_friend while True: print '---------handle_following_rate_limiting------------------' try: rate_limit_status = twitter_friend.get_application_rate_limit_status( resources=['friends']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_friend) app_id_friend, twitter_friend = twutil.twitter_change_auth(
@author: wt Get users' profiles Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the user_id and/or screen_name parameters. """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import pymongo import time from itertools import islice app_id_look_tweet, twitter_look_tweet = twutil.twitter_auth() lookup_remain_tweet, lookup_lock_tweet = 0, 1 def handle_lookup_tweet_rate_limiting(): global app_id_look_tweet, twitter_look_tweet while True: print '---------handle_lookup_rate_limiting------------------' try: rate_limit_status = twitter_look_tweet.get_application_rate_limit_status(resources=['statuses']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_look_tweet) app_id_look_tweet, twitter_look_tweet = twutil.twitter_change_auth(app_id_look_tweet) continue
4. Target users are those have timelines more than 3000 since_id: Returns results with an ID greater than (that is, more recent than) the specified ID. max_id: Returns results with an ID less than (that is, older than) or equal to the specified ID. Each crawl is the most recent 200 timelines. Set the oldest one in each crawl as max_id. SEE: https://unsupervisedlearning.wordpress.com/2014/07/06/scraping-your-twitter-homepage-with-python-and-mongodb/ """ import ohsn.util.twitter_util as twutil import datetime from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import time import pymongo timeline_app_id, timeline_twitter = twutil.twitter_auth() timeline_remain, timeline_lock = 0, 1 def store_tweets(tweets_to_save, collection): """ Simple wrapper to facilitate persisting tweets. Right now, the only pre-processing accomplished is coercing date values to datetime. """ print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + "Size of Stored Timelines: " + str( len(tweets_to_save) ) for tweet in tweets_to_save: tweet["created_at"] = datetime.datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S +0000 %Y") try: collection.insert(tweet)
@author: tw Returns a collection of up to 100 user IDs belonging to users who have retweeted the tweet specified by the id parameter. https://dev.twitter.com/docs/api/1.1/get/statuses/retweeters/ids """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import pymongo import lookup, profiles_check app_id_retweeter, twitter_retweeter = twutil.twitter_auth() retweeter_remain, retweeter_lock = 0, 1 def handle_retweeter_rate_limiting(): global app_id_retweeter, twitter_retweeter while True: print '---------handle__retweeter_rate_limiting------------------' try: rate_limit_status = twitter_retweeter.get_application_rate_limit_status(resources=['statuses']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_retweeter) app_id_retweeter, twitter_retweeter = twutil.twitter_change_auth(app_id_retweeter) continue
@author: tw Returns a collection of up to 100 user IDs belonging to users who have retweeted the tweet specified by the id parameter. https://dev.twitter.com/docs/api/1.1/get/statuses/retweeters/ids """ import ohsn.util.twitter_util as twutil from twython import TwythonRateLimitError, TwythonAuthError, TwythonError import datetime import time import pymongo import lookup, profiles_check app_id_retweeter, twitter_retweeter = twutil.twitter_auth() retweeter_remain, retweeter_lock = 0, 1 def handle_retweeter_rate_limiting(): global app_id_retweeter, twitter_retweeter while True: print '---------handle__retweeter_rate_limiting------------------' try: rate_limit_status = twitter_retweeter.get_application_rate_limit_status( resources=['statuses']) except TwythonRateLimitError as detail: print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \ 'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail) twutil.release_app(app_id_retweeter) app_id_retweeter, twitter_retweeter = twutil.twitter_change_auth(