The type field is not used in dynamic monitors.
@author: wt
"""
import ohsn.util.twitter_util as twutil
import pymongo
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import math
import profiles_check
import lookup
app_id_follower, twitter_follower = twutil.twitter_auth()
follower_remain, follower_lock = 0, 1
def handle_follower_rate_limiting():
global app_id_follower, twitter_follower
while True:
print '---------handle_follower_rate_limiting------------------'
try:
rate_limit_status = twitter_follower.get_application_rate_limit_status(resources=['followers'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_follower)
app_id_follower, twitter_follower = twutil.twitter_change_auth(app_id_follower)
continue
Created on 10:58, 15/02/16
@author: wt
Get users' profiles
Returns fully-hydrated user objects for up to 100 users per request,
as specified by comma-separated values passed to the user_id and/or screen_name parameters.
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import pymongo
import time
import profiles_check
app_id_look, twitter_look = twutil.twitter_auth()
lookup_remain, lookup_lock = 0, 1
def handle_lookup_rate_limiting():
global app_id_look, twitter_look
while True:
print "---------handle_lookup_rate_limiting------------------"
try:
rate_limit_status = twitter_look.get_application_rate_limit_status(resources=["users"])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S"
) + "\t" + "Cannot test due to last incorrect connection, change Twitter APP ID", str(detail)
twutil.release_app(app_id_look)
app_id_look, twitter_look = twutil.twitter_change_auth(app_id_look)
4. Target users are those have timelines more than 3000
since_id: Returns results with an ID greater than (that is, more recent than) the specified ID.
max_id: Returns results with an ID less than (that is, older than) or equal to the specified ID.
Each crawl is the most recent 200 timelines.
Set the oldest one in each crawl as max_id.
SEE: https://unsupervisedlearning.wordpress.com/2014/07/06/scraping-your-twitter-homepage-with-python-and-mongodb/
"""
import ohsn.util.twitter_util as twutil
import datetime
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import time
import pymongo
timeline_app_id, timeline_twitter = twutil.twitter_auth()
timeline_remain, timeline_lock = 0, 1
def store_tweets(tweets_to_save, collection):
"""
Simple wrapper to facilitate persisting tweets. Right now, the only
pre-processing accomplished is coercing date values to datetime.
"""
print datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S") + "\t" + 'Size of Stored Timelines: ' + str(
len(tweets_to_save))
for tweet in tweets_to_save:
tweet['created_at'] = datetime.datetime.strptime(
tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y')
try:
Created on 10:58, 15/02/16
@author: wt
Get users' profiles
Returns fully-hydrated user objects for up to 100 users per request,
as specified by comma-separated values passed to the user_id and/or screen_name parameters.
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import pymongo
import time
import profiles_check
app_id_look, twitter_look = twutil.twitter_auth()
lookup_remain, lookup_lock = 0, 1
def handle_lookup_rate_limiting():
global app_id_look, twitter_look
while True:
print '---------handle_lookup_rate_limiting------------------'
try:
rate_limit_status = twitter_look.get_application_rate_limit_status(
resources=['users'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_look)
app_id_look, twitter_look = twutil.twitter_change_auth(app_id_look)
1: follower.
The type field is not used in dynamic monitors.
@author: wt
"""
import ohsn.util.twitter_util as twutil
import pymongo
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import math
import profiles_check
import lookup
app_id_follower, twitter_follower = twutil.twitter_auth()
follower_remain, follower_lock = 0, 1
def handle_follower_rate_limiting():
global app_id_follower, twitter_follower
while True:
print '---------handle_follower_rate_limiting------------------'
try:
rate_limit_status = twitter_follower.get_application_rate_limit_status(
resources=['followers'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_follower)
app_id_follower, twitter_follower = twutil.twitter_change_auth(
https://dev.twitter.com/docs/api/1.1/get/search/tweets
https://dev.twitter.com/rest/reference/get/search/tweets
https://api.twitter.com/1.1/search/tweets.json?q=%23freebandnames&since_id=24012619984051000&max_id=250126199840518145&result_type=mixed&count=4
"""
import ohsn.util.twitter_util as twutil
import ohsn.util.db_util as dbt
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import pymongo
search_app_id, search_twitter = twutil.twitter_auth()
search_remain, search_lock = 0, 1
def store_tweets(tweets_to_save, collection):
"""
Simple wrapper to facilitate persisting tweets. Right now, the only
pre-processing accomplished is coercing date values to datetime.
"""
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + 'Size of Stored Timelines: ' + str(len(tweets_to_save))
for tweet in tweets_to_save:
tweet['created_at'] = datetime.datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y')
try:
collection.insert(tweet)
except pymongo.errors.DuplicateKeyError:
pass
0: following
1: follower.
@author: wt
"""
import ohsn.util.twitter_util as twutil
import pymongo
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import math
import profiles_check
import lookup
app_id_friend, twitter_friend = twutil.twitter_auth()
following_remain, following_lock = 0, 1
def handle_following_rate_limiting():
global app_id_friend, twitter_friend
while True:
print '---------handle_following_rate_limiting------------------'
try:
rate_limit_status = twitter_friend.get_application_rate_limit_status(resources=['friends'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_friend)
app_id_friend, twitter_friend = twutil.twitter_change_auth(app_id_friend)
continue
Created on 10:58, 15/02/16
@author: wt
Get users' profiles
Returns fully-hydrated user objects for up to 100 users per request,
as specified by comma-separated values passed to the user_id and/or screen_name parameters.
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import pymongo
import time
from itertools import islice
app_id_look_tweet, twitter_look_tweet = twutil.twitter_auth()
lookup_remain_tweet, lookup_lock_tweet = 0, 1
def handle_lookup_tweet_rate_limiting():
global app_id_look_tweet, twitter_look_tweet
while True:
print '---------handle_lookup_rate_limiting------------------'
try:
rate_limit_status = twitter_look_tweet.get_application_rate_limit_status(
resources=['statuses'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_look_tweet)
app_id_look_tweet, twitter_look_tweet = twutil.twitter_change_auth(
# -*- coding: utf-8 -*-
"""
Created on 10:58, 15/02/16
@author: wt
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
app_id_friendship, twitter_friendship = twutil.twitter_auth(56)
friendships_remain, friendships_lock = 0, 1
def handle_friendship_rate_limiting():
global app_id_friendship, twitter_friendship
while True:
print '---------handle_friendship_rate_limiting------------------'
try:
rate_limit_status = twitter_friendship.get_application_rate_limit_status(resources=['friendships'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_friendship)
app_id_friendship, twitter_friendship = twutil.twitter_change_auth(app_id_friendship)
continue
except TwythonAuthError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Author Error, change Twitter APP ID', str(detail)
1: follower.
@author: wt
"""
import ohsn.util.twitter_util as twutil
import pymongo
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import math
import profiles_check
import lookup
import random
app_id_friend, twitter_friend = twutil.twitter_auth()
following_remain, following_lock = 0, 1
def handle_following_rate_limiting():
global app_id_friend, twitter_friend
while True:
print '---------handle_following_rate_limiting------------------'
try:
rate_limit_status = twitter_friend.get_application_rate_limit_status(
resources=['friends'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_friend)
app_id_friend, twitter_friend = twutil.twitter_change_auth(
@author: wt
Get users' profiles
Returns fully-hydrated user objects for up to 100 users per request,
as specified by comma-separated values passed to the user_id and/or screen_name parameters.
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import pymongo
import time
from itertools import islice
app_id_look_tweet, twitter_look_tweet = twutil.twitter_auth()
lookup_remain_tweet, lookup_lock_tweet = 0, 1
def handle_lookup_tweet_rate_limiting():
global app_id_look_tweet, twitter_look_tweet
while True:
print '---------handle_lookup_rate_limiting------------------'
try:
rate_limit_status = twitter_look_tweet.get_application_rate_limit_status(resources=['statuses'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_look_tweet)
app_id_look_tweet, twitter_look_tweet = twutil.twitter_change_auth(app_id_look_tweet)
continue
4. Target users are those have timelines more than 3000
since_id: Returns results with an ID greater than (that is, more recent than) the specified ID.
max_id: Returns results with an ID less than (that is, older than) or equal to the specified ID.
Each crawl is the most recent 200 timelines.
Set the oldest one in each crawl as max_id.
SEE: https://unsupervisedlearning.wordpress.com/2014/07/06/scraping-your-twitter-homepage-with-python-and-mongodb/
"""
import ohsn.util.twitter_util as twutil
import datetime
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import time
import pymongo
timeline_app_id, timeline_twitter = twutil.twitter_auth()
timeline_remain, timeline_lock = 0, 1
def store_tweets(tweets_to_save, collection):
"""
Simple wrapper to facilitate persisting tweets. Right now, the only
pre-processing accomplished is coercing date values to datetime.
"""
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + "Size of Stored Timelines: " + str(
len(tweets_to_save)
)
for tweet in tweets_to_save:
tweet["created_at"] = datetime.datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S +0000 %Y")
try:
collection.insert(tweet)
@author: tw
Returns a collection of up to 100 user IDs belonging to users
who have retweeted the tweet specified by the id parameter.
https://dev.twitter.com/docs/api/1.1/get/statuses/retweeters/ids
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import pymongo
import lookup, profiles_check
app_id_retweeter, twitter_retweeter = twutil.twitter_auth()
retweeter_remain, retweeter_lock = 0, 1
def handle_retweeter_rate_limiting():
global app_id_retweeter, twitter_retweeter
while True:
print '---------handle__retweeter_rate_limiting------------------'
try:
rate_limit_status = twitter_retweeter.get_application_rate_limit_status(resources=['statuses'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_retweeter)
app_id_retweeter, twitter_retweeter = twutil.twitter_change_auth(app_id_retweeter)
continue
@author: tw
Returns a collection of up to 100 user IDs belonging to users
who have retweeted the tweet specified by the id parameter.
https://dev.twitter.com/docs/api/1.1/get/statuses/retweeters/ids
"""
import ohsn.util.twitter_util as twutil
from twython import TwythonRateLimitError, TwythonAuthError, TwythonError
import datetime
import time
import pymongo
import lookup, profiles_check
app_id_retweeter, twitter_retweeter = twutil.twitter_auth()
retweeter_remain, retweeter_lock = 0, 1
def handle_retweeter_rate_limiting():
global app_id_retweeter, twitter_retweeter
while True:
print '---------handle__retweeter_rate_limiting------------------'
try:
rate_limit_status = twitter_retweeter.get_application_rate_limit_status(
resources=['statuses'])
except TwythonRateLimitError as detail:
print datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "\t" + \
'Cannot test due to last incorrect connection, change Twitter APP ID', str(detail)
twutil.release_app(app_id_retweeter)
app_id_retweeter, twitter_retweeter = twutil.twitter_change_auth(
