auth = tA.authorization()
api = API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
#Obtain Geo Code Location of Palo Alto California
#places = api.geo_search(query="USA", granularity="country")
#places = api.geo_search(query="Washington DC", granularity="city")
#place_id = places[0].id
#preventiveString, riskString, elderlyString, sentiments, misc = gA.returnSearchString()
searchString = placeSearch[
index] + ' #COVID-19 OR "COVID-19" OR "pandemic" OR "Corona"'
cursor = Cursor(api.search,
q=searchString,
count=20,
lang="en",
tweet_mode='extended')
api = API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
maxCount = 1000
count = 0
for tweet in cursor.items():
count += 1
fileName = "tweets_" + str(index) + "_" + str(
datetime.datetime.now().date()).replace('-', '_')
file = open(outputPath + fileName + '.txt', 'a')
file.write(str(tweet) + '\n')
print(count, end='')
backline()
def get_friend_list(self, num_friends):
friend_list = []
for friend in Cursor(self.twitter_client.friends,
id=self.twitter_user).items(num_friends):
friend_list.append(friend)
return friend_list
def get_following_handles(self, user, num):
user_friends = []
for friend in Cursor(self.api_instance.friends, id=user).items(num):
user_friends.append(friend)
return user_friends
def testpagecursoritems(self):
items = list(Cursor(self.api.user_timeline).items())
self.assert_(len(items) > 0)
items = list(Cursor(self.api.user_timeline, 'twitter').items(30))
self.assert_(len(items) == 30)
def testcursorcursoritems(self):
items = list(Cursor(self.api.friends_ids).items())
self.assert_(len(items) > 0)
items = list(Cursor(self.api.followers_ids, 'twitter').items(30))
self.assert_(len(items) == 30)
#Prints the first ten items from timeline to terminal
from tweepy import Cursor
from twitter_client import get_twitter_client
if __name__ == '__main__':
client = get_twitter_client()
"""status variable is instance of tweepy.Status (model used by
Tweepy to wrap statuses aka tweets)"""
for status in Cursor(client.home_timeline).items(10):
print(status.text)
print("-----------------")
def get_user_timeline_tweets(self, num_tweets): tweets = [] # How many tweets to get from a users timeline. for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(num_tweets): tweets.append(tweet) return tweets
def testcursorsetstartcursor(self):
c = Cursor(self.api.friends_ids, cursor=123456)
self.assertEqual(c.iterator.next_cursor, 123456)
self.assertFalse('cursor' in c.iterator.kwargs)
def testidcursoritems(self):
items = list(Cursor(self.api.user_timeline).items(2))
self.assertEqual(len(items), 2)
def testcursorcursoritems(self):
items = list(Cursor(self.api.friends_ids).items(2))
self.assertEqual(len(items), 2)
items = list(Cursor(self.api.followers_ids, username).items(1))
self.assertEqual(len(items), 1)
def testcursorcursorpages(self):
pages = list(Cursor(self.api.friends_ids).pages(1))
self.assertTrue(len(pages) == 1)
pages = list(Cursor(self.api.followers_ids, username).pages(1))
self.assertTrue(len(pages) == 1)
def testidcursorpages(self):
pages = list(Cursor(self.api.user_timeline, count=1).pages(2))
self.assertEqual(len(pages), 2)
data.append(['Joined At ', user_profile.created_at])
return data
def GetScreenName(api, user_id):
user = api.get_user(id=user_id)
screen_name = user.screen_name
return screen_name
filename = "C:\\Python27\\tweet2.csv"
f = open(filename, "w+")
headers = "Name,friends,Possitive,Neggative,Neutral"
f.write(headers + "\n")
#list2=[]
for follower_id in Cursor(api.followers_ids,
screen_name='shreya73767208').items():
sc = GetScreenName(api, follower_id)
data = GetUserProfileDetails(api, sc)
print(str(data[0]) + ' : ' + str(data[1]) + ' : ' + str(data[2]))
l1 = str(data[1]).split(',')
print l1[1].replace('u', '')
data1 = getTweets(l1[1].replace('u', ''), 40)
print data1
l3 = str(data[6]).split(',')
f.write(l1[1].replace('u', '') + "," + l3[1].replace(']', '') + "," +
str(data1[0]) + "," + str(data1[1]) + "," + str(data1[2]) + "\n")
f.close()
#print friend list#######################################################
filename = "C:\\Python27\\tweet.csv"
def get_user_timeline_tweets(self, num_tweets):
tweets = [] # looping through every tweet provided
for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(num_tweets):
tweets.append(tweet)
return tweets # list of user timeline tweets
import json
from tweepy import Cursor
from twitter_client import get_twitter_client
if __name__ == '__main__':
client = get_twitter_client()
# retrieve up to 200 statuses per page for 4 pages of recent timeline
with open('home_timeline.jsonl', 'w') as f:
for page in Cursor(client.home_timeline, count=200).pages(4):
for status in page:
f.write(json.dumps(status._json) + "\n")
for file in files:
done_list.append(file[:-4])
except FileNotFoundError:
pass
done_list.extend(no_tweet_list)
users=list(set(users)-set(done_list))
print("City:",city)
print("No. of users remaining:",len(users))
if len(users)==0:
break
for user_id in users:
tweets=Cursor(api.user_timeline,user_id=int(user_id)).items()
last_user_id=user_id
tweet_count=0
try:
print("User id:",user_id)
for tweet in tweets:
tweet_count+=1
print("No. of tweets written:",tweet_count,end='\r')
try:
f=open('tweets/'+city+'/'+user_id+'.txt','a+',encoding='utf-8')
except FileNotFoundError:
os.mkdir('tweets/'+city)
f=open('tweets/'+city+'/'+user_id+'.txt','a+',encoding='utf-8')
f.write('tweet:'+tweet.text+'\n')
tweets = item.statuses_count
account_created_date = item.created_at
delta = datetime.utcnow() - account_created_date
account_age_days = delta.days
# print("Account age (in days): " + str(account_age_days))
tweets_per_day = 0
if account_age_days > 0:
tweets_per_day = "%.2f" % (float(tweets) /
float(account_age_days))
hashtags = []
tweets_list = []
tweet_count = 0
end_date = datetime.utcnow() - timedelta(days=30)
for status in Cursor(auth_api.user_timeline, id=target).items():
tweet_count += 1
if hasattr(status, "text"):
tweets_list.append(status.text)
if hasattr(status, "entities"):
entities = status.entities
if "hashtags" in entities:
for ent in entities["hashtags"]:
if ent is not None:
if "text" in ent:
hashtag = ent["text"]
if hashtag is not None:
hashtags.append(hashtag)
if status.created_at < end_date:
break
client = get_twitter_client()
dirname = "users/{}".format(screen_name)
max_pages = math.ceil(MAX_FRIENDS / 5000)
try:
os.makedirs(dirname, mode=0o755, exist_ok=True)
except OSError:
print("Directory {} already exists".format(dirname))
except Exception as e:
print("Error while creating directory {}".format(dirname))
print(e)
sys.exit(1)
# get followers for a given user
fname = "users/{}/followers.jsonl".format(screen_name)
with open(fname, 'w') as f:
for followers in Cursor(client.followers_ids, screen_name=screen_name).pages(max_pages):
for chunk in paginate(followers, 100):
users = client.lookup_users(user_ids=chunk)
for user in users:
f.write(json.dumps(user._json)+"\n")
if len(followers) == 5000:
print("More results available. Sleeping for 60 seconds to avoid rate limit")
time.sleep(60)
# get friends for a given user
fname = "users/{}/friends.jsonl".format(screen_name)
with open(fname, 'w') as f:
for friends in Cursor(client.friends_ids, screen_name=screen_name).pages(max_pages):
for chunk in paginate(friends, 100):
users = client.lookup_users(user_ids=chunk)
for user in users:
def get_search_cursor(self, q):
temp = []
for tweets in Cursor(self.api.search, q=q, count=200).pages(100):
for gg in tweets:
temp.append(gg._json)
return temp
def get_user_timeline_tweets(self, number_of_tweets): tweets = [] #Cursor is a class which allows us to get the user timeline tweets from the chosen user's timeline. If nothing is given there then it's gonna take the tweets from your timeline. The items part decides how many tweets we want. for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(number_of_tweets): tweets.append(tweet) return tweets
from tweepy import Cursor
import json
from TwitterClient import get_twitter_client
if __name__ == '__main__':
client = get_twitter_client()
with open('home_timeline.jsonl', 'w') as f:
for status in Cursor(client.home_timeline).items(10):
for page in Cursor(client.home_timeline, count=200).pages(2):
for status in page:
f.write(json.dumps(status._json) + "\n")
def get_friend_list(self, number_of_friends): list_of_friends = [] for friend in Cursor(self.twitter_client.friends, id=self.twitter_user).items(number_of_friends): list_of_friends.append(friend) return list_of_friends
def testpagecursorpages(self):
pages = list(Cursor(self.api.user_timeline).pages())
self.assert_(len(pages) > 0)
pages = list(Cursor(self.api.user_timeline, 'twitter').pages(5))
self.assert_(len(pages) == 5)
def get_info(twitter_user):
# screen_name = sys.argv[1]
screen_name = twitter_user
client = get_twitter_client()
dirname = "users/{}".format(screen_name)
max_pages = math.ceil(MAX_FRIENDS / 5000)
try:
os.makedirs(dirname, mode=0o755, exist_ok=True)
except OSError:
print("Directory {} already exists".format(dirname))
except Exception as e:
print("Error while creating directory{}".format(dirname))
print(e)
sys.exit(1)
print('Extrayendo a {} \n'.format(screen_name))
# get followers for a given user
fjson = "users/{}/followers.jsonl".format(screen_name)
fcsv = "users/{}/followers.csv".format(screen_name)
with open(fjson, 'w') as f1, open(fcsv, 'w') as f2:
for followers in Cursor(client.followers_ids,
screen_name=screen_name).pages(max_pages):
for chunk in paginate(followers, 100):
users = client.lookup_users(user_ids=chunk)
# out = [[user.created_at, user.id, user.screen_name,user.name, user.description, user.location] for user in users]
out = [[user.id, user.screen_name] for user in users]
writer = csv.writer(f2)
writer.writerow(["id", "screen_name"])
writer.writerows(out)
for user in users:
f1.write(json.dumps(user._json) + "\n")
if len(followers) == 5000:
print(
"Followers: More results available. Sleeping for 60seconds to avoid rate limit"
)
time.sleep(60)
# get friends for a given user
fjson = "users/{}/friends.jsonl".format(screen_name)
fcsv = "users/{}/friends.csv".format(screen_name)
with open(fjson, 'w') as f1, open(fcsv, 'w') as f2:
for friends in Cursor(client.friends_ids,
screen_name=screen_name).pages(max_pages):
for chunk in paginate(friends, 100):
users = client.lookup_users(user_ids=chunk)
# out = [[user.created_at, user.id, user.screen_name,user.name, user.description, user.location] for user in users]
# writer = csv.writer(f2)
# writer.writerow(["id","screen_name"])
# writer.writerows(out)
for user in users:
f1.write(json.dumps(user._json) + "\n")
if len(friends) == 5000:
print(
"Friends: More results available. Sleeping for 60 seconds to avoid rate limit"
)
time.sleep(60)
# get user's profile
fname = "users/{}/user_profile.json".format(screen_name)
with open(fname, 'w') as f:
profile = client.get_user(screen_name=screen_name)
f.write(json.dumps(profile._json, indent=4))
def testcursorcursorpages(self):
pages = list(Cursor(self.api.friends_ids).pages())
self.assert_(len(pages) > 0)
pages = list(Cursor(self.api.followers_ids, 'twitter').pages(5))
self.assert_(len(pages) == 5)
def get_home_timeline_tweets(self, num_tweets):
home_timeline_tweets = []
for tweet in Cursor(
self.twitter_client.home_timeline).items(num_tweets):
home_timeline_tweets.append(tweet)
return home_timeline_tweets
def get_user_timeline_tweets(self, num_tweets):
tweets = []
for tweet in Cursor(self.twitter_client.user_timeline,
id=self.twitter_user).items(num_tweets):
tweets.append(tweet)
return tweets
#Source Code for Mining data on Twitter with Python Course by TigerStyle Code Academy
import sys
import json
from tweepy import Cursor
from twitter_client import get_twitter_client
if __name__ == '__main__':
user = sys.argv[1]
client = get_twitter_client()
fname = "user_timeline_{}.jsonl".format(user)
with open(fname, 'w') as f:
for page in Cursor(client.user_timeline, screen_name=user, count=200).pages(16):
for status in page:
f.write(json.dumps(status._json)+"\n")
def get_timeline_tweets(self, user, num_tweets):
user_timeline_tweets = []
for tweet in Cursor(self.api_instance.home_timeline, id=user).items(num_tweets):
user_timeline_tweets.append(tweet)
return user_timeline_tweets
city = "Delhi"
access_token = ''
access_token_secret = ''
consumer_key = ''
consumer_secret = ''
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
print('authed')
api = tweepy.API(auth)
places = api.geo_search(query=city)
place_id = places[0].id
coords = (places[0].bounding_box.coordinates)[0]
location = coords[0] + coords[2]
#Cursor implementation
tweets = Cursor(api.search, q='place:' + place_id).items()
i = 0
for tweet in tweets:
with open('tweets/location/' + city + '.txt', 'a+') as f:
f.write('TWEET_TEXT: ' + tweet.text + '\n')
#Stream implementation
'''class StdOutListener(StreamListener):
def on_data(self, tweet):
tweet=json.loads(tweet)
try:
#print(tweet['place']['name'],tweet['user']['name'])
