def test_GetUserTimeline(self): # モックのインタフェースを作成 apiMock = self.mocker.CreateMock(tweepy.API) # モックの戻り値クラスを作成 statusMock = self.mocker.CreateMock(tweepy.models.Status) statusMock.id=100 statusMock.text=u'test status' # Twitterクラスを作成 twitter = Twitter() # 対象クラスにMockインタフェースの設定 twitter.api = apiMock # 動作記録をリセット self.mocker.ResetAll() # 動作の記録 apiMock.user_timeline(id=u'test').AndReturn([statusMock]) # 動作検証の開始 self.mocker.ReplayAll() lst = twitter.getUserTL(u'test') # 実動作の確認(正しくないとエラーが出るよ) self.mocker.VerifyAll() # 戻り値が1件であること assert_equal(1, len(lst)) # 正しいデータが取得できること assert_equal(100, lst[0][0]) assert_equal(u'test status', lst[0][1])
def __init__(self, rootpath, folderpath, start, end, originhashtag):
"""Initialize the parameters.
Parameters
----------
rootpath : str
the root path of the data
folderpath : str
the folder path of the data
start : str
the start date of the query
format: 'yyyy-mm-dd'
end : str
the end date of the query
format: 'yyyy-mm-dd'
originhashtag : str
the original query hashtag
Returns
-------
None
"""
self.rootpath = rootpath
self.folderpath = folderpath
self.start = start
self.end = end
self.filename = "tweets.pkl"
self.originhashtag = originhashtag
self.helper = Utility.Helper(rootpath)
self.addressTweet = Twitter.AddressTweet(rootpath)
self.getTwitter = Twitter.GetTwitterData(rootpath)
def picture_to_twitter(bh_tweet):
print("Taking your picture smile!!")
time.sleep(0.2)
with picamera.PiCamera() as camera:
camera.resolution = (1024, 768)
camera.start_preview()
# Camera warm-up time
time.sleep(1)
camera.capture("foo.jpg")
Twitter.tweet_with_photo("/home/pi/Desktop/MT-MuseumBoerhaave/foo.jpg", bh_tweet)
def test_TweetEnableDisable(self): # Twitter制御クラスの作成 twitter = Twitter() assert_true(twitter.isEnable) twitter.setDisable() assert_false(twitter.isEnable) twitter.setEnable() assert_true(twitter.isEnable)
def picture_to_twitter(bh_tweet):
print("Taking your picture smile!!")
time.sleep(0.2)
with picamera.PiCamera() as camera:
camera.resolution = (1024, 768)
camera.start_preview()
# Camera warm-up time
time.sleep(1)
camera.capture("foo.jpg")
Twitter.tweet_with_photo("/home/pi/Desktop/MT-MuseumBoerhaave/foo.jpg",
bh_tweet)
def test_GetMentions(self): # モックのインタフェースを作成 apiMock = self.mocker.CreateMock(tweepy.API) # モックの戻り値クラスを作成1 userMock1 = self.mocker.CreateMock(tweepy.models.User) userMock1.screen_name = u'testuser1' statusMock1 = self.mocker.CreateMock(tweepy.models.Status) statusMock1.id=100 statusMock1.text=u'test1 status' statusMock1.user=userMock1 # モックの戻り値クラスを作成2 userMock2 = self.mocker.CreateMock(tweepy.models.User) userMock2.screen_name = u'testuser2' statusMock2 = self.mocker.CreateMock(tweepy.models.Status) statusMock2.id=200 statusMock2.text=u'test2 status' statusMock2.user=userMock2 # Twitterクラスを作成 twitter = Twitter() # 対象クラスにMockインタフェースの設定 twitter.api = apiMock # 動作記録をリセット self.mocker.ResetAll() # 動作の記録 apiMock.mentions().AndReturn([statusMock1, statusMock2]) # 動作検証の開始 self.mocker.ReplayAll() lst = twitter.getMentions() # 実動作の確認(正しくないとエラーが出るよ) self.mocker.VerifyAll() # 戻り値が2件であること assert_equal(2, len(lst)) # 正しいデータが取得できること assert_equal(100, lst[0][0]) assert_equal(u'testuser1', lst[0][1]) assert_equal(u'test1 status', lst[0][2]) assert_equal(200, lst[1][0]) assert_equal(u'testuser2', lst[1][1]) assert_equal(u'test2 status', lst[1][2])
def is_bot_spammer_observer(tweets, user_name, debug=0):
if len(tweets) > 2:
is_spam = is_SPAM(tweets, debug=debug)
else:
is_spam = 0
user = Twitter.get_user_data(user_name)
if debug == 1:
print('User name:', user_name)
if (user.followers_count / 4 < user.friends_count
and user.statuses_count < 4
) or (user.followers_count / 4 < user.friends_count and is_spam == 1):
print(
'Attention!\nThis user could be a bot or an observer troll, have',
len(tweets), 'tweets,', user.followers_count,
'followers, and follows', user.friends_count, 'users.')
return 0
elif is_spam:
print(
'Attention!\nThis user could be a spammer troll, always publishes the same.'
)
return 0
else:
print('This user sames a normal user.')
return 1
def data_loading(request, content_type, htmlname):
params = content_type_dict[content_type].copy()
repo = params["repo"]
query = params["query"].format(htmlname)
params.update({"htmlname": htmlname})
if not Github.has_already_created(repo, htmlname):
tweet_list = []
myTwitterAction = Twitter.MyTwitterAction()
if content_type == "legends":
tweet_list = myTwitterAction.search_tweet_list_universal(query,
amount=50)
elif content_type == "overseas":
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
try:
tweet_list = translate_tweet_list(tweet_list)
except:
pass
else:
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload(repo, htmlname, text)
return render(request, f"ranking/ranking.html", params)
def index():
error = None
url = "https://plot.ly/~ccharmander4/69.embed"
#steam = st.plotSteam()
steam = "https://plot.ly/~ccharmander4/80.embed"
try:
text = request.args.get("topic")
tw.twitter_wordcloud(text)
img = "assets/img/"+text+".png"
return render_template("index.html", googleTrend = url, steamTrend = steam, img_source = img)
except Exception as e:
#flash(e)
#url = s.plotInterest("Star wars")
return render_template("index.html", googleTrend = url, steamTrend = steam)
def __call__ (self):
page = Page.Page_Menu()
page.banner += Top_Banner()
main_box = CTK.Box ({'id': 'index-main'})
main_box += Highlights()
page += main_box
side_box = CTK.Box ({'id': 'index-sidebox'})
side_box += Sidebox()
page += side_box
page += CTK.Box({'class': 'clr'})
bar3 = CTK.Box ({'class': 'bar3'})
bar3 += Twitter.Latest_Tweets()
page += bar3
bar3 = CTK.Box ({'class': 'bar3'})
bar3 += Commits.Latest_GIT_Commits()
page += bar3
bar3 = CTK.Box ({'class': 'bar3'})
bar3 += MailingList.Latest_Mailing_List()
page += bar3
page += CTK.Box({'class': 'clr'})
return CTK.HTTP_Cacheable (10, body=page.Render())
def extract(self, user, consumer_key, consumer_secret, access_token,
access_token_secret, test):
self.logger.init_tweet_log(user)
self.logger.log_system("Connecting to Twitter...")
twitter = Twitter(consumer_key, consumer_secret, access_token,
access_token_secret, test)
twitter.auth()
self.logger.log_system("Connected. Extracting user timeline...")
timeline = twitter.timeline(user)
count = 0
for tweet in timeline:
if "text" in tweet:
count = count + 1
self.logger.log_system("Tweet logged: #:%d | ID:%s" %
(count, tweet["id"]))
# print json.dumps(tweet, ensure_ascii=False).encode('utf8')
self.logger.log_tweet(tweet)
if count >= 3180 and test:
self.logger.log_system(
"Approaching 3200 tweet limit for open API; closing connection"
)
twitter.close()
break
elif "message" in tweet:
self.logger.log_system("Message %s" % tweet[message])
break
self.logger.close_tweet_log()
return self.logger.get_output_file()
def connect_to_twitter():
print('Establish connection with Twitter API...')
try:
api = Twitter.get_api()
except Exception:
print('Connection failed')
shutdown_server()
print('Connection Established')
return api
def main():
rootpath = '/local/data/haoxu/Rudetect'
folderpath = 'Gabapentin_0628_0121'
start = '2017-06-28'
end = '2018-01-21'
originhashtag = '#gabapentin'
gettweets = Twitter.GetTweets(rootpath, folderpath, start, end,
originhashtag)
gettweets.start_getTweets()
def test_Refollow(self): # モックのインタフェースを作成 apiMock = self.mocker.CreateMock(tweepy.API) # ユーザーのモックを作成 userMock = self.mocker.CreateMock(tweepy.models.User) userMock.id = 10 userLockMock = self.mocker.CreateMock(tweepy.models.User) userLockMock.id =400 userLockMock.protected = True userUnlockMock = self.mocker.CreateMock(tweepy.models.User) userUnlockMock.id = 500 userUnlockMock.protected = False # Twitterクラスを作成 twitter = Twitter() # 対象クラスにMockインタフェースの設定 twitter.api = apiMock # 動作記録をリセット self.mocker.ResetAll() # 動作の記録 apiMock.me().AndReturn(userMock) apiMock.friends_ids(id=10).AndReturn([100, 200, 300, 1000]) apiMock.followers_ids(id=10).AndReturn([200, 300, 400, 500]) apiMock.get_user(id=400).AndReturn(userLockMock) apiMock.get_user(id=500).AndReturn(userUnlockMock) apiMock.create_friendship(id=500) # 動作検証の開始 self.mocker.ReplayAll() # リフォロー機能 twitter.refollow(); # 実動作の確認(正しくないとエラーが出るよ) self.mocker.VerifyAll()
def twitterEventReceived():
requestJson = request.get_json()
#dump to console for debugging purposes
print(json.dumps(requestJson, indent=4, sort_keys=True))
if 'favorite_events' in requestJson.keys():
#Tweet Favourite Event, process that
likeObject = requestJson['favorite_events'][0]
userId = likeObject.get('user', {}).get('id')
#event is from myself so ignore (Favourite event fires when you send a DM too)
if userId == CURRENT_USER_ID:
return ('', HTTPStatus.OK)
Twitter.processLikeEvent(likeObject)
elif 'direct_message_events' in requestJson.keys():
#DM recieved, process that
eventType = requestJson['direct_message_events'][0].get("type")
messageObject = requestJson['direct_message_events'][0].get(
'message_create', {})
messageSenderId = messageObject.get('sender_id')
#event type isnt new message so ignore
if eventType != 'message_create':
return ('', HTTPStatus.OK)
#message is from myself so ignore (Message create fires when you send a DM too)
if messageSenderId == CURRENT_USER_ID:
return ('', HTTPStatus.OK)
Twitter.processDirectMessageEvent(messageObject)
elif 'tweet_create_events' in requestJson.keys():
for tweet in requestJson['tweet_create_events']:
set_lights_from_a_tweet(tweet)
else:
#Event type not supported
return ('', HTTPStatus.OK)
return ('', HTTPStatus.OK)
def getTweets(self, query, start, end):
"""Get the the final tweets.
Returns
-------
None
"""
gettweets = Twitter.GetTweets(self.rootpath, self.folderpath, start,
end, query)
gettweets.start_getTweets()
def test_TweetControl(self): # APIのモックを作成 apiMock = self.mocker.CreateMock(tweepy.API) # Twitter制御クラスの作成 twitter = Twitter() # モックのインタフェースを設定 twitter.api = apiMock # Enable状態 twitter.setEnable() # 動作記録をリセット self.mocker.ResetAll() # [動作記録]ステータスのアップデート apiMock.update_status(u'test') # 動作検証の開始 self.mocker.ReplayAll() twitter.update(u'test') # 実動作の確認(正しくないとエラーが出るよ) self.mocker.VerifyAll() # Disable状態 twitter.setDisable() # 動作記録をリセット self.mocker.ResetAll() # 動作検証の開始 self.mocker.ReplayAll() twitter.update(u'test') # 実動作の確認(正しくないとエラーが出るよ) self.mocker.VerifyAll()
def get_response(self):
"""
Get a response from the chatbot and display it.
"""
user_input = self.usr_input.get()
self.usr_input.delete(0, tk.END)
if self.user_name == "":
self.user_name = user_input
if self.user_name == "":
self.conversation['state'] = 'normal'
self.conversation.insert(tk.END, "Enter your name:" + "\n")
self.conversation['state'] = 'disabled'
else:
response = chatbot.get_response("Hello")
self.conversation['state'] = 'normal'
self.conversation.insert(
tk.END, "Human: " + user_input + "\n" + "ChatBot: " +
str(response.text) + "\n")
self.conversation['state'] = 'disabled'
else:
command = 'null'
if user_input[0:4] == 'play':
command = 'play'
if user_input[0:5] == 'tweet':
command = 'tweet'
if command == 'play':
if user_input == 'play song':
exec(open('rand_beat.py').read())
elif user_input == 'play game':
exec(open('game_run.py').read())
elif command == 'tweet':
status = "From " + self.user_name + ": " + user_input[6:]
new_tweet = tweet.Tweet(status=status)
self.conversation['state'] = 'normal'
self.conversation.insert(
tk.END, "\n" + self.user_name + " just tweeted: " +
user_input[6:] + "\n" +
"Check out your tweet at: https://twitter.com/project_i_do"
+ "\n" + "\n")
self.conversation['state'] = 'disabled'
else:
response = chatbot.get_response(user_input)
self.conversation['state'] = 'normal'
self.conversation.insert(
tk.END, "Human: " + user_input + "\n" + "ChatBot: " +
str(response.text) + "\n")
self.conversation['state'] = 'disabled'
class TestTwitterMethods(unittest.TestCase):
def setUp(self):
self.twitter = Twitter()
def test_get_tweets_user(self):
"""Test if retrieving specific user tweets returns list correctly.
This test will return an empty list and should still pass if
there is no Twitter Bearer Token available.
"""
user = '******'
result = self.twitter.get_tweets(user)
self.assertIsInstance(result, list)
def test_get_tweets_search(self):
"""Test if retrieving tweets for a generic search term returns list correctly.
This test will return an empty list and should still pass if
there is no Twitter Bearer Token available.
"""
search_term = 'news'
result = self.twitter.get_tweets(search_term)
self.assertIsInstance(result, list)
def test_get_tweets_hashtag(self):
"""Test if retrieving specific hashtag tweets returns list correctly.
This test will return an empty list and should still pass if
there is no Twitter Bearer Token available.
"""
hashtag = '#news'
result = self.twitter.get_tweets(hashtag)
self.assertIsInstance(result, list)
def test_parse_tweets(self):
"""Test if a list of tweets correctly parses into a single string.
"""
tweets = self.twitter.get_tweets('@twitter')
parsed_tweets = self.twitter.parse_tweets(tweets)
self.assertIsInstance(parsed_tweets, str)
def test_UpdateReply(self):
# モックのインタフェースを作成
apiMock = self.mocker.CreateMock(tweepy.API)
# Twitterクラスを作成
twitter = Twitter()
# 対象クラスにMockインタフェースの設定
twitter.api = apiMock
# 動作記録をリセット
self.mocker.ResetAll()
# 動作の記録
apiMock.update_status('test', 12345)
# 動作検証の開始
self.mocker.ReplayAll()
twitter.update('test', 12345);
# 実動作の確認(正しくないとエラーが出るよ)
self.mocker.VerifyAll()
def twitterEventReceived():
requestJson = request.get_json()
if 'favorite_events' in requestJson.keys():
#Tweet Favourite Event, process that
likeObject = requestJson['favorite_events'][0]
userId = likeObject.get('user', {}).get('id')
#event is from myself so ignore (Favourite event fires when you send a DM too)
if userId == CURRENT_USER_ID:
return None
Twitter.processLikeEvent(likeObject)
elif 'direct_message_events' in requestJson.keys():
#DM recieved, process that
eventType = requestJson['direct_message_events'][0].get("type")
messageObject = requestJson['direct_message_events'][0].get(
'message_create', {})
messageSenderId = messageObject.get('sender_id')
#event type isnt new message so ignore
if eventType != 'message_create':
return None
#message is from myself so ignore (Message create fires when you send a DM too)
if messageSenderId == CURRENT_USER_ID:
return None
Twitter.processDirectMessageEvent(messageObject)
else:
#Event type not supported
return None
return None
def test_UpdateDisableDataStore(self):
# モックのインタフェースを作成
apiMock = self.mocker.CreateMock(tweepy.API)
# Twitterクラスを作成
twitter = Twitter()
# 対象クラスにMockインタフェースの設定
twitter.api = apiMock
# ツイート不可に設定
ds = DataStore.DataStore()
ds.setTweetEnable(False)
# 動作記録をリセット
self.mocker.ResetAll()
# 動作記録
self.mocker.ReplayAll()
twitter.update("test");
# 実動作の確認(正しくないとエラーが出るよ)
self.mocker.VerifyAll()
def main2():
repo = "overseas"
for htmlname in Github.seach_page_list(repo):
tweet_list = []
myTwitterAction = Twitter.MyTwitterAction()
query = htmlname + " lang:en min_faves:100"
tweet_list = myTwitterAction.search_tweet_list(query, amount=50)
tweet_list = translate_tweet_list(tweet_list)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.delete_page(repo, htmlname)
Github.upload(repo, htmlname, text)
time.sleep(4)
def main():
tag_list = Github.seach_page_list(repo)
for htmlname in tag_list:
Github.delete_page(repo, htmlname)
print("mainloop:", htmlname, "has deleted")
myTwitterAction = Twitter.MyTwitterAction()
tweet_list = myTwitterAction.search_tweet_list(
'"' + htmlname + '"' + " lang:ja min_faves:100", amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload(repo, htmlname, text)
print("mainloop:", htmlname, "is created")
time.sleep(4)
def getHistoricalTweets(self, username, foldername):
"""Get the historical tweets of user.
Parameters
----------
username : str
the username of account without '@'
Returns
-------
None
save the tweet object in username.pkl file
"""
tweetFlag = True
getTwitterData = Twitter.GetTwitterData(self.rootPath)
criteria = getTwitterData.setCriteria(username=username)
getTwitterData.getTweets(criteria, foldername, username + ".pkl",
tweetFlag)
def get_birthday_hits(self):
username = self.twitterUsernameTextField.text()
self.text = Twitter.birthday_extraction(username)
msg = QMessageBox()
msg.setWindowTitle("Hits From Twitter")
msg.setInformativeText("Hit Copy to Exit As Well")
msg.setText(self.text)
saveFiles = QPushButton("Save As")
copyText = QPushButton("Copy")
saveFiles.clicked.connect(self.save_files)
copyText.clicked.connect(self.copy_text)
msg.addButton(saveFiles, QMessageBox.YesRole)
msg.addButton(copyText, QMessageBox.YesRole)
val = msg.exec_()
def load_missing_ids_thread(self):
while self.missing_ids:
# Pick any id, but do not remove it yet in case this id would
# be requested again by another thread, before we are done.
id = self.missing_ids.pop()
self.missing_ids.add(id)
# Try fetching a description.
print "Before yield", id
yield True
print "After yield", id
try:
buffer = Common.twitter.get_user_info(id)
except Common.Error:
# If we do not get it, just ignore it
buffer = None
if not buffer:
self.missing_ids.remove(id)
continue
# Save the description.
self.db[str(id)] = buffer
# Replace the old strip by a new one wherever appropriate.
new = self.load(id)
assert new is not None
old = User(Twitter.dummy_user(id))
for tab in Tab.Tab.registry.itervalues():
if isinstance(tab.strip_type, User):
if old in tab.strips:
tab.discard_strips([old])
tab.add_strips([new])
# Check!
tab.refresh()
# Ok, we are now done for real with this id.
self.missing_ids.remove(id)
def data_loading(request, htmlname):
result = seach(request)
if result:
return result
params = {
"title": "twitterアカウント分析サイト",
"description": site_explain,
"favicon": favicon,
"img": img,
"repo": repo,
"htmlname": htmlname,
}
if not Github.has_already_created(repo, htmlname):
myTwitterAction = Twitter.MyTwitterAction()
tweet_list = myTwitterAction.search_tweet_list(htmlname, amount=50)
git_json = {}
git_json.update({"tweet_list": tweet_list})
git_json.update({"wordcloud": genWordList(tweet_list)})
text = json.dumps(git_json, ensure_ascii=False, indent=4)
Github.upload("twitter_json", htmlname, text)
return render(request, f"fanstatic/dashboard/dashboard.html", params)
def test_UpdateDisableProperty(self):
# モックのインタフェースを作成
apiMock = self.mocker.CreateMock(tweepy.API)
# Twitterクラスを作成
twitter = Twitter()
# 対象クラスにMockインタフェースの設定
twitter.api = apiMock
# ツイート不可に設定
twitter.setDisable()
# 動作記録をリセット
self.mocker.ResetAll()
# 動作記録
self.mocker.ReplayAll()
twitter.update("test");
# 実動作の確認(正しくないとエラーが出るよ)
self.mocker.VerifyAll()
'''
Created on Nov 21, 2012
@author: kevin
'''
import main
import Twitter
def get_feeds():
main.connect_foursquare()
Twitter.connect_twitter()
if __name__ == '__main__':
main.connect_foursquare()
Twitter.connect_twitter()
pass
def loginTwitter(self,loginList):
self.twitterID = Twitter.Twitter(str(loginList[0]),str(loginList[1]))
self.twitterID.signIn()
def main():
#check folder
if not os.path.exists("dataset_instagram"):
os.makedirs("dataset_instagram")
if not os.path.exists("dataset_twitter"):
os.makedirs("dataset_twitter")
if not os.path.exists("result"):
os.makedirs("result")
network = input("Enter the network you want to use (twitter or instagram) : ")
if network != "twitter" and network != "instagram":
print("Error of network, please choose between twitter or instagram")
sys.exit()
hashtag = input("Enter the hashtag you want to analyse : ")
if network == "twitter":
nbtweet = input("Enter the number of tweets you want : ")
if network == "instagram":
nbtweet = input("Enter the number of post you want : ")
language = input("Enter the language you want (en or fr) : ")
if language != "en" and language != "fr":
print("Error of language, please choose between en or fr")
sys.exit()
if network == "twitter":
path = Twitter.getTweets(hashtag, int(nbtweet), language)
elif network == "instagram":
path = Instagram.get_hash_tag_caption(hashtag, int(nbtweet), "dataset_instagram/"+language+"/"+hashtag+"/"+hashtag+".csv", language)
if language == "en":
Data_Cleaner.DataCleanerEn("pred", hashtag, path)
elif language == "fr":
Data_Cleaner.DataCleanerFr("pred", hashtag, path)
pathpred = Predict.predict(network, hashtag, path, language)
Negative = 0
Neutral = 0
Positive = 0
Total = -1
with open(pathpred, newline='') as csvfile:
predfile = csv.reader(csvfile, delimiter=',')
for row in predfile:
#Total
Total = Total + 1
#Negative
if row[0] == '0':
Negative = Negative + 1
#Neutral
elif row[0] == '2':
Neutral = Neutral + 1
#Positive
elif row[0] == '4':
Positive = Positive + 1
PercentNegative = Negative*100/Total
PercentNeutral = Neutral*100/Total
PercentPositive = Positive*100/Total
print("\nResult for #" + hashtag +" with " + nbtweet + " tweets in " + language + " :")
print("Negative:", round(PercentNegative, 2), "%")
print("Neutral:", round(PercentNeutral, 2), "%")
print("Positive:", round(PercentPositive, 2), "%")
print("\nRandom examples of prediction :")
with open(pathpred) as f:
reader = csv.reader(f)
for index, row in enumerate(reader):
if index == 0:
chosen_row = row
else:
r = random.randint(0, index)
if r == 0:
chosen_row = row
print(chosen_row)
import WeatherApiUi as wau
import TwiText
import Twitter
import WeatherDataExtractor as tc
import APIgetDataException as apie
from time import sleep
import WeatherData as wd
seto = ('35.1815', '137.1087')
toyota = ('35.08', '137.15')
owariasahi = ('35.21', '137.03')
miyoshi = ('35.13', '137.05')
nissin = ('35.17', '136.96')
twitter = Twitter.Twitter()
twitext = TwiText.TwiText()
weatherApi = wau.WeatherApiUi()
extractor = tc.WeatherDataExtractor()
def beSureToDo(city):
while (True):
try:
data = weatherApi.getDataLatLon(city[0], city[1], 1)
weatherData = extractor.getAllData(data)
twitter.twitte(twitext.Build(weatherData))
except apie.APIgetDataException:
sleep(60)
continue
break
def get_feeds():
main.connect_foursquare()
Twitter.connect_twitter()
# Developed and modified by Ye Liang at 11:11 June,3 2017
# All rights reserved
from ExtraTools import write_To_Csv, sleep_Module
import Twitter
import Sqlite
import Parser
import Cluster
CONST_DAYS_INCLUDED = 1
if __name__ == '__main__':
#----------initialize the modules and settings---------------------begin
db = Sqlite.Sqlite("database.db")
parserModule = Parser.Parser()
clusterModule = Cluster.Cluster()
twitterModule = Twitter.Twitter(db, parserModule)
#----------initialize the modules and settings---------------------end
runSequence = True
while(runSequence):
# update all tweets into the database within CONST_DAYS_INCLUDED day
twitterModule.update_All()
# get all updated tweets within CONST_DAYS_INCLUDED day
news24H = db.get_stored_news(CONST_DAYS_INCLUDED)
# cluster, and find the best news
sortedNews, labels, sortedScores = clusterModule.cluster_AP(news24H)
# write to csv file
write_To_Csv(sortedNews, sortedScores, 'Sorted_Best_Tweets (one for each cluster).csv')
# sleeping procedure
runSequence = sleep_Module(600)
db.close()
number of days of coverage
content:
day
date
number of words
number of content
entity
date
number of words
raw text
title (if applicable)
author (if applicable)
other ?
"""
import NYT
import Twitter
import NPR
import Guardian
import Tumblr
NYT.test()
Twitter.test()
NPR.test()
Guardian.test()
Tumblr.test()
#!/usr/bin/python
# wsd project main script
# hardware: ws2801 led strips + raspberry pi + internet adapter
# software pulls twits from an 'admin' (twits and retwits) and
# displays the last result through the led strip
# Written by Pratipo.org, hightly based on Adafruit's IoT Pinter. MIT license.
# MUST BE RUN AS ROOT (due to GPIO access)
import subprocess, time, socket, re
from Wsd2 import *
from Twitter import *
tw = Twitter()
display = Wsd2()
time.sleep(10)
display.setText('#wsd by idensitat+pratipo')
display.rollPixels(True)
time.sleep(0.01)
prev = ""
while(True):
for x in range(0, 3):
print "querying twitter"
body = tw.getNewest()
if __name__ == "__main__":
# Create an HTTP server to host the .LLM files
httpd = StoppableHTTPServer(("0.0.0.0", Settings.server_port), SimpleHTTPServer.SimpleHTTPRequestHandler)
thread.start_new_thread(httpd.serve, ())
# Set a 'storing' counter for audio clips.
storing_c = 0
# Wash, Rinse, Repeat.
while True:
print("Collecting tweets!")
# Retrieves the messages container.
tweets = Twitter.get_tweets()
# Create the llm file
LLM.build_llm(tweets)
# Send a message to the display to retrieve the LLM file.
Update.update_display(Settings.filename)
# Play audio when a 'storing' is discovered.
if storing_c < tweets.storingen:
# Play the audio file.
Audio.storing()
# Set the counter to the 'storing' level.
storing_c = tweets.storingen
def test_13_FollowUpdates_WithArgs_Fail_With_Arg(self):
self.assertRaises(Exception,
Twitter.Twitter('TestStr', 'TestStr').FollowUpdates,
'TestStr0')
def get(self):
self.response.out.write(u'Hello world!')
t=Twitter()
t.update(u"日本語は？")
def test_12_FollowUpdates_WithoutArgs_Pass_With_Only_Valid_Arguments(self):
Twitter.Twitter('TestStr', 'TestStr').FollowUpdates()
def setUp(self):
self.twitter = Twitter()
def buildDict4Tweets(self, foldername):
"""Build dictionary for tweets got from PHEME dataset.
The raw_data.json will be generated in each event folder.
Arguments:
foldername {str} -- folder name of an event
"""
resultDict = defaultdict(dict)
resultPickle = []
folderpath = os.path.join(self.rootPath, foldername)
if not os.path.exists(folderpath):
print("the path {} does not exist.".format(folderpath))
return
rnrsName = [
rnr for rnr in os.listdir(folderpath)
if os.path.isdir(os.path.join(folderpath, rnr))
]
for rnrName in rnrsName:
rnrPath = os.path.join(folderpath, rnrName)
# print("rnrPath ", rnrPath)
rumorsName = [
rumor for rumor in os.listdir(rnrPath)
if os.path.isdir(os.path.join(rnrPath, rumor))
]
for rumorName in rumorsName:
rumorPath = os.path.join(rnrPath, rumorName)
# print("rumorPath ", rumorPath)
reactionPath = os.path.join(rumorPath, "reactions")
# print("reactionPath ", reactionPath)
sourceTweetsPath = os.path.join(rumorPath, "source-tweets")
# print("sourceTweetsPath ", sourceTweetsPath)
reactions = [
reaction for reaction in os.listdir(reactionPath)
if os.path.isfile(os.path.join(reactionPath, reaction))
]
numReaction = len(reactions)
for sourceTweet in os.listdir(sourceTweetsPath):
sourceTweetPath = os.path.join(sourceTweetsPath,
sourceTweet)
# print("sourceTweetPath ", sourceTweetPath)
# avoid .DS_store
if os.path.isfile(
sourceTweetPath) and sourceTweet[0] != ".":
# print("sourceTweetPath ", sourceTweetPath)
data = self.loadJson(sourceTweetPath)
idx = data['id_str']
hashtags = [
'#' + hashtag['text']
for hashtag in data['entities']['hashtags']
]
rumor = True if rnrName == 'rumours' else False
date = data['created_at']
resultDict[idx]['text'] = data['text']
resultDict[idx]['favorite_count'] = data[
'favorite_count']
resultDict[idx]['retweet_count'] = data[
'retweet_count']
resultDict[idx]['hashtags'] = hashtags
resultDict[idx]['comment'] = numReaction
resultDict[idx]['rumor'] = rumor
resultDict[idx]['date'] = date
tweet = Twitter.Tweet()
tweet.id = idx
tweet.text = data['text']
tweet.favorites = data['favorite_count']
tweet.retweets = data['retweet_count']
tweet.reply = numReaction
tweet.rumor = rumor
tweet.setHashtags(hashtags)
tweet.setDate(date)
resultPickle.append(tweet)
self.dumpJson(
folderpath + "/../../pheme_dataset/" + foldername +
"/final/rawData", "raw_data.json", resultDict)
print("raw_data.json has been saved.")
self.dumpPickle(
folderpath + "/../../pheme_dataset/" + foldername +
"/final/rawData", "tweets.pkl", resultPickle)
print("tweets.pkl has been saved.")
import Twitter
#Welcome Screen
print("Twitter Analytics App")
#Take in User Inputs
selection = input("Type#1 to analyze the most used words",
"Type#2 to analyze twitter sentiment",
"Type#3 to analyze twitter stream")
if selection == 1:
print("proceed to analyze most used words")
u = input("Search Account: ")
num = int(input("How many tweets? "))
if selection == 2:
print("proceed to analyze twitter sentiment")
if selection == 3:
print("Proceed to analyze twitter stream")
hash_tag = input("Please enter the hashtag")
Twitter.user_tweets(u, num)
Twitter.polarity() #Not quite sure what to do here.
Twitter.stream_tweets(hash_tag)
def _when_done(self):
if self._options.tweet:
import Twitter
Twitter.tweet(self._options.tweetmsg, self._options.tweetat)
def handle_message(event):
userText = request.json["events"][0]["message"]["text"]
twitterMessage = Twitter.twitterGetMessage(userText)
line_bot_api.push_message('yourBotId', TextSendMessage(text=twitterMessage))
def home(user):
global bow
global users
return render_template('home.html', user=user, bow=bow, title=user + '\'s Home',
background_image=getImageUrl('all'), users=users)
print("Start ElasticSearch server")
client, es = Start()
print("Open connection")
print()
print("Start crawling date")
tw.twitter_scraper()
print("End crawling date")
print()
print("Start create index of tweet's document")
id.createIndexDocuments(client);
print("Index created successfully")
print()
print("Start GUI")
if __name__ == "__main__":
webbrowser.open('http://localhost:5000')
app.run()