def lambda_handler(event, context): with open("credentials.json", "r") as creds_file: creds = json.load(creds_file) oauth_consumer_key = creds["oauth_consumer_key"] oauth_consumer_secret = creds["oauth_consumer_secret"] oauth_token = creds["oauth_token"] oauth_token_secret = creds["oauth_token_secret"] api = TwitterAPI(consumer_key=oauth_consumer_key, consumer_secret=oauth_consumer_secret, access_token_key=oauth_token, access_token_secret=oauth_token_secret) username = event.get('payload') msg = "Remember to LOCK the door!" event = { "event": { "type": "message_create", "message_create": { "target": { "recipient_id": username }, "message_data": { "text": msg } } } } send_msg = api.request('direct_messages/events/new', json.dumps(event)) print('SUCCESS' if send_msg.status_code == 200 else 'PROBLEM: ' + send_msg.text)
def main(): app_settings = settings.get_app_settings() api = TwitterAPI.TwitterAPI(**app_settings) store_data = store.get_data() search_term = 'vnnlp' query = {'screen_name': search_term} filename = 'user_timeline/{}.yaml'.format(search_term) utils.ensure_dir(filename) if 'user_timeline' in store_data and 'max_id' in store_data[ 'user_timeline']: query['max_id'] = store_data['user_timeline']['max_id'] - 1 max_id = None try: with open(filename, 'a') as output_file: r = TwitterAPI.TwitterPager(api, 'statuses/user_timeline', query) for tweet in r.get_iterator(): yaml.dump([tweet], output_file, default_flow_style=False) if 'id' in tweet: max_id = tweet['id'] except KeyboardInterrupt: pass if not 'user_timeline' in store_data: store_data['user_timeline'] = {} store_data['user_timeline']['max_id'] = max_id store.store_data(store_data)
def send_tweet(message_to_tweet: str, url_to_media: str) -> int: """ Send a tweet with an image. :param message_to_tweet: Message to send :param url_to_media: Image to upload :return: Tweet ID (-1 if it failed) :raises Exception: Tweet failed to send for some reason """ twitter_api = TwitterAPI.TwitterAPI( config.get('twitter', 'CONSUMER_KEY'), config.get('twitter', 'CONSUMER_SECRET'), config.get('twitter', 'ACCESS_TOKEN_KEY'), config.get('twitter', 'ACCESS_TOKEN_SECRET'), ) logging.info('Tweet to send: {}'.format(message_to_tweet)) try: if url_to_media is not None: resize_image(url_to_media) photo = open(url_to_media, 'rb') status = twitter_api.request('statuses/update_with_media', {'status': message_to_tweet}, {'media[]': photo}) logging.info('Twitter Status Code: {}'.format(status.status_code)) response = TwitterAPI.TwitterResponse(status, False).json() logging.info('Twitter Response Parsed: {}'.format(response)) return int(response['id_str']) raise Exception("No image attached to tweet") except UnicodeDecodeError: logging.exception( 'Your message could not be encoded. Perhaps it contains non-ASCII characters?' ) raise Exception("Tweet failed to send")
def get_results() -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]: """ Get the results from the competition and print it out :return: Winner's name and their query """ twitter_api = TwitterAPI.TwitterAPI( config.get('twitter', 'CONSUMER_KEY'), config.get('twitter', 'CONSUMER_SECRET'), config.get('twitter', 'ACCESS_TOKEN_KEY'), config.get('twitter', 'ACCESS_TOKEN_SECRET'), ) valid_normal_entries: List[Dict[str, Any]] = [] valid_regex_entries: List[Dict[str, Any]] = [] logging.info('GET RESULTS') json_db: Dict[str, Any] = load_json_db( config.get('scryfallCardGolf', 'TWEET_DATABASE')) max_key: str = max(json_db.keys()) r = TwitterAPI.TwitterPager(twitter_api, 'statuses/mentions_timeline', { 'count': 200, 'since_id': json_db[max_key]['tweet_id'] }) for item in r.get_iterator(): if 'text' not in item: logging.warning('SUSPEND, RATE LIMIT EXCEEDED: ' + item['message']) break logging.info('[TWEET] ' + item['user']['screen_name'] + ': ' + item['text']) for url in item['entities']['urls']: test_url = url['expanded_url'] if 'scryfall.com' not in test_url: continue logging.info('{} submitted solution: {}'.format( item['user']['screen_name'], test_url)) test_query_results = test_query(item['user']['screen_name'], test_url) if test_query_results: user_json_entry: Dict[str, Any] = { 'name': item['user']['screen_name'], 'length': len(test_query_results), 'query': test_query_results } if re.search(r'/.+/', test_query_results): valid_regex_entries.append(user_json_entry) else: valid_normal_entries.append(user_json_entry) return valid_normal_entries, valid_regex_entries
def tweet_download_and_lang_detect(df_tweets, user_id, update_to_invalid_list): """ Calls language detection and checks if enough german tweets remain. If it found almost enough german Tweets it will load more. If it found almost none it will abort. :param df_tweets: 0 during first run, dataframe with tweets during later runs :param user_id: Twitter User_ID for tweet download and language check :param update_to_invalid_list: List of user that can not be downloaded from. Will append to if applicable. :return: df_tweets, update_to_invalid_list, abort_loop, len_df """ if isinstance(df_tweets, int): df_tweets = TwitterAPI.API_tweet_multitool( user_id, 'temp', pages=1, method='user_timeline', append=False, write_to_db=False) # fills DF with 200 tweets of 1 page df_tweets = helper_functions.lang_detect(df_tweets) else: df_tweets_additions = TwitterAPI.API_tweet_multitool( user_id, 'temp', pages=1, method='user_timeline', append=False, write_to_db=False) # fills DF with 200 tweets of 1 page df_tweets_additions = helper_functions.lang_detect( df_tweets_additions) if isinstance(df_tweets_additions, pd.DataFrame): df_tweets = pd.concat([df_tweets, df_tweets_additions]) df_tweets.reset_index(inplace=True) del df_tweets['index'] # if df_tweets is None: #no tweets found or all tweets deleted (non german) # abort_loop = True # return df_tweets, update_to_invalid_list, abort_loop' len_df = helper_functions.dataframe_length(df_tweets) if len_df <= 50: # if almost no tweets are german don't try to get more german tweets from this users. # would take to many page loads update_to_invalid_list.append(user_id) abort_loop = True elif len_df >= 200: abort_loop = True else: # if to few tweets are german load more tweets to get a better result abort_loop = False gc.collect() return df_tweets, update_to_invalid_list, abort_loop, len_df
def check_user_info(self): """ Searches for twitter user profile and saves the data :return: """ #connect to the api api = Tapi.TwitterAPI(cons_key, cons_secret, token, token_secret) #get request out = api.request('users/lookup', {'Name': '{}'.format(self.username), 'screen_name': '{}'.format(self.username)}) if out is not None: for i in out: info = i following = info['friends_count'] followers = info['followers_count'] join_date = info['created_at'] location = info['location'] description = info['description'] user_id = info['id_str'] #create user self.user = TwitterUser(username=info['screen_name'], location=location, description=description, date_joined=join_date, following=following, followers=followers,id=user_id) # save this user TwitterUserDA().save_user(self.user) else: self.user = TwitterUser(username=self.username, location="", description="*manually created*", date_joined="Joined June 2009", following='', followers='')
def __init__(self, twitter_creds, host, port): self.api = TwitterAPI(twitter_creds["consumer_key"], twitter_creds["consumer_secret"], twitter_creds["access_token_key"], twitter_creds["access_token_secret"]) self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.bind((host, port)) print host, port self.s.listen(5) while True: try: c, addr = self.s.accept() self.c = c self.stream() except Exception as e: print e
def searchTwitterAPI(self): query = str(self.filmEntry.get()) releaseDate = str(self.rdEntry.get()) dbTableName = str(query.replace(" ", "")) filmTitleFile = open("filmTitle.txt", "w") filmTitleFile.write(str(dbTableName) + "\n") filmTitleFile.write(releaseDate) filmTitleFile.close() api = TwitterAPI.TwitterClient() weekNo = api.getWeekNO(releaseDate) tableExists = api.createDBTable(dbTableName) if tableExists: #check if there are tweets with the same week number if there are #and the weekNO is differenet then we search for tweets and add them #else we just continue cur.execute( """SELECT COUNT(*) FROM """ + str(dbTableName) + """ WHERE weekNo=?""", (weekNo, )) if cur.fetchone()[0] >= 1: self.goToResults() else: tweets = api.getTweets(query) api.populateDB(dbTableName, tweets, weekNo) api.populateReultsTable(dbTableName, weekNo) self.goToResults() else: tweets = api.getTweets(query) #now we will see if the table is in the database and if it is we will #then check if the data we are adding is new if it is the it will be added to the db api.createDBTable(dbTableName) api.populateDB(dbTableName, tweets, weekNo) api.populateReultsTable(dbTableName, weekNo) self.goToResults()
def test_paging_iterator(self): pager = TwitterAPI.TwitterRestPager(self.api, 'search/tweets', {'q': 'pizza'}) self.assertIsInstance(pager, TwitterAPI.TwitterRestPager) it = pager.get_iterator() self.use_iterator(it)
def populateResultsPage(self): file = open("filmTitle.txt", "r") lines = file.readlines() rdate = lines[1].strip() tableName = lines[0].strip() api = TwitterAPI.TwitterClient() weekNO = api.getWeekNO(rdate) polarity = api.countPostiveTweets( tableName, weekNO) / api.countNegativeTweets(tableName, weekNO) subjectivity = (api.countPostiveTweets(tableName, weekNO) + api.countNegativeTweets(tableName, weekNO) ) / api.countNeutralTweets(tableName, weekNO) self.polarityResult.config(text=str(polarity)) self.subjectivityResult.config(text=str(subjectivity)) self.weekResult.config(text=str(weekNO)) #now to get the chnage in polarity and subjectivity try: polarityChange = polarity - ( api.countPostiveTweets(tableName, (weekNO - 1)) / api.countNegativeTweets(tableName, (weekNO - 1))) subjectivityChange = subjectivity - ( (api.countPostiveTweets(tableName, (weekNO - 1)) + api.countNegativeTweets(tableName, (weekNO - 1))) / api.countNeutralTweets(tableName, (weekNO - 1))) #now i will set all the labels to the new values self.polarityCResult.config(text=str(polarityChange)) self.subjectivityCResult.config(text=str(subjectivityChange)) except ZeroDivisionError: self.polarityCResult.config(text="N/A") self.subjectivityCResult.config(text="N/A")
def main(): app_settings = settings.get_app_settings() api = TwitterAPI.TwitterAPI(**app_settings) search_term = 'dongng' query = {'screen_name': search_term} twitter.fetch_one(api, 'followers/list', search_term, query)
def setUp(self): """Read credentials from TwitterAPI/credentials.txt. You must copy your credentials into this text file. """ oa = TwitterAPI.TwitterOAuth.read_file() self.api = TwitterAPI.TwitterAPI(oa.consumer_key, oa.consumer_secret, oa.access_token_key, oa.access_token_secret)
def tweetImage(message, image_file): api = ta.TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) file = open(image_file, 'rb') data = file.read() r = api.request('statuses/update_with_media', {'status': message}, {'media[]': data}) return 'SUCCESS' if r.status_code == 200 else 'FAILURE'
def main(): app_settings = settings.get_app_settings() api = TwitterAPI.TwitterAPI(**app_settings) search_term = 'dongng' query = {'q': search_term} twitter.fetch(api, 'search/tweets', search_term, query)
def __init__(self, tweet_count=200, twitter_account='stackdevjobs'): self.tweet_count = tweet_count self.twitter_account = twitter_account self.api = TwitterAPI.TwitterAPI(self.consumer_key, self.consumer_secret, auth_type='oAuth2')
def main(): api = TwitterAPI.TwitterAPI(**settings.get_app_settings()) # twitter.fetch(api, 'search/tweets', 'input', {'q': 'coccoc', 'lang': 'vi'}) # twitter.fetch(api, 'search/tweets', 'input', {'q': '#coccoc', 'lang': 'vi'}) # twitter.fetch(api, 'search/tweets', 'input', {'q': 'cntt', 'lang': 'vi'}) # twitter.fetch(api, 'search/tweets', 'input', {'q': 'ifan', 'lang': 'vi'}) # twitter.fetch(api, 'search/tweets', 'input', {'q': 'yêu', 'lang': 'vi'}) twitter.fetch(api, 'search/tweets', 'input', {'q': 'ghét', 'lang': 'vi'})
def test_app_normal(event, mocker): r = requests.Response() r.status_code = 200 r._content = '{}'.encode() res = TwitterAPI.TwitterResponse(r, None) mocker.patch('TwitterAPI.TwitterAPI.request', return_value=res) ret = app.handle(event, config) assert ret == {'MessageIds': ['dummy']}
def test_app_abnormal(event, mocker): r = requests.Response() r.status_code = 401 # authorization error r._content = '{}'.encode() res = TwitterAPI.TwitterResponse(r, None) mocker.patch('TwitterAPI.TwitterAPI.request', return_value=res) with pytest.raises(Exception): app.handle(event, config)
def test_app_normal(event, mocker): r = requests.Response() r.status_code = 200 r._content = '{"id_str": "12345"}'.encode() res = TwitterAPI.TwitterResponse(r, None) mocker.patch('TwitterAPI.TwitterAPI.request', return_value=res) ret = app.handle(event, config) assert ret == {'Results': {'dummy': '12345'}}
def test_app_already_retweeted(event, mocker): r = requests.Response() r.status_code = 403 error = {"errors": [{"code": 187, "message": "Status is a duplicate."}]} r._content = json.dumps(error).encode() res = TwitterAPI.TwitterResponse(r, None) mocker.patch('TwitterAPI.TwitterAPI.request', return_value=res) ret = app.handle(event, config) assert ret == {'Results': {'dummy': None}}
def connect_to_streaming_API(): ''' Connect to twitter streaming API Returns an iterator with all the stream data. ''' api = TwitterAPI.TwitterAPI( 'f8olfZWtAPvgANdP9qecg', 'bSEnCXJuWazjT8S8hZ6BLWMo1C7egIKNgjObHM6Ck', '1726636778-jEn4qUAj2wV60ckbskNSbLJgTRr0c7hiemVOU7x', 'UgwEfM3cukoWIxCWjCiIZiJ0gnQVGH9U42WLfJjnEFODw') r = api.request('statuses/filter', {'track': '#rip'}) return r.get_iterator()
def getTweetCount(q, p=False, debug=False): auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) api = TwitterAPI.TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret) #today's info a = time.time() timeStamp = datetime.datetime.utcnow().date() tDay = timeStamp.day tMonth = timeStamp.strftime("%b") api = TwitterAPI.TwitterAPI(consumer_key, consumer_secret, access_token, access_token_secret) count = 0 r = TwitterPager(api, 'search/tweets', {'q': q, 'count': 100}) for item in r.get_iterator(wait=6): time_stamp = item['created_at'] day = int(time_stamp[8:10]) if (tDay != day and tDay - 1 != day): break if 'message' in item and item['code'] == 88: print('SUSPEND, RATE LIMIT EXCEEDED: %s' % item['message']) break if (tDay - 1 == day): count += 1 if (p): print("Term: " + q + " on " + item["created_at"]) if (debug): b = time.time() c = int(b - a) print("\nThis took " + str(round(c / 60, 2)) + " minutes") return count #res = tweet.getTweetCount("qwertyuiop", False,True) #print(res)
def __init__(self): self.name = 'TwitterApi' self.keys_dict = json.loads(open(twitter_key_path, 'r').read()) self.twitter_api = twitter.TwitterAPI( consumer_key=self.keys_dict['consumer_key'], consumer_secret=self.keys_dict['consumer_secret'], access_token_key=self.keys_dict['access_token_key'], access_token_secret=self.keys_dict['access_token_secret']) self.querying_types = [ # TwitterApi.ResultType.popular, # TwitterApi.ResultType.recent, TwitterApi.ResultType.mixed ]
def search_users(query): "performs a user/search query and returns list of most relevant users name" # needs a connection with oAuth1 ie with user credentials # we use Jafeel2 ;) consumer_key = 'ZuFJRql4R1wLe3vOP9IcD8B5t' consumer_secret = 'XtrinZVQbvHYCcc0IpHDG7NA4DuocdjHIdQ1Dt7u5w8vkxhonk' access_token_key = '137058679-CVBZBXfDCyox60tWZVbHcHK1cSWROXzxjzRfdO6s' access_token_secret = 'yS2Zb2g5pD4QuUbgYRFt966xPqfmB3AZzlbfeKd0Nrbyo' api2 = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) page = 1 nb_results = 10 # r = api2.request('users/search', {'q':query, 'count':nb_results }).json() r = api2.request('users/search', {'q':query, 'page':page, 'count':nb_results }).json() user_list = [] for item in r: user_list.append(item['screen_name']) print user_list return user_list
def test_app_already_retweeted(event, mocker): r = requests.Response() r.status_code = 403 error = { "errors": [{ "code": 327, "message": "You have already retweeted this Tweet." }] } r._content = json.dumps(error).encode() res = TwitterAPI.TwitterResponse(r, None) mocker.patch('TwitterAPI.TwitterAPI.request', return_value=res) ret = app.handle(event, config) assert ret == {'MessageIds': ['dummy']}
def download_user_timelines(political_moderate_list: list, right_wing_populists_list: list): """ Downloads user timelines of users featured in below lists. The downloads are used as training material for AI training. All lists are just incomplete examples. :return: """ #List examples # political_moderate_list = ['_pik_dame_', 'Leekleinkunst', 'MartinaKraus7', 'KAFVKA', 'Volksverpetzer', 'insideX', # 'FranziLucke', 'leonie_stella9', 'Ute631', 'justMPO', 'anouk_avf', 'Komisaar', # 'MenschBernd', 'von_ems', 'lies_das', 'seewanda', 'Rene_Bacher', 'Prasanita93', # 'IgorUllrich', 'AJSalzgitter', 'Bussi72', 'HuWutze', 'strahlgewitter', 'PhilKupi', # 'BaldusEla', 'LarsKlingenberg', 'MichaelSchfer71', 'EddyAges', 'veripot', 'JoernPL', # 'ondreka', 'kleinerJedi', 'DanielKinski', 'wfh7175', 'Sister_records1', 'TinaJergerkamp'] # right_wing_populists_list = ['Junge_Freiheit', 'zaferflocken', 'HelmutWachler', 'M_Briefing', 'TinVonWo', 'mcwd12', # 'EBlume3', 'h_hendrich'] #Political unpolitical stance is currently not used # Tweets of below accounts will be downloaded from twitter. During model a subset of below accounts might be used. # unpolitical_list = ['Podolski10', 'fckoeln', 'FCBayern', 'BVB', 'rtl2', 'DMAX_TV', 'tim_kocht', 'grandcheflafer', # 'bildderfrau', 'gala', 'BUNTE', 'promiflash', 'funny_catvideos', 'BibisBeauty', 'dagibee', # 'siggismallz', 'Gronkh', 'CHIP_online', 'COMPUTERWOCHE', 'SkySportNewsHD', 'MOpdenhoevel', # 'kayefofficial', 'VOGUE_Germany', 'lucycatofficial', 'RealLexyRoxx', 'AnselmSchindler', # 'pentru_tine', 'KaJa80028344'] #unpolitical_list = ['Podolski10'] For Testing # political_list = ['Thomas_Ehrhorn', 'HilseMdb', 'DirkSpaniel', 'MdB_Lucassen', 'RolandTichy', 'UllmannMdB', # 'c_jung77', 'michael_g_link', 'theliberalfrank', 'IreneMihalic', 'KaiGehring', 'RenateKuenast', # 'GoeringEckardt', 'MdB_Freihold', 'ZaklinNastic', 'PetraPauMaHe', 'lgbeutin', 'arnoklare', # 'zierke', 'Timon_Gremmels', 'Johann_Saathoff', 'uhl_markus', 'AnjaKarliczek', 'KLeikert', # 'Junge_Gruppe'] user_lists = { 'political_moderate_list': political_moderate_list, 'right_wing_populists_list': right_wing_populists_list } # List Download for list_name, username_list in user_lists.items(): for element in username_list: TwitterAPI.API_tweet_multitool(element, list_name, pages=10, method='user_timeline', append=True, write_to_db=True)
def fetch(api, resource, search_term, query): filename = '{0}/{1}.yaml'.format(resource.replace('/', '_'), search_term) utils.ensure_dir(filename) tweets = [] try: r = TwitterAPI.TwitterPager(api, resource, query) for tweet in r.get_iterator(): tweets.append(tweet) except KeyboardInterrupt: pass with open(filename, 'w') as output_file: yaml.dump(tweets, output_file, default_flow_style=False)
def get_user_tweet_replies(api, screen_name): params = {'q': screen_name} r = TwitterAPI.TwitterPager(api, 'search/tweets', params) items = [] for item in r.get_iterator(): if 'in_reply_to_screen_name' in item and item[ 'in_reply_to_screen_name'] == screen_name: items.append(item) filename = f'reply_tweets/{screen_name}.yaml' utils.ensure_dir(filename) with open(filename, 'w') as output_file: yaml.dump(items, output_file, default_flow_style=False)
def get_results() -> List[Dict[str, Any]]: """ Get the results from the competition and print it out :return: Winner's name and their query """ valid_entries: List[Dict[str, Any]] = [] logging.info('GET RESULTS') json_db: Dict[str, Any] = load_json_db(TWEET_DATABASE) max_key: str = max(json_db.keys()) r = TwitterAPI.TwitterPager(twitter_api, 'statuses/mentions_timeline', { 'count': 200, 'since_id': json_db[max_key]['tweet_id'] }) for item in r.get_iterator(): if 'text' not in item: logging.warning('SUSPEND, RATE LIMIT EXCEEDED: %s\n' % item['message']) break logging.info('[TWEET] ' + item['user']['screen_name'] + ': ' + item['text']) for url in item['entities']['urls']: test_url = url['expanded_url'] if 'scryfall.com' not in test_url: continue logging.info('{} submitted solution: {}'.format( item['user']['screen_name'], test_url)) test_query_results = test_query(item['user']['screen_name'], test_url) if test_query_results: valid_entries.append({ 'name': item['user']['screen_name'], 'length': len(test_query_results), 'query': test_query_results }) return valid_entries
class Twitter: """Twitter API abstraction class""" CONSUMER_KEY = "" CONSUMER_SECRET = "" ACCESS_TOKEN = "" ACCESS_TOKEN_SECRET = "" test = True def __init__(self, consumer_key, consumer_secret, access_token, access_token_secret, test): self.CONSUMER_KEY = consumer_key self.CONSUMER_SECRET = consumer_secret self.ACCESS_TOKEN = access_token self.ACCESS_TOKEN_SECRET = access_token_secret self.test = test def auth(self): self.api = TwitterAPI(self.CONSUMER_KEY, self.CONSUMER_SECRET, self.ACCESS_TOKEN, self.ACCESS_TOKEN_SECRET) def timeline(self, screen_name): try: if self.test: response = TwitterPager(self.api, \ "statuses/user_timeline", \ {"screen_name": screen_name, \ "count": 200, \ "include_rts": True, \ "trim_user": True}) else: response = TwitterPager(self.api, \ 'tweets/search/%s/:%s' % ("30day", "test"), \ {'query': 'from:' + screen_name}) return response.get_iterator() except Exception as e: print e return response.get_iterator(wait=5) def tweet(self, id): return self.api.request('statuses/show/:%d' % id) def close(self): return None
def fetch(api, resource, search_term, query): filename = '{0}/{1}.yaml'.format(resource.replace('/', '_'), search_term) utils.ensure_dir(filename) count = 0 try: with open(filename, 'a') as output_file: r = TwitterAPI.TwitterPager(api, resource, query) for tweet in r.get_iterator(): if not 'retweeted_status' in tweet and not ( "Tôi đã thích video" or "Tôi đã thêm video") in tweet['text']: yaml.dump([tweet['text']], output_file, default_flow_style=False) count += 1 print(count, tweet['text']) except KeyboardInterrupt: pass
def postTweet(apiKey, apiSecret, accessToken, accessTokenSecret, text): api = TwitterAPI(apiKey, apiSecret, accessToken, accessTokenSecret) r = api.request('statuses/update', {'status': text}) print('SUCCESS' if r.status_code == 200 else 'FAILURE')
def streamTweets(apiKey, apiSecret, accessToken, accessTokenSecret, trackTerm, lang='-', nTweets=100, nFlush=100, media='-'): #, #mSize='medium', saveMedia=False, viewMedia=True, #workDir='cache', saveLog=True, logName='log.txt'): api = TwitterAPI(apiKey, apiSecret, accessToken, accessTokenSecret) # Create directories and files, etc. #curTime = time.strftime("%d_%b_%Y_%H.%M.%S") #if not saveMedia: # workDir = "cache" #if viewMedia: # if not os.path.exists(workDir): # os.makedirs(workDir) # os.chdir(workDir) #if saveLog: # f = open(logName, "w") #print("Started fetching will following parameters:") #print("trackTerm: ", trackTerm) #print("lang: ", lang) #print("nTweets: ", nTweets) #print("nFlush: ", nFlush) #print("media: ", media) #print("mSize: ", mSize) #print("viewMedia: ", viewMedia) #print("saveMedia: ", saveMedia) #print("workDir: ", workDir) #print("saveLog: ", saveLog) #print("logName: ", logName) # Create counters current = 0 total = 0 data = [] stream = [] # Create table header for printing tableHeader=["Name", "Handle", "Text", "Time stamp", "Hashtags", "Retweets", "Favorites", "Media", "Language", "Img Path"] keys = ["name", "handle", "content", "time", "hashtags", "rtCount", "favCount", "media", "lang", "imgName"] # Search r = api.request('statuses/filter', {'track': trackTerm}) # For each tweet for tweet in r.get_iterator(): if 'text' in tweet: # if it's really a tweet and not something else #print(tweet) #break # Check if it fits the media requirements (yes, no, don't care) if media != "-": cMedia = True if media == True else False if ('media' not in tweet['entities']) & (cMedia == True): continue elif ('media' in tweet['entities']) & (cMedia == False): continue else: if 'media' in tweet['entities']: cMedia = True else: cMedia = False # Check if it fits the language requirements (anything or specific) if lang != "-": tLang = lang if tweet['metadata'] != tLang: continue else: tLang = tweet['lang'] # If no hashtags if tweet['entities']['hashtags']: hashtags = "" for tag in tweet['entities']['hashtags']: hashtags = hashtags + tag['text'] + ", " else: hashtags = None fileName = None #if cMedia & viewMedia: # cMedia += len(tweet['entities']['media']) # mediaURL = tweet['entities']['media'][0]['media_url'] # fileName = str(total)+mediaURL[-4:] # last 4 are extension # urllib.request.urlretrieve(mediaURL+":"+mSize, fileName) # Push the tweet onto the stream stack stream.append([tweet['user']['name'], tweet['user']['screen_name'], tweet['text'].replace('\n', ' '), tweet['created_at'], hashtags, tweet['retweet_count'], tweet['favorite_count'], cMedia, tLang, fileName]) # increment the counters current += 1 total += 1 # every 100 tweets, flush the stream to improve performance and add to a big stream if current == nFlush: dictList = [] for i in range(current): dictList.append(dict(zip(keys, stream[i]))) print("Updating database. Total = ", total) dbTest.updateDB(dictList) stream = [] # empty stack current = 0 # reset counter # max number of tweets if total >= nTweets: dictList = [] for i in range(current): dictList.append(dict(zip(keys, stream[i]))) print("Updating database. Total = ", total) dbTest.updateDB(dictList) break # this should not trigger, but just in case # this handles an exception triggered when we send more than 1 request every 5 seconds # this would result in a 15 minute block elif 'message' in item and item['code'] == 88: print('SUSPEND, RATE LIMIT EXCEEDED: %s' % item['message']) break #print(count) # print table #if saveLog: # table = tabulate(data, headers=tableHeader, tablefmt='fancy_grid') # f.write(table) #for i in range(total): # print(data[i]) #os.chdir("..") print("Done Streaming!")
from TwitterAPI import * import requests import os consumer_key = os.environ['TWITTER_CONSUMER_KEY'] consumer_secret = os.environ['TWITTER_SECRET'] access_token_key = os.environ['TWITTER_ACCESS_TOKEN'] access_token_secret = os.environ['TWITTER_ACCESS_TOKEN_SECRET'] api = TwitterAPI(consumer_key, consumer_secret, access_token_key, access_token_secret) gcm_api_key = os.environ['GCM_API_KEY'] gcm_url = 'http://exp-warenix.rhcloud.com/gcm/sendMessage1to1' while True: try: #r = api.request('statuses/filter', {'track': 'hong kong', 'location': '22.25,114.1667'}) r = api.request('user') iterator = r.get_iterator() for item in iterator: if 'text' in item: text = item['text'].encode('utf-8', 'ignore') name = item['user']['name'].encode('utf-8', 'ignore') screen_name = item['user']['screen_name'].encode('utf-8', 'ignore') id_str = item['id_str'] profile_image_url = item['user']['profile_image_url'] print "%s(@%s)\t\t: %s" % (name, screen_name, text) print data = { 'name': name,
access_token_secret) page = 1 nb_results = 10 # r = api2.request('users/search', {'q':query, 'count':nb_results }).json() r = api2.request('users/search', {'q':query, 'page':page, 'count':nb_results }).json() user_list = [] for item in r: user_list.append(item['screen_name']) print user_list return user_list #authentification api = TwitterAPI('BQBZTbY3ugTypaRBq7Is0m6Dh', 'JGeRqs3r42Id4W2Q47NlGwAlNYv0myrBhlUPJeeizQXi56RWBm', auth_type='oAuth2') print("authentification success\n") name_query = "trump" #THIS WILL BE NOT HARDCODED AND WILL BE TAKEN FROM THE FRONT END count = 10 # requests to twitter, one for one's tweet the other for one's personal info r = api.request('statuses/user_timeline', {'screen_name': name_query, 'count':count, 'exclude_replies':'true', 'include_rts':'false'}) personalInfoResponse = api.request('users/show', {'screen_name' : name_query, 'include_entities' : 'false'}).json() if 'errors' in personalInfoResponse: error = personalInfoResponse['errors'] print error[0]['message'] if error[0]['code'] == 50: #user not found search_users(name_query)
def display(request): context = {} bio_output = "" tweets_output = "" name_query = 'kanyewest' if request.method == "POST": name_query = request.POST['query_text'] name_query = name_query.replace('@', '') # Authentification api = TwitterAPI('BQBZTbY3ugTypaRBq7Is0m6Dh', 'JGeRqs3r42Id4W2Q47NlGwAlNYv0myrBhlUPJeeizQXi56RWBm', auth_type='oAuth2') # requests to twitter, one for one's tweet the other for one's personal info count = 200 tweets = api.request('statuses/user_timeline', {'screen_name': name_query, 'count':count, 'exclude_replies':'true', 'include_rts':'false'}) otherTweets = api.request('search/tweets', {'q': '@'+name_query, 'count':count, 'lang': 'en'}); personalInfoResponse = api.request('users/show', {'screen_name' : name_query, 'include_entities' : 'false'}).json() # Error testing, if user not found, we search for one if 'errors' in personalInfoResponse: error = personalInfoResponse['errors'] name_query = 'kanyewest' tweets = api.request('statuses/user_timeline', {'screen_name': name_query, 'count':count, 'exclude_replies':'true', 'include_rts':'false'}) personalInfoResponse = api.request('users/show', {'screen_name' : name_query, 'include_entities' : 'false'}).json() bio_output = json.dumps(build_bio_dict(personalInfoResponse)) tweets_output = json.dumps(build_tweets_dict(tweets)) other_tweets_output = json.dumps(build_other_tweets(otherTweets)) top_hashtags = json.dumps(build_hashtag_freq(tweets)) context = { 'query': name_query } context = RequestContext(request, context) context['bio_data'] = bio_output context['tweet_data'] = tweets_output context['top_hashtags'] = top_hashtags context['other_tweets'] = other_tweets_output return render_to_response('feels/display.html', context) else: bio_output = json.dumps(build_bio_dict(personalInfoResponse)) tweets_output = json.dumps(build_tweets_dict(tweets)) top_hashtags = json.dumps(build_hashtag_freq(tweets)) other_tweets_output = json.dumps(build_other_tweets(otherTweets)) context = { 'query': name_query } context = RequestContext(request, context) context['bio_data'] = bio_output context['tweet_data'] = tweets_output context['top_hashtags'] = top_hashtags context['other_tweets'] = other_tweets_output return render_to_response('feels/display.html', context)
languages.append(PORTUGUESE) languages.append(SPANISH) return languages # Language parameters for the twitter search query FRENCH = 'lang%3Afr' ENGLISH = 'lang%3Aen' PORTUGUESE = 'lang%3Apt' SPANISH = 'lang%3Aes' CATALAN = 'lang%3Acat' consumer_key='rW3r87OsW7V5WcL8uNc9m5kmU', consumer_secret='zsjxaiYf23J5IsJjhZxodrVDVJkBUgWAVfEyBR2bfBwz5GZngV', access_token_key='169260565-tlC8ZpYAD4eyyqPaPpzwYD3UPrxgBV9VQ0zxuu3D', access_token_secret='mKOYFGuklZfo4X5m67pW3LZrB8rnXJlhsKCPasidzSwKt' languages = createLanguagesList() # This method creates a database in Parse.com and performs a search in twitter for 4 languages EN,ES,PT,FR TwitterAPI.performTwitterSearch(consumer_key,consumer_secret,access_token_key,access_token_secret,languages) # After creating the online database, we can retrieve the db items and create local datasets # These methods create the local datasets files localTweets.writeFile("datasets/eng_tweets_x.txt",'en') localTweets.writeFile("datasets/es_tweets_x.txt",'es') localTweets.writeFile("datasets/pt_tweets_x.txt",'pt') localTweets.writeFile("datasets/fr_tweets_x.txt",'fr')