def getTweets(): """ Get list of tweets, with tweet ID and content, from configured Twitter account URL. This function relies on BeautifulSoup to extract the tweet IDs and content of all tweets on the specified page. The data is returned as a list of dictionaries that can be used by other functions. """ all_tweets = [] url = helpers._config('tweets.source_account_url') if not url: helpers._error('getTweets() => The source Twitter account URL (' + url + ') was incorrect. Could not retrieve tweets.') return False headers = {} headers['accept-language'] = 'en-US,en;q=0.9' headers['dnt'] = '1' headers['user-agent'] = helpers._config('gen.APP_NAME') data = requests.get(url) html = BeautifulSoup(data.text, 'html.parser') timeline = html.select('#timeline li.stream-item') if timeline is None: helpers._error( 'getTweets() => Could not retrieve tweets from the page. Please make sure the source Twitter account URL (' + url + ') is correct.') return False helpers._info('getTweets() => Fetched tweets for ' + url + '.') for tweet in timeline: tweet_id = tweet['data-item-id'] try: tweet_text = tweet.select('p.tweet-text')[0].get_text() except: helpers._info('getTweets() => No tweet text found. Moving on...') continue all_tweets.append({"id": tweet_id, "text": tweet_text}) return all_tweets if len(all_tweets) > 0 else None
def getTweets(): """ Get list of tweets, with tweet ID and content, from configured Twitter account URL. This function relies on BeautifulSoup to extract the tweet IDs and content of all tweets on the specified page. The data is returned as a list of dictionaries that can be used by other functions. """ all_tweets = [] url = helpers._config("tweets.source_account_url") if not url: helpers._error( f"getTweets() => The source Twitter account URL ({url}) was incorrect. Could not retrieve tweets." ) return False headers = {} headers["accept-language"] = "en-US,en;q=0.9" headers["dnt"] = "1" headers["user-agent"] = helpers._config("gen.APP_NAME") data = requests.get(url) html = BeautifulSoup(data.text, "html.parser") timeline = html.select("#timeline li.stream-item") if timeline is None: helpers._error( f"getTweets() => Could not retrieve tweets from the page. Please make sure the source Twitter account URL ({url}) is correct." ) return False helpers._info(f"getTweets() => Fetched tweets for {url}.") for tweet in timeline: tweet_id = tweet["data-item-id"] try: tweet_text = tweet.select("p.tweet-text")[0].get_text() except: helpers._info("getTweets() => No tweet text found. Moving on...") continue all_tweets.append({"id": tweet_id, "text": tweet_text}) return all_tweets if len(all_tweets) > 0 else None
def runME(twitter_nametopull, mastodon_secret, mastodon_host): """ This function will get a new Tweet from the configured Twitter account and publish to the configured Mastodon instance. It will only toot once per invokation to avoid flooding the instance. """ try: twitter_nametopull = "https://www.twitter.com/" + twitter_nametopull except: print( "Syntax: twitter_nametopull mastodon_secret mastodon_host No action taken. Exiting." ) sys.exit() print(twitter_nametopull) print(mastodon_secret) print(mastodon_host) if (twitter_nametopull) and (mastodon_secret) and (mastodon_host): tweets = social.getTweets(twitter_nametopull, mastodon_secret, mastodon_host) if not tweets: helpers._error('__main__ => No tweets fetched.') sys.exit() helpers._info('__main__ => ' + str(len(tweets)) + ' tweets fetched.') for tweet in tweets: toot_status = "No Attempt Yet" if social.tootTheTweet(tweet, mastodon_secret, mastodon_host): helpers._info('__main__ => Tooted "' + tweet['text'] + '"') helpers._info( '__main__ => Tooting less is tooting more. Sleeping...') toot_status = "Success" # 070519 sys.exit() else: toot_status = "" # helpers._info('__main__ => /============================' + tweet['id'] + '\n\n') else: print( "Syntax: twitter_nametopull mastodon_secret mastodon_host No action taken. Exiting." ) toot_status = "No Attempt Made. Likely an error in you not sending in all three variables. Exiting." return (toot_status)
import helpers import social import sys if __name__ == '__main__': """ It all starts here... This function will get a new Tweet from the configured Twitter account and publish to the configured Mastodon instance. It will only toot once per invokation to avoid flooding the instance. """ tweets = social.getTweets() if not tweets: helpers._error('__main__ => No tweets fetched.') sys.exit() helpers._info('__main__ => ' + str(len(tweets)) + ' tweets fetched.') for tweet in tweets: if social.tootTheTweet(tweet): helpers._info('__main__ => Tooted "' + tweet['text'] + '"') helpers._info( '__main__ => Tooting less is tooting more. Sleeping...') sys.exit()
def tootTheTweet(tweet): """ Receieve a dictionary containing Tweet ID and text... and TOOT! This function relies on the requests library to post the content to your Mastodon account (human or bot). A boolean success status is returned. Arguments: tweet {dictionary} -- Dictionary containing the "id" and "text" of a single tweet. """ host_instance = helpers._config('toots.host_instance') token = helpers._config('toots.app_secure_token') tweet_id = tweet['id'] if not host_instance: helpers._error('tootTheTweet() => Your host Mastodon instance URL (' + host_instance + ') was incorrect.') return False if not token: helpers._error( 'tootTheTweet() => Your Mastodon access token was incorrect.') return False headers = {} headers['Authorization'] = 'Bearer ' + token headers['Idempotency-Key'] = tweet_id data = {} data['status'] = tweet['text'] data['visibility'] = 'public' tweet_check_file_path = helpers._config('toots.cache_path') + tweet['id'] tweet_check_file = Path(tweet_check_file_path) if tweet_check_file.is_file(): helpers._info('tootTheTweet() => Tweet ' + tweet_id + ' was already posted. Reposting...') return False else: tweet['text'].encode('utf-8') tweet_check = open(tweet_check_file_path, mode='w') tweet_check.write(tweet['text']) tweet_check.close() helpers._info('tootTheTweet() => New tweet ' + tweet_id + ' => "' + tweet['text'] + '".') response = requests.post(url=host_instance + '/api/v1/statuses', data=data, headers=headers) if response.status_code == 200: helpers._info('tootTheTweet() => OK. Posted tweet ' + tweet_id + 'to Mastodon.') helpers._info('tootTheTweet() => Response: ' + response.text) return True else: helpers._info('tootTheTweet() => FAIL. Could not post tweet ' + tweet_id + 'to Mastodon.') helpers._info('tootTheTweet() => Response: ' + response.text) return False
import encodings.idna import helpers import social import sys if __name__ == "__main__": """ It all starts here... This function will get a new Tweet from the configured Twitter account and publish to the configured Mastodon instance. It will only toot once per invokation to avoid flooding the instance. """ tweets = social.getTweets() if not tweets: helpers._error("__main__ => No tweets fetched.") sys.exit() helpers._info(f"__main__ => {len(tweets)} tweets fetched.") for tweet in tweets: if social.tootTheTweet(tweet): helpers._info(f'__main__ => Tooted "{tweet["text"]}"') helpers._info( "__main__ => Tooting less is tooting more. Sleeping...") sys.exit()
def tootTheTweet(tweet): """ Receieve a dictionary containing Tweet ID and text... and TOOT! This function relies on the requests library to post the content to your Mastodon account (human or bot). A boolean success status is returned. Arguments: tweet {dictionary} -- Dictionary containing the "id" and "text" of a single tweet. """ host_instance = helpers._config("toots.host_instance") token = helpers._config("toots.app_secure_token") tweet_id = tweet["id"] if not host_instance: helpers._error( f"tootTheTweet() => Your host Mastodon instance URL ({host_instance}) was incorrect." ) return False if not token: helpers._error( "tootTheTweet() => Your Mastodon access token was incorrect.") return False headers = {} headers["Authorization"] = f"Bearer {token}" headers["Idempotency-Key"] = tweet_id data = {} data["status"] = tweet["text"] data["visibility"] = "public" tweet_check_file_path = helpers._config("toots.cache_path") + tweet["id"] tweet_check_file = Path(tweet_check_file_path) if tweet_check_file.is_file(): helpers._info( f"tootTheTweet() => Tweet {tweet_id} was already posted. Reposting..." ) return False else: tweet["text"].encode("utf-8") tweet_check = open(tweet_check_file_path, mode="w") tweet_check.write(tweet["text"]) tweet_check.close() helpers._info( f'tootTheTweet() => New tweet {tweet_id} => "{tweet["text"]}".') response = requests.post(url=f"{host_instance}/api/v1/statuses", data=data, headers=headers) if response.status_code == 200: helpers._info( f"tootTheTweet() => OK. Posted tweet {tweet_id} to Mastodon.") helpers._info(f"tootTheTweet() => Response: {response.text}") return True else: helpers._info( f"tootTheTweet() => FAIL. Could not post tweet {tweet_id} to Mastodon." ) helpers._info(f"tootTheTweet() => Response: {response.text}") return False
import encodings.idna import helpers import social import sys if __name__ == "__main__": """ It all starts here... This function will get a new Tweet from the configured Twitter account and publish to the configured Mastodon instance. It will only toot once per invokation to avoid flooding the instance. """ tweets = social.getTweets() if not tweets: helpers._error("__main__ => No tweets fetched.") sys.exit() helpers._info("__main__ => {} tweets fetched.".format(len(tweets))) for tweet in tweets: if social.tootTheTweet(tweet): helpers._info('__main__ => Tooted "{}"'.format(tweet["text"])) helpers._info( "__main__ => Tooting less is tooting more. Sleeping...") sys.exit()
def getTweets(twitter_nametopull, mastodon_secret, mastodon_host): """ Get list of tweets, with tweet ID and content, from configured Twitter account URL. This function relies on BeautifulSoup to extract the tweet IDs and content of all tweets on the specified page. The data is returned as a list of dictionaries that can be used by other functions. """ all_tweets = [] tweet_count_max = 1 # set me yes # old way from config.json file url = helpers._config('tweets.source_account_url') url = twitter_nametopull if not url: helpers._error('getTweets() => The source Twitter account URL (' + url + ') was incorrect. Could not retrieve tweets.') return False headers = {} headers['accept-language'] = 'en-US,en;q=0.9' headers['dnt'] = '1' headers['user-agent'] = helpers._config('gen.APP_NAME') # Getting users timeline pulling... data = requests.get(url) html = BeautifulSoup(data.text, 'html.parser') timeline = html.select('#timeline li.stream-item') if timeline is None: helpers._error( 'getTweets() => Could not retrieve tweets from the page. Please make sure the source Twitter account URL (' + url + ') is correct.') return False helpers._info('getTweets() => Fetched tweets for ' + url + '.') tweet_count_loop = 0 tweet_error = 0 for tweet in timeline: # print(tweet) tweet_skip = 0 if (tweet_error == 0) and ( tweet_count_loop <= (tweet_count_max) ): #NOTE: tweet_count_max would be tweet_count_max MINUS 1 if you wanted to do it normally but we want the top 2 tweets in case the top one was PINNED as is a twitter feature and we would want to skip it tweet_count_loop = tweet_count_loop + 1 # print(tweet) tweet_id = tweet['data-item-id'] # suposed to let you dup post as Mastodon will reject if header same twice.... headers['Idempotency-Key'] = tweet_id tweet_text = [] retweet_text = [] tweet_url = [] tweet_datetimestamp = [] tweet_url = url + '/status/' + tweet_id try: tweet_ispinned = retweet_text = tweet.select( 'span.js-pinned-text')[0].get_text() helpers._info( 'getTweets() => This tweet is a pinned tweet. Skipping') except: tweet_ispinned = [] if (tweet_ispinned): tweet_skip = 1 try: tweet_text = tweet.select('p.tweet-text')[0].get_text() # tweet_datetimestamp = tweet.select('a.tweet-timestamp')[0].get_text() tweet_datetimestamp = tweet.select('a.tweet-timestamp')[0] tweet_datetimestamp = tweet_datetimestamp['title'] # print(tweet) try: # Only using to identify in the below ifthen since the tweet text seems to be the same. Not best way but too many scraping variables to keep track of. retweet_text = tweet.select( 'span.js-retweet-text')[0].get_text() retweet_text_itself = tweet.select( 'span.js-retweet-text')[0].get_text() # we are here. not pullnig I think .... retweet_originaltweeter=tweet.select('div.data-screen-name')[0].get_text() except: retweet_text = [] retweet_text_itself = [] if retweet_text: tweet_text = retweet_text_itself.strip( ) + ':\n ' + tweet_text.strip() helpers._info( 'getTweets() => Is Retweet!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' ) #helpers._info('getTweets() =>' + retweet_text) except: helpers._info( 'getTweets() => No tweet text found. Moving on...') continue # cleanup if good tweet if tweet_text: tweet_text = tweet_text.replace( '<a href', ' \n<a href' ) # should be regex this is sloppy in case they already had a space. Doesnt seem to be using this one either. Already stripped it appears tweet_text = tweet_text.replace( 'pic.twitter.com', ' \nhttps://pic.twitter.com' ) # should be regex this is sloppy tweet_text = tweet_text.replace( 'http', ' \nhttp' ) # should be regex this is sloppy in case they already had a space. # change up a specific tweet if you want tweet_text=tweet_text.replace('reassuring.','reassuring. (BOT REPOST)') helpers._info('getTweets() => =============================' + tweet['data-item-id'] + '\n\n') tweet_text = tweet_text + ' \n\nSource: ' + tweet_url + ' ' + tweet_datetimestamp tweet_text = tweet_text + '\n\nEND==================\n\nMy other bots: https://pastebin.com/yuwXfDjZ' helpers._info('getTweets() => TWEET TEXT--> ' + tweet_text) if (tweet_error == 0) and (tweet_skip == 0): all_tweets.append({"id": tweet_id, "text": tweet_text}) else: helpers._info( 'getTweets() => Not adding tweet: Either exists and error or exists and Skipping on purpose' ) return all_tweets if len(all_tweets) > 0 else None