def comment_on_thread(submission, twitter_url_list, imgur_url_list): # Assembles the comment thread_comment = "**Max Resolution Twitter Link(s)**\n\n" for twitter_url in twitter_url_list: thread_comment += ("%s\n\n" % twitter_url) thread_comment += "**Imgur Mirror Link(s)**\n\n" for imgur_url in imgur_url_list: thread_comment += ("%s\n\n" % imgur_url) thread_comment += "---\n\n^^I'm ^^a ^^bot ^^made ^^by ^^[u\/jclishman](https://reddit.com/user/jclishman)!" thread_comment += " [^^[FAQ/Discussion]](http://reddit.com/user/SpaceXMirrorBot/comments/ad36dr/) [^^[Code]](https://github.com/jclishman/SpaceXMirrorBot)" # Posts the comment retries = 0 while retries < 5: try: thread_comment_id = submission.reply(thread_comment) logger.info("Comment made with ID: %s" % thread_comment_id) # Break from the while loop after successful post submission break except praw.exceptions.APIException as e: retries += 1 logger.error("Hit ratelimit, will try again in 5 minutes. (Attempt %d/5)" % retries) time.sleep(300)
def get_twitter_fullres(tweet_url): # RegEx's the status URL for the numeric status ID tweet_id = status_regex.search(tweet_url).group(1) # Get the status JSON from Twitter tweet_data = twitter_api.get_status(tweet_id, include_entities=True, tweet_mode='extended') tweet = tweet_data._json twitter_url_list = [] try: # Check if the tweet has media attached for tweet_media in tweet["extended_entities"]["media"]: if tweet_media["type"] == "photo": # Appends ":orig" to URL for max resolution twitter_url_list.append(tweet_media["media_url_https"]+":orig") else: logger.info("Media attached is not a picture") return -1 return twitter_url_list except: logger.info("Tweet has no media attached") return -1
def post(): send_message_to_channels(msg) logger.info(msg) # Sends how long it took from tweet creation to irc message (debug) time_to_post = round(time.time() - row[6], 5) logger.info(f"Post #{row[0]}, Took {time_to_post}s\n")
def unfollow_account(user_id): try: database = sqlite3.connect('database.db') unfollow_cursor = database.cursor() unfollow_cursor.execute("DELETE FROM following WHERE twitter_id = %s" % user_id) database.commit() logger.info('Deleted from DB') except sqlite3.OperationalError as e: logger.error(str(e))
def set_flags(user_id, retweets, replies): try: database = sqlite3.connect('database.db') set_flags_cursor = database.cursor() set_flags_cursor.execute( "UPDATE following SET retweets = %i, replies = %i WHERE twitter_id = %s" % (retweets, replies, user_id)) logger.info('Updated DB flags') database.commit() except sqlite3.OperationalError as e: logger.error(str(e))
def upload_to_imgur(url_list): imgur_url_list = [] for url in url_list: # Uploads image to Imgur and adds the link to a list image = imgur_api.upload_from_url(url, anon=True) logger.info("Uploaded to Imgur with ID: %s" % image["id"]) imgur_url_list.append(image["link"]) time.sleep(1) return imgur_url_list
def follow_account(username, user_id, retweets, replies): try: database = sqlite3.connect('database.db') follow_cursor = database.cursor() follow_cursor.execute( "INSERT INTO following (username, twitter_id, retweets, replies) VALUES(?,?,?,?)", [username, user_id, retweets, replies]) logger.info('Added to DB') database.commit() except sqlite3.OperationalError as e: logger.error(str(e))
def run(): logger.info("Starting Twitter Stream...") # Makes the stream object myStreamListener = MyStreamListener() myStream = tweepy.Stream(auth, myStreamListener) # Streams tweets try: myStream.filter(follow=following, stall_warnings=True) except Exception as e: logger.error("-----CAUGHT ERROR-----") logger.error(str(e)) logger.error("-----ATTEMPTING RECONNECT-----") run()
def run(): web_api = Client(auto_patch=True, drop_incompat_keys=False) feed = [] startup = True user_dict = { "SpaceX": "20311520", "jclishman.testing": "7400533474" } while True: for id_str in list(user_dict.values()): try: feed.append(web_api.user_feed(id_str, count=1)) time.sleep(5) except Exception as e: #logger.error(str(e)) #logger.error("Error getting feed. Sleeping for 30s") time.sleep(30) for post in feed: post = post[0]["node"] user_id_str = post["owner"]["id"] shortcode = post["shortcode"] timestamp = post["created_time"] # Empty string if there isn't a caption try: caption = post["caption"]["text"] except: caption = '' # Match ID number to screenname for screen_name, id_str in user_dict.items(): if user_id_str == id_str: user_screen_name = screen_name stored_timestamp = db.get_instagram_timestamp(user_screen_name) if int(timestamp) > stored_timestamp: start_time = time.time() db.update_instagram_timestamp(user_screen_name, int(timestamp)) logger.info(f"New Instagram post by @{user_screen_name}, id {user_id_str}") logger.info(f"Post shortcode: {shortcode}") logger.info(f"Post caption: {caption}") logger.info(f"Post timestamp: {timestamp}") url = f"https://instagram.com/p/{shortcode}" if not startup: db.insert_message('Instagram', user_screen_name, caption.replace("\n", " "), url, start_time) time.sleep(10) startup = False
def parse(message_contents): # Default values retweets = 0 replies = 0 # Makes everything lowercase, removes the bot username, and take out all spaces message_clean = message_contents.replace(NICK + ': ', '').lower() # Splits the command into parts command = message_clean.split(' ') action = command[0].strip() # Error handling try: target = command[1].replace('@', '').strip() except: target = None try: retweets = int(command[2].strip()) except: retweets = None try: replies = int(command[3].strip()) except: replies = None if action == 'help': if target == 'follow': return 'Follows Twitter accounts, see MaxQ: help for syntax' elif target == 'unfollow': return 'Unfollows Twitter accounts, see MaxQ: help for syntax' elif target == 'set': return 'Changes flag(s) for accounts I\'m already following, see MaxQ: help for syntax' else: return 'Syntax: MaxQ: follow|unfollow|set @username 0|1 <retweets> 0|1 <replies>' elif action == 'unfollow': user_id = twitter.getID(target) if in_database(user_id): db.unfollow_account(user_id) return 'Unfollowed @%s' % target elif not in_database(user_id): return 'Error: @%s is not in the database' % target if type(retweets) is not int or type(replies) is not int: logger.error('Command error: not an int') return 'Error: Invalid command' if retweets > 1 or replies > 1: logger.error('Command Error: not 0 or 1') return 'Error: Invalid command' # print('Command: ' + str(command)) logger.info( f"Action: {action} | Target: {target} | Retweets: {retweets} | Replies: {replies}" ) if target is None: return 'Error: No target account' user_id = twitter.getID(target) # Is the target a user? Does the user actually exist? if user_id is None: return f"Error: @{target} does not exist" # Are flags specified? if retweets is None or replies is None: return 'Error: Missing valid flag(s)' if action == 'follow': # Is the user not already in the database? if not in_database(user_id): db.follow_account(target, user_id, retweets, replies) return f"@{target} is now being followed on Twitter | Retweets {retweets} | Replies {replies}" return f"@{target} is already being followed on Twitter" if action == 'set': if in_database(user_id): db.set_flags(user_id, retweets, replies) return f"@{target} now has retweets set to {retweets} and replies set to {replies}" return f"Error: @{target} is not in the database"
def insta_thread(): logger.info('Starting Instragram Thread') instagramservice.run()
def twitter_thread(): logger.info('Starting Twitter Thread') twitterservice.run()
def restart_irc(): logger.info('Restarting...') os.system('start cmd /c irc.py') time.sleep(5) irc.send(parse('QUIT :Be right back!')) exit()
def reddit_thread(): logger.info('Starting Reddit Thread') redditservice.run()
# Reddit AMA mode from bot_logging import logger import twitterservice, instagramservice, redditservice import commands, db import socket, ssl import threading import time import json import os logger.info('Now Entering MaxQ...') # Secret credentials :) config = json.load(open('_config.json')) password = config['nickserv_password'] # IRC Config # HOST = 'irc.esper.net' HOST = 'irc.snoonet.org' PORT = 6697 NICK = 'MaxQ' admins = config["admin_hostnames"] channels = ['#groupofthrones'] # Responds to server pings def pong(text): logger.debug('PONG' + text.split()[1]) irc.send(parse('PONG ' + text.split()[1]))
def restart_irc(): logger.info("Restarting...") os.system("nohup python3 irc.py &") time.sleep(5) irc.send(parse("QUIT :Be right back!")) exit()
post_time = submission.created_utc # Is it new? if post_time > last_post_time[subreddit]: last_post_time[subreddit] = post_time # Ignore if it's not a link post if submission.is_self: #logger.info("Not a link post") pass # Check if it's a link to Twitter elif "twitter.com" in submission.url: logger.info("="*30) logger.info("Found a tweet post (%s)" % submission.shortlink,) #logger.info(submission.url) twitter_url_list = get_twitter_fullres(submission.url) # get_twitter_fullres() returns -1 if the tweet has no/incorrect media type if twitter_url_list != -1: imgur_url_list = upload_to_imgur(twitter_url_list) logger.info("Max Res Twitter URLs: ") logger.info(twitter_url_list) logger.info("Mirror Imgur URLs: ") logger.info(imgur_url_list)
def run(): # Bot won't post anything new on startup startup = True class InstagramUser: user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36' rhx_gis = None csrf_token_cookie = None session = requests.Session() def __init__(self): res = self.session.get('https://instagram.com', headers={'User-Agent': self.user_agent}) regex_match = re.search(r'"rhx_gis":"(?P<rhx_gis>[a-f0-9]{32})"', res.text) if regex_match: self.rhx_gis = regex_match.group('rhx_gis') def build_signature(self, query_variables): m = hashlib.md5() m.update('{rhx_gis}:{variables}'.format( rhx_gis=self.rhx_gis, variables=query_variables).encode('utf-8')) return m.hexdigest() def request(self, **kwargs): endpoint_url = None signature_var = None query_params = kwargs.pop('query_params', {}) if kwargs['endpoint'] is "username": endpoint_url = 'https://instagram.com/' + kwargs[ 'username'] + '/' signature_var = urllib.parse.urlparse(endpoint_url).path query_params = {'__a': '1'} elif kwargs['endpoint'] is "graphql": endpoint_url = 'https://instagram.com/graphql/query' query_params['query_hash'] = '42323d64886122307be10013ad2dcc44' signature_var = query_params['variables'] = json.dumps( kwargs['query_variable']) return self.session.get( endpoint_url + "?{}".format(urllib.parse.urlencode(query_params)), headers={ 'User-Agent': self.user_agent, 'X-Instagram-GIS': self.build_signature(signature_var) }) def get_user_id(self, username): response = self.request(username=username, endpoint='username') if response.status_code is not 200: logger.error("Instagram responded with status code %s" % str(response.status_code)) raise Exception("Instagram responded with status_code %s" % str(response.status_code)) json_response = response.json() return json_response['graphql']['user']['id'] def get_recent_post(self, user_id, **kwargs): variables = {"id": user_id, "first": 1} response = self.request(endpoint='graphql', query_variable=variables) if response.status_code is not 200: logger.error("Instagram responded with status code %s" % str(response.status_code)) raise Exception("Instagram responded with status_code %s" % str(response.status_code)) post = response.json( )['data']['user']['edge_owner_to_timeline_media']['edges'][0] post_data = { 'shortcode': post['node']['shortcode'], 'created_at_timestamp': post['node']['taken_at_timestamp'], 'caption': None, } if len(post['node']['edge_media_to_caption']['edges']) != 0: post_data["caption"] = post['node']['edge_media_to_caption'][ 'edges'][0]["node"]["text"] return post_data users_list = db.get_following('instagram') username_list = {x[0] for x in users_list} username = '' while True: for username in username_list: # Gets the ID of the post again, and compares it against the stored value user = InstagramUser() user_id = user.get_user_id(username) post_data = user.get_recent_post(user_id) stored_timestamp = db.get_instagram_timestamp(username) # Is it a new post? if post_data["created_at_timestamp"] > stored_timestamp: start_time = time.time() logger.info('Found new Instagram post') logger.info('Shortcode: ' + post_data["shortcode"]) logger.info('Caption: ' + post_data["caption"]) logger.info('Timestamp: ' + str(post_data["created_at_timestamp"])) db.update_instagram_timestamp( username, post_data["created_at_timestamp"]) logger.info('Updated database\n') if not startup: db.insert_message( 'Instagram', username, post_data["caption"], 'https://instagram.com/p/%s' % post_data["shortcode"], start_time) else: logger.debug('Did not find new post\n') time.sleep(3) startup = False
def on_status(self, status): #print(status.text) #print(json.dumps(status._json)) # Converts the data into usable JSON data = status._json # Puts user attributes into this list if the tweet is from somebody the bot is following # If the tweet isn't from someone the bot is following, set to None # For some reason the twitter API also tells you when someone deletes a tweet # If you try to get the needed properties from a deleted tweet, it throws a KeyError try: user_of_tweet = next( (x for x in users_list if x[1] == data['user']['id_str']), None) except KeyError as e: return if user_of_tweet is None: # Not following this user, do nothing. return start_time = time.time() # Is it a retweet? Is the retweet flag of the user set to 1? if "retweeted_status" in data and user_of_tweet[2] == 1: rt_data = data['retweeted_status'] # Has the retweeted status already been posted? # No, post it if not has_tweet_been_posted(rt_data['user']['screen_name'], rt_data['id_str']): send_tweet_to_db(data, start_time) # Yes, don't post it else: logger.info("Retweet has already been posted") # Is a reply? Is the reply flag of the user set to 1? elif data['in_reply_to_status_id'] is not None and user_of_tweet[ 3] == 1: reply_data = get_status(data['in_reply_to_status_id']) # Has the parent tweet to the reply been posted already? # No, send it as context if not has_tweet_been_posted(reply_data['user']['screen_name'], reply_data['id_str']): # Avoid IRC double pings usernames_to_modify = ['@elonmusk', '@SpaceX'] for username in usernames_to_modify: ZWJ = "" # Inserts a Zero Width Joiner username_zwj = username[:2] + ZWJ + username[2:] reply_data['full_text'] = reply_data['full_text'].replace( username, username_zwj) reply_data['text'] = reply_data['full_text'] # Don't send the ID of the reply tweet reply_data['id_str'] = None send_tweet_to_db(reply_data, start_time) time.sleep(0.5) send_tweet_to_db(data, start_time) # Yes, don't send it again else: send_tweet_to_db(data, start_time) logger.info("Parent tweet already posted") # If it's a normal tweet elif "retweeted_status" not in data and data[ "in_reply_to_status_id"] is None: send_tweet_to_db(data, start_time)
# Reddit AMA mode from bot_logging import logger import twitterservice, redditservice, launchservice, acronymservice import commands, db import socket, ssl import threading import time import math import json import os logger.info("Now Entering MaxQ...") # Secret credentials :) config = json.load(open("_config.json")) password = config["nickserv_password"] # IRC Config HOST = "irc.esper.net" #HOST = "irc.snoonet.org" PORT = 6697 NICK = "MaxQ" admins = config["admin_hostnames"] #channels = ["#lishbot"] channels = ["#SpaceX"] launch_whitelist = config["whitelist"] launchmode = False # Responds to server pings def pong(text):
def on_status(self, status): #print(status.text) #print(json.dumps(status._json)) # Converts the data into usable JSON data = status._json # Puts user attributes into this list if the tweeet is from somebody the bot is following # If the tweet isn't from someone the bot is following, set to None # For some reason the twitter API also tells you when someone deletes a tweet # If you try to get the needed properties from a deleted tweet, it throws a KeyError try: user_of_tweet = next((x for x in users_list if x[1] == data['user']['id_str']), None) except KeyError as e: user_of_tweet = None # Sends the tweet to the database def send_tweet_to_db(tweet_data, start_time): # Gets the full tweet text if it's concatenated by Twitter if "extended_tweet" in tweet_data: text = html.unescape(tweet_data['extended_tweet']['full_text']) # Gets the full retweet text if it's concatenated by Twitter elif "retweeted_status" in tweet_data and "extended_tweet" in tweet_data['retweeted_status']: rt_data = tweet_data['retweeted_status'] text = f"RT @{rt_data['user']['screen_name']}: {html.unescape(rt_data['extended_tweet']['full_text'])}" # Not an extended tweet else: text = html.unescape(tweet_data['text']) # Logs raw JSON #logger.info(json.dumps(data)) # Checks if the tweet is a parent to a reply, which won't have an ID if tweet_data['id_str'] is not None: tweet_url = make_url_from_tweet(tweet_data['user']['screen_name'], tweet_data['id_str']) else: tweet_url = '' db.insert_message('Twitter', tweet_data['user']['screen_name'], text.replace("\n", " "), tweet_url, start_time) # Is the tweet from somebody the bot cares about? if user_of_tweet is not None: start_time = time.time() # Is it a retweet? Is the retweet flag of the user set to 1? if "retweeted_status" in data and user_of_tweet[2] == 1: rt_data = data['retweeted_status'] # Has the retweeted status already been posted? # No, post it if not has_tweet_been_posted(rt_data['user']['screen_name'], rt_data['id_str']): send_tweet_to_db(data, start_time) # Yes, don't post it else: logger.info("Retweet has already been posted") # Is a reply? Is the reply flag of the user set to 1? elif data['in_reply_to_status_id'] is not None and user_of_tweet[3] == 1: reply_data = get_status(data['in_reply_to_status_id']) # Has the parent tweet to the reply been posted already? # No, send it as context if not has_tweet_been_posted(reply_data['user']['screen_name'], reply_data['id_str']): # Avoid IRC double pings usernames_to_modify = ['@elonmusk', '@SpaceX'] for username in usernames_to_modify: ZWJ = "" # Inserts a Zero Width Joiner username_zwj = username[:2] + ZWJ + username[2:] reply_data['full_text'] = reply_data['full_text'].replace(username, username_zwj) reply_data['text'] = reply_data['full_text'] # Don't send the ID of the reply tweet reply_data['id_str'] = None send_tweet_to_db(reply_data, start_time) time.sleep(0.5) send_tweet_to_db(data, start_time) # Yes, don't send it again else: send_tweet_to_db(data, start_time) logger.info("Parent tweet already posted") # If it's a normal tweet elif "retweeted_status" not in data and data["in_reply_to_status_id"] is None: send_tweet_to_db(data, start_time)