def comment_on_thread(submission, twitter_url_list, imgur_url_list):
# Assembles the comment
thread_comment = "**Max Resolution Twitter Link(s)**\n\n"
for twitter_url in twitter_url_list:
thread_comment += ("%s\n\n" % twitter_url)
thread_comment += "**Imgur Mirror Link(s)**\n\n"
for imgur_url in imgur_url_list:
thread_comment += ("%s\n\n" % imgur_url)
thread_comment += "---\n\n^^I'm ^^a ^^bot ^^made ^^by ^^[u\/jclishman](https://reddit.com/user/jclishman)!"
thread_comment += " [^^[FAQ/Discussion]](http://reddit.com/user/SpaceXMirrorBot/comments/ad36dr/) [^^[Code]](https://github.com/jclishman/SpaceXMirrorBot)"
# Posts the comment
retries = 0
while retries < 5:
try:
thread_comment_id = submission.reply(thread_comment)
logger.info("Comment made with ID: %s" % thread_comment_id)
# Break from the while loop after successful post submission
break
except praw.exceptions.APIException as e:
retries += 1
logger.error("Hit ratelimit, will try again in 5 minutes. (Attempt %d/5)" % retries)
time.sleep(300)
def get_twitter_fullres(tweet_url):
# RegEx's the status URL for the numeric status ID
tweet_id = status_regex.search(tweet_url).group(1)
# Get the status JSON from Twitter
tweet_data = twitter_api.get_status(tweet_id, include_entities=True, tweet_mode='extended')
tweet = tweet_data._json
twitter_url_list = []
try:
# Check if the tweet has media attached
for tweet_media in tweet["extended_entities"]["media"]:
if tweet_media["type"] == "photo":
# Appends ":orig" to URL for max resolution
twitter_url_list.append(tweet_media["media_url_https"]+":orig")
else:
logger.info("Media attached is not a picture")
return -1
return twitter_url_list
except:
logger.info("Tweet has no media attached")
return -1
def post():
send_message_to_channels(msg)
logger.info(msg)
# Sends how long it took from tweet creation to irc message (debug)
time_to_post = round(time.time() - row[6], 5)
logger.info(f"Post #{row[0]}, Took {time_to_post}s\n")
def unfollow_account(user_id):
try:
database = sqlite3.connect('database.db')
unfollow_cursor = database.cursor()
unfollow_cursor.execute("DELETE FROM following WHERE twitter_id = %s" %
user_id)
database.commit()
logger.info('Deleted from DB')
except sqlite3.OperationalError as e:
logger.error(str(e))
def set_flags(user_id, retweets, replies):
try:
database = sqlite3.connect('database.db')
set_flags_cursor = database.cursor()
set_flags_cursor.execute(
"UPDATE following SET retweets = %i, replies = %i WHERE twitter_id = %s"
% (retweets, replies, user_id))
logger.info('Updated DB flags')
database.commit()
except sqlite3.OperationalError as e:
logger.error(str(e))
def upload_to_imgur(url_list):
imgur_url_list = []
for url in url_list:
# Uploads image to Imgur and adds the link to a list
image = imgur_api.upload_from_url(url, anon=True)
logger.info("Uploaded to Imgur with ID: %s" % image["id"])
imgur_url_list.append(image["link"])
time.sleep(1)
return imgur_url_list
def follow_account(username, user_id, retweets, replies):
try:
database = sqlite3.connect('database.db')
follow_cursor = database.cursor()
follow_cursor.execute(
"INSERT INTO following (username, twitter_id, retweets, replies) VALUES(?,?,?,?)",
[username, user_id, retweets, replies])
logger.info('Added to DB')
database.commit()
except sqlite3.OperationalError as e:
logger.error(str(e))
def run():
logger.info("Starting Twitter Stream...")
# Makes the stream object
myStreamListener = MyStreamListener()
myStream = tweepy.Stream(auth, myStreamListener)
# Streams tweets
try:
myStream.filter(follow=following, stall_warnings=True)
except Exception as e:
logger.error("-----CAUGHT ERROR-----")
logger.error(str(e))
logger.error("-----ATTEMPTING RECONNECT-----")
run()
def run():
web_api = Client(auto_patch=True, drop_incompat_keys=False)
feed = []
startup = True
user_dict = {
"SpaceX": "20311520",
"jclishman.testing": "7400533474"
}
while True:
for id_str in list(user_dict.values()):
try:
feed.append(web_api.user_feed(id_str, count=1))
time.sleep(5)
except Exception as e:
#logger.error(str(e))
#logger.error("Error getting feed. Sleeping for 30s")
time.sleep(30)
for post in feed:
post = post[0]["node"]
user_id_str = post["owner"]["id"]
shortcode = post["shortcode"]
timestamp = post["created_time"]
# Empty string if there isn't a caption
try:
caption = post["caption"]["text"]
except:
caption = ''
# Match ID number to screenname
for screen_name, id_str in user_dict.items():
if user_id_str == id_str:
user_screen_name = screen_name
stored_timestamp = db.get_instagram_timestamp(user_screen_name)
if int(timestamp) > stored_timestamp:
start_time = time.time()
db.update_instagram_timestamp(user_screen_name, int(timestamp))
logger.info(f"New Instagram post by @{user_screen_name}, id {user_id_str}")
logger.info(f"Post shortcode: {shortcode}")
logger.info(f"Post caption: {caption}")
logger.info(f"Post timestamp: {timestamp}")
url = f"https://instagram.com/p/{shortcode}"
if not startup:
db.insert_message('Instagram', user_screen_name, caption.replace("\n", " "), url, start_time)
time.sleep(10)
startup = False
def parse(message_contents):
# Default values
retweets = 0
replies = 0
# Makes everything lowercase, removes the bot username, and take out all spaces
message_clean = message_contents.replace(NICK + ': ', '').lower()
# Splits the command into parts
command = message_clean.split(' ')
action = command[0].strip()
# Error handling
try:
target = command[1].replace('@', '').strip()
except:
target = None
try:
retweets = int(command[2].strip())
except:
retweets = None
try:
replies = int(command[3].strip())
except:
replies = None
if action == 'help':
if target == 'follow':
return 'Follows Twitter accounts, see MaxQ: help for syntax'
elif target == 'unfollow':
return 'Unfollows Twitter accounts, see MaxQ: help for syntax'
elif target == 'set':
return 'Changes flag(s) for accounts I\'m already following, see MaxQ: help for syntax'
else:
return 'Syntax: MaxQ: follow|unfollow|set @username 0|1 <retweets> 0|1 <replies>'
elif action == 'unfollow':
user_id = twitter.getID(target)
if in_database(user_id):
db.unfollow_account(user_id)
return 'Unfollowed @%s' % target
elif not in_database(user_id):
return 'Error: @%s is not in the database' % target
if type(retweets) is not int or type(replies) is not int:
logger.error('Command error: not an int')
return 'Error: Invalid command'
if retweets > 1 or replies > 1:
logger.error('Command Error: not 0 or 1')
return 'Error: Invalid command'
# print('Command: ' + str(command))
logger.info(
f"Action: {action} | Target: {target} | Retweets: {retweets} | Replies: {replies}"
)
if target is None:
return 'Error: No target account'
user_id = twitter.getID(target)
# Is the target a user? Does the user actually exist?
if user_id is None:
return f"Error: @{target} does not exist"
# Are flags specified?
if retweets is None or replies is None:
return 'Error: Missing valid flag(s)'
if action == 'follow':
# Is the user not already in the database?
if not in_database(user_id):
db.follow_account(target, user_id, retweets, replies)
return f"@{target} is now being followed on Twitter | Retweets {retweets} | Replies {replies}"
return f"@{target} is already being followed on Twitter"
if action == 'set':
if in_database(user_id):
db.set_flags(user_id, retweets, replies)
return f"@{target} now has retweets set to {retweets} and replies set to {replies}"
return f"Error: @{target} is not in the database"
def insta_thread():
logger.info('Starting Instragram Thread')
instagramservice.run()
def twitter_thread():
logger.info('Starting Twitter Thread')
twitterservice.run()
def restart_irc():
logger.info('Restarting...')
os.system('start cmd /c irc.py')
time.sleep(5)
irc.send(parse('QUIT :Be right back!'))
exit()
def reddit_thread():
logger.info('Starting Reddit Thread')
redditservice.run()
# Reddit AMA mode
from bot_logging import logger
import twitterservice, instagramservice, redditservice
import commands, db
import socket, ssl
import threading
import time
import json
import os
logger.info('Now Entering MaxQ...')
# Secret credentials :)
config = json.load(open('_config.json'))
password = config['nickserv_password']
# IRC Config
# HOST = 'irc.esper.net'
HOST = 'irc.snoonet.org'
PORT = 6697
NICK = 'MaxQ'
admins = config["admin_hostnames"]
channels = ['#groupofthrones']
# Responds to server pings
def pong(text):
logger.debug('PONG' + text.split()[1])
irc.send(parse('PONG ' + text.split()[1]))
def restart_irc():
logger.info("Restarting...")
os.system("nohup python3 irc.py &")
time.sleep(5)
irc.send(parse("QUIT :Be right back!"))
exit()
post_time = submission.created_utc
# Is it new?
if post_time > last_post_time[subreddit]:
last_post_time[subreddit] = post_time
# Ignore if it's not a link post
if submission.is_self:
#logger.info("Not a link post")
pass
# Check if it's a link to Twitter
elif "twitter.com" in submission.url:
logger.info("="*30)
logger.info("Found a tweet post (%s)" % submission.shortlink,)
#logger.info(submission.url)
twitter_url_list = get_twitter_fullres(submission.url)
# get_twitter_fullres() returns -1 if the tweet has no/incorrect media type
if twitter_url_list != -1:
imgur_url_list = upload_to_imgur(twitter_url_list)
logger.info("Max Res Twitter URLs: ")
logger.info(twitter_url_list)
logger.info("Mirror Imgur URLs: ")
logger.info(imgur_url_list)
def run():
# Bot won't post anything new on startup
startup = True
class InstagramUser:
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'
rhx_gis = None
csrf_token_cookie = None
session = requests.Session()
def __init__(self):
res = self.session.get('https://instagram.com',
headers={'User-Agent': self.user_agent})
regex_match = re.search(r'"rhx_gis":"(?P<rhx_gis>[a-f0-9]{32})"',
res.text)
if regex_match:
self.rhx_gis = regex_match.group('rhx_gis')
def build_signature(self, query_variables):
m = hashlib.md5()
m.update('{rhx_gis}:{variables}'.format(
rhx_gis=self.rhx_gis,
variables=query_variables).encode('utf-8'))
return m.hexdigest()
def request(self, **kwargs):
endpoint_url = None
signature_var = None
query_params = kwargs.pop('query_params', {})
if kwargs['endpoint'] is "username":
endpoint_url = 'https://instagram.com/' + kwargs[
'username'] + '/'
signature_var = urllib.parse.urlparse(endpoint_url).path
query_params = {'__a': '1'}
elif kwargs['endpoint'] is "graphql":
endpoint_url = 'https://instagram.com/graphql/query'
query_params['query_hash'] = '42323d64886122307be10013ad2dcc44'
signature_var = query_params['variables'] = json.dumps(
kwargs['query_variable'])
return self.session.get(
endpoint_url +
"?{}".format(urllib.parse.urlencode(query_params)),
headers={
'User-Agent': self.user_agent,
'X-Instagram-GIS': self.build_signature(signature_var)
})
def get_user_id(self, username):
response = self.request(username=username, endpoint='username')
if response.status_code is not 200:
logger.error("Instagram responded with status code %s" %
str(response.status_code))
raise Exception("Instagram responded with status_code %s" %
str(response.status_code))
json_response = response.json()
return json_response['graphql']['user']['id']
def get_recent_post(self, user_id, **kwargs):
variables = {"id": user_id, "first": 1}
response = self.request(endpoint='graphql',
query_variable=variables)
if response.status_code is not 200:
logger.error("Instagram responded with status code %s" %
str(response.status_code))
raise Exception("Instagram responded with status_code %s" %
str(response.status_code))
post = response.json(
)['data']['user']['edge_owner_to_timeline_media']['edges'][0]
post_data = {
'shortcode': post['node']['shortcode'],
'created_at_timestamp': post['node']['taken_at_timestamp'],
'caption': None,
}
if len(post['node']['edge_media_to_caption']['edges']) != 0:
post_data["caption"] = post['node']['edge_media_to_caption'][
'edges'][0]["node"]["text"]
return post_data
users_list = db.get_following('instagram')
username_list = {x[0] for x in users_list}
username = ''
while True:
for username in username_list:
# Gets the ID of the post again, and compares it against the stored value
user = InstagramUser()
user_id = user.get_user_id(username)
post_data = user.get_recent_post(user_id)
stored_timestamp = db.get_instagram_timestamp(username)
# Is it a new post?
if post_data["created_at_timestamp"] > stored_timestamp:
start_time = time.time()
logger.info('Found new Instagram post')
logger.info('Shortcode: ' + post_data["shortcode"])
logger.info('Caption: ' + post_data["caption"])
logger.info('Timestamp: ' +
str(post_data["created_at_timestamp"]))
db.update_instagram_timestamp(
username, post_data["created_at_timestamp"])
logger.info('Updated database\n')
if not startup:
db.insert_message(
'Instagram', username, post_data["caption"],
'https://instagram.com/p/%s' % post_data["shortcode"],
start_time)
else:
logger.debug('Did not find new post\n')
time.sleep(3)
startup = False
def on_status(self, status):
#print(status.text)
#print(json.dumps(status._json))
# Converts the data into usable JSON
data = status._json
# Puts user attributes into this list if the tweet is from somebody the bot is following
# If the tweet isn't from someone the bot is following, set to None
# For some reason the twitter API also tells you when someone deletes a tweet
# If you try to get the needed properties from a deleted tweet, it throws a KeyError
try:
user_of_tweet = next(
(x for x in users_list if x[1] == data['user']['id_str']),
None)
except KeyError as e:
return
if user_of_tweet is None:
# Not following this user, do nothing.
return
start_time = time.time()
# Is it a retweet? Is the retweet flag of the user set to 1?
if "retweeted_status" in data and user_of_tweet[2] == 1:
rt_data = data['retweeted_status']
# Has the retweeted status already been posted?
# No, post it
if not has_tweet_been_posted(rt_data['user']['screen_name'],
rt_data['id_str']):
send_tweet_to_db(data, start_time)
# Yes, don't post it
else:
logger.info("Retweet has already been posted")
# Is a reply? Is the reply flag of the user set to 1?
elif data['in_reply_to_status_id'] is not None and user_of_tweet[
3] == 1:
reply_data = get_status(data['in_reply_to_status_id'])
# Has the parent tweet to the reply been posted already?
# No, send it as context
if not has_tweet_been_posted(reply_data['user']['screen_name'],
reply_data['id_str']):
# Avoid IRC double pings
usernames_to_modify = ['@elonmusk', '@SpaceX']
for username in usernames_to_modify:
ZWJ = ""
# Inserts a Zero Width Joiner
username_zwj = username[:2] + ZWJ + username[2:]
reply_data['full_text'] = reply_data['full_text'].replace(
username, username_zwj)
reply_data['text'] = reply_data['full_text']
# Don't send the ID of the reply tweet
reply_data['id_str'] = None
send_tweet_to_db(reply_data, start_time)
time.sleep(0.5)
send_tweet_to_db(data, start_time)
# Yes, don't send it again
else:
send_tweet_to_db(data, start_time)
logger.info("Parent tweet already posted")
# If it's a normal tweet
elif "retweeted_status" not in data and data[
"in_reply_to_status_id"] is None:
send_tweet_to_db(data, start_time)
# Reddit AMA mode
from bot_logging import logger
import twitterservice, redditservice, launchservice, acronymservice
import commands, db
import socket, ssl
import threading
import time
import math
import json
import os
logger.info("Now Entering MaxQ...")
# Secret credentials :)
config = json.load(open("_config.json"))
password = config["nickserv_password"]
# IRC Config
HOST = "irc.esper.net"
#HOST = "irc.snoonet.org"
PORT = 6697
NICK = "MaxQ"
admins = config["admin_hostnames"]
#channels = ["#lishbot"]
channels = ["#SpaceX"]
launch_whitelist = config["whitelist"]
launchmode = False
# Responds to server pings
def pong(text):
def on_status(self, status):
#print(status.text)
#print(json.dumps(status._json))
# Converts the data into usable JSON
data = status._json
# Puts user attributes into this list if the tweeet is from somebody the bot is following
# If the tweet isn't from someone the bot is following, set to None
# For some reason the twitter API also tells you when someone deletes a tweet
# If you try to get the needed properties from a deleted tweet, it throws a KeyError
try:
user_of_tweet = next((x for x in users_list if x[1] == data['user']['id_str']), None)
except KeyError as e:
user_of_tweet = None
# Sends the tweet to the database
def send_tweet_to_db(tweet_data, start_time):
# Gets the full tweet text if it's concatenated by Twitter
if "extended_tweet" in tweet_data:
text = html.unescape(tweet_data['extended_tweet']['full_text'])
# Gets the full retweet text if it's concatenated by Twitter
elif "retweeted_status" in tweet_data and "extended_tweet" in tweet_data['retweeted_status']:
rt_data = tweet_data['retweeted_status']
text = f"RT @{rt_data['user']['screen_name']}: {html.unescape(rt_data['extended_tweet']['full_text'])}"
# Not an extended tweet
else:
text = html.unescape(tweet_data['text'])
# Logs raw JSON
#logger.info(json.dumps(data))
# Checks if the tweet is a parent to a reply, which won't have an ID
if tweet_data['id_str'] is not None:
tweet_url = make_url_from_tweet(tweet_data['user']['screen_name'], tweet_data['id_str'])
else:
tweet_url = ''
db.insert_message('Twitter', tweet_data['user']['screen_name'], text.replace("\n", " "), tweet_url, start_time)
# Is the tweet from somebody the bot cares about?
if user_of_tweet is not None:
start_time = time.time()
# Is it a retweet? Is the retweet flag of the user set to 1?
if "retweeted_status" in data and user_of_tweet[2] == 1:
rt_data = data['retweeted_status']
# Has the retweeted status already been posted?
# No, post it
if not has_tweet_been_posted(rt_data['user']['screen_name'], rt_data['id_str']):
send_tweet_to_db(data, start_time)
# Yes, don't post it
else:
logger.info("Retweet has already been posted")
# Is a reply? Is the reply flag of the user set to 1?
elif data['in_reply_to_status_id'] is not None and user_of_tweet[3] == 1:
reply_data = get_status(data['in_reply_to_status_id'])
# Has the parent tweet to the reply been posted already?
# No, send it as context
if not has_tweet_been_posted(reply_data['user']['screen_name'], reply_data['id_str']):
# Avoid IRC double pings
usernames_to_modify = ['@elonmusk', '@SpaceX']
for username in usernames_to_modify:
ZWJ = ""
# Inserts a Zero Width Joiner
username_zwj = username[:2] + ZWJ + username[2:]
reply_data['full_text'] = reply_data['full_text'].replace(username, username_zwj)
reply_data['text'] = reply_data['full_text']
# Don't send the ID of the reply tweet
reply_data['id_str'] = None
send_tweet_to_db(reply_data, start_time)
time.sleep(0.5)
send_tweet_to_db(data, start_time)
# Yes, don't send it again
else:
send_tweet_to_db(data, start_time)
logger.info("Parent tweet already posted")
# If it's a normal tweet
elif "retweeted_status" not in data and data["in_reply_to_status_id"] is None:
send_tweet_to_db(data, start_time)
