class ImgurRepostBot():
def __init__(self):
os.system('cls')
self.failed_downvotes = [] # Store failed downvotes for later processing
self.failed_comments = [] # Store failed comments for later processing
self.delay_between_requests = 5 # Changed on the fly depending on remaining credits and time until reset
self.thread_lock = threading.Lock()
self.logger = None
self.detected_reposts = 0
self.config = ConfigManager()
self._setup_logging()
self.imgur_client = ImgurClient(self.config.api_details['client_id'],
self.config.api_details['client_secret'],
self.config.api_details['access_token'],
self.config.api_details['refresh_token'])
self.db_conn = ImgurRepostDB(self.config)
self.backfill_progress = 1 if self.config.backfill else 'Disabled'
records, processed_ids = self.db_conn.build_existing_ids()
if self.config.backfill:
threading.Thread(target=self._backfill_database, name='Backfill').start()
self.hash_processing = HashProcessing(self.config, processed_ids, records)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
def _check_thread_status(self):
"""
Check status of critical threads. If they are found dead start them back up
:return:
"""
thread_names = ['configmonitor', 'backfill', 'repostprocessing']
for thrd in threading.enumerate():
if thrd.name.lower() in thread_names:
thread_names.remove(thrd.name.lower())
for i in thread_names:
if i == 'configmonitor':
msg = 'Config Monitor Thread Crashed'
self._output_error(msg)
self.config = ConfigManager()
continue
if i == 'backfill' and self.config.backfill:
msg = 'Backfill Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._backfill_database, name='Backfill').start()
continue
if i == 'repostprocessing':
msg = 'Repost Processing Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
continue
def _setup_logging(self):
if self.config.logging:
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
fhandle = logging.FileHandler('botlog.log')
fhandle.setFormatter(formatter)
self.logger.addHandler(fhandle)
def _output_error(self, msg, output=True):
"""
convenience method to log and/or print an error
:param msg: Message to output/log
:param output: Print error to console
:return:
"""
if output:
print(msg)
if self.config.logging:
self.logger.error(msg)
def _output_info(self, msg):
print(msg)
if self.config.logging:
self.logger.info(msg)
def _backfill_database(self):
"""
Backfill the database with older posts. Useful if script hasn't been run in some time"
:return:
"""
while True:
if self.config.backfill:
original_start_page = self.config.backfill_start_page # So we can detect if it's changed in the config
current_page = self.config.backfill_start_page
while current_page < self.config.backfill_depth + self.config.backfill_start_page:
if not self.db_conn.records_loaded:
continue
self.backfill_progress = current_page
self.insert_latest_images(page=current_page, backfill=True)
current_page += 1
time.sleep(self.delay_between_requests)
if self.config.backfill_start_page != original_start_page:
print('Backfill Start Page Changed In Config')
break
self.backfill_progress = 'Completed'
break
else:
self.backfill_progress = 'Disabled'
time.sleep(5)
def _generate_img(self, url=None):
"""
Generate the image files provided from Imgur. We pass the data straight from the request into PIL.Image
"""
img = None
if not url:
return None
try:
response = request.urlopen(url)
img = Image.open(BytesIO(response.read()))
except (HTTPError, ConnectionError, OSError) as e:
msg = 'Error Generating Image File: \n Error Message: {}'.format(e)
self._output_error(msg)
return None
return img if img else None
def generate_latest_images(self, section='user', sort='time', page=0):
self._adjust_rate_limit_timing()
items = []
try:
temp = self.imgur_client.gallery(section=section, sort=sort, page=page, show_viral=False)
if temp:
items = [i for i in temp if not i.is_album and not self.check_post_title(title=i.title)]
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Error Getting Gallery: {}'.format(e)
self._output_error(msg)
return items
def insert_latest_images(self, section='user', sort='time', page=0, backfill=False):
"""
Pull all current images from user sub, get the hashes and insert into database.
"""
# Don't start inserts until all records are loaded
if not self.db_conn.records_loaded:
return
items = self.generate_latest_images(section=section, sort=sort, page=page)
if not items:
return
# Don't add again if we have already done this image ID
for item in items:
if item.id in self.hash_processing.processed_ids:
continue
img = self._generate_img(url=item.link)
if img:
image_hash = self.hash_processing.generate_hash(img)
if image_hash:
record = {
'image_id': item.id,
'url': item.link,
'gallery_url': 'https://imgur.com/gallery/{}'.format(item.id),
'user': item.account_url,
'submitted': item.datetime,
'hash16': image_hash['hash16'],
'hash64': image_hash['hash64'],
'hash256': image_hash['hash256']
}
self.hash_processing.processed_ids.append(item.id)
self.hash_processing.records.append(record)
# If this is called from back filling don't add hash to be checked
if not backfill:
self.hash_processing.hash_queue.append(record)
print('Processing {}'.format(item.link))
else:
print('Backfill Insert {}'.format(item.link))
self.db_conn.add_entry(record)
def downvote_repost(self, image_id):
"""
Downvote the provided Image ID
"""
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
except ImgurClientError as e:
self.failed_downvotes.append(image_id)
msg = 'Error Voting: {}'.format(e)
self._output_error(msg)
def comment_repost(self, image_id=None, values=None):
"""
Leave a comment on the detected repost.
:param image_id: ID of image to leave comment on.
:param values: Values to be inserted into the message template
:return:
"""
self._output_info('Leaving Comment On {}'.format(image_id))
message = self.build_comment_message(values=values)
if not message:
return
try:
self.imgur_client.gallery_comment(image_id, message)
except (ImgurClientError, ImgurClientRateLimitError) as e:
self.failed_comments.append({'image_id': image_id, 'values': values})
msg = 'Error Posting Commment: {}'.format(e)
self._output_error(msg)
def build_comment_message(self, values=None):
if not values:
return None
# Build up replacement dict
out_dict = {
'count': len(values),
'g_url': values[0]['gallery_url'],
'd_url': values[0]['url'],
'submitted_epoch': values[0]['submitted'],
'submitted_human': time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime(values[0]['submitted'])),
'user': values[0]['user'],
}
try:
final_message = self.config.comment_template.format(**out_dict)
print('Final Message: ' + final_message)
if len(final_message) > 140:
self.logger.warning('Message Length Is Over 140 Chars. Will Be Trimmed')
return final_message
except KeyError as e:
msg = 'Error Generating Message: {}'.format(e)
self._output_error(msg)
return None
def flush_failed_votes_and_comments(self):
"""
If there have been any failed votes or comments (due to imgur server overload) try to redo them
"""
if self.failed_downvotes:
for image_id in self.failed_downvotes:
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
self.failed_downvotes.remove(image_id)
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Downvote On Image {}. \nError: {}'.format(image_id, e)
self._output_error(msg)
if self.failed_comments:
for failed in self.failed_comments:
try:
message = self.build_comment_message(values=failed['values'])
self.imgur_client.gallery_comment(failed['image_id'], message)
self.failed_comments.remove(failed['image_id'])
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Comment On Image {}. \nError: {}'.format(failed['image_id'], e)
self._output_error(msg)
def _repost_processing_thread(self):
"""
Runs in background monitor the queue for detected reposts
:return:
"""
while True:
if len(self.hash_processing.repost_queue) > 0:
current_repost = self.hash_processing.repost_queue.pop(0)
image_id = current_repost[0]['image_id']
sorted_reposts = sorted(current_repost[0]['older_images'], key=itemgetter('submitted'))
if self.config.leave_downvote:
self.downvote_repost(image_id)
if self.config.leave_comment:
self.comment_repost(image_id, values=sorted_reposts)
self.detected_reposts += 1
if self.config.log_reposts:
with open('repost.log', 'a+') as f:
f.write('Repost Image: https://imgur.com/gallery/{}\n'.format(image_id))
f.write('Matching Images:\n')
for r in sorted_reposts:
f.write(r['gallery_url'] + '\n')
def _adjust_rate_limit_timing(self):
"""
Adjust the timing used between request to spread all requests over allowed rate limit
"""
# API Fails To Return This At Times
if not self.imgur_client.credits['ClientRemaining']:
return
remaining_credits_before = int(self.imgur_client.credits['ClientRemaining'])
self.imgur_client.credits = self.imgur_client.get_credits() # Refresh the credit data
# Imgur API sometimes returns 12500 credits remaining in error. If this happens don't update request delay.
# Otherwise the delay will drop to the minimum set in the config and can cause premature credit exhaustion
if int(self.imgur_client.credits['ClientRemaining']) - remaining_credits_before > 100:
"""
print('Imgur API Returned Wrong Remaining Credits. Keeping Last Request Delay Time')
print('API Credits: ' + str(self.imgur_client.credits['ClientRemaining']))
print('Last Credits: ' + str(remaining_credits_before))
"""
return
remaining_credits = self.imgur_client.credits['ClientRemaining']
reset_time = self.imgur_client.credits['UserReset'] + 240 # Add a 4 minute buffer so we don't cut it so close
remaining_seconds = reset_time - round(time.time())
seconds_per_credit = round(remaining_seconds / remaining_credits) # TODO Getting division by zero sometimes
if seconds_per_credit < self.config.min_time_between_requests:
self.delay_between_requests = self.config.min_time_between_requests
else:
self.delay_between_requests = seconds_per_credit
def check_post_title(self, title=None):
"""
Checks the post title for values that we will use to skip over it
This allows us not to flag MRW posts and others as reposts
:return:
"""
if not title:
return None
return [v for v in self.config.title_check_values if v in title.lower()]
def print_current_settings(self):
print('Current Settings')
print('[+] Leave Comments: {}'.format('Enabled' if self.config.leave_comment else 'Disabled'))
print('[+] Leave Downvote: {} '.format('Enabled' if self.config.leave_downvote else 'Disabled'))
print('[+] Backfill: {} '.format('Enabled' if self.config.backfill else 'Disabled'))
print('[+] Backfill Depth: {} '.format(self.config.backfill_depth if self.config.backfill else 'Disabled'))
print('[+] Process Pool Size: {} '.format(self.config.hash_proc_limit))
print('[+] Hash Size: {} bit'.format(self.config.hash_size))
print('[+] Hamming Distance: {}{}'.format(self.config.hamming_cutoff, '\n'))
def print_current_stats(self):
print('Current Stats')
print('[+] Total Images In Database: {}'.format(str(len(self.hash_processing.processed_ids))))
print('[+] Total Hashes Waiting In Pool: {}'.format(str(self.hash_processing.total_in_queue)))
print('[+] Total Hashes In Hash Queue: {}'.format(str(len(self.hash_processing.hash_queue))))
print('[+] Process Pool Status: {}'.format(self.hash_processing.pool_status))
print('[+] Total Reposts Found: {}'.format(str(self.detected_reposts)))
print('[+] Backfill Progress: {}\n'.format('Page ' + str(self.backfill_progress) if self.config.backfill else 'Disabled'))
def print_api_stats(self):
print('API Settings')
print('[+] Remaining Credits: {}'.format(self.imgur_client.credits['ClientRemaining']))
if self.imgur_client.credits['UserReset']:
print('[+] Time Until Credit Reset: {} Minutes'.format(round((int(self.imgur_client.credits['UserReset']) - time.time()) / 60)))
# Make it clear we are overriding the default delay to meet credit refill window
if self.delay_between_requests == self.config.min_time_between_requests:
request_delay = str(self.delay_between_requests) + ' Seconds'
else:
request_delay = str(self.delay_between_requests) + ' Seconds (Overridden By Rate Limit)'
print('[+] Delay Between Requests: {} \n'.format(request_delay))
def run(self):
last_run = round(time.time())
while True:
os.system('cls')
self.print_current_stats()
self.print_current_settings()
self.print_api_stats()
if round(time.time()) - last_run > self.delay_between_requests:
self.insert_latest_images()
self.flush_failed_votes_and_comments()
last_run = round(time.time())
self._check_thread_status()
time.sleep(2)
if __name__ == "__main__":
try:
with open(page_path, "rb") as f:
page_start = pickle.load(f)
print("Starting at page {}.".format(page_start))
except FileNotFoundError:
with open(page_path, "wb") as f:
pickle.dump(page_start, f, pickle.HIGHEST_PROTOCOL)
print("Starting fresh at page 0.")
client = ImgurClient(CLIENT_ID, CLIENT_SECRET)
for page in range(page_start, page_start + page_range):
items = client.memes_subgallery(page=page)
for item in items:
if isValid(item):
media_name = media_path + item.link.split("/")[-1]
r = requests.get(item.link, allow_redirects=True)
with open(media_name, "wb") as f:
f.write(r.content)
data_name = data_path + item.id + ".pkl"
with open(data_name, "wb") as f:
pickle.dump(item, f, pickle.HIGHEST_PROTOCOL)
print(page, item.id, item.width, item.height, item.size,
item.link, item.score)
with open(page_path, "wb") as f:
pickle.dump(page + 1, f, pickle.HIGHEST_PROTOCOL)
print(client.get_credits())
class ImgurUp(object):
def __init__(self):
self.work_left = True
builder = gtk.Builder()
builder.add_from_file('%s/data/main_window.ui' % basepath)
builder.connect_signals(self)
self.windowstate = 1
self.user_auth = 0
self.imagepath = builder.get_object('imagepath')
self.imagetitle = builder.get_object('imagetitle')
self.imagecaption = builder.get_object('imagecaption')
self.image = builder.get_object('imagepreview')
self.filefilter = builder.get_object('filefilter1')
self.filefilter.set_name("Image Files")
self.filefilter.add_pixbuf_formats()
self.menu1 = builder.get_object('menu1')
self.window = builder.get_object('window1')
self.text_info = builder.get_object('label39')
self.statusicon = gtk.StatusIcon()
self.statusicon.set_from_file('%s/data/imgurup.svg' % basepath)
self.statusicon.connect("popup-menu", self.right_click_event)
self.statusicon.connect("activate", self.icon_clicked)
self.statusicon.set_tooltip("Imgur Uploader")
self.window.connect("drag_data_received", self.on_file_dragged)
self.window.drag_dest_set(gtk.DEST_DEFAULT_MOTION | gtk.DEST_DEFAULT_DROP,
dnd_list, gtk.gdk.ACTION_COPY)
self.window.show_all()
if int(config['workmode']) == 0:
self.albums = builder.get_object('albumbutton')
self.albums.set_sensitive(False)
self.user = builder.get_object('authorbutton')
self.user.set_sensitive(False)
else:
self.il = ImgurClient(config['imgurkey'], config['imgursecret'])
if config['usertoken'] and config['usersecret']:
self.il = ImgurClient(config['imgurkey'], config['imgursecret'], config['usertoken'], config['usersecret'])
self.user_auth = 1
else:
self.user_info()
def exit(self, widget=None):
if quit_msg("<b>Are you sure to quit?</b>") == True:
gtk.main_quit()
else:
return False
def on_window1_destroy(self, widget, data=None):
self.window.hide_on_delete()
self.windowstate = 0
return True
def select_image(self, widget, data=None):
filedlg = gtk.FileChooserDialog("Select image file...", None,
gtk.FILE_CHOOSER_ACTION_OPEN,
(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
gtk.STOCK_OPEN, gtk.RESPONSE_OK))
filedlg.add_filter(self.filefilter)
response = filedlg.run()
if response == gtk.RESPONSE_OK:
self.imagepath.set_text(filedlg.get_filename())
imgname = os.path.basename(filedlg.get_filename())
pixbuf = gtk.gdk.pixbuf_new_from_file(filedlg.get_filename())
self.image.set_from_pixbuf(pixbuf.scale_simple(300, 200, gtk.gdk.INTERP_NEAREST))
self.imagetitle.set_text(imgname.split('.')[0].title())
else:
filedlg.destroy()
filedlg.destroy()
def upload_image(self, widget, data=None):
if self.check_fields() == True:
self.text_info.set_markup("<b><i>Uploading...</i></b>")
config = {"title": self.imagetitle.get_text(),
"description": self.imagecaption.get_text()}
info = self.il.upload_from_path(self.imagepath.get_text(),
config)
if show_info(basepath, info) == True:
f = open('%s/recent.txt' % basepath, 'a')
f.write(info+"\n\r")
f.close()
menuItem = gtk.MenuItem(simplejson.loads(info)['images']['image']['title'])
menuItem.connect('activate', lambda term: show_info(basepath, info))
self.menu1.append(menuItem)
self.menu1.show_all()
self.text_info.set_text("")
def check_fields(self):
if not self.imagepath.get_text():
return False
elif not self.imagecaption.get_text():
self.imagecaption.set_text("None")
else:
return True
def clear_fields(self, widget, data=None):
self.imagetitle.set_text("")
self.imagepath.set_text("")
self.imagecaption.set_text("")
self.image.set_from_file("%s/data/imgurup-logo.png" % basepath)
def take_screenshot(self, widget, data=None):
if config['screenshotpath'] != "":
path = config['screenshotpath']
else:
path = os.getenv("HOME")
if self.windowstate == 1:
self.window.hide()
self.windowstate = 0
shot = self.fullscreen_shot(path)
uploadfile = path+"/"+shot
self.window.show_all()
self.windowstate = 1
else:
shot = self.fullscreen_shot(path)
self.window.show_all()
self.windowstate = 1
self.imagepath.set_text(path+"/"+shot)
pixbuf = gtk.gdk.pixbuf_new_from_file(path+"/"+shot)
self.image.set_from_pixbuf(pixbuf.scale_simple(300, 200, gtk.gdk.INTERP_NEAREST))
self.imagetitle.set_text(shot.split('.')[0].title())
if bool(config['captionremove']) == False:
self.imagecaption.set_text("Desktop Screenshot with ImgurUp")
def fullscreen_shot(self, path = os.getenv('HOME')):
from string import Template
imgformat = "png"
width = gtk.gdk.screen_width()
height = gtk.gdk.screen_height()
s = Template(config['screenshotname'])
shotname = s.safe_substitute(date = time.strftime("%Y%m%d%H%M%S", time.localtime()),
time = time.strftime("H%M%S", time.localtime()),
count = config['count'])
time.sleep(float(config['waitscreen']))
screenshot = gtk.gdk.Pixbuf.get_from_drawable(
gtk.gdk.Pixbuf(gtk.gdk.COLORSPACE_RGB, True, 8, width, height),
gtk.gdk.get_default_root_window(),
gtk.gdk.colormap_get_system(),
0, 0, 0, 0, width, height)
screenshot.save(path+"/"+shotname+"."+imgformat, imgformat)
config['count'] = int(config['count']) + 1
config.write()
return shotname+"."+imgformat
def show_albums(self, widget, data=None):
AlbumsDialog(basepath, self.il)
def about_click(self, widget, data=None):
AboutDialog(basepath)
def on_file_dragged(self, widget, context, x, y, select, target, timestamp):
imagepath = "/" + select.data.strip('\r\n\x00').strip("file://")
if mimetypes.guess_type(imagepath)[0].startswith('image') == True:
self.imagepath.set_text(imagepath)
imagename = os.path.basename(imagepath)
pixbuf = gtk.gdk.pixbuf_new_from_file(imagepath)
self.image.set_from_pixbuf(pixbuf.scale_simple(300, 200, gtk.gdk.INTERP_NEAREST))
self.imagetitle.set_text(imagename.split('.')[0].title())
else:
pass
def user_info(self, sender=None, data=None):
builder = gtk.Builder()
builder.add_from_file('%s/data/main_window.ui' % basepath)
userinfo = builder.get_object('userdialog')
authimage = builder.get_object('image1')
authtext = builder.get_object('label15')
username = builder.get_object('label19')
prof = builder.get_object('label18')
privacy = builder.get_object('label21')
credits = builder.get_object('label23')
authbut = builder.get_object('button3')
if self.user_auth == 1:
info = self.il.get_account('me')
authimage.set_from_stock(gtk.STOCK_OK, gtk.ICON_SIZE_SMALL_TOOLBAR)
authtext.set_markup('<span foreground="green">Authenticated</span>')
username.set_text(info.url)
prof.set_text(str(info.pro_expiration))
#privacy.set_text(info['account']['default_album_privacy'])
info = self.il.get_credits()
credits.set_markup('<b>%s</b> credits left' % info['UserRemaining'])
authbut.set_sensitive(False)
else:
authbut.connect("clicked", self.authenticate)
userinfo.add_buttons(gtk.STOCK_CLOSE, gtk.RESPONSE_CLOSE)
userinfo.run()
userinfo.destroy()
def authenticate(self, widget=None, data=None):
authdialog = gtk.Dialog("Authenticate", None, gtk.DIALOG_MODAL|gtk.DIALOG_DESTROY_WITH_PARENT,
(gtk.STOCK_CANCEL, gtk.RESPONSE_REJECT,
gtk.STOCK_OK, gtk.RESPONSE_ACCEPT))
link = gtk.LinkButton(self.il.get_auth_url('pin'), "Click here...")
label = gtk.Label("Visit the following Link and paste the PIN code")
entry = gtk.Entry()
authdialog.vbox.pack_start(label)
authdialog.vbox.pack_start(link)
authdialog.vbox.pack_start(entry)
authdialog.show_all()
response = authdialog.run()
print response
if response == gtk.RESPONSE_ACCEPT:
try:
credentials = self.il.authorize(entry.get_text(), 'pin')
self.user_auth = 1
self.il.set_user_auth(credentials['access_token'], credentials['refresh_token'])
config['usertoken'] = credentials['access_token']
config['usersecret'] = credentials['refresh_token']
config.write()
except:
error_msg("The PIN was not correct")
authdialog.destroy()
authdialog.destroy()
def open_preferences(self, sender, data=None):
PrefsDialog(basepath, config)
def icon_clicked(self, sender, data=None):
if(self.windowstate == 0):
self.window.show_all()
self.windowstate = 1
else:
self.window.hide_on_delete()
self.windowstate = 0
return True
def right_click_event(self, icon, button, time):
menu = gtk.Menu()
about = gtk.ImageMenuItem(gtk.STOCK_ABOUT)
logview = gtk.ImageMenuItem()
logview.set_image(gtk.image_new_from_icon_name('emblem-photos', gtk.ICON_SIZE_MENU))
logview.set_label("Account Albums")
logview.connect("activate", self.show_albums)
quit = gtk.ImageMenuItem(gtk.STOCK_QUIT)
about.connect("activate", self.about_click)
quit.connect("activate", self.exit)
apimenu = gtk.ImageMenuItem(gtk.STOCK_PREFERENCES)
apimenu.set_label("Preferences")
apimenu.connect("activate", self.open_preferences)
shotmenu = gtk.ImageMenuItem(gtk.STOCK_FULLSCREEN)
shotmenu.set_label("Take Screenshot")
shotmenu.connect("activate", self.take_screenshot)
menu.append(about)
menu.append(logview)
menu.append(gtk.SeparatorMenuItem())
menu.append(shotmenu)
menu.append(gtk.SeparatorMenuItem())
menu.append(apimenu)
menu.append(gtk.SeparatorMenuItem())
menu.append(quit)
menu.show_all()
menu.popup(None, None, gtk.status_icon_position_menu,
button, time, self.statusicon)
