def __init__(self):
os.system('cls')
self.failed_downvotes = [] # Store failed downvotes for later processing
self.failed_comments = [] # Store failed comments for later processing
self.delay_between_requests = 5 # Changed on the fly depending on remaining credits and time until reset
self.thread_lock = threading.Lock()
self.logger = None
self.detected_reposts = 0
self.config = ConfigManager()
self._setup_logging()
self.imgur_client = ImgurClient(self.config.api_details['client_id'],
self.config.api_details['client_secret'],
self.config.api_details['access_token'],
self.config.api_details['refresh_token'])
self.db_conn = ImgurRepostDB(self.config)
self.backfill_progress = 1 if self.config.backfill else 'Disabled'
records, processed_ids = self.db_conn.build_existing_ids()
if self.config.backfill:
threading.Thread(target=self._backfill_database, name='Backfill').start()
self.hash_processing = HashProcessing(self.config, processed_ids, records)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
class ImgurRepostBot():
def __init__(self):
os.system('cls')
self.failed_downvotes = [] # Store failed downvotes for later processing
self.failed_comments = [] # Store failed comments for later processing
self.delay_between_requests = 5 # Changed on the fly depending on remaining credits and time until reset
self.thread_lock = threading.Lock()
self.logger = None
self.detected_reposts = 0
self.config = ConfigManager()
self._setup_logging()
self.imgur_client = ImgurClient(self.config.api_details['client_id'],
self.config.api_details['client_secret'],
self.config.api_details['access_token'],
self.config.api_details['refresh_token'])
self.db_conn = ImgurRepostDB(self.config)
self.backfill_progress = 1 if self.config.backfill else 'Disabled'
records, processed_ids = self.db_conn.build_existing_ids()
if self.config.backfill:
threading.Thread(target=self._backfill_database, name='Backfill').start()
self.hash_processing = HashProcessing(self.config, processed_ids, records)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
def _check_thread_status(self):
"""
Check status of critical threads. If they are found dead start them back up
:return:
"""
thread_names = ['configmonitor', 'backfill', 'repostprocessing']
for thrd in threading.enumerate():
if thrd.name.lower() in thread_names:
thread_names.remove(thrd.name.lower())
for i in thread_names:
if i == 'configmonitor':
msg = 'Config Monitor Thread Crashed'
self._output_error(msg)
self.config = ConfigManager()
continue
if i == 'backfill' and self.config.backfill:
msg = 'Backfill Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._backfill_database, name='Backfill').start()
continue
if i == 'repostprocessing':
msg = 'Repost Processing Thread Crashed'
self._output_error(msg)
threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
continue
def _setup_logging(self):
if self.config.logging:
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
fhandle = logging.FileHandler('botlog.log')
fhandle.setFormatter(formatter)
self.logger.addHandler(fhandle)
def _output_error(self, msg, output=True):
"""
convenience method to log and/or print an error
:param msg: Message to output/log
:param output: Print error to console
:return:
"""
if output:
print(msg)
if self.config.logging:
self.logger.error(msg)
def _output_info(self, msg):
print(msg)
if self.config.logging:
self.logger.info(msg)
def _backfill_database(self):
"""
Backfill the database with older posts. Useful if script hasn't been run in some time"
:return:
"""
while True:
if self.config.backfill:
original_start_page = self.config.backfill_start_page # So we can detect if it's changed in the config
current_page = self.config.backfill_start_page
while current_page < self.config.backfill_depth + self.config.backfill_start_page:
if not self.db_conn.records_loaded:
continue
self.backfill_progress = current_page
self.insert_latest_images(page=current_page, backfill=True)
current_page += 1
time.sleep(self.delay_between_requests)
if self.config.backfill_start_page != original_start_page:
print('Backfill Start Page Changed In Config')
break
self.backfill_progress = 'Completed'
break
else:
self.backfill_progress = 'Disabled'
time.sleep(5)
def _generate_img(self, url=None):
"""
Generate the image files provided from Imgur. We pass the data straight from the request into PIL.Image
"""
img = None
if not url:
return None
try:
response = request.urlopen(url)
img = Image.open(BytesIO(response.read()))
except (HTTPError, ConnectionError, OSError) as e:
msg = 'Error Generating Image File: \n Error Message: {}'.format(e)
self._output_error(msg)
return None
return img if img else None
def generate_latest_images(self, section='user', sort='time', page=0):
self._adjust_rate_limit_timing()
items = []
try:
temp = self.imgur_client.gallery(section=section, sort=sort, page=page, show_viral=False)
if temp:
items = [i for i in temp if not i.is_album and not self.check_post_title(title=i.title)]
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Error Getting Gallery: {}'.format(e)
self._output_error(msg)
return items
def insert_latest_images(self, section='user', sort='time', page=0, backfill=False):
"""
Pull all current images from user sub, get the hashes and insert into database.
"""
# Don't start inserts until all records are loaded
if not self.db_conn.records_loaded:
return
items = self.generate_latest_images(section=section, sort=sort, page=page)
if not items:
return
# Don't add again if we have already done this image ID
for item in items:
if item.id in self.hash_processing.processed_ids:
continue
img = self._generate_img(url=item.link)
if img:
image_hash = self.hash_processing.generate_hash(img)
if image_hash:
record = {
'image_id': item.id,
'url': item.link,
'gallery_url': 'https://imgur.com/gallery/{}'.format(item.id),
'user': item.account_url,
'submitted': item.datetime,
'hash16': image_hash['hash16'],
'hash64': image_hash['hash64'],
'hash256': image_hash['hash256']
}
self.hash_processing.processed_ids.append(item.id)
self.hash_processing.records.append(record)
# If this is called from back filling don't add hash to be checked
if not backfill:
self.hash_processing.hash_queue.append(record)
print('Processing {}'.format(item.link))
else:
print('Backfill Insert {}'.format(item.link))
self.db_conn.add_entry(record)
def downvote_repost(self, image_id):
"""
Downvote the provided Image ID
"""
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
except ImgurClientError as e:
self.failed_downvotes.append(image_id)
msg = 'Error Voting: {}'.format(e)
self._output_error(msg)
def comment_repost(self, image_id=None, values=None):
"""
Leave a comment on the detected repost.
:param image_id: ID of image to leave comment on.
:param values: Values to be inserted into the message template
:return:
"""
self._output_info('Leaving Comment On {}'.format(image_id))
message = self.build_comment_message(values=values)
if not message:
return
try:
self.imgur_client.gallery_comment(image_id, message)
except (ImgurClientError, ImgurClientRateLimitError) as e:
self.failed_comments.append({'image_id': image_id, 'values': values})
msg = 'Error Posting Commment: {}'.format(e)
self._output_error(msg)
def build_comment_message(self, values=None):
if not values:
return None
# Build up replacement dict
out_dict = {
'count': len(values),
'g_url': values[0]['gallery_url'],
'd_url': values[0]['url'],
'submitted_epoch': values[0]['submitted'],
'submitted_human': time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime(values[0]['submitted'])),
'user': values[0]['user'],
}
try:
final_message = self.config.comment_template.format(**out_dict)
print('Final Message: ' + final_message)
if len(final_message) > 140:
self.logger.warning('Message Length Is Over 140 Chars. Will Be Trimmed')
return final_message
except KeyError as e:
msg = 'Error Generating Message: {}'.format(e)
self._output_error(msg)
return None
def flush_failed_votes_and_comments(self):
"""
If there have been any failed votes or comments (due to imgur server overload) try to redo them
"""
if self.failed_downvotes:
for image_id in self.failed_downvotes:
try:
self.imgur_client.gallery_item_vote(image_id, vote="down")
self.failed_downvotes.remove(image_id)
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Downvote On Image {}. \nError: {}'.format(image_id, e)
self._output_error(msg)
if self.failed_comments:
for failed in self.failed_comments:
try:
message = self.build_comment_message(values=failed['values'])
self.imgur_client.gallery_comment(failed['image_id'], message)
self.failed_comments.remove(failed['image_id'])
except (ImgurClientError, ImgurClientRateLimitError) as e:
msg = 'Failed To Retry Comment On Image {}. \nError: {}'.format(failed['image_id'], e)
self._output_error(msg)
def _repost_processing_thread(self):
"""
Runs in background monitor the queue for detected reposts
:return:
"""
while True:
if len(self.hash_processing.repost_queue) > 0:
current_repost = self.hash_processing.repost_queue.pop(0)
image_id = current_repost[0]['image_id']
sorted_reposts = sorted(current_repost[0]['older_images'], key=itemgetter('submitted'))
if self.config.leave_downvote:
self.downvote_repost(image_id)
if self.config.leave_comment:
self.comment_repost(image_id, values=sorted_reposts)
self.detected_reposts += 1
if self.config.log_reposts:
with open('repost.log', 'a+') as f:
f.write('Repost Image: https://imgur.com/gallery/{}\n'.format(image_id))
f.write('Matching Images:\n')
for r in sorted_reposts:
f.write(r['gallery_url'] + '\n')
def _adjust_rate_limit_timing(self):
"""
Adjust the timing used between request to spread all requests over allowed rate limit
"""
# API Fails To Return This At Times
if not self.imgur_client.credits['ClientRemaining']:
return
remaining_credits_before = int(self.imgur_client.credits['ClientRemaining'])
self.imgur_client.credits = self.imgur_client.get_credits() # Refresh the credit data
# Imgur API sometimes returns 12500 credits remaining in error. If this happens don't update request delay.
# Otherwise the delay will drop to the minimum set in the config and can cause premature credit exhaustion
if int(self.imgur_client.credits['ClientRemaining']) - remaining_credits_before > 100:
"""
print('Imgur API Returned Wrong Remaining Credits. Keeping Last Request Delay Time')
print('API Credits: ' + str(self.imgur_client.credits['ClientRemaining']))
print('Last Credits: ' + str(remaining_credits_before))
"""
return
remaining_credits = self.imgur_client.credits['ClientRemaining']
reset_time = self.imgur_client.credits['UserReset'] + 240 # Add a 4 minute buffer so we don't cut it so close
remaining_seconds = reset_time - round(time.time())
seconds_per_credit = round(remaining_seconds / remaining_credits) # TODO Getting division by zero sometimes
if seconds_per_credit < self.config.min_time_between_requests:
self.delay_between_requests = self.config.min_time_between_requests
else:
self.delay_between_requests = seconds_per_credit
def check_post_title(self, title=None):
"""
Checks the post title for values that we will use to skip over it
This allows us not to flag MRW posts and others as reposts
:return:
"""
if not title:
return None
return [v for v in self.config.title_check_values if v in title.lower()]
def print_current_settings(self):
print('Current Settings')
print('[+] Leave Comments: {}'.format('Enabled' if self.config.leave_comment else 'Disabled'))
print('[+] Leave Downvote: {} '.format('Enabled' if self.config.leave_downvote else 'Disabled'))
print('[+] Backfill: {} '.format('Enabled' if self.config.backfill else 'Disabled'))
print('[+] Backfill Depth: {} '.format(self.config.backfill_depth if self.config.backfill else 'Disabled'))
print('[+] Process Pool Size: {} '.format(self.config.hash_proc_limit))
print('[+] Hash Size: {} bit'.format(self.config.hash_size))
print('[+] Hamming Distance: {}{}'.format(self.config.hamming_cutoff, '\n'))
def print_current_stats(self):
print('Current Stats')
print('[+] Total Images In Database: {}'.format(str(len(self.hash_processing.processed_ids))))
print('[+] Total Hashes Waiting In Pool: {}'.format(str(self.hash_processing.total_in_queue)))
print('[+] Total Hashes In Hash Queue: {}'.format(str(len(self.hash_processing.hash_queue))))
print('[+] Process Pool Status: {}'.format(self.hash_processing.pool_status))
print('[+] Total Reposts Found: {}'.format(str(self.detected_reposts)))
print('[+] Backfill Progress: {}\n'.format('Page ' + str(self.backfill_progress) if self.config.backfill else 'Disabled'))
def print_api_stats(self):
print('API Settings')
print('[+] Remaining Credits: {}'.format(self.imgur_client.credits['ClientRemaining']))
if self.imgur_client.credits['UserReset']:
print('[+] Time Until Credit Reset: {} Minutes'.format(round((int(self.imgur_client.credits['UserReset']) - time.time()) / 60)))
# Make it clear we are overriding the default delay to meet credit refill window
if self.delay_between_requests == self.config.min_time_between_requests:
request_delay = str(self.delay_between_requests) + ' Seconds'
else:
request_delay = str(self.delay_between_requests) + ' Seconds (Overridden By Rate Limit)'
print('[+] Delay Between Requests: {} \n'.format(request_delay))
def run(self):
last_run = round(time.time())
while True:
os.system('cls')
self.print_current_stats()
self.print_current_settings()
self.print_api_stats()
if round(time.time()) - last_run > self.delay_between_requests:
self.insert_latest_images()
self.flush_failed_votes_and_comments()
last_run = round(time.time())
self._check_thread_status()
time.sleep(2)