def __init__(self):

        os.system('cls')
        self.failed_downvotes = []  # Store failed downvotes for later processing
        self.failed_comments = []  # Store failed comments for later processing
        self.delay_between_requests = 5  # Changed on the fly depending on remaining credits and time until reset
        self.thread_lock = threading.Lock()
        self.logger = None
        self.detected_reposts = 0


        self.config = ConfigManager()
        self._setup_logging()

        self.imgur_client = ImgurClient(self.config.api_details['client_id'],
                                        self.config.api_details['client_secret'],
                                        self.config.api_details['access_token'],
                                        self.config.api_details['refresh_token'])

        self.db_conn = ImgurRepostDB(self.config)

        self.backfill_progress = 1 if self.config.backfill else 'Disabled'

        records, processed_ids = self.db_conn.build_existing_ids()

        if self.config.backfill:
            threading.Thread(target=self._backfill_database, name='Backfill').start()

        self.hash_processing = HashProcessing(self.config, processed_ids, records)

        threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
class ImgurRepostBot():

    def __init__(self):

        os.system('cls')
        self.failed_downvotes = []  # Store failed downvotes for later processing
        self.failed_comments = []  # Store failed comments for later processing
        self.delay_between_requests = 5  # Changed on the fly depending on remaining credits and time until reset
        self.thread_lock = threading.Lock()
        self.logger = None
        self.detected_reposts = 0


        self.config = ConfigManager()
        self._setup_logging()

        self.imgur_client = ImgurClient(self.config.api_details['client_id'],
                                        self.config.api_details['client_secret'],
                                        self.config.api_details['access_token'],
                                        self.config.api_details['refresh_token'])

        self.db_conn = ImgurRepostDB(self.config)

        self.backfill_progress = 1 if self.config.backfill else 'Disabled'

        records, processed_ids = self.db_conn.build_existing_ids()

        if self.config.backfill:
            threading.Thread(target=self._backfill_database, name='Backfill').start()

        self.hash_processing = HashProcessing(self.config, processed_ids, records)

        threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()



    def _check_thread_status(self):
        """
        Check status of critical threads.  If they are found dead start them back up
        :return:
        """
        thread_names = ['configmonitor', 'backfill', 'repostprocessing']

        for thrd in threading.enumerate():
            if thrd.name.lower() in thread_names:
                thread_names.remove(thrd.name.lower())

        for i in thread_names:

            if i == 'configmonitor':
                msg = 'Config Monitor Thread Crashed'
                self._output_error(msg)
                self.config = ConfigManager()
                continue

            if i == 'backfill' and self.config.backfill:
                msg = 'Backfill Thread Crashed'
                self._output_error(msg)
                threading.Thread(target=self._backfill_database, name='Backfill').start()
                continue

            if i == 'repostprocessing':
                msg = 'Repost Processing Thread Crashed'
                self._output_error(msg)
                threading.Thread(target=self._repost_processing_thread, name='RepostProcessing').start()
                continue

    def _setup_logging(self):

        if self.config.logging:
            self.logger = logging.getLogger(__name__)
            self.logger.setLevel(logging.ERROR)
            formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
            fhandle = logging.FileHandler('botlog.log')
            fhandle.setFormatter(formatter)
            self.logger.addHandler(fhandle)

    def _output_error(self, msg, output=True):
        """
        convenience method to log and/or print an error
        :param msg: Message to output/log
        :param output: Print error to console
        :return:
        """
        if output:
            print(msg)
        if self.config.logging:
            self.logger.error(msg)

    def _output_info(self, msg):
        print(msg)
        if self.config.logging:
            self.logger.info(msg)

    def _backfill_database(self):
        """
        Backfill the database with older posts.  Useful if script hasn't been run in some time"
        :return:
        """

        while True:
            if self.config.backfill:
                original_start_page = self.config.backfill_start_page # So we can detect if it's changed in the config
                current_page = self.config.backfill_start_page
                while current_page < self.config.backfill_depth + self.config.backfill_start_page:

                    if not self.db_conn.records_loaded:
                        continue

                    self.backfill_progress = current_page
                    self.insert_latest_images(page=current_page, backfill=True)
                    current_page += 1
                    time.sleep(self.delay_between_requests)

                    if self.config.backfill_start_page != original_start_page:
                        print('Backfill Start Page Changed In Config')
                        break

                self.backfill_progress = 'Completed'
                break
            else:
                self.backfill_progress = 'Disabled'
                time.sleep(5)

    def _generate_img(self, url=None):
        """
        Generate the image files provided from Imgur.  We pass the data straight from the request into PIL.Image
        """

        img = None
        if not url:
            return None

        try:
            response = request.urlopen(url)
            img = Image.open(BytesIO(response.read()))
        except (HTTPError, ConnectionError, OSError) as e:
            msg = 'Error Generating Image File: \n Error Message: {}'.format(e)
            self._output_error(msg)

            return None

        return img if img else None

    def generate_latest_images(self, section='user', sort='time', page=0):

        self._adjust_rate_limit_timing()

        items = []
        try:
            temp = self.imgur_client.gallery(section=section, sort=sort, page=page, show_viral=False)
            if temp:
                items = [i for i in temp if not i.is_album and not self.check_post_title(title=i.title)]
        except (ImgurClientError, ImgurClientRateLimitError) as e:
            msg = 'Error Getting Gallery: {}'.format(e)
            self._output_error(msg)

        return items

    def insert_latest_images(self, section='user', sort='time', page=0, backfill=False):
        """
        Pull all current images from user sub, get the hashes and insert into database.
        """

        # Don't start inserts until all records are loaded
        if not self.db_conn.records_loaded:
            return

        items = self.generate_latest_images(section=section, sort=sort, page=page)

        if not items:
            return

        # Don't add again if we have already done this image ID
        for item in items:
            if item.id in self.hash_processing.processed_ids:
                continue

            img = self._generate_img(url=item.link)
            if img:
                image_hash = self.hash_processing.generate_hash(img)
                if image_hash:

                    record = {
                        'image_id': item.id,
                        'url': item.link,
                        'gallery_url': 'https://imgur.com/gallery/{}'.format(item.id),
                        'user': item.account_url,
                        'submitted': item.datetime,
                        'hash16': image_hash['hash16'],
                        'hash64': image_hash['hash64'],
                        'hash256': image_hash['hash256']
                    }

                    self.hash_processing.processed_ids.append(item.id)
                    self.hash_processing.records.append(record)

                    # If this is called from back filling don't add hash to be checked
                    if not backfill:
                        self.hash_processing.hash_queue.append(record)
                        print('Processing {}'.format(item.link))
                    else:
                        print('Backfill Insert {}'.format(item.link))

                    self.db_conn.add_entry(record)

    def downvote_repost(self, image_id):
        """
        Downvote the provided Image ID
        """
        try:
            self.imgur_client.gallery_item_vote(image_id, vote="down")
        except ImgurClientError as e:
            self.failed_downvotes.append(image_id)
            msg = 'Error Voting: {}'.format(e)
            self._output_error(msg)

    def comment_repost(self, image_id=None, values=None):
        """
        Leave a comment on the detected repost.
        :param image_id: ID of image to leave comment on.
        :param values: Values to be inserted into the message template
        :return:
        """

        self._output_info('Leaving Comment On {}'.format(image_id))
        message = self.build_comment_message(values=values)

        if not message:
            return

        try:
            self.imgur_client.gallery_comment(image_id, message)
        except (ImgurClientError, ImgurClientRateLimitError) as e:
            self.failed_comments.append({'image_id': image_id, 'values': values})
            msg = 'Error Posting Commment: {}'.format(e)
            self._output_error(msg)


    def build_comment_message(self, values=None):

        if not values:
            return None

        # Build up replacement dict
        out_dict = {
            'count': len(values),
            'g_url': values[0]['gallery_url'],
            'd_url': values[0]['url'],
            'submitted_epoch': values[0]['submitted'],
            'submitted_human': time.strftime("%m/%d/%Y, %H:%M:%S", time.localtime(values[0]['submitted'])),
            'user': values[0]['user'],
        }

        try:
            final_message = self.config.comment_template.format(**out_dict)
            print('Final Message: ' + final_message)
            if len(final_message) > 140:
                self.logger.warning('Message Length Is Over 140 Chars.  Will Be Trimmed')
            return final_message
        except KeyError as e:
            msg = 'Error Generating Message: {}'.format(e)
            self._output_error(msg)
            return None


    def flush_failed_votes_and_comments(self):
        """
        If there have been any failed votes or comments (due to imgur server overload) try to redo them
        """

        if self.failed_downvotes:
            for image_id in self.failed_downvotes:
                try:
                    self.imgur_client.gallery_item_vote(image_id, vote="down")
                    self.failed_downvotes.remove(image_id)
                except (ImgurClientError, ImgurClientRateLimitError) as e:
                    msg = 'Failed To Retry Downvote On Image {}.  \nError: {}'.format(image_id, e)
                    self._output_error(msg)

        if self.failed_comments:
            for failed in self.failed_comments:
                try:
                    message = self.build_comment_message(values=failed['values'])
                    self.imgur_client.gallery_comment(failed['image_id'], message)
                    self.failed_comments.remove(failed['image_id'])
                except (ImgurClientError, ImgurClientRateLimitError) as e:
                    msg = 'Failed To Retry Comment On Image {}.  \nError: {}'.format(failed['image_id'], e)
                    self._output_error(msg)

    def _repost_processing_thread(self):
        """
        Runs in background monitor the queue for detected reposts
        :return:
        """
        while True:
            if len(self.hash_processing.repost_queue) > 0:
                current_repost = self.hash_processing.repost_queue.pop(0)
                image_id = current_repost[0]['image_id']
                sorted_reposts = sorted(current_repost[0]['older_images'], key=itemgetter('submitted'))

                if self.config.leave_downvote:
                    self.downvote_repost(image_id)

                if self.config.leave_comment:
                    self.comment_repost(image_id, values=sorted_reposts)

                self.detected_reposts += 1

                if self.config.log_reposts:
                    with open('repost.log', 'a+') as f:
                        f.write('Repost Image: https://imgur.com/gallery/{}\n'.format(image_id))
                        f.write('Matching Images:\n')
                        for r in sorted_reposts:
                            f.write(r['gallery_url'] + '\n')

    def _adjust_rate_limit_timing(self):
        """
        Adjust the timing used between request to spread all requests over allowed rate limit

        """

        # API Fails To Return This At Times
        if not self.imgur_client.credits['ClientRemaining']:
            return

        remaining_credits_before = int(self.imgur_client.credits['ClientRemaining'])
        self.imgur_client.credits = self.imgur_client.get_credits()  # Refresh the credit data

        # Imgur API sometimes returns 12500 credits remaining in error.  If this happens don't update request delay.
        # Otherwise the delay will drop to the minimum set in the config and can cause premature credit exhaustion
        if int(self.imgur_client.credits['ClientRemaining']) - remaining_credits_before > 100:
            """
            print('Imgur API Returned Wrong Remaining Credits.  Keeping Last Request Delay Time')
            print('API Credits: ' + str(self.imgur_client.credits['ClientRemaining']))
            print('Last Credits: ' + str(remaining_credits_before))
            """
            return

        remaining_credits = self.imgur_client.credits['ClientRemaining']
        reset_time = self.imgur_client.credits['UserReset'] + 240  # Add a 4 minute buffer so we don't cut it so close
        remaining_seconds = reset_time - round(time.time())
        seconds_per_credit = round(remaining_seconds / remaining_credits)  # TODO Getting division by zero sometimes

        if seconds_per_credit < self.config.min_time_between_requests:
            self.delay_between_requests = self.config.min_time_between_requests
        else:
            self.delay_between_requests = seconds_per_credit

    def check_post_title(self, title=None):
        """
        Checks the post title for values that we will use to skip over it
        This allows us not to flag MRW posts and others as reposts
        :return:
        """

        if not title:
            return None

        return [v for v in self.config.title_check_values if v in title.lower()]

    def print_current_settings(self):
        print('Current Settings')
        print('[+] Leave Comments: {}'.format('Enabled' if self.config.leave_comment else 'Disabled'))
        print('[+] Leave Downvote: {} '.format('Enabled' if self.config.leave_downvote else 'Disabled'))
        print('[+] Backfill: {} '.format('Enabled' if self.config.backfill else 'Disabled'))
        print('[+] Backfill Depth: {} '.format(self.config.backfill_depth if self.config.backfill else 'Disabled'))
        print('[+] Process Pool Size: {} '.format(self.config.hash_proc_limit))
        print('[+] Hash Size: {} bit'.format(self.config.hash_size))
        print('[+] Hamming Distance: {}{}'.format(self.config.hamming_cutoff, '\n'))

    def print_current_stats(self):
        print('Current Stats')
        print('[+] Total Images In Database: {}'.format(str(len(self.hash_processing.processed_ids))))
        print('[+] Total Hashes Waiting In Pool: {}'.format(str(self.hash_processing.total_in_queue)))
        print('[+] Total Hashes In Hash Queue: {}'.format(str(len(self.hash_processing.hash_queue))))
        print('[+] Process Pool Status: {}'.format(self.hash_processing.pool_status))
        print('[+] Total Reposts Found: {}'.format(str(self.detected_reposts)))
        print('[+] Backfill Progress: {}\n'.format('Page ' + str(self.backfill_progress) if self.config.backfill else 'Disabled'))


    def print_api_stats(self):
        print('API Settings')
        print('[+] Remaining Credits: {}'.format(self.imgur_client.credits['ClientRemaining']))
        if self.imgur_client.credits['UserReset']:
            print('[+] Time Until Credit Reset: {} Minutes'.format(round((int(self.imgur_client.credits['UserReset']) - time.time()) / 60)))

        # Make it clear we are overriding the default delay to meet credit refill window
        if self.delay_between_requests == self.config.min_time_between_requests:
            request_delay = str(self.delay_between_requests) + ' Seconds'
        else:
            request_delay = str(self.delay_between_requests) + ' Seconds (Overridden By Rate Limit)'
        print('[+] Delay Between Requests: {} \n'.format(request_delay))

    def run(self):

        last_run = round(time.time())

        while True:

            os.system('cls')

            self.print_current_stats()
            self.print_current_settings()
            self.print_api_stats()

            if round(time.time()) - last_run > self.delay_between_requests:
                self.insert_latest_images()
                self.flush_failed_votes_and_comments()
                last_run = round(time.time())

            self._check_thread_status()

            time.sleep(2)