def __init__(self, queue, host, port, ssl, user, password): super(Downloader, self).__init__(name='Downloader thread') self._client = IMAP_client_factory(host, port, use_ssl=ssl) self._client.login(user, password) self.daemon = True self._queue = queue self.running = True self._selected_folder = None
class Downloader(Thread): def __init__(self, queue, host, port, ssl, user, password): super(Downloader, self).__init__(name='Downloader thread') self._client = IMAP_client_factory(host, port, use_ssl=ssl) self._client.login(user, password) self.daemon = True self._queue = queue self.running = True self._selected_folder = None def stop(self): self.running = False def select_folder(self, folder, uid_validity): selected = self._client.select_folder(folder) if selected[b'UIDVALIDITY'] != uid_validity: raise UIDExpired('Message UID expired, restart') self._selected_folder = folder def run(self): while self.running: item = self._queue.get() log.debug('Thread %s - got item %s', current_thread().name, item) try: folder = item['folder'] uid_validity = item['uid_validity'] if folder != self._selected_folder: self.select_folder(folder, uid_validity) kwargs = copy(item) del kwargs['folder'] del kwargs['uid_validity'] del kwargs['retry'] kwargs['client'] = self._client download(**kwargs) except SOFT_ERRORS as e: self._queue.task_done() msgid = item['msgid'] log.warn('Download of message %d problem: %s', msgid, e) if item['retry'] < 3: item['retry'] += 1 self._queue.put(item) else: log.exception('Download of message %d failed: %s', msgid, e) except Exception as e: self._queue.task_done() log.exception('Download of message %d failed: %s', item['msgid'], e) else: self._queue.task_done()
class Downloader(Thread): def __init__(self, queue,host, port, ssl, user, password): super(Downloader,self).__init__(name='Downloader thread') self._client=IMAP_client_factory(host,port, use_ssl=ssl) self._client.login(user, password) self.daemon=True self._queue=queue self.running=True self._selected_folder=None def stop(self): self.running=False def select_folder(self,folder, uid_validity): selected=self._client.select_folder(folder) if selected[b'UIDVALIDITY'] != uid_validity: raise UIDExpired('Message UID expired, restart') self._selected_folder=folder def run(self): while self.running: item=self._queue.get() log.debug('Thread %s - got item %s', current_thread().name, item) try: folder=item['folder'] uid_validity = item['uid_validity'] if folder != self._selected_folder: self.select_folder(folder, uid_validity) kwargs=copy(item) del kwargs['folder'] del kwargs['uid_validity'] del kwargs['retry'] kwargs['client']=self._client download(**kwargs) except SOFT_ERRORS as e: self._queue.task_done() msgid=item['msgid'] log.warn('Download of message %d problem: %s', msgid, e) if item['retry'] < 3: item['retry']+=1 self._queue.put(item) else: log.exception('Download of message %d failed: %s', msgid, e) except Exception as e: self._queue.task_done() log.exception('Download of message %d failed: %s', item['msgid'], e) else: self._queue.task_done()
def __init__(self, queue,host, port, ssl, user, password): super(Downloader,self).__init__(name='Downloader thread') self._client=IMAP_client_factory(host,port, use_ssl=ssl) self._client.login(user, password) self.daemon=True self._queue=queue self.running=True self._selected_folder=None
def main(opts): host, port= split_host(opts.host, ssl=not opts.no_ssl) ssl= not opts.no_ssl if opts.insecure_ssl: ssl='insecure' log.debug('SSL status is %s', ssl) # test filter parsing eval_parser=SimpleEvaluator(DUMMY_INFO) filter=opts.filter # @ReservedAssignment try: imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='string') if not opts.no_imap_search else None _ = eval_parser.parse(filter) except ParserSyntaxError as e: msg = "Invalid syntax of filter: %s" %extract_err_msg(e) log.error(msg) # p(msg, file=sys.stderr) sys.exit(1) except ParserEvalError as e: msg = "Invalid semantics of filter: %s" % extract_err_msg(e) log.error(msg) #p(msg, file=sys.stderr) sys.exit(2) charset=None if imap_filter: try: imap_filter.encode('ascii') except UnicodeEncodeError: log.warn('Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers') try: imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='list') [part.encode('utf-8') for part in imap_filter] charset='UTF-8' except UnicodeEncodeError as e: log.error('Invalid characters in filter: %e',e) sys.exit(3) log.debug('IMAP filter: %s', imap_filter) pool=None try: c=IMAP_client_factory(host,port,use_ssl=ssl) if not c.has_capability('imap4rev1'): log.error('This server is not compatible with IMAP4rev1!') sys.exit(4) try: c.login(opts.user, opts.password) if not opts.folder: folders=['INBOX'] else: available=c.list_folders() folders= matched_folders(normalize_folders(available), opts.folder) if not folders: log.warn('No folders patching %s', opts.folder) log.debug('Will process these folders: %s', folders) if opts.move and not c.folder_exists(opts.move): c.create_folder(opts.move) if opts.threads: pool=Pool(opts.threads, host, port, ssl, opts.user, opts.password) for folder in folders: log.info('Processing folder %s', folder) process_folder(c, pool, folder, imap_filter, charset, eval_parser, opts) if pool: if opts.verbose: p('Finished searching, now wait until all parts download in separate threads') def report(): while True: log.debug('Pool status - size %d, unfinished %d', pool._queue.qsize(), pool._queue.unfinished_tasks) time.sleep(5) if logging.getLogger().isEnabledFor(logging.DEBUG): t=Thread(target=report, name='Debug reporting') t.daemon=True t.start() pool.wait_finish() log.debug('Pool is finished') finally: if opts.delete or opts.move: try: c.expunge() # last one shoyld be selected for f in folders[:-1]: c.select_folder(f) c.expunge() log.debug("Expunge folders") except Exception: log.exception('Expunge Failed!') c.logout() except Exception: log.exception('Runtime Error') sys.exit(4)
def main(opts): host, port = split_host(opts.host, ssl=not opts.no_ssl) ssl = not opts.no_ssl if opts.insecure_ssl: ssl = 'insecure' log.debug('SSL status is %s', ssl) # test filter parsing eval_parser = SimpleEvaluator(DUMMY_INFO) filter = opts.filter # @ReservedAssignment try: imap_filter = IMAPFilterGenerator(opts.unsafe_imap_search).parse( filter, serialize='string') if not opts.no_imap_search else None _ = eval_parser.parse(filter) except ParserSyntaxError as e: msg = "Invalid syntax of filter: %s" % extract_err_msg(e) log.error(msg) # p(msg, file=sys.stderr) sys.exit(1) except ParserEvalError as e: msg = "Invalid semantics of filter: %s" % extract_err_msg(e) log.error(msg) #p(msg, file=sys.stderr) sys.exit(2) charset = None if imap_filter: try: imap_filter.encode('ascii') except UnicodeEncodeError: log.warn( 'Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers' ) try: imap_filter = IMAPFilterGenerator( opts.unsafe_imap_search).parse(filter, serialize='list') [part.encode('utf-8') for part in imap_filter] charset = 'UTF-8' except UnicodeEncodeError as e: log.error('Invalid characters in filter: %e', e) sys.exit(3) log.debug('IMAP filter: %s', imap_filter) pool = None try: c = IMAP_client_factory(host, port, use_ssl=ssl) if not c.has_capability('imap4rev1'): log.error('This server is not compatible with IMAP4rev1!') sys.exit(4) try: c.login(opts.user, opts.password) if not opts.folder: folders = ['INBOX'] else: available = c.list_folders() folders = matched_folders(normalize_folders(available), opts.folder) if not folders: log.warn('No folders patching %s', opts.folder) log.debug('Will process these folders: %s', folders) if opts.move and not c.folder_exists(opts.move): c.create_folder(opts.move) if opts.threads: pool = Pool(opts.threads, host, port, ssl, opts.user, opts.password) for folder in folders: log.info('Processing folder %s', folder) process_folder(c, pool, folder, imap_filter, charset, eval_parser, opts) if pool: if opts.verbose: p('Finished searching, now wait until all parts download in separate threads' ) def report(): while True: log.debug('Pool status - size %d, unfinished %d', pool._queue.qsize(), pool._queue.unfinished_tasks) time.sleep(5) if logging.getLogger().isEnabledFor(logging.DEBUG): t = Thread(target=report, name='Debug reporting') t.daemon = True t.start() pool.wait_finish() log.debug('Pool is finished') finally: if opts.delete or opts.move: try: c.expunge() # last one shoyld be selected for f in folders[:-1]: c.select_folder(f) c.expunge() log.debug("Expunge folders") except Exception: log.exception('Expunge Failed!') c.logout() except Exception: log.exception('Runtime Error') sys.exit(4)