def __init__(self, queue, host, port, ssl, user, password):
super(Downloader, self).__init__(name='Downloader thread')
self._client = IMAP_client_factory(host, port, use_ssl=ssl)
self._client.login(user, password)
self.daemon = True
self._queue = queue
self.running = True
self._selected_folder = None
class Downloader(Thread):
def __init__(self, queue, host, port, ssl, user, password):
super(Downloader, self).__init__(name='Downloader thread')
self._client = IMAP_client_factory(host, port, use_ssl=ssl)
self._client.login(user, password)
self.daemon = True
self._queue = queue
self.running = True
self._selected_folder = None
def stop(self):
self.running = False
def select_folder(self, folder, uid_validity):
selected = self._client.select_folder(folder)
if selected[b'UIDVALIDITY'] != uid_validity:
raise UIDExpired('Message UID expired, restart')
self._selected_folder = folder
def run(self):
while self.running:
item = self._queue.get()
log.debug('Thread %s - got item %s', current_thread().name, item)
try:
folder = item['folder']
uid_validity = item['uid_validity']
if folder != self._selected_folder:
self.select_folder(folder, uid_validity)
kwargs = copy(item)
del kwargs['folder']
del kwargs['uid_validity']
del kwargs['retry']
kwargs['client'] = self._client
download(**kwargs)
except SOFT_ERRORS as e:
self._queue.task_done()
msgid = item['msgid']
log.warn('Download of message %d problem: %s', msgid, e)
if item['retry'] < 3:
item['retry'] += 1
self._queue.put(item)
else:
log.exception('Download of message %d failed: %s', msgid,
e)
except Exception as e:
self._queue.task_done()
log.exception('Download of message %d failed: %s',
item['msgid'], e)
else:
self._queue.task_done()
class Downloader(Thread):
def __init__(self, queue,host, port, ssl, user, password):
super(Downloader,self).__init__(name='Downloader thread')
self._client=IMAP_client_factory(host,port, use_ssl=ssl)
self._client.login(user, password)
self.daemon=True
self._queue=queue
self.running=True
self._selected_folder=None
def stop(self):
self.running=False
def select_folder(self,folder, uid_validity):
selected=self._client.select_folder(folder)
if selected[b'UIDVALIDITY'] != uid_validity:
raise UIDExpired('Message UID expired, restart')
self._selected_folder=folder
def run(self):
while self.running:
item=self._queue.get()
log.debug('Thread %s - got item %s', current_thread().name, item)
try:
folder=item['folder']
uid_validity = item['uid_validity']
if folder != self._selected_folder:
self.select_folder(folder, uid_validity)
kwargs=copy(item)
del kwargs['folder']
del kwargs['uid_validity']
del kwargs['retry']
kwargs['client']=self._client
download(**kwargs)
except SOFT_ERRORS as e:
self._queue.task_done()
msgid=item['msgid']
log.warn('Download of message %d problem: %s', msgid, e)
if item['retry'] < 3:
item['retry']+=1
self._queue.put(item)
else:
log.exception('Download of message %d failed: %s', msgid, e)
except Exception as e:
self._queue.task_done()
log.exception('Download of message %d failed: %s', item['msgid'], e)
else:
self._queue.task_done()
def __init__(self, queue,host, port, ssl, user, password):
super(Downloader,self).__init__(name='Downloader thread')
self._client=IMAP_client_factory(host,port, use_ssl=ssl)
self._client.login(user, password)
self.daemon=True
self._queue=queue
self.running=True
self._selected_folder=None
def main(opts):
host, port= split_host(opts.host, ssl=not opts.no_ssl)
ssl= not opts.no_ssl
if opts.insecure_ssl:
ssl='insecure'
log.debug('SSL status is %s', ssl)
# test filter parsing
eval_parser=SimpleEvaluator(DUMMY_INFO)
filter=opts.filter # @ReservedAssignment
try:
imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='string') if not opts.no_imap_search else None
_ = eval_parser.parse(filter)
except ParserSyntaxError as e:
msg = "Invalid syntax of filter: %s" %extract_err_msg(e)
log.error(msg)
# p(msg, file=sys.stderr)
sys.exit(1)
except ParserEvalError as e:
msg = "Invalid semantics of filter: %s" % extract_err_msg(e)
log.error(msg)
#p(msg, file=sys.stderr)
sys.exit(2)
charset=None
if imap_filter:
try:
imap_filter.encode('ascii')
except UnicodeEncodeError:
log.warn('Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers')
try:
imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='list')
[part.encode('utf-8') for part in imap_filter]
charset='UTF-8'
except UnicodeEncodeError as e:
log.error('Invalid characters in filter: %e',e)
sys.exit(3)
log.debug('IMAP filter: %s', imap_filter)
pool=None
try:
c=IMAP_client_factory(host,port,use_ssl=ssl)
if not c.has_capability('imap4rev1'):
log.error('This server is not compatible with IMAP4rev1!')
sys.exit(4)
try:
c.login(opts.user, opts.password)
if not opts.folder:
folders=['INBOX']
else:
available=c.list_folders()
folders= matched_folders(normalize_folders(available), opts.folder)
if not folders:
log.warn('No folders patching %s', opts.folder)
log.debug('Will process these folders: %s', folders)
if opts.move and not c.folder_exists(opts.move):
c.create_folder(opts.move)
if opts.threads:
pool=Pool(opts.threads, host, port, ssl, opts.user, opts.password)
for folder in folders:
log.info('Processing folder %s', folder)
process_folder(c, pool, folder, imap_filter, charset, eval_parser, opts)
if pool:
if opts.verbose:
p('Finished searching, now wait until all parts download in separate threads')
def report():
while True:
log.debug('Pool status - size %d, unfinished %d', pool._queue.qsize(), pool._queue.unfinished_tasks)
time.sleep(5)
if logging.getLogger().isEnabledFor(logging.DEBUG):
t=Thread(target=report, name='Debug reporting')
t.daemon=True
t.start()
pool.wait_finish()
log.debug('Pool is finished')
finally:
if opts.delete or opts.move:
try:
c.expunge() # last one shoyld be selected
for f in folders[:-1]:
c.select_folder(f)
c.expunge()
log.debug("Expunge folders")
except Exception:
log.exception('Expunge Failed!')
c.logout()
except Exception:
log.exception('Runtime Error')
sys.exit(4)
def main(opts):
host, port = split_host(opts.host, ssl=not opts.no_ssl)
ssl = not opts.no_ssl
if opts.insecure_ssl:
ssl = 'insecure'
log.debug('SSL status is %s', ssl)
# test filter parsing
eval_parser = SimpleEvaluator(DUMMY_INFO)
filter = opts.filter # @ReservedAssignment
try:
imap_filter = IMAPFilterGenerator(opts.unsafe_imap_search).parse(
filter, serialize='string') if not opts.no_imap_search else None
_ = eval_parser.parse(filter)
except ParserSyntaxError as e:
msg = "Invalid syntax of filter: %s" % extract_err_msg(e)
log.error(msg)
# p(msg, file=sys.stderr)
sys.exit(1)
except ParserEvalError as e:
msg = "Invalid semantics of filter: %s" % extract_err_msg(e)
log.error(msg)
#p(msg, file=sys.stderr)
sys.exit(2)
charset = None
if imap_filter:
try:
imap_filter.encode('ascii')
except UnicodeEncodeError:
log.warn(
'Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers'
)
try:
imap_filter = IMAPFilterGenerator(
opts.unsafe_imap_search).parse(filter, serialize='list')
[part.encode('utf-8') for part in imap_filter]
charset = 'UTF-8'
except UnicodeEncodeError as e:
log.error('Invalid characters in filter: %e', e)
sys.exit(3)
log.debug('IMAP filter: %s', imap_filter)
pool = None
try:
c = IMAP_client_factory(host, port, use_ssl=ssl)
if not c.has_capability('imap4rev1'):
log.error('This server is not compatible with IMAP4rev1!')
sys.exit(4)
try:
c.login(opts.user, opts.password)
if not opts.folder:
folders = ['INBOX']
else:
available = c.list_folders()
folders = matched_folders(normalize_folders(available),
opts.folder)
if not folders:
log.warn('No folders patching %s', opts.folder)
log.debug('Will process these folders: %s', folders)
if opts.move and not c.folder_exists(opts.move):
c.create_folder(opts.move)
if opts.threads:
pool = Pool(opts.threads, host, port, ssl, opts.user,
opts.password)
for folder in folders:
log.info('Processing folder %s', folder)
process_folder(c, pool, folder, imap_filter, charset,
eval_parser, opts)
if pool:
if opts.verbose:
p('Finished searching, now wait until all parts download in separate threads'
)
def report():
while True:
log.debug('Pool status - size %d, unfinished %d',
pool._queue.qsize(),
pool._queue.unfinished_tasks)
time.sleep(5)
if logging.getLogger().isEnabledFor(logging.DEBUG):
t = Thread(target=report, name='Debug reporting')
t.daemon = True
t.start()
pool.wait_finish()
log.debug('Pool is finished')
finally:
if opts.delete or opts.move:
try:
c.expunge() # last one shoyld be selected
for f in folders[:-1]:
c.select_folder(f)
c.expunge()
log.debug("Expunge folders")
except Exception:
log.exception('Expunge Failed!')
c.logout()
except Exception:
log.exception('Runtime Error')
sys.exit(4)
