def test_imap(self):
g=IMAPFilterGenerator()
p=SimpleEvaluator( CONTEXT)
t1='to ~= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
self.assertTrue(p.parse(t1))
t1='to ^= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
t1='to $= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
t2='mime = "image/png"'
self.assertEqual(g.parse(t2), '')
self.assertTrue(p.parse(t2))
t3='to = "myself"'
self.assertEqual(g.parse(t3), 'TO "myself"')
self.assertFalse(p.parse(t3))
t4='to ~= "myself" & subject ~= "test"'
self.assertEqual(g.parse(t4), '(TO "myself" SUBJECT "test")')
self.assertTrue(p.parse(t4))
t5='to ~= "myself" | subject ~= "test"'
self.assertEqual(g.parse(t5), '(OR TO "myself" SUBJECT "test")')
self.assertTrue(p.parse(t5))
t6='to ~= "myself" & mime ~= "image"'
self.assertEqual(g.parse(t6), 'TO "myself"')
self.assertTrue(p.parse(t6))
t7='mime~="image" & subject ~= "test"'
self.assertEqual(g.parse(t7), 'SUBJECT "test"')
self.assertTrue(p.parse(t7))
def test_text(self):
g=IMAPFilterGenerator(unsafe=True)
ctx={'mime':'application/pdf', 'name':'soubor.pdf'}
p=SimpleEvaluator(ctx)
t='mime="application/pdf" & name ~= "soubor"'
self.assertEqual(g.parse(t), '(TEXT "application/pdf" TEXT "soubor")')
self.assertTrue(p.parse(t))
t='mime^="application" & name $= ".pdf"'
self.assertEqual(g.parse(t), '(TEXT "application" TEXT ".pdf")')
self.assertTrue(p.parse(t))
def test_text(self):
g = IMAPFilterGenerator(unsafe=True)
ctx = {'mime': 'application/pdf', 'name': 'soubor.pdf'}
p = SimpleEvaluator(ctx)
t = 'mime="application/pdf" & name ~= "soubor"'
self.assertEqual(g.parse(t), '(TEXT "application/pdf" TEXT "soubor")')
self.assertTrue(p.parse(t))
t = 'mime^="application" & name $= ".pdf"'
self.assertEqual(g.parse(t), '(TEXT "application" TEXT ".pdf")')
self.assertTrue(p.parse(t))
def test_custom_flags(self):
g = IMAPFilterGenerator(unsafe=True)
ctx = {'flags': TagList(['$Forwarded', 'NotJunk'])}
p = SimpleEvaluator(ctx)
t = 'flags = "$Forwarded"'
self.assertEqual(g.parse(t), 'KEYWORD "$Forwarded"')
self.assertTrue(p.parse(t))
t = 'flags ^= "$Forwarded"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
def test_custom_flags(self):
g=IMAPFilterGenerator(unsafe=True)
ctx={'flags':TagList(['$Forwarded', 'NotJunk'])}
p=SimpleEvaluator(ctx)
t='flags = "$Forwarded"'
self.assertEqual(g.parse(t), 'KEYWORD "$Forwarded"')
self.assertTrue(p.parse(t))
t='flags ^= "$Forwarded"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
def test_notimap(self):
g=IMAPFilterGenerator()
ctx={'mime':'text/plain', 'date': datetime(2015,10,13), 'year':2015, 'subject':'mako','section':'1.1'}
p=SimpleEvaluator(ctx)
t='(section="1" | section~="1.")'
self.assertEqual(g.parse(t), '')
self.assertEqual(g.parse(t, serialize='list'), [])
self.assertTrue(p.parse(t))
t='mime="text/plain" & year=2015 & ! subject^="re:" & (section="1" | section~="1.")'
self.assertEqual(g.parse(t), '(SINCE 1-Jan-2015 BEFORE 1-Jan-2016) (NOT SUBJECT "re:")')
self.assertEqual(g.parse(t, serialize='list'), ['(','SINCE', '1-Jan-2015', 'BEFORE', '1-Jan-2016', ')', '(', 'NOT', 'SUBJECT', "re:", ')'])
self.assertTrue(p.parse(t))
def test_date(self):
g = IMAPFilterGenerator()
ctx = {'date': datetime(2015, 11, 15)}
p = SimpleEvaluator(ctx)
t = "date = 2015-11-15"
self.assertEqual(g.parse(t), "ON 15-Nov-2015")
self.assertTrue(p.parse(t))
t = "date < 2015-11-14"
self.assertEqual(g.parse(t), 'BEFORE 14-Nov-2015')
self.assertFalse(p.parse(t))
t = "date > 2015-11-14"
self.assertEqual(g.parse(t),
'(OR SINCE 14-Nov-2015 NOT (ON 14-Nov-2015))')
self.assertTrue(p.parse(t))
t = "date <= 2015-11-15"
self.assertEqual(g.parse(t), '(OR BEFORE 15-Nov-2015 ON 15-Nov-2015)')
self.assertTrue(p.parse(t))
t = "date >= 2015-11-15"
self.assertEqual(g.parse(t), 'SINCE 15-Nov-2015')
self.assertTrue(p.parse(t))
p = SimpleEvaluator({'date': datetime(2015, 11, 15, 15, 10)})
t = "date <= 2015-11-15"
self.assertEqual(g.parse(t), '(OR BEFORE 15-Nov-2015 ON 15-Nov-2015)')
self.assertTrue(p.parse(t))
def test_imap2(self):
g=IMAPFilterGenerator()
p=SimpleEvaluator( CONTEXT)
t8='to ~= "myself" | mime ~= "image"'
self.assertEqual(g.parse(t8), '')
self.assertTrue(p.parse(t8))
t9='mime~="image" | subject ~= "test"'
self.assertEqual(g.parse(t9), '')
self.assertTrue(p.parse(t9))
t10='! subject ~= "test"'
self.assertEqual(g.parse(t10), '(NOT SUBJECT "test")')
self.assertFalse(p.parse(t10))
def test_imap4(self):
g=IMAPFilterGenerator()
p=SimpleEvaluator( CONTEXT)
t='subject~= "test" & mime ~= "image" | to ~= "myself"'
self.assertEqual(g.parse(t), '(OR SUBJECT "test" TO "myself")')
self.assertTrue(p.parse(t))
t='subject~= "test" | mime ~= "image" & to ~= "myself"'
self.assertEqual(g.parse(t), '(OR SUBJECT "test" TO "myself")')
self.assertTrue(p.parse(t))
t='subject~="test" & seen | to~="myself" & ! recent'
self.assertEqual( g.parse(t), '(OR (SUBJECT "test" SEEN) (TO "myself" (NOT RECENT)))')
self.assertTrue(p.parse(t))
def test_flags(self):
g = IMAPFilterGenerator()
ctx = {
'recent': True,
'seen': True,
'answered': True,
'flagged': True,
'deleted': True,
'draft': True,
}
p = SimpleEvaluator(ctx)
t = "recent & seen & answered & flagged & deleted & draft"
self.assertEqual(g.parse(t),
"(RECENT SEEN ANSWERED FLAGGED DELETED DRAFT)")
self.assertTrue(p.parse(t))
def test_flags(self):
g=IMAPFilterGenerator()
ctx={
'recent': True,
'seen': True,
'answered' : True,
'flagged': True,
'deleted': True,
'draft' : True,
}
p=SimpleEvaluator(ctx)
t="recent & seen & answered & flagged & deleted & draft"
self.assertEqual(g.parse(t), "(RECENT SEEN ANSWERED FLAGGED DELETED DRAFT)")
self.assertTrue(p.parse(t))
def test_list(self):
g=IMAPFilterGenerator()
t1='to ~= "myself"'
self.assertEqual(g.parse(t1, serialize='list'), ['TO', "myself"])
t10='! subject ~= "test"'
self.assertEqual(g.parse(t10, serialize='list'), ['(','NOT', 'SUBJECT', "test",')'])
t="date <= 2015-11-15"
self.assertEqual(g.parse(t, serialize='list'), ['(','OR', 'BEFORE', '15-Nov-2015', 'ON', '15-Nov-2015', ')'])
t='(! subject ~= "test" | from ~= "example.com") & ( ! recent | seen)'
self.assertEqual(g.parse(t, serialize = 'list'),['(','(','(','OR', '(' ,'NOT', 'SUBJECT', "test",')', 'FROM', "example.com",')',')', '(','(','OR', '(', 'NOT', 'RECENT',')', 'SEEN',')',')',')'])
def test_imap2(self):
g = IMAPFilterGenerator()
p = SimpleEvaluator(CONTEXT)
t8 = 'to ~= "myself" | mime ~= "image"'
self.assertEqual(g.parse(t8), '')
self.assertTrue(p.parse(t8))
t9 = 'mime~="image" | subject ~= "test"'
self.assertEqual(g.parse(t9), '')
self.assertTrue(p.parse(t9))
t10 = '! subject ~= "test"'
self.assertEqual(g.parse(t10), '(NOT SUBJECT "test")')
self.assertFalse(p.parse(t10))
def test_imap3(self):
g=IMAPFilterGenerator()
p=SimpleEvaluator( CONTEXT)
t='(! subject ~= "test" | from ~= "example.com") & ( ! recent | seen)'
self.assertEqual(g.parse(t),'(((OR (NOT SUBJECT "test") FROM "example.com")) ((OR (NOT RECENT) SEEN)))')
self.assertTrue(p.parse(t))
t='subject ~= "test" & from ~= "example.com" & to ~= "myself"'
self.assertEqual(g.parse(t), '(SUBJECT "test" FROM "example.com" TO "myself")')
self.assertTrue(p.parse(t))
t='subject ~= "test" | from ~= "example.com" | to ~= "myself"'
self.assertEqual(g.parse(t), '(OR SUBJECT "test" (OR FROM "example.com" TO "myself"))')
self.assertTrue(p.parse(t))
def test_imap4(self):
g = IMAPFilterGenerator()
p = SimpleEvaluator(CONTEXT)
t = 'subject~= "test" & mime ~= "image" | to ~= "myself"'
self.assertEqual(g.parse(t), '(OR SUBJECT "test" TO "myself")')
self.assertTrue(p.parse(t))
t = 'subject~= "test" | mime ~= "image" & to ~= "myself"'
self.assertEqual(g.parse(t), '(OR SUBJECT "test" TO "myself")')
self.assertTrue(p.parse(t))
t = 'subject~="test" & seen | to~="myself" & ! recent'
self.assertEqual(
g.parse(t),
'(OR (SUBJECT "test" SEEN) (TO "myself" (NOT RECENT)))')
self.assertTrue(p.parse(t))
def test_date(self):
g=IMAPFilterGenerator()
ctx={'date' : datetime(2015,11,15)}
p=SimpleEvaluator(ctx)
t="date = 2015-11-15"
self.assertEqual(g.parse(t), "ON 15-Nov-2015")
self.assertTrue(p.parse(t))
t="date < 2015-11-14"
self.assertEqual(g.parse(t), 'BEFORE 14-Nov-2015')
self.assertFalse(p.parse(t))
t="date > 2015-11-14"
self.assertEqual(g.parse(t), '(OR SINCE 14-Nov-2015 NOT (ON 14-Nov-2015))')
self.assertTrue(p.parse(t))
t="date <= 2015-11-15"
self.assertEqual(g.parse(t), '(OR BEFORE 15-Nov-2015 ON 15-Nov-2015)')
self.assertTrue(p.parse(t))
t="date >= 2015-11-15"
self.assertEqual(g.parse(t), 'SINCE 15-Nov-2015')
self.assertTrue(p.parse(t))
p=SimpleEvaluator({'date' : datetime(2015,11,15,15,10)})
t="date <= 2015-11-15"
self.assertEqual(g.parse(t), '(OR BEFORE 15-Nov-2015 ON 15-Nov-2015)')
self.assertTrue(p.parse(t))
def test_or(self):
g = IMAPFilterGenerator()
ctx = {
'mime': 'application/pdf',
'size': 200 * 1024,
'from': '*****@*****.**',
'cc': '',
'subject': 'mako',
'section': '1.1'
}
p = SimpleEvaluator(ctx)
t = 'from~="test" | mime="application/pdf"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t = 'from~="test" | (mime="application/pdf")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t = '(mime="application/pdf") | from~="test"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t = '(mime="application/pdf") | (from~="test")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t = '(mime="application/pdf" & ! from ~= "siska" & ! cc~="nekdo" & size>100k & size<1M) | (mime="image/png")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
def test_size(self):
g=IMAPFilterGenerator()
ctx={'size' : 5 * 1024}
p=SimpleEvaluator(ctx)
t="size > 4k"
self.assertEqual(g.parse(t), "LARGER %d"% (4*1024))
self.assertTrue(p.parse(t))
t="size >= 4k"
self.assertEqual(g.parse(t), "LARGER %d"% (4*1024))
self.assertTrue(p.parse(t))
t="size < 4k"
self.assertEqual(g.parse(t), '')
self.assertFalse(p.parse(t))
t="size <= 4k"
self.assertEqual(g.parse(t), '')
self.assertFalse(p.parse(t))
def test_imap3(self):
g = IMAPFilterGenerator()
p = SimpleEvaluator(CONTEXT)
t = '(! subject ~= "test" | from ~= "example.com") & ( ! recent | seen)'
self.assertEqual(
g.parse(t),
'(((OR (NOT SUBJECT "test") FROM "example.com")) ((OR (NOT RECENT) SEEN)))'
)
self.assertTrue(p.parse(t))
t = 'subject ~= "test" & from ~= "example.com" & to ~= "myself"'
self.assertEqual(g.parse(t),
'(SUBJECT "test" FROM "example.com" TO "myself")')
self.assertTrue(p.parse(t))
t = 'subject ~= "test" | from ~= "example.com" | to ~= "myself"'
self.assertEqual(
g.parse(t),
'(OR SUBJECT "test" (OR FROM "example.com" TO "myself"))')
self.assertTrue(p.parse(t))
def test_size(self):
g = IMAPFilterGenerator()
ctx = {'size': 5 * 1024}
p = SimpleEvaluator(ctx)
t = "size > 4k"
self.assertEqual(g.parse(t), "LARGER %d" % (4 * 1024))
self.assertTrue(p.parse(t))
t = "size >= 4k"
self.assertEqual(g.parse(t), "LARGER %d" % (4 * 1024))
self.assertTrue(p.parse(t))
t = "size < 4k"
self.assertEqual(g.parse(t), '')
self.assertFalse(p.parse(t))
t = "size <= 4k"
self.assertEqual(g.parse(t), '')
self.assertFalse(p.parse(t))
def test_list(self):
g = IMAPFilterGenerator()
t1 = 'to ~= "myself"'
self.assertEqual(g.parse(t1, serialize='list'), ['TO', "myself"])
t10 = '! subject ~= "test"'
self.assertEqual(g.parse(t10, serialize='list'),
['(', 'NOT', 'SUBJECT', "test", ')'])
t = "date <= 2015-11-15"
self.assertEqual(
g.parse(t, serialize='list'),
['(', 'OR', 'BEFORE', '15-Nov-2015', 'ON', '15-Nov-2015', ')'])
t = '(! subject ~= "test" | from ~= "example.com") & ( ! recent | seen)'
self.assertEqual(g.parse(t, serialize='list'), [
'(', '(', '(', 'OR', '(', 'NOT', 'SUBJECT', "test", ')', 'FROM',
"example.com", ')', ')', '(', '(', 'OR', '(', 'NOT', 'RECENT', ')',
'SEEN', ')', ')', ')'
])
def test_year(self):
g = IMAPFilterGenerator(unsafe=True)
ctx = {'year': 2015}
p = SimpleEvaluator(ctx)
t = 'year=2015'
self.assertEqual(g.parse(t), '(SINCE 1-Jan-2015 BEFORE 1-Jan-2016)')
self.assertTrue(p.parse(t))
t = 'year>2015'
self.assertEqual(g.parse(t), 'SINCE 1-Jan-2016')
self.assertFalse(p.parse(t))
t = 'year>=2015'
self.assertEqual(g.parse(t), 'SINCE 1-Jan-2015')
self.assertTrue(p.parse(t))
t = 'year<2016'
self.assertEqual(g.parse(t), 'BEFORE 1-Jan-2016')
self.assertTrue(p.parse(t))
t = 'year<=2016'
self.assertEqual(g.parse(t), 'BEFORE 1-Jan-2017')
self.assertTrue(p.parse(t))
def test_or(self):
g=IMAPFilterGenerator()
ctx={'mime':'application/pdf', 'size': 200*1024, 'from':'*****@*****.**', 'cc':'','subject':'mako','section':'1.1'}
p=SimpleEvaluator(ctx)
t='from~="test" | mime="application/pdf"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t='from~="test" | (mime="application/pdf")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t='(mime="application/pdf") | from~="test"'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t='(mime="application/pdf") | (from~="test")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
t='(mime="application/pdf" & ! from ~= "siska" & ! cc~="nekdo" & size>100k & size<1M) | (mime="image/png")'
self.assertEqual(g.parse(t), '')
self.assertTrue(p.parse(t))
def test_notimap(self):
g = IMAPFilterGenerator()
ctx = {
'mime': 'text/plain',
'date': datetime(2015, 10, 13),
'year': 2015,
'subject': 'mako',
'section': '1.1'
}
p = SimpleEvaluator(ctx)
t = '(section="1" | section~="1.")'
self.assertEqual(g.parse(t), '')
self.assertEqual(g.parse(t, serialize='list'), [])
self.assertTrue(p.parse(t))
t = 'mime="text/plain" & year=2015 & ! subject^="re:" & (section="1" | section~="1.")'
self.assertEqual(
g.parse(t),
'(SINCE 1-Jan-2015 BEFORE 1-Jan-2016) (NOT SUBJECT "re:")')
self.assertEqual(g.parse(t, serialize='list'), [
'(', 'SINCE', '1-Jan-2015', 'BEFORE', '1-Jan-2016', ')', '(',
'NOT', 'SUBJECT', "re:", ')'
])
self.assertTrue(p.parse(t))
def test_year(self):
g=IMAPFilterGenerator(unsafe=True)
ctx={'year':2015}
p=SimpleEvaluator(ctx)
t='year=2015'
self.assertEqual(g.parse(t), '(SINCE 1-Jan-2015 BEFORE 1-Jan-2016)')
self.assertTrue(p.parse(t))
t='year>2015'
self.assertEqual(g.parse(t), 'SINCE 1-Jan-2016')
self.assertFalse(p.parse(t))
t='year>=2015'
self.assertEqual(g.parse(t), 'SINCE 1-Jan-2015')
self.assertTrue(p.parse(t))
t='year<2016'
self.assertEqual(g.parse(t), 'BEFORE 1-Jan-2016')
self.assertTrue(p.parse(t))
t='year<=2016'
self.assertEqual(g.parse(t), 'BEFORE 1-Jan-2017')
self.assertTrue(p.parse(t))
def test_imap(self):
g = IMAPFilterGenerator()
p = SimpleEvaluator(CONTEXT)
t1 = 'to ~= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
self.assertTrue(p.parse(t1))
t1 = 'to ^= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
t1 = 'to $= "myself"'
self.assertEqual(g.parse(t1), 'TO "myself"')
t2 = 'mime = "image/png"'
self.assertEqual(g.parse(t2), '')
self.assertTrue(p.parse(t2))
t3 = 'to = "myself"'
self.assertEqual(g.parse(t3), 'TO "myself"')
self.assertFalse(p.parse(t3))
t4 = 'to ~= "myself" & subject ~= "test"'
self.assertEqual(g.parse(t4), '(TO "myself" SUBJECT "test")')
self.assertTrue(p.parse(t4))
t5 = 'to ~= "myself" | subject ~= "test"'
self.assertEqual(g.parse(t5), '(OR TO "myself" SUBJECT "test")')
self.assertTrue(p.parse(t5))
t6 = 'to ~= "myself" & mime ~= "image"'
self.assertEqual(g.parse(t6), 'TO "myself"')
self.assertTrue(p.parse(t6))
t7 = 'mime~="image" & subject ~= "test"'
self.assertEqual(g.parse(t7), 'SUBJECT "test"')
self.assertTrue(p.parse(t7))
def main(opts):
host, port= split_host(opts.host, ssl=not opts.no_ssl)
ssl= not opts.no_ssl
if opts.insecure_ssl:
ssl='insecure'
log.debug('SSL status is %s', ssl)
# test filter parsing
eval_parser=SimpleEvaluator(DUMMY_INFO)
filter=opts.filter # @ReservedAssignment
try:
imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='string') if not opts.no_imap_search else None
_ = eval_parser.parse(filter)
except ParserSyntaxError as e:
msg = "Invalid syntax of filter: %s" %extract_err_msg(e)
log.error(msg)
# p(msg, file=sys.stderr)
sys.exit(1)
except ParserEvalError as e:
msg = "Invalid semantics of filter: %s" % extract_err_msg(e)
log.error(msg)
#p(msg, file=sys.stderr)
sys.exit(2)
charset=None
if imap_filter:
try:
imap_filter.encode('ascii')
except UnicodeEncodeError:
log.warn('Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers')
try:
imap_filter=IMAPFilterGenerator(opts.unsafe_imap_search).parse(filter, serialize='list')
[part.encode('utf-8') for part in imap_filter]
charset='UTF-8'
except UnicodeEncodeError as e:
log.error('Invalid characters in filter: %e',e)
sys.exit(3)
log.debug('IMAP filter: %s', imap_filter)
pool=None
try:
c=IMAP_client_factory(host,port,use_ssl=ssl)
if not c.has_capability('imap4rev1'):
log.error('This server is not compatible with IMAP4rev1!')
sys.exit(4)
try:
c.login(opts.user, opts.password)
if not opts.folder:
folders=['INBOX']
else:
available=c.list_folders()
folders= matched_folders(normalize_folders(available), opts.folder)
if not folders:
log.warn('No folders patching %s', opts.folder)
log.debug('Will process these folders: %s', folders)
if opts.move and not c.folder_exists(opts.move):
c.create_folder(opts.move)
if opts.threads:
pool=Pool(opts.threads, host, port, ssl, opts.user, opts.password)
for folder in folders:
log.info('Processing folder %s', folder)
process_folder(c, pool, folder, imap_filter, charset, eval_parser, opts)
if pool:
if opts.verbose:
p('Finished searching, now wait until all parts download in separate threads')
def report():
while True:
log.debug('Pool status - size %d, unfinished %d', pool._queue.qsize(), pool._queue.unfinished_tasks)
time.sleep(5)
if logging.getLogger().isEnabledFor(logging.DEBUG):
t=Thread(target=report, name='Debug reporting')
t.daemon=True
t.start()
pool.wait_finish()
log.debug('Pool is finished')
finally:
if opts.delete or opts.move:
try:
c.expunge() # last one shoyld be selected
for f in folders[:-1]:
c.select_folder(f)
c.expunge()
log.debug("Expunge folders")
except Exception:
log.exception('Expunge Failed!')
c.logout()
except Exception:
log.exception('Runtime Error')
sys.exit(4)
def main(opts):
host, port = split_host(opts.host, ssl=not opts.no_ssl)
ssl = not opts.no_ssl
if opts.insecure_ssl:
ssl = 'insecure'
log.debug('SSL status is %s', ssl)
# test filter parsing
eval_parser = SimpleEvaluator(DUMMY_INFO)
filter = opts.filter # @ReservedAssignment
try:
imap_filter = IMAPFilterGenerator(opts.unsafe_imap_search).parse(
filter, serialize='string') if not opts.no_imap_search else None
_ = eval_parser.parse(filter)
except ParserSyntaxError as e:
msg = "Invalid syntax of filter: %s" % extract_err_msg(e)
log.error(msg)
# p(msg, file=sys.stderr)
sys.exit(1)
except ParserEvalError as e:
msg = "Invalid semantics of filter: %s" % extract_err_msg(e)
log.error(msg)
#p(msg, file=sys.stderr)
sys.exit(2)
charset = None
if imap_filter:
try:
imap_filter.encode('ascii')
except UnicodeEncodeError:
log.warn(
'Your search contains non-ascii characters, will try UTF-8, but it may not work on some servers'
)
try:
imap_filter = IMAPFilterGenerator(
opts.unsafe_imap_search).parse(filter, serialize='list')
[part.encode('utf-8') for part in imap_filter]
charset = 'UTF-8'
except UnicodeEncodeError as e:
log.error('Invalid characters in filter: %e', e)
sys.exit(3)
log.debug('IMAP filter: %s', imap_filter)
pool = None
try:
c = IMAP_client_factory(host, port, use_ssl=ssl)
if not c.has_capability('imap4rev1'):
log.error('This server is not compatible with IMAP4rev1!')
sys.exit(4)
try:
c.login(opts.user, opts.password)
if not opts.folder:
folders = ['INBOX']
else:
available = c.list_folders()
folders = matched_folders(normalize_folders(available),
opts.folder)
if not folders:
log.warn('No folders patching %s', opts.folder)
log.debug('Will process these folders: %s', folders)
if opts.move and not c.folder_exists(opts.move):
c.create_folder(opts.move)
if opts.threads:
pool = Pool(opts.threads, host, port, ssl, opts.user,
opts.password)
for folder in folders:
log.info('Processing folder %s', folder)
process_folder(c, pool, folder, imap_filter, charset,
eval_parser, opts)
if pool:
if opts.verbose:
p('Finished searching, now wait until all parts download in separate threads'
)
def report():
while True:
log.debug('Pool status - size %d, unfinished %d',
pool._queue.qsize(),
pool._queue.unfinished_tasks)
time.sleep(5)
if logging.getLogger().isEnabledFor(logging.DEBUG):
t = Thread(target=report, name='Debug reporting')
t.daemon = True
t.start()
pool.wait_finish()
log.debug('Pool is finished')
finally:
if opts.delete or opts.move:
try:
c.expunge() # last one shoyld be selected
for f in folders[:-1]:
c.select_folder(f)
c.expunge()
log.debug("Expunge folders")
except Exception:
log.exception('Expunge Failed!')
c.logout()
except Exception:
log.exception('Runtime Error')
sys.exit(4)
