def scan_mail(getmtime=getmtime):
Directory.createTable(ifNotExists=True)
Directory.createIndexes(ifNotExists=True)
File.createTable(ifNotExists=True)
File.createIndexes(ifNotExists=True)
root_maildirs = []
known_root_maildirs = Directory.selectBy(parent_dir=None)
for mdir in sorted(sources):
mdir = mdir.rstrip('/')
mdir_record = known_root_maildirs.filter(Directory.q.name == mdir)
if mdir_record.count():
root_maildirs.append(mdir_record.getOne())
continue
_last_modified = getmtime(mdir)-1
if root_maildirs:
try:
_find_mdir_source(mdir, fail=True)
except isChildNode:
continue
root_maildirs.append(Directory(name=mdir,
last_modified=_last_modified,
parent_dir=None))
for mdir in (m for m in known_root_maildirs if m not in root_maildirs):
mdir.destroySelf()
print('done getting maildirs')
#trans.commit()
return root_maildirs
def find_missing_and_new(maildirs):
"""
Scans maildirs looking for new and removed mail.
"""
started_at = datetime.now()
imaildirs = iter(maildirs)
stash = []
while 1:
try:
mdir = imaildirs.next()
if type(mdir) is SelectResults:
stash.append(imaildirs)
#maildirs.insert(0, imaildirs)
imaildirs = iter(mdir)
continue
except StopIteration:
try:
imaildirs = iter(stash.pop())
continue
except IndexError:
break
if mdir.has_changed:
mdir_contents = mdir.listdir()
while 1:
try:
item = mdir_contents.next()
itemfp = join(mdir.full_path, item)
except StopIteration:
missing_mail = mdir.files.filter(File.q.last_seen < \
started_at)
for mail in missing_mail:
#pub.sendMessage('nara.xindex.delete', docid=mail.uuid)
mail.destroySelf()
break
if isdir(itemfp):
try:
#item = Directory.selectBy(name=item,
# parent_dir=mdir).getOne()
item = mdir.directories.filter(Directory.q.name == item).getOne()
except SQLObjectNotFound:
item = Directory(name=item, parent_dir=mdir,
last_modified=getmtime(itemfp)-1)
maildirs.append(item)
else:
try:
item = mdir.files.filter(File.q.name == item).getOne()
item.update_seen()
except SQLObjectNotFound:
item = File(name=item, parent_dir=mdir)
#print('add to xapian - %r' % item)
#add to xapian
yield item
def update_maildir_cache(maildirs, colon=':'):
"""
Scans maildirs looking for new and removed mail.
"""
started_at = datetime.now()
imaildirs = iter(maildirs)
stash = []
substash = []
mdirmsg_separator = '%s2,' % colon
while 1:
try:
mdir = imaildirs.next()
if type(mdir) is SelectResults:
stash.append(imaildirs)
imaildirs = iter(mdir)
continue
except StopIteration:
try:
imaildirs = iter(stash.pop())
continue
except IndexError:
break
if mdir.has_changed:
mdir_contents = mdir.listdir()
seen_state = None
while 1:
try:
item = mdir_contents.next()
itemfp = join(mdir.full_path, item)
except StopIteration:
if substash:
mdir_contents = substash.pop()
continue
if seen_state is None:
if not mdir.files.count():
break
missing_mail = mdir.files
else:
missing_mail = mdir.files.filter(File.q.was_seen == \
(not seen_state))
for mail in missing_mail:
mail.destroySelf()
#trans.commit()
break
if isdir(itemfp):
if substash:
# if there's something in substash, that means we're in
# a cur or new folder, and there shouldn't be any
# folders under them. skip it
continue
if item in ('cur', 'new'):
# these aren't maildirs, so don't store them as Directory
# records. they are a component of a maildir though, and
# are where the files we want are located.
substash.append(mdir_contents)
mdir_contents = iter(listdir(itemfp))
continue
if item == 'tmp':
#files in the tmp folder shouldn't be there long
#don't bother
continue
try:
#item = Directory.selectBy(name=item,
# parent_dir=mdir).getOne()
item = mdir.directories.filter(Directory.q.name == item).getOne()
except SQLObjectNotFound:
item = Directory(name=item, parent_dir=mdir,
last_modified=getmtime(itemfp)-1)
maildirs.append(item)
elif substash:
if mdirmsg_separator not in item:
# not a properly formatted Maildir message filename.
# skip it.
continue
item = item.split(mdirmsg_separator)[0]
try:
item = mdir.files.filter(File.q.name == item).getOne()
seen_state = item.update_seen()
except SQLObjectNotFound:
if seen_state is None:
if mdir.files.count():
seen_state = not mdir.files.limit(1).getOne().was_seen
else:
seen_state = True
item = File(name=item, parent_dir=mdir,
was_seen=seen_state)