def sync_mail(self):
"""Iterates through all the mailboxes and scans if necessary."""
config = self.session.config
self._last_rescan_count = rescanned = errors = 0
self._last_rescan_completed = False
self._last_rescan_failed = False
self._interrupt = None
batch = min(self._loop_count * 20, self.RESCAN_BATCH_SIZE)
errors = rescanned = 0
all_completed = True
if not self._check_interrupt():
self._state = 'Waiting... (disco)'
discovered = self.discover_mailboxes()
else:
discovered = 0
ostate = self._state
plan = self._sorted_mailboxes()
self.event.data['plan'] = [[m._key, _('Pending'), m.name] for m in plan]
event_plan = dict((mp[0], mp) for mp in self.event.data['plan'])
if plan and random.randint(0, 10) == 1:
random_plan = [m._key for m in random.sample(plan, 1)]
else:
random_plan = []
for mbx_cfg in plan:
play_nice_with_threads(weak=True)
if self._check_interrupt(clear=False):
all_completed = False
break
try:
with self._lock:
mbx_key = FormatMbxId(mbx_cfg._key)
path = self._path(mbx_cfg)
policy = self._policy(mbx_cfg)
if (path in ('/dev/null', '', None)
or policy in ('ignore', 'unknown')):
event_plan[mbx_cfg._key][1] = _('Skipped')
continue
# Generally speaking, we only rescan if a mailbox looks like
# it has changed. However, every once in a while (see logic
# around random_mailboxes above) we check anyway just in case
# looks are deceiving.
state = {}
if batch < 1:
event_plan[mbx_cfg._key][1] = _('Postponed')
elif (self._has_mailbox_changed(mbx_cfg, state) or
mbx_cfg.local == '!CREATE' or
mbx_cfg._key in random_plan):
event_plan[mbx_cfg._key][1] = _('Working ...')
this_batch = max(5, int(0.7 * batch))
self._state = 'Waiting... (rescan)'
if self._check_interrupt(clear=False):
all_completed = False
break
count = self.rescan_mailbox(mbx_key, mbx_cfg, path,
stop_after=this_batch)
if count >= 0:
self.event.data['counters'
]['indexed_messages'] += count
batch -= count
this_batch -= count
complete = ((count == 0 or this_batch > 0) and
not self._interrupt and
not mailpile.util.QUITTING)
if complete:
rescanned += 1
# If there was a copy, check if it completed
cstate = self.event.data.get('copying') or {}
if not cstate.get('complete', True):
complete = False
# If there was a rescan, check if it completed
rstate = self.event.data.get('rescan') or {}
if not rstate.get('complete', True):
complete = False
# OK, everything looks complete, mark it!
if complete:
event_plan[mbx_cfg._key][1] = _('Completed')
self._mark_mailbox_rescanned(mbx_cfg, state)
else:
event_plan[mbx_cfg._key][1] = _('Indexed %d'
) % count
all_completed = False
if count == 0 and ('sources' in config.sys.debug):
time.sleep(60)
else:
event_plan[mbx_cfg._key][1] = _('Failed')
self._last_rescan_failed = True
all_completed = False
errors += 1
else:
event_plan[mbx_cfg._key][1] = _('Unchanged')
except (NoSuchMailboxError, IOError, OSError) as e:
event_plan[mbx_cfg._key][1] = '%s: %s' % (_('Error'), e)
self._last_rescan_failed = True
errors += 1
except Exception as e:
event_plan[mbx_cfg._key][1] = '%s: %s' % (
_('Internal error'), e)
self._last_rescan_failed = True
self._log_status(_('Internal error'))
raise
self._last_rescan_completed = all_completed
self._state = 'Done'
status = []
if discovered > 0:
status.append(_('Discovered %d mailboxes') % discovered)
self._last_rescan_completed = False
if rescanned > 0:
status.append(_('Processed %d mailboxes') % rescanned)
if errors:
status.append(_('Failed to process %d') % errors)
if not status:
status.append(_('No new mail at %s'
) % datetime.datetime.today().strftime('%H:%M'))
self._log_status(', '.join(status))
self._last_rescan_count = rescanned
self._state = ostate
return rescanned
def _get_mbx_id_and_mfn(self, mbx_cfg):
mbx_id = FormatMbxId(mbx_cfg._key)
return mbx_id, self.session.config.sys.mailbox[mbx_id]
def sync_mail(self):
"""Iterates through all the mailboxes and scans if necessary."""
config = self.session.config
self._last_rescan_count = rescanned = errors = 0
self._last_rescan_completed = True
self._last_rescan_failed = False
self._interrupt = None
batch = self.RESCAN_BATCH_SIZE
errors = rescanned = 0
ostate = self._state
for mbx_cfg in self._sorted_mailboxes():
try:
with self._lock:
mbx_key = FormatMbxId(mbx_cfg._key)
path = self._path(mbx_cfg)
if (path in ('/dev/null', '', None)
or mbx_cfg.policy in ('ignore', 'unknown')):
continue
# Generally speaking, we only rescan if a mailbox looks like
# it has changed. However, 1/50th of the time we take a look
# anyway just in case looks are deceiving.
state = {}
if batch > 0 and (self._has_mailbox_changed(mbx_cfg, state) or
random.randint(0, 50) == 10):
self._state = 'Waiting... (rescan)'
if self._check_interrupt(clear=False):
self._last_rescan_completed = False
break
count = self.rescan_mailbox(mbx_key, mbx_cfg, path,
stop_after=batch)
if count >= 0:
self.event.data['counters'
]['indexed_messages'] += count
batch -= count
complete = ((count == 0 or batch > 0) and
not self._interrupt and
not mailpile.util.QUITTING)
if complete:
rescanned += 1
# If there was a copy, check if it completed
if not self.event.data.get('copying',
{'complete': True}
).get('complete'):
complete = False
# If there was a rescan, check if it completed
if not self.event.data.get('rescan',
{'complete': True}
).get('complete'):
complete = False
# OK, everything looks complete, mark it!
if complete:
self._mark_mailbox_rescanned(mbx_cfg, state)
else:
self._last_rescan_completed = False
else:
self._last_rescan_failed = True
self._last_rescan_completed = False
errors += 1
except (NoSuchMailboxError, IOError, OSError):
self._last_rescan_failed = True
errors += 1
except:
self._last_rescan_failed = True
self._log_status(_('Internal error'))
raise
self._state = 'Waiting... (disco)'
discovered = 0
if not self._check_interrupt():
discovered = self.discover_mailboxes()
status = []
if discovered > 0:
status.append(_('Discovered %d mailboxes') % discovered)
if discovered < 1 or rescanned > 0:
status.append(_('Rescanned %d mailboxes') % rescanned)
if errors:
status.append(_('Failed to rescan %d') % errors)
self._log_status(', '.join(status))
self._last_rescan_count = rescanned
self._state = ostate
return rescanned
def open_mailbox(self, mbx_id, mfn):
if FormatMbxId(mbx_id) in self.my_config.mailbox:
proto_me, path = mfn.split('/', 1)
if proto_me.startswith('src:'):
return SharedImapMailbox(self.session, self, mailbox_path=path)
return False
def _has_mailbox_changed(self, mbx, state):
pop3 = self.session.config.open_mailbox(self.session,
FormatMbxId(mbx._key),
prefer_local=False)
state['stat'] = stat = '%s' % (pop3.stat(), )
return (self.event.data.get('mailbox_state', {}).get(mbx._key) != stat)