def sync_mail(self): """Iterates through all the mailboxes and scans if necessary.""" config = self.session.config self._last_rescan_count = rescanned = errors = 0 self._last_rescan_completed = False self._last_rescan_failed = False self._interrupt = None batch = min(self._loop_count * 20, self.RESCAN_BATCH_SIZE) errors = rescanned = 0 all_completed = True if not self._check_interrupt(): self._state = 'Waiting... (disco)' discovered = self.discover_mailboxes() else: discovered = 0 ostate = self._state plan = self._sorted_mailboxes() self.event.data['plan'] = [[m._key, _('Pending'), m.name] for m in plan] event_plan = dict((mp[0], mp) for mp in self.event.data['plan']) if plan and random.randint(0, 10) == 1: random_plan = [m._key for m in random.sample(plan, 1)] else: random_plan = [] for mbx_cfg in plan: play_nice_with_threads(weak=True) if self._check_interrupt(clear=False): all_completed = False break try: with self._lock: mbx_key = FormatMbxId(mbx_cfg._key) path = self._path(mbx_cfg) policy = self._policy(mbx_cfg) if (path in ('/dev/null', '', None) or policy in ('ignore', 'unknown')): event_plan[mbx_cfg._key][1] = _('Skipped') continue # Generally speaking, we only rescan if a mailbox looks like # it has changed. However, every once in a while (see logic # around random_mailboxes above) we check anyway just in case # looks are deceiving. state = {} if batch < 1: event_plan[mbx_cfg._key][1] = _('Postponed') elif (self._has_mailbox_changed(mbx_cfg, state) or mbx_cfg.local == '!CREATE' or mbx_cfg._key in random_plan): event_plan[mbx_cfg._key][1] = _('Working ...') this_batch = max(5, int(0.7 * batch)) self._state = 'Waiting... (rescan)' if self._check_interrupt(clear=False): all_completed = False break count = self.rescan_mailbox(mbx_key, mbx_cfg, path, stop_after=this_batch) if count >= 0: self.event.data['counters' ]['indexed_messages'] += count batch -= count this_batch -= count complete = ((count == 0 or this_batch > 0) and not self._interrupt and not mailpile.util.QUITTING) if complete: rescanned += 1 # If there was a copy, check if it completed cstate = self.event.data.get('copying') or {} if not cstate.get('complete', True): complete = False # If there was a rescan, check if it completed rstate = self.event.data.get('rescan') or {} if not rstate.get('complete', True): complete = False # OK, everything looks complete, mark it! if complete: event_plan[mbx_cfg._key][1] = _('Completed') self._mark_mailbox_rescanned(mbx_cfg, state) else: event_plan[mbx_cfg._key][1] = _('Indexed %d' ) % count all_completed = False if count == 0 and ('sources' in config.sys.debug): time.sleep(60) else: event_plan[mbx_cfg._key][1] = _('Failed') self._last_rescan_failed = True all_completed = False errors += 1 else: event_plan[mbx_cfg._key][1] = _('Unchanged') except (NoSuchMailboxError, IOError, OSError) as e: event_plan[mbx_cfg._key][1] = '%s: %s' % (_('Error'), e) self._last_rescan_failed = True errors += 1 except Exception as e: event_plan[mbx_cfg._key][1] = '%s: %s' % ( _('Internal error'), e) self._last_rescan_failed = True self._log_status(_('Internal error')) raise self._last_rescan_completed = all_completed self._state = 'Done' status = [] if discovered > 0: status.append(_('Discovered %d mailboxes') % discovered) self._last_rescan_completed = False if rescanned > 0: status.append(_('Processed %d mailboxes') % rescanned) if errors: status.append(_('Failed to process %d') % errors) if not status: status.append(_('No new mail at %s' ) % datetime.datetime.today().strftime('%H:%M')) self._log_status(', '.join(status)) self._last_rescan_count = rescanned self._state = ostate return rescanned
def _get_mbx_id_and_mfn(self, mbx_cfg): mbx_id = FormatMbxId(mbx_cfg._key) return mbx_id, self.session.config.sys.mailbox[mbx_id]
def sync_mail(self): """Iterates through all the mailboxes and scans if necessary.""" config = self.session.config self._last_rescan_count = rescanned = errors = 0 self._last_rescan_completed = True self._last_rescan_failed = False self._interrupt = None batch = self.RESCAN_BATCH_SIZE errors = rescanned = 0 ostate = self._state for mbx_cfg in self._sorted_mailboxes(): try: with self._lock: mbx_key = FormatMbxId(mbx_cfg._key) path = self._path(mbx_cfg) if (path in ('/dev/null', '', None) or mbx_cfg.policy in ('ignore', 'unknown')): continue # Generally speaking, we only rescan if a mailbox looks like # it has changed. However, 1/50th of the time we take a look # anyway just in case looks are deceiving. state = {} if batch > 0 and (self._has_mailbox_changed(mbx_cfg, state) or random.randint(0, 50) == 10): self._state = 'Waiting... (rescan)' if self._check_interrupt(clear=False): self._last_rescan_completed = False break count = self.rescan_mailbox(mbx_key, mbx_cfg, path, stop_after=batch) if count >= 0: self.event.data['counters' ]['indexed_messages'] += count batch -= count complete = ((count == 0 or batch > 0) and not self._interrupt and not mailpile.util.QUITTING) if complete: rescanned += 1 # If there was a copy, check if it completed if not self.event.data.get('copying', {'complete': True} ).get('complete'): complete = False # If there was a rescan, check if it completed if not self.event.data.get('rescan', {'complete': True} ).get('complete'): complete = False # OK, everything looks complete, mark it! if complete: self._mark_mailbox_rescanned(mbx_cfg, state) else: self._last_rescan_completed = False else: self._last_rescan_failed = True self._last_rescan_completed = False errors += 1 except (NoSuchMailboxError, IOError, OSError): self._last_rescan_failed = True errors += 1 except: self._last_rescan_failed = True self._log_status(_('Internal error')) raise self._state = 'Waiting... (disco)' discovered = 0 if not self._check_interrupt(): discovered = self.discover_mailboxes() status = [] if discovered > 0: status.append(_('Discovered %d mailboxes') % discovered) if discovered < 1 or rescanned > 0: status.append(_('Rescanned %d mailboxes') % rescanned) if errors: status.append(_('Failed to rescan %d') % errors) self._log_status(', '.join(status)) self._last_rescan_count = rescanned self._state = ostate return rescanned
def open_mailbox(self, mbx_id, mfn): if FormatMbxId(mbx_id) in self.my_config.mailbox: proto_me, path = mfn.split('/', 1) if proto_me.startswith('src:'): return SharedImapMailbox(self.session, self, mailbox_path=path) return False
def _has_mailbox_changed(self, mbx, state): pop3 = self.session.config.open_mailbox(self.session, FormatMbxId(mbx._key), prefer_local=False) state['stat'] = stat = '%s' % (pop3.stat(), ) return (self.event.data.get('mailbox_state', {}).get(mbx._key) != stat)