def _got_msgs (ms): # opening db per message batch since it takes some time to download each one with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: for m in ms: self.bar_update (1) self.local.store (m, db)
def full_pull (self): total = 1 self.bar_create (leave = True, total = total, desc = 'fetching messages') # NOTE: # this list might grow gigantic for large quantities of e-mail, not really sure # about how much memory this will take. this is just a list of some # simple metadata like message ids. message_gids = [] last_id = self.remote.get_current_history_id (self.local.state.last_historyId) for mset in self.remote.all_messages (): (total, gids) = mset self.bar.total = total self.bar_update (len(gids)) for m in gids: message_gids.append (m['id']) if self.limit is not None and len(message_gids) >= self.limit: break self.bar_close () if self.local.config.remove_local_messages: if self.limit and not self.dry_run: raise ValueError('--limit with "remove_local_messages" will cause lots of messages to be deleted') # removing files that have been deleted remotely all_remote = set (message_gids) all_local = set (self.local.gids.keys ()) remove = list(all_local - all_remote) self.bar_create (leave = True, total = len(remove), desc = 'removing deleted') with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: for m in remove: self.local.remove (m, db) self.bar_update (1) self.bar_close () if len(message_gids) > 0: # get content for new messages updated = self.get_content (message_gids) # get updated labels for the rest needs_update = list(set(message_gids) - set(updated)) self.get_meta (needs_update) else: self.vprint ("pull: no messages.") # set notmuch lastmod time, since we have now synced everything from remote # to local with notmuch.Database () as db: (rev, uuid) = db.get_revision () if not self.dry_run: self.local.state.set_lastmod (rev) self.local.state.set_last_history_id (last_id) self.vprint ('current historyId: %d, current revision: %d' % (last_id, rev))
def _got_msgs (ms): with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: for m in ms: self.bar_update (1) self.local.update_tags (m, None, db)
def partial_pull (self): # get history bar = None history = [] last_id = self.remote.get_current_history_id (self.local.state.last_historyId) try: for hist in self.remote.get_history_since (self.local.state.last_historyId): history.extend (hist) if bar is None: self.bar_create (leave = True, desc = 'fetching changes') self.bar_update (len(hist)) if self.limit is not None and len(history) >= self.limit: break except googleapiclient.errors.HttpError as excep: if excep.resp.status == 404: print ("pull: historyId is too old, full sync required.") self.full_pull () return else: raise except Remote.NoHistoryException as excep: print ("pull: failed, re-try in a bit.") raise finally: if bar is not None: self.bar_close () # figure out which changes need to be applied added_messages = [] # added messages, if they are later deleted they will be # removed from this list deleted_messages = [] # deleted messages, if they are later added they will be # removed from this list labels_changed = [] # list of messages which have had their label changed # the entry will be the last and most recent one in case # of multiple changes. if the message is either deleted # or added after the label change it will be removed from # this list. def remove_from_all (m): nonlocal added_messages, deleted_messages, labels_changed remove_from_list (deleted_messages, m) remove_from_list (labels_changed, m) remove_from_list (added_messages, m) def remove_from_list (lst, m): e = next ((e for e in lst if e['id'] == m['id']), None) if e is not None: lst.remove (e) return True return False if len(history) > 0: self.bar_create (total = len(history), leave = True, desc = 'resolving changes') else: bar = None for h in history: if 'messagesAdded' in h: for m in h['messagesAdded']: mm = m['message'] if not (set(mm.get('labelIds', [])) & self.remote.not_sync): remove_from_all (mm) added_messages.append (mm) if 'messagesDeleted' in h: for m in h['messagesDeleted']: mm = m['message'] # might silently fail to delete this remove_from_all (mm) if self.local.has (mm['id']): deleted_messages.append (mm) # messages that are subsequently deleted by a later action will be removed # from either labels_changed or added_messages. if 'labelsAdded' in h: for m in h['labelsAdded']: mm = m['message'] if not (set(mm.get('labelIds', [])) & self.remote.not_sync): new = remove_from_list (added_messages, mm) or not self.local.has (mm['id']) remove_from_list (labels_changed, mm) if new: added_messages.append (mm) # needs to fetched else: labels_changed.append (mm) else: # in case a not_sync tag has been added to a scheduled message remove_from_list (added_messages, mm) remove_from_list (labels_changed, mm) if self.local.has (mm['id']): remove_from_list (deleted_messages, mm) deleted_messages.append (mm) if 'labelsRemoved' in h: for m in h['labelsRemoved']: mm = m['message'] if not (set(mm.get('labelIds', [])) & self.remote.not_sync): new = remove_from_list (added_messages, mm) or not self.local.has (mm['id']) remove_from_list (labels_changed, mm) if new: added_messages.append (mm) # needs to fetched else: labels_changed.append (mm) else: # in case a not_sync tag has been added remove_from_list (added_messages, mm) remove_from_list (labels_changed, mm) if self.local.has (mm['id']): remove_from_list (deleted_messages, mm) deleted_messages.append (mm) self.bar_update (1) if bar: self.bar_close () changed = False # fetching new messages if len (added_messages) > 0: message_gids = [m['id'] for m in added_messages] updated = self.get_content (message_gids) # updated labels for the messages that already existed needs_update_gid = list(set(message_gids) - set(updated)) needs_update = [m for m in added_messages if m['id'] in needs_update_gid] labels_changed.extend (needs_update) changed = True if self.local.config.remove_local_messages and len(deleted_messages) > 0: with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: for m in tqdm (deleted_messages, leave = True, desc = 'removing messages'): self.local.remove (m['id'], db) changed = True if len (labels_changed) > 0: lchanged = 0 with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: self.bar_create (total = len(labels_changed), leave = True, desc = 'updating tags (0)') for m in labels_changed: r = self.local.update_tags (m, None, db) if r: lchanged += 1 if not self.args.quiet and self.bar: self.bar.set_description ('updating tags (%d)' % lchanged) self.bar_update (1) self.bar_close () changed = True if not changed: self.vprint ("pull: everything is up-to-date.") if not self.dry_run: self.local.state.set_last_history_id (last_id) if (last_id > 0): self.vprint ('current historyId: %d' % last_id)
def push (self, args, setup = False): if not setup: self.setup (args, args.dry_run, True) self.force = args.force self.limit = args.limit self.remote.get_labels () # loading local changes with notmuch.Database () as db: (rev, uuid) = db.get_revision () if rev == self.local.state.lastmod: self.vprint ("push: everything is up-to-date.") return qry = "path:%s/** and lastmod:%d..%d" % (self.local.nm_relative, self.local.state.lastmod, rev) # print ("collecting changes..: %s" % qry) query = notmuch.Query (db, qry) total = query.count_messages () # probably destructive here as well query = notmuch.Query (db, qry) messages = list(query.search_messages ()) if self.limit is not None and len(messages) > self.limit: messages = messages[:self.limit] # get gids and filter out messages outside this repository messages, gids = self.local.messages_to_gids (messages) # get meta-data on changed messages from remote remote_messages = [] self.bar_create (leave = True, total = len(gids), desc = 'receiving metadata') def _got_msgs (ms): for m in ms: self.bar_update (1) remote_messages.append (m) self.remote.get_messages (gids, _got_msgs, 'minimal') self.bar_close () # resolve changes self.bar_create (leave = True, total = len(gids), desc = 'resolving changes') actions = [] for rm, nm in zip(remote_messages, messages): actions.append (self.remote.update (rm, nm, self.local.state.last_historyId, self.force)) self.bar_update (1) self.bar_close () # remove no-ops actions = [ a for a in actions if a ] # limit if self.limit is not None and len(actions) >= self.limit: actions = actions[:self.limit] # push changes if len(actions) > 0: self.bar_create (leave = True, total = len(actions), desc = 'pushing, 0 changed') changed = 0 def cb (resp): nonlocal changed self.bar_update (1) changed += 1 if not self.args.quiet and self.bar: self.bar.set_description ('pushing, %d changed' % changed) self.remote.push_changes (actions, cb) self.bar_close () else: self.vprint ('push: nothing to push') if not self.remote.all_updated: # will not set last_mod, this forces messages to be pushed again at next push print ("push: not all changes could be pushed, will re-try at next push.") else: # TODO: Once we get more confident we might set the last history Id here to # avoid pulling back in the changes we just pushed. Currently there's a race # if something is modified remotely (new email, changed tags), so this might # not really be possible. pass if not self.dry_run and self.remote.all_updated: self.local.state.set_lastmod (rev) self.vprint ("remote historyId: %d" % self.remote.get_current_history_id (self.local.state.last_historyId))
import os from subprocess import call import notmuch with open(os.devnull, "w") as devnull: call(["emacsclient", "-e", "(run-hooks 'notmuch-presync-hook)"], stdout=devnull) db = notmuch.Database() maildir_path = os.path.realpath(os.path.expanduser("~/Maildir")) def move_messages(query_string, destination): query = db.create_query(query_string) for message in query.search_messages(): old_filename = message.get_filename() path, filename = os.path.split(old_filename) cur_new = path.split(os.sep)[-1] new_filename = os.path.join(maildir_path, destination, cur_new, filename) os.rename(old_filename, new_filename) # Archive any messages under INBOX*/ which no longer have the inbox tag inbox_folders = " OR ".join([ "folder:" + path for path in os.listdir(maildir_path) if path.startswith("INBOX") ]) move_messages("NOT tag:inbox AND (" + inbox_folders + ")", "Archive")
def _got_msg (m): bar.update (1) # opening db per message since it takes some time to download each one with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db: self.local.store (m, db)
def load_repository(self): """ Loads the current local repository """ if not os.path.exists(self.state_f): raise Local.RepositoryException( 'local repository not initialized: could not find state file') if not os.path.exists(self.md): raise Local.RepositoryException( 'local repository not initialized: could not find mail dir') self.state = Local.State(self.state_f) ## Check if we are in the notmuch db with notmuch.Database() as db: try: self.nm_dir = db.get_directory(os.path.abspath(self.md)) if self.nm_dir is not None: self.nm_dir = self.nm_dir.path else: # probably empty dir self.nm_dir = os.path.abspath(self.md) self.nm_relative = self.nm_dir[len(db.get_path()) + 1:] except notmuch.errors.FileError: raise Local.RepositoryException( "local mail repository not in notmuch db") ## The Cache: ## ## this cache is used to know which messages we have a physical copy of. ## hopefully this won't grow too gigantic with lots of messages. self.files = [] for (dp, dirnames, fnames) in os.walk(os.path.join(self.md, 'cur')): _fnames = ('cur/' + f for f in fnames) self.files.extend(_fnames) break for (dp, dirnames, fnames) in os.walk(os.path.join(self.md, 'new')): _fnames = ('new/' + f for f in fnames) self.files.extend(_fnames) break # exclude files that are unlikely to be real message files self.files = [f for f in self.files if os.path.basename(f)[0] != '.'] self.gids = {} for f in self.files: m = os.path.basename(f).split(':')[0] self.gids[m] = f # load notmuch config cfg = os.environ.get('NOTMUCH_CONFIG', os.path.expanduser('~/.notmuch-config')) if not os.path.exists(cfg): raise Local.RepositoryException( "could not find notmuch-config: %s" % cfg) self.nmconfig = configparser.ConfigParser() self.nmconfig.read(cfg) self.new_tags = self.nmconfig['new']['tags'].split(';') self.new_tags = [ t.strip() for t in self.new_tags if len(t.strip()) > 0 ] self.loaded = True