def _got_msgs (ms):
# opening db per message batch since it takes some time to download each one
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
for m in ms:
self.bar_update (1)
self.local.store (m, db)
def full_pull (self):
total = 1
self.bar_create (leave = True, total = total, desc = 'fetching messages')
# NOTE:
# this list might grow gigantic for large quantities of e-mail, not really sure
# about how much memory this will take. this is just a list of some
# simple metadata like message ids.
message_gids = []
last_id = self.remote.get_current_history_id (self.local.state.last_historyId)
for mset in self.remote.all_messages ():
(total, gids) = mset
self.bar.total = total
self.bar_update (len(gids))
for m in gids:
message_gids.append (m['id'])
if self.limit is not None and len(message_gids) >= self.limit:
break
self.bar_close ()
if self.local.config.remove_local_messages:
if self.limit and not self.dry_run:
raise ValueError('--limit with "remove_local_messages" will cause lots of messages to be deleted')
# removing files that have been deleted remotely
all_remote = set (message_gids)
all_local = set (self.local.gids.keys ())
remove = list(all_local - all_remote)
self.bar_create (leave = True, total = len(remove), desc = 'removing deleted')
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
for m in remove:
self.local.remove (m, db)
self.bar_update (1)
self.bar_close ()
if len(message_gids) > 0:
# get content for new messages
updated = self.get_content (message_gids)
# get updated labels for the rest
needs_update = list(set(message_gids) - set(updated))
self.get_meta (needs_update)
else:
self.vprint ("pull: no messages.")
# set notmuch lastmod time, since we have now synced everything from remote
# to local
with notmuch.Database () as db:
(rev, uuid) = db.get_revision ()
if not self.dry_run:
self.local.state.set_lastmod (rev)
self.local.state.set_last_history_id (last_id)
self.vprint ('current historyId: %d, current revision: %d' % (last_id, rev))
def _got_msgs (ms):
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
for m in ms:
self.bar_update (1)
self.local.update_tags (m, None, db)
def partial_pull (self):
# get history
bar = None
history = []
last_id = self.remote.get_current_history_id (self.local.state.last_historyId)
try:
for hist in self.remote.get_history_since (self.local.state.last_historyId):
history.extend (hist)
if bar is None:
self.bar_create (leave = True, desc = 'fetching changes')
self.bar_update (len(hist))
if self.limit is not None and len(history) >= self.limit:
break
except googleapiclient.errors.HttpError as excep:
if excep.resp.status == 404:
print ("pull: historyId is too old, full sync required.")
self.full_pull ()
return
else:
raise
except Remote.NoHistoryException as excep:
print ("pull: failed, re-try in a bit.")
raise
finally:
if bar is not None: self.bar_close ()
# figure out which changes need to be applied
added_messages = [] # added messages, if they are later deleted they will be
# removed from this list
deleted_messages = [] # deleted messages, if they are later added they will be
# removed from this list
labels_changed = [] # list of messages which have had their label changed
# the entry will be the last and most recent one in case
# of multiple changes. if the message is either deleted
# or added after the label change it will be removed from
# this list.
def remove_from_all (m):
nonlocal added_messages, deleted_messages, labels_changed
remove_from_list (deleted_messages, m)
remove_from_list (labels_changed, m)
remove_from_list (added_messages, m)
def remove_from_list (lst, m):
e = next ((e for e in lst if e['id'] == m['id']), None)
if e is not None:
lst.remove (e)
return True
return False
if len(history) > 0:
self.bar_create (total = len(history), leave = True, desc = 'resolving changes')
else:
bar = None
for h in history:
if 'messagesAdded' in h:
for m in h['messagesAdded']:
mm = m['message']
if not (set(mm.get('labelIds', [])) & self.remote.not_sync):
remove_from_all (mm)
added_messages.append (mm)
if 'messagesDeleted' in h:
for m in h['messagesDeleted']:
mm = m['message']
# might silently fail to delete this
remove_from_all (mm)
if self.local.has (mm['id']):
deleted_messages.append (mm)
# messages that are subsequently deleted by a later action will be removed
# from either labels_changed or added_messages.
if 'labelsAdded' in h:
for m in h['labelsAdded']:
mm = m['message']
if not (set(mm.get('labelIds', [])) & self.remote.not_sync):
new = remove_from_list (added_messages, mm) or not self.local.has (mm['id'])
remove_from_list (labels_changed, mm)
if new:
added_messages.append (mm) # needs to fetched
else:
labels_changed.append (mm)
else:
# in case a not_sync tag has been added to a scheduled message
remove_from_list (added_messages, mm)
remove_from_list (labels_changed, mm)
if self.local.has (mm['id']):
remove_from_list (deleted_messages, mm)
deleted_messages.append (mm)
if 'labelsRemoved' in h:
for m in h['labelsRemoved']:
mm = m['message']
if not (set(mm.get('labelIds', [])) & self.remote.not_sync):
new = remove_from_list (added_messages, mm) or not self.local.has (mm['id'])
remove_from_list (labels_changed, mm)
if new:
added_messages.append (mm) # needs to fetched
else:
labels_changed.append (mm)
else:
# in case a not_sync tag has been added
remove_from_list (added_messages, mm)
remove_from_list (labels_changed, mm)
if self.local.has (mm['id']):
remove_from_list (deleted_messages, mm)
deleted_messages.append (mm)
self.bar_update (1)
if bar: self.bar_close ()
changed = False
# fetching new messages
if len (added_messages) > 0:
message_gids = [m['id'] for m in added_messages]
updated = self.get_content (message_gids)
# updated labels for the messages that already existed
needs_update_gid = list(set(message_gids) - set(updated))
needs_update = [m for m in added_messages if m['id'] in needs_update_gid]
labels_changed.extend (needs_update)
changed = True
if self.local.config.remove_local_messages and len(deleted_messages) > 0:
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
for m in tqdm (deleted_messages, leave = True, desc = 'removing messages'):
self.local.remove (m['id'], db)
changed = True
if len (labels_changed) > 0:
lchanged = 0
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
self.bar_create (total = len(labels_changed), leave = True, desc = 'updating tags (0)')
for m in labels_changed:
r = self.local.update_tags (m, None, db)
if r:
lchanged += 1
if not self.args.quiet and self.bar:
self.bar.set_description ('updating tags (%d)' % lchanged)
self.bar_update (1)
self.bar_close ()
changed = True
if not changed:
self.vprint ("pull: everything is up-to-date.")
if not self.dry_run:
self.local.state.set_last_history_id (last_id)
if (last_id > 0):
self.vprint ('current historyId: %d' % last_id)
def push (self, args, setup = False):
if not setup:
self.setup (args, args.dry_run, True)
self.force = args.force
self.limit = args.limit
self.remote.get_labels ()
# loading local changes
with notmuch.Database () as db:
(rev, uuid) = db.get_revision ()
if rev == self.local.state.lastmod:
self.vprint ("push: everything is up-to-date.")
return
qry = "path:%s/** and lastmod:%d..%d" % (self.local.nm_relative, self.local.state.lastmod, rev)
# print ("collecting changes..: %s" % qry)
query = notmuch.Query (db, qry)
total = query.count_messages () # probably destructive here as well
query = notmuch.Query (db, qry)
messages = list(query.search_messages ())
if self.limit is not None and len(messages) > self.limit:
messages = messages[:self.limit]
# get gids and filter out messages outside this repository
messages, gids = self.local.messages_to_gids (messages)
# get meta-data on changed messages from remote
remote_messages = []
self.bar_create (leave = True, total = len(gids), desc = 'receiving metadata')
def _got_msgs (ms):
for m in ms:
self.bar_update (1)
remote_messages.append (m)
self.remote.get_messages (gids, _got_msgs, 'minimal')
self.bar_close ()
# resolve changes
self.bar_create (leave = True, total = len(gids), desc = 'resolving changes')
actions = []
for rm, nm in zip(remote_messages, messages):
actions.append (self.remote.update (rm, nm, self.local.state.last_historyId, self.force))
self.bar_update (1)
self.bar_close ()
# remove no-ops
actions = [ a for a in actions if a ]
# limit
if self.limit is not None and len(actions) >= self.limit:
actions = actions[:self.limit]
# push changes
if len(actions) > 0:
self.bar_create (leave = True, total = len(actions), desc = 'pushing, 0 changed')
changed = 0
def cb (resp):
nonlocal changed
self.bar_update (1)
changed += 1
if not self.args.quiet and self.bar:
self.bar.set_description ('pushing, %d changed' % changed)
self.remote.push_changes (actions, cb)
self.bar_close ()
else:
self.vprint ('push: nothing to push')
if not self.remote.all_updated:
# will not set last_mod, this forces messages to be pushed again at next push
print ("push: not all changes could be pushed, will re-try at next push.")
else:
# TODO: Once we get more confident we might set the last history Id here to
# avoid pulling back in the changes we just pushed. Currently there's a race
# if something is modified remotely (new email, changed tags), so this might
# not really be possible.
pass
if not self.dry_run and self.remote.all_updated:
self.local.state.set_lastmod (rev)
self.vprint ("remote historyId: %d" % self.remote.get_current_history_id (self.local.state.last_historyId))
import os
from subprocess import call
import notmuch
with open(os.devnull, "w") as devnull:
call(["emacsclient", "-e", "(run-hooks 'notmuch-presync-hook)"],
stdout=devnull)
db = notmuch.Database()
maildir_path = os.path.realpath(os.path.expanduser("~/Maildir"))
def move_messages(query_string, destination):
query = db.create_query(query_string)
for message in query.search_messages():
old_filename = message.get_filename()
path, filename = os.path.split(old_filename)
cur_new = path.split(os.sep)[-1]
new_filename = os.path.join(maildir_path, destination, cur_new,
filename)
os.rename(old_filename, new_filename)
# Archive any messages under INBOX*/ which no longer have the inbox tag
inbox_folders = " OR ".join([
"folder:" + path for path in os.listdir(maildir_path)
if path.startswith("INBOX")
])
move_messages("NOT tag:inbox AND (" + inbox_folders + ")", "Archive")
def _got_msg (m):
bar.update (1)
# opening db per message since it takes some time to download each one
with notmuch.Database (mode = notmuch.Database.MODE.READ_WRITE) as db:
self.local.store (m, db)
def load_repository(self):
"""
Loads the current local repository
"""
if not os.path.exists(self.state_f):
raise Local.RepositoryException(
'local repository not initialized: could not find state file')
if not os.path.exists(self.md):
raise Local.RepositoryException(
'local repository not initialized: could not find mail dir')
self.state = Local.State(self.state_f)
## Check if we are in the notmuch db
with notmuch.Database() as db:
try:
self.nm_dir = db.get_directory(os.path.abspath(self.md))
if self.nm_dir is not None:
self.nm_dir = self.nm_dir.path
else:
# probably empty dir
self.nm_dir = os.path.abspath(self.md)
self.nm_relative = self.nm_dir[len(db.get_path()) + 1:]
except notmuch.errors.FileError:
raise Local.RepositoryException(
"local mail repository not in notmuch db")
## The Cache:
##
## this cache is used to know which messages we have a physical copy of.
## hopefully this won't grow too gigantic with lots of messages.
self.files = []
for (dp, dirnames, fnames) in os.walk(os.path.join(self.md, 'cur')):
_fnames = ('cur/' + f for f in fnames)
self.files.extend(_fnames)
break
for (dp, dirnames, fnames) in os.walk(os.path.join(self.md, 'new')):
_fnames = ('new/' + f for f in fnames)
self.files.extend(_fnames)
break
# exclude files that are unlikely to be real message files
self.files = [f for f in self.files if os.path.basename(f)[0] != '.']
self.gids = {}
for f in self.files:
m = os.path.basename(f).split(':')[0]
self.gids[m] = f
# load notmuch config
cfg = os.environ.get('NOTMUCH_CONFIG',
os.path.expanduser('~/.notmuch-config'))
if not os.path.exists(cfg):
raise Local.RepositoryException(
"could not find notmuch-config: %s" % cfg)
self.nmconfig = configparser.ConfigParser()
self.nmconfig.read(cfg)
self.new_tags = self.nmconfig['new']['tags'].split(';')
self.new_tags = [
t.strip() for t in self.new_tags if len(t.strip()) > 0
]
self.loaded = True
