def highestmodseq_update( crispin_client, db_session, log, folder_name, last_highestmodseq, status_cb, highestmodseq_fn, syncmanager_lock, c ): account_id = crispin_client.account_id new_highestmodseq = crispin_client.selected_highestmodseq new_uidvalidity = crispin_client.selected_uidvalidity log.info("Starting highestmodseq update on {0} (current HIGHESTMODSEQ: {1})".format(folder_name, new_highestmodseq)) local_uids = account.all_uids(account_id, db_session, folder_name) changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq, c) remote_uids = crispin_client.all_uids(c) if changed_uids: new, updated = new_or_updated(changed_uids, local_uids) log.info("{0} new and {1} updated UIDs".format(len(new), len(updated))) local_uids += new deleted_uids = remove_deleted_uids( account_id, db_session, log, folder_name, local_uids, remote_uids, syncmanager_lock, c ) local_uids = set(local_uids) - deleted_uids update_metadata(crispin_client, db_session, log, folder_name, updated, syncmanager_lock, c) highestmodseq_fn( crispin_client, db_session, log, folder_name, changed_uids, local_uids, status_cb, syncmanager_lock, c ) else: log.info("No new or updated messages") remove_deleted_uids( crispin_client.account_id, db_session, log, folder_name, local_uids, remote_uids, syncmanager_lock, c ) account.update_uidvalidity(account_id, db_session, folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def check_flags(crispin_client, db_session, log, folder_name, local_uids, syncmanager_lock, c): """ Update message flags if folder has changed on the remote. If we have a saved uidvalidity for this folder, make sure the folder hasn't changed since we saved it. Otherwise we need to query for flag changes too. """ saved_validity = account.get_uidvalidity(crispin_client.account_id, db_session, folder_name) if saved_validity is not None: last_highestmodseq = saved_validity.highestmodseq if last_highestmodseq > crispin_client.selected_highestmodseq: uids = crispin_client.new_and_updated_uids(last_highestmodseq, c) if uids: _, updated = new_or_updated(uids, local_uids) update_metadata(crispin_client, db_session, log, folder_name, updated, syncmanager_lock, c)
def update_saved_g_metadata(crispin_client, db_session, log, folder_name, remote_g_metadata, local_uids, syncmanager_lock, c): """ If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache, we need to query for any changes since then and update the saved data. """ log.info("Updating cache with latest changes") # Any uids we don't already have will be downloaded correctly as usual, but # updated uids need to be updated manually. # XXX it may actually be faster to just query for X-GM-MSGID for the # whole folder rather than getting changed UIDs first; MODSEQ queries # are slow on large folders. modified = crispin_client.new_and_updated_uids( crispin_client.selected_highestmodseq, c) log.info("Found {0} modified".format(len(modified))) new, updated = new_or_updated(modified, local_uids) log.info("{} new and {} updated UIDs".format(len(new), len(updated))) if new: remote_g_metadata.update(crispin_client.g_metadata(new, c)) log.info("Updated cache with new messages") else: log.info("No new messages to update metadata for") # Filter out messages that have disappeared. old_len = len(remote_g_metadata) current_remote_uids = set(crispin_client.all_uids(c)) remote_g_metadata = dict((uid, md) for uid, md in remote_g_metadata.iteritems() if uid in current_remote_uids) num_removed = old_len - len(remote_g_metadata) if num_removed > 0: log.info("{} messages removed".format(num_removed)) set_cache(remote_g_metadata_cache_file(crispin_client.account_id, folder_name), remote_g_metadata) if updated: # It's easy and fast to just update these here and now. # Bigger chunk because the data being fetched here is very small. for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE): update_metadata(crispin_client, db_session, log, folder_name, uids, syncmanager_lock, c) log.info("Updated metadata for modified messages") else: log.info("No modified messages to update metadata for")
def update_saved_g_metadata(crispin_client, db_session, log, folder_name, remote_g_metadata, local_uids, syncmanager_lock): """ If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache, we need to query for any changes since then and update the saved data. """ log.info("Updating cache with latest changes") # Any uids we don't already have will be downloaded correctly as usual, but # updated uids need to be updated manually. # XXX it may actually be faster to just query for X-GM-MSGID for the # whole folder rather than getting changed UIDs first; MODSEQ queries # are slow on large folders. modified = crispin_client.new_and_updated_uids( crispin_client.selected_highestmodseq) log.info("Found {0} modified".format(len(modified))) new, updated = new_or_updated(modified, local_uids) log.info("{} new and {} updated UIDs".format(len(new), len(updated))) if new: remote_g_metadata.update(crispin_client.g_metadata(new)) log.info("Updated cache with new messages") else: log.info("No new messages to update metadata for") # Filter out messages that have disappeared. old_len = len(remote_g_metadata) current_remote_uids = set(crispin_client.all_uids()) remote_g_metadata = dict((uid, md) for uid, md in remote_g_metadata.iteritems() if uid in current_remote_uids) num_removed = old_len - len(remote_g_metadata) if num_removed > 0: log.info("{} messages removed".format(num_removed)) set_cache(remote_g_metadata_cache_file(crispin_client.account_id, folder_name), remote_g_metadata) if updated: # It's easy and fast to just update these here and now. # Bigger chunk because the data being fetched here is very small. for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE): update_metadata(crispin_client, db_session, log, folder_name, uids, syncmanager_lock) log.info("Updated metadata for modified messages") else: log.info("No modified messages to update metadata for")