Exemplo n.º 1
0
class SyncableAccount(Account):
    """A syncable email account connecting 2 repositories.

    Derives from :class:`accounts.Account` but contains the additional
    functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`,
    used for syncing.

    In multi-threaded mode, one instance of this object is run per "account"
    thread."""
    def __init__(self, *args, **kwargs):
        Account.__init__(self, *args, **kwargs)
        self._lockfd = None
        self._lockfilepath = os.path.join(self.config.getmetadatadir(),
                                          "%s.lock" % self)

    def __lock(self):
        """Lock the account, throwing an exception if it is locked already."""

        self._lockfd = open(self._lockfilepath, 'w')
        try:
            fcntl.lockf(self._lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except NameError:
            #fcntl not available (Windows), disable file locking... :(
            pass
        except IOError:
            self._lockfd.close()
            six.reraise(
                OfflineImapError,
                OfflineImapError(
                    "Could not lock account %s. Is another "
                    "instance using this account?" % self,
                    OfflineImapError.ERROR.REPO),
                exc_info()[2])

    def _unlock(self):
        """Unlock the account, deleting the lock file"""

        #If we own the lock file, delete it
        if self._lockfd and not self._lockfd.closed:
            self._lockfd.close()
            try:
                os.unlink(self._lockfilepath)
            except OSError:
                pass  # Failed to delete for some reason.

    def syncrunner(self):
        """The target for both single and multi-threaded modes."""

        self.ui.registerthread(self)
        try:
            accountmetadata = self.getaccountmeta()
            if not os.path.exists(accountmetadata):
                os.mkdir(accountmetadata, 0o700)

            self.remoterepos = Repository(self, 'remote')
            self.localrepos = Repository(self, 'local')
            self.statusrepos = Repository(self, 'status')
        except OfflineImapError as e:
            self.ui.error(e, exc_info()[2])
            if e.severity >= OfflineImapError.ERROR.CRITICAL:
                raise
            return

        # Loop account sync if needed (bail out after 3 failures).
        looping = 3
        while looping:
            self.ui.acct(self)
            try:
                self.__lock()
                self.__sync()
            except (KeyboardInterrupt, SystemExit):
                raise
            except OfflineImapError as e:
                # Stop looping and bubble up Exception if needed.
                if e.severity >= OfflineImapError.ERROR.REPO:
                    if looping:
                        looping -= 1
                    if e.severity >= OfflineImapError.ERROR.CRITICAL:
                        raise
                self.ui.error(e, exc_info()[2])
            except Exception as e:
                self.ui.error(e,
                              exc_info()[2],
                              msg="While attempting to sync account '%s'" %
                              self)
            else:
                # After success sync, reset the looping counter to 3.
                if self.refreshperiod:
                    looping = 3
            finally:
                self.ui.acctdone(self)
                self._unlock()
                if looping and self._sleeper() >= 2:
                    looping = 0

    def get_local_folder(self, remotefolder):
        """Return the corresponding local folder for a given remotefolder."""

        return self.localrepos.getfolder(remotefolder.getvisiblename().replace(
            self.remoterepos.getsep(), self.localrepos.getsep()))

    # The syncrunner will loop on this method. This means it is called more than
    # once during the run.
    def __sync(self):
        """Synchronize the account once, then return.

        Assumes that `self.remoterepos`, `self.localrepos`, and
        `self.statusrepos` has already been populated, so it should only
        be called from the :meth:`syncrunner` function."""

        folderthreads = []

        hook = self.getconf('presynchook', '')
        self.callhook(hook)

        if self.utf_8_support and self.remoterepos.getdecodefoldernames():
            raise OfflineImapError(
                "Configuration mismatch in account " +
                "'%s'. " % self.getname() +
                "\nAccount setting 'utf8foldernames' and repository " +
                "setting 'decodefoldernames'\nmay not be used at the " +
                "same time. This account has not been synchronized.\n" +
                "Please check the configuration and documentation.",
                OfflineImapError.ERROR.REPO)

        quickconfig = self.getconfint('quick', 0)
        if quickconfig < 0:
            quick = True
        elif quickconfig > 0:
            if self.quicknum == 0 or self.quicknum > quickconfig:
                self.quicknum = 1
                quick = False
            else:
                self.quicknum = self.quicknum + 1
                quick = True
        else:
            quick = False

        try:
            startedThread = False
            remoterepos = self.remoterepos
            localrepos = self.localrepos
            statusrepos = self.statusrepos

            # Init repos with list of folders, so we have them (and the
            # folder delimiter etc).
            remoterepos.getfolders()
            localrepos.getfolders()

            remoterepos.sync_folder_structure(localrepos, statusrepos)
            # Replicate the folderstructure between REMOTE to LOCAL.
            if not localrepos.getconfboolean('readonly', False):
                self.ui.syncfolders(remoterepos, localrepos)

            # Iterate through all folders on the remote repo and sync.
            for remotefolder in remoterepos.getfolders():
                # Check for CTRL-C or SIGTERM.
                if Account.abort_NOW_signal.is_set():
                    break

                if not remotefolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" % (remotefolder.getname(), remoterepos))
                    continue  # Ignore filtered folder.

                # The remote folder names must not have the local sep char in
                # their names since this would cause troubles while converting
                # the name back (from local to remote).
                sep = localrepos.getsep()
                if (sep != os.path.sep and sep != remoterepos.getsep()
                        and sep in remotefolder.getname()):
                    self.ui.warn(
                        '', "Ignoring folder '%s' due to unsupported "
                        "'%s' character serving as local separator." %
                        (remotefolder.getname(), localrepos.getsep()))
                    continue  # Ignore unsupported folder name.

                localfolder = self.get_local_folder(remotefolder)
                if not localfolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" %
                        (localfolder.getname(), localfolder.repository))
                    continue  # Ignore filtered folder.

                if not globals.options.singlethreading:
                    thread = InstanceLimitedThread(
                        limitNamespace="%s%s" %
                        (FOLDER_NAMESPACE, self.remoterepos.getname()),
                        target=syncfolder,
                        name="Folder %s [acc: %s]" %
                        (remotefolder.getexplainedname(), self),
                        args=(self, remotefolder, quick))
                    thread.start()
                    folderthreads.append(thread)
                else:
                    syncfolder(self, remotefolder, quick)
                startedThread = True
            # Wait for all threads to finish.
            for thr in folderthreads:
                thr.join()
            if startedThread is True:
                mbnames.writeIntermediateFile(
                    self.name)  # Write out mailbox names.
            else:
                msg = "Account {}: no folder to sync (folderfilter issue?)".format(
                    self)
                raise OfflineImapError(msg, OfflineImapError.ERROR.REPO)
            localrepos.forgetfolders()
            remoterepos.forgetfolders()
        except:
            # Error while syncing. Drop all connections that we have, they
            # might be bogus by now (e.g. after suspend).
            localrepos.dropconnections()
            remoterepos.dropconnections()
            raise
        else:
            # Sync went fine. Hold or drop depending on config.
            localrepos.holdordropconnections()
            remoterepos.holdordropconnections()

        hook = self.getconf('postsynchook', '')
        self.callhook(hook)

    def callhook(self, cmd):
        # Check for CTRL-C or SIGTERM and run postsynchook.
        if Account.abort_NOW_signal.is_set():
            return
        if not cmd:
            return
        try:
            self.ui.callhook("Calling hook: " + cmd)
            if self.dryrun:
                return
            p = Popen(cmd,
                      shell=True,
                      stdin=PIPE,
                      stdout=PIPE,
                      stderr=PIPE,
                      close_fds=True)
            r = p.communicate()
            self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r)
            self.ui.callhook("Hook return code: %d" % p.returncode)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            self.ui.error(e, exc_info()[2], msg="Calling hook")
Exemplo n.º 2
0
class SyncableAccount(Account):
    """A syncable email account connecting 2 repositories

    Derives from :class:`accounts.Account` but contains the additional
    functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`,
    used for syncing."""

    def __init__(self, *args, **kwargs):
        Account.__init__(self, *args, **kwargs)
        self._lockfd = None
        self._lockfilepath = os.path.join(self.config.getmetadatadir(),
                                          "%s.lock" % self)

    def lock(self):
        """Lock the account, throwing an exception if it is locked already"""
        self._lockfd = open(self._lockfilepath, 'w')
        try:
            fcntl.lockf(self._lockfd, fcntl.LOCK_EX|fcntl.LOCK_NB)
        except NameError:
            #fcntl not available (Windows), disable file locking... :(
            pass
        except IOError:
            self._lockfd.close()
            raise OfflineImapError("Could not lock account %s. Is another "
                                   "instance using this account?" % self,
                                   OfflineImapError.ERROR.REPO)

    def unlock(self):
        """Unlock the account, deleting the lock file"""
        #If we own the lock file, delete it
        if self._lockfd and not self._lockfd.closed:
            self._lockfd.close()
            try:
                os.unlink(self._lockfilepath)
            except OSError:
                pass #Failed to delete for some reason.

    def syncrunner(self):
        self.ui.registerthread(self)
        try:
            accountmetadata = self.getaccountmeta()
            if not os.path.exists(accountmetadata):
                os.mkdir(accountmetadata, 0o700)

            self.remoterepos = Repository(self, 'remote')
            self.localrepos  = Repository(self, 'local')
            self.statusrepos = Repository(self, 'status')
        except OfflineImapError as e:
            self.ui.error(e, exc_info()[2])
            if e.severity >= OfflineImapError.ERROR.CRITICAL:
                raise
            return

        # Loop account sync if needed (bail out after 3 failures)
        looping = 3
        while looping:
            self.ui.acct(self)
            try:
                self.lock()
                self.sync()
            except (KeyboardInterrupt, SystemExit):
                raise
            except OfflineImapError as e:
                # Stop looping and bubble up Exception if needed.
                if e.severity >= OfflineImapError.ERROR.REPO:
                    if looping:
                        looping -= 1
                    if e.severity >= OfflineImapError.ERROR.CRITICAL:
                        raise
                self.ui.error(e, exc_info()[2])
            except Exception as e:
                self.ui.error(e, exc_info()[2], msg = "While attempting to sync"
                    " account '%s'" % self)
            else:
                # after success sync, reset the looping counter to 3
                if self.refreshperiod:
                    looping = 3
            finally:
                self.ui.acctdone(self)
                self.unlock()
                if looping and self.sleeper() >= 2:
                    looping = 0

    def get_local_folder(self, remotefolder):
        """Return the corresponding local folder for a given remotefolder"""
        return self.localrepos.getfolder(
            remotefolder.getvisiblename().
            replace(self.remoterepos.getsep(), self.localrepos.getsep()))

    def sync(self):
        """Synchronize the account once, then return

        Assumes that `self.remoterepos`, `self.localrepos`, and
        `self.statusrepos` has already been populated, so it should only
        be called from the :meth:`syncrunner` function.
        """
        folderthreads = []

        hook = self.getconf('presynchook', '')
        self.callhook(hook)

        quickconfig = self.getconfint('quick', 0)
        if quickconfig < 0:
            quick = True
        elif quickconfig > 0:
            if self.quicknum == 0 or self.quicknum > quickconfig:
                self.quicknum = 1
                quick = False
            else:
                self.quicknum = self.quicknum + 1
                quick = True
        else:
            quick = False

        try:
            remoterepos = self.remoterepos
            localrepos = self.localrepos
            statusrepos = self.statusrepos

            # init repos with list of folders, so we have them (and the
            # folder delimiter etc)
            remoterepos.getfolders()
            localrepos.getfolders()

            remoterepos.sync_folder_structure(localrepos, statusrepos)
            # replicate the folderstructure between REMOTE to LOCAL
            if not localrepos.getconfboolean('readonly', False):
                self.ui.syncfolders(remoterepos, localrepos)

            # iterate through all folders on the remote repo and sync
            for remotefolder in remoterepos.getfolders():
                # check for CTRL-C or SIGTERM
                if Account.abort_NOW_signal.is_set(): break

                if not remotefolder.sync_this:
                    self.ui.debug('', "Not syncing filtered folder '%s'"
                                  "[%s]" % (remotefolder, remoterepos))
                    continue # Ignore filtered folder
                localfolder = self.get_local_folder(remotefolder)
                if not localfolder.sync_this:
                    self.ui.debug('', "Not syncing filtered folder '%s'"
                                 "[%s]" % (localfolder, localfolder.repository))
                    continue # Ignore filtered folder
                thread = InstanceLimitedThread(\
                    instancename = 'FOLDER_' + self.remoterepos.getname(),
                    target = syncfolder,
                    name = "Folder %s [acc: %s]" % (remotefolder, self),
                    args = (self, remotefolder, quick))
                thread.start()
                folderthreads.append(thread)
            # wait for all threads to finish
            for thr in folderthreads:
                thr.join()
            # Write out mailbox names if required and not in dry-run mode
            if not self.dryrun:
                mbnames.write()
            localrepos.forgetfolders()
            remoterepos.forgetfolders()
        except:
            #error while syncing. Drop all connections that we have, they
            #might be bogus by now (e.g. after suspend)
            localrepos.dropconnections()
            remoterepos.dropconnections()
            raise
        else:
            # sync went fine. Hold or drop depending on config
            localrepos.holdordropconnections()
            remoterepos.holdordropconnections()

        hook = self.getconf('postsynchook', '')
        self.callhook(hook)

    def callhook(self, cmd):
        # check for CTRL-C or SIGTERM and run postsynchook
        if Account.abort_NOW_signal.is_set():
            return
        if not cmd:
            return
        try:
            self.ui.callhook("Calling hook: " + cmd)
            if self.dryrun: # don't if we are in dry-run mode
                return
            p = Popen(cmd, shell=True,
                      stdin=PIPE, stdout=PIPE, stderr=PIPE,
                      close_fds=True)
            r = p.communicate()
            self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r)
            self.ui.callhook("Hook return code: %d" % p.returncode)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            self.ui.error(e, exc_info()[2], msg = "Calling hook")
Exemplo n.º 3
0
class SyncableAccount(Account):
    """A syncable email account connecting 2 repositories

    Derives from :class:`accounts.Account` but contains the additional
    functions :meth:`syncrunner`, :meth:`sync`, :meth:`syncfolders`,
    used for syncing."""
    def __init__(self, *args, **kwargs):
        Account.__init__(self, *args, **kwargs)
        self._lockfd = None
        self._lockfilepath = os.path.join(self.config.getmetadatadir(),
                                          "%s.lock" % self)

    def lock(self):
        """Lock the account, throwing an exception if it is locked already"""
        self._lockfd = open(self._lockfilepath, 'w')
        try:
            fcntl.lockf(self._lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except NameError:
            #fcntl not available (Windows), disable file locking... :(
            pass
        except IOError:
            self._lockfd.close()
            raise OfflineImapError(
                "Could not lock account %s. Is another "
                "instance using this account?" % self,
                OfflineImapError.ERROR.REPO)

    def unlock(self):
        """Unlock the account, deleting the lock file"""
        #If we own the lock file, delete it
        if self._lockfd and not self._lockfd.closed:
            self._lockfd.close()
            try:
                os.unlink(self._lockfilepath)
            except OSError:
                pass  #Failed to delete for some reason.

    def syncrunner(self):
        self.ui.registerthread(self)
        try:
            accountmetadata = self.getaccountmeta()
            if not os.path.exists(accountmetadata):
                os.mkdir(accountmetadata, 0o700)

            self.remoterepos = Repository(self, 'remote')
            self.localrepos = Repository(self, 'local')
            self.statusrepos = Repository(self, 'status')
        except OfflineImapError as e:
            self.ui.error(e, exc_info()[2])
            if e.severity >= OfflineImapError.ERROR.CRITICAL:
                raise
            return

        # Loop account sync if needed (bail out after 3 failures)
        looping = 3
        while looping:
            self.ui.acct(self)
            try:
                self.lock()
                self.sync()
            except (KeyboardInterrupt, SystemExit):
                raise
            except OfflineImapError as e:
                # Stop looping and bubble up Exception if needed.
                if e.severity >= OfflineImapError.ERROR.REPO:
                    if looping:
                        looping -= 1
                    if e.severity >= OfflineImapError.ERROR.CRITICAL:
                        raise
                self.ui.error(e, exc_info()[2])
            except Exception as e:
                self.ui.error(e,
                              exc_info()[2],
                              msg="While attempting to sync"
                              " account '%s'" % self)
            else:
                # after success sync, reset the looping counter to 3
                if self.refreshperiod:
                    looping = 3
            finally:
                self.ui.acctdone(self)
                self.unlock()
                if looping and self.sleeper() >= 2:
                    looping = 0

    def get_local_folder(self, remotefolder):
        """Return the corresponding local folder for a given remotefolder"""
        return self.localrepos.getfolder(remotefolder.getvisiblename().replace(
            self.remoterepos.getsep(), self.localrepos.getsep()))

    def sync(self):
        """Synchronize the account once, then return

        Assumes that `self.remoterepos`, `self.localrepos`, and
        `self.statusrepos` has already been populated, so it should only
        be called from the :meth:`syncrunner` function.
        """
        folderthreads = []

        hook = self.getconf('presynchook', '')
        self.callhook(hook)

        quickconfig = self.getconfint('quick', 0)
        if quickconfig < 0:
            quick = True
        elif quickconfig > 0:
            if self.quicknum == 0 or self.quicknum > quickconfig:
                self.quicknum = 1
                quick = False
            else:
                self.quicknum = self.quicknum + 1
                quick = True
        else:
            quick = False

        try:
            remoterepos = self.remoterepos
            localrepos = self.localrepos
            statusrepos = self.statusrepos

            # init repos with list of folders, so we have them (and the
            # folder delimiter etc)
            remoterepos.getfolders()
            localrepos.getfolders()

            remoterepos.sync_folder_structure(localrepos, statusrepos)
            # replicate the folderstructure between REMOTE to LOCAL
            if not localrepos.getconfboolean('readonly', False):
                self.ui.syncfolders(remoterepos, localrepos)

            # iterate through all folders on the remote repo and sync
            for remotefolder in remoterepos.getfolders():
                # check for CTRL-C or SIGTERM
                if Account.abort_NOW_signal.is_set(): break

                if not remotefolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" % (remotefolder, remoterepos))
                    continue  # Ignore filtered folder
                localfolder = self.get_local_folder(remotefolder)
                if not localfolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" % (localfolder, localfolder.repository))
                    continue  # Ignore filtered folder
                if not globals.options.singlethreading:
                    thread = InstanceLimitedThread(\
                        instancename = 'FOLDER_' + self.remoterepos.getname(),
                        target = syncfolder,
                        name = "Folder %s [acc: %s]" % (remotefolder, self),
                        args = (self, remotefolder, quick))
                    thread.start()
                    folderthreads.append(thread)
                else:
                    syncfolder(self, remotefolder, quick)
            # wait for all threads to finish
            for thr in folderthreads:
                thr.join()
            # Write out mailbox names if required and not in dry-run mode
            if not self.dryrun:
                mbnames.write()
            localrepos.forgetfolders()
            remoterepos.forgetfolders()
        except:
            #error while syncing. Drop all connections that we have, they
            #might be bogus by now (e.g. after suspend)
            localrepos.dropconnections()
            remoterepos.dropconnections()
            raise
        else:
            # sync went fine. Hold or drop depending on config
            localrepos.holdordropconnections()
            remoterepos.holdordropconnections()

        hook = self.getconf('postsynchook', '')
        self.callhook(hook)

    def callhook(self, cmd):
        # check for CTRL-C or SIGTERM and run postsynchook
        if Account.abort_NOW_signal.is_set():
            return
        if not cmd:
            return
        try:
            self.ui.callhook("Calling hook: " + cmd)
            if self.dryrun:  # don't if we are in dry-run mode
                return
            p = Popen(cmd,
                      shell=True,
                      stdin=PIPE,
                      stdout=PIPE,
                      stderr=PIPE,
                      close_fds=True)
            r = p.communicate()
            self.ui.callhook("Hook stdout: %s\nHook stderr:%s\n" % r)
            self.ui.callhook("Hook return code: %d" % p.returncode)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as e:
            self.ui.error(e, exc_info()[2], msg="Calling hook")