Example #1
0
 def upload(self):
     runHook("sync", "upload")
     # make sure it's ok before we try to upload
     assert self.col.db.scalar("pragma integrity_check") == "ok"
     # apply some adjustments, then upload
     self.col.beforeUpload()
     assert self.req("upload", open(self.col.path, "rb")) == "OK"
Example #2
0
 def upload(self):
     runHook("sync", "upload")
     # make sure it's ok before we try to upload
     assert self.col.db.scalar("pragma integrity_check") == "ok"
     # apply some adjustments, then upload
     self.col.beforeUpload()
     assert self.req("upload", open(self.col.path, "rb")) == "OK"
Example #3
0
 def fullSyncFromLocal(self, fields, path):
     global sendProgressHook
     try:
         # write into a temporary file, since POST needs content-length
         src = open(path, "rb")
         (fd, name) = tempfile.mkstemp(prefix="oldanki")
         tmp = open(name, "w+b")
         # post vars
         for (key, value) in fields.items():
             tmp.write('--' + MIME_BOUNDARY + "\r\n")
             tmp.write('Content-Disposition: form-data; name="%s"\r\n' %
                       key)
             tmp.write('\r\n')
             tmp.write(value)
             tmp.write('\r\n')
         # file header
         tmp.write('--' + MIME_BOUNDARY + "\r\n")
         tmp.write(
             'Content-Disposition: form-data; name="deck"; filename="deck"\r\n'
         )
         tmp.write('Content-Type: application/octet-stream\r\n')
         tmp.write('\r\n')
         # data
         comp = zlib.compressobj()
         while 1:
             data = src.read(CHUNK_SIZE)
             if not data:
                 tmp.write(comp.flush())
                 break
             tmp.write(comp.compress(data))
         src.close()
         tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
         size = tmp.tell()
         tmp.seek(0)
         # open http connection
         runHook("fullSyncStarted", size)
         headers = {
             'Content-type':
             'multipart/form-data; boundary=%s' % MIME_BOUNDARY,
             'Content-length': str(size),
             'Host': SYNC_HOST,
         }
         req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers)
         try:
             sendProgressHook = fullSyncProgressHook
             res = urllib2.urlopen(req).read()
             assert res.startswith("OK")
             # update lastSync
             c = sqlite.connect(path)
             c.execute("update decks set lastSync = ?", (res[3:], ))
             c.commit()
             c.close()
         finally:
             sendProgressHook = None
             tmp.close()
             os.close(fd)
             os.unlink(name)
     finally:
         runHook("fullSyncFinished")
Example #4
0
 def fullSyncFromLocal(self, fields, path):
     global sendProgressHook
     try:
         # write into a temporary file, since POST needs content-length
         src = open(path, "rb")
         (fd, name) = tempfile.mkstemp(prefix="anki")
         tmp = open(name, "w+b")
         # post vars
         for (key, value) in fields.items():
             tmp.write('--' + MIME_BOUNDARY + "\r\n")
             tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
             tmp.write('\r\n')
             tmp.write(value)
             tmp.write('\r\n')
         # file header
         tmp.write('--' + MIME_BOUNDARY + "\r\n")
         tmp.write(
             'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
         tmp.write('Content-Type: application/octet-stream\r\n')
         tmp.write('\r\n')
         # data
         comp = zlib.compressobj()
         while 1:
             data = src.read(CHUNK_SIZE)
             if not data:
                 tmp.write(comp.flush())
                 break
             tmp.write(comp.compress(data))
         src.close()
         tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
         size = tmp.tell()
         tmp.seek(0)
         # open http connection
         runHook("fullSyncStarted", size)
         headers = {
             'Content-type': 'multipart/form-data; boundary=%s' %
             MIME_BOUNDARY,
             'Content-length': str(size),
             'Host': SYNC_HOST,
             }
         req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers)
         try:
             sendProgressHook = fullSyncProgressHook
             res = urllib2.urlopen(req).read()
             assert res.startswith("OK")
             # update lastSync
             c = sqlite.connect(path)
             c.execute("update decks set lastSync = ?",
                       (res[3:],))
             c.commit()
             c.close()
         finally:
             sendProgressHook = None
             tmp.close()
             os.close(fd)
             os.unlink(name)
     finally:
         runHook("fullSyncFinished")
Example #5
0
 def fullSyncFromLocal(self, fields, path):
     global sendProgressHook
     try:
         # write into a temporary file, since POST needs content-length
         src = open(path, "rb")
         (fd, name) = tempfile.mkstemp(prefix="anki")
         tmp = open(name, "w+b")
         # post vars
         for (key, value) in fields.items():
             tmp.write('--' + MIME_BOUNDARY + "\r\n")
             tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
             tmp.write('\r\n')
             tmp.write(value)
             tmp.write('\r\n')
         # file header
         tmp.write('--' + MIME_BOUNDARY + "\r\n")
         tmp.write(
             'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
         tmp.write('Content-Type: application/octet-stream\r\n')
         tmp.write('\r\n')
         # data
         comp = zlib.compressobj()
         while 1:
             data = src.read(32768)
             if not data:
                 tmp.write(comp.flush())
                 break
             tmp.write(comp.compress(data))
         tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
         size = tmp.tell()
         tmp.seek(0)
         # open http connection
         socket.setdefaulttimeout(SOCKET_TIMEOUT)
         runHook("fullSyncStarted", size)
         headers = {
             'Content-type': 'multipart/form-data; boundary=%s' %
             MIME_BOUNDARY,
             'Content-length': str(size),
             'Host': SYNC_HOST,
             }
         req = urllib2.Request(SYNC_URL + "fullup", tmp, headers)
         try:
             sendProgressHook = fullSyncProgressHook
             assert urllib2.urlopen(req).read() == "OK"
         finally:
             sendProgressHook = None
             tmp.close()
             os.close(fd)
             os.unlink(name)
     finally:
         socket.setdefaulttimeout(None)
         runHook("fullSyncFinished")
Example #6
0
 def upload(self):
     "True if upload successful."
     runHook("sync", "upload")
     # make sure it's ok before we try to upload
     if self.col.db.scalar("pragma integrity_check") != "ok":
         return False
     if not self.col.basicCheck():
         return False
     # apply some adjustments, then upload
     self.col.beforeUpload()
     if self.req("upload", open(self.col.path, "rb")) != "OK":
         return False
     return True
Example #7
0
 def upload(self):
     "True if upload successful."
     runHook("sync", "upload")
     # make sure it's ok before we try to upload
     if self.col.db.scalar("pragma integrity_check") != "ok":
         return False
     if not self.col.basicCheck():
         return False
     # apply some adjustments, then upload
     self.col.beforeUpload()
     if self.req("upload", open(self.col.path, "rb")) != "OK":
         return False
     return True
Example #8
0
    def _downloadFiles(self, fnames):
        self.col.log("%d files to fetch" % len(fnames))
        while fnames:
            top = fnames[0:SYNC_ZIP_COUNT]
            self.col.log("fetch %s" % top)
            zipData = self.server.downloadFiles(files=top)
            cnt = self.col.media.addFilesFromZip(zipData)
            self.downloadCount += cnt
            self.col.log("received %d files" % cnt)
            fnames = fnames[cnt:]

            n = self.downloadCount
            runHook("syncMsg", ngettext("%d media file downloaded", "%d media files downloaded", n) % n)
Example #9
0
 def download(self):
     runHook("sync", "download")
     self.col.close()
     cont = self.req("download")
     tpath = self.col.path + ".tmp"
     open(tpath, "wb").write(cont)
     # check the received file is ok
     d = DB(tpath)
     assert d.scalar("pragma integrity_check") == "ok"
     d.close()
     # overwrite existing collection
     os.unlink(self.col.path)
     os.rename(tpath, self.col.path)
     self.col = None
Example #10
0
 def download(self):
     runHook("sync", "download")
     self.col.close()
     cont = self.req("download")
     tpath = self.col.path + ".tmp"
     open(tpath, "wb").write(cont)
     # check the received file is ok
     d = DB(tpath)
     assert d.scalar("pragma integrity_check") == "ok"
     d.close()
     # overwrite existing collection
     os.unlink(self.col.path)
     os.rename(tpath, self.col.path)
     self.col = None
Example #11
0
    def _downloadFiles(self, fnames):
        self.col.log("%d files to fetch" % len(fnames))
        while fnames:
            top = fnames[0:SYNC_ZIP_COUNT]
            self.col.log("fetch %s" % top)
            zipData = self.server.downloadFiles(files=top)
            cnt = self.col.media.addFilesFromZip(zipData)
            self.downloadCount += cnt
            self.col.log("received %d files" % cnt)
            fnames = fnames[cnt:]

            n = self.downloadCount
            runHook(
                "syncMsg",
                ngettext("%d media file downloaded",
                         "%d media files downloaded", n) % n)
Example #12
0
 def fullSyncFromServer(self, fields, path):
     try:
         runHook("fullSyncStarted", 0)
         socket.setdefaulttimeout(SOCKET_TIMEOUT)
         fields = urllib.urlencode(fields)
         src = urllib.urlopen(SYNC_URL + "fulldown", fields)
         (fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path),
                                          prefix="fullsync")
         tmp = open(tmpname, "wb")
         decomp = zlib.decompressobj()
         cnt = 0
         while 1:
             data = src.read(32768)
             if not data:
                 tmp.write(decomp.flush())
                 break
             tmp.write(decomp.decompress(data))
             cnt += 32768
             runHook("fullSyncProgress", "fromServer", cnt)
         src.close()
         tmp.close()
         os.close(fd)
         # if we were successful, overwrite old deck
         os.unlink(path)
         os.rename(tmpname, path)
     finally:
         socket.setdefaulttimeout(None)
         runHook("fullSyncFinished")
Example #13
0
 def fullSyncFromServer(self, fields, path):
     try:
         runHook("fullSyncStarted", 0)
         fields = urllib.urlencode(fields)
         src = urllib.urlopen(SYNC_URL + "fulldown", fields)
         (fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path),
                                          prefix="fullsync")
         tmp = open(tmpname, "wb")
         decomp = zlib.decompressobj()
         cnt = 0
         while 1:
             data = src.read(CHUNK_SIZE)
             if not data:
                 tmp.write(decomp.flush())
                 break
             tmp.write(decomp.decompress(data))
             cnt += CHUNK_SIZE
             runHook("fullSyncProgress", "fromServer", cnt)
         src.close()
         tmp.close()
         os.close(fd)
         # if we were successful, overwrite old deck
         os.unlink(path)
         os.rename(tmpname, path)
         # reset the deck name
         c = sqlite.connect(path)
         c.execute("update decks set syncName = ?",
                   [checksum(path.encode("utf-8"))])
         c.commit()
         c.close()
     finally:
         runHook("fullSyncFinished")
Example #14
0
 def fullSyncFromServer(self, fields, path):
     try:
         runHook("fullSyncStarted", 0)
         fields = urllib.urlencode(fields)
         src = urllib2.urlopen(SYNC_URL + "fulldown", fields)
         tmpname = namedtmp("fullsync.anki")
         tmp = open(tmpname, "wb")
         decomp = zlib.decompressobj()
         cnt = 0
         while 1:
             data = src.read(CHUNK_SIZE)
             if not data:
                 tmp.write(decomp.flush())
                 break
             tmp.write(decomp.decompress(data))
             cnt += CHUNK_SIZE
             runHook("fullSyncProgress", "fromServer", cnt)
         src.close()
         tmp.close()
         os.close(fd)
         # if we were successful, overwrite old deck
         os.unlink(path)
         os.rename(tmpname, path)
         # reset the deck name
         c = sqlite.connect(path)
         c.execute("update decks set syncName = ?",
                   [checksum(path.encode("utf-8"))])
         c.commit()
         c.close()
     finally:
         runHook("fullSyncFinished")
Example #15
0
 def fullSyncFromLocal(self, fields, path):
     try:
         # write into a temporary file, since POST needs content-length
         src = open(path, "rb")
         (fd, name) = tempfile.mkstemp(prefix="anki")
         tmp = open(name, "w+b")
         # post vars
         for (key, value) in fields.items():
             tmp.write('--' + MIME_BOUNDARY + "\r\n")
             tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
             tmp.write('\r\n')
             tmp.write(value)
             tmp.write('\r\n')
         # file header
         tmp.write('--' + MIME_BOUNDARY + "\r\n")
         tmp.write(
             'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
         tmp.write('Content-Type: application/octet-stream\r\n')
         tmp.write('\r\n')
         # data
         comp = zlib.compressobj()
         while 1:
             data = src.read(65536)
             if not data:
                 tmp.write(comp.flush())
                 break
             tmp.write(comp.compress(data))
         tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
         size = tmp.tell()
         tmp.seek(0)
         # open http connection
         runHook("fullSyncStarted", size)
         h = httplib.HTTP(SYNC_HOST, SYNC_PORT)
         h.putrequest('POST', "/sync/fullup")
         h.putheader('Content-type', 'multipart/form-data; boundary=%s' %
                     MIME_BOUNDARY)
         h.putheader('Content-length', str(size))
         h.putheader('Host', SYNC_HOST)
         h.endheaders()
         dst = h._conn.sock.makefile("wb", 65536)
         # dump file
         cnt = 0
         while 1:
             runHook("fullSyncProgress", "fromLocal", cnt)
             data = tmp.read(65536)
             if not data:
                 break
             dst.write(data)
             cnt += len(data)
         # wait for reply
         dst.close()
         tmp.close()
         os.close(fd)
         errcode, errmsg, headers = h.getreply()
         assert errcode == 200
     finally:
         runHook("fullSyncFinished")
Example #16
0
def fullSyncProgressHook(cnt):
    runHook("fullSyncProgress", "fromLocal", cnt)
Example #17
0
 def sync(self, mediaUsn):
     # step 1: check if there have been any changes
     runHook("sync", "findMedia")
     lusn = self.col.media.usn()
     # if first sync or resync, clear list of files we think we've sent
     if not lusn:
         self.col.media.forceResync()
     self.col.media.findChanges()
     if lusn == mediaUsn and not self.col.media.hasChanged():
         return "noChanges"
     # step 1.5: if resyncing, we need to get the list of files the server
     # has and remove them from our local list of files to sync
     if not lusn:
         files = self.server.mediaList()
         need = self.col.media.removeExisting(files)
     else:
         need = None
     # step 2: send/recv deletions
     runHook("sync", "removeMedia")
     lrem = self.removed()
     rrem = self.server.remove(fnames=lrem, minUsn=lusn)
     self.remove(rrem)
     # step 3: stream files from server
     runHook("sync", "server")
     while 1:
         runHook("sync", "streamMedia")
         usn = self.col.media.usn()
         zip = self.server.files(minUsn=usn, need=need)
         if self.addFiles(zip=zip):
             break
     # step 4: stream files to the server
     runHook("sync", "client")
     while 1:
         runHook("sync", "streamMedia")
         zip, fnames = self.files()
         if not fnames:
             # finished
             break
         usn = self.server.addFiles(zip=zip)
         # after server has replied, safe to remove from log
         self.col.media.forgetAdded(fnames)
         self.col.media.setUsn(usn)
     # step 5: sanity check during beta testing
     # NOTE: when removing this, need to move server tidyup
     # back from sanity check to addFiles
     s = self.server.mediaSanity()
     c = self.mediaSanity()
     self.col.log("mediaSanity", c, s)
     if c != s:
         # if the sanity check failed, force a resync
         self.col.media.forceResync()
         return "sanityCheckFailed"
Example #18
0
 def sync(self):
     "Returns 'noChanges', 'fullSync', or 'success'."
     # if the deck has any pending changes, flush them first and bump mod
     # time
     self.col.save()
     # step 1: login & metadata
     runHook("sync", "login")
     ret = self.server.meta()
     if not ret:
         return "badAuth"
     self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
     self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
     if abs(rts - lts) > 300:
         return "clockOff"
     if self.lmod == self.rmod:
         return "noChanges"
     elif lscm != rscm:
         return "fullSync"
     self.lnewer = self.lmod > self.rmod
     # step 1.5: check collection is valid
     if not self.col.basicCheck():
         return "basicCheckFailed"
     # step 2: deletions
     runHook("sync", "meta")
     lrem = self.removed()
     rrem = self.server.start(
         minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
     self.remove(rrem)
     # ...and small objects
     lchg = self.changes()
     rchg = self.server.applyChanges(changes=lchg)
     self.mergeChanges(lchg, rchg)
     # step 3: stream large tables from server
     runHook("sync", "server")
     while 1:
         runHook("sync", "stream")
         chunk = self.server.chunk()
         self.applyChunk(chunk=chunk)
         if chunk['done']:
             break
     # step 4: stream to server
     runHook("sync", "client")
     while 1:
         runHook("sync", "stream")
         chunk = self.chunk()
         self.server.applyChunk(chunk=chunk)
         if chunk['done']:
             break
     # step 5: sanity check
     runHook("sync", "sanity")
     c = self.sanityCheck()
     ret = self.server.sanityCheck2(client=c)
     if ret['status'] != "ok":
         # roll back and force full sync
         self.col.rollback()
         self.col.modSchema()
         self.col.save()
         return "sanityCheckFailed"
     # finalize
     runHook("sync", "finalize")
     mod = self.server.finish()
     self.finish(mod)
     return "success"
Example #19
0
    def sync(self, mediaUsn):
        # step 1: check if there have been any changes
        runHook("sync", "findMedia")
        self.col.media.findChanges()
        lusn = self.col.media.usn()
        if lusn == mediaUsn and not self.col.media.hasChanged():
            return "noChanges"
        # step 2: send/recv deletions
        runHook("sync", "removeMedia")
        lrem = self.removed()
        rrem = self.server.remove(fnames=lrem, minUsn=lusn)
        self.remove(rrem)
        # step 3: stream files from server
        runHook("sync", "server")
        while 1:
            runHook("sync", "streamMedia")
            usn = self.col.media.usn()
            zip = self.server.files(minUsn=usn)
            if self.addFiles(zip=zip):
                break
        # step 4: stream files to the server
        runHook("sync", "client")
        while 1:
            runHook("sync", "streamMedia")
            zip, fnames = self.files()
            if not fnames:
                # finished
                break
            usn = self.server.addFiles(zip=zip)
            # after server has replied, safe to remove from log
            self.col.media.forgetAdded(fnames)
            self.col.media.setUsn(usn)
        # step 5: sanity check during beta testing
        # NOTE: when removing this, need to move server tidyup
        # back from sanity check to addFiles
        s = self.server.mediaSanity()
        c = self.mediaSanity()
        if c != s:
            raise Exception("""\
Media sanity check failed. Please copy and paste the text below:\n%s\n%s""" %
                            (c, s))
        return "success"
Example #20
0
    def sync(self):
        "Returns 'noChanges', 'fullSync', or 'success'."
        # if the deck has any pending changes, flush them first and bump mod
        # time
        self.col.save()
        # step 1: login & metadata
        runHook("sync", "login")
        ret = self.server.meta()
        if not ret:
            return "badAuth"
        self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
        self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
        if abs(rts - lts) > 300:
            return "clockOff"
        if self.lmod == self.rmod:
            return "noChanges"
        elif lscm != rscm:
            return "fullSync"
        self.lnewer = self.lmod > self.rmod
        # step 2: deletions
        runHook("sync", "meta")
        lrem = self.removed()
        rrem = self.server.start(
            minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
        self.remove(rrem)
        # ...and small objects
        lchg = self.changes()
        rchg = self.server.applyChanges(changes=lchg)
        self.mergeChanges(lchg, rchg)
        # step 3: stream large tables from server
        runHook("sync", "server")
        while 1:
            runHook("sync", "stream")
            chunk = self.server.chunk()
            self.applyChunk(chunk=chunk)
            if chunk['done']:
                break
        # step 4: stream to server
        runHook("sync", "client")
        while 1:
            runHook("sync", "stream")
            chunk = self.chunk()
            self.server.applyChunk(chunk=chunk)
            if chunk['done']:
                break
        # step 5: sanity check during beta testing
        runHook("sync", "sanity")
        c = self.sanityCheck()
        s = self.server.sanityCheck()
        if c != s:
            raise Exception("""\
Sanity check failed. Please copy and paste the text below:\n%s\n%s""" % (c, s))
        # finalize
        runHook("sync", "finalize")
        mod = self.server.finish()
        self.finish(mod)
        return "success"
Example #21
0
def fullSyncProgressHook(cnt):
    runHook("fullSyncProgress", "fromLocal", cnt)
Example #22
0
    def sync(self):
        # check if there have been any changes
        runHook("sync", "findMedia")
        self.col.log("findChanges")
        self.col.media.findChanges()

        # begin session and check if in sync
        lastUsn = self.col.media.lastUsn()
        ret = self.server.begin()
        srvUsn = ret["usn"]
        if lastUsn == srvUsn and not self.col.media.haveDirty():
            return "noChanges"

        # loop through and process changes from server
        self.col.log("last local usn is %s" % lastUsn)
        self.downloadCount = 0
        while True:
            data = self.server.mediaChanges(lastUsn=lastUsn)

            self.col.log("mediaChanges resp count %d" % len(data))
            if not data:
                break

            need = []
            lastUsn = data[-1][1]
            for fname, rusn, rsum in data:
                lsum, ldirty = self.col.media.syncInfo(fname)
                self.col.log(
                    "check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s"
                    % ((lsum and lsum[0:4]), (rsum and rsum[0:4]), ldirty, rusn, fname)
                )

                if rsum:
                    # added/changed remotely
                    if not lsum or lsum != rsum:
                        self.col.log("will fetch")
                        need.append(fname)
                    else:
                        self.col.log("have same already")
                    ldirty and self.col.media.markClean([fname])
                elif lsum:
                    # deleted remotely
                    if not ldirty:
                        self.col.log("delete local")
                        self.col.media.syncDelete(fname)
                    else:
                        # conflict; local add overrides remote delete
                        self.col.log("conflict; will send")
                else:
                    # deleted both sides
                    self.col.log("both sides deleted")
                    ldirty and self.col.media.markClean([fname])

            self._downloadFiles(need)

            self.col.log("update last usn to %d" % lastUsn)
            self.col.media.setLastUsn(lastUsn)  # commits

        # at this point we're all up to date with the server's changes,
        # and we need to send our own

        updateConflict = False
        toSend = self.col.media.dirtyCount()
        while True:
            zip, fnames = self.col.media.mediaChangesZip()
            if not fnames:
                break

            runHook("syncMsg", ngettext("%d media change to upload", "%d media changes to upload", toSend) % toSend)

            processedCnt, serverLastUsn = self.server.uploadChanges(zip)
            self.col.media.markClean(fnames[0:processedCnt])

            self.col.log("processed %d, serverUsn %d, clientUsn %d" % (processedCnt, serverLastUsn, lastUsn))

            if serverLastUsn - processedCnt == lastUsn:
                self.col.log("lastUsn in sync, updating local")
                lastUsn = serverLastUsn
                self.col.media.setLastUsn(serverLastUsn)  # commits
            else:
                self.col.log("concurrent update, skipping usn update")
                # commit for markClean
                self.col.media.db.commit()
                updateConflict = True

            toSend -= processedCnt

        if updateConflict:
            self.col.log("restart sync due to concurrent update")
            return self.sync()

        lcnt = self.col.media.mediaCount()
        ret = self.server.mediaSanity(local=lcnt)
        if ret == "OK":
            return "OK"
        else:
            self.col.media.forceResync()
            return ret
Example #23
0
 def sync(self, mediaUsn):
     # step 1: check if there have been any changes
     runHook("sync", "findMedia")
     lusn = self.col.media.usn()
     # if first sync or resync, clear list of files we think we've sent
     if not lusn:
         self.col.media.forceResync()
     self.col.media.findChanges()
     if lusn == mediaUsn and not self.col.media.hasChanged():
         return "noChanges"
     # step 1.5: if resyncing, we need to get the list of files the server
     # has and remove them from our local list of files to sync
     if not lusn:
         files = self.server.mediaList()
         need = self.col.media.removeExisting(files)
     else:
         need = None
     # step 2: send/recv deletions
     runHook("sync", "removeMedia")
     lrem = self.removed()
     rrem = self.server.remove(fnames=lrem, minUsn=lusn)
     self.remove(rrem)
     # step 3: stream files from server
     runHook("sync", "server")
     while 1:
         runHook("sync", "streamMedia")
         usn = self.col.media.usn()
         zip = self.server.files(minUsn=usn, need=need)
         if self.addFiles(zip=zip):
             break
     # step 4: stream files to the server
     runHook("sync", "client")
     while 1:
         runHook("sync", "streamMedia")
         zip, fnames = self.files()
         if not fnames:
             # finished
             break
         usn = self.server.addFiles(zip=zip)
         # after server has replied, safe to remove from log
         self.col.media.forgetAdded(fnames)
         self.col.media.setUsn(usn)
     # step 5: sanity check during beta testing
     # NOTE: when removing this, need to move server tidyup
     # back from sanity check to addFiles
     s = self.server.mediaSanity()
     c = self.mediaSanity()
     if c != s:
         # if the sanity check failed, force a resync
         self.col.media.forceResync()
         return "sanityCheckFailed"
Example #24
0
    def sync(self):
        # check if there have been any changes
        runHook("sync", "findMedia")
        self.col.log("findChanges")
        self.col.media.findChanges()

        # begin session and check if in sync
        lastUsn = self.col.media.lastUsn()
        ret = self.server.begin()
        srvUsn = ret['usn']
        if lastUsn == srvUsn and not self.col.media.haveDirty():
            return "noChanges"

        # loop through and process changes from server
        self.col.log("last local usn is %s" % lastUsn)
        self.downloadCount = 0
        while True:
            data = self.server.mediaChanges(lastUsn=lastUsn)

            self.col.log("mediaChanges resp count %d" % len(data))
            if not data:
                break

            need = []
            lastUsn = data[-1][1]
            for fname, rusn, rsum in data:
                lsum, ldirty = self.col.media.syncInfo(fname)
                self.col.log(
                    "check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s" %
                    ((lsum and lsum[0:4]),
                     (rsum and rsum[0:4]), ldirty, rusn, fname))

                if rsum:
                    # added/changed remotely
                    if not lsum or lsum != rsum:
                        self.col.log("will fetch")
                        need.append(fname)
                    else:
                        self.col.log("have same already")
                    ldirty and self.col.media.markClean([fname])
                elif lsum:
                    # deleted remotely
                    if not ldirty:
                        self.col.log("delete local")
                        self.col.media.syncDelete(fname)
                    else:
                        # conflict; local add overrides remote delete
                        self.col.log("conflict; will send")
                else:
                    # deleted both sides
                    self.col.log("both sides deleted")
                    ldirty and self.col.media.markClean([fname])

            self._downloadFiles(need)

            self.col.log("update last usn to %d" % lastUsn)
            self.col.media.setLastUsn(lastUsn)  # commits

        # at this point we're all up to date with the server's changes,
        # and we need to send our own

        updateConflict = False
        toSend = self.col.media.dirtyCount()
        while True:
            zip, fnames = self.col.media.mediaChangesZip()
            if not fnames:
                break

            runHook(
                "syncMsg",
                ngettext("%d media change to upload",
                         "%d media changes to upload", toSend) % toSend)

            processedCnt, serverLastUsn = self.server.uploadChanges(zip)
            self.col.media.markClean(fnames[0:processedCnt])

            self.col.log("processed %d, serverUsn %d, clientUsn %d" %
                         (processedCnt, serverLastUsn, lastUsn))

            if serverLastUsn - processedCnt == lastUsn:
                self.col.log("lastUsn in sync, updating local")
                lastUsn = serverLastUsn
                self.col.media.setLastUsn(serverLastUsn)  # commits
            else:
                self.col.log("concurrent update, skipping usn update")
                # commit for markClean
                self.col.media.db.commit()
                updateConflict = True

            toSend -= processedCnt

        if updateConflict:
            self.col.log("restart sync due to concurrent update")
            return self.sync()

        lcnt = self.col.media.mediaCount()
        ret = self.server.mediaSanity(local=lcnt)
        if ret == "OK":
            return "OK"
        else:
            self.col.media.forceResync()
            return ret
Example #25
0
 def sync(self):
     "Returns 'noChanges', 'fullSync', 'success', etc"
     self.syncMsg = ""
     self.uname = ""
     # if the deck has any pending changes, flush them first and bump mod
     # time
     self.col.save()
     # step 1: login & metadata
     runHook("sync", "login")
     meta = self.server.meta()
     self.col.log("rmeta", meta)
     if not meta:
         return "badAuth"
     # server requested abort?
     self.syncMsg = meta["msg"]
     if not meta["cont"]:
         return "serverAbort"
     else:
         # don't abort, but if 'msg' is not blank, gui should show 'msg'
         # after sync finishes and wait for confirmation before hiding
         pass
     rscm = meta["scm"]
     rts = meta["ts"]
     self.rmod = meta["mod"]
     self.maxUsn = meta["usn"]
     # this is a temporary measure to address the problem of users
     # forgetting which email address they've used - it will be removed
     # when enough time has passed
     self.uname = meta.get("uname", "")
     meta = self.meta()
     self.col.log("lmeta", meta)
     self.lmod = meta["mod"]
     self.minUsn = meta["usn"]
     lscm = meta["scm"]
     lts = meta["ts"]
     if abs(rts - lts) > 300:
         self.col.log("clock off")
         return "clockOff"
     if self.lmod == self.rmod:
         self.col.log("no changes")
         return "noChanges"
     elif lscm != rscm:
         self.col.log("schema diff")
         return "fullSync"
     self.lnewer = self.lmod > self.rmod
     # step 1.5: check collection is valid
     if not self.col.basicCheck():
         self.col.log("basic check")
         return "basicCheckFailed"
     # step 2: deletions
     runHook("sync", "meta")
     lrem = self.removed()
     rrem = self.server.start(minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
     self.remove(rrem)
     # ...and small objects
     lchg = self.changes()
     rchg = self.server.applyChanges(changes=lchg)
     self.mergeChanges(lchg, rchg)
     # step 3: stream large tables from server
     runHook("sync", "server")
     while 1:
         runHook("sync", "stream")
         chunk = self.server.chunk()
         self.col.log("server chunk", chunk)
         self.applyChunk(chunk=chunk)
         if chunk["done"]:
             break
     # step 4: stream to server
     runHook("sync", "client")
     while 1:
         runHook("sync", "stream")
         chunk = self.chunk()
         self.col.log("client chunk", chunk)
         self.server.applyChunk(chunk=chunk)
         if chunk["done"]:
             break
     # step 5: sanity check
     runHook("sync", "sanity")
     c = self.sanityCheck()
     ret = self.server.sanityCheck2(client=c)
     if ret["status"] != "ok":
         # roll back and force full sync
         self.col.rollback()
         self.col.modSchema(False)
         self.col.save()
         return "sanityCheckFailed"
     # finalize
     runHook("sync", "finalize")
     mod = self.server.finish()
     self.finish(mod)
     return "success"
Example #26
0
    def sync(self):
        "Returns 'noChanges', 'fullSync', or 'success'."
        # if the deck has any pending changes, flush them first and bump mod
        # time
        self.col.save()
        # step 1: login & metadata
        runHook("sync", "login")
        ret = self.server.meta()
        if not ret:
            return "badAuth"
        self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
        self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
        if abs(rts - lts) > 300:
            return "clockOff"
        if self.lmod == self.rmod:
            return "noChanges"
        elif lscm != rscm:
            return "fullSync"
        self.lnewer = self.lmod > self.rmod
        # step 2: deletions
        runHook("sync", "meta")
        lrem = self.removed()
        rrem = self.server.start(minUsn=self.minUsn,
                                 lnewer=self.lnewer,
                                 graves=lrem)
        self.remove(rrem)
        # ...and small objects
        lchg = self.changes()
        rchg = self.server.applyChanges(changes=lchg)
        self.mergeChanges(lchg, rchg)
        # step 3: stream large tables from server
        runHook("sync", "server")
        while 1:
            runHook("sync", "stream")
            chunk = self.server.chunk()
            self.applyChunk(chunk=chunk)
            if chunk['done']:
                break
        # step 4: stream to server
        runHook("sync", "client")
        while 1:
            runHook("sync", "stream")
            chunk = self.chunk()
            self.server.applyChunk(chunk=chunk)
            if chunk['done']:
                break
        # step 5: sanity check during beta testing
        runHook("sync", "sanity")
        c = self.sanityCheck()
        s = self.server.sanityCheck()
        if c != s:
            raise Exception("""\
Sanity check failed. Please copy and paste the text below:\n%s\n%s""" % (c, s))
        # finalize
        runHook("sync", "finalize")
        mod = self.server.finish()
        self.finish(mod)
        return "success"
Example #27
0
    def sync(self, mediaUsn):
        # step 1: check if there have been any changes
        runHook("sync", "findMedia")
        self.col.media.findChanges()
        lusn = self.col.media.usn()
        if lusn == mediaUsn and not self.col.media.hasChanged():
            return "noChanges"
        # step 2: send/recv deletions
        runHook("sync", "removeMedia")
        lrem = self.removed()
        rrem = self.server.remove(fnames=lrem, minUsn=lusn)
        self.remove(rrem)
        # step 3: stream files from server
        runHook("sync", "server")
        while 1:
            runHook("sync", "streamMedia")
            usn = self.col.media.usn()
            zip = self.server.files(minUsn=usn)
            if self.addFiles(zip=zip):
                break
        # step 4: stream files to the server
        runHook("sync", "client")
        while 1:
            runHook("sync", "streamMedia")
            zip, fnames = self.files()
            if not fnames:
                # finished
                break
            usn = self.server.addFiles(zip=zip)
            # after server has replied, safe to remove from log
            self.col.media.forgetAdded(fnames)
            self.col.media.setUsn(usn)
        # step 5: sanity check during beta testing
        # NOTE: when removing this, need to move server tidyup
        # back from sanity check to addFiles
        s = self.server.mediaSanity()
        c = self.mediaSanity()
        if c != s:
            raise Exception("""\
Media sanity check failed. Please copy and paste the text below:\n%s\n%s""" %
                            (c, s))
        return "success"
Example #28
0
 def sync(self):
     "Returns 'noChanges', 'fullSync', 'success', etc"
     self.syncMsg = ""
     self.uname = ""
     # if the deck has any pending changes, flush them first and bump mod
     # time
     self.col.save()
     # step 1: login & metadata
     runHook("sync", "login")
     meta = self.server.meta()
     self.col.log("rmeta", meta)
     if not meta:
         return "badAuth"
     rscm = meta['scm']
     rts = meta['ts']
     self.rmod = meta['mod']
     self.maxUsn = meta['usn']
     self.mediaUsn = meta['musn']
     self.syncMsg = meta['msg']
     # this is a temporary measure to address the problem of users
     # forgetting which email address they've used - it will be removed
     # when enough time has passed
     self.uname = meta.get("uname", "")
     # server requested abort?
     if not meta['cont']:
         return "serverAbort"
     else:
         # don't abort, but ui should show message after sync finishes
         # and require confirmation if it's non-empty
         pass
     meta = self.meta()
     self.col.log("lmeta", meta)
     self.lmod = meta['mod']
     self.minUsn = meta['usn']
     lscm = meta['scm']
     lts = meta['ts']
     if abs(rts - lts) > 300:
         self.col.log("clock off")
         return "clockOff"
     if self.lmod == self.rmod:
         self.col.log("no changes")
         return "noChanges"
     elif lscm != rscm:
         self.col.log("schema diff")
         return "fullSync"
     self.lnewer = self.lmod > self.rmod
     # step 1.5: check collection is valid
     if not self.col.basicCheck():
         self.col.log("basic check")
         return "basicCheckFailed"
     # step 2: deletions
     runHook("sync", "meta")
     lrem = self.removed()
     rrem = self.server.start(minUsn=self.minUsn,
                              lnewer=self.lnewer,
                              graves=lrem)
     self.remove(rrem)
     # ...and small objects
     lchg = self.changes()
     rchg = self.server.applyChanges(changes=lchg)
     self.mergeChanges(lchg, rchg)
     # step 3: stream large tables from server
     runHook("sync", "server")
     while 1:
         runHook("sync", "stream")
         chunk = self.server.chunk()
         self.col.log("server chunk", chunk)
         self.applyChunk(chunk=chunk)
         if chunk['done']:
             break
     # step 4: stream to server
     runHook("sync", "client")
     while 1:
         runHook("sync", "stream")
         chunk = self.chunk()
         self.col.log("client chunk", chunk)
         self.server.applyChunk(chunk=chunk)
         if chunk['done']:
             break
     # step 5: sanity check
     runHook("sync", "sanity")
     c = self.sanityCheck()
     ret = self.server.sanityCheck2(client=c)
     if ret['status'] != "ok":
         # roll back and force full sync
         self.col.rollback()
         self.col.modSchema()
         self.col.save()
         return "sanityCheckFailed"
     # finalize
     runHook("sync", "finalize")
     mod = self.server.finish()
     self.finish(mod)
     return "success"