def upload(self):
runHook("sync", "upload")
# make sure it's ok before we try to upload
assert self.col.db.scalar("pragma integrity_check") == "ok"
# apply some adjustments, then upload
self.col.beforeUpload()
assert self.req("upload", open(self.col.path, "rb")) == "OK"
def upload(self):
runHook("sync", "upload")
# make sure it's ok before we try to upload
assert self.col.db.scalar("pragma integrity_check") == "ok"
# apply some adjustments, then upload
self.col.beforeUpload()
assert self.req("upload", open(self.col.path, "rb")) == "OK"
def fullSyncFromLocal(self, fields, path):
global sendProgressHook
try:
# write into a temporary file, since POST needs content-length
src = open(path, "rb")
(fd, name) = tempfile.mkstemp(prefix="oldanki")
tmp = open(name, "w+b")
# post vars
for (key, value) in fields.items():
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write('Content-Disposition: form-data; name="%s"\r\n' %
key)
tmp.write('\r\n')
tmp.write(value)
tmp.write('\r\n')
# file header
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write(
'Content-Disposition: form-data; name="deck"; filename="deck"\r\n'
)
tmp.write('Content-Type: application/octet-stream\r\n')
tmp.write('\r\n')
# data
comp = zlib.compressobj()
while 1:
data = src.read(CHUNK_SIZE)
if not data:
tmp.write(comp.flush())
break
tmp.write(comp.compress(data))
src.close()
tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
size = tmp.tell()
tmp.seek(0)
# open http connection
runHook("fullSyncStarted", size)
headers = {
'Content-type':
'multipart/form-data; boundary=%s' % MIME_BOUNDARY,
'Content-length': str(size),
'Host': SYNC_HOST,
}
req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers)
try:
sendProgressHook = fullSyncProgressHook
res = urllib2.urlopen(req).read()
assert res.startswith("OK")
# update lastSync
c = sqlite.connect(path)
c.execute("update decks set lastSync = ?", (res[3:], ))
c.commit()
c.close()
finally:
sendProgressHook = None
tmp.close()
os.close(fd)
os.unlink(name)
finally:
runHook("fullSyncFinished")
def fullSyncFromLocal(self, fields, path):
global sendProgressHook
try:
# write into a temporary file, since POST needs content-length
src = open(path, "rb")
(fd, name) = tempfile.mkstemp(prefix="anki")
tmp = open(name, "w+b")
# post vars
for (key, value) in fields.items():
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
tmp.write('\r\n')
tmp.write(value)
tmp.write('\r\n')
# file header
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write(
'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
tmp.write('Content-Type: application/octet-stream\r\n')
tmp.write('\r\n')
# data
comp = zlib.compressobj()
while 1:
data = src.read(CHUNK_SIZE)
if not data:
tmp.write(comp.flush())
break
tmp.write(comp.compress(data))
src.close()
tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
size = tmp.tell()
tmp.seek(0)
# open http connection
runHook("fullSyncStarted", size)
headers = {
'Content-type': 'multipart/form-data; boundary=%s' %
MIME_BOUNDARY,
'Content-length': str(size),
'Host': SYNC_HOST,
}
req = urllib2.Request(SYNC_URL + "fullup?v=2", tmp, headers)
try:
sendProgressHook = fullSyncProgressHook
res = urllib2.urlopen(req).read()
assert res.startswith("OK")
# update lastSync
c = sqlite.connect(path)
c.execute("update decks set lastSync = ?",
(res[3:],))
c.commit()
c.close()
finally:
sendProgressHook = None
tmp.close()
os.close(fd)
os.unlink(name)
finally:
runHook("fullSyncFinished")
def fullSyncFromLocal(self, fields, path):
global sendProgressHook
try:
# write into a temporary file, since POST needs content-length
src = open(path, "rb")
(fd, name) = tempfile.mkstemp(prefix="anki")
tmp = open(name, "w+b")
# post vars
for (key, value) in fields.items():
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
tmp.write('\r\n')
tmp.write(value)
tmp.write('\r\n')
# file header
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write(
'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
tmp.write('Content-Type: application/octet-stream\r\n')
tmp.write('\r\n')
# data
comp = zlib.compressobj()
while 1:
data = src.read(32768)
if not data:
tmp.write(comp.flush())
break
tmp.write(comp.compress(data))
tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
size = tmp.tell()
tmp.seek(0)
# open http connection
socket.setdefaulttimeout(SOCKET_TIMEOUT)
runHook("fullSyncStarted", size)
headers = {
'Content-type': 'multipart/form-data; boundary=%s' %
MIME_BOUNDARY,
'Content-length': str(size),
'Host': SYNC_HOST,
}
req = urllib2.Request(SYNC_URL + "fullup", tmp, headers)
try:
sendProgressHook = fullSyncProgressHook
assert urllib2.urlopen(req).read() == "OK"
finally:
sendProgressHook = None
tmp.close()
os.close(fd)
os.unlink(name)
finally:
socket.setdefaulttimeout(None)
runHook("fullSyncFinished")
def upload(self):
"True if upload successful."
runHook("sync", "upload")
# make sure it's ok before we try to upload
if self.col.db.scalar("pragma integrity_check") != "ok":
return False
if not self.col.basicCheck():
return False
# apply some adjustments, then upload
self.col.beforeUpload()
if self.req("upload", open(self.col.path, "rb")) != "OK":
return False
return True
def upload(self):
"True if upload successful."
runHook("sync", "upload")
# make sure it's ok before we try to upload
if self.col.db.scalar("pragma integrity_check") != "ok":
return False
if not self.col.basicCheck():
return False
# apply some adjustments, then upload
self.col.beforeUpload()
if self.req("upload", open(self.col.path, "rb")) != "OK":
return False
return True
def _downloadFiles(self, fnames):
self.col.log("%d files to fetch" % len(fnames))
while fnames:
top = fnames[0:SYNC_ZIP_COUNT]
self.col.log("fetch %s" % top)
zipData = self.server.downloadFiles(files=top)
cnt = self.col.media.addFilesFromZip(zipData)
self.downloadCount += cnt
self.col.log("received %d files" % cnt)
fnames = fnames[cnt:]
n = self.downloadCount
runHook("syncMsg", ngettext("%d media file downloaded", "%d media files downloaded", n) % n)
def download(self):
runHook("sync", "download")
self.col.close()
cont = self.req("download")
tpath = self.col.path + ".tmp"
open(tpath, "wb").write(cont)
# check the received file is ok
d = DB(tpath)
assert d.scalar("pragma integrity_check") == "ok"
d.close()
# overwrite existing collection
os.unlink(self.col.path)
os.rename(tpath, self.col.path)
self.col = None
def download(self):
runHook("sync", "download")
self.col.close()
cont = self.req("download")
tpath = self.col.path + ".tmp"
open(tpath, "wb").write(cont)
# check the received file is ok
d = DB(tpath)
assert d.scalar("pragma integrity_check") == "ok"
d.close()
# overwrite existing collection
os.unlink(self.col.path)
os.rename(tpath, self.col.path)
self.col = None
def _downloadFiles(self, fnames):
self.col.log("%d files to fetch" % len(fnames))
while fnames:
top = fnames[0:SYNC_ZIP_COUNT]
self.col.log("fetch %s" % top)
zipData = self.server.downloadFiles(files=top)
cnt = self.col.media.addFilesFromZip(zipData)
self.downloadCount += cnt
self.col.log("received %d files" % cnt)
fnames = fnames[cnt:]
n = self.downloadCount
runHook(
"syncMsg",
ngettext("%d media file downloaded",
"%d media files downloaded", n) % n)
def fullSyncFromServer(self, fields, path):
try:
runHook("fullSyncStarted", 0)
socket.setdefaulttimeout(SOCKET_TIMEOUT)
fields = urllib.urlencode(fields)
src = urllib.urlopen(SYNC_URL + "fulldown", fields)
(fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path),
prefix="fullsync")
tmp = open(tmpname, "wb")
decomp = zlib.decompressobj()
cnt = 0
while 1:
data = src.read(32768)
if not data:
tmp.write(decomp.flush())
break
tmp.write(decomp.decompress(data))
cnt += 32768
runHook("fullSyncProgress", "fromServer", cnt)
src.close()
tmp.close()
os.close(fd)
# if we were successful, overwrite old deck
os.unlink(path)
os.rename(tmpname, path)
finally:
socket.setdefaulttimeout(None)
runHook("fullSyncFinished")
def fullSyncFromServer(self, fields, path):
try:
runHook("fullSyncStarted", 0)
fields = urllib.urlencode(fields)
src = urllib.urlopen(SYNC_URL + "fulldown", fields)
(fd, tmpname) = tempfile.mkstemp(dir=os.path.dirname(path),
prefix="fullsync")
tmp = open(tmpname, "wb")
decomp = zlib.decompressobj()
cnt = 0
while 1:
data = src.read(CHUNK_SIZE)
if not data:
tmp.write(decomp.flush())
break
tmp.write(decomp.decompress(data))
cnt += CHUNK_SIZE
runHook("fullSyncProgress", "fromServer", cnt)
src.close()
tmp.close()
os.close(fd)
# if we were successful, overwrite old deck
os.unlink(path)
os.rename(tmpname, path)
# reset the deck name
c = sqlite.connect(path)
c.execute("update decks set syncName = ?",
[checksum(path.encode("utf-8"))])
c.commit()
c.close()
finally:
runHook("fullSyncFinished")
def fullSyncFromServer(self, fields, path):
try:
runHook("fullSyncStarted", 0)
fields = urllib.urlencode(fields)
src = urllib2.urlopen(SYNC_URL + "fulldown", fields)
tmpname = namedtmp("fullsync.anki")
tmp = open(tmpname, "wb")
decomp = zlib.decompressobj()
cnt = 0
while 1:
data = src.read(CHUNK_SIZE)
if not data:
tmp.write(decomp.flush())
break
tmp.write(decomp.decompress(data))
cnt += CHUNK_SIZE
runHook("fullSyncProgress", "fromServer", cnt)
src.close()
tmp.close()
os.close(fd)
# if we were successful, overwrite old deck
os.unlink(path)
os.rename(tmpname, path)
# reset the deck name
c = sqlite.connect(path)
c.execute("update decks set syncName = ?",
[checksum(path.encode("utf-8"))])
c.commit()
c.close()
finally:
runHook("fullSyncFinished")
def fullSyncFromLocal(self, fields, path):
try:
# write into a temporary file, since POST needs content-length
src = open(path, "rb")
(fd, name) = tempfile.mkstemp(prefix="anki")
tmp = open(name, "w+b")
# post vars
for (key, value) in fields.items():
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write('Content-Disposition: form-data; name="%s"\r\n' % key)
tmp.write('\r\n')
tmp.write(value)
tmp.write('\r\n')
# file header
tmp.write('--' + MIME_BOUNDARY + "\r\n")
tmp.write(
'Content-Disposition: form-data; name="deck"; filename="deck"\r\n')
tmp.write('Content-Type: application/octet-stream\r\n')
tmp.write('\r\n')
# data
comp = zlib.compressobj()
while 1:
data = src.read(65536)
if not data:
tmp.write(comp.flush())
break
tmp.write(comp.compress(data))
tmp.write('\r\n--' + MIME_BOUNDARY + '--\r\n\r\n')
size = tmp.tell()
tmp.seek(0)
# open http connection
runHook("fullSyncStarted", size)
h = httplib.HTTP(SYNC_HOST, SYNC_PORT)
h.putrequest('POST', "/sync/fullup")
h.putheader('Content-type', 'multipart/form-data; boundary=%s' %
MIME_BOUNDARY)
h.putheader('Content-length', str(size))
h.putheader('Host', SYNC_HOST)
h.endheaders()
dst = h._conn.sock.makefile("wb", 65536)
# dump file
cnt = 0
while 1:
runHook("fullSyncProgress", "fromLocal", cnt)
data = tmp.read(65536)
if not data:
break
dst.write(data)
cnt += len(data)
# wait for reply
dst.close()
tmp.close()
os.close(fd)
errcode, errmsg, headers = h.getreply()
assert errcode == 200
finally:
runHook("fullSyncFinished")
def fullSyncProgressHook(cnt):
runHook("fullSyncProgress", "fromLocal", cnt)
def sync(self, mediaUsn):
# step 1: check if there have been any changes
runHook("sync", "findMedia")
lusn = self.col.media.usn()
# if first sync or resync, clear list of files we think we've sent
if not lusn:
self.col.media.forceResync()
self.col.media.findChanges()
if lusn == mediaUsn and not self.col.media.hasChanged():
return "noChanges"
# step 1.5: if resyncing, we need to get the list of files the server
# has and remove them from our local list of files to sync
if not lusn:
files = self.server.mediaList()
need = self.col.media.removeExisting(files)
else:
need = None
# step 2: send/recv deletions
runHook("sync", "removeMedia")
lrem = self.removed()
rrem = self.server.remove(fnames=lrem, minUsn=lusn)
self.remove(rrem)
# step 3: stream files from server
runHook("sync", "server")
while 1:
runHook("sync", "streamMedia")
usn = self.col.media.usn()
zip = self.server.files(minUsn=usn, need=need)
if self.addFiles(zip=zip):
break
# step 4: stream files to the server
runHook("sync", "client")
while 1:
runHook("sync", "streamMedia")
zip, fnames = self.files()
if not fnames:
# finished
break
usn = self.server.addFiles(zip=zip)
# after server has replied, safe to remove from log
self.col.media.forgetAdded(fnames)
self.col.media.setUsn(usn)
# step 5: sanity check during beta testing
# NOTE: when removing this, need to move server tidyup
# back from sanity check to addFiles
s = self.server.mediaSanity()
c = self.mediaSanity()
self.col.log("mediaSanity", c, s)
if c != s:
# if the sanity check failed, force a resync
self.col.media.forceResync()
return "sanityCheckFailed"
def sync(self):
"Returns 'noChanges', 'fullSync', or 'success'."
# if the deck has any pending changes, flush them first and bump mod
# time
self.col.save()
# step 1: login & metadata
runHook("sync", "login")
ret = self.server.meta()
if not ret:
return "badAuth"
self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
if abs(rts - lts) > 300:
return "clockOff"
if self.lmod == self.rmod:
return "noChanges"
elif lscm != rscm:
return "fullSync"
self.lnewer = self.lmod > self.rmod
# step 1.5: check collection is valid
if not self.col.basicCheck():
return "basicCheckFailed"
# step 2: deletions
runHook("sync", "meta")
lrem = self.removed()
rrem = self.server.start(
minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
self.remove(rrem)
# ...and small objects
lchg = self.changes()
rchg = self.server.applyChanges(changes=lchg)
self.mergeChanges(lchg, rchg)
# step 3: stream large tables from server
runHook("sync", "server")
while 1:
runHook("sync", "stream")
chunk = self.server.chunk()
self.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 4: stream to server
runHook("sync", "client")
while 1:
runHook("sync", "stream")
chunk = self.chunk()
self.server.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 5: sanity check
runHook("sync", "sanity")
c = self.sanityCheck()
ret = self.server.sanityCheck2(client=c)
if ret['status'] != "ok":
# roll back and force full sync
self.col.rollback()
self.col.modSchema()
self.col.save()
return "sanityCheckFailed"
# finalize
runHook("sync", "finalize")
mod = self.server.finish()
self.finish(mod)
return "success"
def sync(self, mediaUsn):
# step 1: check if there have been any changes
runHook("sync", "findMedia")
self.col.media.findChanges()
lusn = self.col.media.usn()
if lusn == mediaUsn and not self.col.media.hasChanged():
return "noChanges"
# step 2: send/recv deletions
runHook("sync", "removeMedia")
lrem = self.removed()
rrem = self.server.remove(fnames=lrem, minUsn=lusn)
self.remove(rrem)
# step 3: stream files from server
runHook("sync", "server")
while 1:
runHook("sync", "streamMedia")
usn = self.col.media.usn()
zip = self.server.files(minUsn=usn)
if self.addFiles(zip=zip):
break
# step 4: stream files to the server
runHook("sync", "client")
while 1:
runHook("sync", "streamMedia")
zip, fnames = self.files()
if not fnames:
# finished
break
usn = self.server.addFiles(zip=zip)
# after server has replied, safe to remove from log
self.col.media.forgetAdded(fnames)
self.col.media.setUsn(usn)
# step 5: sanity check during beta testing
# NOTE: when removing this, need to move server tidyup
# back from sanity check to addFiles
s = self.server.mediaSanity()
c = self.mediaSanity()
if c != s:
raise Exception("""\
Media sanity check failed. Please copy and paste the text below:\n%s\n%s""" %
(c, s))
return "success"
def sync(self):
"Returns 'noChanges', 'fullSync', or 'success'."
# if the deck has any pending changes, flush them first and bump mod
# time
self.col.save()
# step 1: login & metadata
runHook("sync", "login")
ret = self.server.meta()
if not ret:
return "badAuth"
self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
if abs(rts - lts) > 300:
return "clockOff"
if self.lmod == self.rmod:
return "noChanges"
elif lscm != rscm:
return "fullSync"
self.lnewer = self.lmod > self.rmod
# step 2: deletions
runHook("sync", "meta")
lrem = self.removed()
rrem = self.server.start(
minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
self.remove(rrem)
# ...and small objects
lchg = self.changes()
rchg = self.server.applyChanges(changes=lchg)
self.mergeChanges(lchg, rchg)
# step 3: stream large tables from server
runHook("sync", "server")
while 1:
runHook("sync", "stream")
chunk = self.server.chunk()
self.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 4: stream to server
runHook("sync", "client")
while 1:
runHook("sync", "stream")
chunk = self.chunk()
self.server.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 5: sanity check during beta testing
runHook("sync", "sanity")
c = self.sanityCheck()
s = self.server.sanityCheck()
if c != s:
raise Exception("""\
Sanity check failed. Please copy and paste the text below:\n%s\n%s""" % (c, s))
# finalize
runHook("sync", "finalize")
mod = self.server.finish()
self.finish(mod)
return "success"
def fullSyncProgressHook(cnt):
runHook("fullSyncProgress", "fromLocal", cnt)
def sync(self):
# check if there have been any changes
runHook("sync", "findMedia")
self.col.log("findChanges")
self.col.media.findChanges()
# begin session and check if in sync
lastUsn = self.col.media.lastUsn()
ret = self.server.begin()
srvUsn = ret["usn"]
if lastUsn == srvUsn and not self.col.media.haveDirty():
return "noChanges"
# loop through and process changes from server
self.col.log("last local usn is %s" % lastUsn)
self.downloadCount = 0
while True:
data = self.server.mediaChanges(lastUsn=lastUsn)
self.col.log("mediaChanges resp count %d" % len(data))
if not data:
break
need = []
lastUsn = data[-1][1]
for fname, rusn, rsum in data:
lsum, ldirty = self.col.media.syncInfo(fname)
self.col.log(
"check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s"
% ((lsum and lsum[0:4]), (rsum and rsum[0:4]), ldirty, rusn, fname)
)
if rsum:
# added/changed remotely
if not lsum or lsum != rsum:
self.col.log("will fetch")
need.append(fname)
else:
self.col.log("have same already")
ldirty and self.col.media.markClean([fname])
elif lsum:
# deleted remotely
if not ldirty:
self.col.log("delete local")
self.col.media.syncDelete(fname)
else:
# conflict; local add overrides remote delete
self.col.log("conflict; will send")
else:
# deleted both sides
self.col.log("both sides deleted")
ldirty and self.col.media.markClean([fname])
self._downloadFiles(need)
self.col.log("update last usn to %d" % lastUsn)
self.col.media.setLastUsn(lastUsn) # commits
# at this point we're all up to date with the server's changes,
# and we need to send our own
updateConflict = False
toSend = self.col.media.dirtyCount()
while True:
zip, fnames = self.col.media.mediaChangesZip()
if not fnames:
break
runHook("syncMsg", ngettext("%d media change to upload", "%d media changes to upload", toSend) % toSend)
processedCnt, serverLastUsn = self.server.uploadChanges(zip)
self.col.media.markClean(fnames[0:processedCnt])
self.col.log("processed %d, serverUsn %d, clientUsn %d" % (processedCnt, serverLastUsn, lastUsn))
if serverLastUsn - processedCnt == lastUsn:
self.col.log("lastUsn in sync, updating local")
lastUsn = serverLastUsn
self.col.media.setLastUsn(serverLastUsn) # commits
else:
self.col.log("concurrent update, skipping usn update")
# commit for markClean
self.col.media.db.commit()
updateConflict = True
toSend -= processedCnt
if updateConflict:
self.col.log("restart sync due to concurrent update")
return self.sync()
lcnt = self.col.media.mediaCount()
ret = self.server.mediaSanity(local=lcnt)
if ret == "OK":
return "OK"
else:
self.col.media.forceResync()
return ret
def sync(self, mediaUsn):
# step 1: check if there have been any changes
runHook("sync", "findMedia")
lusn = self.col.media.usn()
# if first sync or resync, clear list of files we think we've sent
if not lusn:
self.col.media.forceResync()
self.col.media.findChanges()
if lusn == mediaUsn and not self.col.media.hasChanged():
return "noChanges"
# step 1.5: if resyncing, we need to get the list of files the server
# has and remove them from our local list of files to sync
if not lusn:
files = self.server.mediaList()
need = self.col.media.removeExisting(files)
else:
need = None
# step 2: send/recv deletions
runHook("sync", "removeMedia")
lrem = self.removed()
rrem = self.server.remove(fnames=lrem, minUsn=lusn)
self.remove(rrem)
# step 3: stream files from server
runHook("sync", "server")
while 1:
runHook("sync", "streamMedia")
usn = self.col.media.usn()
zip = self.server.files(minUsn=usn, need=need)
if self.addFiles(zip=zip):
break
# step 4: stream files to the server
runHook("sync", "client")
while 1:
runHook("sync", "streamMedia")
zip, fnames = self.files()
if not fnames:
# finished
break
usn = self.server.addFiles(zip=zip)
# after server has replied, safe to remove from log
self.col.media.forgetAdded(fnames)
self.col.media.setUsn(usn)
# step 5: sanity check during beta testing
# NOTE: when removing this, need to move server tidyup
# back from sanity check to addFiles
s = self.server.mediaSanity()
c = self.mediaSanity()
if c != s:
# if the sanity check failed, force a resync
self.col.media.forceResync()
return "sanityCheckFailed"
def sync(self):
# check if there have been any changes
runHook("sync", "findMedia")
self.col.log("findChanges")
self.col.media.findChanges()
# begin session and check if in sync
lastUsn = self.col.media.lastUsn()
ret = self.server.begin()
srvUsn = ret['usn']
if lastUsn == srvUsn and not self.col.media.haveDirty():
return "noChanges"
# loop through and process changes from server
self.col.log("last local usn is %s" % lastUsn)
self.downloadCount = 0
while True:
data = self.server.mediaChanges(lastUsn=lastUsn)
self.col.log("mediaChanges resp count %d" % len(data))
if not data:
break
need = []
lastUsn = data[-1][1]
for fname, rusn, rsum in data:
lsum, ldirty = self.col.media.syncInfo(fname)
self.col.log(
"check: lsum=%s rsum=%s ldirty=%d rusn=%d fname=%s" %
((lsum and lsum[0:4]),
(rsum and rsum[0:4]), ldirty, rusn, fname))
if rsum:
# added/changed remotely
if not lsum or lsum != rsum:
self.col.log("will fetch")
need.append(fname)
else:
self.col.log("have same already")
ldirty and self.col.media.markClean([fname])
elif lsum:
# deleted remotely
if not ldirty:
self.col.log("delete local")
self.col.media.syncDelete(fname)
else:
# conflict; local add overrides remote delete
self.col.log("conflict; will send")
else:
# deleted both sides
self.col.log("both sides deleted")
ldirty and self.col.media.markClean([fname])
self._downloadFiles(need)
self.col.log("update last usn to %d" % lastUsn)
self.col.media.setLastUsn(lastUsn) # commits
# at this point we're all up to date with the server's changes,
# and we need to send our own
updateConflict = False
toSend = self.col.media.dirtyCount()
while True:
zip, fnames = self.col.media.mediaChangesZip()
if not fnames:
break
runHook(
"syncMsg",
ngettext("%d media change to upload",
"%d media changes to upload", toSend) % toSend)
processedCnt, serverLastUsn = self.server.uploadChanges(zip)
self.col.media.markClean(fnames[0:processedCnt])
self.col.log("processed %d, serverUsn %d, clientUsn %d" %
(processedCnt, serverLastUsn, lastUsn))
if serverLastUsn - processedCnt == lastUsn:
self.col.log("lastUsn in sync, updating local")
lastUsn = serverLastUsn
self.col.media.setLastUsn(serverLastUsn) # commits
else:
self.col.log("concurrent update, skipping usn update")
# commit for markClean
self.col.media.db.commit()
updateConflict = True
toSend -= processedCnt
if updateConflict:
self.col.log("restart sync due to concurrent update")
return self.sync()
lcnt = self.col.media.mediaCount()
ret = self.server.mediaSanity(local=lcnt)
if ret == "OK":
return "OK"
else:
self.col.media.forceResync()
return ret
def sync(self):
"Returns 'noChanges', 'fullSync', 'success', etc"
self.syncMsg = ""
self.uname = ""
# if the deck has any pending changes, flush them first and bump mod
# time
self.col.save()
# step 1: login & metadata
runHook("sync", "login")
meta = self.server.meta()
self.col.log("rmeta", meta)
if not meta:
return "badAuth"
# server requested abort?
self.syncMsg = meta["msg"]
if not meta["cont"]:
return "serverAbort"
else:
# don't abort, but if 'msg' is not blank, gui should show 'msg'
# after sync finishes and wait for confirmation before hiding
pass
rscm = meta["scm"]
rts = meta["ts"]
self.rmod = meta["mod"]
self.maxUsn = meta["usn"]
# this is a temporary measure to address the problem of users
# forgetting which email address they've used - it will be removed
# when enough time has passed
self.uname = meta.get("uname", "")
meta = self.meta()
self.col.log("lmeta", meta)
self.lmod = meta["mod"]
self.minUsn = meta["usn"]
lscm = meta["scm"]
lts = meta["ts"]
if abs(rts - lts) > 300:
self.col.log("clock off")
return "clockOff"
if self.lmod == self.rmod:
self.col.log("no changes")
return "noChanges"
elif lscm != rscm:
self.col.log("schema diff")
return "fullSync"
self.lnewer = self.lmod > self.rmod
# step 1.5: check collection is valid
if not self.col.basicCheck():
self.col.log("basic check")
return "basicCheckFailed"
# step 2: deletions
runHook("sync", "meta")
lrem = self.removed()
rrem = self.server.start(minUsn=self.minUsn, lnewer=self.lnewer, graves=lrem)
self.remove(rrem)
# ...and small objects
lchg = self.changes()
rchg = self.server.applyChanges(changes=lchg)
self.mergeChanges(lchg, rchg)
# step 3: stream large tables from server
runHook("sync", "server")
while 1:
runHook("sync", "stream")
chunk = self.server.chunk()
self.col.log("server chunk", chunk)
self.applyChunk(chunk=chunk)
if chunk["done"]:
break
# step 4: stream to server
runHook("sync", "client")
while 1:
runHook("sync", "stream")
chunk = self.chunk()
self.col.log("client chunk", chunk)
self.server.applyChunk(chunk=chunk)
if chunk["done"]:
break
# step 5: sanity check
runHook("sync", "sanity")
c = self.sanityCheck()
ret = self.server.sanityCheck2(client=c)
if ret["status"] != "ok":
# roll back and force full sync
self.col.rollback()
self.col.modSchema(False)
self.col.save()
return "sanityCheckFailed"
# finalize
runHook("sync", "finalize")
mod = self.server.finish()
self.finish(mod)
return "success"
def sync(self):
"Returns 'noChanges', 'fullSync', or 'success'."
# if the deck has any pending changes, flush them first and bump mod
# time
self.col.save()
# step 1: login & metadata
runHook("sync", "login")
ret = self.server.meta()
if not ret:
return "badAuth"
self.rmod, rscm, self.maxUsn, rts, self.mediaUsn = ret
self.lmod, lscm, self.minUsn, lts, dummy = self.meta()
if abs(rts - lts) > 300:
return "clockOff"
if self.lmod == self.rmod:
return "noChanges"
elif lscm != rscm:
return "fullSync"
self.lnewer = self.lmod > self.rmod
# step 2: deletions
runHook("sync", "meta")
lrem = self.removed()
rrem = self.server.start(minUsn=self.minUsn,
lnewer=self.lnewer,
graves=lrem)
self.remove(rrem)
# ...and small objects
lchg = self.changes()
rchg = self.server.applyChanges(changes=lchg)
self.mergeChanges(lchg, rchg)
# step 3: stream large tables from server
runHook("sync", "server")
while 1:
runHook("sync", "stream")
chunk = self.server.chunk()
self.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 4: stream to server
runHook("sync", "client")
while 1:
runHook("sync", "stream")
chunk = self.chunk()
self.server.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 5: sanity check during beta testing
runHook("sync", "sanity")
c = self.sanityCheck()
s = self.server.sanityCheck()
if c != s:
raise Exception("""\
Sanity check failed. Please copy and paste the text below:\n%s\n%s""" % (c, s))
# finalize
runHook("sync", "finalize")
mod = self.server.finish()
self.finish(mod)
return "success"
def sync(self, mediaUsn):
# step 1: check if there have been any changes
runHook("sync", "findMedia")
self.col.media.findChanges()
lusn = self.col.media.usn()
if lusn == mediaUsn and not self.col.media.hasChanged():
return "noChanges"
# step 2: send/recv deletions
runHook("sync", "removeMedia")
lrem = self.removed()
rrem = self.server.remove(fnames=lrem, minUsn=lusn)
self.remove(rrem)
# step 3: stream files from server
runHook("sync", "server")
while 1:
runHook("sync", "streamMedia")
usn = self.col.media.usn()
zip = self.server.files(minUsn=usn)
if self.addFiles(zip=zip):
break
# step 4: stream files to the server
runHook("sync", "client")
while 1:
runHook("sync", "streamMedia")
zip, fnames = self.files()
if not fnames:
# finished
break
usn = self.server.addFiles(zip=zip)
# after server has replied, safe to remove from log
self.col.media.forgetAdded(fnames)
self.col.media.setUsn(usn)
# step 5: sanity check during beta testing
# NOTE: when removing this, need to move server tidyup
# back from sanity check to addFiles
s = self.server.mediaSanity()
c = self.mediaSanity()
if c != s:
raise Exception("""\
Media sanity check failed. Please copy and paste the text below:\n%s\n%s""" %
(c, s))
return "success"
def sync(self):
"Returns 'noChanges', 'fullSync', 'success', etc"
self.syncMsg = ""
self.uname = ""
# if the deck has any pending changes, flush them first and bump mod
# time
self.col.save()
# step 1: login & metadata
runHook("sync", "login")
meta = self.server.meta()
self.col.log("rmeta", meta)
if not meta:
return "badAuth"
rscm = meta['scm']
rts = meta['ts']
self.rmod = meta['mod']
self.maxUsn = meta['usn']
self.mediaUsn = meta['musn']
self.syncMsg = meta['msg']
# this is a temporary measure to address the problem of users
# forgetting which email address they've used - it will be removed
# when enough time has passed
self.uname = meta.get("uname", "")
# server requested abort?
if not meta['cont']:
return "serverAbort"
else:
# don't abort, but ui should show message after sync finishes
# and require confirmation if it's non-empty
pass
meta = self.meta()
self.col.log("lmeta", meta)
self.lmod = meta['mod']
self.minUsn = meta['usn']
lscm = meta['scm']
lts = meta['ts']
if abs(rts - lts) > 300:
self.col.log("clock off")
return "clockOff"
if self.lmod == self.rmod:
self.col.log("no changes")
return "noChanges"
elif lscm != rscm:
self.col.log("schema diff")
return "fullSync"
self.lnewer = self.lmod > self.rmod
# step 1.5: check collection is valid
if not self.col.basicCheck():
self.col.log("basic check")
return "basicCheckFailed"
# step 2: deletions
runHook("sync", "meta")
lrem = self.removed()
rrem = self.server.start(minUsn=self.minUsn,
lnewer=self.lnewer,
graves=lrem)
self.remove(rrem)
# ...and small objects
lchg = self.changes()
rchg = self.server.applyChanges(changes=lchg)
self.mergeChanges(lchg, rchg)
# step 3: stream large tables from server
runHook("sync", "server")
while 1:
runHook("sync", "stream")
chunk = self.server.chunk()
self.col.log("server chunk", chunk)
self.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 4: stream to server
runHook("sync", "client")
while 1:
runHook("sync", "stream")
chunk = self.chunk()
self.col.log("client chunk", chunk)
self.server.applyChunk(chunk=chunk)
if chunk['done']:
break
# step 5: sanity check
runHook("sync", "sanity")
c = self.sanityCheck()
ret = self.server.sanityCheck2(client=c)
if ret['status'] != "ok":
# roll back and force full sync
self.col.rollback()
self.col.modSchema()
self.col.save()
return "sanityCheckFailed"
# finalize
runHook("sync", "finalize")
mod = self.server.finish()
self.finish(mod)
return "success"