def DownloadDatabaseAndClean(scripts, ftp, item):
DownloadFile(scripts, ftp, item, 'temp')
# Clean up downloaded file
WipeUpdateCon = DatabaseHandler.OpenEntryDatabase('temp')
WipeUpdateCur = WipeUpdateCon.cursor()
WipeUpdateCur.execute(u"UPDATE Text SET updated=0")
WipeUpdateCon.commit()
# Copy it to the right place
Globals.CopyFile( Globals.configData.LocalDatabasePath + '/temp', Globals.configData.LocalDatabasePath + '/{0}'.format(item) )
CompletionTable.CalculateCompletionForDatabase(item)
Globals.Cache.LoadDatabase(item)
return
def LoadDatabase(self, name):
self.databaseAccessRLock.acquire()
Globals.MainWindow.displayStatusMessage('Loading Database: ' + name + '...')
Connection = DatabaseHandler.OpenEntryDatabase(name)
Cursor = Connection.cursor()
Cursor.execute("SELECT ID, StringID, english, comment, status, IdentifyString FROM Text")
table = Cursor.fetchall()
db = [None] * len(table)
for entry in table:
db[ int(entry[0]) - 1 ] = EntryStruct(unicode(entry[2]), unicode(entry[3]), int(entry[1]), int(entry[4]), unicode(entry[5]))
self.Databases[name] = db
Globals.MainWindow.displayStatusMessage('Loaded Database: ' + name)
self.databaseAccessRLock.release()
return
def RevertFromServerWorker(scripts, networkTransferWindow, sendWindowCloseSignal):
Globals.Cache.databaseAccessRLock.acquire()
scripts.WriteDatabaseStorageToHdd()
updateList = list(scripts.update)
for i in range(1, 20):
try:
try:
scripts.ftp = ConnectToFtp()
except:
if i == 20:
print
networkTransferWindow.addListEntry("FTP connection failed, revert didn't succeed.", "< Error >")
Globals.Settings.setValue('update', set(updateList))
Globals.Settings.sync()
Globals.Cache.databaseAccessRLock.release()
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
return
networkTransferWindow.addListEntry('Error during FTP connect, retrying...', "< Error >")
continue
scripts.ftp.cwd('/')
scripts.ftp.cwd(Globals.configData.RemoteDatabasePath)
# "Re-getting changed files from server..."
while len(updateList) > 0:
item = updateList[len(updateList) - 1]
transferWindowIndex = networkTransferWindow.addListEntry("Downloading...", item)
DownloadFile(scripts, scripts.ftp, item, item)
WipeUpdateCon = DatabaseHandler.OpenEntryDatabase(item)
WipeUpdateCur = WipeUpdateCon.cursor()
WipeUpdateCur.execute(u"UPDATE Text SET updated=0")
WipeUpdateCon.commit()
CompletionTable.CalculateCompletionForDatabase(item)
Globals.Cache.LoadDatabase(item)
networkTransferWindow.modifyListEntryStatus(transferWindowIndex, "Complete!")
updateList.remove(item)
scripts.ftp.close()
scripts.update.clear()
Globals.Settings.setValue('update', scripts.update)
Globals.Settings.sync()
Globals.HaveUnsavedChanges = False
scripts.SetWindowTitle()
break
except ftplib.all_errors:
if i == 20:
networkTransferWindow.addListEntry('Error during FTP transfer. Databases may be corrupted, please Revert again as soon as possible.', "< Error >")
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
break
networkTransferWindow.addListEntry('Error during FTP transfer, retrying...', "< Error >")
continue
Globals.Cache.databaseAccessRLock.release()
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(True)
return
def SavetoServerWorker(scripts, networkTransferWindow, sendWindowCloseSignal):
Globals.Cache.databaseAccessRLock.acquire()
scripts.WriteDatabaseStorageToHdd()
if len(scripts.update) == 0:
networkTransferWindow.addListEntry("Nothing to save!", "-")
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
Globals.Cache.databaseAccessRLock.release()
return False
# Beginning Save...
autoRestartAfter = False
for ftperrorcount in range(1, 20):
try:
try:
scripts.ftp = ConnectToFtp()
except:
if ftperrorcount >= 20:
networkTransferWindow.addListEntry("Couldn't connect to FTP Server, stopping upload. Please try again later.", "< Error >")
Globals.Settings.setValue('update', set(scripts.update))
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
Globals.Cache.databaseAccessRLock.release()
return False
networkTransferWindow.addListEntry("Couldn't connect to FTP Server, retrying...", "< Error >")
continue
scripts.ftp.cwd('/')
scripts.ftp.cwd(Globals.configData.RemoteDatabasePath)
# Retrieving any files modified by others...
RetrieveModifiedFilesWorker(scripts, None, networkTransferWindow, False)
# Uploading Files...
LogTable = []
saveUpdate = set()
# stagger upload into multiple 10-file batches
# the way this is written we cannot keep it, but eh
singleFileUploadCounter = 0
for filename in scripts.update:
singleFileUploadCounter = singleFileUploadCounter + 1
if singleFileUploadCounter > 10:
autoRestartAfter = True
saveUpdate.add(filename)
continue
# 'Uploading ' + Globals.GetDatabaseDescriptionString(filename) + ' [' + filename + ']...'
transferWindowIdx = networkTransferWindow.addListEntry("Downloading...", filename)
# Downloading the server version and double checking
DownloadFile(scripts, scripts.ftp, str(filename), 'temp')
try:
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "Merging...")
RemoteMergeCon = DatabaseHandler.OpenEntryDatabase('temp')
DatabaseHandler.MergeDatabaseWithServerVersionBeforeUpload(
DatabaseHandler.OpenEntryDatabase(filename).cursor(),
RemoteMergeCon.cursor()
)
RemoteMergeCon.commit()
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "Uploading...")
for ftpSingleFileUpErrorCount in range(1, 20):
try:
if ftpSingleFileUpErrorCount >= 20:
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "!! Error !! Server file may be corrupted, please manually check and fix or inform someone who can.")
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
Globals.Cache.databaseAccessRLock.release()
return False
result = UploadFile(scripts, scripts.ftp, 'temp', str(filename))
if isinstance(result, str):
continue
break
except ftplib.all_errors:
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "Error, retrying... (" + str(ftpSingleFileUpErrorCount) + ")")
continue
# And copy the new remote over the old local
Globals.CopyFile(Globals.configData.LocalDatabasePath + '/temp', Globals.configData.LocalDatabasePath + '/{0}'.format(filename))
except:
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "Server file corrupted, replacing with local file...")
UploadFile(scripts, scripts.ftp, filename, filename)
LogTable.append(filename)
networkTransferWindow.modifyListEntryStatus(transferWindowIdx, "Complete!")
CompletionTable.CalculateCompletionForDatabase(filename)
Globals.Cache.LoadDatabase(filename)
# Fix up the changelog and upload
transferWindowChangeLogIdx = networkTransferWindow.addListEntry("Modifying...", "ChangeLog")
ChangeLogConnection, ChangeLogCursor = Globals.GetNewChangeLogConnectionAndCursor()
ChangeLogCursor.execute('SELECT Max(ID) as Highest FROM Log')
MaxID = ChangeLogCursor.fetchall()[0][0]
fileString = ''.join(["%s," % (k) for k in LogTable])[:-1]
# 'Uploaded: ', fileString
ChangeLogCursor.execute(u"INSERT INTO Log VALUES({0}, '{1}', '{2}', {3})".format(MaxID + 1, fileString, Globals.Author, "strftime('%s','now')"))
ChangeLogConnection.commit()
ChangeLogConnection.close()
networkTransferWindow.modifyListEntryStatus(transferWindowChangeLogIdx, "Uploading...")
changeLogUploadSuccess = False
for changeup in range(1, 20):
try:
result = UploadFile(scripts, scripts.ftp, 'ChangeLog', 'ChangeLog', False)
if isinstance(result, str) or not result:
if changeup >= 20:
break
else:
networkTransferWindow.modifyListEntryStatus(transferWindowChangeLogIdx, "Error, retrying... (" + str(changeup) + ")")
continue
networkTransferWindow.modifyListEntryStatus(transferWindowChangeLogIdx, "Complete!")
changeLogUploadSuccess = True
break
except ftplib.all_errors:
if changeup >= 20:
break
networkTransferWindow.modifyListEntryStatus(transferWindowChangeLogIdx, "Error, retrying... (" + str(changeup) + ")")
continue
if not changeLogUploadSuccess:
networkTransferWindow.modifyListEntryStatus(transferWindowChangeLogIdx, "!! Error !! Server ChangeLog may be corrupted, please fix immediately.")
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
Globals.Cache.databaseAccessRLock.release()
return False
# Everything is done.
scripts.ftp.close()
if len(saveUpdate) > 0:
Globals.MainWindow.displayStatusMessage( 'Retaining the following files for later upload: ' + str(saveUpdate) )
scripts.update.clear()
scripts.update = set(saveUpdate)
Globals.Settings.setValue('update', scripts.update)
Globals.Settings.sync()
if autoRestartAfter:
retval = SavetoServerWorker(scripts, networkTransferWindow, sendWindowCloseSignal)
Globals.Cache.databaseAccessRLock.release()
return retval
if len(scripts.update) > 0:
Globals.HaveUnsavedChanges = True
else:
Globals.HaveUnsavedChanges = False
scripts.SetWindowTitle()
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(True)
Globals.Cache.databaseAccessRLock.release()
return True
except ftplib.all_errors:
if ftperrorcount >= 20:
networkTransferWindow.addListEntry("Error during FTP transfer. File(s) that were in progress may be corrupted, please confirm and fix.", "< Error >")
break
networkTransferWindow.addListEntry("Error during FTP transfer, retrying...", "< Error >")
continue
if sendWindowCloseSignal:
networkTransferWindow.allowCloseSignal.emit(False)
Globals.Cache.databaseAccessRLock.release()
return False
def CalculateCompletionForDatabaseTreeNode(node):
Globals.Cache.databaseAccessRLock.acquire()
CompletionConnection, CompletionCursor = DatabaseHandler.GetCompletionPercentageConnectionAndCursor(
)
DatabaseConnection = DatabaseHandler.OpenEntryDatabase(node.Name)
DatabaseCursor = DatabaseConnection.cursor()
databaseName = GetCompletionTableDatabaseNameOfTreeNode(node)
for i in range(0, Globals.configData.TranslationStagesCount + 1):
if not node.Subsections:
DatabaseCursor.execute(
'SELECT Count(1) FROM Text WHERE status >= {0}'.format(i))
count = DatabaseCursor.fetchall()[0][0]
else:
count = 0
for sub in node.Subsections:
DatabaseCursor.execute(
'SELECT Count(1) FROM Text WHERE status >= {0} AND ID >= {1} AND ID <= {2}'
.format(i, sub.Start, sub.End))
count += int(DatabaseCursor.fetchall()[0][0])
CompletionCursor.execute(
"SELECT Count(1) FROM StatusData WHERE database = ? AND type = ?",
[databaseName, i])
exists = CompletionCursor.fetchall()[0][0]
if exists > 0:
CompletionCursor.execute(
"UPDATE StatusData SET amount = ? WHERE database = ? AND type = ?",
[count, databaseName, i])
else:
CompletionCursor.execute(
"INSERT INTO StatusData (database, type, amount) VALUES (?, ?, ?)",
[databaseName, i, count])
if not node.Subsections:
DatabaseCursor.execute("SELECT Count(1) FROM Text WHERE comment != ''")
count = DatabaseCursor.fetchall()[0][0]
else:
count = 0
for sub in node.Subsections:
DatabaseCursor.execute(
"SELECT Count(1) FROM Text WHERE comment != '' AND ID >= {0} AND ID <= {1}"
.format(sub.Start, sub.End))
count += int(DatabaseCursor.fetchall()[0][0])
# type == -2 for comment count
CompletionCursor.execute(
"SELECT Count(1) FROM StatusData WHERE database = ? AND type = -2",
[databaseName])
exists = CompletionCursor.fetchall()[0][0]
if exists > 0:
CompletionCursor.execute(
"UPDATE StatusData SET amount = ? WHERE database = ? AND type = -2",
[count, databaseName])
else:
CompletionCursor.execute(
"INSERT INTO StatusData (database, type, amount) VALUES (?, -2, ?)",
[databaseName, count])
CompletionConnection.commit()
Globals.Cache.databaseAccessRLock.release()
