def processMerge(self, dbName, fromVersion):
"""
Merge in log files
Merge fromVersion to lastest version into lastest version in log files
:param dbName: str name of db
:param fromVersion: str from version you want to merger
:return:
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
try:
fromVersion = int(fromVersion)
except ValueError:
raise Exception('Version is not a integer')
if fromVersion <= 0:
raise LogFileException("Version number can not be less than zero")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
if versionedDatabaseRecord.getActualVersion() < fromVersion:
pass
#raise LogFileException("Version number can not be bigger than actual version of database")
files = LogFile.getAllVerFileNameInDirectory(
versionedDatabaseRecord.getDestinationFolder())
#proces for every versed files
for fileName in files.keys():
elementName = files[fileName]
filePath = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(), elementName)
logFile = LogFile(filePath)
logFile.mergeFromVersion(fromVersion)
def processForceMake(self, dbName):
"""
Clear all log files and save into it dump of every element in lastest version
:param dbName:
:return:
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
# get last binary log and last position
binnaryLogNames = BinnaryLogNames(self.dbConnection)
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName)
logDataString = binnarylogs.getAllLogData()
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#version and des folder
destFolder = versionedDatabaseRecord.getDestinationFolder()
newVersion = versionedDatabaseRecord.getActualVersion()
versionedDatabaseRecord.updateMetaDataAboutTables()
# data to transaction
transData = {}
fileNames = LogFile.getAllVerFileNameInDirectory(destFolder)
for fileName in fileNames.keys():
path = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
fileNames[fileName])
logFile = LogFile(path)
transData[fileNames.get(fileName)] = logFile.getAllContent()
try:
#process
LogFile.removeAllVerFilesInDirecotry(destFolder)
dump = DatabaseDump(dbName, self.verDatabaseConnection)
tablesDump = dump.dumpTables(
versionedDatabaseRecord.getAllTables())
for tableName in tablesDump.keys():
logFileName = LogFile.makeLogFilePath(destFolder, tableName)
logFile = LogFile(logFileName)
logFile.insertVersionIntoFile([tablesDump.get(tableName)],
newVersion)
except Exception as e:
#On exception - transaction
LogFile.removeAllVerFilesInDirecotry(destFolder)
for name in transData.keys():
logFileName = LogFile.makeLogFilePath(destFolder, name)
logFile = LogFile(logFileName)
logFile.writeIntoFile(transData[name])
raise
versionedDatabaseRecord.setSomeData(lastBinnaryLogName,
lastBinnaryLogPosition, newVersion)
def processForceSet(self, dbName):
"""
Cleat database and Import everything in log files into database
Edited data will be overwrite
:param dbName: name of force set database
:return:
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
#get ALL sql to process and get last version
files = LogFile.getAllVerFileNameInDirectory(
versionedDatabaseRecord.getDestinationFolder())
newLastVersion = 0
allSql = []
for fileName in files.keys():
elementName = files[fileName]
filePath = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(), elementName)
logFile = LogFile(filePath)
allSql.append(logFile.getAllSql())
if logFile.getLastVersion() > newLastVersion:
newLastVersion = logFile.getLastVersion()
#get last position of binnary logs and last name
binnaryLogNames = BinnaryLogNames(self.dbConnection)
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName)
logDataString = binnarylogs.getAllLogData()
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
# make databaseDump to transaction
dbDump = DatabaseDump(dbName, self.verDatabaseConnection)
dump = dbDump.dumpDatabase()
#proces in into transaction
try:
print("Importing Database...")
self.verDatabaseConnection.clearDatabase()
for sql in allSql:
self.verDatabaseConnection.executeSimpleSQL(sql)
except Exception as e:
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise
# update data in versioned databases
try:
versionedDatabaseRecord = VersionedDatabase.addVersionerDatabase(
self.dbConnection, dbName,
versionedDatabaseRecord.getDestinationFolder(),
lastBinnaryLogName, lastBinnaryLogPosition, newLastVersion)
except Exception as e:
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise
def processImport(self, dbName):
"""
Import logs into database
Local adjustments stay - dont delete it
:param dbName: name of imported database
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
#import slq data
files = LogFile.getAllVerFileNameInDirectory(
versionedDatabaseRecord.getDestinationFolder())
newLastVersion = 0
for fileName in files.keys():
elementName = files[fileName]
filePath = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(), elementName)
logFile = LogFile(filePath)
if fileName == LogFile.OTHER_LOG_NAME_FULL:
#this is other file
newSql = logFile.getSqlFromVersion(
versionedDatabaseRecord.getActualVersion() + 1)
self.verDatabaseConnection.executeSimpleSQL(newSql)
continue
versionedTable = versionedDatabaseRecord.getTableByName(
elementName)
if versionedTable is None:
#this is new tables
newSql = logFile.getAllSql()
if not newSql:
continue
try:
self.verDatabaseConnection.executeSimpleSQL(newSql)
versionedDatabaseRecord.addTable(elementName,
logFile.getLastVersion())
except Exception as e:
self.verDatabaseConnection.deleteElementbyName(elementName)
versionedDatabaseRecord.removeTable(elementName)
raise
else:
#this element has been already in database
newSql = logFile.getSqlFromVersion(
versionedDatabaseRecord.getActualVersion() + 1)
try:
self.verDatabaseConnection.executeSimpleSQL(newSql)
except Exception as e:
for fileName in files.keys():
elementName = files[fileName]
self.__revertByElementName(elementName,
versionedDatabaseRecord)
raise
if logFile.getLastVersion() > newLastVersion:
newLastVersion = logFile.getLastVersion()
#find last version
newVersion = versionedDatabaseRecord.getActualVersion()
if newLastVersion > versionedDatabaseRecord.getActualVersion():
newVersion = newLastVersion
# save metadata
#save only new version
versionedDatabaseRecord.setSomeData(version=newVersion)
def processUp(self, dbName):
"""
Make new database revision
:return:
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
binnaryLogNames = BinnaryLogNames(self.dbConnection)
listOfBinneryLogNames = binnaryLogNames.getBinnaryLogsNamesOlderThan(
versionedDatabaseRecord.getLogFile())
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
lastBinnaryLogPosition = versionedDatabaseRecord.getLogFilePosition()
#check if first log name have name like versionedDatabaseRecord bin name
if versionedDatabaseRecord.getLogFile() not in listOfBinneryLogNames:
lastBinnaryLogPosition = 0
print(
'Warning: Binnary Logs is out of date. Prease increase expire_logs_days'
)
res = TerminalCommand.runDialogYorN(
'All data for version is unavailable. Some data in new version can be missing. Do you want continue'
)
if not res:
return
#check if version in logFiles is same in db
destination = versionedDatabaseRecord.getDestinationFolder()
actualVersion = versionedDatabaseRecord.getActualVersion()
fileNames = LogFile.getAllVerFileNameInDirectory(destination)
for name in fileNames:
logFile = LogFile(
LogFile.makeLogFilePath(destination, fileNames[name]))
fileVersion = logFile.getLastVersion()
if fileVersion > actualVersion:
raise LogFileException(
'Versined files in directory \'' + destination +
"' have newer version then data in database. Please import data into datababe before make new version"
)
#get data from logs
binnarylogs = BinnaryLogs(listOfBinneryLogNames, dbName)
binnarylogs.setStartPositionFirstLog(lastBinnaryLogPosition)
logDataString = binnarylogs.getAllLogData()
#parse data
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#increment actual version - dont save
actualVersion = versionedDatabaseRecord.getActualVersion()
actualVersion = actualVersion + 1
#flush sql into log file
result = binnlogParser.getLogParserResult()
#save to make transaction
logFileData = {}
toDelete = []
try:
#process
for elementName in result.getAllUpdatedElementNames():
elementData = versionedDatabaseRecord.getTableByName(
elementName)
logFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
elementName)
if elementName in result.getCreated():
logFile = LogFile(logFileName)
elif result.getRenameOldNameByNewName(elementName) is not None:
oldName = result.getRenameOldNameByNewName(elementName)
oldLogFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
oldName)
oldLogFile = LogFile(oldLogFileName)
logFileData[oldLogFileName] = oldLogFile.getAllContent()
logFile = LogFile(logFileName)
toDelete.append(logFileName)
logFile.writeIntoFile(oldLogFile.getAllContent())
oldLogFile.deleteFile()
elif versionedDatabaseRecord.getTableByName(elementName):
logFile = LogFile(logFileName)
logFileData[logFileName] = logFile.getAllContent()
else:
#someting wrong
continue
logFile.appendVersionIntoFile(
result.getAllQueriesByName(elementName), actualVersion)
#do others
if result.getOtherQueries():
logFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
LogFile.OTHER_LOG_NAME)
logFile = LogFile(logFileName)
logFileData[logFileName] = logFile.getAllContent()
logFile.appendVersionIntoFile(result.getOtherQueries(),
actualVersion)
#save metadata
if (result.haveSomeData()):
versionedDatabaseRecord.setSomeData(lastBinnaryLogName,
lastBinnaryLogPosition,
actualVersion)
versionedDatabaseRecord.updateMetaDataAboutTables()
else:
versionedDatabaseRecord.setSomeData(lastBinnaryLogName,
lastBinnaryLogPosition)
versionedDatabaseRecord.updateMetaDataAboutTables()
print(
"No data to flush into log files. New version was not make"
)
except Exception as e:
#make transaction
for fileName in toDelete:
logFile = LogFile(fileName)
logFile.deleteFile()
for fileName in logFileData.keys():
logFile = LogFile(fileName)
logFile.writeIntoFile(logFileData[fileName])
raise