def getLastVersion(self):
"""
Return last version number
If last version is no find then return 0
:return: integer
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
list = self.__getAllFileContentByLine()
lastVersion = 0
lenOfFile = len(list)
for key, line in enumerate(reversed(list)):
index = line.find("version:")
if index != -1:
try:
versionStr = line[index + 8:]
versionInt = int(versionStr)
lastVersion = versionInt
break
except:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error on line ' +
str(lenOfFile - key))
return lastVersion
def addVersionerDatabase(dbConnection: Database, dbName, destinationFolder, logFile = None, logFilePosition = 0, actualVersion = 0):
"""
Add database to the versioned databases
Make record in the databaseVersioner database with actual version 0
If database has already versioned then delete old records
:param dbConnection: Database connetion
:param dbName: name of database to versioned
:param destinationFolder: folder to save log files, only abs path.
:param logFile: str of logFile for this database
:param logFilePosition: int number which declare last position in logFile
:return: VersionedDatabase
"""
if logFile is None:
logs = dbConnection.executeSimpleSQL("SHOW BINARY LOGS")
lastLog = logs[-1]
logFile = lastLog.get('Log_name')
if logFile is None:
raise BinnaryLogException('No binnary log file find')
if not dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
if not isDirectoryExist(destinationFolder):
raise FileException("Directory '" + destinationFolder + "' is not exist")
dataNewDatabase = {'db_name': dbName, 'actual_version': actualVersion, 'log_file': logFile,'log_file_position':logFilePosition, 'destination_folder': destinationFolder}
tablesForDatabase = dbConnection.executeSimpleSQL("select table_name from information_schema.tables where TABLE_SCHEMA='" + dbName + "'");
byNameInDb = dbConnection.executeSimpleSQL("SELECT id FROM " + VersionedDatabase.TABLE_NAME + " WHERE `db_name` = %s", (dbName), True)
try:
dbConnection.begin()
# delete od records about vesioning
if byNameInDb.get("id") is not None:
dbConnection.executeSQL("DELETE FROM " + VersionedTable.TABLE_NAME + " WHERE `fk_ver_databases_id` = " + str(byNameInDb.get("id")))
dbConnection.executeSQL("DELETE FROM " + VersionedDatabase.TABLE_NAME + " WHERE `id` = " + str(byNameInDb.get("id")))
# save data about versioning into database
retId = dbConnection.saveIntoTable(VersionedDatabase.TABLE_NAME, dataNewDatabase, withTransaction=False)
if retId is None:
raise DatabaseException('Save record into table \''+VersionedDatabase.TABLE_NAME+'\' faild')
for table in tablesForDatabase:
tableName = table['table_name'];
data = {'name': tableName, 'actual_version': 0, 'fk_ver_databases_id': retId}
dbConnection.saveIntoTable(VersionedTable.TABLE_NAME,data, withTransaction=False)
except Exception as e:
dbConnection.rollback()
raise e
dbConnection.commit()
versionedDatabase = VersionedDatabase(dbConnection, retId)
return versionedDatabase
def getAllSql(self):
"""
Return string which contain all log file
If file dont exist the return empty string
:return: string
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
content = self.getAllContent()
return content
def getSqlFromVersion(self, version: int):
"""
Return sql code from defined version (with version)
If file dont exist the return empty string
:param version: integer of version
:return: string
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
list = self.__getAllFileContentByLine()
numLine = -1
startLine = len(list)
check = False
# find start version
for key, line in enumerate(list):
index = line.find("version:")
if index != -1:
check = True
try:
versionStr = line[index + 8:]
versionInt = int(versionStr)
if versionInt >= version:
numLine = key
break
except:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error on line ' +
str(startLine - key))
if not check:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error - no version exist')
check = False
if numLine != -1:
for key, line in enumerate(list[numLine:]):
if "*/" in line:
startLine = key + numLine + 1
check = True
break
if numLine != -1 and check == False:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error')
string = '\n'.join(list[startLine:])
ret = self.__delMultiLineComments(string)
return ret
def deleteFromVersion(self, version: int):
"""
Delete logs from defined version number with this verison
:param version: integer number of version
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
list = self.__getAllFileContentByLine()
numLine = -1
endLine = len(list)
check = False
# find start of version
for key, line in enumerate(list):
index = line.find("version:")
if index != -1:
check = True
try:
versionStr = line[index + 8:]
versionInt = int(versionStr)
if versionInt >= version:
numLine = key
break
except:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error on line ' +
str(endLine - key))
if not check:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error - no version exist')
check = False
if numLine != -1:
for key, line in (enumerate(reversed(list[numLine:]))):
if "/*" in line:
check = True
endLine = numLine - key
break
if numLine != -1 and check == False:
raise LogFileException('File \'' + self._filePath +
'\' have syntax error')
stringToSave = '\n'.join(list[:endLine])
self.writeIntoFile(stringToSave)
def removeAllFilesInDirectory(directory: str):
"""
Return all files names in directory
:param directory path to directory in abs format
:return: dict {filename: elementName}
"""
if not isDirectoryExist(directory):
raise FileException("Directory '" + directory + "' is not exist")
fileNames = getAllFileNamesInDirectory(directory)
for name in fileNames:
try:
os.remove(directory + "/" + name)
except Exception as e:
pass
return
def getAllVerFileNameInDirectory(directory: str):
"""
Return all log files names in directory
:param directory path to directory in abs format
:return: dict {filename: elementName}
"""
if not isDirectoryExist(directory):
raise FileException("Directory '" + directory + "' is not exist")
fileNames = getAllFileNamesInDirectory(directory)
retNames = {}
extensionLen = len(LogFile.EXTENSION) + 1 # +1 dot in file name
for name in fileNames:
if name.endswith(LogFile.EXTENSION):
retNames[name] = name[0:-extensionLen]
return retNames
def __init__(self, fileName: str):
"""
:param fileName: Name of binnary log
"""
self.__fileDir = config.get('binnaryLogsDir')
self.__fileName = fileName
self.__filePath = self.__fileDir + self.__fileName
self.__command = config.get('mysqlbinlogLocation')
self.__startPosition = None
self.__databaseName = None
if (os.path.isfile(self.__filePath) is False):
raise FileException("File '" + self.__filePath + "' do not exist")
if not os.access(self.__filePath, os.R_OK):
raise BinnaryLogException("Permissinon denied to read file \'" +
self.__filePath + "\'")
def removeAllVerFilesInDirecotry(directory: str):
"""
Remove all versioned files in directory
:param directory: abs path do directory
"""
if not isDirectoryExist(directory):
raise FileException("Directory '" + directory + "' is not exist")
fileNames = getAllFileNamesInDirectory(directory)
extensionLen = len(LogFile.EXTENSION) + 1 # +1 dot in file name
for name in fileNames:
if name.endswith(LogFile.EXTENSION):
try:
os.remove(
LogFile.makeLogFilePath(directory,
name[0:-extensionLen]))
except Exception as e:
pass
def getLastVersionSql(self):
"""
Return last version sql code
On error raise exceptins
:return: string
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
list = self.__getAllFileContentByLine()
numLine = -1
startLine = -1
for key, line in enumerate(reversed(list)):
index = line.find("version:")
if index != -1:
numLine = len(list) - key
break
if numLine >= 0:
for key, line in enumerate(list[numLine:]):
if "*/" in line:
startLine = key + numLine + 1
break
if numLine == -1:
raise LogFileException(
'File \'' + self._filePath +
'\' have syntax error: string \'version:\' was not found in any line'
)
if startLine == -1:
raise LogFileException(
'File \'' + self._filePath +
'\' have syntax error: string \'*/\' was not found in any line'
)
string = '\n'.join(list[startLine:])
ret = self.__delMultiLineComments(string)
return ret
def mergeFromVersion(self, version):
"""
merge sql form secifific version to lastet version and save like last version
This method is transactional
:param version: integer number of version
"""
if self.__isFileLogExist() != True:
raise FileException("File \'" + self._filePath +
"\' is not exists")
if version <= 0:
raise LogFileException("Version number can not be less than zero")
all = self.getAllContent()
try:
lastestVerions = self.getLastVersion()
merged = self.getSqlFromVersion(version)
self.deleteFromVersion(version)
sqlList = merged.split("\n")
self.appendVersionIntoFile(sqlList, lastestVerions)
except Exception as e:
self.writeIntoFile(all)
raise
def processAddExists(self, dbName, sourceFolder):
"""
Import exist versioned logs into local database
:param dbName:
:param sourceFolder: str path to folder where is logs
:return:
"""
destFolder = os.path.abspath(sourceFolder)
if self.dbConnection.isDatabaseExist(dbName) is False:
raise DatabaseException('Database \'' + dbName +
'\' is not exist. You have to make it')
if not isDirectoryExist(destFolder):
raise FileException("Directory '" + destFolder + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
# check if dababase has been versioned
isExist = database.VersionedDatabase.isDatabaseVersioned(
self.dbConnection, dbName)
if isExist:
result = TerminalCommand.runDialogYorN(
"Database '" + dbName +
"' has been versioned. Are you want redeclare it?")
if not result:
return
# get last binary log and last position
binnaryLogNames = BinnaryLogNames(self.dbConnection)
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName)
logDataString = binnarylogs.getAllLogData()
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#make dump of database
dbDump = DatabaseDump(dbName, self.verDatabaseConnection)
dump = dbDump.dumpDatabase()
#cleart db
self.verDatabaseConnection.clearDatabase()
#make and import database
logFilesNames = LogFile.getAllVerFileNameInDirectory(destFolder)
lastVersion = 0
otherSql = ""
for fileName in logFilesNames.keys():
print("Process : " + logFilesNames[fileName])
logFile = LogFile(
LogFile.makeLogFilePath(destFolder, logFilesNames[fileName]))
if logFile.getLastVersion() > lastVersion:
lastVersion = logFile.getLastVersion()
if LogFile.OTHER_LOG_NAME_FULL in fileName:
otherSql = logFile.getAllSql()
else:
#import data into database
try:
allSql = logFile.getAllSql()
if allSql:
self.verDatabaseConnection.executeSimpleSQL(allSql)
except Exception as e:
#special transtaction for ddl
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise
#do OtherSql
try:
self.verDatabaseConnection.executeSimpleSQL(otherSql)
except Exception as e:
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise e
#set database to versioning
versionedDatabaseRecord = database.VersionedDatabase.addVersionerDatabase(
self.dbConnection, dbName, destFolder, lastBinnaryLogName,
lastBinnaryLogPosition, lastVersion)