def processSnapshot(self, dbName, destination):
"""
Dump all database into one file
:param dbName: name of dumped file
:param destination: path to save file
"""
destination = os.path.abspath(destination)
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
isExist = isFileExist(destination)
if isExist:
result = TerminalCommand.runDialogYorN(
'File ' + destination + ' is exist. Do you want rewrite it')
if not result:
return
#get dump of all db
dump = DatabaseDump(dbName, self.dbConnection)
allSql = dump.dumpDatabase()
#get dir path and check path
dirPath = getDirectoryPathFromFilename(destination)
#write into file
dumpFile = DumpFile(destination)
dumpFile.writeIntoFile(allSql)
def processInit(self):
"""
Init the databaseVersioner
:return:
"""
#check if project has been init
check = True
for table in VersionerDatabase.DATABASE_TABLES:
if not self.dbConnection.isTableExist(table):
check = False
if check:
response = TerminalCommand.runDialogYorN(
'Project is initialized. Are you want to init project again? You will lost data !!'
)
if response == False:
return
#delete od elements
toDeletes = self.dbConnection.executeSimpleSQL('SHOW TABLES')
for toDelete in toDeletes:
try:
toDelete = list(toDelete.values())
for item in toDelete:
self.dbConnection.deleteElementbyName(item)
except:
#dont matter
pass
#make new db structure
self.dbConnection.executeSimpleSQL(
VersionerDatabase.DATABASE_STRUCTURE)
def dumpDatabase(self):
"""
Return dump of all database in one string
:return: str
"""
params = [
self.__databaseName, "--user="******"--password="******"--triggers",
"--skip-add-locks", "--skip-comments", "--force"
]
dump = TerminalCommand.runCommandRetStr(self.__command, params)
#dump = utils.removeEmptyLinesFromSql(dump)
return dump
def processAddNonExist(self, dbName, destFolder):
"""
Add database to database versioner and made dump this database
:param dbName: str database name
:param destFolder:
"""
destFolder = os.path.abspath(destFolder)
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
#check if dababase has been versioned
isExist = database.VersionedDatabase.isDatabaseVersioned(
self.dbConnection, dbName)
if isExist:
result = TerminalCommand.runDialogYorN(
"Database '" + dbName +
"' has been versioned. Are you want redeclare it?")
if not result:
return
# get last binary log and last position
binnaryLogNames = BinnaryLogNames(self.dbConnection)
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName)
logDataString = binnarylogs.getAllLogData()
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#make and set new database
newVersion = 1
versionedDatabaseRecord = database.VersionedDatabase.addVersionerDatabase(
self.dbConnection, dbName, destFolder, lastBinnaryLogName,
lastBinnaryLogPosition, newVersion)
# clear and dump new database into logfiles
#todo: dump everything not only tables
LogFileBasic.removeAllFilesInDirectory(destFolder)
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
dump = DatabaseDump(dbName, self.verDatabaseConnection)
tablesDump = dump.dumpTables(versionedDatabaseRecord.getAllTables())
for tableName in tablesDump.keys():
print("Process : " + tableName)
logFileName = LogFile.makeLogFilePath(destFolder, tableName)
logFile = LogFile(logFileName)
logFile.insertVersionIntoFile([tablesDump.get(tableName)],
newVersion)
def getLogContent(self):
"""
Get content of binnary log
:return: string of conten
"""
params = []
if self.__startPosition is not None and self.__startPosition != 0:
params.append("--start-position=" + str(self.__startPosition))
if self.__databaseName is not None:
params.append("--database=" + self.__databaseName)
else:
raise BinnaryLogException('Database name have to be set')
#params.append("--no-defaults")
params.append(self.__filePath)
data = TerminalCommand.runCommandRetStr(self.__command, params)
return data
def dumpTables(self, tables: dict):
"""
Dump tables and views from database
:param tables: list[VersionedTable] list of VersionedTables
:return: dict of key- table name, valua - table/view dump
"""
tablesDump = {}
for table in tables.values():
tableName = table.getValue('name')
params = [
self.__databaseName, "--user="******"--password="******"--triggers",
"--skip-add-locks", "--skip-comments", "--force",
"--add-drop-table", tableName
]
dump = TerminalCommand.runCommandRetStr(self.__command, params)
#dump = utils.removeEmptyLinesFromSql(dump)
tablesDump[tableName] = dump
return tablesDump
def processUp(self, dbName):
"""
Make new database revision
:return:
"""
if not self.dbConnection.isDatabaseExist(dbName):
raise DatabaseException("Database: '" + dbName + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
versionedDatabaseRecord = database.VersionedDatabase(self.dbConnection,
dbName=dbName)
binnaryLogNames = BinnaryLogNames(self.dbConnection)
listOfBinneryLogNames = binnaryLogNames.getBinnaryLogsNamesOlderThan(
versionedDatabaseRecord.getLogFile())
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
lastBinnaryLogPosition = versionedDatabaseRecord.getLogFilePosition()
#check if first log name have name like versionedDatabaseRecord bin name
if versionedDatabaseRecord.getLogFile() not in listOfBinneryLogNames:
lastBinnaryLogPosition = 0
print(
'Warning: Binnary Logs is out of date. Prease increase expire_logs_days'
)
res = TerminalCommand.runDialogYorN(
'All data for version is unavailable. Some data in new version can be missing. Do you want continue'
)
if not res:
return
#check if version in logFiles is same in db
destination = versionedDatabaseRecord.getDestinationFolder()
actualVersion = versionedDatabaseRecord.getActualVersion()
fileNames = LogFile.getAllVerFileNameInDirectory(destination)
for name in fileNames:
logFile = LogFile(
LogFile.makeLogFilePath(destination, fileNames[name]))
fileVersion = logFile.getLastVersion()
if fileVersion > actualVersion:
raise LogFileException(
'Versined files in directory \'' + destination +
"' have newer version then data in database. Please import data into datababe before make new version"
)
#get data from logs
binnarylogs = BinnaryLogs(listOfBinneryLogNames, dbName)
binnarylogs.setStartPositionFirstLog(lastBinnaryLogPosition)
logDataString = binnarylogs.getAllLogData()
#parse data
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#increment actual version - dont save
actualVersion = versionedDatabaseRecord.getActualVersion()
actualVersion = actualVersion + 1
#flush sql into log file
result = binnlogParser.getLogParserResult()
#save to make transaction
logFileData = {}
toDelete = []
try:
#process
for elementName in result.getAllUpdatedElementNames():
elementData = versionedDatabaseRecord.getTableByName(
elementName)
logFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
elementName)
if elementName in result.getCreated():
logFile = LogFile(logFileName)
elif result.getRenameOldNameByNewName(elementName) is not None:
oldName = result.getRenameOldNameByNewName(elementName)
oldLogFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
oldName)
oldLogFile = LogFile(oldLogFileName)
logFileData[oldLogFileName] = oldLogFile.getAllContent()
logFile = LogFile(logFileName)
toDelete.append(logFileName)
logFile.writeIntoFile(oldLogFile.getAllContent())
oldLogFile.deleteFile()
elif versionedDatabaseRecord.getTableByName(elementName):
logFile = LogFile(logFileName)
logFileData[logFileName] = logFile.getAllContent()
else:
#someting wrong
continue
logFile.appendVersionIntoFile(
result.getAllQueriesByName(elementName), actualVersion)
#do others
if result.getOtherQueries():
logFileName = LogFile.makeLogFilePath(
versionedDatabaseRecord.getDestinationFolder(),
LogFile.OTHER_LOG_NAME)
logFile = LogFile(logFileName)
logFileData[logFileName] = logFile.getAllContent()
logFile.appendVersionIntoFile(result.getOtherQueries(),
actualVersion)
#save metadata
if (result.haveSomeData()):
versionedDatabaseRecord.setSomeData(lastBinnaryLogName,
lastBinnaryLogPosition,
actualVersion)
versionedDatabaseRecord.updateMetaDataAboutTables()
else:
versionedDatabaseRecord.setSomeData(lastBinnaryLogName,
lastBinnaryLogPosition)
versionedDatabaseRecord.updateMetaDataAboutTables()
print(
"No data to flush into log files. New version was not make"
)
except Exception as e:
#make transaction
for fileName in toDelete:
logFile = LogFile(fileName)
logFile.deleteFile()
for fileName in logFileData.keys():
logFile = LogFile(fileName)
logFile.writeIntoFile(logFileData[fileName])
raise
def processAddExists(self, dbName, sourceFolder):
"""
Import exist versioned logs into local database
:param dbName:
:param sourceFolder: str path to folder where is logs
:return:
"""
destFolder = os.path.abspath(sourceFolder)
if self.dbConnection.isDatabaseExist(dbName) is False:
raise DatabaseException('Database \'' + dbName +
'\' is not exist. You have to make it')
if not isDirectoryExist(destFolder):
raise FileException("Directory '" + destFolder + "' is not exist")
self.verDatabaseConnection = Database(
Config.getConfig('databaseHost'), Config.getConfig('databasePort'),
Config.getConfig("databaseUser"),
Config.getConfig("databasePassword"), dbName)
# check if dababase has been versioned
isExist = database.VersionedDatabase.isDatabaseVersioned(
self.dbConnection, dbName)
if isExist:
result = TerminalCommand.runDialogYorN(
"Database '" + dbName +
"' has been versioned. Are you want redeclare it?")
if not result:
return
# get last binary log and last position
binnaryLogNames = BinnaryLogNames(self.dbConnection)
lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName()
binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName)
logDataString = binnarylogs.getAllLogData()
binnlogParser = BinnaryLogParser(logDataString)
lastBinnaryLogPosition = binnlogParser.getLastPosition()
#make dump of database
dbDump = DatabaseDump(dbName, self.verDatabaseConnection)
dump = dbDump.dumpDatabase()
#cleart db
self.verDatabaseConnection.clearDatabase()
#make and import database
logFilesNames = LogFile.getAllVerFileNameInDirectory(destFolder)
lastVersion = 0
otherSql = ""
for fileName in logFilesNames.keys():
print("Process : " + logFilesNames[fileName])
logFile = LogFile(
LogFile.makeLogFilePath(destFolder, logFilesNames[fileName]))
if logFile.getLastVersion() > lastVersion:
lastVersion = logFile.getLastVersion()
if LogFile.OTHER_LOG_NAME_FULL in fileName:
otherSql = logFile.getAllSql()
else:
#import data into database
try:
allSql = logFile.getAllSql()
if allSql:
self.verDatabaseConnection.executeSimpleSQL(allSql)
except Exception as e:
#special transtaction for ddl
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise
#do OtherSql
try:
self.verDatabaseConnection.executeSimpleSQL(otherSql)
except Exception as e:
self.verDatabaseConnection.clearDatabase()
self.verDatabaseConnection.executeSimpleSQL(dump)
raise e
#set database to versioning
versionedDatabaseRecord = database.VersionedDatabase.addVersionerDatabase(
self.dbConnection, dbName, destFolder, lastBinnaryLogName,
lastBinnaryLogPosition, lastVersion)