def getLastVersion(self): """ Return last version number If last version is no find then return 0 :return: integer """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") list = self.__getAllFileContentByLine() lastVersion = 0 lenOfFile = len(list) for key, line in enumerate(reversed(list)): index = line.find("version:") if index != -1: try: versionStr = line[index + 8:] versionInt = int(versionStr) lastVersion = versionInt break except: raise LogFileException('File \'' + self._filePath + '\' have syntax error on line ' + str(lenOfFile - key)) return lastVersion
def addVersionerDatabase(dbConnection: Database, dbName, destinationFolder, logFile = None, logFilePosition = 0, actualVersion = 0): """ Add database to the versioned databases Make record in the databaseVersioner database with actual version 0 If database has already versioned then delete old records :param dbConnection: Database connetion :param dbName: name of database to versioned :param destinationFolder: folder to save log files, only abs path. :param logFile: str of logFile for this database :param logFilePosition: int number which declare last position in logFile :return: VersionedDatabase """ if logFile is None: logs = dbConnection.executeSimpleSQL("SHOW BINARY LOGS") lastLog = logs[-1] logFile = lastLog.get('Log_name') if logFile is None: raise BinnaryLogException('No binnary log file find') if not dbConnection.isDatabaseExist(dbName): raise DatabaseException("Database: '" + dbName + "' is not exist") if not isDirectoryExist(destinationFolder): raise FileException("Directory '" + destinationFolder + "' is not exist") dataNewDatabase = {'db_name': dbName, 'actual_version': actualVersion, 'log_file': logFile,'log_file_position':logFilePosition, 'destination_folder': destinationFolder} tablesForDatabase = dbConnection.executeSimpleSQL("select table_name from information_schema.tables where TABLE_SCHEMA='" + dbName + "'"); byNameInDb = dbConnection.executeSimpleSQL("SELECT id FROM " + VersionedDatabase.TABLE_NAME + " WHERE `db_name` = %s", (dbName), True) try: dbConnection.begin() # delete od records about vesioning if byNameInDb.get("id") is not None: dbConnection.executeSQL("DELETE FROM " + VersionedTable.TABLE_NAME + " WHERE `fk_ver_databases_id` = " + str(byNameInDb.get("id"))) dbConnection.executeSQL("DELETE FROM " + VersionedDatabase.TABLE_NAME + " WHERE `id` = " + str(byNameInDb.get("id"))) # save data about versioning into database retId = dbConnection.saveIntoTable(VersionedDatabase.TABLE_NAME, dataNewDatabase, withTransaction=False) if retId is None: raise DatabaseException('Save record into table \''+VersionedDatabase.TABLE_NAME+'\' faild') for table in tablesForDatabase: tableName = table['table_name']; data = {'name': tableName, 'actual_version': 0, 'fk_ver_databases_id': retId} dbConnection.saveIntoTable(VersionedTable.TABLE_NAME,data, withTransaction=False) except Exception as e: dbConnection.rollback() raise e dbConnection.commit() versionedDatabase = VersionedDatabase(dbConnection, retId) return versionedDatabase
def getAllSql(self): """ Return string which contain all log file If file dont exist the return empty string :return: string """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") content = self.getAllContent() return content
def getSqlFromVersion(self, version: int): """ Return sql code from defined version (with version) If file dont exist the return empty string :param version: integer of version :return: string """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") list = self.__getAllFileContentByLine() numLine = -1 startLine = len(list) check = False # find start version for key, line in enumerate(list): index = line.find("version:") if index != -1: check = True try: versionStr = line[index + 8:] versionInt = int(versionStr) if versionInt >= version: numLine = key break except: raise LogFileException('File \'' + self._filePath + '\' have syntax error on line ' + str(startLine - key)) if not check: raise LogFileException('File \'' + self._filePath + '\' have syntax error - no version exist') check = False if numLine != -1: for key, line in enumerate(list[numLine:]): if "*/" in line: startLine = key + numLine + 1 check = True break if numLine != -1 and check == False: raise LogFileException('File \'' + self._filePath + '\' have syntax error') string = '\n'.join(list[startLine:]) ret = self.__delMultiLineComments(string) return ret
def deleteFromVersion(self, version: int): """ Delete logs from defined version number with this verison :param version: integer number of version """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") list = self.__getAllFileContentByLine() numLine = -1 endLine = len(list) check = False # find start of version for key, line in enumerate(list): index = line.find("version:") if index != -1: check = True try: versionStr = line[index + 8:] versionInt = int(versionStr) if versionInt >= version: numLine = key break except: raise LogFileException('File \'' + self._filePath + '\' have syntax error on line ' + str(endLine - key)) if not check: raise LogFileException('File \'' + self._filePath + '\' have syntax error - no version exist') check = False if numLine != -1: for key, line in (enumerate(reversed(list[numLine:]))): if "/*" in line: check = True endLine = numLine - key break if numLine != -1 and check == False: raise LogFileException('File \'' + self._filePath + '\' have syntax error') stringToSave = '\n'.join(list[:endLine]) self.writeIntoFile(stringToSave)
def removeAllFilesInDirectory(directory: str): """ Return all files names in directory :param directory path to directory in abs format :return: dict {filename: elementName} """ if not isDirectoryExist(directory): raise FileException("Directory '" + directory + "' is not exist") fileNames = getAllFileNamesInDirectory(directory) for name in fileNames: try: os.remove(directory + "/" + name) except Exception as e: pass return
def getAllVerFileNameInDirectory(directory: str): """ Return all log files names in directory :param directory path to directory in abs format :return: dict {filename: elementName} """ if not isDirectoryExist(directory): raise FileException("Directory '" + directory + "' is not exist") fileNames = getAllFileNamesInDirectory(directory) retNames = {} extensionLen = len(LogFile.EXTENSION) + 1 # +1 dot in file name for name in fileNames: if name.endswith(LogFile.EXTENSION): retNames[name] = name[0:-extensionLen] return retNames
def __init__(self, fileName: str): """ :param fileName: Name of binnary log """ self.__fileDir = config.get('binnaryLogsDir') self.__fileName = fileName self.__filePath = self.__fileDir + self.__fileName self.__command = config.get('mysqlbinlogLocation') self.__startPosition = None self.__databaseName = None if (os.path.isfile(self.__filePath) is False): raise FileException("File '" + self.__filePath + "' do not exist") if not os.access(self.__filePath, os.R_OK): raise BinnaryLogException("Permissinon denied to read file \'" + self.__filePath + "\'")
def removeAllVerFilesInDirecotry(directory: str): """ Remove all versioned files in directory :param directory: abs path do directory """ if not isDirectoryExist(directory): raise FileException("Directory '" + directory + "' is not exist") fileNames = getAllFileNamesInDirectory(directory) extensionLen = len(LogFile.EXTENSION) + 1 # +1 dot in file name for name in fileNames: if name.endswith(LogFile.EXTENSION): try: os.remove( LogFile.makeLogFilePath(directory, name[0:-extensionLen])) except Exception as e: pass
def getLastVersionSql(self): """ Return last version sql code On error raise exceptins :return: string """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") list = self.__getAllFileContentByLine() numLine = -1 startLine = -1 for key, line in enumerate(reversed(list)): index = line.find("version:") if index != -1: numLine = len(list) - key break if numLine >= 0: for key, line in enumerate(list[numLine:]): if "*/" in line: startLine = key + numLine + 1 break if numLine == -1: raise LogFileException( 'File \'' + self._filePath + '\' have syntax error: string \'version:\' was not found in any line' ) if startLine == -1: raise LogFileException( 'File \'' + self._filePath + '\' have syntax error: string \'*/\' was not found in any line' ) string = '\n'.join(list[startLine:]) ret = self.__delMultiLineComments(string) return ret
def mergeFromVersion(self, version): """ merge sql form secifific version to lastet version and save like last version This method is transactional :param version: integer number of version """ if self.__isFileLogExist() != True: raise FileException("File \'" + self._filePath + "\' is not exists") if version <= 0: raise LogFileException("Version number can not be less than zero") all = self.getAllContent() try: lastestVerions = self.getLastVersion() merged = self.getSqlFromVersion(version) self.deleteFromVersion(version) sqlList = merged.split("\n") self.appendVersionIntoFile(sqlList, lastestVerions) except Exception as e: self.writeIntoFile(all) raise
def processAddExists(self, dbName, sourceFolder): """ Import exist versioned logs into local database :param dbName: :param sourceFolder: str path to folder where is logs :return: """ destFolder = os.path.abspath(sourceFolder) if self.dbConnection.isDatabaseExist(dbName) is False: raise DatabaseException('Database \'' + dbName + '\' is not exist. You have to make it') if not isDirectoryExist(destFolder): raise FileException("Directory '" + destFolder + "' is not exist") self.verDatabaseConnection = Database( Config.getConfig('databaseHost'), Config.getConfig('databasePort'), Config.getConfig("databaseUser"), Config.getConfig("databasePassword"), dbName) # check if dababase has been versioned isExist = database.VersionedDatabase.isDatabaseVersioned( self.dbConnection, dbName) if isExist: result = TerminalCommand.runDialogYorN( "Database '" + dbName + "' has been versioned. Are you want redeclare it?") if not result: return # get last binary log and last position binnaryLogNames = BinnaryLogNames(self.dbConnection) lastBinnaryLogName = binnaryLogNames.getLastBinnaryLogName() binnarylogs = BinnaryLogs([lastBinnaryLogName], dbName) logDataString = binnarylogs.getAllLogData() binnlogParser = BinnaryLogParser(logDataString) lastBinnaryLogPosition = binnlogParser.getLastPosition() #make dump of database dbDump = DatabaseDump(dbName, self.verDatabaseConnection) dump = dbDump.dumpDatabase() #cleart db self.verDatabaseConnection.clearDatabase() #make and import database logFilesNames = LogFile.getAllVerFileNameInDirectory(destFolder) lastVersion = 0 otherSql = "" for fileName in logFilesNames.keys(): print("Process : " + logFilesNames[fileName]) logFile = LogFile( LogFile.makeLogFilePath(destFolder, logFilesNames[fileName])) if logFile.getLastVersion() > lastVersion: lastVersion = logFile.getLastVersion() if LogFile.OTHER_LOG_NAME_FULL in fileName: otherSql = logFile.getAllSql() else: #import data into database try: allSql = logFile.getAllSql() if allSql: self.verDatabaseConnection.executeSimpleSQL(allSql) except Exception as e: #special transtaction for ddl self.verDatabaseConnection.clearDatabase() self.verDatabaseConnection.executeSimpleSQL(dump) raise #do OtherSql try: self.verDatabaseConnection.executeSimpleSQL(otherSql) except Exception as e: self.verDatabaseConnection.clearDatabase() self.verDatabaseConnection.executeSimpleSQL(dump) raise e #set database to versioning versionedDatabaseRecord = database.VersionedDatabase.addVersionerDatabase( self.dbConnection, dbName, destFolder, lastBinnaryLogName, lastBinnaryLogPosition, lastVersion)