def _parseConfigFile(config_file): """ Parse the config file :param config_file: :return: List of dictionaries with each dictionary containing the failed and failover information?? """ rows = [] with open(config_file) as f: for lineno, line in line_reader(f): groups = line.split() # NOT line.split(' ') due to MPP-15675 if len(groups) not in [1, 2]: msg = "line %d of file %s: expected 1 or 2 groups but found %d" % ( lineno, config_file, len(groups)) raise ExceptionNoStackTraceNeeded(msg) parts = groups[0].split('|') if len(parts) != 3: msg = "line %d of file %s: expected 3 parts on failed segment group, obtained %d" % ( lineno, config_file, len(parts)) raise ExceptionNoStackTraceNeeded(msg) address, port, datadir = parts check_values(lineno, address=address, port=port, datadir=datadir) datadir = normalizeAndValidateInputPath( datadir, f.name, lineno) row = { 'failedAddress': address, 'failedPort': port, 'failedDataDirectory': datadir, 'lineno': lineno } if len(groups) == 2: parts2 = groups[1].split('|') if len(parts2) != 3: msg = "line %d of file %s: expected 3 parts on new segment group, obtained %d" % ( lineno, config_file, len(parts2)) raise ExceptionNoStackTraceNeeded(msg) address2, port2, datadir2 = parts2 check_values(lineno, address=address2, port=port2, datadir=datadir2) datadir2 = normalizeAndValidateInputPath( datadir2, f.name, lineno) row.update({ 'newAddress': address2, 'newPort': port2, 'newDataDirectory': datadir2 }) rows.append(row) RecoveryTripletsUserConfigFile._validate(rows) return rows
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 for line in lines: if index == maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but more were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, configFile)) path = normalizeAndValidateInputPath(line, "config file") dirs.append(path) index += 1 if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print( 'Enter mirror segment data directory location %d of %d >' % (len(dirs) + 1, maxPrimariesPerHost)) line = input().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException as e: print("\n%s\n" % e) return dirs
def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) # validate fileData # allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues['contentId']) address = fixedValues['address'] # # read the rest and add the mirror # port = int(fixedValues['port']) dataDir = normalizeAndValidateInputPath(fixedValues['dataDirectory'], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror(toBuild, primary, hostName, address, port, dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 for line in lines: if index == maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but more were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, configFile)) path = normalizeAndValidateInputPath(line, "config file") dirs.append(path) index += 1 if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print 'Enter mirror segment data directory location %d of %d >' % (len(dirs) + 1, maxPrimariesPerHost) line = raw_input().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException, e: print "\n%s\n" % e
def __getMirrorsToBuildFromConfigFile(self, gpArray): filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [row["address"] for row in rows] interfaceLookup = GpInterfaceToHostNameCache( self.__pool, allAddresses, [None] * len(allAddresses)) # # build up the output now # toBuild = [] primaries = [ seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False) ] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in rows: contentId = int(row['contentId']) address = row['address'] dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno']) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) calc.addMirror(toBuild, primary[0], hostName, address, int(row['port']), dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [row["address"] for row in rows] # # build up the output now # toBuild = [] primaries = [ seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False) ] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in rows: contentId = int(row['contentId']) address = row['address'] dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno']) # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file hostName = address primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) calc.addMirror(toBuild, primary[0], hostName, address, int(row['port']), dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild( toBuild, self.__pool, self.__options.quiet, self.__options.batch_size, parallelPerHost=self.__options.segment_batch_size)
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] filespaceOidToPathMaps = [] while len(filespaceOidToPathMaps) < maxPrimariesPerHost: filespaceOidToPathMaps.append({}) filespaceNameToOid = {} for fs in gpArray.getFilespaces(False): filespaceNameToOid[fs.getName()] = fs.getOid() configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 fsOid = gparray.SYSTEM_FILESPACE enteredFilespaces = {} for line in lines: if line.startswith("filespace "): if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) fsName = line[len("filespace "):].strip() labelOfPathsBeingRead = fsName if fsName not in filespaceNameToOid: raise Exception("Unknown filespace %s specified in input file %s" % \ (fsName, configFile)) fsOid = filespaceNameToOid[fsName] if gpArray.isFileSpaceShared(fsOid): raise Exception("Shared filespace %s does not need to specify in input file %s" % \ (fsName, configFile)) if fsName in enteredFilespaces: raise Exception( "Filespace %s specified twice in input file %s" % \ (fsName, configFile)) enteredFilespaces[fsName] = True index = 0 else: if index == maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but more were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, configFile)) path = normalizeAndValidateInputPath(line, "config file") if fsOid == gparray.SYSTEM_FILESPACE: dirs.append(path) else: filespaceOidToPathMaps[index][fsOid] = path index += 1 if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) # The shared filespace path will set later. for (fsName, fsOid) in filespaceNameToOid.iteritems(): if fsName in enteredFilespaces: continue if not gpArray.isFileSpaceShared(fsOid): break for index in range(maxPrimariesPerHost): filespaceOidToPathMaps[index][fsOid] = None enteredFilespaces[fsName] = True if len(enteredFilespaces) != len(filespaceNameToOid): raise Exception("Only read directories for %d of %d filespaces from %s" % \ (len(enteredFilespaces), len(filespaceNameToOid), configFile)) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print 'Enter mirror segment data directory location %d of %d >' % (len(dirs)+1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException, e: print "\n%s\n" % e for fs in gpArray.getFilespaces(False): # Shared storage path can only be set when we know the # relationship between the primary and mirror. fsys = fs.getFsys() delay_set = False if fsys != None and fsys.isShared(): delay_set = True print 'Skip filespace "%s" stored on the shared filesystem "%s"' % (fs.getName(), fsys.getName()) index = 0 while index < maxPrimariesPerHost: if delay_set: filespaceOidToPathMaps[index][fs.getOid()] = None index += 1 continue print "Enter mirror filespace '%s' directory location %d of %d >" % \ (fs.getName(), index+1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: filespaceOidToPathMaps[index][fs.getOid()] = normalizeAndValidateInputPath(line) index += 1 except PathNormalizationException, e: print "\n%s\n" % e
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line( filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(rows) allAddresses = [ row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap() ] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath( fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception( 'Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append( GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] filespaceOidToPathMaps = [] while len(filespaceOidToPathMaps) < maxPrimariesPerHost: filespaceOidToPathMaps.append({}) filespaceNameToOid = {} for fs in gpArray.getFilespaces(False): filespaceNameToOid[fs.getName()] = fs.getOid() configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 fsOid = gparray.SYSTEM_FILESPACE enteredFilespaces = {} for line in lines: if line.startswith("filespace "): if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) fsName = line[len("filespace "):].strip() labelOfPathsBeingRead = fsName if fsName not in filespaceNameToOid: raise Exception("Unknown filespace %s specified in input file %s" % \ (fsName, configFile)) fsOid = filespaceNameToOid[fsName] if fsName in enteredFilespaces: raise Exception( "Filespace %s specified twice in input file %s" % \ (fsName, configFile)) enteredFilespaces[fsName] = True index = 0 else: if index == maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but more were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, configFile)) path = normalizeAndValidateInputPath(line, "config file") if fsOid == gparray.SYSTEM_FILESPACE: dirs.append(path) else: filespaceOidToPathMaps[index][fsOid] = path index += 1 if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) if len(enteredFilespaces) != len(filespaceNameToOid): raise Exception("Only read directories for %d of %d filespaces from %s" % \ (len(enteredFilespaces), len(filespaceNameToOid), configFile)) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print 'Enter mirror segment data directory location %d of %d >' % ( len(dirs) + 1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException, e: print "\n%s\n" % e for fs in gpArray.getFilespaces(False): index = 0 while index < maxPrimariesPerHost: print "Enter mirror filespace '%s' directory location %d of %d >" % \ (fs.getName(), index+1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: filespaceOidToPathMaps[index][fs.getOid( )] = normalizeAndValidateInputPath(line) index += 1 except PathNormalizationException, e: print "\n%s\n" % e
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] filespaceOidToPathMaps = [] while len(filespaceOidToPathMaps) < maxPrimariesPerHost: filespaceOidToPathMaps.append({}) filespaceNameToOid = {} for fs in gpArray.getFilespaces(False): filespaceNameToOid[fs.getName()] = fs.getOid() configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 fsOid = gparray.SYSTEM_FILESPACE enteredFilespaces = {} for line in lines: if line.startswith("filespace "): if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) fsName = line[len("filespace "):].strip() labelOfPathsBeingRead = fsName if fsName not in filespaceNameToOid: raise Exception("Unknown filespace %s specified in input file %s" % \ (fsName, configFile)) fsOid = filespaceNameToOid[fsName] if gpArray.isFileSpaceShared(fsOid): raise Exception("Shared filespace %s does not need to specify in input file %s" % \ (fsName, configFile)) if fsName in enteredFilespaces: raise Exception( "Filespace %s specified twice in input file %s" % \ (fsName, configFile)) enteredFilespaces[fsName] = True index = 0 else: if index == maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but more were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, configFile)) path = normalizeAndValidateInputPath(line, "config file") if fsOid == gparray.SYSTEM_FILESPACE: dirs.append(path) else: filespaceOidToPathMaps[index][fsOid] = path index += 1 if index < maxPrimariesPerHost: raise Exception('Number of %s directories must equal %d but %d were read from %s' % \ (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile)) # The shared filespace path will set later. for (fsName, fsOid) in filespaceNameToOid.iteritems(): if fsName in enteredFilespaces: continue if not gpArray.isFileSpaceShared(fsOid): break for index in range(maxPrimariesPerHost): filespaceOidToPathMaps[index][fsOid] = None enteredFilespaces[fsName] = True if len(enteredFilespaces) != len(filespaceNameToOid): raise Exception("Only read directories for %d of %d filespaces from %s" % \ (len(enteredFilespaces), len(filespaceNameToOid), configFile)) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print 'Enter mirror segment data directory location %d of %d >' % ( len(dirs) + 1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException, e: print "\n%s\n" % e for fs in gpArray.getFilespaces(False): # Shared storage path can only be set when we know the # relationship between the primary and mirror. fsys = fs.getFsys() delay_set = False if fsys != None and fsys.isShared(): delay_set = True print 'Skip filespace "%s" stored on the shared filesystem "%s"' % ( fs.getName(), fsys.getName()) index = 0 while index < maxPrimariesPerHost: if delay_set: filespaceOidToPathMaps[index][fs.getOid()] = None index += 1 continue print "Enter mirror filespace '%s' directory location %d of %d >" % \ (fs.getName(), index+1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: filespaceOidToPathMaps[index][fs.getOid( )] = normalizeAndValidateInputPath(line) index += 1 except PathNormalizationException, e: print "\n%s\n" % e
def __getDataDirectoriesForMirrors(self, maxPrimariesPerHost, gpArray): dirs = [] filespaceOidToPathMaps = [] while len(filespaceOidToPathMaps) < maxPrimariesPerHost: filespaceOidToPathMaps.append({}) filespaceNameToOid = {} for fs in gpArray.getFilespaces(False): filespaceNameToOid[fs.getName()] = fs.getOid() configFile = self.__options.mirrorDataDirConfigFile if configFile is not None: # # load from config file # lines = readAllLinesFromFile(configFile, stripLines=True, skipEmptyLines=True) labelOfPathsBeingRead = "data" index = 0 fsOid = gparray.SYSTEM_FILESPACE enteredFilespaces = {} for line in lines: if line.startswith("filespace "): if index < maxPrimariesPerHost: raise Exception( "Number of %s directories must equal %d but %d were read from %s" % (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile) ) fsName = line[len("filespace ") :].strip() labelOfPathsBeingRead = fsName if fsName not in filespaceNameToOid: raise Exception("Unknown filespace %s specified in input file %s" % (fsName, configFile)) fsOid = filespaceNameToOid[fsName] if fsName in enteredFilespaces: raise Exception("Filespace %s specified twice in input file %s" % (fsName, configFile)) enteredFilespaces[fsName] = True index = 0 else: if index == maxPrimariesPerHost: raise Exception( "Number of %s directories must equal %d but more were read from %s" % (labelOfPathsBeingRead, maxPrimariesPerHost, configFile) ) path = normalizeAndValidateInputPath(line, "config file") if fsOid == gparray.SYSTEM_FILESPACE: dirs.append(path) else: filespaceOidToPathMaps[index][fsOid] = path index += 1 if index < maxPrimariesPerHost: raise Exception( "Number of %s directories must equal %d but %d were read from %s" % (labelOfPathsBeingRead, maxPrimariesPerHost, index, configFile) ) if len(enteredFilespaces) != len(filespaceNameToOid): raise Exception( "Only read directories for %d of %d filespaces from %s" % (len(enteredFilespaces), len(filespaceNameToOid), configFile) ) else: # # get from stdin # while len(dirs) < maxPrimariesPerHost: print "Enter mirror segment data directory location %d of %d >" % (len(dirs) + 1, maxPrimariesPerHost) line = sys.stdin.readline().strip() if len(line) > 0: try: dirs.append(normalizeAndValidateInputPath(line)) except PathNormalizationException, e: print "\n%s\n" % e for fs in gpArray.getFilespaces(False): index = 0 while index < maxPrimariesPerHost: print "Enter mirror filespace '%s' directory location %d of %d >" % ( fs.getName(), index + 1, maxPrimariesPerHost, ) line = sys.stdin.readline().strip() if len(line) > 0: try: filespaceOidToPathMaps[index][fs.getOid()] = normalizeAndValidateInputPath(line) index += 1 except PathNormalizationException, e: print "\n%s\n" % e
def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile fslist = None rows = [] with open(filename) as f: for lineno, line in line_reader(f): if fslist is None: fslist = parse_filespace_order(filename, lineno, line) else: fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line, fslist) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(fslist, rows) # validate fileData # validateFlexibleHeadersListAllFilespaces("Mirror config", gpArray, fileData) filespaceNameToFilespace = dict([(fs.getName(), fs) for fs in gpArray.getFilespaces(False)]) allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, self.__options.mirrorOffset, [], []) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues["contentId"]) address = fixedValues["address"] # # read the rest and add the mirror # port = int(fixedValues["port"]) replicationPort = int(fixedValues["replicationPort"]) primarySegmentReplicationPort = int(fixedValues["primarySegmentReplicationPort"]) dataDir = normalizeAndValidateInputPath(fixedValues["dataDirectory"], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) filespaceOidToPathMap = {} for fsName, path in flexibleValues.iteritems(): path = normalizeAndValidateInputPath(path, "in config file", row.getLine()) filespaceOidToPathMap[filespaceNameToFilespace[fsName].getOid()] = path primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror( toBuild, primary, hostName, address, port, dataDir, replicationPort, primarySegmentReplicationPort, filespaceOidToPathMap, ) if len(toBuild) != len(primaries): raise Exception( "Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % (len(toBuild), len(primaries)) ) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [ row["newAddress"] for row in rows if "newAddress" in row ] failedSegments = [] failoverSegments = [] for row in rows: # find the failed segment failedAddress = row['failedAddress'] failedPort = row['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( row['failedDataDirectory'], "config file", row['lineno']) failedSegment = None for segment in gpArray.getDbList(): if (segment.getSegmentAddress() == failedAddress and str(segment.getSegmentPort()) == failedPort and segment.getSegmentDataDirectory() == failedDataDirectory): if failedSegment is not None: # this could be an assertion -- configuration should not allow multiple entries! raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address|port|directory '%s|%s|%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address|port|directory '%s|%s|%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) failoverSegment = None if "newAddress" in row: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = row["newAddress"] try: port = int(row["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row['lineno'])) dataDirectory = normalizeAndValidateInputPath( row["newDataDirectory"], "config file", row['lineno']) # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file # FIXME: This appears identical to __getMirrorsToBuildFromConfigFilein clsAddMirrors hostName = address # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() if failedSegment.unreachable: continue segs.append( GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild( segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, forceoverwrite=True, progressMode=self.getProgressMode(), parallelPerHost=self.__options.parallelPerHost)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) allAddresses = [row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap()] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath(fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(("A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception('Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath(fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception('Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments(gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append(GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, forceoverwrite=True)
def getMirrorTriples(self): failedSegments = [] failoverSegments = [] for row in self.rows: # find the failed segment failedAddress = row['failedAddress'] failedPort = row['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( row['failedDataDirectory'], "config file", row['lineno']) failedSegment = None for segment in self.gpArray.getDbList(): if (segment.getSegmentAddress() == failedAddress and str(segment.getSegmentPort()) == failedPort and segment.getSegmentDataDirectory() == failedDataDirectory): if failedSegment is not None: # this could be an assertion -- configuration should not allow multiple entries! raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address|port|directory '%s|%s|%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address|port|directory '%s|%s|%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) # TODO: These 2 cases have different behavior which might be confusing to the user. # "<failed_address>|<port>|<data_dir> <recovery_address>|<port>|<data_dir>" does full recovery # "<failed_address>|<port>|<data_dir>" does incremental recovery failoverSegment = None if "newAddress" in row: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = row["newAddress"] try: port = int(row["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row['lineno'])) dataDirectory = normalizeAndValidateInputPath( row["newDataDirectory"], "config file", row['lineno']) # TODO: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file # TODO: This appears identical to __getMirrorsToBuildFromConfigFilein clsAddMirrors hostName = address # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = _findAndValidatePeersForFailedSegments( self.gpArray, failedSegments) for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] if failedSegment.unreachable: continue self.recoveryTriples.append( RecoverTriplet(failedSegment, peerForFailedSegment, failoverSegments[index])) return self.recoveryTriples