def _parseConfigFile(config_file): """ Parse the config file :param config_file: :return: List of dictionaries with each dictionary containing the failed and failover information?? """ rows = [] with open(config_file) as f: for lineno, line in line_reader(f): groups = line.split() # NOT line.split(' ') due to MPP-15675 if len(groups) not in [1, 2]: msg = "line %d of file %s: expected 1 or 2 groups but found %d" % ( lineno, config_file, len(groups)) raise ExceptionNoStackTraceNeeded(msg) parts = groups[0].split('|') if len(parts) != 3: msg = "line %d of file %s: expected 3 parts on failed segment group, obtained %d" % ( lineno, config_file, len(parts)) raise ExceptionNoStackTraceNeeded(msg) address, port, datadir = parts check_values(lineno, address=address, port=port, datadir=datadir) datadir = normalizeAndValidateInputPath( datadir, f.name, lineno) row = { 'failedAddress': address, 'failedPort': port, 'failedDataDirectory': datadir, 'lineno': lineno } if len(groups) == 2: parts2 = groups[1].split('|') if len(parts2) != 3: msg = "line %d of file %s: expected 3 parts on new segment group, obtained %d" % ( lineno, config_file, len(parts2)) raise ExceptionNoStackTraceNeeded(msg) address2, port2, datadir2 = parts2 check_values(lineno, address=address2, port=port2, datadir=datadir2) datadir2 = normalizeAndValidateInputPath( datadir2, f.name, lineno) row.update({ 'newAddress': address2, 'newPort': port2, 'newDataDirectory': datadir2 }) rows.append(row) RecoveryTripletsUserConfigFile._validate(rows) return rows
def test_line_reader(self): mylist = ['', '# test', 'abc:def'] gen = line_reader(mylist) offset, line = gen.next() self.assertTrue(offset == 3 and line == 'abc:def') with self.assertRaises(StopIteration): gen.next() l1 = "first" l2 = "second" gen = line_reader([l1, l2]) offset, line = gen.next() self.assertTrue(offset == 1 and line == l1) offset, line = gen.next() self.assertTrue(offset == 2 and line == l2) with self.assertRaises(StopIteration): gen.next()
def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) # validate fileData # allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues['contentId']) address = fixedValues['address'] # # read the rest and add the mirror # port = int(fixedValues['port']) dataDir = normalizeAndValidateInputPath(fixedValues['dataDirectory'], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror(toBuild, primary, hostName, address, port, dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [row["address"] for row in rows] interfaceLookup = GpInterfaceToHostNameCache( self.__pool, allAddresses, [None] * len(allAddresses)) # # build up the output now # toBuild = [] primaries = [ seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False) ] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in rows: contentId = int(row['contentId']) address = row['address'] dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno']) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) calc.addMirror(toBuild, primary[0], hostName, address, int(row['port']), dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [row["address"] for row in rows] # # build up the output now # toBuild = [] primaries = [ seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False) ] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in rows: contentId = int(row['contentId']) address = row['address'] dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno']) # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file hostName = address primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) calc.addMirror(toBuild, primary[0], hostName, address, int(row['port']), dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild( toBuild, self.__pool, self.__options.quiet, self.__options.batch_size, parallelPerHost=self.__options.segment_batch_size)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line( filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(rows) allAddresses = [ row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap() ] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath( fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception( 'Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append( GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile fslist = None rows = [] with open(filename) as f: for lineno, line in line_reader(f): if fslist is None: fslist = parse_filespace_order(filename, lineno, line) else: fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line, fslist) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(fslist, rows) # validate fileData # validateFlexibleHeadersListAllFilespaces("Mirror config", gpArray, fileData) filespaceNameToFilespace = dict([(fs.getName(), fs) for fs in gpArray.getFilespaces(False)]) allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, self.__options.mirrorOffset, [], []) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues["contentId"]) address = fixedValues["address"] # # read the rest and add the mirror # port = int(fixedValues["port"]) replicationPort = int(fixedValues["replicationPort"]) primarySegmentReplicationPort = int(fixedValues["primarySegmentReplicationPort"]) dataDir = normalizeAndValidateInputPath(fixedValues["dataDirectory"], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) filespaceOidToPathMap = {} for fsName, path in flexibleValues.iteritems(): path = normalizeAndValidateInputPath(path, "in config file", row.getLine()) filespaceOidToPathMap[filespaceNameToFilespace[fsName].getOid()] = path primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror( toBuild, primary, hostName, address, port, dataDir, replicationPort, primarySegmentReplicationPort, filespaceOidToPathMap, ) if len(toBuild) != len(primaries): raise Exception( "Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % (len(toBuild), len(primaries)) ) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [ row["newAddress"] for row in rows if "newAddress" in row ] failedSegments = [] failoverSegments = [] for row in rows: # find the failed segment failedAddress = row['failedAddress'] failedPort = row['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( row['failedDataDirectory'], "config file", row['lineno']) failedSegment = None for segment in gpArray.getDbList(): if (segment.getSegmentAddress() == failedAddress and str(segment.getSegmentPort()) == failedPort and segment.getSegmentDataDirectory() == failedDataDirectory): if failedSegment is not None: # this could be an assertion -- configuration should not allow multiple entries! raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address|port|directory '%s|%s|%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address|port|directory '%s|%s|%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row['lineno'])) failoverSegment = None if "newAddress" in row: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = row["newAddress"] try: port = int(row["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row['lineno'])) dataDirectory = normalizeAndValidateInputPath( row["newDataDirectory"], "config file", row['lineno']) # FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file # FIXME: This appears identical to __getMirrorsToBuildFromConfigFilein clsAddMirrors hostName = address # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() if failedSegment.unreachable: continue segs.append( GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild( segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, forceoverwrite=True, progressMode=self.getProgressMode(), parallelPerHost=self.__options.parallelPerHost)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) allAddresses = [row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap()] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath(fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(("A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception('Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath(fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception('Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments(gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append(GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, forceoverwrite=True)