def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) # validate fileData # allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues['contentId']) address = fixedValues['address'] # # read the rest and add the mirror # port = int(fixedValues['port']) dataDir = normalizeAndValidateInputPath(fixedValues['dataDirectory'], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror(toBuild, primary, hostName, address, port, dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): filename = self.__options.mirrorConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): rows.append(self._getParsedRow(filename, lineno, line)) allAddresses = [row["address"] for row in rows] interfaceLookup = GpInterfaceToHostNameCache( self.__pool, allAddresses, [None] * len(allAddresses)) # # build up the output now # toBuild = [] primaries = [ seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False) ] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, [], self.__options) for row in rows: contentId = int(row['contentId']) address = row['address'] dataDir = normalizeAndValidateInputPath(row['dataDirectory'], "in config file", row['lineno']) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) calc.addMirror(toBuild, primary[0], hostName, address, int(row['port']), dataDir) if len(toBuild) != len(primaries): raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \ (len(toBuild), len(primaries))) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def getRecoveryActionsFromConfiguration(self, gpEnv, gpArray): """ getRecoveryActionsFromConfiguration returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ segments = gpArray.getSegDbList() failedSegments = [seg for seg in segments if seg.isSegmentDown()] peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) # Dictionaries used for building mapping to new hosts recoverAddressMap = {} recoverHostMap = {} interfaceHostnameWarnings = [] # Check if the array is a "standard" array (isStandardArray, _ignore) = gpArray.isStandardArray() recoverHostIdx = 0 if self.__options.newRecoverHosts and len( self.__options.newRecoverHosts) > 0: for seg in failedSegments: segAddress = seg.getSegmentAddress() segHostname = seg.getSegmentHostName() # Haven't seen this hostname before so we put it on a new host if not recoverHostMap.has_key(segHostname): try: recoverHostMap[ segHostname] = self.__options.newRecoverHosts[ recoverHostIdx] except: # If we get here, not enough hosts were specified in the -p option. Need 1 new host # per 1 failed host. raise Exception( 'Not enough new recovery hosts given for recovery.' ) recoverHostIdx += 1 if isStandardArray: # We have a standard array configuration, so we'll try to use the same # interface naming convention. If this doesn't work, we'll correct it # below on name lookup segInterface = segAddress[segAddress.rfind('-'):] destAddress = recoverHostMap[segHostname] + segInterface destHostname = recoverHostMap[segHostname] else: # Non standard configuration so we won't make assumptions on # naming. Instead we'll use the hostname passed in for both # hostname and address and flag for warning later. destAddress = recoverHostMap[segHostname] destHostname = recoverHostMap[segHostname] # Save off the new host/address for this address. recoverAddressMap[segAddress] = (destHostname, destAddress) # Now that we've generated the mapping, look up all the addresses to make # sure they are resolvable. interfaces = [ address for (_ignore, address) in recoverAddressMap.values() ] interfaceLookup = GpInterfaceToHostNameCache( self.__pool, interfaces, [None] * len(interfaces)) for key in recoverAddressMap.keys(): (newHostname, newAddress) = recoverAddressMap[key] try: addressHostnameLookup = interfaceLookup.getHostName( newAddress) # Lookup failed so use hostname passed in for everything. if addressHostnameLookup is None: interfaceHostnameWarnings.append( "Lookup of %s failed. Using %s for both hostname and address." % (newAddress, newHostname)) newAddress = newHostname except: # Catch all exceptions. We will use hostname instead of address # that we generated. interfaceHostnameWarnings.append( "Lookup of %s failed. Using %s for both hostname and address." % (newAddress, newHostname)) newAddress = newHostname # if we've updated the address to use the hostname because of lookup failure # make sure the hostname is resolvable and up if newHostname == newAddress: try: unix.Ping.local("ping new hostname", newHostname) except: raise Exception("Ping of host %s failed." % newHostname) # Save changes in map recoverAddressMap[key] = (newHostname, newAddress) if len(self.__options.newRecoverHosts) != recoverHostIdx: interfaceHostnameWarnings.append( "The following recovery hosts were not needed:") for h in self.__options.newRecoverHosts[recoverHostIdx:]: interfaceHostnameWarnings.append("\t%s" % h) portAssigner = PortAssigner(gpArray) forceFull = self.__options.forceFullResynchronization segs = [] segs_with_persistent_mirroring_disabled = [] for i in range(len(failedSegments)): failoverSegment = None failedSegment = failedSegments[i] liveSegment = peersForFailedSegments[i] if self.__options.newRecoverHosts and len( self.__options.newRecoverHosts) > 0: (newRecoverHost, newRecoverAddress ) = recoverAddressMap[failedSegment.getSegmentAddress()] # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() failoverSegment.setSegmentHostName(newRecoverHost) failoverSegment.setSegmentAddress(newRecoverAddress) port = portAssigner.findAndReservePort(newRecoverHost, newRecoverAddress) failoverSegment.setSegmentPort(port) segs.append( GpMirrorToBuild(failedSegment, liveSegment, failoverSegment, forceFull)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, interfaceHostnameWarnings)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line( filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(rows) allAddresses = [ row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap() ] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath( fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(( "A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception( 'Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath( fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception( 'Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments( gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append( GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray): # create fileData object from config file # filename = self.__options.mirrorConfigFile fslist = None rows = [] with open(filename) as f: for lineno, line in line_reader(f): if fslist is None: fslist = parse_filespace_order(filename, lineno, line) else: fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line, fslist) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile(fslist, rows) # validate fileData # validateFlexibleHeadersListAllFilespaces("Mirror config", gpArray, fileData) filespaceNameToFilespace = dict([(fs.getName(), fs) for fs in gpArray.getFilespaces(False)]) allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()] allNoneArr = [None for a in allAddresses] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) # # build up the output now # toBuild = [] primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)] segsByContentId = GpArray.getSegmentsByContentId(primaries) # note: passed port offset in this call should not matter calc = GpMirrorBuildCalculator(gpArray, self.__options.mirrorOffset, [], []) for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() contentId = int(fixedValues["contentId"]) address = fixedValues["address"] # # read the rest and add the mirror # port = int(fixedValues["port"]) replicationPort = int(fixedValues["replicationPort"]) primarySegmentReplicationPort = int(fixedValues["primarySegmentReplicationPort"]) dataDir = normalizeAndValidateInputPath(fixedValues["dataDirectory"], "in config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception("Segment Host Address %s is unreachable" % address) filespaceOidToPathMap = {} for fsName, path in flexibleValues.iteritems(): path = normalizeAndValidateInputPath(path, "in config file", row.getLine()) filespaceOidToPathMap[filespaceNameToFilespace[fsName].getOid()] = path primary = segsByContentId[contentId] if primary is None: raise Exception("Invalid content %d specified in input file" % contentId) primary = primary[0] calc.addMirror( toBuild, primary, hostName, address, port, dataDir, replicationPort, primarySegmentReplicationPort, filespaceOidToPathMap, ) if len(toBuild) != len(primaries): raise Exception( "Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % (len(toBuild), len(primaries)) ) return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def getRecoveryActionsFromConfiguration(self, gpEnv, gpArray): """ getRecoveryActionsFromConfiguration returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ segments = gpArray.getSegDbList() failedSegments = [seg for seg in segments if seg.isSegmentDown()] peersForFailedSegments = self.findAndValidatePeersForFailedSegments(gpArray, failedSegments) # Dictionaries used for building mapping to new hosts recoverAddressMap = {} recoverHostMap = {} interfaceHostnameWarnings = [] # Check if the array is a "standard" array (isStandardArray, _ignore) = gpArray.isStandardArray() recoverHostIdx = 0 if self.__options.newRecoverHosts and len(self.__options.newRecoverHosts) > 0: for seg in failedSegments: segAddress = seg.getSegmentAddress() segHostname = seg.getSegmentHostName() # Haven't seen this hostname before so we put it on a new host if not recoverHostMap.has_key(segHostname): try: recoverHostMap[segHostname] = self.__options.newRecoverHosts[recoverHostIdx] except: # If we get here, not enough hosts were specified in the -p option. Need 1 new host # per 1 failed host. raise Exception('Not enough new recovery hosts given for recovery.') recoverHostIdx += 1 if isStandardArray: # We have a standard array configuration, so we'll try to use the same # interface naming convention. If this doesn't work, we'll correct it # below on name lookup segInterface = segAddress[segAddress.rfind('-'):] destAddress = recoverHostMap[segHostname] + segInterface destHostname = recoverHostMap[segHostname] else: # Non standard configuration so we won't make assumptions on # naming. Instead we'll use the hostname passed in for both # hostname and address and flag for warning later. destAddress = recoverHostMap[segHostname] destHostname = recoverHostMap[segHostname] # Save off the new host/address for this address. recoverAddressMap[segAddress] = (destHostname, destAddress) # Now that we've generated the mapping, look up all the addresses to make # sure they are resolvable. interfaces = [address for (_ignore, address) in recoverAddressMap.values()] interfaceLookup = GpInterfaceToHostNameCache(self.__pool, interfaces, [None] * len(interfaces)) for key in recoverAddressMap.keys(): (newHostname, newAddress) = recoverAddressMap[key] try: addressHostnameLookup = interfaceLookup.getHostName(newAddress) # Lookup failed so use hostname passed in for everything. if addressHostnameLookup is None: interfaceHostnameWarnings.append( "Lookup of %s failed. Using %s for both hostname and address." % (newAddress, newHostname)) newAddress = newHostname except: # Catch all exceptions. We will use hostname instead of address # that we generated. interfaceHostnameWarnings.append( "Lookup of %s failed. Using %s for both hostname and address." % (newAddress, newHostname)) newAddress = newHostname # if we've updated the address to use the hostname because of lookup failure # make sure the hostname is resolvable and up if newHostname == newAddress: try: unix.Ping.local("ping new hostname", newHostname) except: raise Exception("Ping of host %s failed." % newHostname) # Save changes in map recoverAddressMap[key] = (newHostname, newAddress) if len(self.__options.newRecoverHosts) != recoverHostIdx: interfaceHostnameWarnings.append("The following recovery hosts were not needed:") for h in self.__options.newRecoverHosts[recoverHostIdx:]: interfaceHostnameWarnings.append("\t%s" % h) portAssigner = PortAssigner(gpArray) forceFull = self.__options.forceFullResynchronization segs = [] segs_with_persistent_mirroring_disabled = [] for i in range(len(failedSegments)): failoverSegment = None failedSegment = failedSegments[i] liveSegment = peersForFailedSegments[i] if self.__options.newRecoverHosts and len(self.__options.newRecoverHosts) > 0: (newRecoverHost, newRecoverAddress) = recoverAddressMap[failedSegment.getSegmentAddress()] # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() failoverSegment.setSegmentHostName(newRecoverHost) failoverSegment.setSegmentAddress(newRecoverAddress) port = portAssigner.findAndReservePort(newRecoverHost, newRecoverAddress) failoverSegment.setSegmentPort(port) segs.append(GpMirrorToBuild(failedSegment, liveSegment, failoverSegment, forceFull)) self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, interfaceHostnameWarnings, forceoverwrite=True)
def getRecoveryActionsFromConfigFile(self, gpArray): """ getRecoveryActionsFromConfigFile returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object containing information of segments which are able to recover) """ # create fileData object from config file # filename = self.__options.recoveryConfigFile rows = [] with open(filename) as f: for lineno, line in line_reader(f): fixed, flexible = parse_gprecoverseg_line(filename, lineno, line) rows.append(ParsedConfigFileRow(fixed, flexible, line)) fileData = ParsedConfigFile([], rows) allAddresses = [row.getFixedValuesMap()["newAddress"] for row in fileData.getRows() if "newAddress" in row.getFixedValuesMap()] allNoneArr = [None] * len(allAddresses) interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr) failedSegments = [] failoverSegments = [] for row in fileData.getRows(): fixedValues = row.getFixedValuesMap() flexibleValues = row.getFlexibleValuesMap() # find the failed segment failedAddress = fixedValues['failedAddress'] failedPort = fixedValues['failedPort'] failedDataDirectory = normalizeAndValidateInputPath(fixedValues['failedDataDirectory'], "config file", row.getLine()) failedSegment = None for segment in gpArray.getDbList(): if segment.getSegmentAddress() == failedAddress and \ str(segment.getSegmentPort()) == failedPort and \ segment.getSegmentDataDirectory() == failedDataDirectory: if failedSegment is not None: # # this could be an assertion -- configuration should not allow multiple entries! # raise Exception(("A segment to recover was found twice in configuration. " "This segment is described by address:port:directory '%s:%s:%s' " "on the input line: %s") % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failedSegment = segment if failedSegment is None: raise Exception("A segment to recover was not found in configuration. " \ "This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" % (failedAddress, failedPort, failedDataDirectory, row.getLine())) failoverSegment = None if "newAddress" in fixedValues: """ When the second set was passed, the caller is going to tell us to where we need to failover, so build a failover segment """ # these two lines make it so that failoverSegment points to the object that is registered in gparray failoverSegment = failedSegment failedSegment = failoverSegment.copy() address = fixedValues["newAddress"] try: port = int(fixedValues["newPort"]) except ValueError: raise Exception('Config file format error, invalid number value in line: %s' % (row.getLine())) dataDirectory = normalizeAndValidateInputPath(fixedValues["newDataDirectory"], "config file", row.getLine()) hostName = interfaceLookup.getHostName(address) if hostName is None: raise Exception('Unable to find host name for address %s from line:%s' % (address, row.getLine())) # now update values in failover segment failoverSegment.setSegmentAddress(address) failoverSegment.setSegmentHostName(hostName) failoverSegment.setSegmentPort(port) failoverSegment.setSegmentDataDirectory(dataDirectory) # this must come AFTER the if check above because failedSegment can be adjusted to # point to a different object failedSegments.append(failedSegment) failoverSegments.append(failoverSegment) peersForFailedSegments = self.findAndValidatePeersForFailedSegments(gpArray, failedSegments) segs = [] segs_with_persistent_mirroring_disabled = [] for index, failedSegment in enumerate(failedSegments): peerForFailedSegment = peersForFailedSegments[index] peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId() segs.append(GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index], self.__options.forceFullResynchronization)) self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet, self.__options.parallelDegree, forceoverwrite=True)
from gppylib.commands import base #------------------------------------------------------------------------- if __name__ == '__main__': pool = base.WorkerPool(1) retCode = 0 try: interfaces = [] hostNames = [] for line in sys.stdin: interfaces.append(line.strip()) hostNames.append(None) lookup = GpInterfaceToHostNameCache(pool, interfaces, hostNames) for interface in interfaces: hostname = lookup.getHostName(interface) if hostname is None: sys.stdout.write("__lookup_of_hostname_failed__\n") else: sys.stdout.write(hostname) sys.stdout.write("\n") except Exception, e: sys.stderr.write("Exception converting hostname from cache: %s" % e.__str__()) sys.stderr.write("\n") retCode = 1 except: sys.stderr.write("Exception found converting interface to hostname")
from gppylib.commands import base #------------------------------------------------------------------------- if __name__ == '__main__': pool = base.WorkerPool(1) retCode = 0 try: interfaces = [] hostNames = [] for line in sys.stdin: interfaces.append(line.strip()) hostNames.append(None) lookup = GpInterfaceToHostNameCache(pool, interfaces, hostNames) for interface in interfaces: hostname = lookup.getHostName(interface) if hostname is None: sys.stdout.write("__lookup_of_hostname_failed__\n") else: sys.stdout.write(hostname) sys.stdout.write("\n") except Exception, e: sys.stderr.write("Exception converting hostname from cache: %s" % e.__str__()) sys.stderr.write("\n") retCode = 1 except: sys.stderr.write("Exception found converting interface to hostname") sys.stderr.write("\n")