def _parseConfigFile(config_file):
"""
Parse the config file
:param config_file:
:return: List of dictionaries with each dictionary containing the failed and failover information??
"""
rows = []
with open(config_file) as f:
for lineno, line in line_reader(f):
groups = line.split() # NOT line.split(' ') due to MPP-15675
if len(groups) not in [1, 2]:
msg = "line %d of file %s: expected 1 or 2 groups but found %d" % (
lineno, config_file, len(groups))
raise ExceptionNoStackTraceNeeded(msg)
parts = groups[0].split('|')
if len(parts) != 3:
msg = "line %d of file %s: expected 3 parts on failed segment group, obtained %d" % (
lineno, config_file, len(parts))
raise ExceptionNoStackTraceNeeded(msg)
address, port, datadir = parts
check_values(lineno,
address=address,
port=port,
datadir=datadir)
datadir = normalizeAndValidateInputPath(
datadir, f.name, lineno)
row = {
'failedAddress': address,
'failedPort': port,
'failedDataDirectory': datadir,
'lineno': lineno
}
if len(groups) == 2:
parts2 = groups[1].split('|')
if len(parts2) != 3:
msg = "line %d of file %s: expected 3 parts on new segment group, obtained %d" % (
lineno, config_file, len(parts2))
raise ExceptionNoStackTraceNeeded(msg)
address2, port2, datadir2 = parts2
check_values(lineno,
address=address2,
port=port2,
datadir=datadir2)
datadir2 = normalizeAndValidateInputPath(
datadir2, f.name, lineno)
row.update({
'newAddress': address2,
'newPort': port2,
'newDataDirectory': datadir2
})
rows.append(row)
RecoveryTripletsUserConfigFile._validate(rows)
return rows
def test_line_reader(self):
mylist = ['', '# test', 'abc:def']
gen = line_reader(mylist)
offset, line = gen.next()
self.assertTrue(offset == 3 and line == 'abc:def')
with self.assertRaises(StopIteration):
gen.next()
l1 = "first"
l2 = "second"
gen = line_reader([l1, l2])
offset, line = gen.next()
self.assertTrue(offset == 1 and line == l1)
offset, line = gen.next()
self.assertTrue(offset == 2 and line == l2)
with self.assertRaises(StopIteration):
gen.next()
def __getMirrorsToBuildFromConfigFile(self, gpArray):
# create fileData object from config file
#
filename = self.__options.mirrorConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line)
rows.append(ParsedConfigFileRow(fixed, flexible, line))
fileData = ParsedConfigFile([], rows)
# validate fileData
#
allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()]
allNoneArr = [None for a in allAddresses]
interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr)
#
# build up the output now
#
toBuild = []
primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)]
segsByContentId = GpArray.getSegmentsByContentId(primaries)
# note: passed port offset in this call should not matter
calc = GpMirrorBuildCalculator(gpArray, [], self.__options)
for row in fileData.getRows():
fixedValues = row.getFixedValuesMap()
flexibleValues = row.getFlexibleValuesMap()
contentId = int(fixedValues['contentId'])
address = fixedValues['address']
#
# read the rest and add the mirror
#
port = int(fixedValues['port'])
dataDir = normalizeAndValidateInputPath(fixedValues['dataDirectory'], "in config file", row.getLine())
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception("Segment Host Address %s is unreachable" % address)
primary = segsByContentId[contentId]
if primary is None:
raise Exception("Invalid content %d specified in input file" % contentId)
primary = primary[0]
calc.addMirror(toBuild, primary, hostName, address, port, dataDir)
if len(toBuild) != len(primaries):
raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \
(len(toBuild), len(primaries)))
return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray):
filename = self.__options.mirrorConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
rows.append(self._getParsedRow(filename, lineno, line))
allAddresses = [row["address"] for row in rows]
interfaceLookup = GpInterfaceToHostNameCache(
self.__pool, allAddresses, [None] * len(allAddresses))
#
# build up the output now
#
toBuild = []
primaries = [
seg for seg in gpArray.getDbList()
if seg.isSegmentPrimary(current_role=False)
]
segsByContentId = GpArray.getSegmentsByContentId(primaries)
# note: passed port offset in this call should not matter
calc = GpMirrorBuildCalculator(gpArray, [], self.__options)
for row in rows:
contentId = int(row['contentId'])
address = row['address']
dataDir = normalizeAndValidateInputPath(row['dataDirectory'],
"in config file",
row['lineno'])
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception("Segment Host Address %s is unreachable" %
address)
primary = segsByContentId[contentId]
if primary is None:
raise Exception("Invalid content %d specified in input file" %
contentId)
calc.addMirror(toBuild, primary[0], hostName, address,
int(row['port']), dataDir)
if len(toBuild) != len(primaries):
raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \
(len(toBuild), len(primaries)))
return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet,
self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray):
filename = self.__options.mirrorConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
rows.append(self._getParsedRow(filename, lineno, line))
allAddresses = [row["address"] for row in rows]
#
# build up the output now
#
toBuild = []
primaries = [
seg for seg in gpArray.getDbList()
if seg.isSegmentPrimary(current_role=False)
]
segsByContentId = GpArray.getSegmentsByContentId(primaries)
# note: passed port offset in this call should not matter
calc = GpMirrorBuildCalculator(gpArray, [], self.__options)
for row in rows:
contentId = int(row['contentId'])
address = row['address']
dataDir = normalizeAndValidateInputPath(row['dataDirectory'],
"in config file",
row['lineno'])
# FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file
hostName = address
primary = segsByContentId[contentId]
if primary is None:
raise Exception("Invalid content %d specified in input file" %
contentId)
calc.addMirror(toBuild, primary[0], hostName, address,
int(row['port']), dataDir)
if len(toBuild) != len(primaries):
raise Exception("Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))" % \
(len(toBuild), len(primaries)))
return GpMirrorListToBuild(
toBuild,
self.__pool,
self.__options.quiet,
self.__options.batch_size,
parallelPerHost=self.__options.segment_batch_size)
def getRecoveryActionsFromConfigFile(self, gpArray):
"""
getRecoveryActionsFromConfigFile
returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object
containing information of segments which are able to recover)
"""
# create fileData object from config file
#
filename = self.__options.recoveryConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
fixed, flexible = parse_gprecoverseg_line(
filename, lineno, line)
rows.append(ParsedConfigFileRow(fixed, flexible, line))
fileData = ParsedConfigFile(rows)
allAddresses = [
row.getFixedValuesMap()["newAddress"]
for row in fileData.getRows()
if "newAddress" in row.getFixedValuesMap()
]
allNoneArr = [None] * len(allAddresses)
interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses,
allNoneArr)
failedSegments = []
failoverSegments = []
for row in fileData.getRows():
fixedValues = row.getFixedValuesMap()
flexibleValues = row.getFlexibleValuesMap()
# find the failed segment
failedAddress = fixedValues['failedAddress']
failedPort = fixedValues['failedPort']
failedDataDirectory = normalizeAndValidateInputPath(
fixedValues['failedDataDirectory'], "config file",
row.getLine())
failedSegment = None
for segment in gpArray.getDbList():
if segment.getSegmentAddress() == failedAddress and \
str(segment.getSegmentPort()) == failedPort and \
segment.getSegmentDataDirectory() == failedDataDirectory:
if failedSegment is not None:
#
# this could be an assertion -- configuration should not allow multiple entries!
#
raise Exception((
"A segment to recover was found twice in configuration. "
"This segment is described by address:port:directory '%s:%s:%s' "
"on the input line: %s") %
(failedAddress, failedPort,
failedDataDirectory, row.getLine()))
failedSegment = segment
if failedSegment is None:
raise Exception("A segment to recover was not found in configuration. " \
"This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" %
(failedAddress, failedPort, failedDataDirectory, row.getLine()))
failoverSegment = None
if "newAddress" in fixedValues:
"""
When the second set was passed, the caller is going to tell us to where we need to failover, so
build a failover segment
"""
# these two lines make it so that failoverSegment points to the object that is registered in gparray
failoverSegment = failedSegment
failedSegment = failoverSegment.copy()
address = fixedValues["newAddress"]
try:
port = int(fixedValues["newPort"])
except ValueError:
raise Exception(
'Config file format error, invalid number value in line: %s'
% (row.getLine()))
dataDirectory = normalizeAndValidateInputPath(
fixedValues["newDataDirectory"], "config file",
row.getLine())
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception(
'Unable to find host name for address %s from line:%s'
% (address, row.getLine()))
# now update values in failover segment
failoverSegment.setSegmentAddress(address)
failoverSegment.setSegmentHostName(hostName)
failoverSegment.setSegmentPort(port)
failoverSegment.setSegmentDataDirectory(dataDirectory)
# this must come AFTER the if check above because failedSegment can be adjusted to
# point to a different object
failedSegments.append(failedSegment)
failoverSegments.append(failoverSegment)
peersForFailedSegments = self.findAndValidatePeersForFailedSegments(
gpArray, failedSegments)
segs = []
segs_with_persistent_mirroring_disabled = []
for index, failedSegment in enumerate(failedSegments):
peerForFailedSegment = peersForFailedSegments[index]
peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId()
segs.append(
GpMirrorToBuild(failedSegment, peerForFailedSegment,
failoverSegments[index],
self.__options.forceFullResynchronization))
self._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
self.__options.parallelDegree)
def __getMirrorsToBuildFromConfigFile(self, gpArray):
# create fileData object from config file
#
filename = self.__options.mirrorConfigFile
fslist = None
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
if fslist is None:
fslist = parse_filespace_order(filename, lineno, line)
else:
fixed, flexible = parse_gpaddmirrors_line(filename, lineno, line, fslist)
rows.append(ParsedConfigFileRow(fixed, flexible, line))
fileData = ParsedConfigFile(fslist, rows)
# validate fileData
#
validateFlexibleHeadersListAllFilespaces("Mirror config", gpArray, fileData)
filespaceNameToFilespace = dict([(fs.getName(), fs) for fs in gpArray.getFilespaces(False)])
allAddresses = [row.getFixedValuesMap()["address"] for row in fileData.getRows()]
allNoneArr = [None for a in allAddresses]
interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr)
#
# build up the output now
#
toBuild = []
primaries = [seg for seg in gpArray.getDbList() if seg.isSegmentPrimary(current_role=False)]
segsByContentId = GpArray.getSegmentsByContentId(primaries)
# note: passed port offset in this call should not matter
calc = GpMirrorBuildCalculator(gpArray, self.__options.mirrorOffset, [], [])
for row in fileData.getRows():
fixedValues = row.getFixedValuesMap()
flexibleValues = row.getFlexibleValuesMap()
contentId = int(fixedValues["contentId"])
address = fixedValues["address"]
#
# read the rest and add the mirror
#
port = int(fixedValues["port"])
replicationPort = int(fixedValues["replicationPort"])
primarySegmentReplicationPort = int(fixedValues["primarySegmentReplicationPort"])
dataDir = normalizeAndValidateInputPath(fixedValues["dataDirectory"], "in config file", row.getLine())
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception("Segment Host Address %s is unreachable" % address)
filespaceOidToPathMap = {}
for fsName, path in flexibleValues.iteritems():
path = normalizeAndValidateInputPath(path, "in config file", row.getLine())
filespaceOidToPathMap[filespaceNameToFilespace[fsName].getOid()] = path
primary = segsByContentId[contentId]
if primary is None:
raise Exception("Invalid content %d specified in input file" % contentId)
primary = primary[0]
calc.addMirror(
toBuild,
primary,
hostName,
address,
port,
dataDir,
replicationPort,
primarySegmentReplicationPort,
filespaceOidToPathMap,
)
if len(toBuild) != len(primaries):
raise Exception(
"Wrong number of mirrors specified (specified %s mirror(s) for %s primarie(s))"
% (len(toBuild), len(primaries))
)
return GpMirrorListToBuild(toBuild, self.__pool, self.__options.quiet, self.__options.parallelDegree)
def getRecoveryActionsFromConfigFile(self, gpArray):
"""
getRecoveryActionsFromConfigFile
returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object
containing information of segments which are able to recover)
"""
filename = self.__options.recoveryConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
rows.append(self._getParsedRow(filename, lineno, line))
allAddresses = [
row["newAddress"] for row in rows if "newAddress" in row
]
failedSegments = []
failoverSegments = []
for row in rows:
# find the failed segment
failedAddress = row['failedAddress']
failedPort = row['failedPort']
failedDataDirectory = normalizeAndValidateInputPath(
row['failedDataDirectory'], "config file", row['lineno'])
failedSegment = None
for segment in gpArray.getDbList():
if (segment.getSegmentAddress() == failedAddress
and str(segment.getSegmentPort()) == failedPort
and segment.getSegmentDataDirectory()
== failedDataDirectory):
if failedSegment is not None:
# this could be an assertion -- configuration should not allow multiple entries!
raise Exception((
"A segment to recover was found twice in configuration. "
"This segment is described by address|port|directory '%s|%s|%s' "
"on the input line: %s") %
(failedAddress, failedPort,
failedDataDirectory, row['lineno']))
failedSegment = segment
if failedSegment is None:
raise Exception("A segment to recover was not found in configuration. " \
"This segment is described by address|port|directory '%s|%s|%s' on the input line: %s" %
(failedAddress, failedPort, failedDataDirectory, row['lineno']))
failoverSegment = None
if "newAddress" in row:
"""
When the second set was passed, the caller is going to tell us to where we need to failover, so
build a failover segment
"""
# these two lines make it so that failoverSegment points to the object that is registered in gparray
failoverSegment = failedSegment
failedSegment = failoverSegment.copy()
address = row["newAddress"]
try:
port = int(row["newPort"])
except ValueError:
raise Exception(
'Config file format error, invalid number value in line: %s'
% (row['lineno']))
dataDirectory = normalizeAndValidateInputPath(
row["newDataDirectory"], "config file", row['lineno'])
# FIXME: hostname probably should not be address, but to do so, "hostname" should be added to gpaddmirrors config file
# FIXME: This appears identical to __getMirrorsToBuildFromConfigFilein clsAddMirrors
hostName = address
# now update values in failover segment
failoverSegment.setSegmentAddress(address)
failoverSegment.setSegmentHostName(hostName)
failoverSegment.setSegmentPort(port)
failoverSegment.setSegmentDataDirectory(dataDirectory)
# this must come AFTER the if check above because failedSegment can be adjusted to
# point to a different object
failedSegments.append(failedSegment)
failoverSegments.append(failoverSegment)
peersForFailedSegments = self.findAndValidatePeersForFailedSegments(
gpArray, failedSegments)
segs = []
segs_with_persistent_mirroring_disabled = []
for index, failedSegment in enumerate(failedSegments):
peerForFailedSegment = peersForFailedSegments[index]
peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId()
if failedSegment.unreachable:
continue
segs.append(
GpMirrorToBuild(failedSegment, peerForFailedSegment,
failoverSegments[index],
self.__options.forceFullResynchronization))
self._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
return GpMirrorListToBuild(
segs,
self.__pool,
self.__options.quiet,
self.__options.parallelDegree,
forceoverwrite=True,
progressMode=self.getProgressMode(),
parallelPerHost=self.__options.parallelPerHost)
def getRecoveryActionsFromConfigFile(self, gpArray):
"""
getRecoveryActionsFromConfigFile
returns: a tuple (segments in change tracking disabled mode which are unable to recover, GpMirrorListToBuild object
containing information of segments which are able to recover)
"""
# create fileData object from config file
#
filename = self.__options.recoveryConfigFile
rows = []
with open(filename) as f:
for lineno, line in line_reader(f):
fixed, flexible = parse_gprecoverseg_line(filename, lineno, line)
rows.append(ParsedConfigFileRow(fixed, flexible, line))
fileData = ParsedConfigFile([], rows)
allAddresses = [row.getFixedValuesMap()["newAddress"] for row in fileData.getRows()
if "newAddress" in row.getFixedValuesMap()]
allNoneArr = [None] * len(allAddresses)
interfaceLookup = GpInterfaceToHostNameCache(self.__pool, allAddresses, allNoneArr)
failedSegments = []
failoverSegments = []
for row in fileData.getRows():
fixedValues = row.getFixedValuesMap()
flexibleValues = row.getFlexibleValuesMap()
# find the failed segment
failedAddress = fixedValues['failedAddress']
failedPort = fixedValues['failedPort']
failedDataDirectory = normalizeAndValidateInputPath(fixedValues['failedDataDirectory'],
"config file", row.getLine())
failedSegment = None
for segment in gpArray.getDbList():
if segment.getSegmentAddress() == failedAddress and \
str(segment.getSegmentPort()) == failedPort and \
segment.getSegmentDataDirectory() == failedDataDirectory:
if failedSegment is not None:
#
# this could be an assertion -- configuration should not allow multiple entries!
#
raise Exception(("A segment to recover was found twice in configuration. "
"This segment is described by address:port:directory '%s:%s:%s' "
"on the input line: %s") %
(failedAddress, failedPort, failedDataDirectory, row.getLine()))
failedSegment = segment
if failedSegment is None:
raise Exception("A segment to recover was not found in configuration. " \
"This segment is described by address:port:directory '%s:%s:%s' on the input line: %s" %
(failedAddress, failedPort, failedDataDirectory, row.getLine()))
failoverSegment = None
if "newAddress" in fixedValues:
"""
When the second set was passed, the caller is going to tell us to where we need to failover, so
build a failover segment
"""
# these two lines make it so that failoverSegment points to the object that is registered in gparray
failoverSegment = failedSegment
failedSegment = failoverSegment.copy()
address = fixedValues["newAddress"]
try:
port = int(fixedValues["newPort"])
except ValueError:
raise Exception('Config file format error, invalid number value in line: %s' % (row.getLine()))
dataDirectory = normalizeAndValidateInputPath(fixedValues["newDataDirectory"], "config file",
row.getLine())
hostName = interfaceLookup.getHostName(address)
if hostName is None:
raise Exception('Unable to find host name for address %s from line:%s' % (address, row.getLine()))
# now update values in failover segment
failoverSegment.setSegmentAddress(address)
failoverSegment.setSegmentHostName(hostName)
failoverSegment.setSegmentPort(port)
failoverSegment.setSegmentDataDirectory(dataDirectory)
# this must come AFTER the if check above because failedSegment can be adjusted to
# point to a different object
failedSegments.append(failedSegment)
failoverSegments.append(failoverSegment)
peersForFailedSegments = self.findAndValidatePeersForFailedSegments(gpArray, failedSegments)
segs = []
segs_with_persistent_mirroring_disabled = []
for index, failedSegment in enumerate(failedSegments):
peerForFailedSegment = peersForFailedSegments[index]
peerForFailedSegmentDbId = peerForFailedSegment.getSegmentDbId()
segs.append(GpMirrorToBuild(failedSegment, peerForFailedSegment, failoverSegments[index],
self.__options.forceFullResynchronization))
self._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
return GpMirrorListToBuild(segs, self.__pool, self.__options.quiet,
self.__options.parallelDegree, forceoverwrite=True)
