def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self):
segs_with_persistent_mirroring_disabled = []
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
assert not gprecover_prog.logger.warn.called
def test_check_segment_state_ready_for_recovery_ignores_initial_stdout_warnings(
self, mock_results):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.info = Mock()
mock_results.return_value = CommandResult(
0, '',
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \nmode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n',
False, True)
segment_mock = Mock()
segment_mock.isSegmentQD.return_value = False
segment_mock.isSegmentModeInChangeLogging.return_value = True
segment_mock.getSegmentHostName.return_value = 'foo1'
segment_mock.getSegmentDataDirectory.return_value = 'bar'
segment_mock.getSegmentPort.return_value = 5555
segment_mock.getSegmentDbId.return_value = 2
segment_mock.getSegmentRole.return_value = 'p'
segment_mock.getSegmentMode.return_value = 'c'
segmentList = [segment_mock]
dbsMap = {2: segment_mock}
mock_pool = Mock()
mock_pool.addCommand = Mock()
mock_pool.join = Mock()
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(
segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
gprecover_prog.logger.info.assert_called_once_with(
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of '
)
def test_check_segment_state(self):
options = Mock()
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = CommandResult(1, 'Failed to connect', '',
False, True)
m2 = Mock()
m2.get_results.return_value = CommandResult(0, 'segmentState: Ready',
'', False, True)
mock_pool.getCompletedItems.return_value = []
confProvider = Mock()
m1 = Mock()
m1.getSegmentHostName.return_value = 'foo1'
m1.isSegmentUp.return_value = True
m1.isSegmentMaster.return_value = False
m2 = Mock()
m2.getSegmentHostName.return_value = 'foo2'
m2.isSegmentUp.return_value = True
m2.isSegmentMaster.return_value = False
gparray = Mock()
gparray.getDbList.return_value = [m1, m2]
confProvider.loadSystemConfig.return_value = gparray
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_segment_state(confProvider)
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking_disabled(
self, mock_results):
options = Mock()
mock_results.return_value = CommandResult(
0, '',
'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n',
False, True)
m2 = Mock()
m2.isSegmentQD.return_value = False
m2.isSegmentModeInChangeLogging.return_value = True
m2.getSegmentHostName.return_value = 'foo1'
m2.getSegmentDataDirectory.return_value = 'bar'
m2.getSegmentPort.return_value = 5555
m2.getSegmentDbId.return_value = 2
m2.getSegmentRole.return_value = 'p'
m2.getSegmentMode.return_value = 'c'
segmentList = [m2]
dbsMap = {2: m2}
gprecover_prog = GpRecoverSegmentProgram(options)
mock_pool = Mock()
mock_pool.addCommand = Mock()
mock_pool.join = Mock()
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(
segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_check_segment_state_ready_for_recovery_ignores_initial_stdout_warnings(self, mock_results):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.info = Mock()
mock_results.return_value = CommandResult(0, '', 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \nmode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True)
segment_mock = Mock()
segment_mock.isSegmentQD.return_value = False
segment_mock.isSegmentModeInChangeLogging.return_value = True
segment_mock.getSegmentHostName.return_value = 'foo1'
segment_mock.getSegmentDataDirectory.return_value = 'bar'
segment_mock.getSegmentPort.return_value = 5555
segment_mock.getSegmentDbId.return_value = 2
segment_mock.getSegmentRole.return_value = 'p'
segment_mock.getSegmentMode.return_value = 'c'
segmentList = [segment_mock]
dbsMap = {2:segment_mock}
mock_pool= Mock()
mock_pool.addCommand = Mock()
mock_pool.join = Mock()
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
gprecover_prog.logger.info.assert_called_once_with('Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ')
def test_successful_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0],
[], 1)
self.mock_get_segments_checksum_settings.return_value = ([
self.mirror0
], [])
self.return_one = True
self.mock_build_mirrors.return_value = True
with self.assertRaises(SystemExit) as cm:
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.assertEqual(cm.exception.code, 0)
def test_failed_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0],
[], 1)
self.mock_get_segments_checksum_settings.return_value = ([
self.mirror0
], [])
self.return_one = True
self.mock_build_mirrors.return_value = False
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 1)
def test_check_segment_state_with_segment_not_ready(self):
options = Mock()
m1 = Mock()
m1.get_results.return_value = CommandResult(0, 'Failed to connect', '',
False, True)
m2 = Mock()
m2.get_results.return_value = CommandResult(0,
'segmentState: Not Ready',
'', False, True)
mock_pool = Mock()
mock_pool.getCompletedItems.return_value = [m1, m2]
m1 = Mock()
m1.getSegmentHostName.return_value = 'foo1'
m1.isSegmentUp.return_value = True
m1.isSegmentMaster.return_value = False
m2 = Mock()
m2.getSegmentHostName.return_value = 'foo2'
m2.isSegmentUp.return_value = True
m2.isSegmentMaster.return_value = False
gparray = Mock()
gparray.getDbList.return_value = [m1, m2]
confProvider = Mock()
confProvider.loadSystemConfig.return_value = gparray
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
with self.assertRaisesRegexp(Exception,
'Not ready to connect to database'):
gprecover_prog._check_segment_state(confProvider)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self):
segs_with_persistent_mirroring_disabled = [0, 1]
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
gprecover_prog.logger.warn.assert_called_once_with('Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.')
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(self):
segs_in_change_tracking_disabled = {2:'ChangeTrackingDisabled', 4:'ChangeTrackingDisabled'}
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_in_change_tracking_disabled(segs_in_change_tracking_disabled)
gprecover_prog.logger.warn.assert_called_once_with('Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.')
def test_check_persistent_tables_no_segments(self, mock1):
options = Mock()
segments = []
mock_pool = Mock()
mock_pool.getCompletedItems.return_value = []
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_persistent_tables(segments)
def test_check_persistent_tables_no_segments(self, mock1):
options = Mock()
segments = []
mock_pool = Mock()
mock_pool.getCompletedItems.return_value = []
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_persistent_tables(segments)
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(
self):
segs_with_persistent_mirroring_disabled = []
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
assert not gprecover_prog.logger.warn.called
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(self, mock_sql):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gparray_mock = Mock()
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
segment_mock = Mock()
segment_mock.getSegmentDbId.return_value = 0
result = gprecover_prog.is_segment_mirror_state_mismatched(gparray_mock, segment_mock)
self.assertFalse(result)
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args(args=[])
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpCoordinatorEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL",
return_value=self.execSqlResult),
patch('time.sleep'),
patch(
'gppylib.programs.clsRecoverSegment.GpCoordinatorEnvironment',
GpCoordinatorEnvironmentMock),
# patch('gppylib.system.environment.GpCoordinatorEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpCoordinatorEnvironment.getCoordinatorPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch(
'gppylib.system.configurationInterface.getConfigurationProvider',
self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__',
self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems',
return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(self, mock_sql):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gparray_mock = Mock()
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock()
segment_mock.getSegmentContentId.return_value = 0
result = gprecover_prog.is_segment_mirror_state_mismatched(gparray_mock, segment_mock)
self.assertTrue(result)
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(
self, mock_sql):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gparray_mock = Mock()
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock()
segment_mock.getSegmentContentId.return_value = 0
result = gprecover_prog.is_segment_mirror_state_mismatched(
gparray_mock, segment_mock)
self.assertTrue(result)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(
self):
segs_with_persistent_mirroring_disabled = [0, 1]
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
gprecover_prog.logger.warn.assert_called_once_with(
'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.'
)
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(
self, mock_sql):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gparray_mock = Mock()
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
segment_mock = Mock()
segment_mock.getSegmentDbId.return_value = 0
result = gprecover_prog.is_segment_mirror_state_mismatched(
gparray_mock, segment_mock)
self.assertFalse(result)
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args()
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpMasterEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult),
patch('time.sleep'),
patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock),
# patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_check_persistent_tables(self, mock1):
options = Mock()
segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')]
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = ''
m2 = Mock()
m2.get_results.return_value = ''
mock_pool.getCompletedItems.return_value = [m1, m2]
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_persistent_tables(segments)
def test_check_persistent_tables_error(self, mock1):
options = Mock()
segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')]
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = ['sdfsdf']
m2 = Mock()
m2.get_results.return_value = ['asdfas']
mock_pool.getCompletedItems.return_value = [m1, m2]
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'):
gprecover_prog._check_persistent_tables(segments)
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(
self):
segs_in_change_tracking_disabled = {
2: 'ChangeTrackingDisabled',
4: 'ChangeTrackingDisabled'
}
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog.logger.warn = Mock()
gprecover_prog._output_segments_in_change_tracking_disabled(
segs_in_change_tracking_disabled)
gprecover_prog.logger.warn.assert_called_once_with(
'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.'
)
def test_check_persistent_tables(self, mock1):
options = Mock()
segments = [
self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')
]
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = ''
m2 = Mock()
m2.get_results.return_value = ''
mock_pool.getCompletedItems.return_value = [m1, m2]
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_persistent_tables(segments)
def test_successful_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1)
self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], [])
self.return_one = True
self.mock_build_mirrors.return_value = True
with self.assertRaises(SystemExit) as cm:
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.assertEqual(cm.exception.code, 0)
def test_successful_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit):
self.subject.run()
self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.')
def test_check_persistent_tables_error(self, mock1):
options = Mock()
segments = [
self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')
]
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = ['sdfsdf']
m2 = Mock()
m2.get_results.return_value = ['asdfas']
mock_pool.getCompletedItems.return_value = [m1, m2]
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
with self.assertRaisesRegexp(Exception,
'Please fix the persistent tables issue'):
gprecover_prog._check_persistent_tables(segments)
def test_failed_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 0)
self.subject.logger.info.assert_any_call('The rebalance operation has completed with WARNINGS. '
'Please review the output in the gprecoverseg log.')
def test_check_segment_state(self):
options = Mock()
mock_pool = Mock()
m1 = Mock()
m1.get_results.return_value = CommandResult(1, 'Failed to connect', '', False, True)
m2 = Mock()
m2.get_results.return_value = CommandResult(0, 'segmentState: Ready', '', False, True)
mock_pool.getCompletedItems.return_value = []
confProvider = Mock()
m1 = Mock()
m1.getSegmentHostName.return_value = 'foo1'
m1.isSegmentUp.return_value = True
m1.isSegmentMaster.return_value = False
m2 = Mock()
m2.getSegmentHostName.return_value = 'foo2'
m2.isSegmentUp.return_value = True
m2.isSegmentMaster.return_value = False
gparray = Mock()
gparray.getDbList.return_value = [m1, m2]
confProvider.loadSystemConfig.return_value = gparray
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
gprecover_prog._check_segment_state_for_connection(confProvider)
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking_disabled(self, mock_results):
options = Mock()
mock_results.return_value = CommandResult(0, '', 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True)
m2 = Mock()
m2.isSegmentQD.return_value = False
m2.isSegmentModeInChangeLogging.return_value = True
m2.getSegmentHostName.return_value = 'foo1'
m2.getSegmentDataDirectory.return_value = 'bar'
m2.getSegmentPort.return_value = 5555
m2.getSegmentDbId.return_value = 2
m2.getSegmentRole.return_value = 'p'
m2.getSegmentMode.return_value = 'c'
segmentList = [m2]
dbsMap = {2:m2}
gprecover_prog = GpRecoverSegmentProgram(options)
mock_pool= Mock()
mock_pool.addCommand = Mock()
mock_pool.join = Mock()
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_check_segment_state_with_segment_not_ready(self):
options = Mock()
m1 = Mock()
m1.get_results.return_value = CommandResult(0, 'Failed to connect', '', False, True)
m2 = Mock()
m2.get_results.return_value = CommandResult(0, 'segmentState: Not Ready', '', False, True)
mock_pool = Mock()
mock_pool.getCompletedItems.return_value = [m1, m2]
m1 = Mock()
m1.getSegmentHostName.return_value = 'foo1'
m1.isSegmentUp.return_value = True
m1.isSegmentMaster.return_value = False
m2 = Mock()
m2.getSegmentHostName.return_value = 'foo2'
m2.isSegmentUp.return_value = True
m2.isSegmentMaster.return_value = False
gparray = Mock()
gparray.getDbList.return_value = [m1, m2]
confProvider = Mock()
confProvider.loadSystemConfig.return_value = gparray
gprecover_prog = GpRecoverSegmentProgram(options)
gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool
with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'):
gprecover_prog._check_segment_state_for_connection(confProvider)
def test_successful_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit):
self.subject.run()
self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.')
def test_failed_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1)
self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], [])
self.return_one = True
self.mock_build_mirrors.return_value = False
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 1)
def test_check_segment_change_tracking_disabled_state_return_false(self):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
res = gprecover_prog.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_READY)
self.assertEquals(res, False)
def test_check_segment_change_tracking_disabled_state_return_true(self):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
res = gprecover_prog.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED)
self.assertEquals(res, True)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.config_file_path = os.path.join(self.temp_dir, "foo")
with open(self.config_file_path, "w") as config_file:
config_file.write("")
self.conn = Mock()
self.conn.__enter__ = Mock(return_value=(Mock(), None))
self.conn.__exit__ = Mock(return_value=None)
self.cursor = FakeCursor()
self.db_singleton = Mock()
self.os_env = dict(USER="******")
self.os_env["MASTER_DATA_DIRECTORY"] = self.temp_dir
self.os_env["GPHOME"] = self.temp_dir
self.gparray = self._create_gparray_with_2_primary_2_mirrors()
self.pool = Mock()
self.pool.getCompletedItems.return_value = []
self.pgconf_dict = gucdict()
self.pgconf_dict["port"] = setting("port", "123", None, None, None)
self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None)
self.config_provider_mock = MagicMock(spec=GpConfigurationProvider)
self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock
self.gpArrayMock = MagicMock(spec=GpArray)
self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]]
self.gpArrayMock.hasMirrors = True
self.gpArrayMock.isStandardArray.return_value = (True, None)
self.gpArrayMock.master = self.gparray.master
self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock
self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False)
self.apply_patches([
patch('os.environ', new=self.os_env),
patch('gppylib.db.dbconn.connect', return_value=self.conn),
patch('gppylib.db.dbconn.execSQL', return_value=self.cursor),
patch('gppylib.db.dbconn.execSQLForSingletonRow', return_value=["foo"]),
patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict),
patch('gppylib.commands.gp.GpVersion'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch('gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock),
patch('gppylib.commands.base.WorkerPool', return_value=self.pool),
patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}),
patch('gppylib.gplog.get_default_logger'),
patch.object(GpMirrorListToBuild, "__init__", return_value=None),
patch.object(GpMirrorListToBuild, "buildMirrors"),
patch.object(GpMirrorListToBuild, "getAdditionalWarnings"),
patch.object(GpMirrorListToBuild, "getMirrorsToBuild"),
patch.object(HeapChecksum, "check_segment_consistency"),
patch.object(HeapChecksum, "get_segments_checksum_settings"),
])
self.call_count = 0
self.return_one = True
self.mock_build_mirrors = self.get_mock_from_apply_patch("buildMirrors")
self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch('getMirrorsToBuild')
self.mock_heap_checksum_init = self.get_mock_from_apply_patch("__init__")
self.mock_check_segment_consistency = self.get_mock_from_apply_patch('check_segment_consistency')
self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch('get_segments_checksum_settings')
sys.argv = ["gprecoverseg"] # reset to relatively empty args list
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = self.config_file_path
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
faultProberInterface.gFaultProber = Mock()
def rebalance(self):
self.logger.info("Determining primary and mirror segment pairs to rebalance")
# The current implementation of rebalance calls "gprecoverseg -a" below.
# Thus, if another balanced pair is not synchronized, or has a down mirror
# that pair will be recovered as a side-effect of rebalancing.
unbalanced_primary_segs = []
for segmentPair in self.gpArray.segmentPairs:
if segmentPair.balanced():
continue
if segmentPair.up() and segmentPair.reachable() and segmentPair.synchronized():
unbalanced_primary_segs.append(segmentPair.primaryDB)
else:
self.logger.warning(
"Not rebalancing primary segment dbid %d with its mirror dbid %d because one is either down, unreachable, or not synchronized" \
% (segmentPair.primaryDB.dbid, segmentPair.mirrorDB.dbid))
if not len(unbalanced_primary_segs):
self.logger.info("No segments to rebalance")
return True
unbalanced_primary_segs = GpArray.getSegmentsByHostName(unbalanced_primary_segs)
pool = base.WorkerPool(min(len(unbalanced_primary_segs), self.batch_size))
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.logger.info("Stopping unbalanced primary segments...")
for hostname in list(unbalanced_primary_segs.keys()):
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=base.REMOTE,
remoteHost=hostname,
timeout=600,
segment_batch_size=self.segment_batch_size)
pool.addCommand(cmd)
base.join_and_indicate_progress(pool)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
allSegmentsStopped = (failed_count == 0)
if not allSegmentsStopped:
self.logger.warn("%d segments failed to stop. A full rebalance of the" % failed_count)
self.logger.warn("system is not possible at this time. Please check the")
self.logger.warn("log files, correct the problem, and run gprecoverseg -r")
self.logger.warn("again.")
self.logger.info("gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
segment_reconfigurer = SegmentReconfigurer(logger=self.logger,
worker_pool=pool, timeout=MIRROR_PROMOTION_TIMEOUT)
segment_reconfigurer.reconfigure()
# Final step is to issue a recoverseg operation to resync segments
self.logger.info("Starting segment synchronization")
original_sys_args = sys.argv[:]
self.logger.info("=============================START ANOTHER RECOVER=========================================")
# import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
cmd_args = ['gprecoverseg', '-a', '-B', str(self.batch_size), '-b', str(self.segment_batch_size)]
sys.argv = cmd_args[:]
local_parser = GpRecoverSegmentProgram.createParser()
local_options, args = local_parser.parse_args()
recover_cmd = GpRecoverSegmentProgram.createProgram(local_options, args)
try:
recover_cmd.run()
except SystemExit as e:
if e.code != 0:
self.logger.error("Failed to start the synchronization step of the segment rebalance.")
self.logger.error("Check the gprecoverseg log file, correct any problems, and re-run")
self.logger.error(' '.join(cmd_args))
raise Exception("Error synchronizing.\nError: %s" % str(e))
finally:
if recover_cmd:
recover_cmd.cleanup()
sys.argv = original_sys_args
self.logger.info("==============================END ANOTHER RECOVER==========================================")
except Exception as ex:
raise ex
finally:
pool.join()
pool.haltWork()
pool.joinWorkers()
signal.signal(signal.SIGINT, signal.default_int_handler)
return allSegmentsStopped # if all segments stopped, then a full rebalance was done
def test_check_segment_change_tracking_disabled_state_return_true(self):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
res = gprecover_prog.check_segment_change_tracking_disabled_state(
gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED)
self.assertEquals(res, True)
class GpRecoverSegmentProgramTestCase(GpTestCase):
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args()
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpMasterEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult),
patch('time.sleep'),
patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock),
# patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_check_persistent_tables__when_no_errors_detected__succeeds(self):
segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')]
command1 = Mock(spec=Command)
command1.get_results.return_value = ''
command2 = Mock(spec=Command)
command2.get_results.return_value = ''
self.execSqlResult.fetchall.return_value = [['template1']]
self.worker_pool.getCompletedItems.return_value = [command1, command2]
self.subject._check_persistent_tables(segments)
def test_check_persistent_tables__with_no_segments__succeeds(self):
self.execSqlResult.fetchall.return_value = [['template1']]
self.subject._check_persistent_tables([])
def test_check_persistent_tables__when_error_exists__raises(self):
self.execSqlResult.fetchall.return_value = [['template1']]
segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')]
command1 = Mock()
command1.get_results.return_value = ['sdfsdf']
command2 = Mock()
command2.get_results.return_value = ['asdfas']
self.worker_pool.getCompletedItems.return_value = [command1, command2]
with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'):
self.subject._check_persistent_tables(segments)
def test_check_database_connection__when_all_segments_are_ready_to_connect__returns_true(self):
self.gparray.getDbList.return_value = []
conf_provider = self._get_mock_conf_provider(self.gparray)
self.assertTrue(self.subject._check_database_connection(conf_provider))
def test_check_database_connection__when_raises_beyond_max_retries__returns_false(self):
conf_provider_that_raises = Mock(spec=GpConfigurationProvider)
self.assertFalse(self.subject._check_database_connection(conf_provider_that_raises))
def test_check_segment_state__when_all_segments_ready__succeeds(self):
conf_provider = self._get_mock_conf_provider(self.gparray)
command1 = Mock(spec=Command)
command1.get_results.return_value = CommandResult(0, '', 'segmentState: Ready', False, True)
command2 = Mock(spec=Command)
command2.get_results.return_value = CommandResult(1, '', 'segmentState: Ready', False, True)
self.worker_pool.getCompletedItems.return_value = [command1, command2]
self.subject._check_segment_state_for_connection(conf_provider)
def test_check_segment_state__when_one_segment_not_ready__raises(self):
segment1 = Mock(spec=GpDB)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=GpDB)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
gparray_mock = Mock(spec=GpArray)
gparray_mock.getDbList.return_value = [segment1, segment2]
conf_provider = self._get_mock_conf_provider(gparray_mock)
command1 = Mock(spec=Command)
command1.get_results.return_value = CommandResult(0, '', 'segmentState: Ready', False, True)
command2 = Mock(spec=Command)
command2.get_results.return_value = CommandResult(1, '', 'Failed to connect', False, True)
self.worker_pool.getCompletedItems.return_value = [command1, command2]
with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'):
self.subject._check_segment_state_for_connection(conf_provider)
@patch('gppylib.commands.base.Command.get_results')
def test_check_segment_state_ready_for_recovery_ignores_initial_stderr_warnings(self, mock_results):
mock_results.return_value = CommandResult(0, '',
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \n'
'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n'
'dataState: InChangeTracking\n',
False, True)
segment_mock = Mock(spec=GpDB)
segment_mock.isSegmentQD.return_value = False
segment_mock.isSegmentModeInChangeLogging.return_value = True
segment_mock.getSegmentHostName.return_value = 'foo1'
segment_mock.getSegmentDataDirectory.return_value = 'bar'
segment_mock.getSegmentPort.return_value = 5555
segment_mock.getSegmentDbId.return_value = 2
segment_mock.getSegmentRole.return_value = 'p'
segment_mock.getSegmentMode.return_value = 'c'
segmentList = [segment_mock]
dbsMap = {2: segment_mock}
segmentStates = self.subject.check_segment_state_ready_for_recovery(segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
self.subject.logger.info.assert_called_once_with(
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ')
@patch('gppylib.commands.base.Command.get_results')
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking__sets_disabled_state(self,
mock_results):
mock_results.return_value = CommandResult(0, '',
'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n'
'dataState: InChangeTracking\n',
False, True)
segment = Mock(spec=GpDB)
segment.isSegmentQD.return_value = False
segment.isSegmentModeInChangeLogging.return_value = True
segment.getSegmentHostName.return_value = 'foo1'
segment.getSegmentDataDirectory.return_value = 'bar'
segment.getSegmentPort.return_value = 5555
segment.getSegmentDbId.return_value = 2
segment.getSegmentRole.return_value = 'p'
segment.getSegmentMode.return_value = 'c'
segmentList = [segment]
dbsMap = {2: segment}
segmentStates = self.subject.check_segment_state_ready_for_recovery(segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(self):
segs_in_change_tracking_disabled = {2: 'ChangeTrackingDisabled', 4: 'ChangeTrackingDisabled'}
self.subject._output_segments_in_change_tracking_disabled(segs_in_change_tracking_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.')
def test_check_segment_change_tracking_disabled_state_return_true(self):
res = self.subject.check_segment_change_tracking_disabled_state(
gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED)
self.assertEquals(res, True)
def test_check_segment_change_tracking_disabled_state_return_false(self):
res = self.subject.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_READY)
self.assertEquals(res, False)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self):
segs_with_persistent_mirroring_disabled = [0, 1]
self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.')
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self):
segs_with_persistent_mirroring_disabled = []
self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
assert not self.subject.logger.warn.called
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(self):
self.execSqlResult.fetchall.return_value = [[3], [1]]
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentContentId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock)
self.assertTrue(mismatched)
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_enabled(self):
self.execSqlResult.fetchall.return_value = [[3]]
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentContentId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock)
self.assertFalse(mismatched)
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(self):
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentDbId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock)
self.assertFalse(mismatched)
def test__run__when_no_replication_is_setup__raises(self):
self.gparray.getSegDbList.return_value = []
self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
with self.assertRaisesRegexp(Exception, 'replication is not configured'):
self.subject.run()
def test__run__when_fault_strategy_is_SAN__calls_SAN_failback(self):
with patch.object(GpRecoverSegmentProgram, 'SanFailback', return_value=None) as san_failback:
self.gparray.getSegDbList.return_value = []
self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_SAN
self.subject.run()
san_failback.assert_called_once_with(self.gparray, self.gp_env)
############################################################
# Private
def _get_mock_segment(self, name, port, address, datadir):
segment = Mock(spec=GpDB)
segment.getSegmentHostName.return_value = name
segment.getSegmentAddress.return_value = address
segment.getSegmentPort.return_value = port
segment.getSegmentDataDirectory.return_value = datadir
return segment
def _get_mock_conf_provider(self, gparray_result=None):
conf_provider = Mock(spec=GpConfigurationProvider)
conf_provider.loadSystemConfig.return_value = gparray_result
return conf_provider
def _segments_mock(self):
segment1 = Mock(spec=GpDB)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=GpDB)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
return [segment1, segment2]
def rebalance(self):
# Get the unbalanced primary segments grouped by hostname
# These segments are what we will shutdown.
self.logger.info("Getting unbalanced segments")
unbalanced_primary_segs = GpArray.getSegmentsByHostName(self.gpArray.get_unbalanced_primary_segdbs())
pool = base.WorkerPool()
count = 0
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.logger.info("Stopping unbalanced primary segments...")
for hostname in unbalanced_primary_segs.keys():
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=base.REMOTE,
remoteHost=hostname,
timeout=600)
pool.addCommand(cmd)
count += 1
pool.wait_and_printdots(count, False)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
allSegmentsStopped = (failed_count == 0)
if not allSegmentsStopped:
self.logger.warn("%d segments failed to stop. A full rebalance of the")
self.logger.warn("system is not possible at this time. Please check the")
self.logger.warn("log files, correct the problem, and run gprecoverseg -r")
self.logger.warn("again.")
self.logger.info("gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
# issue a distributed query to make sure we pick up the fault
# that we just caused by shutting down segments
conn = None
try:
self.logger.info("Triggering segment reconfiguration")
dburl = dbconn.DbURL()
conn = dbconn.connect(dburl)
cmd = ReconfigDetectionSQLQueryCommand(conn)
pool.addCommand(cmd)
pool.wait_and_printdots(1, False)
except Exception:
# This exception is expected
pass
finally:
if conn:
conn.close()
# Final step is to issue a recoverseg operation to resync segments
self.logger.info("Starting segment synchronization")
original_sys_args = sys.argv[:]
try:
self.logger.info("=============================START ANOTHER RECOVER=========================================")
# import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
sys.argv = ['gprecoverseg', '-a']
local_parser = GpRecoverSegmentProgram.createParser()
local_options, args = local_parser.parse_args()
cmd = GpRecoverSegmentProgram.createProgram(local_options, args)
cmd.run()
except SystemExit as e:
if e.code != 0:
self.logger.error("Failed to start the synchronization step of the segment rebalance.")
self.logger.error("Check the gprecoverseg log file, correct any problems, and re-run")
self.logger.error("'gprecoverseg -a'.")
raise Exception("Error synchronizing.\nError: %s" % str(e))
finally:
if cmd:
cmd.cleanup()
sys.argv = original_sys_args
self.logger.info("==============================END ANOTHER RECOVER==========================================")
except Exception, ex:
raise ex
def rebalance(self):
# Get the unbalanced primary segments grouped by hostname
# These segments are what we will shutdown.
self.logger.info("Getting unbalanced segments")
unbalanced_primary_segs = GpArray.getSegmentsByHostName(
self.gpArray.get_unbalanced_primary_segdbs())
pool = base.WorkerPool()
try:
# Disable ctrl-c
signal.signal(signal.SIGINT, signal.SIG_IGN)
self.logger.info("Stopping unbalanced primary segments...")
for hostname in unbalanced_primary_segs.keys():
cmd = GpSegStopCmd("stop unbalanced primary segs",
self.gpEnv.getGpHome(),
self.gpEnv.getGpVersion(),
'fast',
unbalanced_primary_segs[hostname],
ctxt=base.REMOTE,
remoteHost=hostname,
timeout=600)
pool.addCommand(cmd)
base.join_and_indicate_progress(pool)
failed_count = 0
completed = pool.getCompletedItems()
for res in completed:
if not res.get_results().wasSuccessful():
failed_count += 1
allSegmentsStopped = (failed_count == 0)
if not allSegmentsStopped:
self.logger.warn(
"%d segments failed to stop. A full rebalance of the")
self.logger.warn(
"system is not possible at this time. Please check the")
self.logger.warn(
"log files, correct the problem, and run gprecoverseg -r")
self.logger.warn("again.")
self.logger.info(
"gprecoverseg will continue with a partial rebalance.")
pool.empty_completed_items()
segment_reconfigurer = SegmentReconfigurer(
logger=self.logger,
worker_pool=pool,
timeout=MIRROR_PROMOTION_TIMEOUT)
segment_reconfigurer.reconfigure()
# Final step is to issue a recoverseg operation to resync segments
self.logger.info("Starting segment synchronization")
original_sys_args = sys.argv[:]
try:
self.logger.info(
"=============================START ANOTHER RECOVER========================================="
)
# import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
sys.argv = ['gprecoverseg', '-a']
local_parser = GpRecoverSegmentProgram.createParser()
local_options, args = local_parser.parse_args()
cmd = GpRecoverSegmentProgram.createProgram(
local_options, args)
cmd.run()
except SystemExit as e:
if e.code != 0:
self.logger.error(
"Failed to start the synchronization step of the segment rebalance."
)
self.logger.error(
"Check the gprecoverseg log file, correct any problems, and re-run"
)
self.logger.error("'gprecoverseg -a'.")
raise Exception("Error synchronizing.\nError: %s" % str(e))
finally:
if cmd:
cmd.cleanup()
sys.argv = original_sys_args
self.logger.info(
"==============================END ANOTHER RECOVER=========================================="
)
except Exception, ex:
raise ex
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.config_file_path = os.path.join(self.temp_dir, "foo")
with open(self.config_file_path, "w") as config_file:
config_file.write("")
self.conn = Mock()
self.conn.__enter__ = Mock(return_value=(Mock(), None))
self.conn.__exit__ = Mock(return_value=None)
self.cursor = FakeCursor()
self.db_singleton = Mock()
self.os_env = dict(USER="******")
self.os_env["COORDINATOR_DATA_DIRECTORY"] = self.temp_dir
self.os_env["GPHOME"] = self.temp_dir
self.gparray = self._create_gparray_with_2_primary_2_mirrors()
self.pool = Mock()
self.pool.getCompletedItems.return_value = []
self.pgconf_dict = gucdict()
self.pgconf_dict["port"] = setting("port", "123", None, None, None)
self.pgconf_dict["max_connection"] = setting("max_connections", "1",
None, None, None)
self.config_provider_mock = MagicMock(spec=GpConfigurationProvider)
self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock
self.gpArrayMock = MagicMock(spec=GpArray)
self.gpArrayMock.getDbList.side_effect = [[self.primary0],
[self.primary0],
[self.primary0]]
self.gpArrayMock.segmentPairs = []
self.gpArrayMock.hasMirrors = True
self.gpArrayMock.isStandardArray.return_value = (True, None)
self.gpArrayMock.coordinator = self.gparray.coordinator
self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock
self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0,
None, False)
self.apply_patches([
patch('os.environ', new=self.os_env),
patch('gppylib.db.dbconn.connect', return_value=self.conn),
patch('gppylib.db.dbconn.query', return_value=self.cursor),
patch('gppylib.db.dbconn.queryRow', return_value=["foo"]),
patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict),
patch('gppylib.commands.gp.GpVersion'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch(
'gppylib.system.configurationInterface.getConfigurationProvider',
return_value=self.config_provider_mock),
patch('gppylib.commands.base.WorkerPool', return_value=self.pool),
patch('gppylib.gparray.GpArray.getSegmentsByHostName',
return_value={}),
patch('gppylib.gplog.get_default_logger'),
patch.object(GpMirrorListToBuild, "__init__", return_value=None),
patch.object(GpMirrorListToBuild, "buildMirrors"),
patch.object(GpMirrorListToBuild, "getAdditionalWarnings"),
patch.object(GpMirrorListToBuild, "getMirrorsToBuild"),
patch.object(HeapChecksum, "check_segment_consistency"),
patch.object(HeapChecksum, "get_segments_checksum_settings"),
])
self.call_count = 0
self.return_one = True
self.mock_build_mirrors = self.get_mock_from_apply_patch(
"buildMirrors")
self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch(
'getMirrorsToBuild')
self.mock_heap_checksum_init = self.get_mock_from_apply_patch(
"__init__")
self.mock_check_segment_consistency = self.get_mock_from_apply_patch(
'check_segment_consistency')
self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch(
'get_segments_checksum_settings')
sys.argv = ["gprecoverseg"] # reset to relatively empty args list
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = self.config_file_path
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
faultProberInterface.gFaultProber = Mock()
class GpRecoversegTestCase(GpTestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.config_file_path = os.path.join(self.temp_dir, "foo")
with open(self.config_file_path, "w") as config_file:
config_file.write("")
self.conn = Mock()
self.conn.__enter__ = Mock(return_value=(Mock(), None))
self.conn.__exit__ = Mock(return_value=None)
self.cursor = FakeCursor()
self.db_singleton = Mock()
self.os_env = dict(USER="******")
self.os_env["COORDINATOR_DATA_DIRECTORY"] = self.temp_dir
self.os_env["GPHOME"] = self.temp_dir
self.gparray = self._create_gparray_with_2_primary_2_mirrors()
self.pool = Mock()
self.pool.getCompletedItems.return_value = []
self.pgconf_dict = gucdict()
self.pgconf_dict["port"] = setting("port", "123", None, None, None)
self.pgconf_dict["max_connection"] = setting("max_connections", "1",
None, None, None)
self.config_provider_mock = MagicMock(spec=GpConfigurationProvider)
self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock
self.gpArrayMock = MagicMock(spec=GpArray)
self.gpArrayMock.getDbList.side_effect = [[self.primary0],
[self.primary0],
[self.primary0]]
self.gpArrayMock.segmentPairs = []
self.gpArrayMock.hasMirrors = True
self.gpArrayMock.isStandardArray.return_value = (True, None)
self.gpArrayMock.coordinator = self.gparray.coordinator
self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock
self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0,
None, False)
self.apply_patches([
patch('os.environ', new=self.os_env),
patch('gppylib.db.dbconn.connect', return_value=self.conn),
patch('gppylib.db.dbconn.query', return_value=self.cursor),
patch('gppylib.db.dbconn.queryRow', return_value=["foo"]),
patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict),
patch('gppylib.commands.gp.GpVersion'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch(
'gppylib.system.configurationInterface.getConfigurationProvider',
return_value=self.config_provider_mock),
patch('gppylib.commands.base.WorkerPool', return_value=self.pool),
patch('gppylib.gparray.GpArray.getSegmentsByHostName',
return_value={}),
patch('gppylib.gplog.get_default_logger'),
patch.object(GpMirrorListToBuild, "__init__", return_value=None),
patch.object(GpMirrorListToBuild, "buildMirrors"),
patch.object(GpMirrorListToBuild, "getAdditionalWarnings"),
patch.object(GpMirrorListToBuild, "getMirrorsToBuild"),
patch.object(HeapChecksum, "check_segment_consistency"),
patch.object(HeapChecksum, "get_segments_checksum_settings"),
])
self.call_count = 0
self.return_one = True
self.mock_build_mirrors = self.get_mock_from_apply_patch(
"buildMirrors")
self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch(
'getMirrorsToBuild')
self.mock_heap_checksum_init = self.get_mock_from_apply_patch(
"__init__")
self.mock_check_segment_consistency = self.get_mock_from_apply_patch(
'check_segment_consistency')
self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch(
'get_segments_checksum_settings')
sys.argv = ["gprecoverseg"] # reset to relatively empty args list
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = self.config_file_path
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
faultProberInterface.gFaultProber = Mock()
def _get_test_mirrors(self):
if self.return_one:
return [self.mirror_to_build]
if self.call_count == 0:
self.call_count += 1
return [self.mirror_to_build]
else:
self.call_count += 1
return []
def tearDown(self):
shutil.rmtree(self.temp_dir)
super(GpRecoversegTestCase, self).tearDown()
def test_when_checksums_mismatch_it_raises(self):
self.primary0.heap_checksum = 0
self.mock_check_segment_consistency.return_value = ([],
[self.primary0], 1)
self.mock_get_segments_checksum_settings.return_value = ([
self.mirror0
], [])
self.return_one = True
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.assertTrue(self.gparray.coordinator.isSegmentCoordinator(True))
with self.assertRaisesRegex(
Exception,
"Heap checksum setting differences reported on segments"):
self.subject.run()
self.mock_get_segments_checksum_settings.assert_called_with(
[self.primary0])
self.subject.logger.fatal.assert_any_call(
'Heap checksum setting differences reported on segments')
self.subject.logger.fatal.assert_any_call(
'Failed checksum consistency validation:')
self.subject.logger.fatal.assert_any_call(
'sdw1 checksum set to 0 differs from coordinator checksum set to 1'
)
@patch.object(HeapChecksum, "__init__", return_value=None)
def test_when_cannot_determine_checksums_it_raises(
self, mock_heap_checksum_init):
self.mock_check_segment_consistency.return_value = ([], [1], True)
self.mock_get_segments_checksum_settings.return_value = ([], [])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.return_one = True
self.assertTrue(self.gparray.coordinator.isSegmentCoordinator(True))
with self.assertRaisesRegex(
Exception,
"No segments responded to ssh query for heap checksum validation."
):
self.subject.run()
self.mock_get_segments_checksum_settings.assert_called_with(
[self.primary0])
mock_heap_checksum_init.assert_called_with(self.gpArrayMock,
logger=self.subject.logger,
num_workers=1)
@patch("os._exit")
def test_when_no_segments_to_recover_validation_succeeds(self, _):
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.return_one = False
with self.assertRaises(SystemExit):
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.subject.logger.info.assert_any_call(
'No checksum validation necessary when '
'there are no segments to recover.')
@patch.object(TableLogger, "info")
@patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=True)
@patch("os._exit")
def test_successful_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit):
self.subject.run()
self.subject.logger.info.assert_any_call(
'The rebalance operation has completed successfully.')
@patch.object(TableLogger, "info")
@patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=False)
@patch("os._exit")
def test_failed_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 0)
self.subject.logger.info.assert_any_call(
'The rebalance operation has completed with WARNINGS. '
'Please review the output in the gprecoverseg log.')
@patch.object(TableLogger, "info")
def test_failed_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0],
[], 1)
self.mock_get_segments_checksum_settings.return_value = ([
self.mirror0
], [])
self.return_one = True
self.mock_build_mirrors.return_value = False
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 1)
@patch.object(TableLogger, "info")
def test_successful_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.coordinatorDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(
spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0],
[], 1)
self.mock_get_segments_checksum_settings.return_value = ([
self.mirror0
], [])
self.return_one = True
self.mock_build_mirrors.return_value = True
with self.assertRaises(SystemExit) as cm:
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.assertEqual(cm.exception.code, 0)
def _create_gparray_with_2_primary_2_mirrors(self):
coordinator = Segment.initFromString(
"1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator")
self.primary0 = Segment.initFromString(
"2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0")
primary1 = Segment.initFromString(
"3|1|p|p|s|u|sdw2|sdw2|40001|/data/primary1")
self.mirror0 = Segment.initFromString(
"4|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0")
mirror1 = Segment.initFromString(
"5|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1")
return GpArray(
[coordinator, self.primary0, primary1, self.mirror0, mirror1])
class GpRecoversegTestCase(GpTestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.config_file_path = os.path.join(self.temp_dir, "foo")
with open(self.config_file_path, "w") as config_file:
config_file.write("")
self.conn = Mock()
self.conn.__enter__ = Mock(return_value=(Mock(), None))
self.conn.__exit__ = Mock(return_value=None)
self.cursor = FakeCursor()
self.db_singleton = Mock()
self.os_env = dict(USER="******")
self.os_env["MASTER_DATA_DIRECTORY"] = self.temp_dir
self.os_env["GPHOME"] = self.temp_dir
self.gparray = self._create_gparray_with_2_primary_2_mirrors()
self.pool = Mock()
self.pool.getCompletedItems.return_value = []
self.pgconf_dict = gucdict()
self.pgconf_dict["port"] = setting("port", "123", None, None, None)
self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None)
self.config_provider_mock = MagicMock(spec=GpConfigurationProvider)
self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock
self.gpArrayMock = MagicMock(spec=GpArray)
self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]]
self.gpArrayMock.hasMirrors = True
self.gpArrayMock.isStandardArray.return_value = (True, None)
self.gpArrayMock.master = self.gparray.master
self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock
self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False)
self.apply_patches([
patch('os.environ', new=self.os_env),
patch('gppylib.db.dbconn.connect', return_value=self.conn),
patch('gppylib.db.dbconn.execSQL', return_value=self.cursor),
patch('gppylib.db.dbconn.execSQLForSingletonRow', return_value=["foo"]),
patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict),
patch('gppylib.commands.gp.GpVersion'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch('gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock),
patch('gppylib.commands.base.WorkerPool', return_value=self.pool),
patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}),
patch('gppylib.gplog.get_default_logger'),
patch.object(GpMirrorListToBuild, "__init__", return_value=None),
patch.object(GpMirrorListToBuild, "buildMirrors"),
patch.object(GpMirrorListToBuild, "getAdditionalWarnings"),
patch.object(GpMirrorListToBuild, "getMirrorsToBuild"),
patch.object(HeapChecksum, "check_segment_consistency"),
patch.object(HeapChecksum, "get_segments_checksum_settings"),
])
self.call_count = 0
self.return_one = True
self.mock_build_mirrors = self.get_mock_from_apply_patch("buildMirrors")
self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch('getMirrorsToBuild')
self.mock_heap_checksum_init = self.get_mock_from_apply_patch("__init__")
self.mock_check_segment_consistency = self.get_mock_from_apply_patch('check_segment_consistency')
self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch('get_segments_checksum_settings')
sys.argv = ["gprecoverseg"] # reset to relatively empty args list
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = self.config_file_path
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
faultProberInterface.gFaultProber = Mock()
def _get_test_mirrors(self):
if self.return_one:
return [self.mirror_to_build]
if self.call_count == 0:
self.call_count += 1
return [self.mirror_to_build]
else:
self.call_count += 1
return []
def tearDown(self):
shutil.rmtree(self.temp_dir)
super(GpRecoversegTestCase, self).tearDown()
def test_when_checksums_mismatch_it_raises(self):
self.primary0.heap_checksum = 0
self.mock_check_segment_consistency.return_value = ([], [self.primary0], 1)
self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], [])
self.return_one = True
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.assertTrue(self.gparray.master.isSegmentMaster(True))
with self.assertRaisesRegexp(Exception, "Heap checksum setting differences reported on segments"):
self.subject.run()
self.mock_get_segments_checksum_settings.assert_called_with([self.primary0])
self.subject.logger.fatal.assert_any_call('Heap checksum setting differences reported on segments')
self.subject.logger.fatal.assert_any_call('Failed checksum consistency validation:')
self.subject.logger.fatal.assert_any_call('sdw1 checksum set to 0 differs from master checksum set to 1')
@patch.object(HeapChecksum, "__init__", return_value=None)
def test_when_cannot_determine_checksums_it_raises(self, mock_heap_checksum_init):
self.mock_check_segment_consistency.return_value = ([], [1], True)
self.mock_get_segments_checksum_settings.return_value = ([], [])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.return_one = True
self.assertTrue(self.gparray.master.isSegmentMaster(True))
with self.assertRaisesRegexp(Exception, "No segments responded to ssh query for heap checksum validation."):
self.subject.run()
self.mock_get_segments_checksum_settings.assert_called_with([self.primary0])
mock_heap_checksum_init.assert_called_with(self.gpArrayMock, logger=self.subject.logger, num_workers=1)
@patch("os._exit")
def test_when_no_segments_to_recover_validation_succeeds(self, _):
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.return_one = False
with self.assertRaises(SystemExit):
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.subject.logger.info.assert_any_call('No checksum validation necessary when '
'there are no segments to recover.')
@patch.object(TableLogger, "info")
@patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=True)
@patch("os._exit")
def test_successful_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit):
self.subject.run()
self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.')
@patch.object(TableLogger, "info")
@patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=False)
@patch("os._exit")
def test_failed_rebalance(self, _, __, ___):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.rebalanceSegments = True
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 0)
self.subject.logger.info.assert_any_call('The rebalance operation has completed with WARNINGS. '
'Please review the output in the gprecoverseg log.')
@patch.object(TableLogger, "info")
def test_failed_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1)
self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], [])
self.return_one = True
self.mock_build_mirrors.return_value = False
with self.assertRaises(SystemExit) as cm:
self.subject.run()
self.assertEqual(cm.exception.code, 1)
@patch.object(TableLogger, "info")
def test_successful_recover(self, _):
self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0]
options = Options()
options.masterDataDirectory = self.temp_dir
options.spareDataDirectoryFile = None
options.showProgress = True
options.showProgressInplace = True
# import HERE so that patches are already in place!
from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram
self.subject = GpRecoverSegmentProgram(options)
self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal'])
self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors
self.primary0.heap_checksum = 1
self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1)
self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], [])
self.return_one = True
self.mock_build_mirrors.return_value = True
with self.assertRaises(SystemExit) as cm:
# XXX Disable live FTS probes. The fact that we have to do this
# indicates that these are not really unit tests.
with patch.object(self.subject, 'trigger_fts_probe'):
self.subject.run()
self.assertEqual(cm.exception.code, 0)
def _create_gparray_with_2_primary_2_mirrors(self):
master = Segment.initFromString(
"1|-1|p|p|s|u|mdw|mdw|5432|/data/master")
self.primary0 = Segment.initFromString(
"2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0")
primary1 = Segment.initFromString(
"3|1|p|p|s|u|sdw2|sdw2|40001|/data/primary1")
self.mirror0 = Segment.initFromString(
"4|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0")
mirror1 = Segment.initFromString(
"5|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1")
return GpArray([master, self.primary0, primary1, self.mirror0, mirror1])
def test_check_database_connection_exceed_max_retries(self, mock1, mock2):
options = Mock()
confProvider = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
self.assertFalse(gprecover_prog._check_database_connection(confProvider))
def test_check_segment_change_tracking_disabled_state_return_false(self):
options = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
res = gprecover_prog.check_segment_change_tracking_disabled_state(
gparray.SEGMENT_STATE_READY)
self.assertEquals(res, False)
def test_check_database_connection(self, mock1, mock2):
options = Mock()
confProvider = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
self.assertTrue(gprecover_prog._check_database_connection(confProvider))
class GpRecoverSegmentProgramTestCase(GpTestCase):
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args()
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpMasterEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL",
return_value=self.execSqlResult),
patch('time.sleep'),
patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment',
GpMasterEnvironmentMock),
# patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch(
'gppylib.system.configurationInterface.getConfigurationProvider',
self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__',
self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems',
return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_check_persistent_tables__when_no_errors_detected__succeeds(self):
segments = [
self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')
]
command1 = Mock(spec=Command)
command1.get_results.return_value = ''
command2 = Mock(spec=Command)
command2.get_results.return_value = ''
self.execSqlResult.fetchall.return_value = [['template1']]
self.worker_pool.getCompletedItems.return_value = [command1, command2]
self.subject._check_persistent_tables(segments)
def test_check_persistent_tables__with_no_segments__succeeds(self):
self.execSqlResult.fetchall.return_value = [['template1']]
self.subject._check_persistent_tables([])
def test_check_persistent_tables__when_error_exists__raises(self):
self.execSqlResult.fetchall.return_value = [['template1']]
segments = [
self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'),
self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')
]
command1 = Mock()
command1.get_results.return_value = ['sdfsdf']
command2 = Mock()
command2.get_results.return_value = ['asdfas']
self.worker_pool.getCompletedItems.return_value = [command1, command2]
with self.assertRaisesRegexp(Exception,
'Please fix the persistent tables issue'):
self.subject._check_persistent_tables(segments)
def test_check_database_connection__when_all_segments_are_ready_to_connect__returns_true(
self):
self.gparray.getDbList.return_value = []
conf_provider = self._get_mock_conf_provider(self.gparray)
self.assertTrue(self.subject._check_database_connection(conf_provider))
def test_check_database_connection__when_raises_beyond_max_retries__returns_false(
self):
conf_provider_that_raises = Mock(spec=GpConfigurationProvider)
self.assertFalse(
self.subject._check_database_connection(conf_provider_that_raises))
def test_check_segment_state__when_all_segments_ready__succeeds(self):
conf_provider = self._get_mock_conf_provider(self.gparray)
command1 = Mock(spec=Command)
command1.get_results.return_value = CommandResult(
0, '', 'segmentState: Ready', False, True)
command2 = Mock(spec=Command)
command2.get_results.return_value = CommandResult(
1, '', 'segmentState: Ready', False, True)
self.worker_pool.getCompletedItems.return_value = [command1, command2]
self.subject._check_segment_state_for_connection(conf_provider)
def test_check_segment_state__when_one_segment_not_ready__raises(self):
segment1 = Mock(spec=GpDB)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=GpDB)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
gparray_mock = Mock(spec=GpArray)
gparray_mock.getDbList.return_value = [segment1, segment2]
conf_provider = self._get_mock_conf_provider(gparray_mock)
command1 = Mock(spec=Command)
command1.get_results.return_value = CommandResult(
0, '', 'segmentState: Ready', False, True)
command2 = Mock(spec=Command)
command2.get_results.return_value = CommandResult(
1, '', 'Failed to connect', False, True)
self.worker_pool.getCompletedItems.return_value = [command1, command2]
with self.assertRaisesRegexp(Exception,
'Not ready to connect to database'):
self.subject._check_segment_state_for_connection(conf_provider)
@patch('gppylib.commands.base.Command.get_results')
def test_check_segment_state_ready_for_recovery_ignores_initial_stderr_warnings(
self, mock_results):
mock_results.return_value = CommandResult(
0, '',
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \n'
'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n'
'dataState: InChangeTracking\n', False, True)
segment_mock = Mock(spec=GpDB)
segment_mock.isSegmentQD.return_value = False
segment_mock.isSegmentModeInChangeLogging.return_value = True
segment_mock.getSegmentHostName.return_value = 'foo1'
segment_mock.getSegmentDataDirectory.return_value = 'bar'
segment_mock.getSegmentPort.return_value = 5555
segment_mock.getSegmentDbId.return_value = 2
segment_mock.getSegmentRole.return_value = 'p'
segment_mock.getSegmentMode.return_value = 'c'
segmentList = [segment_mock]
dbsMap = {2: segment_mock}
segmentStates = self.subject.check_segment_state_ready_for_recovery(
segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
self.subject.logger.info.assert_called_once_with(
'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of '
)
@patch('gppylib.commands.base.Command.get_results')
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking__sets_disabled_state(
self, mock_results):
mock_results.return_value = CommandResult(
0, '',
'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n'
'dataState: InChangeTracking\n', False, True)
segment = Mock(spec=GpDB)
segment.isSegmentQD.return_value = False
segment.isSegmentModeInChangeLogging.return_value = True
segment.getSegmentHostName.return_value = 'foo1'
segment.getSegmentDataDirectory.return_value = 'bar'
segment.getSegmentPort.return_value = 5555
segment.getSegmentDbId.return_value = 2
segment.getSegmentRole.return_value = 'p'
segment.getSegmentMode.return_value = 'c'
segmentList = [segment]
dbsMap = {2: segment}
segmentStates = self.subject.check_segment_state_ready_for_recovery(
segmentList, dbsMap)
self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(
self):
segs_in_change_tracking_disabled = {
2: 'ChangeTrackingDisabled',
4: 'ChangeTrackingDisabled'
}
self.subject._output_segments_in_change_tracking_disabled(
segs_in_change_tracking_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.'
)
def test_check_segment_change_tracking_disabled_state_return_true(self):
res = self.subject.check_segment_change_tracking_disabled_state(
gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED)
self.assertEquals(res, True)
def test_check_segment_change_tracking_disabled_state_return_false(self):
res = self.subject.check_segment_change_tracking_disabled_state(
gparray.SEGMENT_STATE_READY)
self.assertEquals(res, False)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(
self):
segs_with_persistent_mirroring_disabled = [0, 1]
self.subject._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.'
)
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(
self):
segs_with_persistent_mirroring_disabled = []
self.subject._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
assert not self.subject.logger.warn.called
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(
self):
self.execSqlResult.fetchall.return_value = [[3], [1]]
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentContentId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(
gparray_mock, segment_mock)
self.assertTrue(mismatched)
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_enabled(
self):
self.execSqlResult.fetchall.return_value = [[3]]
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentContentId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(
gparray_mock, segment_mock)
self.assertFalse(mismatched)
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(
self):
gparray_mock = Mock(spec=GpArray)
gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
segment_mock = Mock(spec=GpDB)
segment_mock.getSegmentDbId.return_value = 0
mismatched = self.subject.is_segment_mirror_state_mismatched(
gparray_mock, segment_mock)
self.assertFalse(mismatched)
def test__run__when_no_replication_is_setup__raises(self):
self.gparray.getSegDbList.return_value = []
self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE
with self.assertRaisesRegexp(Exception,
'replication is not configured'):
self.subject.run()
def test__run__when_fault_strategy_is_SAN__calls_SAN_failback(self):
with patch.object(GpRecoverSegmentProgram,
'SanFailback',
return_value=None) as san_failback:
self.gparray.getSegDbList.return_value = []
self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_SAN
self.subject.run()
san_failback.assert_called_once_with(self.gparray, self.gp_env)
############################################################
# Private
def _get_mock_segment(self, name, port, address, datadir):
segment = Mock(spec=GpDB)
segment.getSegmentHostName.return_value = name
segment.getSegmentAddress.return_value = address
segment.getSegmentPort.return_value = port
segment.getSegmentDataDirectory.return_value = datadir
return segment
def _get_mock_conf_provider(self, gparray_result=None):
conf_provider = Mock(spec=GpConfigurationProvider)
conf_provider.loadSystemConfig.return_value = gparray_result
return conf_provider
def _segments_mock(self):
segment1 = Mock(spec=GpDB)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=GpDB)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
return [segment1, segment2]
class GpRecoverSegmentProgramTestCase(GpTestCase):
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args(args=[])
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpMasterEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult),
patch('time.sleep'),
patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock),
# patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self):
segs_with_persistent_mirroring_disabled = [0, 1]
self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.')
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self):
segs_with_persistent_mirroring_disabled = []
self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled)
assert not self.subject.logger.warn.called
############################################################
# Private
def _get_mock_segment(self, name, port, address, datadir):
segment = Mock(spec=Segment)
segment.getSegmentHostName.return_value = name
segment.getSegmentAddress.return_value = address
segment.getSegmentPort.return_value = port
segment.getSegmentDataDirectory.return_value = datadir
return segment
def _get_mock_conf_provider(self, gparray_result=None):
conf_provider = Mock(spec=GpConfigurationProvider)
conf_provider.loadSystemConfig.return_value = gparray_result
return conf_provider
def _segments_mock(self):
segment1 = Mock(spec=Segment)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=Segment)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
return [segment1, segment2]
def test_check_database_connection_exceed_max_retries(self, mock1, mock2):
options = Mock()
confProvider = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
self.assertFalse(
gprecover_prog._check_database_connection(confProvider))
class GpRecoverSegmentProgramTestCase(GpTestCase):
def setUp(self):
raw_options = GpRecoverSegmentProgram.createParser()
(options, _) = raw_options.parse_args(args=[])
options.spareDataDirectoryFile = None
options.newRecoverHosts = None
self.subject = GpRecoverSegmentProgram(options)
self.execSqlResult = Mock(spec=['fetchall'])
self.gp_env = Mock()
GpMasterEnvironmentMock = Mock(return_value=self.gp_env)
self.gparray = Mock(spec=GpArray)
self.gparray.getDbList.return_value = self._segments_mock()
configProviderMock = Mock(spec=GpConfigurationProvider)
configProviderMock.initializeProvider.return_value = configProviderMock
configProviderMock.loadSystemConfig.return_value = self.gparray
self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider)
self.getConfigProviderFunctionMock.return_value = configProviderMock
self.subject.logger = Mock()
self.worker_pool = Mock(spec=WorkerPool, return_value=None)
self.worker_pool.getCompletedItems.return_value = []
self.worker_pool.logger = self.subject.logger
self.worker_pool.addCommand.return_value = None
self.pool_completed = []
self.apply_patches([
patch("gppylib.db.dbconn.connect"),
patch("gppylib.db.dbconn.DbURL"),
patch("gppylib.db.dbconn.execSQL",
return_value=self.execSqlResult),
patch('time.sleep'),
patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment',
GpMasterEnvironmentMock),
# patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env),
# patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'),
patch('gppylib.system.faultProberInterface.getFaultProber'),
patch(
'gppylib.system.configurationInterface.getConfigurationProvider',
self.getConfigProviderFunctionMock),
patch('gppylib.commands.base.WorkerPool.__init__',
self.worker_pool),
patch('gppylib.commands.base.WorkerPool.getCompletedItems',
return_value=self.pool_completed),
patch('gppylib.commands.base.WorkerPool.addCommand'),
patch('gppylib.commands.base.WorkerPool.join'),
])
# tests make use of a workaround to access a python attribute that is normally
# name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute>
# such as self.subject._GpRecoverSegmentProgram__pool = mock_pool
self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(
self):
segs_with_persistent_mirroring_disabled = [0, 1]
self.subject._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
self.subject.logger.warn.assert_called_once_with(
'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.'
)
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(
self):
segs_with_persistent_mirroring_disabled = []
self.subject._output_segments_with_persistent_mirroring_disabled(
segs_with_persistent_mirroring_disabled)
assert not self.subject.logger.warn.called
############################################################
# Private
def _get_mock_segment(self, name, port, address, datadir):
segment = Mock(spec=Segment)
segment.getSegmentHostName.return_value = name
segment.getSegmentAddress.return_value = address
segment.getSegmentPort.return_value = port
segment.getSegmentDataDirectory.return_value = datadir
return segment
def _get_mock_conf_provider(self, gparray_result=None):
conf_provider = Mock(spec=GpConfigurationProvider)
conf_provider.loadSystemConfig.return_value = gparray_result
return conf_provider
def _segments_mock(self):
segment1 = Mock(spec=Segment)
segment1.getSegmentHostName.return_value = 'foo1'
segment1.isSegmentUp.return_value = True
segment1.isSegmentMaster.return_value = False
segment1.isSegmentStandby.return_value = False
segment2 = Mock(spec=Segment)
segment2.getSegmentHostName.return_value = 'foo2'
segment2.isSegmentUp.return_value = True
segment2.isSegmentMaster.return_value = False
segment2.isSegmentStandby.return_value = False
return [segment1, segment2]
def test_check_database_connection(self, mock1, mock2):
options = Mock()
confProvider = Mock()
gprecover_prog = GpRecoverSegmentProgram(options)
self.assertTrue(
gprecover_prog._check_database_connection(confProvider))
