def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self): segs_with_persistent_mirroring_disabled = [] options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) assert not gprecover_prog.logger.warn.called
def test_check_segment_state_ready_for_recovery_ignores_initial_stdout_warnings( self, mock_results): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.info = Mock() mock_results.return_value = CommandResult( 0, '', 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \nmode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True) segment_mock = Mock() segment_mock.isSegmentQD.return_value = False segment_mock.isSegmentModeInChangeLogging.return_value = True segment_mock.getSegmentHostName.return_value = 'foo1' segment_mock.getSegmentDataDirectory.return_value = 'bar' segment_mock.getSegmentPort.return_value = 5555 segment_mock.getSegmentDbId.return_value = 2 segment_mock.getSegmentRole.return_value = 'p' segment_mock.getSegmentMode.return_value = 'c' segmentList = [segment_mock] dbsMap = {2: segment_mock} mock_pool = Mock() mock_pool.addCommand = Mock() mock_pool.join = Mock() gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool segmentStates = gprecover_prog.check_segment_state_ready_for_recovery( segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) gprecover_prog.logger.info.assert_called_once_with( 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ' )
def test_check_segment_state(self): options = Mock() mock_pool = Mock() m1 = Mock() m1.get_results.return_value = CommandResult(1, 'Failed to connect', '', False, True) m2 = Mock() m2.get_results.return_value = CommandResult(0, 'segmentState: Ready', '', False, True) mock_pool.getCompletedItems.return_value = [] confProvider = Mock() m1 = Mock() m1.getSegmentHostName.return_value = 'foo1' m1.isSegmentUp.return_value = True m1.isSegmentMaster.return_value = False m2 = Mock() m2.getSegmentHostName.return_value = 'foo2' m2.isSegmentUp.return_value = True m2.isSegmentMaster.return_value = False gparray = Mock() gparray.getDbList.return_value = [m1, m2] confProvider.loadSystemConfig.return_value = gparray gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_segment_state(confProvider)
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking_disabled( self, mock_results): options = Mock() mock_results.return_value = CommandResult( 0, '', 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True) m2 = Mock() m2.isSegmentQD.return_value = False m2.isSegmentModeInChangeLogging.return_value = True m2.getSegmentHostName.return_value = 'foo1' m2.getSegmentDataDirectory.return_value = 'bar' m2.getSegmentPort.return_value = 5555 m2.getSegmentDbId.return_value = 2 m2.getSegmentRole.return_value = 'p' m2.getSegmentMode.return_value = 'c' segmentList = [m2] dbsMap = {2: m2} gprecover_prog = GpRecoverSegmentProgram(options) mock_pool = Mock() mock_pool.addCommand = Mock() mock_pool.join = Mock() gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool segmentStates = gprecover_prog.check_segment_state_ready_for_recovery( segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_check_segment_state_ready_for_recovery_ignores_initial_stdout_warnings(self, mock_results): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.info = Mock() mock_results.return_value = CommandResult(0, '', 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \nmode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True) segment_mock = Mock() segment_mock.isSegmentQD.return_value = False segment_mock.isSegmentModeInChangeLogging.return_value = True segment_mock.getSegmentHostName.return_value = 'foo1' segment_mock.getSegmentDataDirectory.return_value = 'bar' segment_mock.getSegmentPort.return_value = 5555 segment_mock.getSegmentDbId.return_value = 2 segment_mock.getSegmentRole.return_value = 'p' segment_mock.getSegmentMode.return_value = 'c' segmentList = [segment_mock] dbsMap = {2:segment_mock} mock_pool= Mock() mock_pool.addCommand = Mock() mock_pool.join = Mock() gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) gprecover_prog.logger.info.assert_called_once_with('Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ')
def test_successful_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([ self.mirror0 ], []) self.return_one = True self.mock_build_mirrors.return_value = True with self.assertRaises(SystemExit) as cm: # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.assertEqual(cm.exception.code, 0)
def test_failed_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([ self.mirror0 ], []) self.return_one = True self.mock_build_mirrors.return_value = False with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 1)
def test_check_segment_state_with_segment_not_ready(self): options = Mock() m1 = Mock() m1.get_results.return_value = CommandResult(0, 'Failed to connect', '', False, True) m2 = Mock() m2.get_results.return_value = CommandResult(0, 'segmentState: Not Ready', '', False, True) mock_pool = Mock() mock_pool.getCompletedItems.return_value = [m1, m2] m1 = Mock() m1.getSegmentHostName.return_value = 'foo1' m1.isSegmentUp.return_value = True m1.isSegmentMaster.return_value = False m2 = Mock() m2.getSegmentHostName.return_value = 'foo2' m2.isSegmentUp.return_value = True m2.isSegmentMaster.return_value = False gparray = Mock() gparray.getDbList.return_value = [m1, m2] confProvider = Mock() confProvider.loadSystemConfig.return_value = gparray gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'): gprecover_prog._check_segment_state(confProvider)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self): segs_with_persistent_mirroring_disabled = [0, 1] options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) gprecover_prog.logger.warn.assert_called_once_with('Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.')
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(self): segs_in_change_tracking_disabled = {2:'ChangeTrackingDisabled', 4:'ChangeTrackingDisabled'} options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_in_change_tracking_disabled(segs_in_change_tracking_disabled) gprecover_prog.logger.warn.assert_called_once_with('Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.')
def test_check_persistent_tables_no_segments(self, mock1): options = Mock() segments = [] mock_pool = Mock() mock_pool.getCompletedItems.return_value = [] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_persistent_tables(segments)
def test_check_persistent_tables_no_segments(self, mock1): options = Mock() segments = [] mock_pool = Mock() mock_pool.getCompletedItems.return_value = [] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_persistent_tables(segments)
def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments( self): segs_with_persistent_mirroring_disabled = [] options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) assert not gprecover_prog.logger.warn.called
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(self, mock_sql): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gparray_mock = Mock() gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE segment_mock = Mock() segment_mock.getSegmentDbId.return_value = 0 result = gprecover_prog.is_segment_mirror_state_mismatched(gparray_mock, segment_mock) self.assertFalse(result)
def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args(args=[]) options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpCoordinatorEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch( 'gppylib.programs.clsRecoverSegment.GpCoordinatorEnvironment', GpCoordinatorEnvironmentMock), # patch('gppylib.system.environment.GpCoordinatorEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpCoordinatorEnvironment.getCoordinatorPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch( 'gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(self, mock_sql): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gparray_mock = Mock() gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock() segment_mock.getSegmentContentId.return_value = 0 result = gprecover_prog.is_segment_mirror_state_mismatched(gparray_mock, segment_mock) self.assertTrue(result)
def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled( self, mock_sql): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gparray_mock = Mock() gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock() segment_mock.getSegmentContentId.return_value = 0 result = gprecover_prog.is_segment_mirror_state_mismatched( gparray_mock, segment_mock) self.assertTrue(result)
def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments( self): segs_with_persistent_mirroring_disabled = [0, 1] options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) gprecover_prog.logger.warn.assert_called_once_with( 'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.' )
def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled( self, mock_sql): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gparray_mock = Mock() gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE segment_mock = Mock() segment_mock.getSegmentDbId.return_value = 0 result = gprecover_prog.is_segment_mirror_state_mismatched( gparray_mock, segment_mock) self.assertFalse(result)
def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args() options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpMasterEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock), # patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool
def test_check_persistent_tables(self, mock1): options = Mock() segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')] mock_pool = Mock() m1 = Mock() m1.get_results.return_value = '' m2 = Mock() m2.get_results.return_value = '' mock_pool.getCompletedItems.return_value = [m1, m2] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_persistent_tables(segments)
def test_check_persistent_tables_error(self, mock1): options = Mock() segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')] mock_pool = Mock() m1 = Mock() m1.get_results.return_value = ['sdfsdf'] m2 = Mock() m2.get_results.return_value = ['asdfas'] mock_pool.getCompletedItems.return_value = [m1, m2] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'): gprecover_prog._check_persistent_tables(segments)
def test_output_segments_in_change_tracking_disabled_should_print_failed_segments( self): segs_in_change_tracking_disabled = { 2: 'ChangeTrackingDisabled', 4: 'ChangeTrackingDisabled' } options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog.logger.warn = Mock() gprecover_prog._output_segments_in_change_tracking_disabled( segs_in_change_tracking_disabled) gprecover_prog.logger.warn.assert_called_once_with( 'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.' )
def test_check_persistent_tables(self, mock1): options = Mock() segments = [ self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2') ] mock_pool = Mock() m1 = Mock() m1.get_results.return_value = '' m2 = Mock() m2.get_results.return_value = '' mock_pool.getCompletedItems.return_value = [m1, m2] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_persistent_tables(segments)
def test_successful_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], []) self.return_one = True self.mock_build_mirrors.return_value = True with self.assertRaises(SystemExit) as cm: # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.assertEqual(cm.exception.code, 0)
def test_successful_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit): self.subject.run() self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.')
def test_check_persistent_tables_error(self, mock1): options = Mock() segments = [ self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2') ] mock_pool = Mock() m1 = Mock() m1.get_results.return_value = ['sdfsdf'] m2 = Mock() m2.get_results.return_value = ['asdfas'] mock_pool.getCompletedItems.return_value = [m1, m2] gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'): gprecover_prog._check_persistent_tables(segments)
def test_failed_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 0) self.subject.logger.info.assert_any_call('The rebalance operation has completed with WARNINGS. ' 'Please review the output in the gprecoverseg log.')
def test_check_segment_state(self): options = Mock() mock_pool = Mock() m1 = Mock() m1.get_results.return_value = CommandResult(1, 'Failed to connect', '', False, True) m2 = Mock() m2.get_results.return_value = CommandResult(0, 'segmentState: Ready', '', False, True) mock_pool.getCompletedItems.return_value = [] confProvider = Mock() m1 = Mock() m1.getSegmentHostName.return_value = 'foo1' m1.isSegmentUp.return_value = True m1.isSegmentMaster.return_value = False m2 = Mock() m2.getSegmentHostName.return_value = 'foo2' m2.isSegmentUp.return_value = True m2.isSegmentMaster.return_value = False gparray = Mock() gparray.getDbList.return_value = [m1, m2] confProvider.loadSystemConfig.return_value = gparray gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool gprecover_prog._check_segment_state_for_connection(confProvider)
def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking_disabled(self, mock_results): options = Mock() mock_results.return_value = CommandResult(0, '', 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\ndataState: InChangeTracking\n', False, True) m2 = Mock() m2.isSegmentQD.return_value = False m2.isSegmentModeInChangeLogging.return_value = True m2.getSegmentHostName.return_value = 'foo1' m2.getSegmentDataDirectory.return_value = 'bar' m2.getSegmentPort.return_value = 5555 m2.getSegmentDbId.return_value = 2 m2.getSegmentRole.return_value = 'p' m2.getSegmentMode.return_value = 'c' segmentList = [m2] dbsMap = {2:m2} gprecover_prog = GpRecoverSegmentProgram(options) mock_pool= Mock() mock_pool.addCommand = Mock() mock_pool.join = Mock() gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool segmentStates = gprecover_prog.check_segment_state_ready_for_recovery(segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'})
def test_check_segment_state_with_segment_not_ready(self): options = Mock() m1 = Mock() m1.get_results.return_value = CommandResult(0, 'Failed to connect', '', False, True) m2 = Mock() m2.get_results.return_value = CommandResult(0, 'segmentState: Not Ready', '', False, True) mock_pool = Mock() mock_pool.getCompletedItems.return_value = [m1, m2] m1 = Mock() m1.getSegmentHostName.return_value = 'foo1' m1.isSegmentUp.return_value = True m1.isSegmentMaster.return_value = False m2 = Mock() m2.getSegmentHostName.return_value = 'foo2' m2.isSegmentUp.return_value = True m2.isSegmentMaster.return_value = False gparray = Mock() gparray.getDbList.return_value = [m1, m2] confProvider = Mock() confProvider.loadSystemConfig.return_value = gparray gprecover_prog = GpRecoverSegmentProgram(options) gprecover_prog._GpRecoverSegmentProgram__pool = mock_pool with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'): gprecover_prog._check_segment_state_for_connection(confProvider)
def test_successful_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit): self.subject.run() self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.')
def test_failed_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], []) self.return_one = True self.mock_build_mirrors.return_value = False with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 1)
def test_check_segment_change_tracking_disabled_state_return_false(self): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) res = gprecover_prog.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_READY) self.assertEquals(res, False)
def test_check_segment_change_tracking_disabled_state_return_true(self): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) res = gprecover_prog.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED) self.assertEquals(res, True)
def setUp(self): self.temp_dir = tempfile.mkdtemp() self.config_file_path = os.path.join(self.temp_dir, "foo") with open(self.config_file_path, "w") as config_file: config_file.write("") self.conn = Mock() self.conn.__enter__ = Mock(return_value=(Mock(), None)) self.conn.__exit__ = Mock(return_value=None) self.cursor = FakeCursor() self.db_singleton = Mock() self.os_env = dict(USER="******") self.os_env["MASTER_DATA_DIRECTORY"] = self.temp_dir self.os_env["GPHOME"] = self.temp_dir self.gparray = self._create_gparray_with_2_primary_2_mirrors() self.pool = Mock() self.pool.getCompletedItems.return_value = [] self.pgconf_dict = gucdict() self.pgconf_dict["port"] = setting("port", "123", None, None, None) self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None) self.config_provider_mock = MagicMock(spec=GpConfigurationProvider) self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock self.gpArrayMock = MagicMock(spec=GpArray) self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]] self.gpArrayMock.hasMirrors = True self.gpArrayMock.isStandardArray.return_value = (True, None) self.gpArrayMock.master = self.gparray.master self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False) self.apply_patches([ patch('os.environ', new=self.os_env), patch('gppylib.db.dbconn.connect', return_value=self.conn), patch('gppylib.db.dbconn.execSQL', return_value=self.cursor), patch('gppylib.db.dbconn.execSQLForSingletonRow', return_value=["foo"]), patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict), patch('gppylib.commands.gp.GpVersion'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch('gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock), patch('gppylib.commands.base.WorkerPool', return_value=self.pool), patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}), patch('gppylib.gplog.get_default_logger'), patch.object(GpMirrorListToBuild, "__init__", return_value=None), patch.object(GpMirrorListToBuild, "buildMirrors"), patch.object(GpMirrorListToBuild, "getAdditionalWarnings"), patch.object(GpMirrorListToBuild, "getMirrorsToBuild"), patch.object(HeapChecksum, "check_segment_consistency"), patch.object(HeapChecksum, "get_segments_checksum_settings"), ]) self.call_count = 0 self.return_one = True self.mock_build_mirrors = self.get_mock_from_apply_patch("buildMirrors") self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch('getMirrorsToBuild') self.mock_heap_checksum_init = self.get_mock_from_apply_patch("__init__") self.mock_check_segment_consistency = self.get_mock_from_apply_patch('check_segment_consistency') self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch('get_segments_checksum_settings') sys.argv = ["gprecoverseg"] # reset to relatively empty args list options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = self.config_file_path options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) faultProberInterface.gFaultProber = Mock()
def rebalance(self): self.logger.info("Determining primary and mirror segment pairs to rebalance") # The current implementation of rebalance calls "gprecoverseg -a" below. # Thus, if another balanced pair is not synchronized, or has a down mirror # that pair will be recovered as a side-effect of rebalancing. unbalanced_primary_segs = [] for segmentPair in self.gpArray.segmentPairs: if segmentPair.balanced(): continue if segmentPair.up() and segmentPair.reachable() and segmentPair.synchronized(): unbalanced_primary_segs.append(segmentPair.primaryDB) else: self.logger.warning( "Not rebalancing primary segment dbid %d with its mirror dbid %d because one is either down, unreachable, or not synchronized" \ % (segmentPair.primaryDB.dbid, segmentPair.mirrorDB.dbid)) if not len(unbalanced_primary_segs): self.logger.info("No segments to rebalance") return True unbalanced_primary_segs = GpArray.getSegmentsByHostName(unbalanced_primary_segs) pool = base.WorkerPool(min(len(unbalanced_primary_segs), self.batch_size)) try: # Disable ctrl-c signal.signal(signal.SIGINT, signal.SIG_IGN) self.logger.info("Stopping unbalanced primary segments...") for hostname in list(unbalanced_primary_segs.keys()): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=base.REMOTE, remoteHost=hostname, timeout=600, segment_batch_size=self.segment_batch_size) pool.addCommand(cmd) base.join_and_indicate_progress(pool) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count += 1 allSegmentsStopped = (failed_count == 0) if not allSegmentsStopped: self.logger.warn("%d segments failed to stop. A full rebalance of the" % failed_count) self.logger.warn("system is not possible at this time. Please check the") self.logger.warn("log files, correct the problem, and run gprecoverseg -r") self.logger.warn("again.") self.logger.info("gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() segment_reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=pool, timeout=MIRROR_PROMOTION_TIMEOUT) segment_reconfigurer.reconfigure() # Final step is to issue a recoverseg operation to resync segments self.logger.info("Starting segment synchronization") original_sys_args = sys.argv[:] self.logger.info("=============================START ANOTHER RECOVER=========================================") # import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram cmd_args = ['gprecoverseg', '-a', '-B', str(self.batch_size), '-b', str(self.segment_batch_size)] sys.argv = cmd_args[:] local_parser = GpRecoverSegmentProgram.createParser() local_options, args = local_parser.parse_args() recover_cmd = GpRecoverSegmentProgram.createProgram(local_options, args) try: recover_cmd.run() except SystemExit as e: if e.code != 0: self.logger.error("Failed to start the synchronization step of the segment rebalance.") self.logger.error("Check the gprecoverseg log file, correct any problems, and re-run") self.logger.error(' '.join(cmd_args)) raise Exception("Error synchronizing.\nError: %s" % str(e)) finally: if recover_cmd: recover_cmd.cleanup() sys.argv = original_sys_args self.logger.info("==============================END ANOTHER RECOVER==========================================") except Exception as ex: raise ex finally: pool.join() pool.haltWork() pool.joinWorkers() signal.signal(signal.SIGINT, signal.default_int_handler) return allSegmentsStopped # if all segments stopped, then a full rebalance was done
def test_check_segment_change_tracking_disabled_state_return_true(self): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) res = gprecover_prog.check_segment_change_tracking_disabled_state( gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED) self.assertEquals(res, True)
class GpRecoverSegmentProgramTestCase(GpTestCase): def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args() options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpMasterEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock), # patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool def test_check_persistent_tables__when_no_errors_detected__succeeds(self): segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')] command1 = Mock(spec=Command) command1.get_results.return_value = '' command2 = Mock(spec=Command) command2.get_results.return_value = '' self.execSqlResult.fetchall.return_value = [['template1']] self.worker_pool.getCompletedItems.return_value = [command1, command2] self.subject._check_persistent_tables(segments) def test_check_persistent_tables__with_no_segments__succeeds(self): self.execSqlResult.fetchall.return_value = [['template1']] self.subject._check_persistent_tables([]) def test_check_persistent_tables__when_error_exists__raises(self): self.execSqlResult.fetchall.return_value = [['template1']] segments = [self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2')] command1 = Mock() command1.get_results.return_value = ['sdfsdf'] command2 = Mock() command2.get_results.return_value = ['asdfas'] self.worker_pool.getCompletedItems.return_value = [command1, command2] with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'): self.subject._check_persistent_tables(segments) def test_check_database_connection__when_all_segments_are_ready_to_connect__returns_true(self): self.gparray.getDbList.return_value = [] conf_provider = self._get_mock_conf_provider(self.gparray) self.assertTrue(self.subject._check_database_connection(conf_provider)) def test_check_database_connection__when_raises_beyond_max_retries__returns_false(self): conf_provider_that_raises = Mock(spec=GpConfigurationProvider) self.assertFalse(self.subject._check_database_connection(conf_provider_that_raises)) def test_check_segment_state__when_all_segments_ready__succeeds(self): conf_provider = self._get_mock_conf_provider(self.gparray) command1 = Mock(spec=Command) command1.get_results.return_value = CommandResult(0, '', 'segmentState: Ready', False, True) command2 = Mock(spec=Command) command2.get_results.return_value = CommandResult(1, '', 'segmentState: Ready', False, True) self.worker_pool.getCompletedItems.return_value = [command1, command2] self.subject._check_segment_state_for_connection(conf_provider) def test_check_segment_state__when_one_segment_not_ready__raises(self): segment1 = Mock(spec=GpDB) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=GpDB) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False gparray_mock = Mock(spec=GpArray) gparray_mock.getDbList.return_value = [segment1, segment2] conf_provider = self._get_mock_conf_provider(gparray_mock) command1 = Mock(spec=Command) command1.get_results.return_value = CommandResult(0, '', 'segmentState: Ready', False, True) command2 = Mock(spec=Command) command2.get_results.return_value = CommandResult(1, '', 'Failed to connect', False, True) self.worker_pool.getCompletedItems.return_value = [command1, command2] with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'): self.subject._check_segment_state_for_connection(conf_provider) @patch('gppylib.commands.base.Command.get_results') def test_check_segment_state_ready_for_recovery_ignores_initial_stderr_warnings(self, mock_results): mock_results.return_value = CommandResult(0, '', 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \n' 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n' 'dataState: InChangeTracking\n', False, True) segment_mock = Mock(spec=GpDB) segment_mock.isSegmentQD.return_value = False segment_mock.isSegmentModeInChangeLogging.return_value = True segment_mock.getSegmentHostName.return_value = 'foo1' segment_mock.getSegmentDataDirectory.return_value = 'bar' segment_mock.getSegmentPort.return_value = 5555 segment_mock.getSegmentDbId.return_value = 2 segment_mock.getSegmentRole.return_value = 'p' segment_mock.getSegmentMode.return_value = 'c' segmentList = [segment_mock] dbsMap = {2: segment_mock} segmentStates = self.subject.check_segment_state_ready_for_recovery(segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) self.subject.logger.info.assert_called_once_with( 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ') @patch('gppylib.commands.base.Command.get_results') def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking__sets_disabled_state(self, mock_results): mock_results.return_value = CommandResult(0, '', 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n' 'dataState: InChangeTracking\n', False, True) segment = Mock(spec=GpDB) segment.isSegmentQD.return_value = False segment.isSegmentModeInChangeLogging.return_value = True segment.getSegmentHostName.return_value = 'foo1' segment.getSegmentDataDirectory.return_value = 'bar' segment.getSegmentPort.return_value = 5555 segment.getSegmentDbId.return_value = 2 segment.getSegmentRole.return_value = 'p' segment.getSegmentMode.return_value = 'c' segmentList = [segment] dbsMap = {2: segment} segmentStates = self.subject.check_segment_state_ready_for_recovery(segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) def test_output_segments_in_change_tracking_disabled_should_print_failed_segments(self): segs_in_change_tracking_disabled = {2: 'ChangeTrackingDisabled', 4: 'ChangeTrackingDisabled'} self.subject._output_segments_in_change_tracking_disabled(segs_in_change_tracking_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.') def test_check_segment_change_tracking_disabled_state_return_true(self): res = self.subject.check_segment_change_tracking_disabled_state( gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED) self.assertEquals(res, True) def test_check_segment_change_tracking_disabled_state_return_false(self): res = self.subject.check_segment_change_tracking_disabled_state(gparray.SEGMENT_STATE_READY) self.assertEquals(res, False) def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self): segs_with_persistent_mirroring_disabled = [0, 1] self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.') def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self): segs_with_persistent_mirroring_disabled = [] self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) assert not self.subject.logger.warn.called def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled(self): self.execSqlResult.fetchall.return_value = [[3], [1]] gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock(spec=GpDB) segment_mock.getSegmentContentId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock) self.assertTrue(mismatched) def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_enabled(self): self.execSqlResult.fetchall.return_value = [[3]] gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock(spec=GpDB) segment_mock.getSegmentContentId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock) self.assertFalse(mismatched) def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled(self): gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE segment_mock = Mock(spec=GpDB) segment_mock.getSegmentDbId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched(gparray_mock, segment_mock) self.assertFalse(mismatched) def test__run__when_no_replication_is_setup__raises(self): self.gparray.getSegDbList.return_value = [] self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE with self.assertRaisesRegexp(Exception, 'replication is not configured'): self.subject.run() def test__run__when_fault_strategy_is_SAN__calls_SAN_failback(self): with patch.object(GpRecoverSegmentProgram, 'SanFailback', return_value=None) as san_failback: self.gparray.getSegDbList.return_value = [] self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_SAN self.subject.run() san_failback.assert_called_once_with(self.gparray, self.gp_env) ############################################################ # Private def _get_mock_segment(self, name, port, address, datadir): segment = Mock(spec=GpDB) segment.getSegmentHostName.return_value = name segment.getSegmentAddress.return_value = address segment.getSegmentPort.return_value = port segment.getSegmentDataDirectory.return_value = datadir return segment def _get_mock_conf_provider(self, gparray_result=None): conf_provider = Mock(spec=GpConfigurationProvider) conf_provider.loadSystemConfig.return_value = gparray_result return conf_provider def _segments_mock(self): segment1 = Mock(spec=GpDB) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=GpDB) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False return [segment1, segment2]
def rebalance(self): # Get the unbalanced primary segments grouped by hostname # These segments are what we will shutdown. self.logger.info("Getting unbalanced segments") unbalanced_primary_segs = GpArray.getSegmentsByHostName(self.gpArray.get_unbalanced_primary_segdbs()) pool = base.WorkerPool() count = 0 try: # Disable ctrl-c signal.signal(signal.SIGINT, signal.SIG_IGN) self.logger.info("Stopping unbalanced primary segments...") for hostname in unbalanced_primary_segs.keys(): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=base.REMOTE, remoteHost=hostname, timeout=600) pool.addCommand(cmd) count += 1 pool.wait_and_printdots(count, False) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count += 1 allSegmentsStopped = (failed_count == 0) if not allSegmentsStopped: self.logger.warn("%d segments failed to stop. A full rebalance of the") self.logger.warn("system is not possible at this time. Please check the") self.logger.warn("log files, correct the problem, and run gprecoverseg -r") self.logger.warn("again.") self.logger.info("gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() # issue a distributed query to make sure we pick up the fault # that we just caused by shutting down segments conn = None try: self.logger.info("Triggering segment reconfiguration") dburl = dbconn.DbURL() conn = dbconn.connect(dburl) cmd = ReconfigDetectionSQLQueryCommand(conn) pool.addCommand(cmd) pool.wait_and_printdots(1, False) except Exception: # This exception is expected pass finally: if conn: conn.close() # Final step is to issue a recoverseg operation to resync segments self.logger.info("Starting segment synchronization") original_sys_args = sys.argv[:] try: self.logger.info("=============================START ANOTHER RECOVER=========================================") # import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram sys.argv = ['gprecoverseg', '-a'] local_parser = GpRecoverSegmentProgram.createParser() local_options, args = local_parser.parse_args() cmd = GpRecoverSegmentProgram.createProgram(local_options, args) cmd.run() except SystemExit as e: if e.code != 0: self.logger.error("Failed to start the synchronization step of the segment rebalance.") self.logger.error("Check the gprecoverseg log file, correct any problems, and re-run") self.logger.error("'gprecoverseg -a'.") raise Exception("Error synchronizing.\nError: %s" % str(e)) finally: if cmd: cmd.cleanup() sys.argv = original_sys_args self.logger.info("==============================END ANOTHER RECOVER==========================================") except Exception, ex: raise ex
def rebalance(self): # Get the unbalanced primary segments grouped by hostname # These segments are what we will shutdown. self.logger.info("Getting unbalanced segments") unbalanced_primary_segs = GpArray.getSegmentsByHostName( self.gpArray.get_unbalanced_primary_segdbs()) pool = base.WorkerPool() try: # Disable ctrl-c signal.signal(signal.SIGINT, signal.SIG_IGN) self.logger.info("Stopping unbalanced primary segments...") for hostname in unbalanced_primary_segs.keys(): cmd = GpSegStopCmd("stop unbalanced primary segs", self.gpEnv.getGpHome(), self.gpEnv.getGpVersion(), 'fast', unbalanced_primary_segs[hostname], ctxt=base.REMOTE, remoteHost=hostname, timeout=600) pool.addCommand(cmd) base.join_and_indicate_progress(pool) failed_count = 0 completed = pool.getCompletedItems() for res in completed: if not res.get_results().wasSuccessful(): failed_count += 1 allSegmentsStopped = (failed_count == 0) if not allSegmentsStopped: self.logger.warn( "%d segments failed to stop. A full rebalance of the") self.logger.warn( "system is not possible at this time. Please check the") self.logger.warn( "log files, correct the problem, and run gprecoverseg -r") self.logger.warn("again.") self.logger.info( "gprecoverseg will continue with a partial rebalance.") pool.empty_completed_items() segment_reconfigurer = SegmentReconfigurer( logger=self.logger, worker_pool=pool, timeout=MIRROR_PROMOTION_TIMEOUT) segment_reconfigurer.reconfigure() # Final step is to issue a recoverseg operation to resync segments self.logger.info("Starting segment synchronization") original_sys_args = sys.argv[:] try: self.logger.info( "=============================START ANOTHER RECOVER=========================================" ) # import here because GpRecoverSegmentProgram and GpSegmentRebalanceOperation have a circular dependency from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram sys.argv = ['gprecoverseg', '-a'] local_parser = GpRecoverSegmentProgram.createParser() local_options, args = local_parser.parse_args() cmd = GpRecoverSegmentProgram.createProgram( local_options, args) cmd.run() except SystemExit as e: if e.code != 0: self.logger.error( "Failed to start the synchronization step of the segment rebalance." ) self.logger.error( "Check the gprecoverseg log file, correct any problems, and re-run" ) self.logger.error("'gprecoverseg -a'.") raise Exception("Error synchronizing.\nError: %s" % str(e)) finally: if cmd: cmd.cleanup() sys.argv = original_sys_args self.logger.info( "==============================END ANOTHER RECOVER==========================================" ) except Exception, ex: raise ex
def setUp(self): self.temp_dir = tempfile.mkdtemp() self.config_file_path = os.path.join(self.temp_dir, "foo") with open(self.config_file_path, "w") as config_file: config_file.write("") self.conn = Mock() self.conn.__enter__ = Mock(return_value=(Mock(), None)) self.conn.__exit__ = Mock(return_value=None) self.cursor = FakeCursor() self.db_singleton = Mock() self.os_env = dict(USER="******") self.os_env["COORDINATOR_DATA_DIRECTORY"] = self.temp_dir self.os_env["GPHOME"] = self.temp_dir self.gparray = self._create_gparray_with_2_primary_2_mirrors() self.pool = Mock() self.pool.getCompletedItems.return_value = [] self.pgconf_dict = gucdict() self.pgconf_dict["port"] = setting("port", "123", None, None, None) self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None) self.config_provider_mock = MagicMock(spec=GpConfigurationProvider) self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock self.gpArrayMock = MagicMock(spec=GpArray) self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]] self.gpArrayMock.segmentPairs = [] self.gpArrayMock.hasMirrors = True self.gpArrayMock.isStandardArray.return_value = (True, None) self.gpArrayMock.coordinator = self.gparray.coordinator self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False) self.apply_patches([ patch('os.environ', new=self.os_env), patch('gppylib.db.dbconn.connect', return_value=self.conn), patch('gppylib.db.dbconn.query', return_value=self.cursor), patch('gppylib.db.dbconn.queryRow', return_value=["foo"]), patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict), patch('gppylib.commands.gp.GpVersion'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch( 'gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock), patch('gppylib.commands.base.WorkerPool', return_value=self.pool), patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}), patch('gppylib.gplog.get_default_logger'), patch.object(GpMirrorListToBuild, "__init__", return_value=None), patch.object(GpMirrorListToBuild, "buildMirrors"), patch.object(GpMirrorListToBuild, "getAdditionalWarnings"), patch.object(GpMirrorListToBuild, "getMirrorsToBuild"), patch.object(HeapChecksum, "check_segment_consistency"), patch.object(HeapChecksum, "get_segments_checksum_settings"), ]) self.call_count = 0 self.return_one = True self.mock_build_mirrors = self.get_mock_from_apply_patch( "buildMirrors") self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch( 'getMirrorsToBuild') self.mock_heap_checksum_init = self.get_mock_from_apply_patch( "__init__") self.mock_check_segment_consistency = self.get_mock_from_apply_patch( 'check_segment_consistency') self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch( 'get_segments_checksum_settings') sys.argv = ["gprecoverseg"] # reset to relatively empty args list options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = self.config_file_path options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) faultProberInterface.gFaultProber = Mock()
class GpRecoversegTestCase(GpTestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() self.config_file_path = os.path.join(self.temp_dir, "foo") with open(self.config_file_path, "w") as config_file: config_file.write("") self.conn = Mock() self.conn.__enter__ = Mock(return_value=(Mock(), None)) self.conn.__exit__ = Mock(return_value=None) self.cursor = FakeCursor() self.db_singleton = Mock() self.os_env = dict(USER="******") self.os_env["COORDINATOR_DATA_DIRECTORY"] = self.temp_dir self.os_env["GPHOME"] = self.temp_dir self.gparray = self._create_gparray_with_2_primary_2_mirrors() self.pool = Mock() self.pool.getCompletedItems.return_value = [] self.pgconf_dict = gucdict() self.pgconf_dict["port"] = setting("port", "123", None, None, None) self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None) self.config_provider_mock = MagicMock(spec=GpConfigurationProvider) self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock self.gpArrayMock = MagicMock(spec=GpArray) self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]] self.gpArrayMock.segmentPairs = [] self.gpArrayMock.hasMirrors = True self.gpArrayMock.isStandardArray.return_value = (True, None) self.gpArrayMock.coordinator = self.gparray.coordinator self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False) self.apply_patches([ patch('os.environ', new=self.os_env), patch('gppylib.db.dbconn.connect', return_value=self.conn), patch('gppylib.db.dbconn.query', return_value=self.cursor), patch('gppylib.db.dbconn.queryRow', return_value=["foo"]), patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict), patch('gppylib.commands.gp.GpVersion'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch( 'gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock), patch('gppylib.commands.base.WorkerPool', return_value=self.pool), patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}), patch('gppylib.gplog.get_default_logger'), patch.object(GpMirrorListToBuild, "__init__", return_value=None), patch.object(GpMirrorListToBuild, "buildMirrors"), patch.object(GpMirrorListToBuild, "getAdditionalWarnings"), patch.object(GpMirrorListToBuild, "getMirrorsToBuild"), patch.object(HeapChecksum, "check_segment_consistency"), patch.object(HeapChecksum, "get_segments_checksum_settings"), ]) self.call_count = 0 self.return_one = True self.mock_build_mirrors = self.get_mock_from_apply_patch( "buildMirrors") self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch( 'getMirrorsToBuild') self.mock_heap_checksum_init = self.get_mock_from_apply_patch( "__init__") self.mock_check_segment_consistency = self.get_mock_from_apply_patch( 'check_segment_consistency') self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch( 'get_segments_checksum_settings') sys.argv = ["gprecoverseg"] # reset to relatively empty args list options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = self.config_file_path options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) faultProberInterface.gFaultProber = Mock() def _get_test_mirrors(self): if self.return_one: return [self.mirror_to_build] if self.call_count == 0: self.call_count += 1 return [self.mirror_to_build] else: self.call_count += 1 return [] def tearDown(self): shutil.rmtree(self.temp_dir) super(GpRecoversegTestCase, self).tearDown() def test_when_checksums_mismatch_it_raises(self): self.primary0.heap_checksum = 0 self.mock_check_segment_consistency.return_value = ([], [self.primary0], 1) self.mock_get_segments_checksum_settings.return_value = ([ self.mirror0 ], []) self.return_one = True self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.assertTrue(self.gparray.coordinator.isSegmentCoordinator(True)) with self.assertRaisesRegex( Exception, "Heap checksum setting differences reported on segments"): self.subject.run() self.mock_get_segments_checksum_settings.assert_called_with( [self.primary0]) self.subject.logger.fatal.assert_any_call( 'Heap checksum setting differences reported on segments') self.subject.logger.fatal.assert_any_call( 'Failed checksum consistency validation:') self.subject.logger.fatal.assert_any_call( 'sdw1 checksum set to 0 differs from coordinator checksum set to 1' ) @patch.object(HeapChecksum, "__init__", return_value=None) def test_when_cannot_determine_checksums_it_raises( self, mock_heap_checksum_init): self.mock_check_segment_consistency.return_value = ([], [1], True) self.mock_get_segments_checksum_settings.return_value = ([], []) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.return_one = True self.assertTrue(self.gparray.coordinator.isSegmentCoordinator(True)) with self.assertRaisesRegex( Exception, "No segments responded to ssh query for heap checksum validation." ): self.subject.run() self.mock_get_segments_checksum_settings.assert_called_with( [self.primary0]) mock_heap_checksum_init.assert_called_with(self.gpArrayMock, logger=self.subject.logger, num_workers=1) @patch("os._exit") def test_when_no_segments_to_recover_validation_succeeds(self, _): self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.return_one = False with self.assertRaises(SystemExit): # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.subject.logger.info.assert_any_call( 'No checksum validation necessary when ' 'there are no segments to recover.') @patch.object(TableLogger, "info") @patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=True) @patch("os._exit") def test_successful_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit): self.subject.run() self.subject.logger.info.assert_any_call( 'The rebalance operation has completed successfully.') @patch.object(TableLogger, "info") @patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=False) @patch("os._exit") def test_failed_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 0) self.subject.logger.info.assert_any_call( 'The rebalance operation has completed with WARNINGS. ' 'Please review the output in the gprecoverseg log.') @patch.object(TableLogger, "info") def test_failed_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([ self.mirror0 ], []) self.return_one = True self.mock_build_mirrors.return_value = False with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 1) @patch.object(TableLogger, "info") def test_successful_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.coordinatorDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock( spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([ self.mirror0 ], []) self.return_one = True self.mock_build_mirrors.return_value = True with self.assertRaises(SystemExit) as cm: # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.assertEqual(cm.exception.code, 0) def _create_gparray_with_2_primary_2_mirrors(self): coordinator = Segment.initFromString( "1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator") self.primary0 = Segment.initFromString( "2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0") primary1 = Segment.initFromString( "3|1|p|p|s|u|sdw2|sdw2|40001|/data/primary1") self.mirror0 = Segment.initFromString( "4|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0") mirror1 = Segment.initFromString( "5|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1") return GpArray( [coordinator, self.primary0, primary1, self.mirror0, mirror1])
class GpRecoversegTestCase(GpTestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() self.config_file_path = os.path.join(self.temp_dir, "foo") with open(self.config_file_path, "w") as config_file: config_file.write("") self.conn = Mock() self.conn.__enter__ = Mock(return_value=(Mock(), None)) self.conn.__exit__ = Mock(return_value=None) self.cursor = FakeCursor() self.db_singleton = Mock() self.os_env = dict(USER="******") self.os_env["MASTER_DATA_DIRECTORY"] = self.temp_dir self.os_env["GPHOME"] = self.temp_dir self.gparray = self._create_gparray_with_2_primary_2_mirrors() self.pool = Mock() self.pool.getCompletedItems.return_value = [] self.pgconf_dict = gucdict() self.pgconf_dict["port"] = setting("port", "123", None, None, None) self.pgconf_dict["max_connection"] = setting("max_connections", "1", None, None, None) self.config_provider_mock = MagicMock(spec=GpConfigurationProvider) self.config_provider_mock.initializeProvider.return_value = self.config_provider_mock self.gpArrayMock = MagicMock(spec=GpArray) self.gpArrayMock.getDbList.side_effect = [[self.primary0], [self.primary0], [self.primary0]] self.gpArrayMock.hasMirrors = True self.gpArrayMock.isStandardArray.return_value = (True, None) self.gpArrayMock.master = self.gparray.master self.config_provider_mock.loadSystemConfig.return_value = self.gpArrayMock self.mirror_to_build = GpMirrorToBuild(self.mirror0, self.primary0, None, False) self.apply_patches([ patch('os.environ', new=self.os_env), patch('gppylib.db.dbconn.connect', return_value=self.conn), patch('gppylib.db.dbconn.execSQL', return_value=self.cursor), patch('gppylib.db.dbconn.execSQLForSingletonRow', return_value=["foo"]), patch('gppylib.pgconf.readfile', return_value=self.pgconf_dict), patch('gppylib.commands.gp.GpVersion'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch('gppylib.system.configurationInterface.getConfigurationProvider', return_value=self.config_provider_mock), patch('gppylib.commands.base.WorkerPool', return_value=self.pool), patch('gppylib.gparray.GpArray.getSegmentsByHostName', return_value={}), patch('gppylib.gplog.get_default_logger'), patch.object(GpMirrorListToBuild, "__init__", return_value=None), patch.object(GpMirrorListToBuild, "buildMirrors"), patch.object(GpMirrorListToBuild, "getAdditionalWarnings"), patch.object(GpMirrorListToBuild, "getMirrorsToBuild"), patch.object(HeapChecksum, "check_segment_consistency"), patch.object(HeapChecksum, "get_segments_checksum_settings"), ]) self.call_count = 0 self.return_one = True self.mock_build_mirrors = self.get_mock_from_apply_patch("buildMirrors") self.mock_get_mirrors_to_build = self.get_mock_from_apply_patch('getMirrorsToBuild') self.mock_heap_checksum_init = self.get_mock_from_apply_patch("__init__") self.mock_check_segment_consistency = self.get_mock_from_apply_patch('check_segment_consistency') self.mock_get_segments_checksum_settings = self.get_mock_from_apply_patch('get_segments_checksum_settings') sys.argv = ["gprecoverseg"] # reset to relatively empty args list options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = self.config_file_path options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) faultProberInterface.gFaultProber = Mock() def _get_test_mirrors(self): if self.return_one: return [self.mirror_to_build] if self.call_count == 0: self.call_count += 1 return [self.mirror_to_build] else: self.call_count += 1 return [] def tearDown(self): shutil.rmtree(self.temp_dir) super(GpRecoversegTestCase, self).tearDown() def test_when_checksums_mismatch_it_raises(self): self.primary0.heap_checksum = 0 self.mock_check_segment_consistency.return_value = ([], [self.primary0], 1) self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], []) self.return_one = True self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.assertTrue(self.gparray.master.isSegmentMaster(True)) with self.assertRaisesRegexp(Exception, "Heap checksum setting differences reported on segments"): self.subject.run() self.mock_get_segments_checksum_settings.assert_called_with([self.primary0]) self.subject.logger.fatal.assert_any_call('Heap checksum setting differences reported on segments') self.subject.logger.fatal.assert_any_call('Failed checksum consistency validation:') self.subject.logger.fatal.assert_any_call('sdw1 checksum set to 0 differs from master checksum set to 1') @patch.object(HeapChecksum, "__init__", return_value=None) def test_when_cannot_determine_checksums_it_raises(self, mock_heap_checksum_init): self.mock_check_segment_consistency.return_value = ([], [1], True) self.mock_get_segments_checksum_settings.return_value = ([], []) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.return_one = True self.assertTrue(self.gparray.master.isSegmentMaster(True)) with self.assertRaisesRegexp(Exception, "No segments responded to ssh query for heap checksum validation."): self.subject.run() self.mock_get_segments_checksum_settings.assert_called_with([self.primary0]) mock_heap_checksum_init.assert_called_with(self.gpArrayMock, logger=self.subject.logger, num_workers=1) @patch("os._exit") def test_when_no_segments_to_recover_validation_succeeds(self, _): self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.return_one = False with self.assertRaises(SystemExit): # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.subject.logger.info.assert_any_call('No checksum validation necessary when ' 'there are no segments to recover.') @patch.object(TableLogger, "info") @patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=True) @patch("os._exit") def test_successful_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit): self.subject.run() self.subject.logger.info.assert_any_call('The rebalance operation has completed successfully.') @patch.object(TableLogger, "info") @patch.object(GpSegmentRebalanceOperation, "rebalance", return_value=False) @patch("os._exit") def test_failed_rebalance(self, _, __, ___): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.rebalanceSegments = True options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 0) self.subject.logger.info.assert_any_call('The rebalance operation has completed with WARNINGS. ' 'Please review the output in the gprecoverseg log.') @patch.object(TableLogger, "info") def test_failed_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], []) self.return_one = True self.mock_build_mirrors.return_value = False with self.assertRaises(SystemExit) as cm: self.subject.run() self.assertEqual(cm.exception.code, 1) @patch.object(TableLogger, "info") def test_successful_recover(self, _): self.gpArrayMock.get_unbalanced_segdbs.return_value = [self.primary0] options = Options() options.masterDataDirectory = self.temp_dir options.spareDataDirectoryFile = None options.showProgress = True options.showProgressInplace = True # import HERE so that patches are already in place! from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram self.subject = GpRecoverSegmentProgram(options) self.subject.logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) self.mock_get_mirrors_to_build.side_effect = self._get_test_mirrors self.primary0.heap_checksum = 1 self.mock_check_segment_consistency.return_value = ([self.primary0], [], 1) self.mock_get_segments_checksum_settings.return_value = ([self.mirror0], []) self.return_one = True self.mock_build_mirrors.return_value = True with self.assertRaises(SystemExit) as cm: # XXX Disable live FTS probes. The fact that we have to do this # indicates that these are not really unit tests. with patch.object(self.subject, 'trigger_fts_probe'): self.subject.run() self.assertEqual(cm.exception.code, 0) def _create_gparray_with_2_primary_2_mirrors(self): master = Segment.initFromString( "1|-1|p|p|s|u|mdw|mdw|5432|/data/master") self.primary0 = Segment.initFromString( "2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0") primary1 = Segment.initFromString( "3|1|p|p|s|u|sdw2|sdw2|40001|/data/primary1") self.mirror0 = Segment.initFromString( "4|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0") mirror1 = Segment.initFromString( "5|1|m|m|s|u|sdw1|sdw1|50001|/data/mirror1") return GpArray([master, self.primary0, primary1, self.mirror0, mirror1])
def test_check_database_connection_exceed_max_retries(self, mock1, mock2): options = Mock() confProvider = Mock() gprecover_prog = GpRecoverSegmentProgram(options) self.assertFalse(gprecover_prog._check_database_connection(confProvider))
def test_check_segment_change_tracking_disabled_state_return_false(self): options = Mock() gprecover_prog = GpRecoverSegmentProgram(options) res = gprecover_prog.check_segment_change_tracking_disabled_state( gparray.SEGMENT_STATE_READY) self.assertEquals(res, False)
def test_check_database_connection(self, mock1, mock2): options = Mock() confProvider = Mock() gprecover_prog = GpRecoverSegmentProgram(options) self.assertTrue(gprecover_prog._check_database_connection(confProvider))
class GpRecoverSegmentProgramTestCase(GpTestCase): def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args() options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpMasterEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock), # patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch( 'gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool def test_check_persistent_tables__when_no_errors_detected__succeeds(self): segments = [ self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2') ] command1 = Mock(spec=Command) command1.get_results.return_value = '' command2 = Mock(spec=Command) command2.get_results.return_value = '' self.execSqlResult.fetchall.return_value = [['template1']] self.worker_pool.getCompletedItems.return_value = [command1, command2] self.subject._check_persistent_tables(segments) def test_check_persistent_tables__with_no_segments__succeeds(self): self.execSqlResult.fetchall.return_value = [['template1']] self.subject._check_persistent_tables([]) def test_check_persistent_tables__when_error_exists__raises(self): self.execSqlResult.fetchall.return_value = [['template1']] segments = [ self._get_mock_segment('seg1', '1234', 'seg1', '/tmp/seg1'), self._get_mock_segment('seg2', '2345', 'seg2', '/tmp/seg2') ] command1 = Mock() command1.get_results.return_value = ['sdfsdf'] command2 = Mock() command2.get_results.return_value = ['asdfas'] self.worker_pool.getCompletedItems.return_value = [command1, command2] with self.assertRaisesRegexp(Exception, 'Please fix the persistent tables issue'): self.subject._check_persistent_tables(segments) def test_check_database_connection__when_all_segments_are_ready_to_connect__returns_true( self): self.gparray.getDbList.return_value = [] conf_provider = self._get_mock_conf_provider(self.gparray) self.assertTrue(self.subject._check_database_connection(conf_provider)) def test_check_database_connection__when_raises_beyond_max_retries__returns_false( self): conf_provider_that_raises = Mock(spec=GpConfigurationProvider) self.assertFalse( self.subject._check_database_connection(conf_provider_that_raises)) def test_check_segment_state__when_all_segments_ready__succeeds(self): conf_provider = self._get_mock_conf_provider(self.gparray) command1 = Mock(spec=Command) command1.get_results.return_value = CommandResult( 0, '', 'segmentState: Ready', False, True) command2 = Mock(spec=Command) command2.get_results.return_value = CommandResult( 1, '', 'segmentState: Ready', False, True) self.worker_pool.getCompletedItems.return_value = [command1, command2] self.subject._check_segment_state_for_connection(conf_provider) def test_check_segment_state__when_one_segment_not_ready__raises(self): segment1 = Mock(spec=GpDB) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=GpDB) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False gparray_mock = Mock(spec=GpArray) gparray_mock.getDbList.return_value = [segment1, segment2] conf_provider = self._get_mock_conf_provider(gparray_mock) command1 = Mock(spec=Command) command1.get_results.return_value = CommandResult( 0, '', 'segmentState: Ready', False, True) command2 = Mock(spec=Command) command2.get_results.return_value = CommandResult( 1, '', 'Failed to connect', False, True) self.worker_pool.getCompletedItems.return_value = [command1, command2] with self.assertRaisesRegexp(Exception, 'Not ready to connect to database'): self.subject._check_segment_state_for_connection(conf_provider) @patch('gppylib.commands.base.Command.get_results') def test_check_segment_state_ready_for_recovery_ignores_initial_stderr_warnings( self, mock_results): mock_results.return_value = CommandResult( 0, '', 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of \n' 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n' 'dataState: InChangeTracking\n', False, True) segment_mock = Mock(spec=GpDB) segment_mock.isSegmentQD.return_value = False segment_mock.isSegmentModeInChangeLogging.return_value = True segment_mock.getSegmentHostName.return_value = 'foo1' segment_mock.getSegmentDataDirectory.return_value = 'bar' segment_mock.getSegmentPort.return_value = 5555 segment_mock.getSegmentDbId.return_value = 2 segment_mock.getSegmentRole.return_value = 'p' segment_mock.getSegmentMode.return_value = 'c' segmentList = [segment_mock] dbsMap = {2: segment_mock} segmentStates = self.subject.check_segment_state_ready_for_recovery( segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) self.subject.logger.info.assert_called_once_with( 'Warning: Permanently added "a4eb06fc188f,172.17.0.2" (RSA) to the list of ' ) @patch('gppylib.commands.base.Command.get_results') def test_check_segment_state_ready_for_recovery_with_segment_in_change_tracking__sets_disabled_state( self, mock_results): mock_results.return_value = CommandResult( 0, '', 'mode: PrimarySegment\nsegmentState: ChangeTrackingDisabled\n' 'dataState: InChangeTracking\n', False, True) segment = Mock(spec=GpDB) segment.isSegmentQD.return_value = False segment.isSegmentModeInChangeLogging.return_value = True segment.getSegmentHostName.return_value = 'foo1' segment.getSegmentDataDirectory.return_value = 'bar' segment.getSegmentPort.return_value = 5555 segment.getSegmentDbId.return_value = 2 segment.getSegmentRole.return_value = 'p' segment.getSegmentMode.return_value = 'c' segmentList = [segment] dbsMap = {2: segment} segmentStates = self.subject.check_segment_state_ready_for_recovery( segmentList, dbsMap) self.assertEquals(segmentStates, {2: 'ChangeTrackingDisabled'}) def test_output_segments_in_change_tracking_disabled_should_print_failed_segments( self): segs_in_change_tracking_disabled = { 2: 'ChangeTrackingDisabled', 4: 'ChangeTrackingDisabled' } self.subject._output_segments_in_change_tracking_disabled( segs_in_change_tracking_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 2 ,4 in change tracking disabled state, need to run recoverseg with -F option.' ) def test_check_segment_change_tracking_disabled_state_return_true(self): res = self.subject.check_segment_change_tracking_disabled_state( gparray.SEGMENT_STATE_CHANGE_TRACKING_DISABLED) self.assertEquals(res, True) def test_check_segment_change_tracking_disabled_state_return_false(self): res = self.subject.check_segment_change_tracking_disabled_state( gparray.SEGMENT_STATE_READY) self.assertEquals(res, False) def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments( self): segs_with_persistent_mirroring_disabled = [0, 1] self.subject._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.' ) def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments( self): segs_with_persistent_mirroring_disabled = [] self.subject._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) assert not self.subject.logger.warn.called def test_is_segment_mirror_state_mismatched_cluster_mirroring_enabled_segment_mirroring_disabled( self): self.execSqlResult.fetchall.return_value = [[3], [1]] gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock(spec=GpDB) segment_mock.getSegmentContentId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched( gparray_mock, segment_mock) self.assertTrue(mismatched) def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_enabled( self): self.execSqlResult.fetchall.return_value = [[3]] gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_FILE_REPLICATION segment_mock = Mock(spec=GpDB) segment_mock.getSegmentContentId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched( gparray_mock, segment_mock) self.assertFalse(mismatched) def test_is_segment_mirror_state_mismatched_cluster_and_segments_mirroring_disabled( self): gparray_mock = Mock(spec=GpArray) gparray_mock.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE segment_mock = Mock(spec=GpDB) segment_mock.getSegmentDbId.return_value = 0 mismatched = self.subject.is_segment_mirror_state_mismatched( gparray_mock, segment_mock) self.assertFalse(mismatched) def test__run__when_no_replication_is_setup__raises(self): self.gparray.getSegDbList.return_value = [] self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_NONE with self.assertRaisesRegexp(Exception, 'replication is not configured'): self.subject.run() def test__run__when_fault_strategy_is_SAN__calls_SAN_failback(self): with patch.object(GpRecoverSegmentProgram, 'SanFailback', return_value=None) as san_failback: self.gparray.getSegDbList.return_value = [] self.gparray.getFaultStrategy.return_value = gparray.FAULT_STRATEGY_SAN self.subject.run() san_failback.assert_called_once_with(self.gparray, self.gp_env) ############################################################ # Private def _get_mock_segment(self, name, port, address, datadir): segment = Mock(spec=GpDB) segment.getSegmentHostName.return_value = name segment.getSegmentAddress.return_value = address segment.getSegmentPort.return_value = port segment.getSegmentDataDirectory.return_value = datadir return segment def _get_mock_conf_provider(self, gparray_result=None): conf_provider = Mock(spec=GpConfigurationProvider) conf_provider.loadSystemConfig.return_value = gparray_result return conf_provider def _segments_mock(self): segment1 = Mock(spec=GpDB) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=GpDB) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False return [segment1, segment2]
class GpRecoverSegmentProgramTestCase(GpTestCase): def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args(args=[]) options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpMasterEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock), # patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch('gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments(self): segs_with_persistent_mirroring_disabled = [0, 1] self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.') def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments(self): segs_with_persistent_mirroring_disabled = [] self.subject._output_segments_with_persistent_mirroring_disabled(segs_with_persistent_mirroring_disabled) assert not self.subject.logger.warn.called ############################################################ # Private def _get_mock_segment(self, name, port, address, datadir): segment = Mock(spec=Segment) segment.getSegmentHostName.return_value = name segment.getSegmentAddress.return_value = address segment.getSegmentPort.return_value = port segment.getSegmentDataDirectory.return_value = datadir return segment def _get_mock_conf_provider(self, gparray_result=None): conf_provider = Mock(spec=GpConfigurationProvider) conf_provider.loadSystemConfig.return_value = gparray_result return conf_provider def _segments_mock(self): segment1 = Mock(spec=Segment) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=Segment) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False return [segment1, segment2]
def test_check_database_connection_exceed_max_retries(self, mock1, mock2): options = Mock() confProvider = Mock() gprecover_prog = GpRecoverSegmentProgram(options) self.assertFalse( gprecover_prog._check_database_connection(confProvider))
class GpRecoverSegmentProgramTestCase(GpTestCase): def setUp(self): raw_options = GpRecoverSegmentProgram.createParser() (options, _) = raw_options.parse_args(args=[]) options.spareDataDirectoryFile = None options.newRecoverHosts = None self.subject = GpRecoverSegmentProgram(options) self.execSqlResult = Mock(spec=['fetchall']) self.gp_env = Mock() GpMasterEnvironmentMock = Mock(return_value=self.gp_env) self.gparray = Mock(spec=GpArray) self.gparray.getDbList.return_value = self._segments_mock() configProviderMock = Mock(spec=GpConfigurationProvider) configProviderMock.initializeProvider.return_value = configProviderMock configProviderMock.loadSystemConfig.return_value = self.gparray self.getConfigProviderFunctionMock = Mock(GpConfigurationProvider) self.getConfigProviderFunctionMock.return_value = configProviderMock self.subject.logger = Mock() self.worker_pool = Mock(spec=WorkerPool, return_value=None) self.worker_pool.getCompletedItems.return_value = [] self.worker_pool.logger = self.subject.logger self.worker_pool.addCommand.return_value = None self.pool_completed = [] self.apply_patches([ patch("gppylib.db.dbconn.connect"), patch("gppylib.db.dbconn.DbURL"), patch("gppylib.db.dbconn.execSQL", return_value=self.execSqlResult), patch('time.sleep'), patch('gppylib.programs.clsRecoverSegment.GpMasterEnvironment', GpMasterEnvironmentMock), # patch('gppylib.system.environment.GpMasterEnvironment.__init__', self.gp_env), # patch('gppylib.system.environment.GpMasterEnvironment.getMasterPort'), patch('gppylib.system.faultProberInterface.getFaultProber'), patch( 'gppylib.system.configurationInterface.getConfigurationProvider', self.getConfigProviderFunctionMock), patch('gppylib.commands.base.WorkerPool.__init__', self.worker_pool), patch('gppylib.commands.base.WorkerPool.getCompletedItems', return_value=self.pool_completed), patch('gppylib.commands.base.WorkerPool.addCommand'), patch('gppylib.commands.base.WorkerPool.join'), ]) # tests make use of a workaround to access a python attribute that is normally # name mangled when specified with a "__" prefix. That workaround is to use _<class>__<attribute> # such as self.subject._GpRecoverSegmentProgram__pool = mock_pool self.subject._GpRecoverSegmentProgram__pool = self.worker_pool def test_output_segments_with_persistent_mirroring_disabled_should_print_failed_segments( self): segs_with_persistent_mirroring_disabled = [0, 1] self.subject._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) self.subject.logger.warn.assert_called_once_with( 'Segments with dbid 0, 1 not recovered; persistent mirroring state is disabled.' ) def test_output_segments_with_persistent_mirroring_disabled_should_not_print_if_no_segments( self): segs_with_persistent_mirroring_disabled = [] self.subject._output_segments_with_persistent_mirroring_disabled( segs_with_persistent_mirroring_disabled) assert not self.subject.logger.warn.called ############################################################ # Private def _get_mock_segment(self, name, port, address, datadir): segment = Mock(spec=Segment) segment.getSegmentHostName.return_value = name segment.getSegmentAddress.return_value = address segment.getSegmentPort.return_value = port segment.getSegmentDataDirectory.return_value = datadir return segment def _get_mock_conf_provider(self, gparray_result=None): conf_provider = Mock(spec=GpConfigurationProvider) conf_provider.loadSystemConfig.return_value = gparray_result return conf_provider def _segments_mock(self): segment1 = Mock(spec=Segment) segment1.getSegmentHostName.return_value = 'foo1' segment1.isSegmentUp.return_value = True segment1.isSegmentMaster.return_value = False segment1.isSegmentStandby.return_value = False segment2 = Mock(spec=Segment) segment2.getSegmentHostName.return_value = 'foo2' segment2.isSegmentUp.return_value = True segment2.isSegmentMaster.return_value = False segment2.isSegmentStandby.return_value = False return [segment1, segment2]
def test_check_database_connection(self, mock1, mock2): options = Mock() confProvider = Mock() gprecover_prog = GpRecoverSegmentProgram(options) self.assertTrue( gprecover_prog._check_database_connection(confProvider))