class TestRdirClient(BaseTestCase): @classmethod def setUpClass(cls): super(TestRdirClient, cls).setUpClass() cls._service('@indexer', 'stop') @classmethod def tearDownClass(cls): super(TestRdirClient, cls).tearDownClass() cls._service('@indexer', 'start') def _push_chunks(self): max_mtime = 16 self.incident_date = random.randrange(2, max_mtime - 1) expected_entries = list() for _ in range(4): cid = random_id(64) for _ in range(random.randrange(2, 5)): content_id = random_id(32) for _ in range(random.randrange(2, 5)): chunk_id = random_id(63) mtime = random.randrange(0, max_mtime + 1) if mtime <= self.incident_date: chunk_id += '0' else: chunk_id += '1' self.rdir.chunk_push(self.rawx_id, cid, content_id, chunk_id, mtime=mtime) entry = (cid, content_id, chunk_id, {'mtime': mtime}) expected_entries.append(entry) self.expected_entries = sorted(expected_entries) def setUp(self): super(TestRdirClient, self).setUp() self.rawx_conf = random.choice(self.conf['services']['rawx']) self.rawx_id = self.rawx_conf.get('service_id', self.rawx_conf['addr']) self.rdir = RdirClient(self.conf) self.rdir.admin_clear(self.rawx_id, clear_all=True) self._push_chunks() self.rdir._direct_request = Mock(side_effect=self.rdir._direct_request) def _assert_chunk_fetch(self, expected_entries, entries, limit=0): self.assertListEqual(expected_entries, list(entries)) nb_requests = 1 if limit > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(limit))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_fetch(self): entries = self.rdir.chunk_fetch(self.rawx_id) self._assert_chunk_fetch(self.expected_entries, entries) def test_chunk_fetch_with_limit(self): entries = self.rdir.chunk_fetch(self.rawx_id, limit=2) self._assert_chunk_fetch(self.expected_entries, entries, limit=2) def test_chunk_fetch_with_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid) self._assert_chunk_fetch(expected_entries_cid, entries) def test_chunk_fetch_with_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] entries = self.rdir.chunk_fetch(self.rawx_id, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid, entries, limit=2) def test_chunk_fetch_with_start_after(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_limit(self): start_after_index = random.randrange(0, len(self.expected_entries)) start_after = '|'.join(self.expected_entries[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, limit=2) self._assert_chunk_fetch(self.expected_entries[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_start_after_container_id(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_start_after_container_id_limit(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] start_after_index = random.randrange(0, len(expected_entries_cid)) start_after = '|'.join(expected_entries_cid[start_after_index][:3]) entries = self.rdir.chunk_fetch(self.rawx_id, start_after=start_after, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_cid[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_no_incident(self): entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(list(), entries) def test_chunk_fetch_with_rebuild(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True) self._assert_chunk_fetch(expected_entries_rebuild, entries) def test_chunk_fetch_with_rebuild_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, limit=2) self._assert_chunk_fetch(expected_entries_rebuild, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries) def test_chunk_fetch_with_rebuild_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_contaier_id_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, limit=2) self._assert_chunk_fetch(expected_entries_rebuild_cid, entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries) def test_chunk_fetch_with_rebuild_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] if expected_entries_rebuild: start_after_index = random.randrange(0, len(expected_entries_rebuild)) start_after = '|'.join( expected_entries_rebuild[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, start_after=start_after, limit=2) self._assert_chunk_fetch(expected_entries_rebuild[start_after_index + 1:], entries, limit=2) def test_chunk_fetch_with_rebuild_container_id_start_after_limit(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() expected_entries_rebuild = [ entry for entry in self.expected_entries if entry[2][-1] == '0' ] cid = random.choice(self.expected_entries)[0] expected_entries_rebuild_cid = \ [entry for entry in expected_entries_rebuild if entry[0] == cid] if expected_entries_rebuild_cid: start_after_index = random.randrange( 0, len(expected_entries_rebuild_cid)) start_after = '|'.join( expected_entries_rebuild_cid[start_after_index][:3]) else: start_after_index = 0 start_after = '|'.join( (random_id(64), random_id(32), random_id(64))) entries = self.rdir.chunk_fetch(self.rawx_id, rebuild=True, container_id=cid, start_after=start_after, limit=2) self._assert_chunk_fetch( expected_entries_rebuild_cid[start_after_index + 1:], entries, limit=2) def _assert_chunk_status(self, expected_entries, status, max=0, incident=False): expected_status = dict() expected_status['chunk'] = {'total': len(expected_entries)} expected_status['container'] = dict() for entry in expected_entries: expected_status['container'][entry[0]]['total'] = \ expected_status['container'].setdefault( entry[0], dict()).get('total', 0) + 1 if incident: expected_entries_rebuild = [ entry for entry in expected_entries if entry[2][-1] == '0' ] expected_status['chunk']['to_rebuild'] = \ len(expected_entries_rebuild) for entry in expected_entries_rebuild: expected_status['container'][entry[0]]['to_rebuild'] = \ expected_status['container'][entry[0]].get( 'to_rebuild', 0) + 1 self.assertDictEqual(expected_status, status) nb_requests = 1 if max > 0 and len(expected_entries) > 0: nb_requests = int(math.ceil(len(expected_entries) / float(max))) self.assertEqual(nb_requests, self.rdir._direct_request.call_count) self.rdir._direct_request.reset_mock() def test_chunk_status(self): status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status) def test_chunk_status_with_max(self): status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, max=2) def test_chunk_status_with_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status) def test_chunk_status_with_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, max=2) def test_chunk_status_with_marker(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status) def test_chunk_status_with_marker_max(self): marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, max=2) def test_chunk_status_marker_prefix(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status) def test_chunk_status_with_marker_prefix_max(self): cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, max=2) def test_chunk_status_with_incident(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id) self._assert_chunk_status(self.expected_entries, status, incident=True) def test_chunk_status_with_incident_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() status = self.rdir.status(self.rawx_id, max=2) self._assert_chunk_status(self.expected_entries, status, incident=True, max=2) def test_chunk_status_with_incident_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid) self._assert_chunk_status(expected_entries_cid, status, incident=True) def test_chunk_status_with_incident_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] status = self.rdir.status(self.rawx_id, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid, status, incident=True, max=2) def test_chunk_status_with_incident_marker(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() marker_index = random.randrange(0, len(self.expected_entries)) marker = '|'.join(self.expected_entries[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, max=2) self._assert_chunk_status(self.expected_entries[marker_index + 1:], status, incident=True, max=2) def test_chunk_status_with_incident_marker_prefix(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True) def test_chunk_status_with_incident_marker_prefix_max(self): self.rdir.admin_incident_set(self.rawx_id, self.incident_date) self.rdir._direct_request.reset_mock() cid = random.choice(self.expected_entries)[0] expected_entries_cid = [ entry for entry in self.expected_entries if entry[0] == cid ] marker_index = random.randrange(0, len(expected_entries_cid)) marker = '|'.join(expected_entries_cid[marker_index][:3]) status = self.rdir.status(self.rawx_id, marker=marker, prefix=cid, max=2) self._assert_chunk_status(expected_entries_cid[marker_index + 1:], status, incident=True, max=2)
class RawxRebuildJob(XcuteRdirJob): JOB_TYPE = 'rawx-rebuild' TASK_CLASS = RawxRebuildTask DEFAULT_RAWX_TIMEOUT = 60.0 DEFAULT_DRY_RUN = False DEFAULT_ALLOW_SAME_RAWX = True DEFAULT_TRY_CHUNK_DELETE = False DEFAULT_ALLOW_FROZEN_CT = False DEFAULT_DECLARE_INCIDENT_DATE = False @classmethod def sanitize_params(cls, job_params): sanitized_job_params, _ = super(RawxRebuildJob, cls).sanitize_params(job_params) # specific configuration service_id = job_params.get('service_id') if not service_id: raise ValueError('Missing service ID') sanitized_job_params['service_id'] = service_id sanitized_job_params['rawx_timeout'] = float_value( job_params.get('rawx_timeout'), cls.DEFAULT_RAWX_TIMEOUT) sanitized_job_params['dry_run'] = boolean_value( job_params.get('dry_run'), cls.DEFAULT_DRY_RUN) sanitized_job_params['allow_same_rawx'] = boolean_value( job_params.get('allow_same_rawx'), cls.DEFAULT_ALLOW_SAME_RAWX) sanitized_job_params['try_chunk_delete'] = boolean_value( job_params.get('try_chunk_delete'), cls.DEFAULT_TRY_CHUNK_DELETE) sanitized_job_params['allow_frozen_container'] = boolean_value( job_params.get('allow_frozen_container'), cls.DEFAULT_ALLOW_FROZEN_CT) set_specific_incident_date = int_value( job_params.get('set_specific_incident_date'), None) if set_specific_incident_date is None: set_incident_date = boolean_value( job_params.get('set_incident_date'), cls.DEFAULT_DECLARE_INCIDENT_DATE) if set_incident_date: set_specific_incident_date = int(time.time()) else: set_incident_date = True sanitized_job_params['set_incident_date'] = set_incident_date sanitized_job_params['set_specific_incident_date'] = \ set_specific_incident_date return sanitized_job_params, 'rawx/%s' % service_id def __init__(self, conf, logger=None): super(RawxRebuildJob, self).__init__(conf, logger=logger) self.rdir_client = RdirClient(self.conf, logger=self.logger) def prepare(self, job_params): service_id = job_params['service_id'] rdir_timeout = job_params['rdir_timeout'] set_incident_date = job_params['set_incident_date'] set_specific_incident_date = job_params['set_specific_incident_date'] if not set_incident_date: return self.rdir_client.admin_incident_set(service_id, set_specific_incident_date, timeout=rdir_timeout) def get_tasks(self, job_params, marker=None): chunk_infos = self.get_chunk_infos(job_params, marker=marker) for container_id, content_id, chunk_id, _ in chunk_infos: task_id = '|'.join((container_id, content_id, chunk_id)) yield task_id, { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } def get_total_tasks(self, job_params, marker=None): chunk_infos = self.get_chunk_infos(job_params, marker=marker) i = 0 for i, (container_id, content_id, chunk_id, _) \ in enumerate(chunk_infos, 1): if i % 1000 == 0: yield '|'.join((container_id, content_id, chunk_id)), 1000 remaining = i % 1000 if remaining > 0: yield '|'.join((container_id, content_id, chunk_id)), remaining def get_chunk_infos(self, job_params, marker=None): service_id = job_params['service_id'] rdir_fetch_limit = job_params['rdir_fetch_limit'] rdir_timeout = job_params['rdir_timeout'] chunk_infos = self.rdir_client.chunk_fetch(service_id, rebuild=True, timeout=rdir_timeout, limit=rdir_fetch_limit, start_after=marker) return chunk_infos