class TestFrontendClient(object): def setup_method(self, method): self.opts = Munch( frontend_base_url="http://example.com/", frontend_auth="12345678", ) self.fc = FrontendClient(self.opts) self.data = { "foo": "bar", "bar": [1, 3, 5], } self.url_path = "sub_path" self.build_id = 12345 self.chroot_name = "fedora-20-x86_64" @pytest.fixture def mask_post_to_fe(self): self.ptf = MagicMock() self.fc._post_to_frontend = self.ptf def test_post_to_frontend(self, post_req): post_req.return_value.status_code = 200 self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_not_200(self, post_req): post_req.return_value.status_code = 501 with pytest.raises(RequestException): self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_post_error(self, post_req): post_req.side_effect = RequestException() with pytest.raises(RequestException): self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_repeated_first_try_ok(self, mask_post_to_fe, mc_time): response = "ok\n" self.ptf.return_value = response assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response assert not mc_time.sleep.called def test_post_to_frontend_repeated_second_try_ok(self, mask_post_to_fe, mc_time): response = "ok\n" self.ptf.side_effect = [ RequestException(), response, ] assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response assert mc_time.sleep.called def test_post_to_frontend_repeated_all_attempts_failed(self, mask_post_to_fe, mc_time): self.ptf.side_effect = RequestException() with pytest.raises(RequestException): self.fc._post_to_frontend_repeatedly(self.data, self.url_path) assert mc_time.sleep.called def test_update(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.update(self.data) assert ptfr.call_args == mock.call(self.data, "update") def test_starting_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr for val in [True, False]: ptfr.return_value.json.return_value = {"can_start": val} assert self.fc.starting_build(self.build_id, self.chroot_name) == val def test_starting_build_err(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr with pytest.raises(RequestException): self.fc.starting_build(self.build_id, self.chroot_name) def test_starting_build_err_2(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr ptfr.return_value.json.return_value = {} with pytest.raises(RequestException): self.fc.starting_build(self.build_id, self.chroot_name) def test_reschedule_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.reschedule_build(self.build_id, self.chroot_name) expected = mock.call({'build_id': self.build_id, 'chroot': self.chroot_name}, 'reschedule_build_chroot') assert ptfr.call_args == expected
class TestFrontendClient(object): def setup_method(self, method): self.opts = Munch( frontend_base_url="http://example.com/", frontend_auth="12345678", ) self.fc = FrontendClient(self.opts) self.data = { "foo": "bar", "bar": [1, 3, 5], } self.url_path = "sub_path" self.build_id = 12345 self.task_id = "12345-fedora-20-x86_64" self.chroot_name = "fedora-20-x86_64" @pytest.fixture def mask_frontend_request(self): self.f_r = MagicMock() self.fc._frontend_request = self.f_r def test_post_to_frontend(self, f_request_method): name, method = f_request_method method.return_value.status_code = 200 self.fc._frontend_request(self.url_path, self.data, method=name) assert method.called def test_post_to_frontend_wrappers(self, f_request_method): name, method = f_request_method method.return_value.status_code = 200 call = getattr(self.fc, name) if name == 'get': call(self.url_path) else: call(self.url_path, self.data) assert method.called def test_post_to_frontend_not_200(self, post_req): post_req.return_value.status_code = 501 with pytest.raises(FrontendClientRetryError): self.fc._frontend_request(self.url_path, self.data) assert post_req.called def test_post_to_frontend_post_error(self, post_req): post_req.side_effect = RequestException() with pytest.raises(FrontendClientRetryError): self.fc._frontend_request(self.url_path, self.data) assert post_req.called def test_post_to_frontend_repeated_first_try_ok(self, mask_frontend_request, mc_time): response = "ok\n" self.f_r.return_value = response mc_time.time.return_value = 0 assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response assert not mc_time.sleep.called def test_post_to_frontend_repeated_second_try_ok(self, f_request_method, mask_frontend_request, mc_time): method_name, method = f_request_method response = "ok\n" self.f_r.side_effect = [ FrontendClientRetryError(), response, ] mc_time.time.return_value = 0 assert self.fc._frontend_request_repeatedly( self.url_path, data=self.data, method=method_name) == response assert mc_time.sleep.called def test_post_to_frontend_err_400(self, post_req, mc_time): response = Response() response.status_code = 404 response.reason = 'NOT FOUND' post_req.side_effect = [ FrontendClientRetryError(), response, ] mc_time.time.return_value = 0 with pytest.raises(FrontendClientException): assert self.fc._post_to_frontend_repeatedly( self.data, self.url_path) == response assert mc_time.sleep.called @mock.patch('backend.frontend.BACKEND_TIMEOUT', 100) def test_post_to_frontend_repeated_all_attempts_failed( self, mask_frontend_request, caplog, mc_time): mc_time.time.side_effect = [ 0, 0, 5, 5 + 10, 5 + 10 + 15, 5 + 10 + 15 + 20, 1000 ] self.f_r.side_effect = FrontendClientRetryError() with pytest.raises(FrontendClientException): self.fc._post_to_frontend_repeatedly(self.data, self.url_path) assert mc_time.sleep.call_args_list == [ mock.call(x) for x in [5, 10, 15, 20, 25] ] assert len(caplog.records) == 5 def test_post_to_frontend_repeated_indefinitely(self, mask_frontend_request, caplog, mc_time): mc_time.time.return_value = 1 self.fc.try_indefinitely = True self.f_r.side_effect = [FrontendClientRetryError() for _ in range(100)] \ + [FrontendClientException()] # e.g. 501 eventually with pytest.raises(FrontendClientException): self.fc._post_to_frontend_repeatedly(self.data, self.url_path) assert mc_time.sleep.called assert len(caplog.records) == 100 def test_reschedule_300(self, mask_frontend_request, post_req): response = Response() response.status_code = 302 response.reason = 'whatever' post_req.side_effect = response with pytest.raises(FrontendClientException) as ex: self.fc.reschedule_all_running() assert 'Failed to reschedule builds' in str(ex) def test_update(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.update(self.data) assert ptfr.call_args == mock.call(self.data, "update") def test_starting_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr for val in [True, False]: ptfr.return_value.json.return_value = {"can_start": val} assert self.fc.starting_build(self.data) == val def test_starting_build_err(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr with pytest.raises(FrontendClientException): self.fc.starting_build(self.data) def test_starting_build_err_2(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr ptfr.return_value.json.return_value = {} with pytest.raises(FrontendClientException): self.fc.starting_build(self.data) def test_reschedule_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.reschedule_build(self.build_id, self.task_id, self.chroot_name) expected = mock.call( { 'build_id': self.build_id, 'task_id': self.task_id, 'chroot': self.chroot_name }, 'reschedule_build_chroot') assert ptfr.call_args == expected
class TestFrontendClient(object): def setup_method(self, method): self.opts = Munch( frontend_base_url="http://example.com/", frontend_auth="12345678", ) self.fc = FrontendClient(self.opts) self.data = { "foo": "bar", "bar": [1, 3, 5], } self.url_path = "sub_path" self.build_id = 12345 self.task_id = "12345-fedora-20-x86_64" self.chroot_name = "fedora-20-x86_64" @pytest.fixture def mask_post_to_fe(self): self.ptf = MagicMock() self.fc._post_to_frontend = self.ptf def test_post_to_frontend(self, post_req): post_req.return_value.status_code = 200 self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_not_200(self, post_req): post_req.return_value.status_code = 501 with pytest.raises(RequestException): self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_post_error(self, post_req): post_req.side_effect = RequestException() with pytest.raises(RequestException): self.fc._post_to_frontend(self.data, self.url_path) assert post_req.called def test_post_to_frontend_repeated_first_try_ok(self, mask_post_to_fe, mc_time): response = "ok\n" self.ptf.return_value = response assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response assert not mc_time.sleep.called def test_post_to_frontend_repeated_second_try_ok(self, mask_post_to_fe, mc_time): response = "ok\n" self.ptf.side_effect = [ RequestException(), response, ] assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response assert mc_time.sleep.called def test_post_to_frontend_repeated_all_attempts_failed(self, mask_post_to_fe, mc_time): self.ptf.side_effect = RequestException() with pytest.raises(RequestException): self.fc._post_to_frontend_repeatedly(self.data, self.url_path) assert mc_time.sleep.called def test_update(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.update(self.data) assert ptfr.call_args == mock.call(self.data, "update") def test_starting_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr for val in [True, False]: ptfr.return_value.json.return_value = {"can_start": val} assert self.fc.starting_build(self.data) == val def test_starting_build_err(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr with pytest.raises(RequestException): self.fc.starting_build(self.data) def test_starting_build_err_2(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr ptfr.return_value.json.return_value = {} with pytest.raises(RequestException): self.fc.starting_build(self.data) def test_reschedule_build(self): ptfr = MagicMock() self.fc._post_to_frontend_repeatedly = ptfr self.fc.reschedule_build(self.build_id, self.task_id, self.chroot_name) expected = mock.call({'build_id': self.build_id, 'task_id': self.task_id, 'chroot': self.chroot_name}, 'reschedule_build_chroot') assert ptfr.call_args == expected
class Pruner(object): def __init__(self, opts, cmdline_opts=None): self.opts = opts self.prune_days = getattr(self.opts, "prune_days", DEF_DAYS) self.chroots = {} self.frontend_client = FrontendClient(self.opts) self.mtime_optimization = True if cmdline_opts: self.mtime_optimization = not cmdline_opts.no_mtime_optimization def run(self): response = self.frontend_client._post_to_frontend_repeatedly("", "chroots-prunerepo-status") self.chroots = json.loads(response.content) results_dir = self.opts.destdir loginfo("Pruning results dir: {} ".format(results_dir)) user_dir_names, user_dirs = list_subdir(results_dir) loginfo("Going to process total number: {} of user's directories".format(len(user_dir_names))) loginfo("Going to process user's directories: {}".format(user_dir_names)) loginfo("--------------------------------------------") for username, subpath in zip(user_dir_names, user_dirs): loginfo("For user `{}` exploring path: {}".format(username, subpath)) for projectdir, project_path in zip(*list_subdir(subpath)): loginfo("Exploring projectdir `{}` with path: {}".format(projectdir, project_path)) self.prune_project(project_path, username, projectdir) loginfo("--------------------------------------------") loginfo("Setting final_prunerepo_done for deactivated chroots") chroots_to_prune = [] for chroot, active in self.chroots.items(): if not active: chroots_to_prune.append(chroot) self.frontend_client._post_to_frontend_repeatedly(chroots_to_prune, "final-prunerepo-done") loginfo("--------------------------------------------") loginfo("Pruning finished") def prune_project(self, project_path, username, projectdir): loginfo("Going to prune {}/{}".format(username, projectdir)) projectname = projectdir.split(':', 1)[0] loginfo("projectname = {}".format(projectname)) try: if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname): loginfo("Skipped {}/{} since auto createrepo option is disabled" .format(username, projectdir)) return if get_persistent_status(self.opts.frontend_base_url, username, projectname): loginfo("Skipped {}/{} since the project is persistent" .format(username, projectdir)) return if not get_auto_prune_status(self.opts.frontend_base_url, username, projectname): loginfo("Skipped {}/{} since auto-prunning is disabled for the project" .format(username, projectdir)) return except (CoprException, CoprRequestException) as exception: logerror("Failed to get project details for {}/{} with error: {}".format( username, projectdir, exception)) return for sub_dir_name in os.listdir(project_path): chroot_path = os.path.join(project_path, sub_dir_name) if sub_dir_name == 'modules': continue if not os.path.isdir(chroot_path): continue if sub_dir_name not in self.chroots: loginfo("Final pruning already done for chroot {}/{}:{}".format(username, projectdir, sub_dir_name)) continue if self.mtime_optimization: # We only ever remove builds that were done at least # 'self.prune_days' ago. And because we run prunerepo _daily_ # we know that the candidates for removal (if there are such) # are removed about a day after "build_time + self.prune_days". touched_before = time.time()-os.stat(chroot_path).st_mtime touched_before = touched_before/3600/24 # seconds -> days # Because it might happen that prunerepo has some problems to # successfully go through the directory for some time (bug, user # error, I/O problems...) we rather wait 10 more days till we # really start to ignore the directory. if touched_before > int(self.prune_days) + 10: loginfo("Skipping {} - not changed for {} days".format( sub_dir_name, touched_before)) continue try: cmd = ['prunerepo', '--verbose', '--days', str(self.prune_days), '--nocreaterepo', chroot_path] stdout = runcmd(cmd) loginfo(stdout) createrepo(path=chroot_path, front_url=self.opts.frontend_base_url, username=username, projectname=projectname, override_acr_flag=True) clean_copr(chroot_path, self.prune_days, verbose=True) except Exception as err: logexception(err) logerror("Error pruning chroot {}/{}:{}".format(username, projectdir, sub_dir_name)) loginfo("Pruning done for chroot {}/{}:{}".format(username, projectdir, sub_dir_name)) loginfo("Pruning finished for projectdir {}/{}".format(username, projectdir))