예제 #1
0
class TestDispatcher(object):
    def setup_method(self, method):
        self.test_time = time.time()
        subdir = "test_createrepo_{}".format(time.time())
        self.tmp_dir_path = os.path.join(tempfile.gettempdir(), subdir)
        os.mkdir(self.tmp_dir_path)

        self.pkg_pdn = "foobar"
        self.pkg_name = "{}.src.rpm".format(self.pkg_pdn)
        self.pkg_path = os.path.join(self.tmp_dir_path, self.pkg_name)
        with open(self.pkg_path, "w") as handle:
            handle.write("1")

        self.CHROOT = "fedora-20-x86_64"
        self.vm_ip = "192.168.1.2"
        self.vm_name = "VM_{}".format(self.test_time)

        self.DESTDIR = os.path.join(self.tmp_dir_path, COPR_OWNER, COPR_NAME)
        self.DESTDIR_CHROOT = os.path.join(self.DESTDIR, self.CHROOT)
        self.FRONT_URL = "htt://front.example.com"
        self.BASE_URL = "http://example.com/results"
        self.PKG_NAME = "foobar"
        self.PKG_VERSION = "1.2.3"
        self.HOST = "127.0.0.1"
        self.SRC_PKG_URL = "http://example.com/{}-{}.src.rpm".format(
            self.PKG_NAME, self.PKG_VERSION)
        self.job_build_id = 12345
        self.task = {
            "project_owner": COPR_OWNER,
            "project_name": COPR_NAME,
            "pkgs": self.SRC_PKG_URL,
            "repos": "",
            "build_id": self.job_build_id,
            "chroot": self.CHROOT,
        }

        self.spawn_pb = "/spawn.yml"
        self.terminate_pb = "/terminate.yml"
        self.opts = Bunch(
            ssh=Bunch(transport="paramiko"),
            spawn_in_advance=False,
            frontend_url="http://example.com/",
            frontend_auth="12345678",
            build_groups={
                "3": {
                    "spawn_playbook": self.spawn_pb,
                    "terminate_playbook": self.terminate_pb,
                    "name": "3"
                }
            },
            terminate_vars=[],
            fedmsg_enabled=False,
            sleeptime=0.1,
            do_sign=True,
            worker_logdir=self.tmp_dir_path,
            timeout=1800,
            destdir=self.tmp_dir_path,
            results_baseurl="/tmp",
            consecutive_failure_threshold=10,
        )
        self.job = BuildJob(self.task, self.opts)

        self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)

        self.worker_num = 2
        self.group_id = "3"
        self.events = multiprocessing.Queue()
        self.ip = "192.168.1.1"
        self.worker_callback = MagicMock()
        self.worker_fe_callback = MagicMock()
        self.events = multiprocessing.Queue()
        self.logfile_path = os.path.join(self.tmp_dir_path, "test.log")

    @pytest.fixture
    def init_worker(self):
        self.worker = Worker(
            opts=self.opts,
            events=self.events,
            worker_num=self.worker_num,
            group_id=self.group_id,
            callback=self.worker_callback,
        )

        def set_ip(*args, **kwargs):
            self.worker.vm_ip = self.vm_ip

        def erase_ip(*args, **kwargs):
            self.worker.vm_ip = None

        self.set_ip = set_ip

    @pytest.fixture
    def reg_vm(self):
        # call only with init_worker fixture
        self.worker.vm_name = self.vm_name
        self.worker.vm_ip = self.vm_ip

    def teardown_method(self, method):
        # print("\nremove: {}".format(self.tmp_dir_path))
        shutil.rmtree(self.tmp_dir_path)

    def test_init_worker_wo_callback(self):
        worker = Worker(
            opts=self.opts,
            events=self.events,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )
        assert worker.callback

    def test_pkg_built_before(self):
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        target_dir = os.path.join(self.tmp_dir_path, self.CHROOT, self.pkg_pdn)
        os.makedirs(target_dir)
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        with open(os.path.join(target_dir, "fail"), "w") as handle:
            handle.write("undone")
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        os.remove(os.path.join(target_dir, "fail"))
        with open(os.path.join(target_dir, "success"), "w") as handle:
            handle.write("done")
        assert Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                       self.tmp_dir_path)

    def test_spawn_instance_with_check(self, init_worker):
        self.worker.spawn_instance = MagicMock()

        self.worker.spawn_instance.side_effect = self.set_ip

        self.worker.spawn_instance_with_check()
        assert self.vm_ip == self.worker.vm_ip

    def test_spawn_instance_with_check_no_ip(self, init_worker):
        self.worker.spawn_instance = MagicMock()

        with pytest.raises(CoprWorkerError):
            self.worker.spawn_instance_with_check()

    def test_spawn_instance_with_check_ansible_error_reraised(
            self, init_worker, mc_register_build_result):
        self.worker.spawn_instance = MagicMock()
        self.worker.spawn_instance.side_effect = AnsibleError("foobar")

        # with pytest.raises():
        with pytest.raises(AnsibleError):
            self.worker.spawn_instance_with_check()

    def test_spawn_instance_missing_playbook_for_group_id(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.validate_vm = MagicMock()
        self.worker.group_id = "175"

        self.worker.spawn_instance()
        assert self.worker.vm_ip is None
        assert not self.worker.try_spawn.called
        assert not self.worker.validate_vm.called

    def test_spawn_instance_ok_immediately(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip
        assert self.worker.try_spawn.called
        assert self.worker.validate_vm.called

    def test_spawn_instance_error_once_try_spawn(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.side_effect = [
            CoprWorkerSpawnFailError("foobar"), self.vm_ip
        ]
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip

        assert len(self.worker.try_spawn.call_args_list) == 2
        assert len(self.worker.validate_vm.call_args_list) == 1

    def test_spawn_instance_error_once_validate_new_vm(self, init_worker):
        self.worker.run_ansible_playbook = MagicMock()

        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()
        self.worker.validate_vm.side_effect = [
            CoprWorkerSpawnFailError("foobar"),
            None,
        ]

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip

        assert len(self.worker.try_spawn.call_args_list) == 2
        assert len(self.worker.validate_vm.call_args_list) == 2

    def test_spawn_instance_ok_immediately_passed_args(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()

        assert self.worker.try_spawn.called
        assert self.worker.validate_vm.called

        assert self.worker.try_spawn.call_args == mock.call(
            self.try_spawn_args)

    def test_try_spawn_ansible_error_no_result(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = None

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.try_spawn(self.try_spawn_args)

    def test_try_spawn_ansible_ok_no_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = "foobar IP={}".format(self.vm_ip)

        assert self.worker.try_spawn(self.try_spawn_args) == self.vm_ip

    def test_try_spawn_ansible_ok_with_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = "foobar \"IP={}\" adsf \"vm_name={}\"".format(
            self.vm_ip, self.vm_name)

        assert self.worker.try_spawn(self.try_spawn_args) == self.vm_ip
        assert self.worker.vm_name == self.vm_name

    def test_try_spawn_ansible_bad_ip_no_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        for bad_ip in ["256.0.0.2", "not-an-ip", "example.com", ""]:
            mc_run_ans.return_value = "foobar IP={}".format(bad_ip)

            with pytest.raises(CoprWorkerSpawnFailError):
                self.worker.try_spawn(self.try_spawn_args)

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm(self, mc_runner, init_worker, reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.return_value = {"contacted": {self.vm_ip: "ok"}}
        mc_runner.return_value = mc_ans_conn

        self.worker.validate_vm()
        assert mc_ans_conn.run.called

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm_ans_error(self, mc_runner, init_worker, reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.side_effect = IOError()
        mc_runner.return_value = mc_ans_conn

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.validate_vm()
        assert mc_ans_conn.run.called

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm_bad_response(self, mc_runner, init_worker,
                                          reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.return_value = {"contacted": {}}
        mc_runner.return_value = mc_ans_conn

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.validate_vm()
        assert mc_ans_conn.run.called

    def test_terminate_instance(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans

        self.worker.terminate_instance()
        assert mc_run_ans.called
        expected_call = mock.call("-c ssh {} ".format(self.terminate_pb),
                                  'terminate instance')
        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.opts.terminate_vars = ["vm_name"]

        self.worker.terminate_instance()
        assert mc_run_ans.called
        expected_call = mock.call(
            '-c ssh {} --extra-vars=\'{{"copr_task": {{"vm_name": "{}"}}}}\''.
            format(self.terminate_pb, self.vm_name), 'terminate instance')

        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.opts.terminate_vars = ["ip", "vm_name"]

        self.worker.terminate_instance()
        assert mc_run_ans.called

        expected_call = mock.call(
            '-c ssh {} --extra-vars=\''
            '{{"copr_task": {{"vm_name": "{}", "ip": "{}"}}}}\''.format(
                self.terminate_pb, self.vm_name, self.vm_ip),
            'terminate instance')

        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.worker.group_id = "322"

        with pytest.raises(SystemExit):
            self.worker.terminate_instance()
        assert not mc_run_ans.called

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"

        self.worker.opts.fedmsg_enabled = True
        self.worker.event(topic, template, content)
        el = self.worker.events.get()

        assert el["who"] == "worker-2"
        assert el["what"] == "foo: foo, bar: bar"

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event_error(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"
        mc_fedmsg.publish.side_effect = IOError()

        self.worker.opts.fedmsg_enabled = True
        self.worker.event(topic, template, content)
        el = self.worker.events.get()

        assert el["who"] == "worker-2"
        assert el["what"] == "foo: foo, bar: bar"

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event_disable_fedmsg(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"
        mc_fedmsg.publish.side_effect = IOError()

        self.worker.event(topic, template, content)
        assert not mc_fedmsg.publish.called

    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_first_try_ok(self, mc_subprocess,
                                               init_worker):
        exptected_result = "ok"
        mc_subprocess.check_output.return_value = exptected_result

        assert self.worker.run_ansible_playbook(
            self.try_spawn_args) == exptected_result

        assert mc_subprocess.check_output.called_once
        assert mc_subprocess.check_output.call_args == mock.call(
            'ansible-playbook -c ssh /spawn.yml', shell=True)

    @mock.patch("backend.daemons.dispatcher.time")
    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_first_second_ok(self, mc_subprocess, mc_time,
                                                  init_worker, capsys):
        expected_result = "ok"
        mc_subprocess.check_output.side_effect = [
            CalledProcessError(1, ""),
            expected_result,
        ]

        self.worker.run_ansible_playbook(self.try_spawn_args)
        stdout, stderr = capsys.readouterr()
        assert len(mc_subprocess.check_output.call_args_list) == 2

    @mock.patch("backend.daemons.dispatcher.time")
    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_all_attempts_failed(self, mc_subprocess,
                                                      mc_time, init_worker,
                                                      capsys):
        expected_result = "ok"
        mc_subprocess.check_output.side_effect = [
            CalledProcessError(1, ""),
            CalledProcessError(1, ""),
            expected_result,
        ]

        assert self.worker.run_ansible_playbook(self.try_spawn_args,
                                                attempts=2) is None
        assert len(mc_subprocess.check_output.call_args_list) == 2
        stdout, stderr = capsys.readouterr()

    def test_worker_callback(self):
        wc = WorkerCallback(self.logfile_path)

        assert not os.path.exists(self.logfile_path)
        msg = "foobar"
        wc.log(msg)

        with open(self.logfile_path) as handle:
            obtained = handle.read()
            assert msg in obtained

    @mock.patch("backend.daemons.dispatcher.open", create=True)
    def test_worker_callback_error(self, mc_open, capsys):
        wc = WorkerCallback(self.logfile_path)
        mc_open.side_effect = IOError()

        wc.log("foobar")
        stdout, stderr = capsys.readouterr()

        assert "Could not write to logfile" in stderr

        assert not os.path.exists(self.logfile_path)

    def test_mark_started(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.mark_started(self.job)

        expected_call = mock.call({
            'builds': [{
                'status': 3,
                'build_id': self.job_build_id,
                'project_name': 'copr_name',
                'submitter': None,
                'project_owner': 'copr_owner',
                'repos': [],
                'results': u'/tmp/copr_owner/copr_name/',
                'destdir': self.DESTDIR,
                'started_on': None,
                'submitted_on': None,
                'chroot': 'fedora-20-x86_64',
                'ended_on': None,
                'built_packages': '',
                'timeout': 1800,
                'pkg_version': '',
                'pkg_epoch': None,
                'pkg_main_version': '',
                'pkg_release': None,
                'memory_reqs': None,
                'buildroot_pkgs': None,
                'id': self.job_build_id,
                'pkg': self.SRC_PKG_URL,
                "enable_net": True
            }]
        })

        assert expected_call == self.worker_fe_callback.update.call_args

    def test_mark_started_error(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.mark_started(self.job)

    def test_return_results(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.mark_started(self.job)

        expected_call = mock.call({
            'builds': [{
                'status': 3,
                'build_id': self.job_build_id,
                'project_name': 'copr_name',
                'submitter': None,
                'project_owner': 'copr_owner',
                'repos': [],
                'results': u'/tmp/copr_owner/copr_name/',
                'destdir': self.DESTDIR,
                'started_on': self.job.started_on,
                'submitted_on': None,
                'chroot': 'fedora-20-x86_64',
                'ended_on': self.job.ended_on,
                'built_packages': '',
                'timeout': 1800,
                'pkg_version': '',
                'pkg_epoch': None,
                'pkg_main_version': '',
                'pkg_release': None,
                'memory_reqs': None,
                'buildroot_pkgs': None,
                'id': self.job_build_id,
                'pkg': self.SRC_PKG_URL,
                "enable_net": True
            }]
        })

        assert expected_call == self.worker_fe_callback.update.call_args

    def test_return_results_error(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.return_results(self.job)

    def test_starting_builds(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.starting_build(self.job)

        expected_call = mock.call(self.job_build_id, self.CHROOT)
        assert expected_call == self.worker_fe_callback.starting_build.call_args

    def test_starting_build_error(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.starting_build.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.starting_build(self.job)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    @mock.patch("backend.daemons.dispatcher.os")
    def test_do_job_failure_on_mkdirs(self, mc_os, mc_mr, init_worker, reg_vm):
        mc_os.path.exists.return_value = False
        mc_os.makedirs.side_effect = IOError()

        self.worker.frontend_callback = self.worker_fe_callback

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE
        assert not mc_mr.called

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job(self, mc_mr_class, init_worker, reg_vm,
                    mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert os.path.exists(self.DESTDIR_CHROOT)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job_updates_details(self, mc_mr_class, init_worker, reg_vm,
                                    mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)
        mc_mr_class.return_value.build_pkg.return_value = {
            "results": self.test_time,
        }

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert self.job.results == self.test_time
        assert os.path.exists(self.DESTDIR_CHROOT)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job_mr_error(self, mc_mr_class, init_worker, reg_vm,
                             mc_register_build_result):
        mc_mr_class.return_value.build_pkg.side_effect = MockRemoteError(
            "foobar")

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_init_fedmsg(self, mc_fedmsg, init_worker):
        self.worker.init_fedmsg()
        assert not mc_fedmsg.init.called
        self.worker.opts.fedmsg_enabled = True
        self.worker.init_fedmsg()
        assert mc_fedmsg.init.called

        mc_fedmsg.init.side_effect = KeyError()
        self.worker.init_fedmsg()

    def test_obtain_job(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        obtained_job = self.worker.obtain_job()
        assert obtained_job.__dict__ == self.job.__dict__
        assert self.worker.pkg_built_before.called

    def test_obtain_job_skip_pkg(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = True
        self.worker.mark_started = MagicMock()
        self.worker.return_results = MagicMock()

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        assert self.worker.obtain_job() is None
        assert self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_type_error(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.side_effect = TypeError()
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_none_result(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = None
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_on_starting_build(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.starting_build.return_value = False
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        assert self.worker.obtain_job() is None
        assert not self.worker.pkg_built_before.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job

        def validate_not_spawn():
            assert not self.worker.spawn_instance_with_check.called
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_not_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()
        assert mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_spawn_in_advance(self, mc_time, init_worker):
        self.worker.opts.spawn_in_advance = True
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.side_effect = self.set_ip

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job

        def validate_spawn():
            assert self.worker.spawn_instance_with_check.called
            self.worker.spawn_instance_with_check.reset_mock()
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            assert not self.worker.spawn_instance_with_check.called
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_spawn_in_advance_with_existing_vm(self, mc_time, init_worker):
        self.worker.opts.spawn_in_advance = True
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.side_effect = self.set_ip

        self.worker.check_vm_still_alive = MagicMock()

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.side_effect = [
            None,
            self.job,
        ]

        def validate_spawn():
            assert self.worker.spawn_instance_with_check.called
            self.worker.spawn_instance_with_check.reset_mock()
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            assert not self.worker.spawn_instance_with_check.called
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()
        assert self.worker.check_vm_still_alive.called
        assert self.worker.spawn_instance_with_check.called_once

    def test_check_vm_still_alive(self, init_worker):
        self.worker.validate_vm = MagicMock()
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert not self.worker.validate_vm.called
        assert not self.worker.terminate_instance.called

    def test_check_vm_still_alive_ok(self, init_worker, reg_vm):
        self.worker.validate_vm = MagicMock()
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert self.worker.validate_vm.called
        assert not self.worker.terminate_instance.called

    def test_check_vm_still_alive_not_ok(self, init_worker, reg_vm):
        self.worker.validate_vm = MagicMock()
        self.worker.validate_vm.side_effect = CoprWorkerSpawnFailError(
            "foobar")
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert self.worker.validate_vm.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_finalize(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True
            raise IOError()

        mc_do_job.side_effect = stop_loop

        self.worker.run()

        assert mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_no_job(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = None
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True

        mc_time.sleep.side_effect = stop_loop

        self.worker.run()

        assert not mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert not self.worker.terminate_instance.called
예제 #2
0
class TestDispatcher(object):
    def setup_method(self, method):
        self.test_time = time.time()
        subdir = "test_createrepo_{}".format(time.time())
        self.tmp_dir_path = os.path.join(tempfile.gettempdir(), subdir)
        os.mkdir(self.tmp_dir_path)

        self.pkg_pdn = "foobar"
        self.pkg_name = "{}.src.rpm".format(self.pkg_pdn)
        self.pkg_path = os.path.join(self.tmp_dir_path, self.pkg_name)
        with open(self.pkg_path, "w") as handle:
            handle.write("1")

        self.CHROOT = "fedora-20-x86_64"
        self.vm_ip = "192.168.1.2"
        self.vm_name = "VM_{}".format(self.test_time)

        self.DESTDIR = os.path.join(self.tmp_dir_path, COPR_OWNER, COPR_NAME)
        self.DESTDIR_CHROOT = os.path.join(self.DESTDIR, self.CHROOT)
        self.FRONT_URL = "htt://front.example.com"
        self.BASE_URL = "http://example.com/results"
        self.PKG_NAME = "foobar"
        self.PKG_VERSION = "1.2.3"
        self.HOST = "127.0.0.1"
        self.SRC_PKG_URL = "http://example.com/{}-{}.src.rpm".format(
            self.PKG_NAME, self.PKG_VERSION)
        self.job_build_id = 12345

        self.GIT_HASH = "1234r"
        self.GIT_BRANCH = "f20"
        self.GIT_REPO = "foo/bar/xyz"

        self.task = {
            "project_owner": COPR_OWNER,
            "project_name": COPR_NAME,
            "pkgs": self.SRC_PKG_URL,
            "repos": "",
            "build_id": self.job_build_id,
            "chroot": self.CHROOT,
            "task_id": "{}-{}".format(self.job_build_id, self.CHROOT),
            "git_repo": self.GIT_REPO,
            "git_hash": self.GIT_HASH,
            "git_branch": self.GIT_BRANCH,
            "package_name": self.PKG_NAME,
            "package_version": self.PKG_VERSION
        }

        self.spawn_pb = "/spawn.yml"
        self.terminate_pb = "/terminate.yml"
        self.opts = Munch(
            ssh=Munch(transport="paramiko"),
            frontend_base_url="http://example.com",
            frontend_auth="12345678",
            build_groups={
                3: {
                    "spawn_playbook": self.spawn_pb,
                    "terminate_playbook": self.terminate_pb,
                    "name": "3"
                }
            },
            fedmsg_enabled=False,
            sleeptime=0.1,
            do_sign=True,
            timeout=1800,
            destdir=self.tmp_dir_path,
            results_baseurl="/tmp",
            consecutive_failure_threshold=10,
            redis_host="127.0.0.1",
            redis_port=6379,
            redis_db=0,
        )
        self.job = BuildJob(self.task, self.opts)

        self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)

        self.worker_num = 2
        self.group_id = 3

        self.ip = "192.168.1.1"
        self.worker_callback = MagicMock()

        self.logfile_path = os.path.join(self.tmp_dir_path, "test.log")

        self.frontend_client = MagicMock()

    @pytest.yield_fixture
    def mc_vmm(self):
        with mock.patch("{}.VmManager".format(MODULE_REF)) as handle:
            self.vmm = MagicMock()
            handle.return_value = self.vmm
            yield self.vmm

    @pytest.fixture
    def init_worker(self):
        self.worker = Worker(
            opts=self.opts,
            frontend_client=self.frontend_client,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )

        self.worker.vmm = MagicMock()

        def set_ip(*args, **kwargs):
            self.worker.vm_ip = self.vm_ip

        def erase_ip(*args, **kwargs):
            self.worker.vm_ip = None

        self.set_ip = set_ip
        self.erase_ip = erase_ip

    @pytest.fixture
    def reg_vm(self):
        # call only with init_worker fixture
        self.worker.vm_name = self.vm_name
        self.worker.vm_ip = self.vm_ip

    def teardown_method(self, method):
        # print("\nremove: {}".format(self.tmp_dir_path))
        shutil.rmtree(self.tmp_dir_path)

    def test_init_worker_wo_callback(self):
        worker = Worker(
            opts=self.opts,
            frontend_client=self.frontend_client,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )
        worker.vmm = MagicMock()

    def test_pkg_built_before(self):
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        target_dir = os.path.join(self.tmp_dir_path, self.CHROOT, self.pkg_pdn)
        os.makedirs(target_dir)
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        with open(os.path.join(target_dir, "fail"), "w") as handle:
            handle.write("undone")
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                           self.tmp_dir_path)
        os.remove(os.path.join(target_dir, "fail"))
        with open(os.path.join(target_dir, "success"), "w") as handle:
            handle.write("done")
        assert Worker.pkg_built_before(self.pkg_path, self.CHROOT,
                                       self.tmp_dir_path)

    def test_mark_started(self, init_worker):
        self.worker.mark_started(self.job)
        assert self.frontend_client.update.called

    def test_mark_started_error(self, init_worker):
        self.frontend_client.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.mark_started(self.job)

    def test_return_results(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.mark_started(self.job)

        # expected_call = mock.call({'builds': [
        #     {'status': 3, 'build_id': self.job_build_id,
        #      'project_name': 'copr_name', 'submitter': None,
        #      'project_owner': 'copr_owner', 'repos': [],
        #      'results': u'/tmp/copr_owner/copr_name/',
        #      'destdir': self.DESTDIR,
        #      'started_on': self.job.started_on, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
        #      'ended_on': self.job.ended_on, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
        #      'pkg_epoch': None, 'pkg_main_version': '', 'pkg_release': None,
        #      'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
        #      'pkg': self.SRC_PKG_URL, "enable_net": True,
        #      'task_id': self.job.task_id, 'mockchain_macros': {
        #         'copr_username': '******',
        #         'copr_projectname': 'copr_name',
        #         'vendor': 'Fedora Project COPR (copr_owner/copr_name)'}
        #      }
        # ]})

        assert self.frontend_client.update.called

    def test_return_results_error(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10
        self.frontend_client.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.return_results(self.job)

    def test_starting_builds(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.starting_build(self.job)

        # expected_call = mock.call(self.job_build_id, self.CHROOT)
        assert self.frontend_client.starting_build.called

    def test_starting_build_error(self, init_worker):
        self.frontend_client.starting_build.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.starting_build(self.job)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    @mock.patch("backend.daemons.dispatcher.os")
    def test_do_job_failure_on_mkdirs(self, mc_os, mc_mr, init_worker, reg_vm):
        mc_os.path.exists.return_value = False
        mc_os.makedirs.side_effect = IOError()

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE
        assert not mc_mr.called

    def test_do_job(self, mc_mr_class, init_worker, reg_vm,
                    mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert os.path.exists(self.DESTDIR_CHROOT)

    def test_do_job_updates_details(self, mc_mr_class, init_worker, reg_vm,
                                    mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)
        mc_mr_class.return_value.build_pkg_and_process_results.return_value = {
            "results": self.test_time,
        }

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert self.job.results == self.test_time
        assert os.path.exists(self.DESTDIR_CHROOT)

    def test_do_job_mr_error(self, mc_mr_class, init_worker, reg_vm,
                             mc_register_build_result):
        mc_mr_class.return_value.build_pkg_and_process_results.side_effect = MockRemoteError(
            "foobar")

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE

    def test_copy_mock_logs(self, mc_mr_class, init_worker, reg_vm,
                            mc_register_build_result):
        os.makedirs(self.job.results_dir)
        for filename in ["build-00012345.log", "build-00012345.rsync.log"]:
            open(os.path.join(self.job.chroot_dir, filename), "w")

        self.worker.copy_mock_logs(self.job)
        assert set(os.listdir(self.job.results_dir)) == set(
            ["rsync.log.gz", "mockchain.log.gz"])

    def test_copy_mock_logs_missing_files(self, mc_mr_class, init_worker,
                                          reg_vm, mc_register_build_result):
        os.makedirs(self.job.results_dir)
        self.worker.copy_mock_logs(self.job)
        assert set(os.listdir(self.job.results_dir)) == set()

    def test_clean_previous_build_results(self, mc_mr_class, init_worker,
                                          reg_vm, mc_register_build_result):
        os.makedirs(self.job.results_dir)

        files = [
            "fail", "foo.rpm", "build.log.gz", "root.log.gz", "state.log.gz"
        ]
        for filename in files:
            open(os.path.join(self.job.results_dir, filename), "w")

        with open(os.path.join(self.job.results_dir, "build.info"),
                  "w") as build_info:
            build_info.writelines(["build_id=123\n", "builder_ip=<bar>"])

        self.worker.clean_result_directory(self.job)
        backup_dir = os.path.join(
            os.path.join(self.job.results_dir, "prev_build_backup"))
        assert os.path.isdir(backup_dir)
        assert set(os.listdir(backup_dir)) == set(files[2:] + ["build.info"])
        assert "foo.rpm" in os.listdir(self.job.results_dir)

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_init_fedmsg(self, mc_fedmsg, init_worker):
        self.worker.init_fedmsg()
        assert not mc_fedmsg.init.called
        self.worker.opts.fedmsg_enabled = True
        self.worker.init_fedmsg()
        assert mc_fedmsg.init.called

        mc_fedmsg.init.side_effect = KeyError()
        self.worker.init_fedmsg()

    def test_obtain_job(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()

        self.worker.jg.get_build = MagicMock(return_value=self.task)
        obtained_job = self.worker.obtain_job()
        assert obtained_job.__dict__ == self.job.__dict__

    def test_obtain_job_dequeue_type_error(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.side_effect = TypeError()
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_none_result(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = None
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_dummy_run(self, init_worker, mc_time, mc_grc):
        self.worker.init_fedmsg = MagicMock()
        self.worker.run_cycle = MagicMock()
        self.worker.update_process_title = MagicMock()

        def on_run_cycle(*args, **kwargs):
            self.worker.kill_received = True

        self.worker.run_cycle.side_effect = on_run_cycle
        self.worker.run()

        assert self.worker.init_fedmsg.called

        assert mc_grc.called
        assert self.worker.run_cycle.called

    def test_group_name_error(self, init_worker):
        self.opts.build_groups[self.group_id].pop("name")
        assert self.worker.group_name == str(self.group_id)

    def test_update_process_title(self, init_worker, mc_setproctitle):
        self.worker.update_process_title()
        base_title = 'worker-{} {} '.format(self.group_id, self.worker_num)
        assert mc_setproctitle.call_args[0][0] == base_title
        #mc_setproctitle.reset_mock()
        self.worker.vm_ip = self.vm_ip
        self.worker.update_process_title()
        title_with_ip = base_title + "VM_IP={} ".format(self.vm_ip)
        assert mc_setproctitle.call_args[0][0] == title_with_ip

        self.worker.vm_name = self.vm_name
        self.worker.update_process_title()
        title_with_name = title_with_ip + "VM_NAME={} ".format(self.vm_name)
        assert mc_setproctitle.call_args[0][0] == title_with_name

        self.worker.update_process_title("foobar")
        assert mc_setproctitle.call_args[0][0] == title_with_name + "foobar"

    def test_dummy_notify_job_grab_about_task_end(self, init_worker):
        self.worker.rc = MagicMock()
        self.worker.notify_job_grab_about_task_end(self.job)
        expected = json.dumps({
            "action": "remove",
            "build_id": 12345,
            "chroot": "fedora-20-x86_64",
            "task_id": "12345-fedora-20-x86_64"
        })
        assert self.worker.rc.publish.call_args == mock.call(
            JOB_GRAB_TASK_END_PUBSUB, expected)

        self.worker.notify_job_grab_about_task_end(self.job, True)
        expected2 = json.dumps({
            "action": "reschedule",
            "build_id": 12345,
            "chroot": "fedora-20-x86_64",
            "task_id": "12345-fedora-20-x86_64"
        })
        assert self.worker.rc.publish.call_args == mock.call(
            JOB_GRAB_TASK_END_PUBSUB, expected2)

    def test_run_cycle(self, init_worker, mc_time):
        self.worker.update_process_title = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.do_job = MagicMock()
        self.worker.notify_job_grab_about_task_end = MagicMock()

        self.worker.obtain_job.return_value = None
        self.worker.run_cycle()
        assert self.worker.obtain_job.called
        assert mc_time.sleep.called
        assert not mc_time.time.called

        vmd = VmDescriptor(self.vm_ip, self.vm_name, 0, "ready")
        vmd.vm_ip = self.vm_ip
        vmd.vm_name = self.vm_name

        self.worker.obtain_job.return_value = self.job
        self.worker.vmm.acquire_vm.side_effect = [
            IOError(),
            None,
            NoVmAvailable("foobar"),
            vmd,
        ]

        self.worker.run_cycle()
        assert not self.worker.do_job.called
        assert self.worker.notify_job_grab_about_task_end.called_once
        assert self.worker.notify_job_grab_about_task_end.call_args[1][
            "do_reschedule"]
        self.worker.notify_job_grab_about_task_end.reset_mock()

        ###  normal work
        def on_release_vm(*args, **kwargs):
            assert self.worker.vm_ip == self.vm_ip
            assert self.worker.vm_name == self.vm_name

        self.worker.vmm.release_vm.side_effect = on_release_vm
        self.worker.run_cycle()
        assert self.worker.do_job.called_once
        assert self.worker.notify_job_grab_about_task_end.called_once
        assert not self.worker.notify_job_grab_about_task_end.call_args[1].get(
            "do_reschedule")

        assert self.worker.vmm.release_vm.called

        self.worker.vmm.acquire_vm = MagicMock()
        self.worker.vmm.acquire_vm.return_value = vmd

        ### handle VmError
        self.worker.notify_job_grab_about_task_end.reset_mock()
        self.worker.vmm.release_vm.reset_mock()
        self.worker.do_job.side_effect = VmError("foobar")
        self.worker.run_cycle()

        assert self.worker.notify_job_grab_about_task_end.call_args[1][
            "do_reschedule"]
        assert self.worker.vmm.release_vm.called

        ### handle other errors
        self.worker.notify_job_grab_about_task_end.reset_mock()
        self.worker.vmm.release_vm.reset_mock()
        self.worker.do_job.side_effect = IOError()
        self.worker.run_cycle()

        assert self.worker.notify_job_grab_about_task_end.call_args[1][
            "do_reschedule"]
        assert self.worker.vmm.release_vm.called

    def test_run_cycle_halt_on_can_start_job_false(self, init_worker):
        self.worker.notify_job_grab_about_task_end = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job
        self.worker.starting_build = MagicMock()
        self.worker.starting_build.return_value = False
        self.worker.acquire_vm_for_job = MagicMock()

        self.worker.run_cycle()
        assert self.worker.starting_build.called
        assert not self.worker.acquire_vm_for_job.called
예제 #3
0
class TestDispatcher(object):

    def setup_method(self, method):
        self.test_time = time.time()
        subdir = "test_createrepo_{}".format(time.time())
        self.tmp_dir_path = os.path.join(tempfile.gettempdir(), subdir)
        os.mkdir(self.tmp_dir_path)

        self.pkg_pdn = "foobar"
        self.pkg_name = "{}.src.rpm".format(self.pkg_pdn)
        self.pkg_path = os.path.join(self.tmp_dir_path, self.pkg_name)
        with open(self.pkg_path, "w") as handle:
            handle.write("1")

        self.CHROOT = "fedora-20-x86_64"
        self.vm_ip = "192.168.1.2"
        self.vm_name = "VM_{}".format(self.test_time)

        self.DESTDIR = os.path.join(self.tmp_dir_path, COPR_OWNER, COPR_NAME)
        self.DESTDIR_CHROOT = os.path.join(self.DESTDIR, self.CHROOT)
        self.FRONT_URL = "htt://front.example.com"
        self.BASE_URL = "http://example.com/results"
        self.PKG_NAME = "foobar"
        self.PKG_VERSION = "1.2.3"
        self.HOST = "127.0.0.1"
        self.SRC_PKG_URL = "http://example.com/{}-{}.src.rpm".format(self.PKG_NAME, self.PKG_VERSION)
        self.job_build_id = 12345

        self.GIT_HASH = "1234r"
        self.GIT_BRANCH = "f20"
        self.GIT_REPO = "foo/bar/xyz"

        self.task = {
            "project_owner": COPR_OWNER,
            "project_name": COPR_NAME,
            "pkgs": self.SRC_PKG_URL,
            "repos": "",
            "build_id": self.job_build_id,
            "chroot": self.CHROOT,
            "task_id": "{}-{}".format(self.job_build_id, self.CHROOT),

            "git_repo": self.GIT_REPO,
            "git_hash": self.GIT_HASH,
            "git_branch": self.GIT_BRANCH,

            "package_name": self.PKG_NAME,
            "package_version": self.PKG_VERSION
        }

        self.spawn_pb = "/spawn.yml"
        self.terminate_pb = "/terminate.yml"
        self.opts = Munch(
            ssh=Munch(transport="paramiko"),
            frontend_base_url="http://example.com",
            frontend_auth="12345678",
            build_groups={
                3: {
                    "spawn_playbook": self.spawn_pb,
                    "terminate_playbook": self.terminate_pb,
                    "name": "3"
                }
            },

            fedmsg_enabled=False,
            sleeptime=0.1,
            do_sign=True,
            timeout=1800,
            destdir=self.tmp_dir_path,
            results_baseurl="/tmp",

            consecutive_failure_threshold=10,

            redis_host="127.0.0.1",
            redis_port=6379,
            redis_db=0,
        )
        self.job = BuildJob(self.task, self.opts)

        self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)

        self.worker_num = 2
        self.group_id = 3

        self.ip = "192.168.1.1"
        self.worker_callback = MagicMock()

        self.logfile_path = os.path.join(self.tmp_dir_path, "test.log")

        self.frontend_client = MagicMock()

    @pytest.yield_fixture
    def mc_vmm(self):
        with mock.patch("{}.VmManager".format(MODULE_REF)) as handle:
            self.vmm = MagicMock()
            handle.return_value = self.vmm
            yield self.vmm

    @pytest.fixture
    def init_worker(self):
        self.worker = Worker(
            opts=self.opts,
            frontend_client=self.frontend_client,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )

        self.worker.vmm = MagicMock()

        def set_ip(*args, **kwargs):
            self.worker.vm_ip = self.vm_ip

        def erase_ip(*args, **kwargs):
            self.worker.vm_ip = None

        self.set_ip = set_ip
        self.erase_ip = erase_ip

    @pytest.fixture
    def reg_vm(self):
        # call only with init_worker fixture
        self.worker.vm_name = self.vm_name
        self.worker.vm_ip = self.vm_ip

    def teardown_method(self, method):
        # print("\nremove: {}".format(self.tmp_dir_path))
        shutil.rmtree(self.tmp_dir_path)

    def test_init_worker_wo_callback(self):
        worker = Worker(
            opts=self.opts,
            frontend_client=self.frontend_client,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )
        worker.vmm = MagicMock()

    def test_pkg_built_before(self):
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        target_dir = os.path.join(self.tmp_dir_path, self.CHROOT, self.pkg_pdn)
        os.makedirs(target_dir)
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        with open(os.path.join(target_dir, "fail"), "w") as handle:
            handle.write("undone")
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        os.remove(os.path.join(target_dir, "fail"))
        with open(os.path.join(target_dir, "success"), "w") as handle:
            handle.write("done")
        assert Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)

    def test_mark_started(self, init_worker):
        self.worker.mark_started(self.job)
        assert self.frontend_client.update.called

    def test_mark_started_error(self, init_worker):
        self.frontend_client.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.mark_started(self.job)

    def test_return_results(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.mark_started(self.job)

        # expected_call = mock.call({'builds': [
        #     {'status': 3, 'build_id': self.job_build_id,
        #      'project_name': 'copr_name', 'submitter': None,
        #      'project_owner': 'copr_owner', 'repos': [],
        #      'results': u'/tmp/copr_owner/copr_name/',
        #      'destdir': self.DESTDIR,
        #      'started_on': self.job.started_on, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
        #      'ended_on': self.job.ended_on, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
        #      'pkg_epoch': None, 'pkg_main_version': '', 'pkg_release': None,
        #      'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
        #      'pkg': self.SRC_PKG_URL, "enable_net": True,
        #      'task_id': self.job.task_id, 'mockchain_macros': {
        #         'copr_username': '******',
        #         'copr_projectname': 'copr_name',
        #         'vendor': 'Fedora Project COPR (copr_owner/copr_name)'}
        #      }
        # ]})

        assert self.frontend_client.update.called

    def test_return_results_error(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10
        self.frontend_client.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.return_results(self.job)

    def test_starting_builds(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.starting_build(self.job)

        # expected_call = mock.call(self.job_build_id, self.CHROOT)
        assert self.frontend_client.starting_build.called

    def test_starting_build_error(self, init_worker):
        self.frontend_client.starting_build.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.starting_build(self.job)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    @mock.patch("backend.daemons.dispatcher.os")
    def test_do_job_failure_on_mkdirs(self, mc_os, mc_mr, init_worker, reg_vm):
        mc_os.path.exists.return_value = False
        mc_os.makedirs.side_effect = IOError()

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE
        assert not mc_mr.called

    def test_do_job(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert os.path.exists(self.DESTDIR_CHROOT)

    def test_do_job_updates_details(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)
        mc_mr_class.return_value.build_pkg_and_process_results.return_value = {
            "results": self.test_time,
        }

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert self.job.results == self.test_time
        assert os.path.exists(self.DESTDIR_CHROOT)

    def test_do_job_mr_error(self, mc_mr_class, init_worker,
                             reg_vm, mc_register_build_result):
        mc_mr_class.return_value.build_pkg_and_process_results.side_effect = MockRemoteError("foobar")

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE

    def test_copy_mock_logs(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        os.makedirs(self.job.results_dir)
        for filename in ["build-00012345.log", "build-00012345.rsync.log"]:
            open(os.path.join(self.job.chroot_dir, filename), "w")

        self.worker.copy_mock_logs(self.job)
        assert set(os.listdir(self.job.results_dir)) == set(["rsync.log.gz", "mockchain.log.gz"])

    def test_copy_mock_logs_missing_files(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        os.makedirs(self.job.results_dir)
        self.worker.copy_mock_logs(self.job)
        assert set(os.listdir(self.job.results_dir)) == set()

    def test_clean_previous_build_results(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        os.makedirs(self.job.results_dir)

        files = ["fail", "foo.rpm", "build.log.gz", "root.log.gz", "state.log.gz"]
        for filename in files:
            open(os.path.join(self.job.results_dir, filename), "w")

        with open(os.path.join(self.job.results_dir, "build.info"), "w") as build_info:
            build_info.writelines(["build_id=123\n", "builder_ip=<bar>"])

        self.worker.clean_result_directory(self.job)
        backup_dir = os.path.join(os.path.join(self.job.results_dir, "prev_build_backup"))
        assert os.path.isdir(backup_dir)
        assert set(os.listdir(backup_dir)) == set(files[2:] + ["build.info"])
        assert "foo.rpm" in os.listdir(self.job.results_dir)

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_init_fedmsg(self, mc_fedmsg, init_worker):
        self.worker.init_fedmsg()
        assert not mc_fedmsg.init.called
        self.worker.opts.fedmsg_enabled = True
        self.worker.init_fedmsg()
        assert mc_fedmsg.init.called

        mc_fedmsg.init.side_effect = KeyError()
        self.worker.init_fedmsg()

    def test_obtain_job(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()

        self.worker.jg.get_build = MagicMock(return_value=self.task)
        obtained_job = self.worker.obtain_job()
        assert obtained_job.__dict__ == self.job.__dict__

    def test_obtain_job_dequeue_type_error(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.side_effect = TypeError()
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_none_result(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = None
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_dummy_run(self, init_worker, mc_time, mc_grc):
        self.worker.init_fedmsg = MagicMock()
        self.worker.run_cycle = MagicMock()
        self.worker.update_process_title = MagicMock()

        def on_run_cycle(*args, **kwargs):
            self.worker.kill_received = True

        self.worker.run_cycle.side_effect = on_run_cycle
        self.worker.run()

        assert self.worker.init_fedmsg.called

        assert mc_grc.called
        assert self.worker.run_cycle.called

    def test_group_name_error(self, init_worker):
        self.opts.build_groups[self.group_id].pop("name")
        assert self.worker.group_name == str(self.group_id)

    def test_update_process_title(self, init_worker, mc_setproctitle):
        self.worker.update_process_title()
        base_title = 'worker-{} {} '.format(self.group_id, self.worker_num)
        assert mc_setproctitle.call_args[0][0] == base_title
        #mc_setproctitle.reset_mock()
        self.worker.vm_ip = self.vm_ip
        self.worker.update_process_title()
        title_with_ip = base_title + "VM_IP={} ".format(self.vm_ip)
        assert mc_setproctitle.call_args[0][0] == title_with_ip

        self.worker.vm_name = self.vm_name
        self.worker.update_process_title()
        title_with_name = title_with_ip + "VM_NAME={} ".format(self.vm_name)
        assert mc_setproctitle.call_args[0][0] == title_with_name

        self.worker.update_process_title("foobar")
        assert mc_setproctitle.call_args[0][0] == title_with_name + "foobar"

    def test_dummy_notify_job_grab_about_task_end(self, init_worker):
        self.worker.rc = MagicMock()
        self.worker.notify_job_grab_about_task_end(self.job)
        expected = json.dumps({
            "action": "remove",
            "build_id": 12345,
            "chroot": "fedora-20-x86_64",
            "task_id": "12345-fedora-20-x86_64"
        })
        assert self.worker.rc.publish.call_args == mock.call(JOB_GRAB_TASK_END_PUBSUB, expected)

        self.worker.notify_job_grab_about_task_end(self.job, True)
        expected2 = json.dumps({
            "action": "reschedule",
            "build_id": 12345,
            "chroot": "fedora-20-x86_64",
            "task_id": "12345-fedora-20-x86_64"
        })
        assert self.worker.rc.publish.call_args == mock.call(JOB_GRAB_TASK_END_PUBSUB, expected2)

    def test_run_cycle(self, init_worker, mc_time):
        self.worker.update_process_title = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.do_job = MagicMock()
        self.worker.notify_job_grab_about_task_end = MagicMock()

        self.worker.obtain_job.return_value = None
        self.worker.run_cycle()
        assert self.worker.obtain_job.called
        assert mc_time.sleep.called
        assert not mc_time.time.called

        vmd = VmDescriptor(self.vm_ip, self.vm_name, 0, "ready")
        vmd.vm_ip = self.vm_ip
        vmd.vm_name = self.vm_name

        self.worker.obtain_job.return_value = self.job
        self.worker.vmm.acquire_vm.side_effect = [
            IOError(),
            None,
            NoVmAvailable("foobar"),
            vmd,
        ]

        self.worker.run_cycle()
        assert not self.worker.do_job.called
        assert self.worker.notify_job_grab_about_task_end.called_once
        assert self.worker.notify_job_grab_about_task_end.call_args[1]["do_reschedule"]
        self.worker.notify_job_grab_about_task_end.reset_mock()

        ###  normal work
        def on_release_vm(*args, **kwargs):
            assert self.worker.vm_ip == self.vm_ip
            assert self.worker.vm_name == self.vm_name

        self.worker.vmm.release_vm.side_effect = on_release_vm
        self.worker.run_cycle()
        assert self.worker.do_job.called_once
        assert self.worker.notify_job_grab_about_task_end.called_once
        assert not self.worker.notify_job_grab_about_task_end.call_args[1].get("do_reschedule")

        assert self.worker.vmm.release_vm.called

        self.worker.vmm.acquire_vm = MagicMock()
        self.worker.vmm.acquire_vm.return_value = vmd

        ### handle VmError
        self.worker.notify_job_grab_about_task_end.reset_mock()
        self.worker.vmm.release_vm.reset_mock()
        self.worker.do_job.side_effect = VmError("foobar")
        self.worker.run_cycle()

        assert self.worker.notify_job_grab_about_task_end.call_args[1]["do_reschedule"]
        assert self.worker.vmm.release_vm.called

        ### handle other errors
        self.worker.notify_job_grab_about_task_end.reset_mock()
        self.worker.vmm.release_vm.reset_mock()
        self.worker.do_job.side_effect = IOError()
        self.worker.run_cycle()

        assert self.worker.notify_job_grab_about_task_end.call_args[1]["do_reschedule"]
        assert self.worker.vmm.release_vm.called

    def test_run_cycle_halt_on_can_start_job_false(self, init_worker):
        self.worker.notify_job_grab_about_task_end = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job
        self.worker.starting_build = MagicMock()
        self.worker.starting_build.return_value = False
        self.worker.acquire_vm_for_job = MagicMock()

        self.worker.run_cycle()
        assert self.worker.starting_build.called
        assert not self.worker.acquire_vm_for_job.called
예제 #4
0
class TestDispatcher(object):

    def setup_method(self, method):
        self.test_time = time.time()
        subdir = "test_createrepo_{}".format(time.time())
        self.tmp_dir_path = os.path.join(tempfile.gettempdir(), subdir)
        os.mkdir(self.tmp_dir_path)

        self.pkg_pdn = "foobar"
        self.pkg_name = "{}.src.rpm".format(self.pkg_pdn)
        self.pkg_path = os.path.join(self.tmp_dir_path, self.pkg_name)
        with open(self.pkg_path, "w") as handle:
            handle.write("1")

        self.CHROOT = "fedora-20-x86_64"
        self.vm_ip = "192.168.1.2"
        self.vm_name = "VM_{}".format(self.test_time)

        self.DESTDIR = os.path.join(self.tmp_dir_path, COPR_OWNER, COPR_NAME)
        self.DESTDIR_CHROOT = os.path.join(self.DESTDIR, self.CHROOT)
        self.FRONT_URL = "htt://front.example.com"
        self.BASE_URL = "http://example.com/results"
        self.PKG_NAME = "foobar"
        self.PKG_VERSION = "1.2.3"
        self.HOST = "127.0.0.1"
        self.SRC_PKG_URL = "http://example.com/{}-{}.src.rpm".format(self.PKG_NAME, self.PKG_VERSION)
        self.job_build_id = 12345
        self.task = {
            "project_owner": COPR_OWNER,
            "project_name": COPR_NAME,
            "pkgs": self.SRC_PKG_URL,
            "repos": "",
            "build_id": self.job_build_id,
            "chroot": self.CHROOT,
        }

        self.spawn_pb = "/spawn.yml"
        self.terminate_pb = "/terminate.yml"
        self.opts = Bunch(
            ssh=Bunch(transport="paramiko"),
            spawn_in_advance=False,
            frontend_url="http://example.com/",
            frontend_auth="12345678",
            build_groups={
                "3": {
                    "spawn_playbook": self.spawn_pb,
                    "terminate_playbook": self.terminate_pb,
                    "name": "3"
                }
            },
            terminate_vars=[],

            fedmsg_enabled=False,
            sleeptime=0.1,
            do_sign=True,
            worker_logdir=self.tmp_dir_path,
            timeout=1800,
            destdir=self.tmp_dir_path,
            results_baseurl="/tmp",

            consecutive_failure_threshold=10,
        )
        self.job = BuildJob(self.task, self.opts)

        self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb)

        self.worker_num = 2
        self.group_id = "3"
        self.events = multiprocessing.Queue()
        self.ip = "192.168.1.1"
        self.worker_callback = MagicMock()
        self.worker_fe_callback = MagicMock()
        self.events = multiprocessing.Queue()
        self.logfile_path = os.path.join(self.tmp_dir_path, "test.log")

    @pytest.fixture
    def init_worker(self):
        self.worker = Worker(
            opts=self.opts,
            events=self.events,
            worker_num=self.worker_num,
            group_id=self.group_id,
            callback=self.worker_callback,
        )

        def set_ip(*args, **kwargs):
            self.worker.vm_ip = self.vm_ip

        def erase_ip(*args, **kwargs):
            self.worker.vm_ip = None

        self.set_ip = set_ip

    @pytest.fixture
    def reg_vm(self):
        # call only with init_worker fixture
        self.worker.vm_name = self.vm_name
        self.worker.vm_ip = self.vm_ip

    def teardown_method(self, method):
        # print("\nremove: {}".format(self.tmp_dir_path))
        shutil.rmtree(self.tmp_dir_path)

    def test_init_worker_wo_callback(self):
        worker = Worker(
            opts=self.opts,
            events=self.events,
            worker_num=self.worker_num,
            group_id=self.group_id,
        )
        assert worker.callback

    def test_pkg_built_before(self):
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        target_dir = os.path.join(self.tmp_dir_path, self.CHROOT, self.pkg_pdn)
        os.makedirs(target_dir)
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        with open(os.path.join(target_dir, "fail"), "w") as handle:
            handle.write("undone")
        assert not Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)
        os.remove(os.path.join(target_dir, "fail"))
        with open(os.path.join(target_dir, "success"), "w") as handle:
            handle.write("done")
        assert Worker.pkg_built_before(self.pkg_path, self.CHROOT, self.tmp_dir_path)

    def test_spawn_instance_with_check(self, init_worker):
        self.worker.spawn_instance = MagicMock()

        self.worker.spawn_instance.side_effect = self.set_ip

        self.worker.spawn_instance_with_check()
        assert self.vm_ip == self.worker.vm_ip

    def test_spawn_instance_with_check_no_ip(self, init_worker):
        self.worker.spawn_instance = MagicMock()

        with pytest.raises(CoprWorkerError):
            self.worker.spawn_instance_with_check()

    def test_spawn_instance_with_check_ansible_error_reraised(self, init_worker, mc_register_build_result):
        self.worker.spawn_instance = MagicMock()
        self.worker.spawn_instance.side_effect = AnsibleError("foobar")

        # with pytest.raises():
        with pytest.raises(AnsibleError):
            self.worker.spawn_instance_with_check()

    def test_spawn_instance_missing_playbook_for_group_id(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.validate_vm = MagicMock()
        self.worker.group_id = "175"

        self.worker.spawn_instance()
        assert self.worker.vm_ip is None
        assert not self.worker.try_spawn.called
        assert not self.worker.validate_vm.called

    def test_spawn_instance_ok_immediately(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip
        assert self.worker.try_spawn.called
        assert self.worker.validate_vm.called

    def test_spawn_instance_error_once_try_spawn(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.side_effect = [
            CoprWorkerSpawnFailError("foobar"),
            self.vm_ip
        ]
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip

        assert len(self.worker.try_spawn.call_args_list) == 2
        assert len(self.worker.validate_vm.call_args_list) == 1

    def test_spawn_instance_error_once_validate_new_vm(self, init_worker):
        self.worker.run_ansible_playbook = MagicMock()

        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()
        self.worker.validate_vm.side_effect = [
            CoprWorkerSpawnFailError("foobar"),
            None,
        ]

        self.worker.spawn_instance()
        assert self.worker.vm_ip == self.vm_ip

        assert len(self.worker.try_spawn.call_args_list) == 2
        assert len(self.worker.validate_vm.call_args_list) == 2

    def test_spawn_instance_ok_immediately_passed_args(self, init_worker):
        self.worker.try_spawn = MagicMock()
        self.worker.try_spawn.return_value = self.vm_ip
        self.worker.validate_vm = MagicMock()

        self.worker.spawn_instance()

        assert self.worker.try_spawn.called
        assert self.worker.validate_vm.called

        assert self.worker.try_spawn.call_args == mock.call(self.try_spawn_args)

    def test_try_spawn_ansible_error_no_result(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = None

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.try_spawn(self.try_spawn_args)

    def test_try_spawn_ansible_ok_no_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = "foobar IP={}".format(self.vm_ip)

        assert self.worker.try_spawn(self.try_spawn_args) == self.vm_ip

    def test_try_spawn_ansible_ok_with_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        mc_run_ans.return_value = "foobar \"IP={}\" adsf \"vm_name={}\"".format(
            self.vm_ip, self.vm_name)

        assert self.worker.try_spawn(self.try_spawn_args) == self.vm_ip
        assert self.worker.vm_name == self.vm_name

    def test_try_spawn_ansible_bad_ip_no_vm_name(self, init_worker):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        for bad_ip in ["256.0.0.2", "not-an-ip", "example.com", ""]:
            mc_run_ans.return_value = "foobar IP={}".format(bad_ip)

            with pytest.raises(CoprWorkerSpawnFailError):
                self.worker.try_spawn(self.try_spawn_args)

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm(self, mc_runner, init_worker, reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.return_value = {"contacted": {self.vm_ip: "ok"}}
        mc_runner.return_value = mc_ans_conn

        self.worker.validate_vm()
        assert mc_ans_conn.run.called

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm_ans_error(self, mc_runner, init_worker, reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.side_effect = IOError()
        mc_runner.return_value = mc_ans_conn

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.validate_vm()
        assert mc_ans_conn.run.called

    @mock.patch("backend.daemons.dispatcher.ansible.runner.Runner")
    def test_validate_new_vm_bad_response(self, mc_runner, init_worker, reg_vm):
        mc_ans_conn = MagicMock()
        mc_ans_conn.run.return_value = {"contacted": {}}
        mc_runner.return_value = mc_ans_conn

        with pytest.raises(CoprWorkerSpawnFailError):
            self.worker.validate_vm()
        assert mc_ans_conn.run.called

    def test_terminate_instance(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans

        self.worker.terminate_instance()
        assert mc_run_ans.called
        expected_call = mock.call(
            "-c ssh {} ".format(self.terminate_pb),
            'terminate instance')
        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_with_vm_name(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.opts.terminate_vars = ["vm_name"]

        self.worker.terminate_instance()
        assert mc_run_ans.called
        expected_call = mock.call(
            '-c ssh {} --extra-vars=\'{{"copr_task": {{"vm_name": "{}"}}}}\''
            .format(self.terminate_pb, self.vm_name),
            'terminate instance')

        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_with_ip_and_vm_name(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.opts.terminate_vars = ["ip", "vm_name"]

        self.worker.terminate_instance()
        assert mc_run_ans.called

        expected_call = mock.call(
            '-c ssh {} --extra-vars=\''
            '{{"copr_task": {{"vm_name": "{}", "ip": "{}"}}}}\''
            .format(self.terminate_pb, self.vm_name, self.vm_ip),
            'terminate instance')

        assert expected_call == mc_run_ans.call_args
        assert self.worker.vm_ip is None
        assert self.worker.vm_name is None

    def test_terminate_instance_missed_playbook(self, init_worker, reg_vm):
        mc_run_ans = MagicMock()
        self.worker.run_ansible_playbook = mc_run_ans
        self.worker.group_id = "322"

        with pytest.raises(SystemExit):
            self.worker.terminate_instance()
        assert not mc_run_ans.called

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"

        self.worker.opts.fedmsg_enabled = True
        self.worker.event(topic, template, content)
        el = self.worker.events.get()

        assert el["who"] == "worker-2"
        assert el["what"] == "foo: foo, bar: bar"

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event_error(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"
        mc_fedmsg.publish.side_effect = IOError()

        self.worker.opts.fedmsg_enabled = True
        self.worker.event(topic, template, content)
        el = self.worker.events.get()

        assert el["who"] == "worker-2"
        assert el["what"] == "foo: foo, bar: bar"

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_event_disable_fedmsg(self, mc_fedmsg, init_worker):
        template = "foo: {foo}, bar: {bar}"
        content = {"foo": "foo", "bar": "bar"}
        topic = "copr_test"
        mc_fedmsg.publish.side_effect = IOError()

        self.worker.event(topic, template, content)
        assert not mc_fedmsg.publish.called

    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_first_try_ok(self, mc_subprocess, init_worker):
        exptected_result = "ok"
        mc_subprocess.check_output.return_value = exptected_result

        assert self.worker.run_ansible_playbook(self.try_spawn_args) == exptected_result

        assert mc_subprocess.check_output.called_once
        assert mc_subprocess.check_output.call_args == mock.call(
            'ansible-playbook -c ssh /spawn.yml', shell=True)

    @mock.patch("backend.daemons.dispatcher.time")
    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_first_second_ok(self, mc_subprocess,
                                                  mc_time, init_worker, capsys):
        expected_result = "ok"
        mc_subprocess.check_output.side_effect = [
            CalledProcessError(1, ""),
            expected_result,
        ]

        self.worker.run_ansible_playbook(self.try_spawn_args)
        stdout, stderr = capsys.readouterr()
        assert len(mc_subprocess.check_output.call_args_list) == 2

    @mock.patch("backend.daemons.dispatcher.time")
    @mock.patch("backend.daemons.dispatcher.subprocess")
    def test_run_ansible_playbook_all_attempts_failed(self, mc_subprocess,
                                                      mc_time, init_worker, capsys):
        expected_result = "ok"
        mc_subprocess.check_output.side_effect = [
            CalledProcessError(1, ""),
            CalledProcessError(1, ""),
            expected_result,
        ]

        assert self.worker.run_ansible_playbook(self.try_spawn_args, attempts=2) is None
        assert len(mc_subprocess.check_output.call_args_list) == 2
        stdout, stderr = capsys.readouterr()

    def test_worker_callback(self):
        wc = WorkerCallback(self.logfile_path)

        assert not os.path.exists(self.logfile_path)
        msg = "foobar"
        wc.log(msg)

        with open(self.logfile_path) as handle:
            obtained = handle.read()
            assert msg in obtained

    @mock.patch("backend.daemons.dispatcher.open", create=True)
    def test_worker_callback_error(self, mc_open, capsys):
        wc = WorkerCallback(self.logfile_path)
        mc_open.side_effect = IOError()

        wc.log("foobar")
        stdout, stderr = capsys.readouterr()

        assert "Could not write to logfile" in stderr

        assert not os.path.exists(self.logfile_path)

    def test_mark_started(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.mark_started(self.job)

        expected_call = mock.call({'builds': [
            {'status': 3, 'build_id': self.job_build_id,
             'project_name': 'copr_name', 'submitter': None,
             'project_owner': 'copr_owner', 'repos': [],
             'results': u'/tmp/copr_owner/copr_name/',
             'destdir': self.DESTDIR,
             'started_on': None, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
             'ended_on': None, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
             'pkg_epoch': None, 'pkg_main_version': '', 'pkg_release': None,
             'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
             'pkg': self.SRC_PKG_URL, "enable_net": True}
        ]})

        assert expected_call == self.worker_fe_callback.update.call_args

    def test_mark_started_error(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.mark_started(self.job)

    def test_return_results(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.mark_started(self.job)

        expected_call = mock.call({'builds': [
            {'status': 3, 'build_id': self.job_build_id,
             'project_name': 'copr_name', 'submitter': None,
             'project_owner': 'copr_owner', 'repos': [],
             'results': u'/tmp/copr_owner/copr_name/',
             'destdir': self.DESTDIR,
             'started_on': self.job.started_on, 'submitted_on': None, 'chroot': 'fedora-20-x86_64',
             'ended_on': self.job.ended_on, 'built_packages': '', 'timeout': 1800, 'pkg_version': '',
             'pkg_epoch': None, 'pkg_main_version': '', 'pkg_release': None,
             'memory_reqs': None, 'buildroot_pkgs': None, 'id': self.job_build_id,
             'pkg': self.SRC_PKG_URL, "enable_net": True}
        ]})

        assert expected_call == self.worker_fe_callback.update.call_args

    def test_return_results_error(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.update.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.return_results(self.job)

    def test_starting_builds(self, init_worker):
        self.job.started_on = self.test_time
        self.job.ended_on = self.test_time + 10

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.starting_build(self.job)

        expected_call = mock.call(self.job_build_id, self.CHROOT)
        assert expected_call == self.worker_fe_callback.starting_build.call_args

    def test_starting_build_error(self, init_worker):
        self.worker.frontend_callback = self.worker_fe_callback
        self.worker_fe_callback.starting_build.side_effect = IOError()

        with pytest.raises(CoprWorkerError):
            self.worker.starting_build(self.job)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    @mock.patch("backend.daemons.dispatcher.os")
    def test_do_job_failure_on_mkdirs(self, mc_os, mc_mr, init_worker, reg_vm):
        mc_os.path.exists.return_value = False
        mc_os.makedirs.side_effect = IOError()

        self.worker.frontend_callback = self.worker_fe_callback

        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE
        assert not mc_mr.called

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert os.path.exists(self.DESTDIR_CHROOT)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job_updates_details(self, mc_mr_class, init_worker, reg_vm, mc_register_build_result):
        assert not os.path.exists(self.DESTDIR_CHROOT)
        mc_mr_class.return_value.build_pkg.return_value = {
            "results": self.test_time,
        }

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.SUCCEEDED
        assert self.job.results == self.test_time
        assert os.path.exists(self.DESTDIR_CHROOT)

    @mock.patch("backend.daemons.dispatcher.MockRemote")
    def test_do_job_mr_error(self, mc_mr_class, init_worker,
                             reg_vm, mc_register_build_result):
        mc_mr_class.return_value.build_pkg.side_effect = MockRemoteError("foobar")

        self.worker.frontend_callback = self.worker_fe_callback
        self.worker.do_job(self.job)
        assert self.job.status == BuildStatus.FAILURE

    @mock.patch("backend.daemons.dispatcher.fedmsg")
    def test_init_fedmsg(self, mc_fedmsg, init_worker):
        self.worker.init_fedmsg()
        assert not mc_fedmsg.init.called
        self.worker.opts.fedmsg_enabled = True
        self.worker.init_fedmsg()
        assert mc_fedmsg.init.called

        mc_fedmsg.init.side_effect = KeyError()
        self.worker.init_fedmsg()

    def test_obtain_job(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        obtained_job = self.worker.obtain_job()
        assert obtained_job.__dict__ == self.job.__dict__
        assert self.worker.pkg_built_before.called

    def test_obtain_job_skip_pkg(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = True
        self.worker.mark_started = MagicMock()
        self.worker.return_results = MagicMock()

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        assert self.worker.obtain_job() is None
        assert self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_type_error(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.side_effect = TypeError()
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_dequeue_none_result(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = None
        assert self.worker.obtain_job() is None
        assert not self.worker.starting_build.called
        assert not self.worker.pkg_built_before.called

    def test_obtain_job_on_starting_build(self, init_worker):
        mc_tq = MagicMock()
        self.worker.task_queue = mc_tq
        self.worker.starting_build = MagicMock()
        self.worker.starting_build.return_value = False
        self.worker.pkg_built_before = MagicMock()
        self.worker.pkg_built_before.return_value = False

        mc_tq.dequeue.return_value = MagicMock(data=self.task)
        assert self.worker.obtain_job() is None
        assert not self.worker.pkg_built_before.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job

        def validate_not_spawn():
            assert not self.worker.spawn_instance_with_check.called
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_not_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()
        assert mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_spawn_in_advance(self, mc_time, init_worker):
        self.worker.opts.spawn_in_advance = True
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.side_effect = self.set_ip

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job

        def validate_spawn():
            assert self.worker.spawn_instance_with_check.called
            self.worker.spawn_instance_with_check.reset_mock()
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            assert not self.worker.spawn_instance_with_check.called
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_spawn_in_advance_with_existing_vm(self, mc_time, init_worker):
        self.worker.opts.spawn_in_advance = True
        self.worker.init_fedmsg = MagicMock()
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.side_effect = self.set_ip

        self.worker.check_vm_still_alive = MagicMock()

        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.side_effect = [
            None,
            self.job,
        ]

        def validate_spawn():
            assert self.worker.spawn_instance_with_check.called
            self.worker.spawn_instance_with_check.reset_mock()
            return mock.DEFAULT

        self.worker.obtain_job.side_effect = validate_spawn
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            assert not self.worker.spawn_instance_with_check.called
            self.worker.kill_received = True

        mc_do_job.side_effect = stop_loop

        self.worker.run()
        assert self.worker.check_vm_still_alive.called
        assert self.worker.spawn_instance_with_check.called_once

    def test_check_vm_still_alive(self, init_worker):
        self.worker.validate_vm = MagicMock()
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert not self.worker.validate_vm.called
        assert not self.worker.terminate_instance.called

    def test_check_vm_still_alive_ok(self, init_worker, reg_vm):
        self.worker.validate_vm = MagicMock()
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert self.worker.validate_vm.called
        assert not self.worker.terminate_instance.called

    def test_check_vm_still_alive_not_ok(self, init_worker, reg_vm):
        self.worker.validate_vm = MagicMock()
        self.worker.validate_vm.side_effect = CoprWorkerSpawnFailError("foobar")
        self.worker.terminate_instance = MagicMock()

        self.worker.check_vm_still_alive()

        assert self.worker.validate_vm.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_finalize(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = self.job
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True
            raise IOError()

        mc_do_job.side_effect = stop_loop

        self.worker.run()

        assert mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert self.worker.terminate_instance.called

    @mock.patch("backend.daemons.dispatcher.time")
    def test_run_no_job(self, mc_time, init_worker):
        self.worker.init_fedmsg = MagicMock()
        self.worker.obtain_job = MagicMock()
        self.worker.obtain_job.return_value = None
        self.worker.spawn_instance_with_check = MagicMock()
        self.worker.spawn_instance_with_check.return_value = self.vm_ip
        self.worker.terminate_instance = MagicMock()

        mc_do_job = MagicMock()
        self.worker.do_job = mc_do_job

        def stop_loop(*args, **kwargs):
            self.worker.kill_received = True

        mc_time.sleep.side_effect = stop_loop

        self.worker.run()

        assert not mc_do_job.called
        assert self.worker.init_fedmsg.called
        assert self.worker.obtain_job.called
        assert not self.worker.terminate_instance.called