def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups_count=1, build_groups={0: {"name": "base", "archs": ["i386", "x86_64"], "max_vm_per_user": 3}}, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.queue = Queue() self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.rc = get_redis_connection(self.opts) self.ps = None self.log_msg_list = [] self.callback = TestCallback() self.queue = Queue() self.vmm = VmManager(self.opts) self.vmm.post_init() self.vmm.log = MagicMock() self.pid = 12345
def setup_method(self, method): self.test_root_path = tempfile.mkdtemp() self.spawn_pb_path = "{}/spawn.yml".format(self.test_root_path) self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups={0: {"spawn_playbook": self.spawn_pb_path, "name": "base", "archs": ["i386", "x86_64"]}}, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.try_spawn_args = "-c ssh {}".format(self.spawn_pb_path) self.grl_patcher = mock.patch("{}.get_redis_logger".format(MODULE_REF)) self.grl_patcher.start() self.checker = MagicMock() self.terminator = MagicMock() self.spawner = Spawner(self.opts) self.spawner.recycle = types.MethodType(mock.MagicMock, self.spawner) self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.rc = get_redis_connection(self.opts) self.logger = MagicMock()
def do_spawn_and_publish(opts, spawn_playbook, group): log = get_redis_logger(opts, "spawner.detached", "spawner") try: log.debug("Going to spawn") spawn_result = spawn_instance(spawn_playbook, log) log.debug("Spawn finished") except CoprSpawnFailError as err: log.info("Spawning a builder with pb: {}".format(err.msg)) vm_ip = get_ip_from_log(err.msg) vm_name = get_vm_name_from_log(err.msg) if vm_ip and vm_name: # VM started but failed later during ansible run. try: log.exception("Trying to terminate: {}({}).".format(vm_name, vm_ip)) terminate.terminate_vm(opts, opts.build_groups[int(group)]["terminate_playbook"], group, vm_name, vm_ip) except Exception: # ignore all errors raise log.exception("Error during ansible invocation: {}".format(err.msg)) return except Exception as err: log.exception("[Unexpected] Failed to spawn builder: {}".format(err)) return spawn_result["group"] = group spawn_result["topic"] = EventTopics.VM_SPAWNED try: rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(spawn_result)) except Exception as err: log.exception("Failed to publish msg about new VM: {} with error: {}" .format(spawn_result, err))
def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ) rc = get_redis_connection(self.opts) self.channel = rc.pubsub(ignore_subscribe_messages=True) self.channel.subscribe(LOG_PUB_SUB)
def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ) self.rc = get_redis_connection(self.opts) # remove leftovers from previous tests self.rc.delete(LOG_REDIS_FIFO)
def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups={0: {"spawn_playbook": "/spawn.yml", "name": "base", "archs": ["i386", "x86_64"]}}, ) self.executor = Executor(self.opts) self.rc = get_redis_connection(self.opts)
def post_init(self): """ Self configuration. Should be called before usage of some methods. """ self.rc = get_redis_connection(self.opts) self.lua_scripts["set_checking_state"] = self.rc.register_script(set_checking_state_lua) self.lua_scripts["acquire_vm"] = self.rc.register_script(acquire_vm_lua) self.lua_scripts["release_vm"] = self.rc.register_script(release_vm_lua) self.lua_scripts["terminate_vm"] = self.rc.register_script(terminate_vm_lua) self.lua_scripts["mark_vm_check_failed"] = self.rc.register_script(mark_vm_check_failed_lua)
def check_health(opts, vm_name, vm_ip): """ Test connectivity to the VM :param vm_ip: ip address to the newly created VM :raises: :py:class:`~backend.exceptions.CoprWorkerSpawnFailError`: validation fails """ # setproctitle("check VM: {}".format(vm_ip)) log = get_redis_logger(opts, "vmm.check_health.detached", "vmm") runner_options = dict( remote_user=opts.build_user or "root", host_list="{},".format(vm_ip), pattern=vm_ip, forks=1, transport=opts.ssh.transport, timeout=opts.vm_ssh_check_timeout ) connection = Runner(**runner_options) connection.module_name = "shell" connection.module_args = "echo hello" result = { "vm_ip": vm_ip, "vm_name": vm_name, "msg": "", "result": "OK", "topic": EventTopics.HEALTH_CHECK } err_msg = None try: res = connection.run() if vm_ip not in res.get("contacted", {}): err_msg = ( "VM is not responding to the testing playbook." "Runner options: {}".format(runner_options) + "Ansible raw response:\n{}".format(res)) except Exception as error: err_msg = "Failed to check VM ({})due to ansible error: {}".format(vm_ip, error) log.exception(err_msg) try: if err_msg: result["result"] = "failed" result["msg"] = err_msg rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(result)) except Exception as err: log.exception("Failed to publish msg health check result: {} with error: {}" .format(result, err))
def check_health(opts, vm_name, vm_ip): """ Test connectivity to the VM :param vm_ip: ip address to the newly created VM :raises: :py:class:`~backend.exceptions.CoprWorkerSpawnFailError`: validation fails """ # setproctitle("check VM: {}".format(vm_ip)) log = get_redis_logger(opts, "vmm.check_health.detached", "vmm") runner_options = dict(remote_user=opts.build_user or "root", host_list="{},".format(vm_ip), pattern=vm_ip, forks=1, transport=opts.ssh.transport, timeout=opts.vm_ssh_check_timeout) connection = Runner(**runner_options) connection.module_name = "shell" connection.module_args = "echo hello" result = { "vm_ip": vm_ip, "vm_name": vm_name, "msg": "", "result": "OK", "topic": EventTopics.HEALTH_CHECK } err_msg = None try: res = connection.run() if vm_ip not in res.get("contacted", {}): err_msg = ("VM is not responding to the testing playbook." "Runner options: {}".format(runner_options) + "Ansible raw response:\n{}".format(res)) except Exception as error: err_msg = "Failed to check VM ({})due to ansible error: {}".format( vm_ip, error) log.exception(err_msg) try: if err_msg: result["result"] = "failed" result["msg"] = err_msg rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(result)) except Exception as err: log.exception( "Failed to publish msg health check result: {} with error: {}". format(result, err))
def setup_method(self, method): self.test_root_path = tempfile.mkdtemp() self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path) self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch( transport="ssh" ), build_groups={ 0: { "terminate_playbook": self.terminate_pb_path, "name": "base", "archs": ["i386", "x86_64"], "vm_max_check_fails": 2, } }, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.rc = get_redis_connection(self.opts) self.checker = MagicMock() self.spawner = MagicMock() self.terminator = MagicMock() self.queue = Queue() self.vmm = MagicMock() self.vmm.rc = self.rc self.grl_patcher = mock.patch("{}.get_redis_logger".format(MODULE_REF)) self.grl_patcher.start() self.eh = EventHandler(self.opts, self.vmm, self.terminator) self.eh.post_init() self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.msg = {"vm_ip": self.vm_ip, "vm_name": self.vm_name, "group": self.group} self.stage = 0
def setup_method(self, method): self.test_root_path = tempfile.mkdtemp() self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path) self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups={ 0: { "terminate_playbook": self.terminate_pb_path, "name": "base", "archs": ["i386", "x86_64"], "vm_max_check_fails": 2, } }, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.rc = get_redis_connection(self.opts) self.checker = MagicMock() self.spawner = MagicMock() self.terminator = MagicMock() self.queue = Queue() self.vmm = MagicMock() self.vmm.rc = self.rc self.grl_patcher = mock.patch("{}.get_redis_logger".format(MODULE_REF)) self.grl_patcher.start() self.eh = EventHandler(self.opts, self.vmm, self.terminator) self.eh.post_init() self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.msg = { "vm_ip": self.vm_ip, "vm_name": self.vm_name, "group": self.group } self.stage = 0
def __init__(self, opts, logger=None): self.opts = weakref.proxy(opts) self.lua_scripts = {} self.rc = None self.log = logger or get_redis_logger(self.opts, "vmm.lib", "vmm") self.rc = get_redis_connection(self.opts) self.lua_scripts["set_checking_state"] = self.rc.register_script(set_checking_state_lua) self.lua_scripts["acquire_vm"] = self.rc.register_script(acquire_vm_lua) self.lua_scripts["release_vm"] = self.rc.register_script(release_vm_lua) self.lua_scripts["terminate_vm"] = self.rc.register_script(terminate_vm_lua) self.lua_scripts["mark_vm_check_failed"] = self.rc.register_script(mark_vm_check_failed_lua)
def main(): opts = BackendConfigReader().read() conn = get_redis_connection(opts) key = CONSECUTIVE_FAILURE_REDIS_KEY value = int(conn.get(key) or 0) if value > opts.consecutive_failure_threshold: print("Critical") sys.exit(2) elif value > int(0.5 * opts.consecutive_failure_threshold): print("Warning") sys.exit(1) else: print("OK") sys.exit(0)
def post_init(self): """ Self configuration. Should be called before usage of some methods. """ self.rc = get_redis_connection(self.opts) self.lua_scripts["set_checking_state"] = self.rc.register_script( set_checking_state_lua) self.lua_scripts["acquire_vm"] = self.rc.register_script( acquire_vm_lua) self.lua_scripts["release_vm"] = self.rc.register_script( release_vm_lua) self.lua_scripts["terminate_vm"] = self.rc.register_script( terminate_vm_lua) self.lua_scripts["mark_vm_check_failed"] = self.rc.register_script( mark_vm_check_failed_lua)
def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch( transport="ssh" ), build_groups={ 0: { "spawn_playbook": "/spawn.yml", "name": "base", "archs": ["i386", "x86_64"] } } ) self.executor = Executor(self.opts) self.rc = get_redis_connection(self.opts)
def setup_method(self, method): self.test_root_path = tempfile.mkdtemp() self.terminate_pb_path = "{}/terminate.yml".format(self.test_root_path) self.opts = Munch( redis_port=7777, ssh=Munch( transport="ssh" ), build_groups={ 0: { "terminate_playbook": self.terminate_pb_path, "name": "base", "archs": ["i386", "x86_64"], } }, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.grl_patcher = mock.patch("{}.get_redis_logger".format(MODULE_REF)) self.grl_patcher.start() # self.try_spawn_args = '-c ssh {}'.format(self.spawn_pb_path) # self.callback = TestCallback() self.checker = MagicMock() self.terminator = MagicMock() self.terminator = Terminator(self.opts) self.terminator.recycle = types.MethodType(mock.MagicMock, self.terminator) self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.rc = get_redis_connection(self.opts) self.log_msg_list = [] self.logger = MagicMock()
def setup_method(self, method): self.opts = Munch( redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups_count=2, build_groups={ GID1: { "name": "base", "archs": ["i386", "x86_64"], "max_vm_per_user": 3, }, GID2: { "name": "arm", "archs": [ "armV7", ] } }, fedmsg_enabled=False, sleeptime=0.1, do_sign=True, timeout=1800, # destdir=self.tmp_dir_path, results_baseurl="/tmp", ) self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.vm2_ip = "127.0.0.2" self.vm2_name = "localhost2" self.ownername = "bob" self.rc = get_redis_connection(self.opts) self.ps = None self.log_msg_list = [] self.vmm = VmManager(self.opts) self.vmm.log = MagicMock() self.pid = 12345
def __init__(self, opts, logger=None): self.opts = weakref.proxy(opts) self.lua_scripts = {} self.rc = None self.log = logger or get_redis_logger(self.opts, "vmm.lib", "vmm") self.rc = get_redis_connection(self.opts) self.lua_scripts["set_checking_state"] = self.rc.register_script( set_checking_state_lua) self.lua_scripts["acquire_vm"] = self.rc.register_script( acquire_vm_lua) self.lua_scripts["release_vm"] = self.rc.register_script( release_vm_lua) self.lua_scripts["terminate_vm"] = self.rc.register_script( terminate_vm_lua) self.lua_scripts["mark_vm_check_failed"] = self.rc.register_script( mark_vm_check_failed_lua)
def setup_method(self, method): self.redis = get_redis_connection(REDIS_OPTS) self.redis.flushall() self.worker_manager = ToyWorkerManager(redis_connection=self.redis, max_workers=5, log=log) prefix = 'toy:' + str(time.time()) self.worker_manager.worker_prefix = prefix prefix += ':' self.wprefix = prefix self.w0 = prefix + '0' self.w1 = prefix + '1' self.worker_manager.frontend_client = MagicMock() raw_actions = [0, 1, 2, 3, 3, 3, 4, 5, 6, 7, 8, 9] actions = [ActionQueueTask(action) for action in raw_actions] for action in actions: self.worker_manager.add_task(action)
def terminate_vm(opts, terminate_playbook, group, vm_name, vm_ip): """ Call the terminate playbook to destroy the instance """ log = get_redis_logger(opts, "terminator.detached", "terminator") term_args = {"ip": vm_ip, "vm_name": vm_name} args = "-c ssh {} {}".format( # self.vm_ip, terminate_playbook, ans_extra_vars_encode(term_args, "copr_task")) result = { "vm_ip": vm_ip, "vm_name": vm_name, "group": group, "topic": EventTopics.VM_TERMINATED, "result": "OK" } start_time = time.time() try: log.info("starting terminate vm with args: %s", term_args) run_ansible_playbook_cli(args, "terminate instance", log) result["result"] = "OK" except Exception as error: result["result"] = "failed" msg = "Failed to terminate an instance: vm_name={}, vm_ip={}, error: {}".format( vm_name, vm_ip, error) result["msg"] = msg log.exception(msg) try: log.info("VM terminated %s, time elapsed: %s ", term_args, time.time() - start_time) rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(result)) except Exception as error: log.exception("Failed to publish msg about new VM: %s with error: %s", result, error)
def terminate_vm(opts, terminate_playbook, group, vm_name, vm_ip): """ Call the terminate playbook to destroy the instance """ # setproctitle("Terminating VM") log = get_redis_logger(opts, "terminator.detached", "terminator") term_args = {"ip": vm_ip, "vm_name": vm_name} args = "-c ssh {} {}".format( # self.vm_ip, terminate_playbook, ans_extra_vars_encode(term_args, "copr_task")) result = { "vm_ip": vm_ip, "vm_name": vm_name, "group": group, "topic": EventTopics.VM_TERMINATED, "result": "OK" } start_time = time.time() try: log.info("starting terminate vm with args: {}".format(term_args)) run_ansible_playbook_cli(args, "terminate instance", log) result["result"] = "OK" except Exception as error: result["result"] = "failed" msg = "Failed to terminate an instance: vm_name={}, vm_ip={}, error: {}".format(vm_name, vm_ip, error) result["msg"] = msg log.exception(msg) try: log.info("VM terminated {}, time elapsed: {} ".format(term_args, time.time() - start_time)) rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(result)) except Exception as error: log.exception("Failed to publish msg about new VM: {} with error: {}".format(result, error))
def do_spawn_and_publish(opts, spawn_playbook, group): log = get_redis_logger(opts, "spawner.detached", "spawner") try: log.debug("Going to spawn") spawn_result = spawn_instance(spawn_playbook, log) log.debug("Spawn finished") except CoprSpawnFailError as err: log.info("Spawning a builder with pb: {}".format(err.msg)) vm_ip = get_ip_from_log(err.msg) vm_name = get_vm_name_from_log(err.msg) if vm_ip and vm_name: # VM started but failed later during ansible run. try: log.exception("Trying to terminate: {}({}).".format( vm_name, vm_ip)) terminate.terminate_vm( opts, opts.build_groups[int(group)]["terminate_playbook"], group, vm_name, vm_ip) except Exception: # ignore all errors raise log.exception("Error during ansible invocation: {}".format(err.msg)) return except Exception as err: log.exception("[Unexpected] Failed to spawn builder: {}".format(err)) return spawn_result["group"] = group spawn_result["topic"] = EventTopics.VM_SPAWNED try: rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(spawn_result)) except Exception as err: log.exception( "Failed to publish msg about new VM: {} with error: {}".format( spawn_result, err))
def do_spawn_and_publish(opts, spawn_playbook, group): log = get_redis_logger(opts, "spawner.detached", "spawner") try: log.debug("Going to spawn") spawn_result = spawn_instance(spawn_playbook, log) log.debug("Spawn finished") except CoprSpawnFailError as err: log.exception("Failed to spawn builder: {}".format(err)) return except Exception as err: log.exception("[Unexpected] Failed to spawn builder: {}".format(err)) return spawn_result["group"] = group spawn_result["topic"] = EventTopics.VM_SPAWNED try: rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(spawn_result)) except Exception as err: log.exception("Failed to publish msg about new VM: {} with error: {}" .format(spawn_result, err))
def check_health(opts, vm_name, vm_ip): """ Test connectivity to the VM :param vm_ip: ip address to the newly created VM :raises: :py:class:`~backend.exceptions.CoprWorkerSpawnFailError`: validation fails """ log = get_redis_logger(opts, "vmm.check_health.detached", "vmm") result = { "vm_ip": vm_ip, "vm_name": vm_name, "msg": "", "result": "OK", "topic": EventTopics.HEALTH_CHECK } err_msg = None try: conn = SSHConnection(opts.build_user or "root", vm_ip, config_file=opts.ssh.builder_config) rc, stdout, _ = conn.run_expensive("echo hello") if rc != 0 or stdout != "hello\n": err_msg = "Unexpected check output" except Exception as error: err_msg = "Healtcheck failed for VM {} with error {}".format(vm_ip, error) log.exception(err_msg) try: if err_msg: result["result"] = "failed" result["msg"] = err_msg rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(result)) except Exception as err: log.exception("Failed to publish msg health check result: %s with error: %s", result, err)
def do_spawn_and_publish(opts, spawn_playbook, group): log = get_redis_logger(opts, "spawner.detached", "spawner") try: log.debug("Going to spawn") spawn_result = spawn_instance(spawn_playbook, log) log.debug("Spawn finished") except CoprSpawnFailError as err: log.exception("Failed to spawn builder: {}".format(err)) return except Exception as err: log.exception("[Unexpected] Failed to spawn builder: {}".format(err)) return spawn_result["group"] = group spawn_result["topic"] = EventTopics.VM_SPAWNED try: rc = get_redis_connection(opts) rc.publish(PUBSUB_MB, json.dumps(spawn_result)) except Exception as err: log.exception( "Failed to publish msg about new VM: {} with error: {}".format( spawn_result, err))
def setup_method(self, method): self.vm_spawn_min_interval = 30 self.opts = Munch( redis_host="127.0.0.1", redis_db=9, redis_port=7777, ssh=Munch( transport="ssh" ), build_groups_count=2, build_groups={ 0: { "name": "base", "archs": ["i386", "x86_64"], "max_vm_total": 5, "max_spawn_processes": 3, "vm_spawn_min_interval": self.vm_spawn_min_interval, "vm_dirty_terminating_timeout": 120, "vm_health_check_period": 10, "vm_health_check_max_time": 60, "vm_terminating_timeout": 300, }, 1: { "name": "arm", "archs": ["armV7"], "vm_spawn_min_interval": self.vm_spawn_min_interval, "vm_dirty_terminating_timeout": 120, "vm_health_check_period": 10, "vm_health_check_max_time": 60, "vm_terminating_timeout": 300, } }, fedmsg_enabled=False, sleeptime=0.1, vm_cycle_timeout=10, ) self.queue = Queue() self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.rc = get_redis_connection(self.opts) self.ps = None self.log_msg_list = [] self.callback = TestCallback() # checker = HealthChecker(self.opts, self.callback) self.checker = MagicMock() self.spawner = MagicMock() self.terminator = MagicMock() self.mc_logger = MagicMock() self.vmm = VmManager(self.opts, logger=self.mc_logger) self.event_handler = MagicMock() self.vm_master = VmMaster( self.opts, self.vmm, self.spawner, self.checker, ) self.vm_master.event_handler = MagicMock() self.pid = 12345 self.vm_ip = "127.0.0.1" self.vm_name = "build 12345"
def setup_method(self, method): self.vm_spawn_min_interval = 30 self.opts = Munch( redis_host="127.0.0.1", redis_db=9, redis_port=7777, ssh=Munch(transport="ssh"), build_groups_count=2, build_groups={ 0: { "name": "base", "archs": ["i386", "x86_64"], "max_vm_total": 5, "max_spawn_processes": 3, "vm_spawn_min_interval": self.vm_spawn_min_interval, "vm_dirty_terminating_timeout": 120, "vm_health_check_period": 10, "vm_health_check_max_time": 60, "vm_terminating_timeout": 300, }, 1: { "name": "arm", "archs": ["armV7"], "vm_spawn_min_interval": self.vm_spawn_min_interval, "vm_dirty_terminating_timeout": 120, "vm_health_check_period": 10, "vm_health_check_max_time": 60, "vm_terminating_timeout": 300, } }, fedmsg_enabled=False, sleeptime=0.1, vm_cycle_timeout=10, ) self.queue = Queue() self.vm_ip = "127.0.0.1" self.vm_name = "localhost" self.group = 0 self.username = "******" self.rc = get_redis_connection(self.opts) self.ps = None self.log_msg_list = [] self.callback = TestCallback() # checker = HealthChecker(self.opts, self.callback) self.checker = MagicMock() self.spawner = MagicMock() self.terminator = MagicMock() self.mc_logger = MagicMock() self.vmm = VmManager(self.opts, logger=self.mc_logger) self.vmm.post_init() self.event_handler = MagicMock() self.vm_master = VmMaster( self.opts, self.vmm, self.spawner, self.checker, ) self.vm_master.event_handler = MagicMock() self.pid = 12345 self.vm_ip = "127.0.0.1" self.vm_name = "build 12345"