def test_timed_popen(self): time.sleep(2) cmd = ["python", op.join(self.cur_dir, "timed_cli_test.py"), "2"] output = tp.timed_popen(cmd, 1) self.assertTrue(output[-1]) output = tp.timed_popen(cmd, 3) self.assertFalse(output[-1]) self.assertIsNotNone(output[0]) output = tp.timed_popen(cmd) self.assertFalse(output[-1]) self.assertIsNotNone(output[0])
def _get_agents(self): if self.agents: return for ip in (self.ip, self.ip2): if not ip: continue cli = [ ip, "-np", "getagent", "-ver", "-name", "-node", "-rev", "-model", "-type", "-mem", "-serial", "-spid", "-cabinet", "-os", ] output = timed_popen(self.cli_common + cli, 60) if output.count(None) == 2: self._timed_out_count += 1 _logger.error(self._log_template % (ip, "getagent", "timed_out")) continue self._timed_out_count = 0 if output[1]: _logger.error(self._log_template % (ip, "getagent", output[1])) _logger.info("retry with legacy navicli") output = timed_popen(self.legacy_cli_common + cli) if output.count(None) == 2: self._timed_out_count += 1 _logger.error(self._log_template % (ip, "getagent", "timed_out")) continue self._timed_out_count = 0 if output[1]: _logger.error(self._log_template % (ip, "getagent", output[1])) else: self._support_naviseccli = False agent = vbo.Agent(self.site, ip, output[0].split("\n")) if agent.is_valid(): self.agents.append(agent) self._dump(agent, cli, output[0]) else: reason = "stdout:%s stderr:%s" % (output[0], output[1]) _logger.error(self._log_template % (ip, "getagent", reason))
def collect_sp_perf_metrics(self): self._get_agents() if not self.is_valid(): _logger.warn("VNX Block %s is not valid, ignore" % self.ip) return {} _logger.info("start collect sp perf for %s" % self.ip) perf_metrics = [] cli_common = self._get_cli() random.shuffle(self.agents) for agent in self.agents: cli = cli_common + [agent.ip, "-np", "getcontrol", "-all"] output = timed_popen(cli, 60) if output.count(None) == 2: _logger.error(self._log_template % (agent.ip, "getcontrol", "timed_out")) self._timed_out_count += 1 continue self._timed_out_count = 0 if output[1]: _logger.error(self._log_template % (agent.ip, "getcontrol", output[1])) else: self._dump(agent, "getcontrol", output[0]) perf = vbo.ProcessorPerfMetrics(agent, output[0].split("\n")) if perf.is_valid(): perf_metrics.append(perf) else: _logger.error(self._log_template % (agent.ip, "getcontrol", output[0])) _logger.info("end collect sp perf for %s" % self.ip) return {"sp_perf": perf_metrics}
def test_timed_popen(self): run_thr = threading.Thread(target=self.data_loader.run) caped_stdout = _utils.CaptureStdout() with caped_stdout as stdout: run_thr.start() cmd = ["python", "_timed_cli.py", "2"] output = tp.timed_popen(cmd, 1) self.assertIsNone(output[0]) self.assertIsNone(output[1]) output = tp.timed_popen(cmd, 3) self.assertIsNotNone(output[0]) output = tp.timed_popen(cmd) self.assertIsNotNone(output[0]) stdout.clear() self.data_loader.tear_down() run_thr.join()
def _get_nas_control_devices(self, storage_groups, devices): if not self._support_naviseccli: return {"devices": []} sg_devs = set((dev_id for sg in storage_groups for dev_id in sg.device_ids)) all_devs = set((dev.id for dev in devices)) nas_control_devs = sg_devs - all_devs opts = [ "", "-np", "getlun", "-messner", "", "-name", "-capacity", "-private", "-state", "-drivetype", "-owner", "-default", "-rg", "-type", "-uid", "-ismetalun", ] if self._support_virtual_provisioning(): opts.extend(("-isthinlun", "-ispoollun")) _logger.info("start collect nas device inventory for %s" % self.ip) devs = [] random.shuffle(self.agents) for dev_id in nas_control_devs: opts[4] = dev_id for agent in self.agents: opts[0] = agent.ip cli = self.cli_common + opts output = timed_popen(cli, 60) if output.count(None) == 2: self._timed_out_count += 1 _logger.error(self._log_template % (agent.ip, "getlun", "timed_out")) continue self._timed_out_count = 0 if output[1]: _logger.error(self._log_template % (agent.ip, "getlun", output[1])) else: output = "LOGICAL UNIT NUMBER %s\n%s" % (dev_id, output[0]) devs.extend(vbo.parse_block_objects(agent, output, r"^\s*LOGICAL UNIT NUMBER\s+", vbo.Device)) self._dump(agent, "getlun", output) break _logger.info("end collect nas device inventory for %s" % self.ip) return {"devices": devs}
def _do_collect_perf_metrics(self, cmd, interval, svr_name, handler): template = ("export NAS_DB=/nas; /nas/bin/server_stats %s -monitor %s " "-interval %s -count 1 -format csv -terminationsummary no") if svr_name is None: svrs = (svr.name for svr in self._nas_servers if svr.type == "nas" and svr.status == "enabled" and svr.statusactual in ("online, active", "online, ready")) else: svrs = (svr_name,) perf_metrics = [] self._get_system_info() if not self.is_valid(): _logger.warn("system %s is not in valid state, ignore" % self.ip) return perf_metrics timed_out = 0 for svr in svrs: for ip in (self.ip, self._ip2): if ip: cli = template % (svr, cmd, interval) cli = [self.SSH, "%s@%s" % (self._username, ip), cli] output = timed_popen(cli, interval + 60) if output.count(None) == 2: _logger.error(self._log_template % (ip, "server_stats", "timed_out")) timed_out += 1 self._timed_out_count += 1 continue self._timed_out_count = 0 perf = handler(self, svr, output[0]) if isinstance(perf, list): perf_metrics.extend(perf) elif perf.is_valid(): perf_metrics.append(perf) else: _logger.info("Get no result for %s from %s %s" % (cmd, svr, ip)) _logger.debug("Ingore data from %s %s for %s: %s" % (svr, ip, cmd, output[0])) self._dump(cli, output[0]) # if not timed out, break always break if timed_out == 1 and self.ip and self._ip2: timed_out = 0 self.ip, self._ip2 = self._ip2, self.ip _logger.warn("Swapping primary and secondary %s <-> %s" % (self.ip, self._ip2)) return perf_metrics
def _do_collect(self, ip, batch_cmd, handlers, timeout=60): res = {} start_pat = reco(r'<<(\w+)>>') end_pat = reco(r'<</(\w+)>>') cmd = [self.SSH, "%s@%s" % (self._username, ip), batch_cmd] output = timed_popen(cmd, timeout) if output.count(None) == 2: self._timed_out_count += 1 _logger.error(self._log_template % (ip, batch_cmd, "timed_out")) return res self._timed_out_count = 0 current_cmd = None cmd_output = [] for lin in output[0].split("\n"): lin = lin.rstrip() m = end_pat.search(lin) if m: if current_cmd and cmd_output: if current_cmd in handlers: (objname, objs) = handlers[current_cmd](cmd_output) res.setdefault(objname, []).extend(objs) else: _logger.warn("I don't understand this cmd: %s" % current_cmd) del cmd_output[:] current_cmd = None continue m = start_pat.search(lin) if m: current_cmd = m.group(1) continue if current_cmd: cmd_output.append(lin) self._dump(batch_cmd, output[0]) return res
def _do_collect(self, cli_opts, start_tag, StorageClass, timeout=90): self._get_agents() if not self.is_valid(): _logger.warn("VNX Block %s is not valid, ignore" % self.ip) return [] cli_common = self._get_cli() random.shuffle(self.agents) for agent in self.agents: cli_opts[0] = agent.ip cli = cli_common + cli_opts output = timed_popen(cli, timeout) if output.count(None) == 2: self._timed_out_count += 1 _logger.error(self._log_template % (agent.ip, cli_opts[2], "timed_out")) continue self._timed_out_count = 0 if output[1]: _logger.error(self._log_template % (agent.ip, cli_ops[2], output[1])) else: self._dump(agent, cli, output[0]) return vbo.parse_block_objects(agent, output[0], start_tag, StorageClass) return []