def __init__(self, host, port, xprt, max_recv_len = MAX_RECV_LEN, auth=None, auth_opt=None): try: from ovis_ldms import ldms except: raise ImportError("Failed to import ovis_ldms.ldms. " "Please make sure that ldms is built with --enable-swig") else: self.ldms_module = ldms if xprt is None: raise ArgumentError("xprt is required to create an LDMS transport") self.socket = None self.host = host self.port = port self.xprt = xprt self.state = "INIT" if auth: self.ldms = ldms.LDMS_xprt_new_with_auth(self.xprt, auth, auth_opt) else: self.ldms = ldms.LDMS_xprt_new(self.xprt) if not self.ldms: raise ValueError("Failed to create LDMS transport") self.state = "NEW" self.max_recv_len = self.ldms.msg_max_get() self.rc = ldms.LDMS_xprt_connect_by_name(self.ldms, self.host, str(self.port)) if self.rc != 0: raise RuntimeError("Failed to connect to ldmsd. %s" % (self.ldms.event_errcode2str(self.rc))) self.type = "inband" self.state = "CONNECTED"
def test_01_verify(self): """Verify data in the storage""" x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", self.SMP_PORT) if rc: log.error("rc: %d" % rc) assert(rc == 0) dlist = ldms.LDMS_xprt_dir(x) _sets = [] log.info("Looking up sets") self.assertEqual(len(dlist), 1) s = ldms.LDMS_xprt_lookup(x, dlist[0], 0) log.info("Collecting data from LDMS for comparison") data = set() for i in range(0, 10): # update first s.update() l = ldms_set_as_tuple(s) data.add(l) time.sleep(1) time.sleep(1) # to make sure that the last data point has been stored log.info("Verifying...") csv_data = LdmsCsv("test_store_csv/csv") csv_data = set(csv_data) self.assertLessEqual(data, csv_data)
def test_cfg_good(self): cfg = """\ prdcr_add name=%(prdcr)s xprt=%(xprt)s host=%(host)s port=%(port)s \ type=active interval=1000000 prdcr_start name=%(prdcr)s updtr_add name=%(updtr)s interval=1000000 offset=500000 updtr_prdcr_add name=%(updtr)s regex=%(prdcr)s updtr_start name=%(updtr)s """ % { "prdcr": "prdcr", "updtr": "updtr", "xprt": "sock", "host": "localhost", "port": self.SMP_PORT, } daemon = LDMSD(port="10000", auth=self.AUTH, auth_opt=self.LDMSD_AUTH_OPT, cfg=cfg) daemon.run() time.sleep(0.5) xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.LDMSD_AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", "10000") assert (rc == 0) dir_resp = ldms.LDMS_xprt_dir(xprt) daemon.term() self.assertEqual(dir_resp, ["smp/meminfo"])
def test_dir_owner(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertEqual(rc, 0) _dir = ldms.LDMS_xprt_dir(xprt) self.assertEqual(_dir, ["smp/meminfo"])
def test_env(self): os.putenv("LDMS_AUTH_FILE", self.AUTH_OPT["conf"]) xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, None) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertEqual(rc, 0) _dir = ldms.LDMS_xprt_dir(xprt) self.assertEqual(_dir, ["smp/meminfo"])
def __verify(self, lvl, _id, failover=False, empty=False): prdcr = LVX_prdcr(lvl, _id) log.info("Verifying %s" % prdcr) port = LVX_port(lvl, _id) x = ldms.LDMS_xprt_new(XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", str(port)) DEBUG.x = x DEBUG.rc = rc assert (rc == 0) s0 = set() N = 2**lvl off = N * _id if empty: s0 = set() else: s0 = set([LVX_prdcr(0, i) + "/" + s \ for i in range(off, off + N) \ for s in ["meminfo", "vmstat"] ]) if failover: off = N * (_id ^ 1) s0.update([LVX_prdcr(0, i) + "/" + s \ for i in range(off, off + N) \ for s in ["meminfo", "vmstat"] ]) dirs = ldms.LDMS_xprt_dir(x) s1 = set(dirs) DEBUG.s0 = s0 DEBUG.s1 = s1 msg = "ldmsd (%d, %d) verification failed, expecting %s, but got %s" % ( lvl, _id, str(s0), str(s1)) self.assertEqual(s0, s1, msg)
def test_01_verify(self): """Verify data in the storage""" x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", self.SMP_PORT) if rc: log.error("rc: %d" % rc) assert (rc == 0) dlist = ldms.LDMS_xprt_dir(x) self.assertEqual(len(dlist), 1) log.info("Looking up sets") _set = ldms.LDMS_xprt_lookup(x, dlist[0], 0) assert (_set) log.info("Collecting data from LDMS for comparison") data = [] for i in range(0, 10): # update first _set.update() d = ldms_set_as_dict(_set) data.append(d) time.sleep(1) time.sleep(1) # to make sure that the last data point has been stored log.info("Verifying...") keys = data[0].keys() for d in data: self.assertEqual(set(keys), set(d.keys())) for d in self.amqp_sink.data: self.assertEqual(set(keys), set(d.keys())) data = set(tuple_from_dict(d, keys) for d in data) amqp_data = set(tuple_from_dict(d, keys) for d in self.amqp_sink.data) self.assertGreater(len(data), 0) self.assertLessEqual(data, amqp_data)
def test_01_default_with_job(self): """Test default options with jobinfo set""" cfg = """ load name=jobinfo plugin=faux_job config name=jobinfo smplr_add name=smp_job instance=jobinfo interval=2000000 smplr_start name=smp_job load name=clk plugin=clock config name=clk smplr_add name=smp_clk instance=clk interval=2000000 smplr_start name=smp_clk """ % { "host": HOSTNAME, "port": self.SMP_PORT, } daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH, auth_opt = self.LDMSD_AUTH_OPT, cfg = cfg) daemon.run() time.sleep(2.0) xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.LDMSD_AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT) assert(rc == 0) self.__verify(xprt, job = self.PRDCR + "/jobinfo", clk = self.PRDCR + "/clk") daemon.term()
def test_01_verify(self): """Verify data in the storage""" x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", self.SMP_PORT) if rc: log.error("rc: %d" % rc) assert (rc == 0) dlist = ldms.LDMS_xprt_dir(x) _sets = [] log.info("Looking up sets") for name in dlist: s = ldms.LDMS_xprt_lookup(x, name, 0) assert (s) _sets.append(s) log.info("Collecting data from LDMS for comparison") data = set() for i in range(0, 10): # update first for s in _sets: s.update() for s in _sets: l = ldms_set_as_tuple(s, with_ts=True) data.add(l) dlen = len(l) time.sleep(1) time.sleep(1) # to make sure that the last data point has been stored log.info("Verifying...") rf = ResultFile(self.STORE_PATH) # Verify the computed results rf.verify() # Now, verify that the stored raw is good names = [s.metric_name_get(k) for k, v in s.iter_items()] names = ["#Time"] + names csv_data = set(r.as_tuple(names) for r in rf) self.assertLessEqual(data, csv_data)
def test_ls_owner(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertEqual(rc, 0) _set = ldms.LDMS_xprt_lookup(xprt, "smp/meminfo", 0) self.assertIsNotNone(_set)
def test_00_verify_cfg(self): """Verify sampler config, cmd-expand only env command""" host = socket.gethostname() xprt = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", "10001") self.assertEqual(rc, 0) dir_resp = ldms.LDMS_xprt_dir(xprt) self.assertEqual(dir_resp, [host + "/$(whoami)/meminfo"])
def test_dir_owner(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) if rc: raise RuntimeError("LDMS connect failed: %d" % rc) _dir = ldms.LDMS_xprt_dir(xprt) self.assertEqual(set(_dir), set(self.SETS)) ldms.ldms_xprt_close(xprt)
def test_dir_other(self): auth_opt = {"uid": "5555", "gid": "5555"} xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, auth_opt) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) if rc: raise RuntimeError("LDMS connect failed: %d" % rc) r = re.compile("0..6") _sets = [_s for _s, _p in zip(self.SETS, self.PERMS) if r.match(_p)] _dir = ldms.LDMS_xprt_dir(xprt) self.assertEqual(set(_dir), set(_sets)) ldms.ldms_xprt_close(xprt)
def test_lookup_owner(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) if rc: raise RuntimeError("LDMS connect failed: %d" % rc) for _name, _perm in zip(self.SETS, self.PERMS): _set = ldms.LDMS_xprt_lookup(xprt, _name, ldms.LDMS_LOOKUP_BY_INSTANCE) self.assertIsNotNone(_set) ldms.ldms_xprt_close(xprt)
def __init__(self, port, xprt="sock", hostname="localhost"): self.xprt = ldms.LDMS_xprt_new(xprt) rc = ldms.LDMS_xprt_connect_by_name(self.xprt, hostname, port) assert (rc == 0) self.sets = [] self._dict = {} _dirs = ldms.LDMS_xprt_dir(self.xprt) for d in _dirs: s = ldms.LDMS_xprt_lookup(self.xprt, d, ldms.LDMS_LOOKUP_BY_INSTANCE) self.sets.append(s) self._dict[d] = s
def test_tsampler(self): time.sleep(1) # wait for sampler to populate the set x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", self.SMP_PORT) self.assertEqual(rc, 0) s = ldms.LDMS_xprt_lookup(x, "smp/hfclock", 0) s.update() clk = s["clock"] DEBUG.clk = clk tmp = s["clock_timeval"] clk_tv = [ tmp[2*i] + tmp[2*i+1]*1e-6 for i in range(0, len(tmp)/2) ] DEBUG.clk_tv = clk_tv for (a, b) in zip(clk, clk_tv): self.assertGreater(a, 0) self.assertLess(abs(a-b), 0.001)
def test_lookup_other(self): auth_opt = {"uid": "5555", "gid": "5555"} xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, auth_opt) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) if rc: raise RuntimeError("LDMS connect failed: %d" % rc) for _name, _perm in zip(self.SETS, self.PERMS): _set = ldms.LDMS_xprt_lookup(xprt, _name, ldms.LDMS_LOOKUP_BY_INSTANCE) DEBUG.name = _name DEBUG.perm = _perm DEBUG.set = _set if _perm[3] != "0": self.assertIsNotNone(_set) else: self.assertIsNone(_set) ldms.ldms_xprt_close(xprt)
def test_000(self): """Verify that the agg collects from both smp0 and smp1""" x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", str(self.AGG_PORT_BASE)) self.assertEqual(rc, 0) dirs = ldms.LDMS_xprt_dir(x) self.assertEqual( set(dirs), set(["smp%d/meminfo" % i for i in range(0, self.SMP_NUM)])) sets = {d: ldms.LDMS_xprt_lookup(x, d, 0) for d in dirs} for k, s in sets.iteritems(): s.update() grp = re.match(r"smp(\d+)/meminfo", k).groups() comp_id = int(grp[0]) self.assertEqual(s['component_id'], comp_id) ts = s.ts_get() self.assertGreater(ts.sec, 0) pass
def test_00_default(self): """Test default options.""" cfg = """ load name=clk plugin=clock config name=clk smplr_add name=smp_clk instance=clk interval=2000000 smplr_start name=smp_clk """ daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH, auth_opt = self.LDMSD_AUTH_OPT, cfg = cfg) daemon.run() time.sleep(2.0) xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.LDMSD_AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT) assert(rc == 0) self.__verify(xprt, clk = self.PRDCR + "/clk") daemon.term()
def test_01_req_noexp(self): """Request over xprt shall not be command-expanded""" ctrl = LDMSD_Controller(port=self.SMP_PORT, xprt=self.XPRT) ctrl.run() ctrl.read_pty() ctrl.write_pty("env X=$(hostname)\n") ctrl.write_pty("load name=vmstat\n") ctrl.write_pty("config name=vmstat producer=${X} \ instance=${X}/vmstat\ schema=vmstat\n") ctrl.write_pty("start name=vmstat interval=1000000 offset=0\n") time.sleep(0.2) host = socket.gethostname() xprt = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", "10001") self.assertEqual(rc, 0) dir_resp = ldms.LDMS_xprt_dir(xprt) dir_resp.sort() expected = [host + "/$(whoami)/meminfo", "$(hostname)/vmstat"] expected.sort() self.assertEqual(dir_resp, expected)
def test_update(self): x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", self.SMP_PORT) self.assertEqual(rc, 0) s = ldms.LDMS_xprt_lookup(x, self.SET_NAME, 0) time.sleep(self.UPD_INT * uS) log.info("First update ...") ts_list0 = self.__update(s) time.sleep(self.UPD_INT * uS) log.info("Second update ...") ts_list1 = self.__update(s) ts_list = ts_list0 + ts_list1 DEBUG.ts_list = ts_list DEBUG.ts_list0 = ts_list0 DEBUG.ts_list1 = ts_list1 log.info("Verifying data ...") for a, b in zip(ts_list, ts_list[1:]): DEBUG.a = a DEBUG.b = b self.assertLess(a, b) d = b - a self.assertLess(abs(d - self.SMP_INT * uS), 0.001) log.info("%d data timestamps verified" % len(ts_list)) pass
def test_wrong_secret(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, self.OTHER_AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertNotEqual(rc, 0)
def test_02_customized_options(self): """Test customized options""" pass cfg = """ load name=job plugin=faux_job_alt config name=job instance=myjob smplr_add name=smp_job instance=job interval=2000000 offset=0 smplr_start name=smp_job load name=clk plugin=clock config name=clk producer=myprdcr component_id=20 instance=myclk \ uid=2222 \ gid=3333 \ perm=0660 \ job_set=myjob \ job_id=alt_job_id \ app_id=alt_app_id \ job_start=alt_job_start \ job_end=alt_job_end smplr_add name=smp_clk instance=clk interval=2000000 offset=0 smplr_start name=smp_clk """ % { "host": HOSTNAME, "port": self.SMP_PORT, } daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH, auth_opt = self.LDMSD_AUTH_OPT, cfg = cfg) log.info("") log.info("Starting ldmsd") daemon.run() time.sleep(4.0) # verify with uid/gid 2222 log.info("Verifying with uid/gid 2222") xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, {"uid": "2222", "gid": "2222"}) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT) assert(rc == 0) self.__verify(xprt, job="myjob", clk="myclk", component_id=20, job_id="alt_job_id", app_id="alt_app_id", job_start="alt_job_start", job_end="alt_job_end") # verify with uid/gid 3333 log.info("Verifying with uid/gid 3333") xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, {"uid": "3333", "gid": "3333"}) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT) assert(rc == 0) self.__verify(xprt, job="myjob", clk="myclk", component_id=20, job_id="alt_job_id", app_id="alt_app_id", job_start="alt_job_start", job_end="alt_job_end") log.info("Verifying with uid/gid 4444") xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, {"uid": "4444", "gid": "4444"}) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT) assert(rc == 0) # shouldn't see 'myclk' self.__verify(xprt, job="myjob", clk=None, component_id=20, job_id="alt_job_id", app_id="alt_app_id", job_start="alt_job_start", job_end="alt_job_end") # terminate ldmsd daemon.term()
def _get_sets(self, port): x = ldms.LDMS_xprt_new(self.XPRT) rc = ldms.LDMS_xprt_connect_by_name(x, "localhost", port) _dirs = ldms.LDMS_xprt_dir(x) for d in _dirs: pass
def setUpClass(cls): # Need 3 ldmsd .. the config objects are for aggregators log.info("Setting up TestLDMSDPerm") try: # samplers (producers) for prdcr in cls.PRDCRS: smp_cfg = """ load name=meminfo config name=meminfo producer=%(prdcr)s \ instance=%(prdcr)s/meminfo schema=meminfo start name=meminfo interval=1000000 offset=0 """ % prdcr log.debug("smp_cfg: %s" % smp_cfg) ldmsd = LDMSD(port=prdcr["port"], xprt=cls.XPRT, auth=cls.AUTH, auth_opt=cls.LDMSD_AUTH_OPT, cfg=smp_cfg, logfile=prdcr["logfile"]) log.info("starting %s" % prdcr["prdcr"]) ldmsd.run() cls.prdcrs.append(ldmsd) # aggregator cls.agg = LDMSD(port=cls.AGG_PORT, xprt=cls.XPRT, auth=cls.AUTH, auth_opt=cls.LDMSD_AUTH_OPT, logfile=cls.AGG_LOG) log.info("starting aggregator") cls.agg.run() time.sleep(1) # need to config separately so that prdcr,updtr pairs are owned by # different users. log.info("configuring aggregator") for prdcr in cls.PRDCRS: log.info("....adding %(prdcr)s" % prdcr) agg_cfg = """\ prdcr_add name=%(prdcr)s xprt=%(xprt)s host=localhost \ port=%(port)s type=active interval=1000000 \ perm=0600 prdcr_start name=%(prdcr)s updtr_add name=%(updtr)s interval=1000000 offset=500000 \ perm=0600 updtr_prdcr_add name=%(updtr)s regex=%(prdcr)s updtr_start name=%(updtr)s """ % prdcr log.debug("agg_cfg: %s" % agg_cfg) ctrl = ldmsdInbandConfig(host="localhost", port=cls.AGG_PORT, xprt=prdcr["xprt"], auth=prdcr["auth"], auth_opt=prdcr["auth_opt"]) for cmd in agg_cfg.splitlines(): cmd = cmd.strip() if not cmd: continue log.debug("cmd: %s" % cmd) req = LDMSD_Request.from_str(cmd) req.send(ctrl) resp = req.receive(ctrl) errcode = resp["errcode"] if errcode: raise RuntimeError("LDMSD Ctrl errcode: %d" % errcode) ctrl.close() time.sleep(1) # Verify that the agg is working as configured log.info("verifying aggregator") xprt = ldms.LDMS_xprt_new_with_auth(cls.XPRT, cls.AUTH, cls.LDMSD_AUTH_OPT) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", cls.AGG_PORT) if rc: raise RuntimeError("LDMS connect failed: %d" % rc) _dir = ldms.LDMS_xprt_dir(xprt) log.debug("dirs: %s" % str(_dir)) ldms.ldms_xprt_close(xprt) _edirs = [p["prdcr"] + "/meminfo" for p in cls.PRDCRS] if set(_dir) != set(_edirs): raise RuntimeError("Bad set ...") except: del cls.agg del cls.prdcrs raise log.info("TestLDMSDPerm set up done")
def test_wrong_auth(self): xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, "none", None) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertNotEqual(rc, 0)
def test_env_wrong_secret(self): os.putenv("LDMS_AUTH_FILE", self.OTHER_AUTH_OPT["conf"]) xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH, None) rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.PORT) self.assertNotEqual(rc, 0)