def test_02_ldmsctl(self):
logfile = "ldmsctl.log"
self.smp = LDMSD(port=self.SMP_PORT,
xprt=self.XPRT,
verbose="DEBUG",
logfile=logfile)
self.smp.run()
if not on_timeout(self.is_logfile_ready, filename=logfile):
raise Exception(
"ldmsd log file isn't created within 1 second after ldmsd is started."
)
name_len = self.getMaxMsgSz("ldmsctl")
line = self.getGreetingCfgCmd(name_len)
cfg = tempfile.NamedTemporaryFile()
cfg.write(line)
cfg.file.flush()
# ldmsd_controller subprocess
ctrl = LDMSD_Controller(port=self.SMP_PORT,
xprt=self.XPRT,
source=cfg.name,
ldmsctl=True)
ctrl.run()
msg = self.getLogMsg(name_len)
self.assertTrue(
on_timeout(self.is_msg_in_logfile, filename=logfile, msg=msg))
def test_05_bad_pair(self):
port = LVX_port(1, 0)
pname = LVX_prdcr(1, 0)
cfg = """\
failover_config host=localhost port=%(port)d xprt=%(xprt)s \
auto_switch=1 interval=1000000 \
peer_name=%(name)s
failover_start
""" % {
"port": port,
"xprt": XPRT,
"name": pname,
}
p = LDMSD(9999, cfg=cfg, name="bad")
p.run()
time.sleep(4)
ctrl = ldmsdInbandConfig(host="localhost", port=9999, xprt=XPRT)
resp = ctrl.comm("failover_status")
DEBUG.resp = resp
ctrl.close()
obj = json.loads(resp['msg'])
self.assertIn(obj['conn_state'], [
'DISCONNECTED',
'CONNECTING',
'PAIRING',
'PAIRING_RETRY',
])
self.assertEqual(int(obj['flags']['PEERCFG_RECEIVED']), 0)
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
try:
# samplers (producers)
smp_cfg = """
load name=test_sampler
config name=test_sampler producer=smp \
base=set action=default num_sets=%(num_sets)d \
push=%(push)d
start name=test_sampler interval=%(interval_us)d offset=0
""" % {
"interval_us": cls.INTERVAL_US,
"num_sets": cls.NUM_SETS,
"push": cls.SMP_PUSH_EVERY,
}
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port=cls.SMP_PORT,
xprt=cls.XPRT,
cfg=smp_cfg,
logfile=cls.SMP_LOG,
gdb_port=cls.SMP_GDB_PORT)
log.info("starting sampler")
cls.smp.run()
# aggregator
agg_cfg = """
prdcr_add name=prdcr xprt=%(xprt)s \
host=localhost port=%(port)s \
interval=%(interval_us)d type=active
prdcr_start name=prdcr
updtr_add name=updtr push=onpush \
interval=%(interval_us)d
updtr_prdcr_add name=updtr regex=prdcr.*
updtr_start name=updtr
""" % {
"xprt": cls.XPRT,
"port": cls.SMP_PORT,
"interval_us": cls.INTERVAL_US,
}
cls.agg = LDMSD(port=cls.AGG_PORT,
xprt=cls.XPRT,
cfg=agg_cfg,
logfile=cls.AGG_LOG,
gdb_port=cls.AGG_GDB_PORT)
log.info("starting aggregator")
cls.agg.run()
if cls.AGG_GDB_PORT or cls.SMP_GDB_PORT:
log.info("AGG_GDB_PORT: %s" % str(cls.AGG_GDB_PORT))
log.info("SMP_GDB_PORT: %s" % str(cls.SMP_GDB_PORT))
raw_input("Press ENTER to continue after the gdb is attached")
time.sleep(2 * cls.INTERVAL_SEC)
except:
del cls.agg
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(self):
self.expt = None
self.samp = None
self.slurm_set = None # the `slurm` set
self.mem_set = None # the `mem` (meminfo) set
self.conn = None # ldms connection
ldms.ldms_init(512*1024*1024) # 512MB should suffice
try:
cfg = """\
load name=slurm plugin=slurm_sampler
config name=slurm instance=slurm stream=slurm \
job_count=%(JOB_SIZE)d task_count=%(TASK_SIZE)d
load name=mem plugin=meminfo
config name=mem job_set=slurm instance=mem
smplr_add name=mem_smplr instance=mem interval=1000000 offset=0
smplr_start name=mem_smplr
""" % { k: getattr(self, k) for k in dir(self) }
self.samp = LDMSD(port=self.SAMP_PORT, cfg = cfg,
logfile = self.SAMP_LOG,
gdb_port = self.SAMP_GDB_PORT)
D.samp = self.samp
log.info("Starting sampler")
self.samp.run()
self.conn = ldms.LDMS_xprt_new(self.XPRT)
D.conn = self.conn
self.conn.connectByName("localhost", self.SAMP_PORT)
self.slurm_set = self.conn.lookupSet("slurm", 0)
D.slurm_set = self.slurm_set
self.mem_set = self.conn.lookupSet("mem", 0)
D.mem_set = self.mem_set
expt = {
"component_id" : [0L] * self.JOB_SIZE,
"job_id" : [0L] * self.JOB_SIZE,
"app_id" : [0L] * self.JOB_SIZE,
"current_slot" : 0L,
"job_state" : [0L] * self.JOB_SIZE,
"job_tstamp" : [0L] * self.JOB_SIZE,
"job_size" : [0L] * self.JOB_SIZE,
"job_uid" : [0L] * self.JOB_SIZE,
"job_gid" : [0L] * self.JOB_SIZE,
"job_start" : [0L] * self.JOB_SIZE,
"job_end" : [0L] * self.JOB_SIZE,
"node_count" : [0L] * self.JOB_SIZE,
"task_count" : [0L] * self.JOB_SIZE,
}
task_keys = ["task_pid", "task_rank", "task_exit_status"]
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
ldms.ldms_init(256*1024*1024)
try:
# samplers (producers)
smp_cfg = """
load name=hfclock
config name=hfclock producer=smp \
instance=smp/hfclock schema=hfclock \
hfinterval=100000 \
hfcount=10
start name=hfclock interval=1000000 offset=0
"""
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port = cls.SMP_PORT, xprt = cls.XPRT,
cfg = smp_cfg,
logfile = cls.SMP_LOG)
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
# aggregator will be configured later in the test cases
except:
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
try:
# samplers (producers)
smp_cfg = """
load name=meminfo
config name=meminfo producer=smp \
instance=smp/meminfo schema=meminfo
start name=meminfo interval=1000000 offset=0
"""
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port=cls.SMP_PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
cfg=smp_cfg,
logfile=cls.SMP_LOG)
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
# aggregator will be configured later in the test cases
except:
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
cls.smp = None
cls.agg = None
log.info("--- Setting up TestLDMSDSetArray ---")
try:
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
set_array_card=20 \
uid=0 gid=0 perm=0777
start name=meminfo interval=%(interval)d offset=0
""" % {
"interval": cls.SMP_INT,
}
cls.smp = LDMSD(port=cls.SMP_PORT,
cfg=cfg,
logfile=cls.SMP_LOG,
gdb_port=cls.SMP_GDB_PORT)
DEBUG.smp = cls.smp
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
log.info("--- Done setting up TestLDMSDSetArray ---")
except:
cls.tearDownClass()
raise
def setUpClass(cls):
# 1 sampler
log.info("Setting up " + cls.__name__)
try:
# samplers (producers)
smp_cfg = """
env H=$(hostname)
load name=meminfo
config name=meminfo producer=${H} \
instance=${H}/$(whoami)/meminfo schema=meminfo
start name=meminfo interval=1000000 offset=0
"""
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port=cls.SMP_PORT,
xprt=cls.XPRT,
cfg=smp_cfg,
logfile=cls.SMP_LOG)
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
except:
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
log.info("--- Setting up TestLDMSAuthOvis ---")
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
# auth conf
conf_text = "secretword=this_is_a_secret"
f = open(cls.CONF, "w")
f.write(conf_text)
f.close()
os.chmod(cls.CONF, 0600)
# other conf
conf_text = "secretword=this_is_another_secret"
f = open(cls.OTHER_CONF, "w")
f.write(conf_text)
f.close()
os.chmod(cls.OTHER_CONF, 0600)
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
uid=0 gid=0 perm=0600
start name=meminfo interval=1000000
"""
cls.ldmsd = LDMSD(port=cls.PORT,
cfg=cfg,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT)
log.info("starting ldmsd")
cls.ldmsd.run()
time.sleep(1)
log.info("--- Done setting up TestLDMSAuthOvis ---")
def setUpClass(cls):
log.info("Setting up " + cls.__name__)
try:
cls.ldmsd = LDMSD(port=cls.PORT,
xprt=cls.XPRT,
logfile=cls.LOG,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT)
log.info("starting ldmsd")
cls.ldmsd.run()
sleep(1)
cls.ldmsd_interface = LDMSD_Controller(port=cls.PORT,
host=cls.HOST,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT,
ldmsctl=cls.is_ldmsctl)
cls.ldmsd_interface.run()
cls.ldmsd_interface.read_pty(
) # Read the welcome message and the prompt
except:
if cls.ldmsd:
del cls.ldmsd
if cls.ldmsd_interface:
del cls.ldmsd_interface
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
log.info("Setting up TestLDMSAuthNaive")
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
cls.ldmsd = LDMSD(port=cls.PORT, auth=cls.AUTH, auth_opt=cls.AUTH_OPT)
log.info("starting ldmsd")
cls.ldmsd.run()
cls.cfg = ldmsdInbandConfig(host="localhost",
port=cls.PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT)
# NOTE: cls.cfg automatically create an LDMS xprt and connect to the
# target ldmsd.
cmds = []
for _set, _perm in zip(cls.SETS, cls.PERMS):
cmds.append("load name=%s" % _set)
cmds.append("config name=%(set)s producer=test1 instance=%(set)s \
schema=%(set)s component_id=1 \
uid=%(uid)s gid=%(gid)s perm=%(perm)s" % {
"set": _set,
"uid": cls.UID,
"gid": cls.GID,
"perm": _perm,
})
cmds.append("start name=%s interval=1000000" % _set)
log.info("configuring ldmsd over LDMS xprt")
for cmd in cmds:
req = LDMSD_Request.from_str(cmd)
req.send(cls.cfg)
resp = req.receive(cls.cfg)
errcode = resp["errcode"] if "errcode" in resp else 0
if errcode:
raise RuntimeError("LDMSD request errcode: %d" % errcode)
time.sleep(1)
log.info("TestLDMSAuthNaive setup complete")
def setUpClass(cls):
if os.path.exists(cls.STORE_PATH):
os.remove(cls.STORE_PATH)
with open(cls.DERIVED_CONF, "w") as f:
f.write(derived_conf)
try:
smpcfg = """
load name=test plugin=test_sampler
config name=test component_id=20
config name=test action=add_schema schema=sch \
metrics=s0:data:U64:0,s1:data:U64:1,s2:data:U64:2,\
v0:data:U64_ARRAY:0:3,v1:data:U64_ARRAY:1:3,v2:data:U64_ARRAY:2:3
config name=test action=add_set schema=sch \
instance=test
smplr_add name=smp_test instance=test interval=1000000 offset=0
smplr_start name=smp_test
"""
cls.smp = LDMSD(port=cls.SMP_PORT, cfg=smpcfg, logfile=cls.SMP_LOG)
cls.smp.run()
time.sleep(2.0)
aggcfg = """
prdcr_add name=smp type=active xprt=%(XPRT)s port=%(SMP_PORT)s \
host=localhost interval=1000000
prdcr_start name=smp
updtr_add name=upd interval=1000000 offset=500000
updtr_prdcr_add name=upd regex=.*
updtr_start name=upd
load name=csv plugin=store_function_csv
config name=csv path=%(STORE_PATH)s \
derivedconf=%(DERIVED_CONF)s buffer=0
strgp_add name=test_csv container=csv schema=sch
strgp_prdcr_add name=test_csv regex=.*
strgp_start name=test_csv
""" % vars(cls)
cls.agg = LDMSD(port=cls.AGG_PORT, cfg=aggcfg, logfile=cls.AGG_LOG)
cls.agg.run()
time.sleep(4.0) # make sure that it starts storing something
except:
cls.tearDownClass()
raise
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
try:
# samplers (producers)
smp_cfg = """
load name=meminfo
config name=meminfo producer=smp \
instance=smp/meminfo schema=meminfo
start name=meminfo interval=1000000 offset=0
"""
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port=cls.SMP_PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
cfg=smp_cfg,
logfile=cls.SMP_LOG)
log.info("starting sampler")
cls.smp.run()
# aggregator
cls.agg = LDMSD(port=cls.AGG_PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
logfile=cls.AGG_LOG)
log.info("starting aggregator")
cls.agg.run()
time.sleep(1)
# Now, setup the ldmsd_controller subprocess
cls.ctrl = LDMSD_Controller(port=cls.AGG_PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
ldmsctl=True)
cls.ctrl.run()
cls.ctrl.read_pty() # discard the welcome message and the prompt
except:
del cls.agg
del cls.smp
del cls.ctrl
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
log.info("Setting up " + cls.__name__)
try:
cls.smp = LDMSD(port = cls.SMP_PORT, xprt = cls.SMP_XPRT,
logfile = cls.LOG, auth = cls.AUTH,
auth_opt = cls.AUTH_OPT, host_name = cls.SMP_NAME)
log.info("starting sampler")
cls.smp.run()
agg1_cfg = """
prdcr_add name=samplerd host=%(host)s xprt=%(xprt)s port=%(port)s type=active interval=20000000
prdcr_start name=samplerd""" % {
"host": cls.SMP_HOST,
"xprt": cls.SMP_XPRT,
"port": cls.SMP_PORT,
}
cls.agg1 = LDMSD(port = cls.AGG1_PORT, xprt = cls.AGG1_XPRT,
logfile = cls.LOG, auth = cls.AUTH, cfg = agg1_cfg,
auth_opt = cls.AUTH_OPT, host_name = cls.AGG1_NAME)
log.info("Starting aggregator 1")
cls.agg1.run()
agg2_cfg = """
prdcr_add name=agg1 host=%(host)s xprt=%(xprt)s port=%(port)s \
type=active interval=20000000
prdcr_start name=agg1
""" % {
"host": cls.AGG1_HOST,
"xprt": cls.AGG1_XPRT,
"port": cls.AGG1_PORT,
}
cls.agg2 = LDMSD(port = cls.AGG2_PORT, xprt = cls.AGG2_XPRT,
logfile = cls.LOG, auth = cls.AUTH, cfg = agg2_cfg,
auth_opt = cls.AUTH_OPT, host_name = cls.AGG2_NAME)
log.info("Starting aggregator 2")
cls.agg2.run()
sleep(1)
except:
del cls.smp
del cls.agg1
del cls.agg2
raise
log.info(cls.__name__ + " set up done")
def test_00_config_file(self):
name_len = self.getMaxMsgSz("configFile")
logfile = "config_file.log"
smp_cfg = self.getGreetingCfgCmd(name_len)
self.smp = LDMSD(port=self.SMP_PORT,
xprt=self.XPRT,
cfg=smp_cfg,
logfile=logfile,
verbose="DEBUG")
log.info("starting sampler")
self.smp.run()
if not on_timeout(self.is_logfile_ready, filename=logfile):
raise Exception(
"ldmsd log file isn't created within 1 second after ldmsd is started."
)
msg = self.getLogMsg(name_len)
self.assertTrue(
on_timeout(self.is_msg_in_logfile, filename=logfile, msg=msg))
def setUpClass(cls):
if os.path.exists(cls.CSV_PATH):
os.remove(cls.CSV_PATH)
try:
smpcfg = """
load name=test plugin=test_sampler
config name=test component_id=100
config name=test action=add_all metric_array_sz=4 schema=sch
config name=test action=add_set schema=sch instance=test
smplr_add name=smp_test instance=test interval=1000000 offset=0
smplr_start name=smp_test
"""
cls.smp = LDMSD(port = cls.SMP_PORT, cfg = smpcfg,
logfile = cls.SMP_LOG)
cls.smp.run()
time.sleep(2.0)
aggcfg = """
load name=csv plugin=store_csv
config name=csv path=%(CSV_PATH)s
prdcr_add name=smp xprt=%(XPRT)s host=localhost \
port=%(SMP_PORT)s \
type=active interval=1000000
prdcr_start name=smp
updtr_add name=upd interval=1000000 offset=500000
updtr_prdcr_add name=upd regex=.*
updtr_start name=upd
strgp_add name=strgp container=csv schema=sch
strgp_prdcr_add name=strgp regex=.*
strgp_start name=strgp
""" % vars(cls)
cls.agg = LDMSD(port = cls.AGG_PORT, cfg = aggcfg,
logfile = cls.AGG_LOG)
cls.agg.run()
time.sleep(4.0) # make sure that it starts storing something
except:
cls.tearDownClass()
raise
def setUpClass(cls):
try:
smpcfg = """
load name=test plugin=test_sampler
config name=test component_id=100
config name=test action=add_all metric_array_sz=4 \
schema=test
config name=test action=add_set schema=test instance=test
smplr_add name=smp_test instance=test interval=1000000 offset=0
smplr_start name=smp_test
"""
cls.smp = LDMSD(port=cls.SMP_PORT, cfg=smpcfg, logfile=cls.SMP_LOG)
cls.smp.run()
time.sleep(2.0)
aggcfg = """
load name=amqp plugin=store_amqp
config name=amqp host=localhost
prdcr_add name=smp xprt=%(XPRT)s host=localhost \
port=%(SMP_PORT)s type=active interval=1000000
prdcr_start name=smp
updtr_add name=upd interval=1000000 offset=500000
updtr_prdcr_add name=upd regex=.*
updtr_start name=upd
strgp_add name=strgp container=amqp schema=test
strgp_prdcr_add name=strgp regex=.*
strgp_start name=strgp
""" % vars(cls)
cls.agg = LDMSD(port=cls.AGG_PORT, cfg=aggcfg, logfile=cls.AGG_LOG)
cls.agg.run()
time.sleep(4.0) # make sure that it starts storing something
cls.amqp_sink = AMQPSink("LDMS.set.test", "JSON")
cls.amqp_sink.start()
except:
cls.tearDownClass()
raise
def test_cfg_good(self):
cfg = """\
prdcr_add name=%(prdcr)s xprt=%(xprt)s host=%(host)s port=%(port)s \
type=active interval=1000000
prdcr_start name=%(prdcr)s
updtr_add name=%(updtr)s interval=1000000 offset=500000
updtr_prdcr_add name=%(updtr)s regex=%(prdcr)s
updtr_start name=%(updtr)s
""" % {
"prdcr": "prdcr",
"updtr": "updtr",
"xprt": "sock",
"host": "localhost",
"port": self.SMP_PORT,
}
daemon = LDMSD(port="10000",
auth=self.AUTH,
auth_opt=self.LDMSD_AUTH_OPT,
cfg=cfg)
daemon.run()
time.sleep(0.5)
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
self.LDMSD_AUTH_OPT)
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", "10000")
assert (rc == 0)
dir_resp = ldms.LDMS_xprt_dir(xprt)
daemon.term()
self.assertEqual(dir_resp, ["smp/meminfo"])
def test_01_default_with_job(self):
"""Test default options with jobinfo set"""
cfg = """
load name=jobinfo plugin=faux_job
config name=jobinfo
smplr_add name=smp_job instance=jobinfo interval=2000000
smplr_start name=smp_job
load name=clk plugin=clock
config name=clk
smplr_add name=smp_clk instance=clk interval=2000000
smplr_start name=smp_clk
""" % {
"host": HOSTNAME,
"port": self.SMP_PORT,
}
daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH,
auth_opt = self.LDMSD_AUTH_OPT,
cfg = cfg)
daemon.run()
time.sleep(2.0)
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
self.LDMSD_AUTH_OPT)
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT)
assert(rc == 0)
self.__verify(xprt, job = self.PRDCR + "/jobinfo",
clk = self.PRDCR + "/clk")
daemon.term()
def LVX_ldmsd_new(lvl, _id, failover=True, gdb=False):
cfg = LVX_cfg(lvl, _id, failover)
prdcr = LVX_prdcr(lvl, _id)
logfile = LOGDIR + "/" + prdcr
port = LVX_port(lvl, _id)
gdb_port = LVX_gdb_port(lvl, _id) if gdb else None
ldmsd = LDMSD(port=port,
xprt=XPRT,
logfile=logfile,
cfg=cfg,
gdb_port=gdb_port,
name=prdcr,
verbose=VERBOSE)
return ldmsd
def test_cfg_semi_bad(self):
cfg = """\
prdcr_add name=abc bogus=bogus
"""
daemon = LDMSD(port="10000",
auth=self.AUTH,
auth_opt=self.LDMSD_AUTH_OPT,
cfg=cfg)
daemon.run()
time.sleep(1)
# bad config should not terminate the daemon
self.assertTrue(daemon.is_running())
daemon.term()
def setUpClass(cls):
if os.getuid() != 0 or os.getgid() != 0:
raise RuntimeError("This test needs to run as `root`, "
"suggestion: `sudo -E bash` to become root "
"with your current environments, then execute "
"this script.")
usr = pwd.getpwall()[1]
cls.usr = usr
cls.munged = None
cls.ldmsd = None
try:
log.info("--- Setting up %s ---" % cls.__name__)
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
cls.munged = Munged(key=''.join('0' for i in range(0, 1024)))
cls.munged.start()
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
uid=0 gid=0 perm=0600
start name=meminfo interval=1000000
load name=vmstat
config name=vmstat producer=smp instance=smp/vmstat \
schema=vmstat component_id=1 \
uid=%(usr_uid)d gid=%(usr_gid)d perm=0600
start name=vmstat interval=1000000
""" % {
'usr_uid': usr.pw_uid,
'usr_gid': usr.pw_gid,
}
cls.AUTH_OPT = {"socket": cls.munged.socket}
cls.ldmsd = LDMSD(port=cls.PORT,
cfg=cfg,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT,
logfile=DIR + "/ldmsd.log")
log.info("starting ldmsd")
cls.ldmsd.run()
time.sleep(1)
log.info("--- Done setting up %s ---" % cls.__name__)
except Exception, e:
del cls.ldmsd
del cls.munged
raise
def test_lv1_store(self):
# ldmsd_aaggregator conf
shutil.rmtree("csv/csv1", ignore_errors=True)
os.makedirs("csv/csv1")
cfg = """\
prdcr_add name=prdcr xprt=%(xprt)s host=localhost port=%(port)s \
interval=1000000 type=active
prdcr_start name=prdcr
updtr_add name=updtr interval=%(interval)d offset=50000
updtr_prdcr_add name=updtr regex=prdcr
updtr_start name=updtr
load name=store_csv
config name=store_csv action=init path=csv buffer=0
strgp_add name=strgp plugin=store_csv container=csv1 schema=meminfo
strgp_prdcr_add name=strgp regex=prdcr
strgp_start name=strgp
""" % {
"xprt": self.XPRT,
"port": self.SMP_PORT,
"interval": self.AGG_INT,
}
agg = LDMSD(port=self.AGG_PORT,
cfg=cfg,
logfile=self.AGG_LOG,
gdb_port=self.AGG_GDB_PORT)
DEBUG.agg = agg
log.info("starting aggregator")
agg.run()
log.info("collecting data")
time.sleep(2 + 2 * self.AGG_INT * uS)
agg.term()
time.sleep(0.25)
log.info("Verifying Data")
# expecting to see a bunch of data, with dt ~ self.SMP_INT usec
f = open("csv/csv1/meminfo")
lines = f.readlines()
lines = lines[1:] # the [0] is the header
rexp = re.compile("^(\d+\.\d+),.*$")
ts = [float(rexp.match(l).group(1)) for l in lines]
for a, b in zip(ts, ts[1:]):
dt = b - a
# allowing 1 millisec error
self.assertLess(abs(dt - self.SMP_INT * uS), 0.001)
log.info("%d data timestamps verified" % len(ts))
def test_00_default(self):
"""Test default options."""
cfg = """
load name=clk plugin=clock
config name=clk
smplr_add name=smp_clk instance=clk interval=2000000
smplr_start name=smp_clk
"""
daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH,
auth_opt = self.LDMSD_AUTH_OPT,
cfg = cfg)
daemon.run()
time.sleep(2.0)
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
self.LDMSD_AUTH_OPT)
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT)
assert(rc == 0)
self.__verify(xprt, clk = self.PRDCR + "/clk")
daemon.term()
class SlurmTest(unittest.TestCase):
"""A test case for slurm sampler"""
XPRT = "sock"
SAMP_PORT = "10001"
SAMP_LOG = DIR + "/samp.log" # set "<PATH>" to enable sampler logging
SAMP_GDB_PORT = None # set to "20001" and remote-attach for debugging
JOB_SIZE = 4
TASK_SIZE = 8
@classmethod
def setUpClass(self):
self.expt = None
self.samp = None
self.slurm_set = None # the `slurm` set
self.mem_set = None # the `mem` (meminfo) set
self.conn = None # ldms connection
ldms.ldms_init(512*1024*1024) # 512MB should suffice
try:
cfg = """\
load name=slurm plugin=slurm_sampler
config name=slurm instance=slurm stream=slurm \
job_count=%(JOB_SIZE)d task_count=%(TASK_SIZE)d
load name=mem plugin=meminfo
config name=mem job_set=slurm instance=mem
smplr_add name=mem_smplr instance=mem interval=1000000 offset=0
smplr_start name=mem_smplr
""" % { k: getattr(self, k) for k in dir(self) }
self.samp = LDMSD(port=self.SAMP_PORT, cfg = cfg,
logfile = self.SAMP_LOG,
gdb_port = self.SAMP_GDB_PORT)
D.samp = self.samp
log.info("Starting sampler")
self.samp.run()
self.conn = ldms.LDMS_xprt_new(self.XPRT)
D.conn = self.conn
self.conn.connectByName("localhost", self.SAMP_PORT)
self.slurm_set = self.conn.lookupSet("slurm", 0)
D.slurm_set = self.slurm_set
self.mem_set = self.conn.lookupSet("mem", 0)
D.mem_set = self.mem_set
expt = {
"component_id" : [0L] * self.JOB_SIZE,
"job_id" : [0L] * self.JOB_SIZE,
"app_id" : [0L] * self.JOB_SIZE,
"current_slot" : 0L,
"job_state" : [0L] * self.JOB_SIZE,
"job_tstamp" : [0L] * self.JOB_SIZE,
"job_size" : [0L] * self.JOB_SIZE,
"job_uid" : [0L] * self.JOB_SIZE,
"job_gid" : [0L] * self.JOB_SIZE,
"job_start" : [0L] * self.JOB_SIZE,
"job_end" : [0L] * self.JOB_SIZE,
"node_count" : [0L] * self.JOB_SIZE,
"task_count" : [0L] * self.JOB_SIZE,
}
task_keys = ["task_pid", "task_rank", "task_exit_status"]
expt.update({ "%s_%d" % (k, i) : [0L] * self.TASK_SIZE \
for k in task_keys \
for i in range(0, self.JOB_SIZE)
})
self.expt = expt
log.info("--- Done setting up SlurmTest ---")
def test_02_customized_options(self):
"""Test customized options"""
pass
cfg = """
load name=job plugin=faux_job_alt
config name=job instance=myjob
smplr_add name=smp_job instance=job interval=2000000 offset=0
smplr_start name=smp_job
load name=clk plugin=clock
config name=clk producer=myprdcr component_id=20 instance=myclk \
uid=2222 \
gid=3333 \
perm=0660 \
job_set=myjob \
job_id=alt_job_id \
app_id=alt_app_id \
job_start=alt_job_start \
job_end=alt_job_end
smplr_add name=smp_clk instance=clk interval=2000000 offset=0
smplr_start name=smp_clk
""" % {
"host": HOSTNAME,
"port": self.SMP_PORT,
}
daemon = LDMSD(port = self.SMP_PORT, auth = self.AUTH,
auth_opt = self.LDMSD_AUTH_OPT,
cfg = cfg)
log.info("")
log.info("Starting ldmsd")
daemon.run()
time.sleep(4.0)
# verify with uid/gid 2222
log.info("Verifying with uid/gid 2222")
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
{"uid": "2222", "gid": "2222"})
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT)
assert(rc == 0)
self.__verify(xprt, job="myjob", clk="myclk", component_id=20,
job_id="alt_job_id", app_id="alt_app_id",
job_start="alt_job_start", job_end="alt_job_end")
# verify with uid/gid 3333
log.info("Verifying with uid/gid 3333")
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
{"uid": "3333", "gid": "3333"})
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT)
assert(rc == 0)
self.__verify(xprt, job="myjob", clk="myclk", component_id=20,
job_id="alt_job_id", app_id="alt_app_id",
job_start="alt_job_start", job_end="alt_job_end")
log.info("Verifying with uid/gid 4444")
xprt = ldms.LDMS_xprt_new_with_auth(self.XPRT, self.AUTH,
{"uid": "4444", "gid": "4444"})
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost", self.SMP_PORT)
assert(rc == 0)
# shouldn't see 'myclk'
self.__verify(xprt, job="myjob", clk=None, component_id=20,
job_id="alt_job_id", app_id="alt_app_id",
job_start="alt_job_start", job_end="alt_job_end")
# terminate ldmsd
daemon.term()
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
shutil.rmtree(DIR + "/sos", ignore_errors=True)
try:
# samplers (producers)
for i in range(0, cls.SMP_NUM):
smp_cfg = """
load name=meminfo
config name=meminfo producer=smp \
instance=smp%(id)d/meminfo schema=meminfo \
component_id=%(id)d
start name=meminfo interval=1000000 offset=0
""" % {
"id": i,
}
log.debug("smp_cfg: %s" % smp_cfg)
logfile = DIR + ('/%d.log' % i)
port = cls.SMP_PORT_BASE + i
cls.smp[i] = LDMSD(port=str(port),
xprt=cls.XPRT,
cfg=smp_cfg,
logfile=logfile)
log.info("starting sampler %d" % i)
cls.smp[i].run()
time.sleep(1)
except:
cls.tearDownClass()
raise
try:
sio = StringIO() # holding cfg for agg
for i in range(0, cls.SMP_NUM):
prdcr_cfg = """
prdcr_add name=%(prdcr)s xprt=%(xprt)s host=localhost \
port=%(port)d type=active interval=1000000
prdcr_start name=%(prdcr)s
""" % {
"prdcr": "smp%d" % i,
"xprt": cls.XPRT,
"port": cls.SMP_PORT_BASE + i,
}
sio.write(prdcr_cfg)
cfg = """
updtr_add name=all interval=1000000 offset=500000
updtr_prdcr_add name=all regex=.*
updtr_start name=all
load name=store_sos
config name=store_sos path=%(path)s
strgp_add name=all_but_0 plugin=store_sos container=cont \
schema=meminfo
strgp_prdcr_add name=all_but_0 regex=smp[^0]+
strgp_start name=all_but_0
""" % {
"path": DIR + "/sos"
}
sio.write(cfg)
logfile = DIR + "/agg.log"
cfg = sio.getvalue()
cls.agg = LDMSD(
port=str(cls.AGG_PORT_BASE),
xprt=cls.XPRT,
cfg=cfg,
logfile=logfile,
#gdb_port = 21000,
)
cls.agg.run()
time.sleep(2)
except:
cls.tearDownClass()
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
try:
# samplers (producers)
smp_cfg = """
load name=meminfo
config name=meminfo producer=smp \
instance=smp/meminfo schema=meminfo
start name=meminfo interval=%(interval_us)d offset=0
load name=grptest
config name=grptest producer=smp instance=NA schema=NA \
prefix=smp
config name=grptest members=0xF
start name=grptest interval=%(interval_us)d offset=0
""" % {
"interval_us": cls.INTERVAL_US,
}
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port=cls.SMP_PORT,
xprt=cls.XPRT,
cfg=smp_cfg,
logfile=cls.SMP_LOG,
gdb_port=cls.SMP_GDB_PORT)
log.info("starting sampler")
cls.smp.run()
# aggregator
agg_cfg = """
prdcr_add name=prdcr xprt=%(xprt)s \
host=localhost port=%(port)s \
interval=%(interval_us)d type=active
prdcr_start name=prdcr
updtr_add name=updtr interval=%(interval_us)d offset=%(offset)d\
auto_interval=true
updtr_match_add name=updtr regex=.*/grp match=inst
updtr_prdcr_add name=updtr regex=prdcr.*
updtr_start name=updtr
""" % {
"xprt": cls.XPRT,
"port": cls.SMP_PORT,
"interval_us": cls.INTERVAL_US,
"offset": int(cls.INTERVAL_US / 2),
}
cls.agg = LDMSD(port=cls.AGG_PORT,
xprt=cls.XPRT,
cfg=agg_cfg,
logfile=cls.AGG_LOG,
gdb_port=cls.AGG_GDB_PORT)
log.info("starting aggregator")
cls.agg.run()
if cls.AGG_GDB_PORT or cls.SMP_GDB_PORT:
log.info("AGG_GDB_PORT: %s" % str(cls.AGG_GDB_PORT))
log.info("SMP_GDB_PORT: %s" % str(cls.SMP_GDB_PORT))
raw_input("Press ENTER to continue after the gdb is attached")
time.sleep(2 * cls.INTERVAL_SEC)
except:
del cls.agg
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(cls):
# Need 3 ldmsd .. the config objects are for aggregators
log.info("Setting up TestLDMSDPerm")
try:
# samplers (producers)
for prdcr in cls.PRDCRS:
smp_cfg = """
load name=meminfo
config name=meminfo producer=%(prdcr)s \
instance=%(prdcr)s/meminfo schema=meminfo
start name=meminfo interval=1000000 offset=0
""" % prdcr
log.debug("smp_cfg: %s" % smp_cfg)
ldmsd = LDMSD(port=prdcr["port"],
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
cfg=smp_cfg,
logfile=prdcr["logfile"])
log.info("starting %s" % prdcr["prdcr"])
ldmsd.run()
cls.prdcrs.append(ldmsd)
# aggregator
cls.agg = LDMSD(port=cls.AGG_PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.LDMSD_AUTH_OPT,
logfile=cls.AGG_LOG)
log.info("starting aggregator")
cls.agg.run()
time.sleep(1)
# need to config separately so that prdcr,updtr pairs are owned by
# different users.
log.info("configuring aggregator")
for prdcr in cls.PRDCRS:
log.info("....adding %(prdcr)s" % prdcr)
agg_cfg = """\
prdcr_add name=%(prdcr)s xprt=%(xprt)s host=localhost \
port=%(port)s type=active interval=1000000 \
perm=0600
prdcr_start name=%(prdcr)s
updtr_add name=%(updtr)s interval=1000000 offset=500000 \
perm=0600
updtr_prdcr_add name=%(updtr)s regex=%(prdcr)s
updtr_start name=%(updtr)s
""" % prdcr
log.debug("agg_cfg: %s" % agg_cfg)
ctrl = ldmsdInbandConfig(host="localhost",
port=cls.AGG_PORT,
xprt=prdcr["xprt"],
auth=prdcr["auth"],
auth_opt=prdcr["auth_opt"])
for cmd in agg_cfg.splitlines():
cmd = cmd.strip()
if not cmd:
continue
log.debug("cmd: %s" % cmd)
req = LDMSD_Request.from_str(cmd)
req.send(ctrl)
resp = req.receive(ctrl)
errcode = resp["errcode"]
if errcode:
raise RuntimeError("LDMSD Ctrl errcode: %d" % errcode)
ctrl.close()
time.sleep(1)
# Verify that the agg is working as configured
log.info("verifying aggregator")
xprt = ldms.LDMS_xprt_new_with_auth(cls.XPRT, cls.AUTH,
cls.LDMSD_AUTH_OPT)
rc = ldms.LDMS_xprt_connect_by_name(xprt, "localhost",
cls.AGG_PORT)
if rc:
raise RuntimeError("LDMS connect failed: %d" % rc)
_dir = ldms.LDMS_xprt_dir(xprt)
log.debug("dirs: %s" % str(_dir))
ldms.ldms_xprt_close(xprt)
_edirs = [p["prdcr"] + "/meminfo" for p in cls.PRDCRS]
if set(_dir) != set(_edirs):
raise RuntimeError("Bad set ...")
except:
del cls.agg
del cls.prdcrs
raise
log.info("TestLDMSDPerm set up done")
class TestLDMSDLongConfig(unittest.TestCase):
"""Test cases focusing on long configuration line"""
XPRT = "sock"
SMP_PORT = "10001"
SMP_LOG = "smp.log"
LEN = 65536
# LDMSD instances
smp = None
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def setUp(self):
log.debug("---- %s ----" % self._testMethodName)
def tearDown(self):
log.debug("----------------------------")
if self.smp:
del self.smp
def getMaxMsgSz(self, mode=["configFile", "ldmsctl", "ldmsd_controller"]):
if mode == "configFile":
return 65536 # ldmsd config file max rec len is 8192.
else:
return 1100000 # ldms_xprt_msg_max() for socket is 1048536. This could be varied by system to system.
def getGreetingCfgCmd(self, _len):
return """greeting name=%s""" % ("a" * _len)
def is_logfile_ready(self, filename):
"""Check if the ldmsd log file is created.
"""
if os.path.isfile(filename):
return True
else:
return False
def is_msg_in_logfile(self, filename, msg):
"""Find a line in the ldmsd log file that contains the given msg
@param msg message to search for in the log file
"""
with open(filename, 'r') as loghandle:
line = loghandle.readline()
while line:
if msg in line:
return True
line = loghandle.readline()
return False
def getLogMsg(self, len):
return "strlen(name)=%s" % len
def test_00_config_file(self):
name_len = self.getMaxMsgSz("configFile")
logfile = "config_file.log"
smp_cfg = self.getGreetingCfgCmd(name_len)
self.smp = LDMSD(port=self.SMP_PORT,
xprt=self.XPRT,
cfg=smp_cfg,
logfile=logfile,
verbose="DEBUG")
log.info("starting sampler")
self.smp.run()
if not on_timeout(self.is_logfile_ready, filename=logfile):
raise Exception(
"ldmsd log file isn't created within 1 second after ldmsd is started."
)
msg = self.getLogMsg(name_len)
self.assertTrue(
on_timeout(self.is_msg_in_logfile, filename=logfile, msg=msg))
def test_01_ldmsd_controller(self):
logfile = "ldmsd_controller.log"
self.smp = LDMSD(port=self.SMP_PORT,
xprt=self.XPRT,
verbose="DEBUG",
logfile=logfile)
self.smp.run()
if not on_timeout(self.is_logfile_ready, filename=logfile):
raise Exception(
"ldmsd log file isn't created within 1 second after ldmsd is started."
)
name_len = self.getMaxMsgSz("ldmsd_controller")
line = self.getGreetingCfgCmd(name_len)
cfg = tempfile.NamedTemporaryFile()
cfg.write(line)
cfg.file.flush()
# ldmsd_controller subprocess
ctrl = LDMSD_Controller(port=self.SMP_PORT,
xprt=self.XPRT,
source=cfg.name)
ctrl.run()
msg = self.getLogMsg(name_len)
self.assertTrue(
on_timeout(self.is_msg_in_logfile, filename=logfile, msg=msg))
def test_02_ldmsctl(self):
logfile = "ldmsctl.log"
self.smp = LDMSD(port=self.SMP_PORT,
xprt=self.XPRT,
verbose="DEBUG",
logfile=logfile)
self.smp.run()
if not on_timeout(self.is_logfile_ready, filename=logfile):
raise Exception(
"ldmsd log file isn't created within 1 second after ldmsd is started."
)
name_len = self.getMaxMsgSz("ldmsctl")
line = self.getGreetingCfgCmd(name_len)
cfg = tempfile.NamedTemporaryFile()
cfg.write(line)
cfg.file.flush()
# ldmsd_controller subprocess
ctrl = LDMSD_Controller(port=self.SMP_PORT,
xprt=self.XPRT,
source=cfg.name,
ldmsctl=True)
ctrl.run()
msg = self.getLogMsg(name_len)
self.assertTrue(
on_timeout(self.is_msg_in_logfile, filename=logfile, msg=msg))