def main():
try:
parser = ArgumentParser()
parser.add_argument('--mem', help = "Pre-allocated Memory",
default = DEFAULT_PRE_ALLOC_MEM)
args = parser.parse_args()
#===================
# This will pre-allocate memory to create metrics and set instances.
#===================
ldms.ldms_init(args.mem) # This is mandatory call when using libldms.
instance = create_ldms_set_instance()
update_metric_values(instance)
#===================
# Print set meta data and data
#
# See more in pytest.py
#===================
pytest.show_meta(instance)
get_metric_values(instance)
#===================
# Delete the set instance
#===================
ldms.ldms_set_delete(instance)
return 0
except:
traceback.print_exc()
def setUpClass(cls):
log.info("--- Setting up TestLDMSAuthOvis ---")
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
# auth conf
conf_text = "secretword=this_is_a_secret"
f = open(cls.CONF, "w")
f.write(conf_text)
f.close()
os.chmod(cls.CONF, 0600)
# other conf
conf_text = "secretword=this_is_another_secret"
f = open(cls.OTHER_CONF, "w")
f.write(conf_text)
f.close()
os.chmod(cls.OTHER_CONF, 0600)
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
uid=0 gid=0 perm=0600
start name=meminfo interval=1000000
"""
cls.ldmsd = LDMSD(port=cls.PORT,
cfg=cfg,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT)
log.info("starting ldmsd")
cls.ldmsd.run()
time.sleep(1)
log.info("--- Done setting up TestLDMSAuthOvis ---")
def setUpClass(cls):
cls.smp = None
cls.agg = None
log.info("--- Setting up TestLDMSDSetArray ---")
try:
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
set_array_card=20 \
uid=0 gid=0 perm=0777
start name=meminfo interval=%(interval)d offset=0
""" % {
"interval": cls.SMP_INT,
}
cls.smp = LDMSD(port=cls.SMP_PORT,
cfg=cfg,
logfile=cls.SMP_LOG,
gdb_port=cls.SMP_GDB_PORT)
DEBUG.smp = cls.smp
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
log.info("--- Done setting up TestLDMSDSetArray ---")
except:
cls.tearDownClass()
raise
def setUpClass(cls):
log.info("Setting up TestLDMSAuthNaive")
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
cls.ldmsd = LDMSD(port=cls.PORT, auth=cls.AUTH, auth_opt=cls.AUTH_OPT)
log.info("starting ldmsd")
cls.ldmsd.run()
cls.cfg = ldmsdInbandConfig(host="localhost",
port=cls.PORT,
xprt=cls.XPRT,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT)
# NOTE: cls.cfg automatically create an LDMS xprt and connect to the
# target ldmsd.
cmds = []
for _set, _perm in zip(cls.SETS, cls.PERMS):
cmds.append("load name=%s" % _set)
cmds.append("config name=%(set)s producer=test1 instance=%(set)s \
schema=%(set)s component_id=1 \
uid=%(uid)s gid=%(gid)s perm=%(perm)s" % {
"set": _set,
"uid": cls.UID,
"gid": cls.GID,
"perm": _perm,
})
cmds.append("start name=%s interval=1000000" % _set)
log.info("configuring ldmsd over LDMS xprt")
for cmd in cmds:
req = LDMSD_Request.from_str(cmd)
req.send(cls.cfg)
resp = req.receive(cls.cfg)
errcode = resp["errcode"] if "errcode" in resp else 0
if errcode:
raise RuntimeError("LDMSD request errcode: %d" % errcode)
time.sleep(1)
log.info("TestLDMSAuthNaive setup complete")
def setUpClass(cls):
# 1 sampler, 1 aggregator
log.info("Setting up " + cls.__name__)
ldms.ldms_init(256*1024*1024)
try:
# samplers (producers)
smp_cfg = """
load name=hfclock
config name=hfclock producer=smp \
instance=smp/hfclock schema=hfclock \
hfinterval=100000 \
hfcount=10
start name=hfclock interval=1000000 offset=0
"""
log.debug("smp_cfg: %s" % smp_cfg)
cls.smp = LDMSD(port = cls.SMP_PORT, xprt = cls.XPRT,
cfg = smp_cfg,
logfile = cls.SMP_LOG)
log.info("starting sampler")
cls.smp.run()
time.sleep(1)
# aggregator will be configured later in the test cases
except:
del cls.smp
raise
log.info(cls.__name__ + " set up done")
def setUpClass(self):
self.expt = None
self.samp = None
self.slurm_set = None # the `slurm` set
self.mem_set = None # the `mem` (meminfo) set
self.conn = None # ldms connection
ldms.ldms_init(512*1024*1024) # 512MB should suffice
try:
cfg = """\
load name=slurm plugin=slurm_sampler
config name=slurm instance=slurm stream=slurm \
job_count=%(JOB_SIZE)d task_count=%(TASK_SIZE)d
load name=mem plugin=meminfo
config name=mem job_set=slurm instance=mem
smplr_add name=mem_smplr instance=mem interval=1000000 offset=0
smplr_start name=mem_smplr
""" % { k: getattr(self, k) for k in dir(self) }
self.samp = LDMSD(port=self.SAMP_PORT, cfg = cfg,
logfile = self.SAMP_LOG,
gdb_port = self.SAMP_GDB_PORT)
D.samp = self.samp
log.info("Starting sampler")
self.samp.run()
self.conn = ldms.LDMS_xprt_new(self.XPRT)
D.conn = self.conn
self.conn.connectByName("localhost", self.SAMP_PORT)
self.slurm_set = self.conn.lookupSet("slurm", 0)
D.slurm_set = self.slurm_set
self.mem_set = self.conn.lookupSet("mem", 0)
D.mem_set = self.mem_set
expt = {
"component_id" : [0L] * self.JOB_SIZE,
"job_id" : [0L] * self.JOB_SIZE,
"app_id" : [0L] * self.JOB_SIZE,
"current_slot" : 0L,
"job_state" : [0L] * self.JOB_SIZE,
"job_tstamp" : [0L] * self.JOB_SIZE,
"job_size" : [0L] * self.JOB_SIZE,
"job_uid" : [0L] * self.JOB_SIZE,
"job_gid" : [0L] * self.JOB_SIZE,
"job_start" : [0L] * self.JOB_SIZE,
"job_end" : [0L] * self.JOB_SIZE,
"node_count" : [0L] * self.JOB_SIZE,
"task_count" : [0L] * self.JOB_SIZE,
}
task_keys = ["task_pid", "task_rank", "task_exit_status"]
def setUpClass(cls):
if os.getuid() != 0 or os.getgid() != 0:
raise RuntimeError("This test needs to run as `root`, "
"suggestion: `sudo -E bash` to become root "
"with your current environments, then execute "
"this script.")
usr = pwd.getpwall()[1]
cls.usr = usr
cls.munged = None
cls.ldmsd = None
try:
log.info("--- Setting up %s ---" % cls.__name__)
ldms.ldms_init(512 * 1024 * 1024) # 512MB should suffice
cls.munged = Munged(key=''.join('0' for i in range(0, 1024)))
cls.munged.start()
# ldmsd sampler conf
cfg = """\
load name=meminfo
config name=meminfo producer=smp instance=smp/meminfo \
schema=meminfo component_id=1 \
uid=0 gid=0 perm=0600
start name=meminfo interval=1000000
load name=vmstat
config name=vmstat producer=smp instance=smp/vmstat \
schema=vmstat component_id=1 \
uid=%(usr_uid)d gid=%(usr_gid)d perm=0600
start name=vmstat interval=1000000
""" % {
'usr_uid': usr.pw_uid,
'usr_gid': usr.pw_gid,
}
cls.AUTH_OPT = {"socket": cls.munged.socket}
cls.ldmsd = LDMSD(port=cls.PORT,
cfg=cfg,
auth=cls.AUTH,
auth_opt=cls.AUTH_OPT,
logfile=DIR + "/ldmsd.log")
log.info("starting ldmsd")
cls.ldmsd.run()
time.sleep(1)
log.info("--- Done setting up %s ---" % cls.__name__)
except Exception, e:
del cls.ldmsd
del cls.munged
raise
from ovis_ldms import ldms
from ldmsd_util import LDMSD
from ldmsd_config import ldmsdInbandConfig
from ldmsd_request import LDMSD_Request
from distutils.spawn import find_executable as which
class Debug(object):
pass
D = Debug()
ldms.ldms_init(64 * 1024 * 1024)
Src = namedtuple('Src', ['path', 'content'])
SrcData = namedtuple('SrcData', ['src', 'data'])
# SrcData.src is the Src (or list of Src) describing file(s) and the contents
# SrcData.data is the LDMSData corresponding to the source
def prog_to_prefix(prog_name):
path = which(prog_name)
(prefix, bindir, pname) = path.rsplit('/', 2)
assert (bindir in ["bin", "sbin"])
return prefix
LDMS_PREFIX = prog_to_prefix("ldms-pedigree")
import pdb
import collections
from StringIO import StringIO
from ovis_ldms import ldms
from ldmsd.ldmsd_util import LDMSD
from ldmsd.ldmsd_config import ldmsdInbandConfig
from ldmsd.ldmsd_request import LDMSD_Request
import csv
DIR = "test_store_csv" if not sys.path[0] else sys.path[0] + "/test_store_csv"
log = logging.getLogger(__name__)
HOSTNAME = socket.gethostname()
ldms.ldms_init(512*1024*1024) # 512MB should suffice
class Debug(object):
pass
D = Debug()
def ldms_set_as_dict(_set):
return { _set.metric_name_get(_mid): _val \
for (_mid, _val) in _set.iter_items() }
def ldms_set_as_tuple(_set):
t = tuple( _val for (_mid, _val) in _set.iter_items() )
# add special fields
ts = _set.ts_get()
ts_float = ts.sec + ts.usec*1e-6
def ldmsd_test_init(request):
size = request.config.getoption("--pre-mem")
ldms_init(size)
from ovis_ldms import ldms
from ldmsd.ldmsd_util import LDMSD, LDMSD_Controller
from ldmsd.ldmsd_config import ldmsdInbandConfig
from ldmsd.ldmsd_request import LDMSD_Request
log = logging.getLogger(__name__)
class Debug(object):
pass
DEBUG = Debug()
ldms.ldms_init(128 * 1024 * 1024)
class LdmsWrap(object):
"""Conveneint LDMS xprt wrapper"""
def __init__(self, port, xprt="sock", hostname="localhost"):
self.xprt = ldms.LDMS_xprt_new(xprt)
rc = ldms.LDMS_xprt_connect_by_name(self.xprt, hostname, port)
assert (rc == 0)
self.sets = []
self._dict = {}
_dirs = ldms.LDMS_xprt_dir(self.xprt)
for d in _dirs:
s = ldms.LDMS_xprt_lookup(self.xprt, d,
ldms.LDMS_LOOKUP_BY_INSTANCE)
self.sets.append(s)
import time
import shutil
import logging
import unittest
from StringIO import StringIO
from ovis_ldms import ldms
from ldmsd.ldmsd_util import LDMSD
from ldmsd.ldmsd_config import ldmsdInbandConfig
from ldmsd.ldmsd_request import LDMSD_Request
from sosdb import Sos
log = logging.getLogger(__name__)
DIR = "ldmsd_strgp"
ldms.ldms_init(16 * 1024 * 1024)
class TestLDMSDStrgp(unittest.TestCase):
"""Test cases focusing on strgp"""
XPRT = "sock"
SMP_PORT_BASE = 10000
AGG_PORT_BASE = 11000
SMP_NUM = 4
# LDMSD instances
smp = dict()
agg = None
@classmethod
def setUpClass(cls):