def test_2ndLevel_store_after_1stLevel_revived(self, logger, cfg, obj):
sleep(4 * microsec2sec(cfg.LDMSD_UPDATE_INTERVAL))
sosc = SOS.Container("{0}/{1}".format(cfg.STORE_PATH,
obj['container']))
for schema_name, schema in sosc.schemas.iteritems():
if schema_name == obj['schema']:
break
schema = None
# Not a test point
assert (schema is not None,
"The testing sos schema does not exist. Check the test code")
tstp_attr = schema.attr("Timestamp")
tstp_iter = tstp_attr.iterator()
last_record_time = tstp_iter.end().values["Timestamp"].__int__()
logger.debug("last timestamp: {0}".format(last_record_time))
assert (last_record_time is not None)
assert (last_record_time != "")
prev_last_record_time = last_record_time
sleep(20 * microsec2sec(cfg.LDMSD_UPDATE_INTERVAL))
last_record_time = tstp_iter.end().values["Timestamp"].__int__()
logger.debug("last timestamp: {0}".format(last_record_time))
assert (last_record_time > prev_last_record_time)
def test_1stLevel_agg_after_revived(self, logger, cfg, obj, agg_1stLevel_conn):
passed_sec = 0
timeout = 2 * microsec2sec(cfg.LDMSD_UPDATE_INTERVAL) # lookup + update. 2 times for safety
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert(inst_dir is not None)
while (len(inst_dir) == 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert(inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert(obj['instance'] in inst_dir)
def test_2ndLevel_agg_instance_existed(self, logger, cfg, obj, agg_2ndLevel_conn):
passed_sec = 0
timeout = 2 * microsec2sec(cfg.LDMSD_UPDATE_INTERVAL)
inst_dir = ldms.LDMS_xprt_dir(agg_2ndLevel_conn)
assert(inst_dir is not None)
while (len(inst_dir) == 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_2ndLevel_conn)
assert(inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert(obj['instance'] in inst_dir)
def test_2ndLevel_instance_after_1stLevel_revived(self, logger, cfg, obj,
agg_2ndLevel_conn):
passed_sec = 0
timeout = microsec2sec(cfg.LDMSD_RECONNECT_INTERVAL) + 1
inst_dir = ldms.LDMS_xprt_dir(agg_2ndLevel_conn)
assert (inst_dir is not None)
while (len(inst_dir) == 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_2ndLevel_conn)
assert (inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert (obj['instance'] in inst_dir)
def test_agg_1stLevel_instance_after_samplerd_die(self, logger, cfg, obj, agg_1stLevel_conn):
passed_sec = 0
timeout = microsec2sec(cfg.LDMSD_UPDATE_INTERVAL)
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert(inst_dir is not None)
while (len(inst_dir) > 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert(inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert(obj['instance'] not in inst_dir)
def test_agg_after_samplerd_die(self, logger, cfg, obj, agg_conn):
passed_sec = 0
# the latest aggregator recognize that the samplerd is gone
# would be at the update time.
timeout = microsec2sec(cfg.LDMSD_UPDATE_INTERVAL)
inst_dir = ldms.LDMS_xprt_dir(agg_conn)
assert (inst_dir is not None)
while (len(inst_dir) > 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_conn)
assert (inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert (obj['instance'] not in inst_dir)
def test_agg_after_samplerd_revived(self, logger, obj, cfg, agg_conn):
passed_sec = 0
# The default reconnect time is 20 seconds. There is no
# way to change this from the static configuration
timeout = 20 + 2 * microsec2sec(cfg.LDMSD_UPDATE_INTERVAL)
inst_dir = ldms.LDMS_xprt_dir(agg_conn)
assert (inst_dir is not None)
while (len(inst_dir) == 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_conn)
assert (inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert (obj['instance'] in inst_dir)
def test_1stLevel_instance_after_samplerd_revived(self, logger, cfg, obj,
agg_1stLevel_conn):
passed_sec = 0
# wait time must be at least the reconnect interval.
# + 2 is time buffer to do dir and lookup
timeout = microsec2sec(cfg.LDMSD_RECONNECT_INTERVAL) + 1
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert (inst_dir is not None)
while (len(inst_dir) == 0) and (passed_sec < timeout):
passed_sec += 1
sleep(1)
inst_dir = ldms.LDMS_xprt_dir(agg_1stLevel_conn)
assert (inst_dir is not None)
logger.info("Wait time: {0}".format(passed_sec))
assert (obj['instance'] in inst_dir)
