def Test_C_poll(self):
log_level = self.log.getEffectiveLevel()
poll_fd, poll_send = self.self_pipe()
poll = taskforce.poll.poll()
poll.register(poll_fd, taskforce.poll.POLLIN)
# Check active poll
os.write(poll_send, '\0'.encode('utf-8'))
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'active poll', evlist)
assert evlist
assert len(os.read(poll_fd, 10)) == 1
# Check timeout
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'timeout poll', evlist)
assert evlist == []
# Check timeout accuracy
start = time.time()
delay = 500
evlist = poll.poll(timeout=500)
self.dump_evlist(poll, 'timeout accuracy', evlist)
assert evlist == []
delta = abs(time.time() - start - delay/1000.0)
self.log.info("%s poll timeout delta from wall clock %s", my(self), deltafmt(delta, decimals=6))
assert delta < 0.1
if poll.get_mode() == taskforce.poll.PL_SELECT:
self.log.warning("%s Default mode is PL_SELECT so retest skipped", my(self))
else:
poll = taskforce.poll.poll()
poll.set_mode(taskforce.poll.PL_SELECT)
poll.register(poll_fd, taskforce.poll.POLLIN)
# Check active poll
os.write(poll_send, '\0'.encode('utf-8'))
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'select active poll', evlist)
assert evlist
assert len(os.read(poll_fd, 10)) == 1
# Check timeout
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'select timeout poll', evlist)
assert evlist == []
# Check timeout accuracy
start = time.time()
delay = 500
evlist = poll.poll(timeout=500)
self.dump_evlist(poll, 'select timeout accuracy', evlist)
assert evlist == []
delta = abs(time.time() - start - delay/1000.0)
self.log.info("%s select poll timeout delta from wall clock %s", my(self), deltafmt(delta, decimals=6))
assert delta < 0.1
self.close_pipe()
def get_process_count(self, httpc, taskname, expect=None):
count = None
initial_delay = 0.5
backoff_delay = 3.0
max_delay = 8.0
max_attempts = 10
delay = initial_delay
for attempt in range(max_attempts):
self.log.debug("%s Attempt %d", my(self), attempt + 1)
resp = httpc.getmap('/status/tasks')
if taskname in resp and 'processes' in resp[taskname]:
count = 0
for proc in resp[taskname]['processes']:
if proc.get('pid'):
count += 1
if attempt + 1 < max_attempts:
self.log.info(
"%s Task '%s' has %d process%s, expecting %s, next attempt in %s",
my(self), taskname, count, ses(count, 'es'), expect,
deltafmt(delay))
else:
self.log.error(
"%s Task '%s' has %d process%s, expecting %s, giving up",
my(self), taskname, count, ses(count, 'es'), expect)
else:
count = None
self.log.info("%s Task '%s' not found, expecting %s processes",
my(self), taskname, expect)
if expect is None:
self.log.info(
"%s Task '%s' has %d process%s with no expectation",
my(self), taskname, count, ses(count, 'es'))
return count
elif count == expect:
self.log.info("%s Task '%s' has all %d expected process%s",
my(self), taskname, count, ses(count, 'es'))
return count
for tname in resp.keys():
count = 0
for proc in resp[tname]['processes']:
if proc.get('pid'):
count += 1
self.log.debug("While waiting, task '%s' has %d process%s",
tname, count, ses(count, 'es'))
self.log.debug("%s Next attempt in %s", my(self), deltafmt(delay))
time.sleep(delay)
delay += backoff_delay
if delay > max_delay: delay = max_delay
if count is None:
raise Exception("No processes seen for task '%s', %d expected" %
(taskname, expected))
else:
raise Exception("Task '%s' has %d process%s, %d expected" %
(taskname, count, ses(count, 'es'), expect))
def Test_C_poll(self):
log_level = self.log.getEffectiveLevel()
poll_fd, poll_send = self.self_pipe()
poll = taskforce.poll.poll()
poll.register(poll_fd, taskforce.poll.POLLIN)
# Check active poll
os.write(poll_send, '\0'.encode('utf-8'))
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'active poll', evlist)
assert evlist
assert len(os.read(poll_fd, 10)) == 1
# Check timeout
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'timeout poll', evlist)
assert evlist == []
# Check timeout accuracy
start = time.time()
delay = 500
evlist = poll.poll(timeout=500)
self.dump_evlist(poll, 'timeout accuracy', evlist)
assert evlist == []
delta = abs(time.time() - start - delay/1000.0)
self.log.info("%s poll timeout delta from wall clock %s", my(self), deltafmt(delta, decimals=6))
assert delta < 0.1
if poll.get_mode() == taskforce.poll.PL_SELECT:
self.log.warning("%s Default mode is PL_SELECT so retest skipped", my(self))
else:
poll = taskforce.poll.poll()
poll.set_mode(taskforce.poll.PL_SELECT)
poll.register(poll_fd, taskforce.poll.POLLIN)
# Check active poll
os.write(poll_send, '\0'.encode('utf-8'))
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'select active poll', evlist)
assert evlist
assert len(os.read(poll_fd, 10)) == 1
# Check timeout
evlist = poll.poll(timeout=30)
self.dump_evlist(poll, 'select timeout poll', evlist)
assert evlist == []
# Check timeout accuracy
start = time.time()
delay = 500
evlist = poll.poll(timeout=500)
self.dump_evlist(poll, 'select timeout accuracy', evlist)
assert evlist == []
delta = abs(time.time() - start - delay/1000.0)
self.log.info("%s select poll timeout delta from wall clock %s", my(self), deltafmt(delta, decimals=6))
assert delta < 0.1
self.close_pipe()
def get_process_count(self, httpc, taskname, expect=None):
count = None
initial_delay = 0.5
backoff_delay = 3.0
max_delay = 8.0
max_attempts = 10
delay = initial_delay
for attempt in range(max_attempts):
self.log.debug("%s Attempt %d", my(self), attempt+1)
resp = httpc.getmap('/status/tasks')
if taskname in resp and 'processes' in resp[taskname]:
count = 0
for proc in resp[taskname]['processes']:
if proc.get('pid'):
count += 1
if attempt+1 < max_attempts:
self.log.info("%s Task '%s' has %d process%s, expecting %s, next attempt in %s",
my(self), taskname, count, ses(count, 'es'), expect, deltafmt(delay))
else:
self.log.error("%s Task '%s' has %d process%s, expecting %s, giving up",
my(self), taskname, count, ses(count, 'es'), expect)
else:
count = None
self.log.info("%s Task '%s' not found, expecting %s processes", my(self), taskname, expect)
if expect is None:
self.log.info("%s Task '%s' has %d process%s with no expectation",
my(self), taskname, count, ses(count, 'es'))
return count
elif count == expect:
self.log.info("%s Task '%s' has all %d expected process%s",
my(self), taskname, count, ses(count, 'es'))
return count
for tname in resp.keys():
count = 0
for proc in resp[tname]['processes']:
if proc.get('pid'):
count += 1
self.log.debug("While waiting, task '%s' has %d process%s", tname, count, ses(count, 'es'))
self.log.debug("%s Next attempt in %s", my(self), deltafmt(delay))
time.sleep(delay)
delay += backoff_delay
if delay > max_delay: delay = max_delay
if count is None:
raise Exception("No processes seen for task '%s', %d expected" % (taskname, expected))
else:
raise Exception("Task '%s' has %d process%s, %d expected" % (taskname, count, ses(count, 'es'), expect))
def start_client(self, address, use_ssl=None):
start = time.time()
give_up = start + 10
last_exc = None
while time.time() < give_up:
try:
httpc = taskforce.http.Client(address=address, use_ssl=use_ssl, log=self.log)
last_exc = None
break
except Exception as e:
last_exc = e
self.log.debug("%s Connection attempt failed after %s -- %s",
my(self), deltafmt(time.time() - start), str(e))
time.sleep(0.5)
if last_exc:
self.log.error("%s Connection attempt failed after %s -- %s",
my(self), deltafmt(time.time() - start), str(e), exc_info=True)
raise last_exc
return httpc
def start_client(self, address, use_ssl=None):
start = time.time()
give_up = start + 10
last_exc = None
while time.time() < give_up:
try:
httpc = taskforce.http.Client(address=address, use_ssl=use_ssl, log=self.log)
last_exc = None
break
except Exception as e:
last_exc = e
self.log.debug("%s Connection attempt failed after %s -- %s",
my(self), deltafmt(time.time() - start), str(e))
time.sleep(0.5)
if last_exc:
self.log.error("%s Connection attempt failed after %s -- %s",
my(self), deltafmt(time.time() - start), str(e), exc_info=True)
raise last_exc
return httpc
def Test_D_deltafmt(self):
short_match = re.compile(r'^0\.\d+s$')
now = time.time()
delta = utils.deltafmt(time.time()-now, decimals=6)
self.log.info("Delta = %s", delta)
assert short_match.match(delta)
assert utils.deltafmt(time.time()-now) == '0.00s'
assert utils.deltafmt('abc') == '(bad delta: abc)'
known_deltas = {
60.0: '1m0.0s',
600.0: '10m0s',
3600.0: '1h0m0s',
2*24*3600+60.0: '48h1m0s'
}
for delta, expected in known_deltas.items():
res = utils.deltafmt(delta)
self.log.info("Delta of %.2f = %s", delta, res)
assert res == expected
def Test_A_https_tcp_status(self):
httpc = self.start_tf(self.tcp_address, use_ssl=False)
# Check the version info is sane
resp = httpc.getmap('/status/version')
self.log.info("Version info: %s", str(resp))
assert 'taskforce' in resp
assert 'platform' in resp
# This is not a control path, so the os release and platform should be hidden
assert 'release' not in resp['platform']
assert 'platform' not in resp['platform']
# Try a bogus format
try:
resp = httpc.getmap('/status/version?indent=4&fmt=xml')
assert "No 'version' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'version' exception on bad format: %s",
my(self), str(e))
give_up = time.time() + 30
toi = 'db_server'
toi_started = None
while time.time() < give_up:
resp = httpc.getmap('/status/tasks')
self.log.debug('Resp %s', json.dumps(resp, indent=4))
if toi in resp:
if 'processes' in resp[toi] and len(
resp[toi]['processes']) > 0:
if 'started_t' in resp[toi]['processes'][0]:
toi_started = resp[toi]['processes'][0]['started_t']
self.log.info(
"%s Task of interest '%s' started %s ago",
my(self), toi, deltafmt(time.time() - toi_started))
break
else:
self.log.info("%s Task of interest '%s' is has procs",
my(self), toi)
else:
self.log.info("%s Task of interest '%s' is known",
my(self), toi)
time.sleep(9)
# Try a bogus format
try:
resp = httpc.getmap('/status/tasks?indent=4&fmt=xml')
assert "No 'tasks' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'tasks' exception on bad format: %s",
my(self), str(e))
# Check the config info is sane
resp = httpc.getmap('/status/config?pending=0')
assert 'tasks' in resp
# Try a bogus format
try:
resp = httpc.getmap('/status/config?indent=4&fmt=xml')
assert "No 'config' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'config' exception on bad format: %s",
my(self), str(e))
support.check_procsim_errors(self.__module__, env, log=self.log)
self.stop_tf()
assert toi_started is not None
def Test_A_https_tcp_status(self):
httpc = self.start_tf(self.tcp_address, use_ssl=False)
# Check the version info is sane
resp = httpc.getmap('/status/version')
self.log.info("Version info: %s", str(resp))
assert 'taskforce' in resp
assert 'platform' in resp
# This is not a control path, so the os release and platform should be hidden
assert 'release' not in resp['platform']
assert 'platform' not in resp['platform']
# Try a bogus format
try:
resp = httpc.getmap('/status/version?indent=4&fmt=xml')
assert "No 'version' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'version' exception on bad format: %s", my(self), str(e))
give_up = time.time() + 30
toi = 'db_server'
toi_started = None
while time.time() < give_up:
resp = httpc.getmap('/status/tasks')
self.log.debug('Resp %s', json.dumps(resp, indent=4))
if toi in resp:
if 'processes' in resp[toi] and len(resp[toi]['processes']) > 0:
if 'started_t' in resp[toi]['processes'][0]:
toi_started = resp[toi]['processes'][0]['started_t']
self.log.info("%s Task of interest '%s' started %s ago",
my(self), toi, deltafmt(time.time() - toi_started))
break
else:
self.log.info("%s Task of interest '%s' is has procs", my(self), toi)
else:
self.log.info("%s Task of interest '%s' is known", my(self), toi)
time.sleep(9)
# Try a bogus format
try:
resp = httpc.getmap('/status/tasks?indent=4&fmt=xml')
assert "No 'tasks' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'tasks' exception on bad format: %s", my(self), str(e))
# Check the config info is sane
resp = httpc.getmap('/status/config?pending=0')
assert 'tasks' in resp
# Try a bogus format
try:
resp = httpc.getmap('/status/config?indent=4&fmt=xml')
assert "No 'config' exception on bad 'fmt'" is False
except taskforce.http.HttpError as e:
self.log.info("%s Expected 'config' exception on bad format: %s", my(self), str(e))
support.check_procsim_errors(self.__module__, env, log=self.log)
self.stop_tf()
assert toi_started is not None
def Test_L_pidclaim(self):
args = list(sys.argv)
args.pop(0)
pidfile = './%s-%s.pid' % (utils.appname(), env.edition)
# Run the test as a forked process so we can test for
# pid file creation and removal.
#
start = time.time()
pid = os.fork()
if pid == 0:
args = ['pidclaim']
self.set_path('PYTHONPATH', env.base_dir)
if 'NOSE_WITH_COVERAGE' in os.environ:
exe = 'coverage'
args.append('run')
else:
exe = 'python'
args.extend(['tests/scripts/pidclaim', pidfile])
self.log.info("%s child, running '%s' %s", my(self), exe, args)
os.execvp(exe, args)
else:
time.sleep(1)
self.log.info("Child PID is: %d", pid)
with open(pidfile, 'r') as f:
claim_pid = int(f.readline().strip())
self.log.info("PID read back as: %d", claim_pid)
assert claim_pid == pid
(wpid, status) = os.wait()
self.log.info("Child ran %s", utils.deltafmt(time.time() - start, decimals=3))
self.log.info("Child %s", utils.statusfmt(status))
assert status == 0
assert not os.path.exists(pidfile)
# Bad pid param
#
log_level = self.log.getEffectiveLevel()
try:
# Mask the log message as we expect a failure
self.log.setLevel(logging.CRITICAL)
utils.pidclaim(pid='abc')
self.log.setLevel(log_level)
expected_error_occurred = False
except Exception as e:
self.log.setLevel(log_level)
self.log.info("%s Received expected bad pid error -- %s", my(self), str(e))
expected_error_occurred = True
assert expected_error_occurred
# Invalid pid param
#
log_level = self.log.getEffectiveLevel()
try:
# Mask the log message as we expect a failure
self.log.setLevel(logging.CRITICAL)
utils.pidclaim(pid=0)
self.log.setLevel(log_level)
expected_error_occurred = False
except Exception as e:
self.log.setLevel(log_level)
self.log.info("%s Received expected invalid pid error -- %s", my(self), str(e))
expected_error_occurred = True
assert expected_error_occurred