def it_restarts_on_unready():
def it_is_ready():
assert_svstat('playground/slow-startup', state='ready')
def it_becomes_unready():
from testfixtures import Comparison as C
from pgctl.daemontools import svstat
assert svstat('playground/slow-startup') != C(SvStat, {'state': 'ready'}, strict=False)
it_can_succeed()
os.remove('playground/slow-startup/readyfile')
wait_for(it_becomes_unready)
wait_for(it_is_ready)
assert_command(
('pgctl-2015', 'log'),
'''\
==> playground/slow-startup/log <==
{TIMESTAMP} pgctl-poll-ready: service's ready check succeeded
{TIMESTAMP} pgctl-poll-ready: service's ready check failed -- we are restarting it for you
{TIMESTAMP} [pgctl] Stopping: slow-startup
{TIMESTAMP} [pgctl] Stopped: slow-startup
{TIMESTAMP} [pgctl] Starting: slow-startup
{TIMESTAMP} pgctl-poll-ready: service's ready check succeeded
{TIMESTAMP} [pgctl] Started: slow-startup
''',
'',
0,
norm=norm.pgctl,
)
def it_can_start_via_abspath_service(self, in_example_dir, scratch_dir,
tmpdir):
with tmpdir.ensure_dir('arbitrary').as_cwd():
assert not scratch_dir.join('now.date').isfile()
date_abspath = str(in_example_dir.join('playground/date'))
check_call(('pgctl', 'start', date_abspath))
wait_for(lambda: scratch_dir.join('now.date').isfile())
def it_can_start_via_abspath_pgdir(self, in_example_dir, scratch_dir, tmpdir):
with tmpdir.ensure_dir("arbitrary").as_cwd():
assert not scratch_dir.join("now.date").isfile()
env = os.environ.copy()
env["PGCTL_PGDIR"] = str(in_example_dir.join("playground"))
check_call(("pgctl-2015", "start", "date"), env=env)
wait_for(lambda: scratch_dir.join("now.date").isfile())
def it_works(self, in_example_dir):
assert_command(
('pgctl', 'start', 'A'),
'',
'''\
[pgctl] Starting: A
[pgctl] Started: A
''',
0,
)
wait_for(lambda: assert_command(
('pgctl', 'log', 'A'),
'''\
==> playground/A/logs/current <==
{TIMESTAMP} [pgctl] Starting: B
{TIMESTAMP} [pgctl] DEBUG: parentlock: '%s/playground/A'
{TIMESTAMP} [pgctl] DEBUG: LOCK: ${LOCK}
{TIMESTAMP} [pgctl] DEBUG: loop: check_time $TIME
{TIMESTAMP} [pgctl] Started: B
{TIMESTAMP} this is stdout
{TIMESTAMP} this is stderr
''' % in_example_dir,
'',
0,
norm=norm.pgctl,
))
assert_command(
('pgctl', 'stop', 'A'),
'',
'''\
[pgctl] Stopping: A
[pgctl] Stopped: A
''',
0,
)
def it_can_start_via_abspath_pgdir(self, in_example_dir, scratch_dir, tmpdir):
with tmpdir.ensure_dir('arbitrary').as_cwd():
assert not scratch_dir.join('now.date').isfile()
env = os.environ.copy()
env['PGCTL_PGDIR'] = str(in_example_dir.join('playground'))
check_call(('pgctl', 'start', 'date'), env=env)
wait_for(lambda: scratch_dir.join('now.date').isfile())
def it_fails_twice_but_doesnt_restart():
def it_is_ready():
assert_svstat('playground/unreliable', state='ready')
assert_command(
('pgctl-2015', 'start'),
'',
'[pgctl] Starting: unreliable\n[pgctl] Started: unreliable\n',
0
)
wait_for(it_is_ready)
assert_command(
('pgctl-2015', 'log'),
'''\
==> playground/unreliable/log <==
{TIMESTAMP} pgctl-poll-ready: service's ready check succeeded
{TIMESTAMP} pgctl-poll-ready: failed (restarting in 2.00 seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in 1.99 seconds)
''',
'',
0,
norm=norm.timestamp,
)
def it_restarts_on_unready():
def it_is_ready():
assert_svstat('playground/slow-startup', state='ready')
def it_stopped():
assert_svstat('playground/slow-startup', state=SvStat.UNSUPERVISED)
it_can_succeed()
os.remove('playground/slow-startup/readyfile')
wait_for(it_stopped, limit=5) # TODO see why it takes so long, we found python taking .5 seconds to start
wait_for(it_is_ready, limit=5)
assert_command(
('pgctl-2015', 'log'),
'''\
==> playground/slow-startup/stdout.log <==
==> playground/slow-startup/stderr.log <==
pgctl-poll-ready: service's ready check succeeded
pgctl-poll-ready: service's ready check failed -- we are restarting it for you
[pgctl] Stopping: slow-startup
[pgctl] Stopped: slow-startup
[pgctl] Starting: slow-startup
pgctl-poll-ready: service's ready check succeeded
[pgctl] Started: slow-startup
''',
'',
0,
)
def it_works(self, in_example_dir):
assert_command(
('pgctl', 'start', 'A'),
'',
'''\
[pgctl] Starting: A
[pgctl] Started: A
''',
0,
)
wait_for(lambda: assert_command(
('pgctl', 'log', 'A'),
'''\
==> playground/A/log <==
{TIMESTAMP} [pgctl] Starting: B
{TIMESTAMP} [pgctl] DEBUG: parentlock: '%s/playground/A'
{TIMESTAMP} [pgctl] DEBUG: LOCK: ${LOCK}
{TIMESTAMP} [pgctl] DEBUG: loop: check_time $TIME
{TIMESTAMP} [pgctl] Started: B
{TIMESTAMP} this is stdout
{TIMESTAMP} this is stderr
''' % in_example_dir,
'',
0,
norm=norm.pgctl,
))
assert_command(
('pgctl', 'stop', 'A'),
'',
'''\
[pgctl] Stopping: A
[pgctl] Stopped: A
''',
0,
)
def it_can_start_via_abspath_pgdir(self, in_example_dir, scratch_dir,
tmpdir):
with tmpdir.ensure_dir('arbitrary').as_cwd():
assert not scratch_dir.join('now.date').isfile()
env = os.environ.copy()
env['PGCTL_PGDIR'] = str(in_example_dir.join('playground'))
check_call(('pgctl', 'start', 'date'), env=env)
wait_for(lambda: scratch_dir.join('now.date').isfile())
def it_does_start(self, in_example_dir):
test_string = 'oh, hi there.\n'
with open('input', 'w') as input:
input.write(test_string)
assert not os.path.isfile('output')
check_call(('pgctl', 'start', 'tail'))
wait_for(lambda: os.path.isfile('output'))
assert open('output').read() == test_string
def it_does_start(self, in_example_dir):
test_string = "oh, hi there.\n"
with open("input", "w") as input:
input.write(test_string)
assert not os.path.isfile("output")
check_call(("pgctl-2015", "start", "tail"))
wait_for(lambda: os.path.isfile("output"))
assert open("output").read() == test_string
def it_does_start(self, in_example_dir):
test_string = 'oh, hi there.\n'
with open('input', 'w') as input:
input.write(test_string)
assert not os.path.isfile('output')
check_call(('pgctl-2015', 'start', 'tail'))
wait_for(lambda: os.path.isfile('output'))
assert open('output').read() == test_string
def it_restarts_on_unready():
def it_is_ready():
assert_svstat('playground/slow-startup', state='ready')
def it_becomes_unready():
from testfixtures import Comparison as C
from pgctl.daemontools import svstat
assert svstat('playground/slow-startup') != C(
SvStat, {'state': 'ready'}, strict=False)
it_can_succeed()
os.remove('playground/slow-startup/readyfile')
wait_for(it_becomes_unready)
wait_for(it_is_ready)
assert_command(
('pgctl', 'log'),
'''\
==> playground/slow-startup/log <==
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in {TIME} seconds)
{TIMESTAMP} pgctl-poll-ready: failed for more than {TIME} seconds -- we are restarting this service for you
{TIMESTAMP} [pgctl] Stopping: slow-startup
{TIMESTAMP} [pgctl] Stopped: slow-startup
{TIMESTAMP} [pgctl] Starting: slow-startup
{TIMESTAMP} pgctl-poll-ready: service's ready check succeeded
{TIMESTAMP} [pgctl] Started: slow-startup
''',
'',
0,
norm=norm.pgctl,
)
def it_kills_processes_holding_the_lock(self, tmpdir):
lockfile = tmpdir.ensure('lock')
lock = lockfile.open()
process = Popen(('sleep', 'infinity'))
lock.close()
# assert `process` has `lock`
assert list(fuser(lockfile.strpath)) == [process.pid]
with mock.patch('sys.stderr', new_callable=StringIO.StringIO) as mock_stderr:
terminate_runaway_processes(lockfile.strpath)
assert 'WARNING: Killing these runaway ' in mock_stderr.getvalue()
wait_for(lambda: process.poll() == -9)
def it_disables_polling():
stderr = open('stderr', 'w')
proc = Popen(('pgctl', 'debug', 'unreliable'), stdin=open(os.devnull), stdout=PIPE, stderr=stderr)
def check_file_contents():
expected = '''\
pgctl-poll-ready: disabled during debug -- quitting
'''
with open('stderr') as fd:
actual = fd.read()
return expected == actual
wait_for(check_file_contents)
proc.terminate()
stderr.close()
def it_disables_polling():
stderr = open('stderr', 'w')
proc = Popen(('pgctl', 'debug', 'unreliable'), stdin=open(os.devnull), stdout=PIPE, stderr=stderr)
def check_file_contents():
expected = '''\
[pgctl] Stopping: unreliable
[pgctl] Stopped: unreliable
pgctl-poll-ready: disabled during debug -- quitting
'''
with open('stderr') as fd:
actual = fd.read()
return expected == actual
wait_for(check_file_contents)
proc.terminate()
stderr.close()
def it_disables_polling_heartbeat():
from mock import patch
with patch.dict(os.environ, [('PGCTL_TIMEOUT', '5')]):
proc = Popen(('pgctl-2015', 'debug', 'slow-startup'), stdin=open(os.devnull), stdout=PIPE, stderr=PIPE)
from testing.assertions import wait_for
wait_for(lambda: assert_svstat('playground/slow-startup', state='ready'))
check_call(('pgctl-2015', 'stop'))
stdout, stderr = proc.communicate()
stdout, stderr = stdout.decode('UTF-8'), stderr.decode('UTF-8')
assert stderr == '''\
[pgctl] Stopping: slow-startup
[pgctl] Stopped: slow-startup
pgctl-poll-ready: service's ready check succeeded
pgctl-poll-ready: heartbeat is disabled during debug -- quitting
'''
assert stdout == ''
assert proc.returncode == 0
def it_fails_twice_but_doesnt_restart():
def it_is_ready():
assert_svstat('playground/unreliable', state='ready')
assert_command(
('pgctl', 'start'), '',
'[pgctl] Starting: unreliable\n[pgctl] Started: unreliable\n', 0)
wait_for(it_is_ready)
assert_command(
('pgctl', 'log'),
'''\
==> playground/unreliable/log <==
{TIMESTAMP} pgctl-poll-ready: service's ready check succeeded
{TIMESTAMP} pgctl-poll-ready: failed (restarting in 2.00 seconds)
{TIMESTAMP} pgctl-poll-ready: failed (restarting in 1.99 seconds)
''',
'',
0,
norm=norm.timestamp,
)
def it_does_start(self, in_example_dir, scratch_dir):
assert not scratch_dir.join('now.date').isfile()
check_call(('pgctl', 'start', 'date'))
wait_for(lambda: scratch_dir.join('now.date').isfile())
def it_does_start(self, in_example_dir, scratch_dir):
assert not scratch_dir.join('now.date').isfile()
check_call(('pgctl-2015', 'start', 'date'))
wait_for(lambda: scratch_dir.join('now.date').isfile())
def it_does_start(self, in_example_dir, scratch_dir):
assert not scratch_dir.join("now.date").isfile()
check_call(("pgctl-2015", "start", "date"))
wait_for(lambda: scratch_dir.join("now.date").isfile())
def it_can_start_via_abspath_service(self, in_example_dir, scratch_dir, tmpdir):
with tmpdir.ensure_dir("arbitrary").as_cwd():
assert not scratch_dir.join("now.date").isfile()
date_abspath = str(in_example_dir.join("playground/date"))
check_call(("pgctl-2015", "start", date_abspath))
wait_for(lambda: scratch_dir.join("now.date").isfile())
def it_can_start_via_abspath_service(self, in_example_dir, scratch_dir, tmpdir):
with tmpdir.ensure_dir('arbitrary').as_cwd():
assert not scratch_dir.join('now.date').isfile()
date_abspath = str(in_example_dir.join('playground/date'))
check_call(('pgctl', 'start', date_abspath))
wait_for(lambda: scratch_dir.join('now.date').isfile())
