def test_list(reqmanager):
r1 = Request(Activity(), "14m", "pending request")
reqmanager.add(r1)
r2 = Request(Activity(), "2h", "due request")
r2.state = State.due
r2.next_due = datetime.datetime(2016, 4, 20, 12, tzinfo=pytz.UTC)
reqmanager.add(r2)
r3 = Request(Activity(), "1m 30s", "error request")
r3.state = State.error
r3.next_due = datetime.datetime(2016, 4, 20, 11, tzinfo=pytz.UTC)
att = Attempt()
att.duration = datetime.timedelta(seconds=75)
att.returncode = 1
r3.attempts = [att]
reqmanager.add(r3)
assert (
str(reqmanager)
== """\
St Id Scheduled Estimate Comment
e {id3} 2016-04-20 11:00 UTC 1m 30s error request (duration: 1m 15s)
* {id2} 2016-04-20 12:00 UTC 2h due request
- {id1} --- TBA --- 14m pending request\
""".format(
id1=r1.id[:7], id2=r2.id[:7], id3=r3.id[:7]
)
)
def test_do_add_two_reqs_with_identical_comments(reqmanager):
with reqmanager as rm:
assert rm.add(Request(Activity(), 1, "comment 1")) is not None
assert (
rm.add(Request(Activity(), 1, "comment 1"), skip_same_comment=False)
is not None
)
assert len(rm.requests) == 2
def test_execute_logs_exception(reqmanager, caplog):
req = reqmanager.add(Request(Activity(), 1))
req.state = State.due
os.chmod(req.dir, 0o000) # simulates I/O error
reqmanager.execute()
assert 'Permission denied' in caplog.text
os.chmod(req.dir, 0o755) # py.test cannot clean up 0o000 dirs
def test_execute_marks_service_status(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 1))
req.state = State.due
reqmanager.execute()
assert [
unittest.mock.call(unittest.mock.ANY, False),
unittest.mock.call(unittest.mock.ANY, True),
] == connect().mark_node_service_status.call_args_list
def test_explicitly_deleted(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 90))
req.state = State.deleted
arch = connect().end_maintenance
reqmanager.archive()
arch.assert_called_once_with(
{req.id: {"duration": None, "result": "deleted"}}
)
def test_duration_from_started_finished(utcnow, tmpdir):
utcnow.side_effect = [
datetime.datetime(2016, 4, 20, 6, 0),
datetime.datetime(2016, 4, 20, 6, 2),
]
r = Request(Activity(), 1, dir=str(tmpdir))
r.execute()
assert r.duration == 120.0
def test_postpone(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 90))
req.state = State.postpone
postp = connect().postpone_maintenance
reqmanager.postpone()
postp.assert_called_once_with({req.id: {'postpone_by': 180}})
assert req.state == State.postpone
assert req.next_due is None
def test_execute_not_performed_on_connection_error(
execute, connect, reqmanager
):
connect().mark_node_service_status.side_effect = socket.error()
req = reqmanager.add(Request(Activity(), 1))
req.state = State.due
with pytest.raises(OSError):
reqmanager.execute()
assert execute.mock_calls == []
def test_duration():
r = Request(Activity(), 1)
a = Attempt()
a.duration = 10
r.attempts.append(a)
a = Attempt()
a.duration = 5
r.attempts.append(a)
assert r.duration == 5 # last attempt counts
def test_execute_logs_exception(connect, reqmanager, log):
req = reqmanager.add(Request(Activity(), 1))
req.state = State.due
os.chmod(req.dir, 0o000) # simulates I/O error
reqmanager.execute()
log.has(
"execute-request-failed", request=req.id, level="error", exc_info=True
)
os.chmod(req.dir, 0o755) # py.test cannot clean up 0o000 dirs
def test_delete_disappeared_requests(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 1, "comment"))
sched = connect().schedule_maintenance
sched.return_value = {
req.id: {"time": "2016-04-20T15:12:40.9+00:00"},
"123abc": {"time": None},
}
endm = connect().end_maintenance
reqmanager.schedule()
endm.assert_called_once_with({"123abc": {"result": "deleted"}})
def test_external_activity_state(tmpdir, logger):
r = Request(ExternalStateActivity(), 1, dir=str(tmpdir))
r.save()
extstate = str(tmpdir / "external_state")
with open(extstate) as f:
assert "foo\n" == f.read()
with open(extstate, "w") as f:
print("bar", file=f)
r2 = Request.load(str(tmpdir), logger)
assert r2.activity.external == "bar\n"
def test_external_activity_state(tmpdir):
r = Request(ExternalStateActivity(), 1, dir=str(tmpdir))
r.save()
extstate = str(tmpdir / 'external_state')
with open(extstate) as f:
assert 'foo\n' == f.read()
with open(extstate, 'w') as f:
print('bar', file=f)
r2 = Request.load(str(tmpdir))
assert r2.activity.external == 'bar\n'
def test_schedule_requests(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 1, "comment"))
rpccall = connect().schedule_maintenance
rpccall.return_value = {req.id: {"time": "2016-04-20T15:12:40.9+00:00"}}
reqmanager.schedule()
rpccall.assert_called_once_with(
{req.id: {"estimate": 1, "comment": "comment"}}
)
assert req.next_due == datetime.datetime(
2016, 4, 20, 15, 12, 40, 900000, tzinfo=pytz.UTC
)
def test_execute_obeys_retrylimit(tmpdir):
Request.MAX_RETRIES = 3
r = Request(Activity(), 1, dir=str(tmpdir))
results = []
for i in range(Request.MAX_RETRIES + 1):
r.execute()
assert len(r.attempts) == i + 1
r.update_state()
results.append(r.state)
assert results[0] == State.success
assert results[-2] == State.success
assert results[-1] == State.retrylimit
def test_execute_activity_no_reboot(connect, run, reqmanager, log):
activity = Activity()
req = reqmanager.add(Request(activity, 1))
req.state = State.due
reqmanager.execute(run_all_now=True)
run.assert_has_calls(
[
call('echo "entering demo"', shell=True, check=True),
call('echo "leaving demo"', shell=True, check=True),
]
)
assert log.has("enter-maintenance")
assert log.has("leave-maintenance")
def request_population(n, dir):
"""Creates a ReqManager with a pregenerated population of N requests.
The ReqManager and a list of Requests are passed to the calling code.
"""
with ReqManager(str(dir)) as reqmanager:
requests = []
for i in range(n):
req = Request(Activity(), 60, comment=str(i))
req._reqid = shortuuid.encode(uuid.UUID(int=i))
reqmanager.add(req)
requests.append(req)
yield (reqmanager, requests)
def test_delete_disappeared_requests(connect, reqmanager):
req = reqmanager.add(Request(Activity(), 1, 'comment'))
sched = connect().schedule_maintenance
sched.return_value = {
req.id: {
'time': '2016-04-20T15:12:40.9+00:00'
},
'123abc': {
'time': None
},
}
endm = connect().end_maintenance
reqmanager.schedule()
endm.assert_called_once_with({'123abc': {'result': 'deleted'}})
def test_save_yaml(tmpdir):
r = Request(Activity(), 10, 'my comment', dir=str(tmpdir))
assert r.id is not None
r.save()
with open(str(tmpdir / 'request.yaml')) as f:
assert f.read() == """\
!!python/object:fc.maintenance.request.Request
_reqid: {id}
activity: !!python/object:fc.maintenance.activity.Activity {{}}
attempts: []
comment: my comment
dir: {tmpdir}
estimate: !!python/object:fc.maintenance.estimate.Estimate {{value: 10.0}}
""".format(id=r.id, tmpdir=str(tmpdir))
def test_execute_activity_with_reboot(connect, run, reqmanager, log):
activity = Activity()
activity.reboot_needed = RebootType.WARM
req = reqmanager.add(Request(activity, 1))
req.state = State.due
reqmanager.execute(run_all_now=True)
run.assert_has_calls(
[
call('echo "entering demo"', shell=True, check=True),
call("reboot", check=True, capture_output=True, text=True),
]
)
assert log.has("enter-maintenance")
assert log.has("maintenance-reboot")
assert not log.has("leave-maintenance")
def main():
a = argparse.ArgumentParser(description=__doc__,
epilog="""
If neither a script file or --exec is given, read the activity script from
stdin.
""")
a.add_argument('-c',
'--comment',
metavar='TEXT',
default='',
help='announce upcoming maintenance with this message')
a.add_argument('-d',
'--spooldir',
metavar='DIR',
default=DEFAULT_DIR,
help='request spool dir (default: %(default)s)')
a.add_argument('estimate',
metavar='ESTIMATE',
help='estimate activity duration (suffixes: s, m, h)')
g = a.add_mutually_exclusive_group()
g.add_argument('-e',
'--exec',
metavar='SHELLCMD',
default=False,
help='execute shell command as maintenance activity')
g.add_argument('file',
metavar='FILE',
default=None,
nargs='?',
type=argparse.FileType('r'),
help='execute FILE as maintenance activity')
args = a.parse_args()
if args.file:
act = args.file
elif args.exec:
act = io.StringIO(args.exec + '\n')
else:
act = sys.stdin
rm = ReqManager(spooldir=args.spooldir)
rm.add(
Request(ShellScriptActivity(act), args.estimate, comment=args.comment))
def test_save_yaml(tmpdir):
r = Request(Activity(), 10, "my comment", dir=str(tmpdir))
assert r.id is not None
r.save()
with open(str(tmpdir / "request.yaml")) as f:
assert (
f.read()
== """\
&id001 !!python/object:fc.maintenance.request.Request
_reqid: {id}
_reqmanager: null
activity: !!python/object:fc.maintenance.activity.Activity
request: *id001
attempts: []
comment: my comment
dir: {tmpdir}
estimate: !!python/object:fc.maintenance.estimate.Estimate
value: 10.0
""".format(
id=r.id, tmpdir=str(tmpdir)
)
)
def main():
a = argparse.ArgumentParser(description=__doc__)
a.add_argument('-c',
'--comment',
metavar='TEXT',
default=None,
help='announce upcoming reboot with this message')
a.add_argument('-p',
'--poweroff',
default=False,
action='store_true',
help='power off instead of reboot')
a.add_argument('-d',
'--spooldir',
metavar='DIR',
default=DEFAULT_DIR,
help='request spool dir (default: %(default)s)')
args = a.parse_args()
action = 'poweroff' if args.poweroff else 'reboot'
comment = args.comment if args.comment else (
'Scheduled machine {}'.format(action))
rm = ReqManager(spooldir=args.spooldir)
rm.add(Request(RebootActivity(action), 10 * 60, comment=comment))
def test_dont_add_two_reqs_with_identical_comments(reqmanager):
with reqmanager as rm:
assert rm.add(Request(Activity(), 1, 'comment 1')) is not None
assert rm.add(Request(Activity(), 1, 'comment 1')) is None
assert len(rm.requests) == 1
def test_execute_proceeds_on_connection_error(connect, reqmanager):
connect().mark_node_service_status.side_effect = socket.error()
req = reqmanager.add(Request(Activity(), 1))
req.state = State.due
reqmanager.execute()
assert req.state == State.success
def test_duration_from_activity_duration(tmpdir):
r = Request(FixedDurationActivity(), 1, dir=str(tmpdir))
r.execute()
assert r.duration == 90
def test_list_other_requests(reqmanager):
with reqmanager as rm:
first = rm.add(Request(Activity(), 1))
second = rm.add(Request(Activity(), 1))
assert first.other_requests() == [second]
def test_update_due_should_not_accept_naive_datetimes():
r = Request(Activity(), 1)
with pytest.raises(TypeError):
r.update_due(datetime.datetime(2016, 4, 20, 12, 00))
def test_find_by_comment(reqmanager):
with reqmanager as rm:
rm.add(Request(Activity(), 1, 'comment 1'))
req2 = rm.add(Request(Activity(), 1, 'comment 2'))
with reqmanager as rm:
assert req2 == rm.find_by_comment('comment 2')
def test_execute_catches_errors(tmpdir):
r = Request(FailingActivity(), 1, dir=str(tmpdir))
r.execute()
assert len(r.attempts) == 1
assert 'activity failing' in r.attempts[0].stderr
assert r.attempts[0].returncode != 0