def _check_retries(self, get_pairs_data):
errors = {
"errors": [
server_not_member(self.lbs[0].upper(), self.nodes[0]),
"Cloud Server {} does not exist".format(self.nodes[1]),
"Load Balancer Pool {} does not exist".format(
self.lbs[2].upper())
]
}
lbr1 = "d6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2"
noder1 = "a95ae0c4-6ab8-4873-b82f-f8433840cff2"
lbr2 = "e6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2"
noder2 = "e95ae0c4-6ab8-4873-b82f-f8433840cff2"
pairs, data = get_pairs_data(lbr1, noder1, lbr2, noder2)
retried_data = r._sorted_data([(lbr1, noder1), (lbr2, noder2)])
success_resp = {"good": "response"}
seq = [
(self.svc_req_intent(data),
const(stub_json_response(errors, 409))),
(log_intent(
"request-rcv3-bulk", errors, req_body=("jsonified", data)),
noop),
(self.svc_req_intent(retried_data),
const(stub_json_response(success_resp, 204))),
(log_intent(
"request-rcv3-bulk", success_resp,
req_body=("jsonified", retried_data)),
noop)
]
self.assertEqual(
perform_sequence(seq, r.bulk_delete(pairs)), success_resp)
def test_acquire_timeout(self):
"""
acquire_eff creates child node and keeps checking if it is smallest and
eventually gives up by raising `LockTimeout`. It deletes child node
before returning.
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)),
(Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.12)),
(Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.4)),
(DeleteNode(path="/testlock/prefix0000000001", version=-1), noop)
]
self.assertRaises(
LockTimeout, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def test_acquire_blocking_success(self):
"""
acquire_eff creates child, realizes its not the smallest. Tries again
every 0.01 seconds until it succeeds
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)),
(Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0.2)),
(Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))
]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(True, 1)))
def test_all_retries(self):
"""
If bulk_delete returns "server not a member", lb or server deleted
for all attempted pairs then there is no retry and returns None
"""
errors = {
"errors": [
server_not_member(self.lbs[0].upper(), self.nodes[0]),
"Cloud Server {} does not exist".format(self.nodes[1]),
"Load Balancer Pool {} does not exist".format(
self.lbs[2].upper())
]
}
pairs = pset([
(self.lbs[0], self.nodes[1]), # test same server pairs
(self.lbs[2], self.nodes[0]) # test same lb pairs
])
pairs = self.pairs | pairs
data = r._sorted_data(pairs)
seq = [
(self.svc_req_intent(data),
const(stub_json_response(errors, 409))),
(log_intent(
"request-rcv3-bulk", errors, req_body=("jsonified", data)),
noop)
]
self.assertIsNone(perform_sequence(seq, r.bulk_delete(pairs)))
def test_acquire_create_path_success(self):
"""
acquire_eff creates provided path if it doesn't exist
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000000")),
(GetChildren("/testlock"), const(["prefix0000000000"]))
]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_acquire_success(self):
"""
acquire_eff creates child and gets lock as it is the smallest one
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), conste(NodeExistsError())),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000000")),
(GetChildren("/testlock"), const(["prefix0000000000"]))
]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_success(self):
"""
Returns todays scaling groups based on number of tenants fetched
since last time. Updates the current fetch in file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), const(["2", "0.0"])),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [7, 86400.0]), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)],
self.groups))
def test_added(self):
"""
total desired, pending and actual are added to cloud metrics
"""
td = 10
ta = 20
tp = 3
tt = 7
tg = 13
m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60}
md = merge(m, {'metricValue': td, 'metricName': 'ord.desired'})
ma = merge(m, {'metricValue': ta, 'metricName': 'ord.actual'})
mp = merge(m, {'metricValue': tp, 'metricName': 'ord.pending'})
mt = merge(m, {'metricValue': tt, 'metricName': 'ord.tenants'})
mg = merge(m, {'metricValue': tg, 'metricName': 'ord.groups'})
req_data = [md, ma, mp, mt, mg]
log = object()
seq = [
(Func(time.time), const(100)),
(service_request(
ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest",
data=req_data, log=log).intent, noop)
]
eff = add_to_cloud_metrics(
m['ttlInSeconds'], 'ord', td, ta, tp, tt, tg, log=log)
self.assertIsNone(perform_sequence(seq, eff))
def test_added(self):
"""
total desired, pending and actual are added to cloud metrics
"""
metrics = [GroupMetrics('t1', 'g1', 3, 2, 0),
GroupMetrics('t2', 'g1', 4, 4, 1),
GroupMetrics('t2', 'g', 100, 20, 0)]
m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60}
md = merge(m, {'metricValue': 107, 'metricName': 'ord.desired'})
ma = merge(m, {'metricValue': 26, 'metricName': 'ord.actual'})
mp = merge(m, {'metricValue': 1, 'metricName': 'ord.pending'})
mt = merge(m, {'metricValue': 2, 'metricName': 'ord.tenants'})
mg = merge(m, {'metricValue': 3, 'metricName': 'ord.groups'})
mt1d = merge(m, {'metricValue': 3, 'metricName': 'ord.t1.desired'})
mt1a = merge(m, {'metricValue': 2, 'metricName': 'ord.t1.actual'})
mt1p = merge(m, {'metricValue': 0, 'metricName': 'ord.t1.pending'})
mt2d = merge(m, {'metricValue': 104, 'metricName': 'ord.t2.desired'})
mt2a = merge(m, {'metricValue': 24, 'metricName': 'ord.t2.actual'})
mt2p = merge(m, {'metricValue': 1, 'metricName': 'ord.t2.pending'})
req_data = [md, ma, mp, mt, mg, mt1d, mt1a, mt1p, mt2d, mt2a, mt2p]
log = mock_log()
seq = [
(Func(time.time), const(100)),
(service_request(
ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest",
data=req_data, log=log).intent, noop)
]
eff = add_to_cloud_metrics(m['ttlInSeconds'], 'ord', metrics, 2, log)
self.assertIsNone(perform_sequence(seq, eff))
log.msg.assert_called_once_with(
'total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=107, ta=26, tp=1)
def test_multiple_errors(self):
"""
If bulk add returns 409 then multiple errors returned are collected and
raised as a single `BulkErrors`
"""
errors = {
"errors": [
lb_inactive(self.lbs[0]),
"Load Balancer Pool {} does not exist".format(self.lbs[1]),
"Cloud Server {} is unprocessable".format(self.nodes[2])
]
}
seq = [
(self.svc_req_intent(self.data),
const(stub_json_response(errors, 409))),
(log_intent(
"request-rcv3-bulk", errors,
req_body=("jsonified", self.data)),
noop)
]
with self.assertRaises(r.BulkErrors) as ec:
perform_sequence(seq, r.bulk_add(self.pairs))
self.assertEqual(
ec.exception.errors,
pset([r.LBInactive(self.lbs[0]),
r.NoSuchLBError(self.lbs[1]),
r.ServerUnprocessableError(self.nodes[2])])
)
def test_is_acquired_no_children(self):
"""
is_acquired_eff returns False if there are no children
"""
self.lock._node = "/testlock/prefix000000000"
seq = [(GetChildren("/testlock"), const([]))]
self.assertFalse(perform_sequence(seq, self.lock.is_acquired_eff()))
def _check_retries(self, pairs, data, retried_data, errors):
resp = {"response": "yo"}
seq = [
(self.svc_req_intent(data),
const(stub_json_response(errors, 409))),
(log_intent(
"request-rcv3-bulk", errors,
req_body=("jsonified", data)),
noop),
(self.svc_req_intent(retried_data),
const(stub_json_response(resp, 201))),
(log_intent(
"request-rcv3-bulk", resp,
req_body=("jsonified", retried_data)),
noop)
]
self.assertEqual(perform_sequence(seq, r.bulk_add(pairs)), resp)
def test_acquire_delete_child(self):
"""
acquire_eff deletes existing child if it exists
"""
self.lock._node = "/testlock/prefix000000002"
seq = [
(DeleteNode(path="/testlock/prefix000000002", version=-1), noop),
(zk.CreateNode("/testlock"), conste(NodeExistsError())),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000000")),
(GetChildren("/testlock"), const(["prefix0000000000"]))
]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_error_writing(self):
"""
Logs and ignores error writing to the file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), const(["2", "0.0"])),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [7, 86400.0]),
lambda i: raise_(IOError("bad"))),
(LogErr(mock.ANY, "error updating number of tenants", {}), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)],
self.groups))
def test_release_performs(self):
"""
release performs effect from release_eff
"""
self.lock.dispatcher = SequenceDispatcher([
(("release",), const("ret"))])
self.lock.release_eff = intent_func("release")
self.assertEqual(self.successResultOf(self.lock.release()), "ret")
def test_is_acquired_performs(self):
"""
is_acquired performs effect from is_acquired_eff
"""
self.lock.dispatcher = SequenceDispatcher([
(("is_acquired",), const("ret"))])
self.lock.is_acquired_eff = intent_func("is_acquired")
self.assertEqual(self.successResultOf(self.lock.is_acquired()), "ret")
def test_acquire_blocking_no_timeout(self):
"""
When acquire_eff is called without timeout, it creates child, realizes
its not the smallest, tries again every 0.1 seconds without checking
time and succeeds if its the smallest node
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Func(time.time), const(0)),
(Delay(0.1), noop),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(Delay(0.1), noop),
(GetChildren("/testlock"), const(["prefix0000000001"]))
]
self.assertTrue(
perform_sequence(seq, self.lock.acquire_eff(True, None)))
def test_setup_no_groups(self):
"""
``self.s.setup()`` gets groups and does nothing if there are no groups
"""
self.s.dispatcher = SequenceDispatcher([(("ggtc", "cf"), const([]))])
d = self.s.setup()
self.successResultOf(d)
self.assertEqual(self.s._calls, [])
self.assertEqual(self.clock.getDelayedCalls(), [])
def test_added(self):
"""
total desired, pending and actual are added to cloud metrics
"""
metrics = [
GroupMetrics('t1', 'g1', 3, 2, 0),
GroupMetrics('t2', 'g1', 4, 4, 1),
GroupMetrics('t2', 'g', 100, 20, 0),
GroupMetrics('t3', 'g3', 5, 3, 0)
]
config = {"non-convergence-tenants": ["t1"]}
m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60}
md = merge(m, {'metricValue': 112, 'metricName': 'ord.desired'})
ma = merge(m, {'metricValue': 29, 'metricName': 'ord.actual'})
mp = merge(m, {'metricValue': 1, 'metricName': 'ord.pending'})
mt = merge(m, {'metricValue': 3, 'metricName': 'ord.tenants'})
mg = merge(m, {'metricValue': 4, 'metricName': 'ord.groups'})
mt1d = merge(m, {'metricValue': 3, 'metricName': 'ord.t1.desired'})
mt1a = merge(m, {'metricValue': 2, 'metricName': 'ord.t1.actual'})
mt1p = merge(m, {'metricValue': 0, 'metricName': 'ord.t1.pending'})
mt2d = merge(m, {'metricValue': 104, 'metricName': 'ord.t2.desired'})
mt2a = merge(m, {'metricValue': 24, 'metricName': 'ord.t2.actual'})
mt2p = merge(m, {'metricValue': 1, 'metricName': 'ord.t2.pending'})
mt3d = merge(m, {'metricValue': 5, 'metricName': 'ord.t3.desired'})
mt3a = merge(m, {'metricValue': 3, 'metricName': 'ord.t3.actual'})
mt3p = merge(m, {'metricValue': 0, 'metricName': 'ord.t3.pending'})
cd = merge(m, {'metricValue': 109, 'metricName': 'ord.conv_desired'})
ca = merge(m, {'metricValue': 27, 'metricName': 'ord.conv_actual'})
cdiv = merge(m, {
'metricValue': 82,
'metricName': 'ord.conv_divergence'
})
req_data = [
md, ma, mp, mt, mg, mt1d, mt1a, mt1p, mt2d, mt2a, mt2p, mt3d, mt3a,
mt3p, cd, ca, cdiv
]
log = mock_log()
seq = [(Func(time.time), const(100)),
(service_request(ServiceType.CLOUD_METRICS_INGEST,
"POST",
"ingest",
data=req_data,
log=log).intent, noop)]
eff = add_to_cloud_metrics(
m['ttlInSeconds'],
'ord',
metrics,
3, # number of tenants
config,
log)
self.assertIsNone(perform_sequence(seq, eff))
log.msg.assert_called_once_with(
'total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=112,
ta=29,
tp=1)
def test_no_last_info(self):
"""
Returns first 5 non-convergence tenants if could not fetch last info
from file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), lambda i: raise_(IOError("e"))),
(LogErr(mock.ANY, "error reading previous number of tenants", {}),
noop),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [5, 86400.0]), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 7)],
self.groups))
def test_acquire_performs(self):
"""
acquire performs effect from acquire_eff
"""
self.lock.dispatcher = SequenceDispatcher([
(("acquire", "blocking", "timeout"), const("ret"))])
self.lock.acquire_eff = intent_func("acquire")
self.assertEqual(
self.successResultOf(self.lock.acquire("blocking", "timeout")),
"ret")
def test_is_acquired_not_first_child(self):
"""
is_acquired_eff returns False if its not is not the first child
"""
self.lock._node = "/testlock/prefix0000000001"
seq = [
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"]))
]
self.assertFalse(perform_sequence(seq, self.lock.is_acquired_eff()))
def test_is_acquired_first_child(self):
"""
is_acquired_eff returns True if it's node is the first child
"""
self.lock._node = "/testlock/prefix0000000000"
seq = [
(GetChildren("/testlock"),
const(["prefix0000000001", "prefix0000000000"]))
]
self.assertTrue(perform_sequence(seq, self.lock.is_acquired_eff()))
def test_lock_already_acquired(self):
"""
If lock is already acquired, it will just call eff
"""
self.lb.acquired = True
seq = [("call", const("eff_return"))]
self.assertEqual(
perform_sequence(
seq,
zk.call_if_acquired(self.lock, Effect("call"))),
("eff_return", False))
def test_inactive_group(self):
"""
Convergence is not triggerred on in-ACTIVE group
"""
self.state.status = ScalingGroupStatus.ERROR
seq = [
(GetScalingGroupInfo(tenant_id="tid", group_id="gid"),
const(("group", self.manifest))),
]
self.assertIsNone(
perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def test_active_paused(self):
"""
Convergence is not triggerred on ACTIVE paused group
"""
self.state.paused = True
seq = [
(GetScalingGroupInfo(tenant_id="tid", group_id="gid"),
const(("group", self.manifest))),
]
self.assertIsNone(
perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def test_acquire_nonblocking_fails(self):
"""
acquire_eff creates child and returns False immediately after finding
its not the smallest child when blocking=False. It deletes child node
before returning.
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"),
const(["prefix0000000000", "prefix0000000001"])),
(DeleteNode(path="/testlock/prefix0000000001", version=-1), noop)
]
self.assertFalse(
perform_sequence(seq, self.lock.acquire_eff(False, None)))
def test_acquire_other_error(self):
"""
If acquire_eff internally raises any error then it tries to delete
child node before returning.
"""
seq = [
(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode(
"/testlock/prefix", value="id",
ephemeral=True, sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"), conste(SessionExpiredError())),
(DeleteNode(path="/testlock/prefix0000000001", version=-1),
conste(SessionExpiredError()))
]
self.assertRaises(
SessionExpiredError, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def test_without_metrics(self):
"""
Doesnt add metrics to blueflood if metrics config is not there
"""
sequence = SequenceDispatcher([(GetAllValidGroups(), const(self.groups))])
self.get_dispatcher.return_value = sequence
del self.config["metrics"]
with sequence.consume():
d = collect_metrics("reactor", self.config, self.log)
self.assertEqual(self.successResultOf(d), "metrics")
self.assertFalse(self.add_to_cloud_metrics.called)
def test_filtered(self):
"""
Only convgergence enabled tenants are returned
"""
conf = {"non-convergence-tenants": ["t1"]}
groups = [{"tenantId": "t1", "groupId": "g1"},
{"tenantId": "t1", "groupId": "g12"},
{"tenantId": "t2", "groupId": "g2"},
{"tenantId": "t3", "groupId": "g3"}]
eff = sh.get_groups_to_converge(conf.get)
seq = [(GetAllValidGroups(), const(groups))]
self.assertEqual(perform_sequence(seq, eff), groups[2:])
def test_lock_acquired(self):
"""
When lock is not acquired, it is tried and if successful calls eff
"""
self.lb.acquired = False
self.lb.acquire_call = (False, None, True)
seq = [("call", const("eff_return"))]
self.assertEqual(
perform_sequence(
seq,
zk.call_if_acquired(self.lock, Effect("call"))),
("eff_return", True))
def test_without_metrics(self):
"""
Doesnt add metrics to blueflood if metrics config is not there
"""
sequence = SequenceDispatcher([(GetAllValidGroups(),
const(self.groups))])
self.get_dispatcher.return_value = sequence
del self.config["metrics"]
with sequence.consume():
d = collect_metrics("reactor", self.config, self.log)
self.assertEqual(self.successResultOf(d), "metrics")
self.assertFalse(self.add_to_cloud_metrics.called)
def setUp(self):
"""
mock dependent functions
"""
self.connect_cass_servers = patch(
self, 'otter.metrics.connect_cass_servers')
self.client = mock.Mock(spec=['disconnect'])
self.client.disconnect.return_value = succeed(None)
self.connect_cass_servers.return_value = self.client
self.log = mock_log()
self.get_all_metrics = patch(self,
'otter.metrics.get_all_metrics',
return_value=succeed("metrics"))
self.groups = [{
"tenantId": "t1",
"groupId": "g1",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g2",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g12",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t3",
"groupId": "g3",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t2",
"groupId": "g11",
"launch_config": '{"type": "launch_server"}'
}]
self.lc_groups = {"t1": self.groups[:2], "t2": [self.groups[-1]]}
self.add_to_cloud_metrics = patch(self,
'otter.metrics.add_to_cloud_metrics',
side_effect=intent_func("atcm"))
self.config = {
'cassandra': 'c',
'identity': identity_config,
'metrics': {
'service': 'ms',
'tenant_id': 'tid',
'region': 'IAD',
'ttl': 200,
"last_tenant_fpath": "lpath"
},
'region': 'r',
'cloudServersOpenStack': 'nova',
'cloudLoadBalancers': 'clb',
'cloudOrchestration': 'orch',
'rackconnect': 'rc',
"non-convergence-tenants": ["ct"]
}
self.sequence = SequenceDispatcher([
(GetAllValidGroups(), const(self.groups)),
(TenantScope(mock.ANY, "tid"),
nested_sequence([(("atcm", 200, "r", "metrics", 2, self.config,
self.log, False), noop)]))
])
self.get_dispatcher = patch(self,
"otter.metrics.get_dispatcher",
return_value=self.sequence)