def test_promise(self):
promise = Promise()
self.assertEqual(promise.prepare, None)
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
promise = Promise(prepare=prepare)
self.assertEqual(promise.prepare, prepare)
self.assertEqual(promise.to_json(), {'prepare': prepare.to_json()})
def test_success(self):
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
success = Success(prepare=prepare)
self.assertEqual(success.to_json(), {
'status': 'SUCCESS',
'prepare': prepare.to_json()
})
def test_prepare(self):
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
self.assertEqual(prepare.to_json(), {
'id': 3,
'key': 'biz',
'predicate': 'set',
'argument': 'a'
})
target = Prepare._id
prepare = Prepare(key='buzz', predicate='a', argument='b')
self.assertEqual(prepare.to_json(), {
'id': target,
'key': 'buzz',
'predicate': 'a',
'argument': 'b'
})
_ = yield self.assert_send_works(prepare, '/prepare')
request = tornado.httpclient.HTTPRequest(
body=json.dumps(prepare.to_json()),
method='POST',
headers={'Content-Type': 'application/json'},
url='/prepare')
target = Prepare.from_request(request)
self.assertEqual(target.to_json(), prepare.to_json())
def post(self):
prepare = Prepare.from_request(self.request)
in_progress = current_promises.get(prepare.key)
last_accepted = completed_rounds.highest_numbered(prepare.key)
if in_progress:
logger.info("Promise in progress already %s", in_progress)
if in_progress.prepare.id == prepare.id:
raise Exception("Prepare IDs match.")
if in_progress.prepare.id > prepare.id:
# Some replica has issued a higher promise
# than ours. Abort.
logger.warning("Existing promise is higher.")
self.respond(code=400, message=in_progress)
elif last_accepted is None or (
in_progress.prepare.id > last_accepted.prepare.id
): # >= since we could have just learned but not removed the existing process because this is all async
# Complete the in-progress promise first
# Possible for the incoming promise to have the same ID as the existing one.
logger.info("Must complete earlier promise first: %s",
in_progress)
self.respond(code=200, message=in_progress)
else:
logger.info("New promise is higher. Issuing promise.")
self.respond(code=200, message=Promise())
elif last_accepted is None or prepare.id > last_accepted.prepare.id:
logger.info("Adding a new promise for prepare %s", prepare)
current_promises.add(Promise(prepare=prepare))
self.respond(code=200, message=Promise())
else:
logger.warning(
"Prepare has a lower ID than the last accepted proposal")
logger.warning("prepare: %s, last_accepted: %s", prepare,
last_accepted)
self.respond(code=400, message=last_accepted)
def test_allows_non_conflicting_writes(self):
prepare = Prepare(id=0, key='foo', predicate='set', argument='a')
promise = Promise()
prepare_success = mock.Mock()
prepare_success.code = 200
prepare_success.body = json.dumps(promise.to_json())
fut = tornado.concurrent.Future()
fut.set_result([prepare_success, prepare_success])
propose_success = mock.Mock()
propose_success.code = 200
propose_success.body = json.dumps(Promise(prepare=prepare).to_json())
propose_fut = tornado.concurrent.Future()
propose_fut.set_result([propose_success, propose_success])
learn_success = mock.Mock()
learn_success.code = 200
learn_success.body = ''
learn_fut = tornado.concurrent.Future()
learn_fut.set_result([learn_success, learn_success])
with mock.patch('models.Prepare.send', return_value=fut) as send:
with mock.patch('models.Propose.send',
return_value=propose_fut) as propose_send:
with mock.patch('models.Learn.fanout',
return_value=learn_fut) as learn_fanout:
response = self.post('/write',
body={
'key': 'foo',
'predicate': 'set',
'argument': 'a'
})
self.assertEqual(response.code, 200)
def test_accept(self):
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
accept = Accept(prepare=prepare)
response = mock.Mock()
response.body = json.dumps(accept.to_json())
target = Accept.from_response(response)
self.assertEqual(accept.to_json(), target.to_json())
def test_promises(self):
prepare1 = Prepare(id=1, key='biz', predicate='pa', argument='a')
prepare2 = Prepare(id=2, key='biz', predicate='pb', argument='b')
prepare3 = Prepare(id=3, key='biz', predicate='pc', argument='c')
prepare4 = Prepare(id=4, key='baz', predicate='pd', argument='d')
promise1 = Promise(prepare=prepare1)
promise2 = Promise(prepare=prepare2)
promise3 = Promise(prepare=prepare3)
promise4 = Promise(prepare=prepare4)
promises = Promises([promise1, promise2, promise3, promise4])
self.assertEqual(promises.highest_numbered().to_json(),
promise4.to_json())
self.assertEqual(
promises.highest_numbered(key='biz').to_json(), promise3.to_json())
def test_learn(self):
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
learn = Learn(prepare=prepare)
request = tornado.httpclient.HTTPRequest(
body=json.dumps(learn.to_json()),
method='POST',
headers={'Content-Type': 'application/json'},
url='/learn')
self.assertEqual(learn.to_json(),
Learn.from_request(request).to_json())
def test_to_json(self):
prepare = Prepare(id=1, key='foo', predicate='incr', argument=1)
phase = Phase(prepare=prepare)
self.assertEqual(phase.to_json(), {
'prepare': {
'id': 1,
'key': 'foo',
'predicate': 'incr',
'argument': 1
}
})
def test_propose(self):
prepare = Prepare(id=3, key='biz', predicate='set', argument='a')
propose = Propose(prepare=prepare)
self.assert_send_works(propose, '/propose')
request = tornado.httpclient.HTTPRequest(
body=json.dumps(propose.to_json()),
method='POST',
headers={'Content-Type': 'application/json'},
url='/propose')
target = Propose.from_request(request)
self.assertEqual(propose.to_json(), target.to_json())
def test_returns_lower_numbered_in_progress_promises(self):
lower_prepare = Prepare(id=0, key='foo', predicate='set', argument='a')
higher_prepare = Prepare(id=1,
key='foo',
predicate='set',
argument='b')
success = self.post('/prepare', lower_prepare.to_json())
self.assertEqual(success.code, 200)
self.assertEqual(
Promise.from_response(success).to_json(), {'prepare': None})
self.assertEqual(agent.current_promises.highest_numbered().to_json(),
{'prepare': lower_prepare.to_json()})
failure = self.post('/prepare', higher_prepare.to_json())
self.assertEqual(failure.code, 200)
target = Promise.from_response(failure)
self.assertEqual(target.to_json(),
{'prepare': lower_prepare.to_json()})
def test_send(self):
prepare = Prepare(id=1, key='foo', predicate='incr', argument=1)
phase = Phase(prepare=prepare)
fut = tornado.concurrent.Future()
response = mock.Mock()
response.body = json.dumps(phase.to_json())
fut.set_result(response)
phase.endpoint = '/testing'
client = mock.Mock()
client.fetch = mock.Mock()
client.fetch.return_value = fut
with mock.patch('tornado.httpclient.AsyncHTTPClient',
return_value=client):
responses = yield phase.send(agents.quorum())
self.assertEqual(len(responses), len(agents.quorum()))
def post(self):
"""
{
key: <str>,
predicate: <str>,
argument: <str|int>
}
"""
request = json.loads(self.request.body)
prepare = Prepare(**request)
prepares = collections.deque([prepare])
current_promises.add(Promise(prepare=prepare))
quorum = agents.quorum(excluding=options.port)
while prepares: # TODO: Timeout here.
prepare = prepares.popleft()
logging.info("Sending prepare for %s", prepare)
responses, issued, conflicting = yield prepare.send(quorum)
logger.info("Got %s issued and %s conflicting", len(issued),
len(conflicting))
logger.info("Response codes: %s",
", ".join([str(r.code) for r in responses]))
if conflicting: # Issue another promise.
logger.warning(
"%s was pre-empted by a higher ballot. retrying.".format(
prepare.id))
prepares.append(
Prepare(key=prepare.key,
predicate=prepare.predicate,
argument=prepare.argument))
continue
elif len(issued) != len(quorum):
raise tornado.web.HTTPError(
status_code=500,
log_message='FAILED to acquire quorum on Promise')
promises = Promises.from_responses(responses)
earlier_promise = promises.highest_numbered()
if earlier_promise and earlier_promise not in current_promises: # Repair.
prepares.append(prepare)
prepare = earlier_promise.prepare
# Now we have a promise.
responses, issued, conflicting = yield Propose(
prepare=prepare).send(quorum)
if len(issued) == len(quorum):
logger.info("Got success for propose %s. Learning...", prepare)
successes = yield Learn(prepare).fanout(expected=Success)
elif conflicting:
logger.error("Conflicting promise detected. Will re-issue.")
raise Exception("Conflicting promise detected.")
else:
raise tornado.web.HTTPError(
status_code=500,
log_message='Failed to acquire quorum on Accept')
if len(successes) == len(agents.all()):
current_promises.remove(prepare)
self.respond(Success(prepare))
else:
logger.error("Got %s successes with a required quorum of %s",
len(successes), len(agents.all()))
raise tornado.web.HTTPError(
status_code=500,
log_message='Failed to acquire quorum on Learn')
def test_fanout_raises_not_implemented(self):
prepare = Prepare(id=1, key='foo', predicate='incr', argument=1)
phase = Phase(prepare=prepare)
with self.assertRaises(NotImplementedError):
should_fail = yield phase.fanout()
def get_prepare(self):
return Prepare(id=1, key='foo', predicate='set', argument='a')