def test_read_hole_timeout(self):
measure = Measure(utils.nanotime())
msg = OffsetMessage(measure, is_full=False, offset=1)
self.flash_unit.write_offset(data_id, msg)
time.sleep(config.FLASH_HOLE_DELAY_THRESHOLD)
self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.read,
offset)
self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.write,
data_id, data)
def test_read_hole_timeout(self):
measure = Measure(utils.nanotime())
msg = OffsetMessage(measure, is_full=False, offset=1)
self.flash_unit.write_offset(data_id, msg)
time.sleep(config.FLASH_HOLE_DELAY_THRESHOLD)
self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.read,
offset)
self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.write,
data_id, data)
def _is_likely_hole(self, offset):
"""Checks whether this offset is likely a hole.
An offset is considered a hole when it has been written by the
sequencer and at least config.FLASH_HOLE_DELAY_THRESHOLD seconds
have passed.
"""
try:
time_since_offset = (utils.nanotime() -
self.offset_timestamps[offset])
except KeyError:
# This offset has not been received from the sequencer.
return False
else:
return time_since_offset > self.flash_hole_delay_threshold
def append(self, data):
''' string -> (int list * float list)
Checking length of data and for every piece of the data, try
guessing a server to write to and waiting for response. If a
guess fails, retry writing. Every time some feedback helps to
guess better next time. Return the list of tokens storing the
data.
Failure Handling:
Sequencer returns specific token ID when the guessing server
is already full.
Exception Handling:
NONE
'''
data_len = len(data)
if data_len == 0:
return []
number_of_tokens = (data_len - 1) // self.config.FLASH_PAGE_SIZE + 1
token_list = []
# try send every piece of data by guessing
for i in xrange(number_of_tokens):
piece_beg = i * self.config.FLASH_PAGE_SIZE
piece_end = (i + 1) * self.config.FLASH_PAGE_SIZE
if i == number_of_tokens - 1:
piece_end = data_len
piece_data = data[piece_beg:piece_end]
while True:
server_w = self.guess_server()
piece_id = self.client_id
self.send_to_sequencer(server_w, piece_id)
request_timestamp = utils.nanotime()
response_w = self.write_to_flash(server_w, piece_data,
piece_id)
self.update_guess_info(response_w, request_timestamp, server_w)
if response_w.status == flash_service_pb2.WriteResponse.SUCCESS:
token_list.append(response_w.measure.token)
break
return token_list
def append(self, data):
''' string -> (int list * float list)
Checking length of data and for every piece of the data, try
guessing a server to write to and waiting for response. If a
guess fails, retry writing. Every time some feedback helps to
guess better next time. Return the list of tokens storing the
data.
Failure Handling:
Sequencer returns specific token ID when the guessing server
is already full.
Exception Handling:
NONE
'''
data_len = len(data)
if data_len == 0:
return []
number_of_tokens = (data_len - 1) // self.config.FLASH_PAGE_SIZE + 1
token_list = []
# try send every piece of data by guessing
for i in xrange(number_of_tokens):
piece_beg = i * self.config.FLASH_PAGE_SIZE
piece_end = (i + 1) * self.config.FLASH_PAGE_SIZE
if i == number_of_tokens - 1:
piece_end = data_len
piece_data = data[piece_beg:piece_end]
while True:
server_w = self.guess_server()
piece_id = self.client_id
self.send_to_sequencer(server_w, piece_id)
request_timestamp = utils.nanotime()
response_w = self.write_to_flash(server_w, piece_data, piece_id)
self.update_guess_info(response_w, request_timestamp, server_w)
if response_w.status == flash_service_pb2.WriteResponse.SUCCESS:
token_list.append(response_w.measure.token)
break
return token_list
def guess_server(self):
''' None -> int
Guess the next server that should be written to.
'''
if self.last_state == INITIAL:
return random.randint(0, self.config.FLASH_PER_GROUP - 1)
elif self.last_state == FULL:
return self.last_server + 1
else:
# SUCCESS or FAIL here
guess_inc = (utils.nanos_to_sec(self.delay + utils.nanotime() -
self.largest_timestamp) * self.latest_ips + 1 +
self.config.CLIENT_GUESS_OVERESTIMATION)
guess_token = int(math.ceil(self.largest_token + guess_inc))
guess_token += random.randint(1, 2)
(guess_server, _, _, _) = self.projection.translate(guess_token)
return guess_server
def guess_server(self):
''' None -> int
Guess the next server that should be written to.
'''
if self.last_state == INITIAL:
return random.randint(0, self.config.FLASH_PER_GROUP - 1)
elif self.last_state == FULL:
return self.last_server + 1
else:
# SUCCESS or FAIL here
guess_inc = (
utils.nanos_to_sec(self.delay + utils.nanotime() -
self.largest_timestamp) * self.latest_ips +
1 + self.config.CLIENT_GUESS_OVERESTIMATION)
guess_token = int(math.ceil(self.largest_token + guess_inc))
guess_token += random.randint(1, 2)
(guess_server, _, _, _) = self.projection.translate(guess_token)
return guess_server
def send_to_flash(self, request, token, is_full=False):
"""If is_full is True, token does not have any meaning,
should be ignored.
"""
(host, port) = self.config.SERVER_ADDR_LIST[request.flash_unit_index]
# TODO: initialize RpcService in init
service_f = RpcService(flash_service_pb2.FlashService_Stub, port, host)
request_f = flash_service_pb2.WriteOffsetRequest()
request_f.data_id = request.data_id
(_, _, _, request_f.offset) = self.projection.translate(token)
request_f.is_full = is_full
request_f.measure.token = token
request_f.measure.request_timestamp = request.request_timestamp
# timestamp is the timestamp for ips
token_timestamp = utils.nanotime()
request_f.measure.token_timestamp = token_timestamp
request_f.measure.ips = self.ips_thread.get_ips()
service_f.WriteOffset(request_f, callback=self.callback)
sequencing_overhead = token_timestamp - request.request_timestamp
self.logger.debug("sequencing overhead for {0}: {1}ns".format(
request.data_id, sequencing_overhead))
def send_to_flash(self, request, token, is_full=False):
"""If is_full is True, token does not have any meaning,
should be ignored.
"""
(host, port) = self.config.SERVER_ADDR_LIST[request.flash_unit_index]
# TODO: initialize RpcService in init
service_f = RpcService(flash_service_pb2.FlashService_Stub, port, host)
request_f = flash_service_pb2.WriteOffsetRequest()
request_f.data_id = request.data_id
(_, _, _, request_f.offset) = self.projection.translate(token)
request_f.is_full = is_full
request_f.measure.token = token
request_f.measure.request_timestamp = request.request_timestamp
# timestamp is the timestamp for ips
token_timestamp = utils.nanotime()
request_f.measure.token_timestamp = token_timestamp
request_f.measure.ips = self.ips_thread.get_ips()
service_f.WriteOffset(request_f, callback=self.callback)
sequencing_overhead = token_timestamp - request.request_timestamp
self.logger.debug("sequencing overhead for {0}: {1}ns".format(
request.data_id, sequencing_overhead))
from fawnlog import flashlib
from fawnlog.flash_unit import FlashUnit
from fawnlog import utils
from test import config
Measure = namedtuple("Measure", ["timestamp"])
OffsetMessage = namedtuple("OffsetMessage", ["measure", "is_full", "offset"])
data = "abc"
data_id = "id_1"
offset = 1
measure = Measure(utils.nanotime())
msg = OffsetMessage(measure, is_full=False, offset=1)
full_msg = OffsetMessage(None, is_full=True, offset=-1)
class TestFlashUnit(unittest.TestCase):
def setUp(self):
self.flash_unit = FlashUnit(0, config)
self.flash_unit.reset()
def tearDown(self):
self.flash_unit.close()
def test_write_offset_write(self):
self.flash_unit.write_offset(data_id, msg)
def __init__(self, data_id, flash_unit_index):
self.data_id = data_id
self.flash_unit_index = flash_unit_index
# when we get the request
self.request_timestamp = utils.nanotime()
def __init__(self, data_id, flash_unit_index):
self.data_id = data_id
self.flash_unit_index = flash_unit_index
# when we get the request
self.request_timestamp = utils.nanotime()
from collections import namedtuple
from fawnlog import flashlib
from fawnlog.flash_unit import FlashUnit
from fawnlog import utils
from test import config
Measure = namedtuple("Measure", ["timestamp"])
OffsetMessage = namedtuple("OffsetMessage", ["measure", "is_full", "offset"])
data = "abc"
data_id = "id_1"
offset = 1
measure = Measure(utils.nanotime())
msg = OffsetMessage(measure, is_full=False, offset=1)
full_msg = OffsetMessage(None, is_full=True, offset=-1)
class TestFlashUnit(unittest.TestCase):
def setUp(self):
self.flash_unit = FlashUnit(0, config)
self.flash_unit.reset()
def tearDown(self):
self.flash_unit.close()
def test_write_offset_write(self):
self.flash_unit.write_offset(data_id, msg)
self._write_and_assert()
def write_offset(self, data_id, offset_message):
"""Writes the offset message for the given data id."""
if not offset_message.is_full:
self.offset_timestamps[offset_message.offset] = utils.nanotime()
self.offset_buffer.put_offset_message(data_id, offset_message)