def test_read_hole_timeout(self): measure = Measure(utils.nanotime()) msg = OffsetMessage(measure, is_full=False, offset=1) self.flash_unit.write_offset(data_id, msg) time.sleep(config.FLASH_HOLE_DELAY_THRESHOLD) self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.read, offset) self.assertRaises(flashlib.ErrorFilledHole, self.flash_unit.write, data_id, data)
def _is_likely_hole(self, offset): """Checks whether this offset is likely a hole. An offset is considered a hole when it has been written by the sequencer and at least config.FLASH_HOLE_DELAY_THRESHOLD seconds have passed. """ try: time_since_offset = (utils.nanotime() - self.offset_timestamps[offset]) except KeyError: # This offset has not been received from the sequencer. return False else: return time_since_offset > self.flash_hole_delay_threshold
def append(self, data): ''' string -> (int list * float list) Checking length of data and for every piece of the data, try guessing a server to write to and waiting for response. If a guess fails, retry writing. Every time some feedback helps to guess better next time. Return the list of tokens storing the data. Failure Handling: Sequencer returns specific token ID when the guessing server is already full. Exception Handling: NONE ''' data_len = len(data) if data_len == 0: return [] number_of_tokens = (data_len - 1) // self.config.FLASH_PAGE_SIZE + 1 token_list = [] # try send every piece of data by guessing for i in xrange(number_of_tokens): piece_beg = i * self.config.FLASH_PAGE_SIZE piece_end = (i + 1) * self.config.FLASH_PAGE_SIZE if i == number_of_tokens - 1: piece_end = data_len piece_data = data[piece_beg:piece_end] while True: server_w = self.guess_server() piece_id = self.client_id self.send_to_sequencer(server_w, piece_id) request_timestamp = utils.nanotime() response_w = self.write_to_flash(server_w, piece_data, piece_id) self.update_guess_info(response_w, request_timestamp, server_w) if response_w.status == flash_service_pb2.WriteResponse.SUCCESS: token_list.append(response_w.measure.token) break return token_list
def guess_server(self): ''' None -> int Guess the next server that should be written to. ''' if self.last_state == INITIAL: return random.randint(0, self.config.FLASH_PER_GROUP - 1) elif self.last_state == FULL: return self.last_server + 1 else: # SUCCESS or FAIL here guess_inc = (utils.nanos_to_sec(self.delay + utils.nanotime() - self.largest_timestamp) * self.latest_ips + 1 + self.config.CLIENT_GUESS_OVERESTIMATION) guess_token = int(math.ceil(self.largest_token + guess_inc)) guess_token += random.randint(1, 2) (guess_server, _, _, _) = self.projection.translate(guess_token) return guess_server
def guess_server(self): ''' None -> int Guess the next server that should be written to. ''' if self.last_state == INITIAL: return random.randint(0, self.config.FLASH_PER_GROUP - 1) elif self.last_state == FULL: return self.last_server + 1 else: # SUCCESS or FAIL here guess_inc = ( utils.nanos_to_sec(self.delay + utils.nanotime() - self.largest_timestamp) * self.latest_ips + 1 + self.config.CLIENT_GUESS_OVERESTIMATION) guess_token = int(math.ceil(self.largest_token + guess_inc)) guess_token += random.randint(1, 2) (guess_server, _, _, _) = self.projection.translate(guess_token) return guess_server
def send_to_flash(self, request, token, is_full=False): """If is_full is True, token does not have any meaning, should be ignored. """ (host, port) = self.config.SERVER_ADDR_LIST[request.flash_unit_index] # TODO: initialize RpcService in init service_f = RpcService(flash_service_pb2.FlashService_Stub, port, host) request_f = flash_service_pb2.WriteOffsetRequest() request_f.data_id = request.data_id (_, _, _, request_f.offset) = self.projection.translate(token) request_f.is_full = is_full request_f.measure.token = token request_f.measure.request_timestamp = request.request_timestamp # timestamp is the timestamp for ips token_timestamp = utils.nanotime() request_f.measure.token_timestamp = token_timestamp request_f.measure.ips = self.ips_thread.get_ips() service_f.WriteOffset(request_f, callback=self.callback) sequencing_overhead = token_timestamp - request.request_timestamp self.logger.debug("sequencing overhead for {0}: {1}ns".format( request.data_id, sequencing_overhead))
from fawnlog import flashlib from fawnlog.flash_unit import FlashUnit from fawnlog import utils from test import config Measure = namedtuple("Measure", ["timestamp"]) OffsetMessage = namedtuple("OffsetMessage", ["measure", "is_full", "offset"]) data = "abc" data_id = "id_1" offset = 1 measure = Measure(utils.nanotime()) msg = OffsetMessage(measure, is_full=False, offset=1) full_msg = OffsetMessage(None, is_full=True, offset=-1) class TestFlashUnit(unittest.TestCase): def setUp(self): self.flash_unit = FlashUnit(0, config) self.flash_unit.reset() def tearDown(self): self.flash_unit.close() def test_write_offset_write(self): self.flash_unit.write_offset(data_id, msg)
def __init__(self, data_id, flash_unit_index): self.data_id = data_id self.flash_unit_index = flash_unit_index # when we get the request self.request_timestamp = utils.nanotime()
from collections import namedtuple from fawnlog import flashlib from fawnlog.flash_unit import FlashUnit from fawnlog import utils from test import config Measure = namedtuple("Measure", ["timestamp"]) OffsetMessage = namedtuple("OffsetMessage", ["measure", "is_full", "offset"]) data = "abc" data_id = "id_1" offset = 1 measure = Measure(utils.nanotime()) msg = OffsetMessage(measure, is_full=False, offset=1) full_msg = OffsetMessage(None, is_full=True, offset=-1) class TestFlashUnit(unittest.TestCase): def setUp(self): self.flash_unit = FlashUnit(0, config) self.flash_unit.reset() def tearDown(self): self.flash_unit.close() def test_write_offset_write(self): self.flash_unit.write_offset(data_id, msg) self._write_and_assert()
def write_offset(self, data_id, offset_message): """Writes the offset message for the given data id.""" if not offset_message.is_full: self.offset_timestamps[offset_message.offset] = utils.nanotime() self.offset_buffer.put_offset_message(data_id, offset_message)