def run(self): self.stopped = False logger.info('Thread started!') while not self.stopped: try: neighbours = self.operator.get_neighbours(NT_SUPERIOR) if not neighbours: time.sleep(INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT) continue for neighbour in neighbours: logger.debug('Checking range table at %s'%neighbour) mod_index = self.operator.ranges_table.get_mod_index() params = {'mod_index': mod_index} packet_obj = FabnetPacketRequest(method='CheckHashRangeTable', sender=self.operator.self_address, parameters=params) rcode, rmsg = self.operator.call_node(neighbour, packet_obj) for i in xrange(CHECK_HASH_TABLE_TIMEOUT): if self.stopped: break time.sleep(1) except Exception, err: logger.error(str(err))
def load(self, ranges_dump): self.__lock.acquire() try: is_old_ex = False if self.__ranges: is_old_ex = self.__ranges self.__ranges, self.__last_dm, self.__mod_index = pickle.loads(ranges_dump) if is_old_ex: s = 'OLD(-)/NEW(+) HASHES IN TABLES:\n' for range_o in self.iter_table(): for old_range in is_old_ex: if old_range.to_str() == range_o.to_str(): break else: s += '+ %s\n'%range_o.to_str() for old_range in is_old_ex: for range_o in self.iter_table(): if old_range.to_str() == range_o.to_str(): break else: s += '- %s\n'%old_range.to_str() logger.info(s) self.__blocked.clear() finally: self.__lock.release()
def _move_range(self, range_obj): logger.info('Node %s went from DHT. Updating hash range table on network...'%range_obj.node_address) rm_lst = [(range_obj.start, range_obj.end, range_obj.node_address)] parameters = {'append': [], 'remove': rm_lst} req = FabnetPacketRequest(method='UpdateHashRangeTable', sender=self.self_address, parameters=parameters) self.call_network(req)
def check_neighbours(self): ka_packet = FabnetPacketRequest(method=KEEP_ALIVE_METHOD, sender=self.self_address, sync=True) superiors = self.get_neighbours(NT_SUPERIOR) remove_nodes = [] for superior in superiors: resp = self.fri_client.call_sync(superior, ka_packet) cnt = 0 self.__lock.acquire() try: if self.__superior_keep_alives.get(superior, None) is None: self.__superior_keep_alives[superior] = 0 if resp.ret_code == RC_OK: self.__superior_keep_alives[superior] = 0 elif resp.ret_code == RC_NOT_MY_NEIGHBOUR: del self.__superior_keep_alives[superior] remove_nodes.append((NT_SUPERIOR, superior, False)) else: self.__superior_keep_alives[superior] += 1 cnt = self.__superior_keep_alives[superior] finally: self.__lock.release() if cnt == KEEP_ALIVE_TRY_COUNT: logger.info('Neighbour %s does not respond. removing it...'%superior) remove_nodes.append((NT_SUPERIOR, superior, True)) self.__lock.acquire() del self.__superior_keep_alives[superior] self.__lock.release() #check upper nodes... uppers = self.get_neighbours(NT_UPPER) self.__lock.acquire() try: cur_dt = datetime.now() for upper in uppers: ka_dt = self.__upper_keep_alives.get(upper, None) if ka_dt == None: self.__upper_keep_alives[upper] = datetime.now() continue delta = cur_dt - ka_dt if delta.total_seconds() >= KEEP_ALIVE_MAX_WAIT_TIME: logger.info('No keep alive packets from upper neighbour %s. removing it...'%upper) remove_nodes.append((NT_UPPER, upper, True)) del self.__upper_keep_alives[upper] finally: self.__lock.release() for n_type, nodeaddr, is_not_respond in remove_nodes: self.remove_neighbour(n_type, nodeaddr) if is_not_respond: self.on_neigbour_not_respond(n_type, nodeaddr) if remove_nodes: self._rebalance_nodes()
def run(self): try: self.__server = wsgiserver.CherryPyWSGIServer((self.__host, self.__port), self.__service.web_app,) logger.info('CAServer thread is started') self.__server.start() except Exception, err: logger.error('[CAServer] %s'%err)
def _send_to_neighbours(self, packet): packet.sender = self.self_address neighbours = self.get_neighbours(NT_SUPERIOR) for neighbour in neighbours: ret, message = self.fri_client.call(neighbour, packet) if ret != RC_OK: logger.info('Neighbour %s does not respond! Details: %s'%(neighbour, message))
def _process_replicas(self): dht_range = self.operator.get_dht_range() for digest, data, file_path in dht_range.iter_replicas(): logger.info('Processing replica %s'%digest) if self._put_data(digest, data, is_replica=True): logger.debug('Removing %s from local replicas'%digest) os.remove(file_path) else: logger.debug('data block with key=%s is send from reservation range'%digest)
def _get_ranges_table(self, from_addr, mod_index): if not self.operator.ranges_table.empty(): for i in xrange(RANGES_TABLE_FLAPPING_TIMEOUT): time.sleep(1) c_mod_index = self.operator.ranges_table.get_mod_index() if c_mod_index == mod_index: return logger.info('Ranges table is invalid! Requesting table from %s'% from_addr) self._init_operation(from_addr, 'GetRangesTable', {})
def update_dht_range(self, new_dht_range): self._lock() old_dht_range = self.__dht_range self.__dht_range = new_dht_range self._unlock() old_dht_range.move_to_trash() dht_range = self.get_dht_range() logger.info('New node range: %040x-%040x'%(dht_range.get_start(), dht_range.get_end()))
def _manage_new_neighbours(self): logger.info('Discovered neigbours: %s and %s'%(self.__new_superior, self.__new_upper)) parameters = { 'neighbour_type': NT_SUPERIOR, 'operation': MNO_APPEND, 'node_address': self.operator.self_address } self._init_operation(self.__new_superior, 'ManageNeighbour', parameters) parameters = { 'neighbour_type': NT_UPPER, 'operation': MNO_APPEND, 'node_address': self.operator.self_address } self._init_operation(self.__new_upper, 'ManageNeighbour', parameters)
def start_as_dht_member(self): if self.status == DS_DESTROYING: return self.status = DS_INITIALIZE dht_range = self.get_dht_range() nochange = False curr_start = dht_range.get_start() curr_end = dht_range.get_end() if dht_range.is_max_range() or self.__split_requests_cache: new_range = self.__get_next_max_range() else: new_range = self.__get_next_range_near(curr_start, curr_end) if new_range: if (new_range.start != curr_start or new_range.end != curr_end): nochange = True if new_range.node_address == self.self_address: self.set_status_to_normalwork() return if new_range is None: #wait and try again if self.__start_dht_try_count == DHT_CYCLE_TRY_COUNT: logger.error('Cant initialize node as a part of DHT') self.__start_dht_try_count = 0 return logger.info('No ready range for me on network... So, sleep and try again') self.__start_dht_try_count += 1 self.__split_requests_cache = [] time.sleep(WAIT_RANGE_TIMEOUT) return self.start_as_dht_member() if nochange: new_dht_range = dht_range else: new_dht_range = FSHashRanges(long(new_range.start + new_range.length()/2+1), long(new_range.end), self.save_path) self.update_dht_range(new_dht_range) new_dht_range.restore_from_trash() #try getting new range data from trash self.__split_requests_cache.append(new_range.node_address) logger.info('Call SplitRangeRequest to %s'%(new_range.node_address,)) parameters = { 'start_key': new_dht_range.get_start(), 'end_key': new_dht_range.get_end() } req = FabnetPacketRequest(method='SplitRangeRequest', sender=self.self_address, parameters=parameters) ret_code, ret_msg = self.call_node(new_range.node_address, req) if ret_code != RC_OK: logger.error('Cant start SplitRangeRequest operation on node %s. Details: %s'%(new_range.node_address, ret_msg)) return self.start_as_dht_member()
def check_dht_range(self): if self.status == DS_INITIALIZE: return dht_range = self.get_dht_range() start = dht_range.get_start() end = dht_range.get_end() range_obj = self.ranges_table.find(start) if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address: logger.error('DHT range on this node is not found in ranges_table') logger.info('Trying reinit node as DHT member...') self.start_as_dht_member()
def run(self): logger.info('started') self.stopped = False while not self.stopped: try: #self._check_range_free_size() logger.debug('MonitorDHTRanges iteration...') self._process_reservation_range() self._process_replicas() except Exception, err: logger.error('[MonitorDHTRanges] %s'% err) finally:
def callback(self, packet, sender=None): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ logger.info('Trying select other hash range...') self.operator.start_as_dht_member()
def callback(self, packet, sender=None): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ if packet.ret_code != RC_OK: logger.info('[GetRangesTableCallback] Node %s does not initialized yet...'%packet.from_node) return logger.info('Recevied ranges table') self.operator.ranges_table.load(str(packet.ret_parameters['ranges_table'])) logger.info('Ranges table is loaded to fabnet node') if self.operator.status == DS_INITIALIZE: logger.info('Starting node as DHT member') self.operator.start_as_dht_member() else: self.operator.check_dht_range()
def process(self, packet): """In this method should be implemented logic of processing reuqest packet from sender node @param packet - object of FabnetPacketRequest class @return object of FabnetPacketResponse or None for disabling packet response to sender """ logger.info('Canceled range splitting! Joining subranges.') dht_range = self.operator.get_dht_range() dht_range.join_subranges() return FabnetPacketResponse()
def run(self): self.stopped = False logger.info('Check neighbours thread is started!') while not self.stopped: try: t0 = datetime.now() self.operator.check_neighbours() proc_dt = datetime.now() - t0 except Exception, err: logger.error('[CheckNeighboursThread] %s'%err) finally:
def process(self, packet): """In this method should be implemented logic of processing reuqest packet from sender node @param packet - object of FabnetPacketRequest class @return object of FabnetPacketResponse or None for disabling packet response to sender """ if self.operator.status == DS_INITIALIZE: return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Node is not initialized yet!') ranges_table = self.operator.ranges_table.dump() logger.info('Sending ranges table to %s'%packet.sender) return FabnetPacketResponse(ret_parameters={'ranges_table': ranges_table})
def run(self): logger.info('Starting connection handler thread...') self.__bind_socket() logger.info('Connection handler thread started!') while not self.stopped: try: (sock, addr) = self.sock.accept() if self.stopped: sock.close() break self.queue.put(sock) except Exception, err: logger.error('[accept] %s'%err)
def remove_neighbour(self, neighbour_type, address): self.__lock.acquire() try: if neighbour_type == NT_SUPERIOR: if address not in self.superior_neighbours: return del self.superior_neighbours[self.superior_neighbours.index(address)] logger.info('Superior neighbour %s removed'%address) elif neighbour_type == NT_UPPER: if address not in self.upper_neighbours: return del self.upper_neighbours[self.upper_neighbours.index(address)] logger.info('Upper neighbour %s removed'%address) else: raise OperException('Neigbour type "%s" is invalid!'%neighbour_type) finally: self.__lock.release()
def start(self): self.stopped = False self.__workers_manager_thread.start() self.__conn_handler_thread.start() self.__check_neighbours_thread.start() while self.__conn_handler_thread.status == S_PENDING: time.sleep(.1) if self.__conn_handler_thread.status == S_ERROR: self.stop() logger.error('FriServer does not started!') return False else: logger.info('FriServer started!') return True
def run(self): self.stopped = False self.__lock.acquire() try: for i in range(self.min_count): thread = FriWorker(self.queue, self.operator, self.keystorage, self.sessions) thread.setName('%s-FriWorkerThread#%i' % (self.workers_name, i)) self.__threads.append(thread) self.__threads_idx = self.min_count for thread in self.__threads: thread.start() finally: self.__lock.release() logger.info('Started work threads (min_count)!') not_empty_queue_count = 0 empty_queue_count = 0 while not self.stopped: try: time.sleep(.2) if self.queue.qsize() > 0: not_empty_queue_count += 1 empty_queue_count = 0 else: not_empty_queue_count = 0 empty_queue_count += 1 act, busy = self.get_workers_stat() if not_empty_queue_count >= 5: if act == busy: self.__spawn_work_threads() elif empty_queue_count >= 15: if (act - busy) > self.min_count: self.__stop_work_thread() except Exception, err: ret_message = '%s error: %s' % (self.getName(), err) logger.error(ret_message)
def _check_near_range(self): if self.operator.status != DS_NORMALWORK: return self_dht_range = self.operator.get_dht_range() if self_dht_range.get_end() != MAX_HASH: next_range = self.operator.ranges_table.find(self_dht_range.get_end()+1) if not next_range: next_exists_range = self.operator.ranges_table.find_next(self_dht_range.get_end()-1) if next_exists_range: end = next_exists_range.start-1 else: end = MAX_HASH new_dht_range = self_dht_range.extend(self_dht_range.get_end()+1, end) self.operator.update_dht_range(new_dht_range) rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.operator.self_address)] append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.operator.self_address)] logger.info('Extended range by next neighbours') time.sleep(WAIT_DHT_TABLE_UPDATE) self._init_network_operation('UpdateHashRangeTable', {'append': append_lst, 'remove': rm_lst}) return self._check_near_range() first_range = self.operator.ranges_table.find(MIN_HASH) if not first_range: first_range = self.operator.ranges_table.get_first() if not first_range: return if first_range.node_address == self.operator.self_address: new_dht_range = self_dht_range.extend(MIN_HASH, first_range.start-1) self.operator.update_dht_range(new_dht_range) rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.operator.self_address)] append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.operator.self_address)] logger.info('Extended range by first range') self._init_network_operation('UpdateHashRangeTable', {'append': append_lst, 'remove': rm_lst})
def run(self): self.stopped = False logger.info('Thread started!') while not self.stopped: time.sleep(1) continue#FIXME try: try: tc_oper = self.operator.get_operation_instance('TopologyCognition') except OperException, err: time.sleep(1) continue while True: last_processed_dt = tc_oper.get_last_processed_dt() dt = datetime.now() - last_processed_dt if dt.total_seconds() < NO_TOPOLOGY_DYSCOVERY_WINDOW: w_seconds = random.randint(MIN_TOPOLOGY_DISCOVERY_WAIT, MAX_TOPOLOGY_DISCOVERY_WAIT) for i in xrange(w_seconds): time.sleep(1) if self.stopped: return else: break logger.info('Starting topology discovery...') packet = FabnetPacketRequest(method='TopologyCognition', parameters={"need_rebalance": 1}) self.operator.call_network(packet) for i in xrange(DISCOVERY_TOPOLOGY_TIMEOUT): time.sleep(1) if self.stopped: return except Exception, err: logger.error(str(err))
def callback(self, packet, sender=None): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ if packet.ret_code != RC_OK: logger.info('Trying select other hash range...') self.operator.start_as_dht_member() else: free_size = self.operator.get_dht_range().get_free_size() if (int(packet.ret_parameters['range_size']) * 100. / free_size) > ALLOW_FREE_SIZE_PERCENTS: logger.info('Requested range is huge for me :( canceling...') self._init_operation(packet.from_node, 'SplitRangeCancel', {}) else: logger.info('Requesting new range data from %s...'%packet.from_node) self._init_operation(packet.from_node, 'GetRangeDataRequest', {})
while not self.stopped: try: (sock, addr) = self.sock.accept() if self.stopped: sock.close() break self.queue.put(sock) except Exception, err: logger.error('[accept] %s'%err) if self.sock: self.sock.close() del self.sock logger.info('Connection handler thread stopped!') def stop(self): self.stopped = True class FriWorkersManager(threading.Thread): def __init__(self, queue, operator, keystorage, sessions, min_count=MIN_WORKERS_COUNT, \ max_count=MAX_WORKERS_COUNT, workers_name='unnamed'): threading.Thread.__init__(self) self.queue = queue self.operator = operator self.keystorage = keystorage self.min_count = min_count self.max_count = max_count self.workers_name = workers_name
continue while True: last_processed_dt = tc_oper.get_last_processed_dt() dt = datetime.now() - last_processed_dt if dt.total_seconds() < NO_TOPOLOGY_DYSCOVERY_WINDOW: w_seconds = random.randint(MIN_TOPOLOGY_DISCOVERY_WAIT, MAX_TOPOLOGY_DISCOVERY_WAIT) for i in xrange(w_seconds): time.sleep(1) if self.stopped: return else: break logger.info('Starting topology discovery...') packet = FabnetPacketRequest(method='TopologyCognition', parameters={"need_rebalance": 1}) self.operator.call_network(packet) for i in xrange(DISCOVERY_TOPOLOGY_TIMEOUT): time.sleep(1) if self.stopped: return except Exception, err: logger.error(str(err)) logger.info('Thread stopped!') def stop(self): self.stopped = True
def set_status_to_normalwork(self): logger.info('Changing node status to NORMALWORK') self.status = DS_NORMALWORK self.__split_requests_cache = [] self.__start_dht_try_count = 0
if resp.ret_code != RC_OK: logger.error('PutDataBlock error on %s: %s'%(k_range.node_address, resp.ret_message)) return False return True def run(self): logger.info('started') self.stopped = False while not self.stopped: try: #self._check_range_free_size() logger.debug('MonitorDHTRanges iteration...') self._process_reservation_range() self._process_replicas() except Exception, err: logger.error('[MonitorDHTRanges] %s'% err) finally: for i in xrange(MONITOR_DHT_RANGES_TIMEOUT): if self.stopped: break time.sleep(1) logger.info('stopped') def stop(self): self.stopped = True
def run(self): logger.info('%s started!'%self.getName()) ok_packet = FabnetPacketResponse(ret_code=RC_OK, ret_message='ok') ok_msg = ok_packet.dump() while True: ret_code = RC_OK ret_message = '' data = '' sock = None try: self.__busy_flag.clear() sock = self.queue.get() if sock == STOP_THREAD_EVENT: logger.info('%s stopped!'%self.getName()) break self.__busy_flag.set() packet = self.handle_connection(sock) session_id = packet.get('session_id', None) role = self.check_session(sock, session_id) is_sync = packet.get('sync', False) if not is_sync: sock.sendall(ok_msg) sock.close() sock = None if not packet.has_key('ret_code'): pack = FabnetPacketRequest(**packet) ret_packet = self.operator.process(pack, role) try: if not is_sync: if ret_packet: self.operator.send_to_sender(pack.sender, ret_packet) else: if not ret_packet: ret_packet = FabnetPacketResponse() sock.sendall(ret_packet.dump()) sock.close() sock = None finally: self.operator.after_process(pack, ret_packet) else: self.operator.callback(FabnetPacketResponse(**packet)) except Exception, err: ret_message = '%s error: %s' % (self.getName(), err) logger.error(ret_message) try: if sock: err_packet = FabnetPacketResponse(ret_code=RC_ERROR, ret_message=ret_message) sock.sendall(err_packet.dump()) sock.close() except Exception, err: logger.error("Can't send error message to socket: %s"%err) self._close_socket(sock)