def run(self):
self.stopped = False
logger.info('Thread started!')
while not self.stopped:
try:
neighbours = self.operator.get_neighbours(NT_SUPERIOR)
if not neighbours:
time.sleep(INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT)
continue
for neighbour in neighbours:
logger.debug('Checking range table at %s'%neighbour)
mod_index = self.operator.ranges_table.get_mod_index()
params = {'mod_index': mod_index}
packet_obj = FabnetPacketRequest(method='CheckHashRangeTable',
sender=self.operator.self_address, parameters=params)
rcode, rmsg = self.operator.call_node(neighbour, packet_obj)
for i in xrange(CHECK_HASH_TABLE_TIMEOUT):
if self.stopped:
break
time.sleep(1)
except Exception, err:
logger.error(str(err))
def load(self, ranges_dump):
self.__lock.acquire()
try:
is_old_ex = False
if self.__ranges:
is_old_ex = self.__ranges
self.__ranges, self.__last_dm, self.__mod_index = pickle.loads(ranges_dump)
if is_old_ex:
s = 'OLD(-)/NEW(+) HASHES IN TABLES:\n'
for range_o in self.iter_table():
for old_range in is_old_ex:
if old_range.to_str() == range_o.to_str():
break
else:
s += '+ %s\n'%range_o.to_str()
for old_range in is_old_ex:
for range_o in self.iter_table():
if old_range.to_str() == range_o.to_str():
break
else:
s += '- %s\n'%old_range.to_str()
logger.info(s)
self.__blocked.clear()
finally:
self.__lock.release()
def _move_range(self, range_obj):
logger.info('Node %s went from DHT. Updating hash range table on network...'%range_obj.node_address)
rm_lst = [(range_obj.start, range_obj.end, range_obj.node_address)]
parameters = {'append': [], 'remove': rm_lst}
req = FabnetPacketRequest(method='UpdateHashRangeTable', sender=self.self_address, parameters=parameters)
self.call_network(req)
def check_neighbours(self):
ka_packet = FabnetPacketRequest(method=KEEP_ALIVE_METHOD, sender=self.self_address, sync=True)
superiors = self.get_neighbours(NT_SUPERIOR)
remove_nodes = []
for superior in superiors:
resp = self.fri_client.call_sync(superior, ka_packet)
cnt = 0
self.__lock.acquire()
try:
if self.__superior_keep_alives.get(superior, None) is None:
self.__superior_keep_alives[superior] = 0
if resp.ret_code == RC_OK:
self.__superior_keep_alives[superior] = 0
elif resp.ret_code == RC_NOT_MY_NEIGHBOUR:
del self.__superior_keep_alives[superior]
remove_nodes.append((NT_SUPERIOR, superior, False))
else:
self.__superior_keep_alives[superior] += 1
cnt = self.__superior_keep_alives[superior]
finally:
self.__lock.release()
if cnt == KEEP_ALIVE_TRY_COUNT:
logger.info('Neighbour %s does not respond. removing it...'%superior)
remove_nodes.append((NT_SUPERIOR, superior, True))
self.__lock.acquire()
del self.__superior_keep_alives[superior]
self.__lock.release()
#check upper nodes...
uppers = self.get_neighbours(NT_UPPER)
self.__lock.acquire()
try:
cur_dt = datetime.now()
for upper in uppers:
ka_dt = self.__upper_keep_alives.get(upper, None)
if ka_dt == None:
self.__upper_keep_alives[upper] = datetime.now()
continue
delta = cur_dt - ka_dt
if delta.total_seconds() >= KEEP_ALIVE_MAX_WAIT_TIME:
logger.info('No keep alive packets from upper neighbour %s. removing it...'%upper)
remove_nodes.append((NT_UPPER, upper, True))
del self.__upper_keep_alives[upper]
finally:
self.__lock.release()
for n_type, nodeaddr, is_not_respond in remove_nodes:
self.remove_neighbour(n_type, nodeaddr)
if is_not_respond:
self.on_neigbour_not_respond(n_type, nodeaddr)
if remove_nodes:
self._rebalance_nodes()
def run(self):
try:
self.__server = wsgiserver.CherryPyWSGIServer((self.__host, self.__port), self.__service.web_app,)
logger.info('CAServer thread is started')
self.__server.start()
except Exception, err:
logger.error('[CAServer] %s'%err)
def _send_to_neighbours(self, packet):
packet.sender = self.self_address
neighbours = self.get_neighbours(NT_SUPERIOR)
for neighbour in neighbours:
ret, message = self.fri_client.call(neighbour, packet)
if ret != RC_OK:
logger.info('Neighbour %s does not respond! Details: %s'%(neighbour, message))
def _process_replicas(self):
dht_range = self.operator.get_dht_range()
for digest, data, file_path in dht_range.iter_replicas():
logger.info('Processing replica %s'%digest)
if self._put_data(digest, data, is_replica=True):
logger.debug('Removing %s from local replicas'%digest)
os.remove(file_path)
else:
logger.debug('data block with key=%s is send from reservation range'%digest)
def _get_ranges_table(self, from_addr, mod_index):
if not self.operator.ranges_table.empty():
for i in xrange(RANGES_TABLE_FLAPPING_TIMEOUT):
time.sleep(1)
c_mod_index = self.operator.ranges_table.get_mod_index()
if c_mod_index == mod_index:
return
logger.info('Ranges table is invalid! Requesting table from %s'% from_addr)
self._init_operation(from_addr, 'GetRangesTable', {})
def update_dht_range(self, new_dht_range):
self._lock()
old_dht_range = self.__dht_range
self.__dht_range = new_dht_range
self._unlock()
old_dht_range.move_to_trash()
dht_range = self.get_dht_range()
logger.info('New node range: %040x-%040x'%(dht_range.get_start(), dht_range.get_end()))
def _manage_new_neighbours(self):
logger.info('Discovered neigbours: %s and %s'%(self.__new_superior, self.__new_upper))
parameters = { 'neighbour_type': NT_SUPERIOR, 'operation': MNO_APPEND,
'node_address': self.operator.self_address }
self._init_operation(self.__new_superior, 'ManageNeighbour', parameters)
parameters = { 'neighbour_type': NT_UPPER, 'operation': MNO_APPEND,
'node_address': self.operator.self_address }
self._init_operation(self.__new_upper, 'ManageNeighbour', parameters)
def start_as_dht_member(self):
if self.status == DS_DESTROYING:
return
self.status = DS_INITIALIZE
dht_range = self.get_dht_range()
nochange = False
curr_start = dht_range.get_start()
curr_end = dht_range.get_end()
if dht_range.is_max_range() or self.__split_requests_cache:
new_range = self.__get_next_max_range()
else:
new_range = self.__get_next_range_near(curr_start, curr_end)
if new_range:
if (new_range.start != curr_start or new_range.end != curr_end):
nochange = True
if new_range.node_address == self.self_address:
self.set_status_to_normalwork()
return
if new_range is None:
#wait and try again
if self.__start_dht_try_count == DHT_CYCLE_TRY_COUNT:
logger.error('Cant initialize node as a part of DHT')
self.__start_dht_try_count = 0
return
logger.info('No ready range for me on network... So, sleep and try again')
self.__start_dht_try_count += 1
self.__split_requests_cache = []
time.sleep(WAIT_RANGE_TIMEOUT)
return self.start_as_dht_member()
if nochange:
new_dht_range = dht_range
else:
new_dht_range = FSHashRanges(long(new_range.start + new_range.length()/2+1), long(new_range.end), self.save_path)
self.update_dht_range(new_dht_range)
new_dht_range.restore_from_trash() #try getting new range data from trash
self.__split_requests_cache.append(new_range.node_address)
logger.info('Call SplitRangeRequest to %s'%(new_range.node_address,))
parameters = { 'start_key': new_dht_range.get_start(), 'end_key': new_dht_range.get_end() }
req = FabnetPacketRequest(method='SplitRangeRequest', sender=self.self_address, parameters=parameters)
ret_code, ret_msg = self.call_node(new_range.node_address, req)
if ret_code != RC_OK:
logger.error('Cant start SplitRangeRequest operation on node %s. Details: %s'%(new_range.node_address, ret_msg))
return self.start_as_dht_member()
def check_dht_range(self):
if self.status == DS_INITIALIZE:
return
dht_range = self.get_dht_range()
start = dht_range.get_start()
end = dht_range.get_end()
range_obj = self.ranges_table.find(start)
if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address:
logger.error('DHT range on this node is not found in ranges_table')
logger.info('Trying reinit node as DHT member...')
self.start_as_dht_member()
def run(self):
logger.info('started')
self.stopped = False
while not self.stopped:
try:
#self._check_range_free_size()
logger.debug('MonitorDHTRanges iteration...')
self._process_reservation_range()
self._process_replicas()
except Exception, err:
logger.error('[MonitorDHTRanges] %s'% err)
finally:
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
logger.info('Trying select other hash range...')
self.operator.start_as_dht_member()
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code != RC_OK:
logger.info('[GetRangesTableCallback] Node %s does not initialized yet...'%packet.from_node)
return
logger.info('Recevied ranges table')
self.operator.ranges_table.load(str(packet.ret_parameters['ranges_table']))
logger.info('Ranges table is loaded to fabnet node')
if self.operator.status == DS_INITIALIZE:
logger.info('Starting node as DHT member')
self.operator.start_as_dht_member()
else:
self.operator.check_dht_range()
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
logger.info('Canceled range splitting! Joining subranges.')
dht_range = self.operator.get_dht_range()
dht_range.join_subranges()
return FabnetPacketResponse()
def run(self):
self.stopped = False
logger.info('Check neighbours thread is started!')
while not self.stopped:
try:
t0 = datetime.now()
self.operator.check_neighbours()
proc_dt = datetime.now() - t0
except Exception, err:
logger.error('[CheckNeighboursThread] %s'%err)
finally:
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
if self.operator.status == DS_INITIALIZE:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Node is not initialized yet!')
ranges_table = self.operator.ranges_table.dump()
logger.info('Sending ranges table to %s'%packet.sender)
return FabnetPacketResponse(ret_parameters={'ranges_table': ranges_table})
def run(self):
logger.info('Starting connection handler thread...')
self.__bind_socket()
logger.info('Connection handler thread started!')
while not self.stopped:
try:
(sock, addr) = self.sock.accept()
if self.stopped:
sock.close()
break
self.queue.put(sock)
except Exception, err:
logger.error('[accept] %s'%err)
def remove_neighbour(self, neighbour_type, address):
self.__lock.acquire()
try:
if neighbour_type == NT_SUPERIOR:
if address not in self.superior_neighbours:
return
del self.superior_neighbours[self.superior_neighbours.index(address)]
logger.info('Superior neighbour %s removed'%address)
elif neighbour_type == NT_UPPER:
if address not in self.upper_neighbours:
return
del self.upper_neighbours[self.upper_neighbours.index(address)]
logger.info('Upper neighbour %s removed'%address)
else:
raise OperException('Neigbour type "%s" is invalid!'%neighbour_type)
finally:
self.__lock.release()
def start(self):
self.stopped = False
self.__workers_manager_thread.start()
self.__conn_handler_thread.start()
self.__check_neighbours_thread.start()
while self.__conn_handler_thread.status == S_PENDING:
time.sleep(.1)
if self.__conn_handler_thread.status == S_ERROR:
self.stop()
logger.error('FriServer does not started!')
return False
else:
logger.info('FriServer started!')
return True
def run(self):
self.stopped = False
self.__lock.acquire()
try:
for i in range(self.min_count):
thread = FriWorker(self.queue, self.operator, self.keystorage, self.sessions)
thread.setName('%s-FriWorkerThread#%i' % (self.workers_name, i))
self.__threads.append(thread)
self.__threads_idx = self.min_count
for thread in self.__threads:
thread.start()
finally:
self.__lock.release()
logger.info('Started work threads (min_count)!')
not_empty_queue_count = 0
empty_queue_count = 0
while not self.stopped:
try:
time.sleep(.2)
if self.queue.qsize() > 0:
not_empty_queue_count += 1
empty_queue_count = 0
else:
not_empty_queue_count = 0
empty_queue_count += 1
act, busy = self.get_workers_stat()
if not_empty_queue_count >= 5:
if act == busy:
self.__spawn_work_threads()
elif empty_queue_count >= 15:
if (act - busy) > self.min_count:
self.__stop_work_thread()
except Exception, err:
ret_message = '%s error: %s' % (self.getName(), err)
logger.error(ret_message)
def _check_near_range(self):
if self.operator.status != DS_NORMALWORK:
return
self_dht_range = self.operator.get_dht_range()
if self_dht_range.get_end() != MAX_HASH:
next_range = self.operator.ranges_table.find(self_dht_range.get_end()+1)
if not next_range:
next_exists_range = self.operator.ranges_table.find_next(self_dht_range.get_end()-1)
if next_exists_range:
end = next_exists_range.start-1
else:
end = MAX_HASH
new_dht_range = self_dht_range.extend(self_dht_range.get_end()+1, end)
self.operator.update_dht_range(new_dht_range)
rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.operator.self_address)]
append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.operator.self_address)]
logger.info('Extended range by next neighbours')
time.sleep(WAIT_DHT_TABLE_UPDATE)
self._init_network_operation('UpdateHashRangeTable', {'append': append_lst, 'remove': rm_lst})
return self._check_near_range()
first_range = self.operator.ranges_table.find(MIN_HASH)
if not first_range:
first_range = self.operator.ranges_table.get_first()
if not first_range:
return
if first_range.node_address == self.operator.self_address:
new_dht_range = self_dht_range.extend(MIN_HASH, first_range.start-1)
self.operator.update_dht_range(new_dht_range)
rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.operator.self_address)]
append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.operator.self_address)]
logger.info('Extended range by first range')
self._init_network_operation('UpdateHashRangeTable', {'append': append_lst, 'remove': rm_lst})
def run(self):
self.stopped = False
logger.info('Thread started!')
while not self.stopped:
time.sleep(1)
continue#FIXME
try:
try:
tc_oper = self.operator.get_operation_instance('TopologyCognition')
except OperException, err:
time.sleep(1)
continue
while True:
last_processed_dt = tc_oper.get_last_processed_dt()
dt = datetime.now() - last_processed_dt
if dt.total_seconds() < NO_TOPOLOGY_DYSCOVERY_WINDOW:
w_seconds = random.randint(MIN_TOPOLOGY_DISCOVERY_WAIT, MAX_TOPOLOGY_DISCOVERY_WAIT)
for i in xrange(w_seconds):
time.sleep(1)
if self.stopped:
return
else:
break
logger.info('Starting topology discovery...')
packet = FabnetPacketRequest(method='TopologyCognition', parameters={"need_rebalance": 1})
self.operator.call_network(packet)
for i in xrange(DISCOVERY_TOPOLOGY_TIMEOUT):
time.sleep(1)
if self.stopped:
return
except Exception, err:
logger.error(str(err))
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code != RC_OK:
logger.info('Trying select other hash range...')
self.operator.start_as_dht_member()
else:
free_size = self.operator.get_dht_range().get_free_size()
if (int(packet.ret_parameters['range_size']) * 100. / free_size) > ALLOW_FREE_SIZE_PERCENTS:
logger.info('Requested range is huge for me :( canceling...')
self._init_operation(packet.from_node, 'SplitRangeCancel', {})
else:
logger.info('Requesting new range data from %s...'%packet.from_node)
self._init_operation(packet.from_node, 'GetRangeDataRequest', {})
while not self.stopped:
try:
(sock, addr) = self.sock.accept()
if self.stopped:
sock.close()
break
self.queue.put(sock)
except Exception, err:
logger.error('[accept] %s'%err)
if self.sock:
self.sock.close()
del self.sock
logger.info('Connection handler thread stopped!')
def stop(self):
self.stopped = True
class FriWorkersManager(threading.Thread):
def __init__(self, queue, operator, keystorage, sessions, min_count=MIN_WORKERS_COUNT, \
max_count=MAX_WORKERS_COUNT, workers_name='unnamed'):
threading.Thread.__init__(self)
self.queue = queue
self.operator = operator
self.keystorage = keystorage
self.min_count = min_count
self.max_count = max_count
self.workers_name = workers_name
continue
while True:
last_processed_dt = tc_oper.get_last_processed_dt()
dt = datetime.now() - last_processed_dt
if dt.total_seconds() < NO_TOPOLOGY_DYSCOVERY_WINDOW:
w_seconds = random.randint(MIN_TOPOLOGY_DISCOVERY_WAIT, MAX_TOPOLOGY_DISCOVERY_WAIT)
for i in xrange(w_seconds):
time.sleep(1)
if self.stopped:
return
else:
break
logger.info('Starting topology discovery...')
packet = FabnetPacketRequest(method='TopologyCognition', parameters={"need_rebalance": 1})
self.operator.call_network(packet)
for i in xrange(DISCOVERY_TOPOLOGY_TIMEOUT):
time.sleep(1)
if self.stopped:
return
except Exception, err:
logger.error(str(err))
logger.info('Thread stopped!')
def stop(self):
self.stopped = True
def set_status_to_normalwork(self):
logger.info('Changing node status to NORMALWORK')
self.status = DS_NORMALWORK
self.__split_requests_cache = []
self.__start_dht_try_count = 0
if resp.ret_code != RC_OK:
logger.error('PutDataBlock error on %s: %s'%(k_range.node_address, resp.ret_message))
return False
return True
def run(self):
logger.info('started')
self.stopped = False
while not self.stopped:
try:
#self._check_range_free_size()
logger.debug('MonitorDHTRanges iteration...')
self._process_reservation_range()
self._process_replicas()
except Exception, err:
logger.error('[MonitorDHTRanges] %s'% err)
finally:
for i in xrange(MONITOR_DHT_RANGES_TIMEOUT):
if self.stopped:
break
time.sleep(1)
logger.info('stopped')
def stop(self):
self.stopped = True
def run(self):
logger.info('%s started!'%self.getName())
ok_packet = FabnetPacketResponse(ret_code=RC_OK, ret_message='ok')
ok_msg = ok_packet.dump()
while True:
ret_code = RC_OK
ret_message = ''
data = ''
sock = None
try:
self.__busy_flag.clear()
sock = self.queue.get()
if sock == STOP_THREAD_EVENT:
logger.info('%s stopped!'%self.getName())
break
self.__busy_flag.set()
packet = self.handle_connection(sock)
session_id = packet.get('session_id', None)
role = self.check_session(sock, session_id)
is_sync = packet.get('sync', False)
if not is_sync:
sock.sendall(ok_msg)
sock.close()
sock = None
if not packet.has_key('ret_code'):
pack = FabnetPacketRequest(**packet)
ret_packet = self.operator.process(pack, role)
try:
if not is_sync:
if ret_packet:
self.operator.send_to_sender(pack.sender, ret_packet)
else:
if not ret_packet:
ret_packet = FabnetPacketResponse()
sock.sendall(ret_packet.dump())
sock.close()
sock = None
finally:
self.operator.after_process(pack, ret_packet)
else:
self.operator.callback(FabnetPacketResponse(**packet))
except Exception, err:
ret_message = '%s error: %s' % (self.getName(), err)
logger.error(ret_message)
try:
if sock:
err_packet = FabnetPacketResponse(ret_code=RC_ERROR, ret_message=ret_message)
sock.sendall(err_packet.dump())
sock.close()
except Exception, err:
logger.error("Can't send error message to socket: %s"%err)
self._close_socket(sock)