def run(self):
logger.info('started')
while True:
for i in xrange(Config.MONITOR_DHT_RANGES_TIMEOUT):
if self.stopped.is_set():
break
time.sleep(1)
if self.stopped.is_set():
break
try:
logger.debug('MonitorDHTRanges iteration...')
self._check_range_free_size()
if self.stopped.is_set():
break
self._process_reservation_range()
if self.stopped.is_set():
break
self._process_replicas()
if self.stopped.is_set():
break
except Exception, err:
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[MonitorDHTRanges] %s'% err)
def run(self):
logger.info('Thread started!')
while not self.stopped.is_set():
dt = 0
try:
t0 = datetime.now()
logger.debug('Collecting %s nodes statistic...'%self.check_status)
nodeaddrs = self.operator.get_nodes_list(self.check_status)
for nodeaddr in nodeaddrs:
logger.debug('Get statistic from %s'%nodeaddr)
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = self.client.call_sync(nodeaddr, packet_obj)
if self.check_status == UP and ret_packet.ret_code:
logger.warning('Node with address %s does not response... Details: %s'%(nodeaddr, ret_packet))
self.operator.change_node_status(nodeaddr, DOWN)
else:
stat = json.dumps(ret_packet.ret_parameters)
self.operator.update_node_stat(nodeaddr, stat)
dt = total_seconds(datetime.now() - t0)
logger.info('Nodes (with status=%s) stat is collected. Processed secs: %s'%(self.check_status, dt))
except Exception, err:
logger.error(str(err))
finally:
def _move_range(self, range_obj):
logger.info('Node %s went from DHT. Updating hash range table on network...'%range_obj.node_address)
rm_lst = [(range_obj.start, range_obj.end, range_obj.node_address)]
parameters = {'append': [], 'remove': rm_lst}
req = FabnetPacketRequest(method='UpdateHashRangeTable', sender=self.self_address, parameters=parameters)
self.call_network(req)
def check_dht_range(self, reinit=True):
'''check current DHT range
return True if current DHT range has 'unstable' status (initializing, spliting, invalid)
return False if current DHT range is OK
if current DHT range is invalid and reinit==True -> start init DHT process
'''
if self.status == DS_INITIALIZE:
return True
dht_range = self.get_dht_range()
if dht_range.get_subranges():
return True
start = dht_range.get_start()
end = dht_range.get_end()
range_obj = self.ranges_table.find(start)
if not range_obj:
range_obj = self.ranges_table.find(end)
if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address:
msg = 'Invalid self range!'
if range_obj:
msg += ' hash table range - [%040x-%040x]%s... my range - [%040x-%040x]%s'% \
(range_obj.start, range_obj.end, range_obj.node_address, start, end, self.self_address)
else:
msg += 'Not found in hash table [%040x-%040x]%s'%(start, end, self.self_address)
logger.info(msg)
if (not range_obj) or reinit:
logger.info('Trying reinit node as DHT member...')
self.start_as_dht_member()
return True
def _take_range(self, range_obj):
logger.info('Take node old range. Updating hash range table on network...')
app_lst = [(range_obj.start, range_obj.end, range_obj.node_address)]
parameters = {'append': app_lst, 'remove': []}
req = FabnetPacketRequest(method='UpdateHashRangeTable', sender=self.self_address, parameters=parameters)
self.call_network(req)
def run(self):
logger.info('started')
while True:
for i in xrange(int(Config.MONITOR_DHT_RANGES_TIMEOUT)):
if self.stopped.is_set():
break
if self.interrupt.is_set():
self.interrupt.clear()
break
time.sleep(1)
if self.stopped.is_set():
break
if self.operator.status == DS_INITIALIZE:
continue
try:
logger.debug('MonitorDHTRanges iteration...')
self._process_foreign()
if self.stopped.is_set():
break
self._check_range_free_size()
if self.stopped.is_set():
break
except Exception, err:
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[MonitorDHTRanges] %s'% err)
def load(self, ranges_dump):
self.__lock.acquire()
try:
is_old_ex = False
if self.__ranges:
is_old_ex = self.__ranges
self.__ranges, self.__last_dm, self.__mod_index = pickle.loads(ranges_dump)
logger.debug('HASH RANGES: %s'%'\n'.join([r.to_str() for r in self.__ranges]))
if is_old_ex and len(is_old_ex) > 1:
log_s = 'OLD(-)/NEW(+) HASHES IN TABLES:\n'
for range_o in self.iter_table():
for old_range in is_old_ex:
if old_range.to_str() == range_o.to_str():
break
else:
log_s += '+ %s\n' % range_o.to_str()
for old_range in is_old_ex:
for range_o in self.iter_table():
if old_range.to_str() == range_o.to_str():
break
else:
log_s += '- %s\n' % old_range.to_str()
logger.info(log_s)
self.__blocked.clear()
if is_old_ex:
return len(is_old_ex)
return 0
finally:
self.__lock.release()
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
try:
event_type = packet.parameters.get('event_type', None)
event_provider = packet.parameters.get('event_provider', None)
event_topic = packet.parameters.get('event_topic', None)
if event_provider is None:
raise Exception('event_provider does not found!')
event_message = packet.parameters.get('event_message', None)
if packet.sender is None: #this is sender
if event_type == ET_ALERT:
logger.warning('[ALERT][%s] *%s* %s'%(event_provider, event_topic, event_message))
elif event_type == ET_INFO:
logger.info('[INFORMATION][%s] *%s* %s'%(event_provider, event_topic, event_message))
else:
logger.info('[NOTIFICATION.%s][%s] *%s* %s'%(event_type, event_provider, event_topic, event_message))
self.on_network_notify(event_type, event_provider, event_topic, event_message)
except Exception, err:
logger.error('[NotifyOperation] %s'%err)
def set_status_to_normalwork(self, save_range=False):
logger.info('Changing node status to NORMALWORK')
self.status = DS_NORMALWORK
self.__split_requests_cache = []
self.__start_dht_try_count = 0
if save_range:
dht_range = self.get_dht_range()
dht_range.save_range()
def update_dht_range(self, new_dht_range):
self._lock()
old_dht_range = self.__dht_range
self.__dht_range = new_dht_range
self._unlock()
dht_range = self.get_dht_range()
logger.info('New node range: %040x-%040x' % (dht_range.get_start(), dht_range.get_end()))
def check_neighbours(self):
ka_packet = FabnetPacketRequest(method=KEEP_ALIVE_METHOD, sender=self.self_address, sync=True)
superiors = self.get_neighbours(NT_SUPERIOR)
remove_nodes = []
for superior in superiors:
resp = self.fri_client.call_sync(superior, ka_packet)
cnt = 0
self._lock()
try:
if self.__superior_keep_alives.get(superior, None) is None:
self.__superior_keep_alives[superior] = 0
if resp.ret_code == RC_OK:
self.__superior_keep_alives[superior] = 0
elif resp.ret_code == RC_NOT_MY_NEIGHBOUR:
remove_nodes.append((NT_SUPERIOR, superior, False))
continue
else:
self.__superior_keep_alives[superior] += 1
cnt = self.__superior_keep_alives[superior]
finally:
self._unlock()
if cnt == KEEP_ALIVE_TRY_COUNT:
logger.info('Neighbour %s does not respond. removing it...'%superior)
remove_nodes.append((NT_SUPERIOR, superior, True))
#check upper nodes...
uppers = self.get_neighbours(NT_UPPER)
self._lock()
try:
cur_dt = datetime.now()
for upper in uppers:
ka_dt = self.__upper_keep_alives.get(upper, None)
if ka_dt == None:
self.__upper_keep_alives[upper] = datetime.now()
continue
if total_seconds(cur_dt - ka_dt) >= KEEP_ALIVE_MAX_WAIT_TIME:
logger.info('No keep alive packets from upper neighbour %s. removing it...'%upper)
remove_nodes.append((NT_UPPER, upper, False))
finally:
self._unlock()
for n_type, nodeaddr, is_not_respond in remove_nodes:
self.remove_neighbour(n_type, nodeaddr)
if is_not_respond:
self.on_neigbour_not_respond(n_type, nodeaddr)
if remove_nodes or \
(len(uppers) < ONE_DIRECT_NEIGHBOURS_COUNT) or \
(len(superiors) < ONE_DIRECT_NEIGHBOURS_COUNT):
self._rebalance_nodes()
def run(self):
logger.info('Thread started!')
try:
self.__server.serve_forever()
except Exception, err:
self.is_error.set()
import traceback
logger.write = logger.info
traceback.print_exc(file=logger)
logger.error('Unexpected error: %s'%err)
def _manage_new_neighbours(self):
logger.info('Discovered neigbours: %s and %s'%(self.__new_superior, self.__new_upper))
parameters = { 'neighbour_type': NT_SUPERIOR, 'operation': MNO_APPEND,
'node_address': self.operator.self_address, 'operator_type': self.operator.OPTYPE }
self.operator.async_remote_call(self.__new_superior, 'ManageNeighbour', parameters)
parameters = { 'neighbour_type': NT_UPPER, 'operation': MNO_APPEND,
'node_address': self.operator.self_address, 'operator_type': self.operator.OPTYPE }
self.operator.async_remote_call(self.__new_upper, 'ManageNeighbour', parameters)
def _process_replicas(self):
self.__full_nodes = []
dht_range = self.operator.get_dht_range()
for digest, data, file_path in dht_range.iter_replicas():
if self.stopped.is_set():
break
logger.info('Processing replica %s'%digest)
if self._put_data(digest, data, is_replica=True):
logger.debug('data block with key=%s is send from replicas range'%digest)
os.unlink(file_path)
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
auth_key = packet.parameters['auth_key']
self.operator.set_auth_key(auth_key)
logger.info('ChangeAuthKey: new auth key is installed!')
def accept_foreign_subrange(self, foreign_node, subrange_size):
dht_range = self.get_dht_range()
estimated_data_size_perc = dht_range.get_estimated_data_percents(subrange_size)
if estimated_data_size_perc >= float(Config.ALLOW_USED_SIZE_PERCENTS):
logger.info('Requested range is huge for me :( canceling...')
req = FabnetPacketRequest(method='SplitRangeCancel', sender=self.self_address)
self.call_node(foreign_node, req)
else:
logger.info('Requesting new range data from %s...'%foreign_node)
req = FabnetPacketRequest(method='GetRangeDataRequest', sender=self.self_address)
self.call_node(foreign_node, req)
def _get_ranges_table(self, from_addr, mod_index, ranges_count, force=False):
for i in xrange(self.operator.get_config_value('RANGES_TABLE_FLAPPING_TIMEOUT')):
if force:
break
c_mod_index, c_ranges_count = self.operator.get_ranges_table_status()
if c_ranges_count == 0:
break
if c_mod_index == mod_index and ranges_count == c_ranges_count:
return
time.sleep(1)
logger.info('Ranges table is invalid! Requesting table from %s'% from_addr)
self._init_operation(from_addr, 'GetRangesTable', {})
def stop(self):
try:
logger.info('stopping operator...')
self.stopped.set()
self.__check_neighbours_thread.stop()
uppers = self.get_neighbours(NT_UPPER)
superiors = self.get_neighbours(NT_SUPERIOR)
self.__unbind_neighbours(uppers, NT_SUPERIOR)
self.__unbind_neighbours(superiors, NT_UPPER)
except Exception, err:
logger.error('Operator stopping failed. Details: %s'%err)
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
logger.info("Canceled range splitting! Joining subranges.")
self.operator.join_subranges()
return FabnetPacketResponse()
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
logger.info("Trying select other hash range...")
self.operator.start_as_dht_member()
def check_near_range(self, reinit_dht=False):
if self.status != DS_NORMALWORK:
return
failed_range = self.check_dht_range(reinit=reinit_dht)
if failed_range:
return
self._lock()
try:
self_dht_range = self.get_dht_range()
if self_dht_range.get_end() != MAX_KEY:
next_range = self.ranges_table.find(self_dht_range.get_end()+1)
if not next_range:
next_exists_range = self.ranges_table.find_next(self_dht_range.get_end()-1)
if next_exists_range:
end = next_exists_range.start-1
else:
end = MAX_KEY
new_dht_range = self_dht_range.extend(self_dht_range.get_end()+1, end)
self.update_dht_range(new_dht_range)
rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.self_address)]
append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.self_address)]
logger.info('Extended range by next neighbours')
req = FabnetPacketRequest(method='UpdateHashRangeTable', \
sender=self.self_address, parameters={'append': append_lst, 'remove': rm_lst})
self.call_network(req)
return
first_range = self.ranges_table.find(MIN_KEY)
if not first_range:
first_range = self.ranges_table.get_first()
if not first_range:
return
if first_range.node_address == self.self_address:
new_dht_range = self_dht_range.extend(MIN_KEY, first_range.start-1)
self.update_dht_range(new_dht_range)
rm_lst = [(self_dht_range.get_start(), self_dht_range.get_end(), self.self_address)]
append_lst = [(new_dht_range.get_start(), new_dht_range.get_end(), self.self_address)]
logger.info('Extended range by first range')
req = FabnetPacketRequest(method='UpdateHashRangeTable', \
sender=self.self_address, parameters={'append': append_lst, 'remove': rm_lst})
self.call_network(req)
finally:
self._unlock()
def start_as_dht_member(self):
if self.status == DS_DESTROYING:
return
self.status = DS_INITIALIZE
dht_range = self.get_dht_range()
curr_start = dht_range.get_start()
curr_end = dht_range.get_end()
last_range = dht_range.get_last_range()
if last_range and not self.__split_requests_cache:
new_range = self.__get_next_range_near(last_range[0], last_range[1])
elif dht_range.is_max_range() or self.__split_requests_cache:
new_range = self.__get_next_max_range()
else:
new_range = self.__get_next_range_near(curr_start, curr_end)
if new_range is None:
#wait and try again
if self.__start_dht_try_count == Config.DHT_CYCLE_TRY_COUNT:
logger.error('Cant initialize node as a part of DHT')
self.__start_dht_try_count = 0
return
logger.info('No ready range for me on network... So, sleep and try again')
self.__start_dht_try_count += 1
self.__split_requests_cache = []
time.sleep(Config.WAIT_RANGE_TIMEOUT)
return self.start_as_dht_member()
if (new_range.start == curr_start and new_range.end == curr_end):
new_dht_range = dht_range
else:
new_dht_range = FSHashRanges(long(new_range.start), long(new_range.end), self.save_path)
self.update_dht_range(new_dht_range)
new_dht_range.restore_from_reservation() #try getting new range data from reservation
if new_range.node_address == self.self_address:
self._take_range(new_range)
self.set_status_to_normalwork()
return
self.__split_requests_cache.append(new_range.node_address)
logger.info('Call SplitRangeRequest [%040x-%040x] to %s'% \
(new_dht_range.get_start(), new_dht_range.get_end(), new_range.node_address,))
parameters = { 'start_key': new_dht_range.get_start(), 'end_key': new_dht_range.get_end() }
req = FabnetPacketRequest(method='SplitRangeRequest', sender=self.self_address, parameters=parameters)
self.call_node(new_range.node_address, req)
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code != RC_OK:
logger.info("[GetRangesTableCallback] Node %s does not initialized yet..." % packet.from_node)
return
logger.info("Recevied ranges table")
self.operator.restore_ranges_table(str(packet.ret_parameters["ranges_table"]))
logger.info("Ranges table is loaded to fabnet node")
if self.operator.get_status() == DS_INITIALIZE:
logger.info("Starting node as DHT member")
self.operator.start_as_dht_member()
else:
self.operator.check_near_range(True) # check with reinit_dht=True
def _process_foreign(self):
self.__full_nodes = []
dht_range = self.operator.get_dht_range()
cnt = 0
for digest, dbct, file_path in dht_range.iterator(foreign_only=True):
cnt += 1
if self.stopped.is_set():
break
logger.info('Processing foreign data block %s %s'%(digest, dbct))
if self._put_data(digest, file_path, dbct):
logger.debug('data block with key=%s is send'%digest)
os.remove(file_path)
if cnt == 0:
self.__changed_range = False
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
if self.operator.get_status() == DS_INITIALIZE:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Node is not initialized yet!')
ranges_table = self.operator.dump_ranges_table()
logger.info('Sending ranges table to %s'%packet.sender)
return FabnetPacketResponse(ret_parameters={'ranges_table': ranges_table})
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code != RC_OK:
logger.info('GetRangeData failed "%s"! Trying select other hash range...'%packet.ret_message)
self.operator.start_as_dht_member()
else:
self.operator.set_status_to_normalwork(True) #with save_range=True
def run(self):
logger.info('Check neighbours thread is started!')
proc_dt = timedelta(0)
while not self.stopped.is_set():
try:
t0 = datetime.now()
self.operator.check_neighbours()
proc_dt = datetime.now() - t0
logger.debug('CheckNeighbours process time: %s'%proc_dt)
except Exception, err:
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[CheckNeighboursThread] %s'%err)
finally:
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code != RC_OK:
logger.error('Cant split range from %s. Details: %s'%(sender, packet.ret_message))
logger.info('SplitRangeRequest failed! Trying select other hash range...')
self.operator.start_as_dht_member()
else:
subrange_size = int(packet.ret_parameters['range_size'])
self.operator.accept_foreign_subrange(packet.from_node, subrange_size)
def _manage_new_neighbours(self):
logger.info("Discovered neigbours: %s and %s" % (self.__new_superior, self.__new_upper))
parameters = {
"neighbour_type": NT_SUPERIOR,
"operation": MNO_APPEND,
"node_address": self.operator.self_address,
"operator_type": self.operator.OPTYPE,
}
self.operator.async_remote_call(self.__new_superior, "ManageNeighbour", parameters)
parameters = {
"neighbour_type": NT_UPPER,
"operation": MNO_APPEND,
"node_address": self.operator.self_address,
"operator_type": self.operator.OPTYPE,
}
self.operator.async_remote_call(self.__new_upper, "ManageNeighbour", parameters)
def remove_neighbour(self, neighbour_type, address):
self.__lock.acquire()
try:
neighbours = self.__neighbours.get(neighbour_type, None)
if neighbours is None:
raise OperException('Neigbour type "%s" is invalid!'%neighbour_type)
for optype, n_list in neighbours.items():
try:
del n_list[n_list.index(address)]
logger.info('%s neighbour %s with type "%s" is removed'%(NT_MAP[neighbour_type], address, optype))
except ValueError, err:
pass
if address in self.__upper_keep_alives:
del self.__upper_keep_alives[address]
if address in self.__superior_keep_alives:
del self.__superior_keep_alives[address]
