def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
dht_range = self.operator.get_dht_range()
subranges = dht_range.get_subranges()
if not subranges:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message="Range is not splitted!")
ret_range, new_range = subranges
try:
node_address = packet.sender
logger.debug("Starting subrange data transfering to %s" % node_address)
for key, data in ret_range.iter_range():
checksum = hashlib.sha1(data).hexdigest()
params = {"key": key, "checksum": checksum}
resp = self._init_operation(node_address, "PutDataBlock", params, binary_data=data, sync=True)
if resp.ret_code:
raise Exception(
"Init PutDataBlock operation on %s error. Details: %s" % (node_address, resp.ret_message)
)
self.operator.update_dht_range(new_range)
except Exception, err:
logger.error("GetRangeDataRequestOperation error: %s" % err)
dht_range.join_subranges()
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message="Send range data failed: %s" % err)
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
if self.operator.ranges_table.empty():
logger.debug('Received update for hash ranges table, but it is not initialized yet. Skip operation...')
return
append_lst = packet.parameters.get('append', [])
rm_lst = packet.parameters.get('remove', [])
rm_obj_list = [HashRange(r[0], r[1], r[2]) for r in rm_lst]
ap_obj_list = [HashRange(a[0], a[1], a[2]) for a in append_lst]
self._lock()
try:
self.operator.ranges_table.apply_changes(rm_obj_list, ap_obj_list)
logger.debug('RM RANGE: %s'%', '.join([r.to_str() for r in rm_obj_list]))
logger.debug('APP RANGE: %s'%', '.join([a.to_str() for a in ap_obj_list]))
except Exception, err:
logger.error('UpdateHashRangeTable error: %s'%err)
def callback(self, packet):
"""process callback fabnet packet
@param packet - object of FabnetPacketResponse class
"""
msg_info = self.msg_container.get(packet.message_id)
self.__lock.acquire()
try:
if not msg_info:
raise OperException('Message with ID %s does not found! {Packet: %s}'%(packet.message_id, packet))
msg_info['responses_count'] += 1
operation = msg_info['operation']
sender = msg_info['sender']
finally:
self.__lock.release()
operation_obj = self.__operations.get(operation, None)
if operation_obj is None:
raise OperException('Method "%s" does not implemented!'%operation)
s_packet = None
try:
s_packet = operation_obj.callback(packet, sender)
except Exception, err:
logger.error('%s callback failed. Details: %s'%(operation, err))
def run(self):
self.stopped = False
logger.info('Thread started!')
while not self.stopped:
try:
neighbours = self.operator.get_neighbours(NT_SUPERIOR)
if not neighbours:
time.sleep(INIT_DHT_WAIT_NEIGHBOUR_TIMEOUT)
continue
for neighbour in neighbours:
logger.debug('Checking range table at %s'%neighbour)
mod_index = self.operator.ranges_table.get_mod_index()
params = {'mod_index': mod_index}
packet_obj = FabnetPacketRequest(method='CheckHashRangeTable',
sender=self.operator.self_address, parameters=params)
rcode, rmsg = self.operator.call_node(neighbour, packet_obj)
for i in xrange(CHECK_HASH_TABLE_TIMEOUT):
if self.stopped:
break
time.sleep(1)
except Exception, err:
logger.error(str(err))
def send_to_sender(self, sender, packet):
if sender is None:
self.callback(packet)
return
rcode, rmsg = self.fri_client.call(sender, packet)
if rcode:
logger.error('[Operator.send_back to %s] %s %s.'%(sender, rcode, rmsg))
def _rebalance_nodes(self):
operation_obj = self.__operations.get('ManageNeighbour', None)
if not operation_obj:
logger.error('ManageNeighbour does not found. Cant rebalance node!')
return
operation_obj.rebalance_append({'neighbour_type': NT_SUPERIOR})
operation_obj.rebalance_append({'neighbour_type': NT_UPPER})
def run(self):
try:
self.__server = wsgiserver.CherryPyWSGIServer((self.__host, self.__port), self.__service.web_app,)
logger.info('CAServer thread is started')
self.__server.start()
except Exception, err:
logger.error('[CAServer] %s'%err)
def process(self, packet, role=None):
"""process request fabnet packet
@param packet - object of FabnetPacketRequest class
@param role - requestor role (None for disable auth)
"""
try:
if packet.method == KEEP_ALIVE_METHOD:
return self._process_keep_alive(packet)
inserted = self.msg_container.put_safe(packet.message_id,
{'operation': packet.method,
'sender': packet.sender,
'responses_count': 0,
'datetime': datetime.now()})
if not inserted:
#this message is already processing/processed
#logger.debug('packet is already processing/processed: %s'%packet)
return
operation_obj = self.__operations.get(packet.method, None)
if operation_obj is None:
raise OperException('Method "%s" does not implemented!'%packet.method)
operation_obj.check_role(role)
logger.debug('processing packet %s'%packet)
if self.__stat is not None:
self._lock()
try:
self.__stat[packet.method] += 1
finally:
self._unlock()
message_id = packet.message_id
n_packet = operation_obj.before_resend(packet)
if n_packet:
n_packet.message_id = message_id
n_packet.sync = False
self._send_to_neighbours(n_packet)
s_packet = operation_obj.process(packet)
if s_packet:
s_packet.message_id = packet.message_id
s_packet.from_node = self.self_address
return s_packet
except Exception, err:
err_packet = FabnetPacketResponse(from_node=self.self_address,
message_id=packet.message_id,
ret_code=1, ret_message= '[Operator.process] %s'%err)
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[Operator.process] %s'%err)
return err_packet
def _init_operation(self, node_address, operation, parameters, sync=False, binary_data=''):
"""Initiate new operation"""
req = FabnetPacketRequest(method=operation, sender=self.operator.self_address, \
parameters=parameters, binary_data=binary_data, sync=sync)
resp = self.operator.call_node(node_address, req, sync)
if sync:
return resp
else:
rcode, rmsg = resp
if rcode:
logger.error('Operation %s failed on %s. Details: %s'%(operation, node_address, rmsg))
return rcode, rmsg
def start_as_dht_member(self):
if self.status == DS_DESTROYING:
return
self.status = DS_INITIALIZE
dht_range = self.get_dht_range()
nochange = False
curr_start = dht_range.get_start()
curr_end = dht_range.get_end()
if dht_range.is_max_range() or self.__split_requests_cache:
new_range = self.__get_next_max_range()
else:
new_range = self.__get_next_range_near(curr_start, curr_end)
if new_range:
if (new_range.start != curr_start or new_range.end != curr_end):
nochange = True
if new_range.node_address == self.self_address:
self.set_status_to_normalwork()
return
if new_range is None:
#wait and try again
if self.__start_dht_try_count == DHT_CYCLE_TRY_COUNT:
logger.error('Cant initialize node as a part of DHT')
self.__start_dht_try_count = 0
return
logger.info('No ready range for me on network... So, sleep and try again')
self.__start_dht_try_count += 1
self.__split_requests_cache = []
time.sleep(WAIT_RANGE_TIMEOUT)
return self.start_as_dht_member()
if nochange:
new_dht_range = dht_range
else:
new_dht_range = FSHashRanges(long(new_range.start + new_range.length()/2+1), long(new_range.end), self.save_path)
self.update_dht_range(new_dht_range)
new_dht_range.restore_from_trash() #try getting new range data from trash
self.__split_requests_cache.append(new_range.node_address)
logger.info('Call SplitRangeRequest to %s'%(new_range.node_address,))
parameters = { 'start_key': new_dht_range.get_start(), 'end_key': new_dht_range.get_end() }
req = FabnetPacketRequest(method='SplitRangeRequest', sender=self.self_address, parameters=parameters)
ret_code, ret_msg = self.call_node(new_range.node_address, req)
if ret_code != RC_OK:
logger.error('Cant start SplitRangeRequest operation on node %s. Details: %s'%(new_range.node_address, ret_msg))
return self.start_as_dht_member()
def check_dht_range(self):
if self.status == DS_INITIALIZE:
return
dht_range = self.get_dht_range()
start = dht_range.get_start()
end = dht_range.get_end()
range_obj = self.ranges_table.find(start)
if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address:
logger.error('DHT range on this node is not found in ranges_table')
logger.info('Trying reinit node as DHT member...')
self.start_as_dht_member()
def run(self):
logger.info('started')
self.stopped = False
while not self.stopped:
try:
#self._check_range_free_size()
logger.debug('MonitorDHTRanges iteration...')
self._process_reservation_range()
self._process_replicas()
except Exception, err:
logger.error('[MonitorDHTRanges] %s'% err)
finally:
def run(self):
self.stopped = False
logger.info('Check neighbours thread is started!')
while not self.stopped:
try:
t0 = datetime.now()
self.operator.check_neighbours()
proc_dt = datetime.now() - t0
except Exception, err:
logger.error('[CheckNeighboursThread] %s'%err)
finally:
def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if packet.ret_code == RC_ERROR:
logger.error('CheckHashRangeTable failed on %s. Details: %s %s'%(packet.from_node, packet.ret_code, packet.ret_message))
elif packet.ret_code == RC_NEED_UPDATE:
self._get_ranges_table(packet.from_node, packet.ret_parameters['mod_index'])
def __bind_socket(self):
try:
if self.keystorage:
context = self.keystorage.get_node_context()
self.sock = Connection(context)
else:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.sock.bind((self.hostname, self.port))
self.sock.listen(5)
except Exception, err:
self.status = S_ERROR
logger.error('[__bind_socket] %s'%err)
def run(self):
logger.info('Starting connection handler thread...')
self.__bind_socket()
logger.info('Connection handler thread started!')
while not self.stopped:
try:
(sock, addr) = self.sock.accept()
if self.stopped:
sock.close()
break
self.queue.put(sock)
except Exception, err:
logger.error('[accept] %s'%err)
def start(self):
self.stopped = False
self.__workers_manager_thread.start()
self.__conn_handler_thread.start()
self.__check_neighbours_thread.start()
while self.__conn_handler_thread.status == S_PENDING:
time.sleep(.1)
if self.__conn_handler_thread.status == S_ERROR:
self.stop()
logger.error('FriServer does not started!')
return False
else:
logger.info('FriServer started!')
return True
def _put_data(self, key, data, is_replica=False):
k_range = self.operator.ranges_table.find(long(key, 16))
if not k_range:
logger.debug('No range found for reservation key %s'%key)
return False
checksum = hashlib.sha1(data).hexdigest()
params = {'key': key, 'checksum': checksum, 'is_replica': is_replica}
req = FabnetPacketRequest(method='PutDataBlock', sender=self.operator.self_address,\
parameters=params, binary_data=data, sync=True)
resp = self.operator.call_node(k_range.node_address, req, sync=True)
if resp.ret_code != RC_OK:
logger.error('PutDataBlock error on %s: %s'%(k_range.node_address, resp.ret_message))
return False
return True
def stop(self):
self.__lock.acquire()
self.stopped = True
try:
act_count = 0
for thread in self.__threads:
if thread.is_alive():
act_count += 1
for i in xrange(act_count):
self.queue.put(STOP_THREAD_EVENT)
self.queue.join()
for thread in self.__threads:
if thread.is_alive():
thread.join()
except Exception, err:
logger.error('stopping error: %s'%err)
def _check_stat(self, address):
client = FriClient()
while True:
try:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
time.sleep(.5)
continue
uppers_balance = int(ret_packet.ret_parameters[u'uppers_balance'])
superiors_balance = int(ret_packet.ret_parameters[u'superiors_balance'])
if uppers_balance >= 0 and superiors_balance >= 0:
break
print 'Node %s is not balanced yet! Waiting...'%address
time.sleep(.5)
except Exception, err:
logger.error('ERROR: %s'%err)
raise err
def _check_stat(self, address):
client = FriClient()
while True:
try:
packet_obj = FabnetPacketRequest(method="NodeStatistic", sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
time.sleep(0.5)
continue
uppers_balance = int(ret_packet.ret_parameters["NeighboursInfo"][u"uppers_balance"])
superiors_balance = int(ret_packet.ret_parameters["NeighboursInfo"][u"superiors_balance"])
if uppers_balance >= 0 and superiors_balance >= 0:
return
else:
print "Node %s is not balanced yet! Waiting..." % address
time.sleep(0.5)
except Exception, err:
logger.error("ERROR: %s" % err)
raise err
def _check_stat(self, address):
key_storage = init_keystore(KS_PATH, KS_PASSWD)
client = FriClient(key_storage)
while True:
try:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
time.sleep(.5)
continue
uppers_balance = int(ret_packet.ret_parameters['NeighboursInfo'][u'uppers_balance'])
superiors_balance = int(ret_packet.ret_parameters['NeighboursInfo'][u'superiors_balance'])
if uppers_balance >= 0 and superiors_balance >= 0:
return
else:
print 'Node %s is not balanced yet! Waiting...'%address
time.sleep(.5)
except Exception, err:
logger.error('ERROR: %s'%err)
raise err
def run(self):
self.stopped = False
self.__lock.acquire()
try:
for i in range(self.min_count):
thread = FriWorker(self.queue, self.operator, self.keystorage, self.sessions)
thread.setName('%s-FriWorkerThread#%i' % (self.workers_name, i))
self.__threads.append(thread)
self.__threads_idx = self.min_count
for thread in self.__threads:
thread.start()
finally:
self.__lock.release()
logger.info('Started work threads (min_count)!')
not_empty_queue_count = 0
empty_queue_count = 0
while not self.stopped:
try:
time.sleep(.2)
if self.queue.qsize() > 0:
not_empty_queue_count += 1
empty_queue_count = 0
else:
not_empty_queue_count = 0
empty_queue_count += 1
act, busy = self.get_workers_stat()
if not_empty_queue_count >= 5:
if act == busy:
self.__spawn_work_threads()
elif empty_queue_count >= 15:
if (act - busy) > self.min_count:
self.__stop_work_thread()
except Exception, err:
ret_message = '%s error: %s' % (self.getName(), err)
logger.error(ret_message)
def after_process(self, packet, ret_packet):
"""process some logic after response send"""
if packet.method == KEEP_ALIVE_METHOD:
return
msg_info = self.msg_container.get(packet.message_id)
self.__lock.acquire()
try:
if not msg_info:
raise OperException('Message with ID %s does not found in after_process! {Packet: %s}'%(packet.message_id, packet))
operation = msg_info['operation']
finally:
self.__lock.release()
operation_obj = self.__operations.get(operation, None)
if operation_obj is None:
raise OperException('Method "%s" does not implemented!'%operation)
try:
operation_obj.after_process(packet, ret_packet)
except Exception, err:
logger.error('%s after_process routine failed. Details: %s'%(operation, err))
def check_stat(address):
client = FriClient()
while True:
try:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
time.sleep(.5)
continue
uppers_balance = int(ret_packet.ret_parameters['NeighboursInfo'][u'uppers_balance'])
superiors_balance = int(ret_packet.ret_parameters['NeighboursInfo'][u'superiors_balance'])
if uppers_balance >= 0 and superiors_balance >= 0:
if ret_packet.ret_parameters['DHTInfo']['status'] == 'normwork':
break
print 'Node %s is not initialized as DHT member yet! Waiting...'%(address)
else:
print 'Node %s is not balanced yet! Waiting...'%address
time.sleep(.5)
except Exception, err:
logger.error('ERROR: %s'%err)
raise err
def run(self):
self.stopped = False
logger.info('Thread started!')
while not self.stopped:
time.sleep(1)
continue#FIXME
try:
try:
tc_oper = self.operator.get_operation_instance('TopologyCognition')
except OperException, err:
time.sleep(1)
continue
while True:
last_processed_dt = tc_oper.get_last_processed_dt()
dt = datetime.now() - last_processed_dt
if dt.total_seconds() < NO_TOPOLOGY_DYSCOVERY_WINDOW:
w_seconds = random.randint(MIN_TOPOLOGY_DISCOVERY_WAIT, MAX_TOPOLOGY_DISCOVERY_WAIT)
for i in xrange(w_seconds):
time.sleep(1)
if self.stopped:
return
else:
break
logger.info('Starting topology discovery...')
packet = FabnetPacketRequest(method='TopologyCognition', parameters={"need_rebalance": 1})
self.operator.call_network(packet)
for i in xrange(DISCOVERY_TOPOLOGY_TIMEOUT):
time.sleep(1)
if self.stopped:
return
except Exception, err:
logger.error(str(err))
def _close_socket(self, sock):
try:
sock.close()
except Exception, err:
logger.error('Closing client socket error: %s'%err)
def run(self):
logger.info('%s started!'%self.getName())
ok_packet = FabnetPacketResponse(ret_code=RC_OK, ret_message='ok')
ok_msg = ok_packet.dump()
while True:
ret_code = RC_OK
ret_message = ''
data = ''
sock = None
try:
self.__busy_flag.clear()
sock = self.queue.get()
if sock == STOP_THREAD_EVENT:
logger.info('%s stopped!'%self.getName())
break
self.__busy_flag.set()
packet = self.handle_connection(sock)
session_id = packet.get('session_id', None)
role = self.check_session(sock, session_id)
is_sync = packet.get('sync', False)
if not is_sync:
sock.sendall(ok_msg)
sock.close()
sock = None
if not packet.has_key('ret_code'):
pack = FabnetPacketRequest(**packet)
ret_packet = self.operator.process(pack, role)
try:
if not is_sync:
if ret_packet:
self.operator.send_to_sender(pack.sender, ret_packet)
else:
if not ret_packet:
ret_packet = FabnetPacketResponse()
sock.sendall(ret_packet.dump())
sock.close()
sock = None
finally:
self.operator.after_process(pack, ret_packet)
else:
self.operator.callback(FabnetPacketResponse(**packet))
except Exception, err:
ret_message = '%s error: %s' % (self.getName(), err)
logger.error(ret_message)
try:
if sock:
err_packet = FabnetPacketResponse(ret_code=RC_ERROR, ret_message=ret_message)
sock.sendall(err_packet.dump())
sock.close()
except Exception, err:
logger.error("Can't send error message to socket: %s"%err)
self._close_socket(sock)
