def wait_topology(node_i, nodes_count): conn = None while True: try: db = '/tmp/node_%s/fabnet_topology.db'%node_i while not os.path.exists(db): print '%s not exists!'%db time.sleep(0.2) time.sleep(.5) data = SafeJsonFile(db).read() cnt = 0 for value in data.values(): if int(value.get('old_data', 0)) == 0: cnt += 1 if cnt != nodes_count: time.sleep(.5) else: break finally: if conn: curs.close() conn.close()
def get_discovered_nodes(self): db = os.path.join(self.operator.home_dir, TOPOLOGY_DB) if not os.path.exists(db): return {} db = SafeJsonFile(db) nodes = db.read() return nodes
def callback(self, packet, sender): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ if sender: return packet node_address = packet.ret_parameters.get('node_address', None) superior_neighbours = packet.ret_parameters.get('superior_neighbours', None) upper_neighbours = packet.ret_parameters.get('upper_neighbours', None) node_name = packet.ret_parameters.get('node_name', '') if (node_address is None) or (superior_neighbours is None) or (upper_neighbours is None): raise Exception('TopologyCognition response packet is invalid! Packet: %s'%str(packet.to_dict())) self._lock() try: db = SafeJsonFile(os.path.join(self.home_dir, TOPOLOGY_DB)) data = db.read() data[node_address] = {'node_name': node_name, 'superiors': superior_neighbours, \ 'uppers': upper_neighbours, 'old_data': 0} db.write(data) finally: self._unlock() if packet.ret_parameters.get('need_rebalance', False): self.operator.smart_neighbours_rebalance(node_address, superior_neighbours, upper_neighbours)
def run(self): for i in self.cnt: f = SafeJsonFile(self.filepath) try: f.write(self.obj) time.sleep(.1) except Exception, err: self.errors.append(str(err))
def on_network_notify(self, event_type, event_provider, event_topic, event_message): self._lock() try: db = SafeJsonFile(os.path.join(self.home_dir, NOTIFICATIONS_DB)) data = db.read() n_list = data.get('notifications', []) n_list.append({'event_type': event_type, 'event_provider': event_provider, \ 'event_topic': event_topic, 'event_message': event_message, 'notify_dt': datetime.now().isoformat()}) data['notifications'] = n_list db.write(data) finally: self._unlock()
def run(self): for i in self.cnt: f = SafeJsonFile(self.filepath) try: data = f.read() if not data: continue #if data != self.obj: # raise Exception('invalid data read!') time.sleep(.05) except Exception, err: print 'error: %s'%err self.errors.append(err)
def test_topology_cognition(self): servers = [] addresses = [] #os.system('rm /tmp/fabnet_topology.db') try: for i in range(1900, 1900+NODES_COUNT): address = '127.0.0.1:%s'%i if not addresses: neighbour = None else: neighbour = random.choice(addresses) server = TestServerThread(i, neighbour) server.start() servers.append(server) addresses.append(address) time.sleep(1.5) time.sleep(1) packet = { 'method': 'TopologyCognition', 'sender': None, 'parameters': {}} packet_obj = FabnetPacketRequest(**packet) fri_client = FriClient() addr = random.choice(addresses) fri_client.call(addr, packet_obj) time.sleep(1) operator = OperatorClient('node%s'%addr.split(':')[-1]) home_dir = operator.get_home_dir() db = SafeJsonFile(os.path.join(home_dir, 'fabnet_topology.db')) nodes = db.read() print 'NODES LIST: %s'%str(nodes) for i in range(1900, 1900+NODES_COUNT): address = '127.0.0.1:%s'%i self.assertTrue(nodes.has_key(address)) self.assertTrue(len(nodes[address]['uppers']) >= 2) self.assertTrue(len(nodes[address]['superiors']) >= 2) self.assertEqual(nodes[address]['node_name'], 'node%s'%i) finally: for server in servers: if server: server.stop() server.join() time.sleep(1)
def get_discovered_nodes(self): ''' return nodes in following struct: { node_addr: { uppers: [addr, ...], superiors: [addr, ...] } ... } ''' db = os.path.join(self.home_dir, TOPOLOGY_DB) if not os.path.exists(db): return {} db = SafeJsonFile(db) nodes = db.read() return nodes
def before_resend(self, packet): """In this method should be implemented packet transformation for resend it to neighbours @params packet - object of FabnetPacketRequest class @return object of FabnetPacketRequest class or None for disabling packet resend to neigbours """ if packet.sender is None: db = SafeJsonFile(os.path.join(self.home_dir, TOPOLOGY_DB)) data = db.read() if data: for item in data.values(): item['old_data'] = 1 db.write(data) return packet
def call_repair_data(address, out_streem, expect_res, invalid_node=None): if os.path.exists(NDB): os.remove(NDB) client = FriClient() params = {} packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = client.call_sync(address, packet_obj) dht_info = ret_packet.ret_parameters['DHTInfo'] free_size = dht_info['free_size_percents'] if invalid_node: packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = client.call_sync(invalid_node, packet_obj) dht_info = ret_packet.ret_parameters['DHTInfo'] start = dht_info['range_start'] end = dht_info['range_end'] params = {'check_range_start': start, 'check_range_end': end} t0 = datetime.now() packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params) rcode, rmsg = client.call(address, packet_obj) if rcode != 0: raise Exception('RepairDataBlocks does not started. Details: %s'%rmsg) cnt = 0 try_cnt = 0 while cnt != expect_res: if try_cnt == 100: print 'Notifications count: %s, but expected: %s'%(cnt, expect_res) try_cnt = 0 try_cnt += 1 db = SafeJsonFile(NDB) data = db.read() events = data.get('notifications', []) events = filter(lambda e: e['event_topic']=='RepairDataBlocks', events) cnt = len(events) if cnt == 2: break time.sleep(.2) dt = datetime.now() - t0 db = SafeJsonFile(NDB) data = db.read() events = data.get('notifications', []) events = filter(lambda e: e['event_topic']=='RepairDataBlocks', events) for event in events: out_streem.write('[%s][%s][%s] %s\n'%(event['event_provider'], event['notify_dt'], event['event_type'], event['event_message'])) #packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) #client = FriClient() #ret_packet = client.call_sync(event[0], packet_obj) #out_streem.write('[%s] stat: %s\n'%(event[0], ret_packet.ret_parameters['methods_stat'])) packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = client.call_sync(address, packet_obj) dht_info = ret_packet.ret_parameters['DHTInfo'] post_free_size = dht_info['free_size_percents'] if post_free_size < free_size: out_streem.write('INVALID RESTORED RANGE FREE SIZE. BEFORE=%s, AFTER=%s\n'%(free_size, post_free_size)) out_streem.write('Process time: %s\n'%dt)