def __init__(self, self_address, home_dir='/tmp/', certfile=None, is_init_node=False, node_name='unknown'): Operator.__init__(self, self_address, home_dir, certfile, is_init_node, node_name) self.status = DS_INITIALIZE self.ranges_table = HashRangesTable() if is_init_node: self.ranges_table.append(MIN_HASH, MAX_HASH, self.self_address) self.save_path = os.path.join(home_dir, 'dht_range') if not os.path.exists(self.save_path): os.mkdir(self.save_path) self.__dht_range = FSHashRanges.discovery_range(self.save_path) self.__split_requests_cache = [] self.__start_dht_try_count = 0 self.__init_dht_thread = None if is_init_node: self.status = DS_NORMALWORK self.__check_hash_table_thread = CheckLocalHashTableThread(self) self.__check_hash_table_thread.setName('%s-CheckLocalHashTableThread'%self.node_name) self.__check_hash_table_thread.start() self.__monitor_dht_ranges = MonitorDHTRanges(self) self.__monitor_dht_ranges.setName('%s-MonitorDHTRanges'%self.node_name) self.__monitor_dht_ranges.start()
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, \ is_init_node=False, node_name='unknown', config={}): cur_cfg = {} cur_cfg.update(DEFAULT_DHT_CONFIG) cur_cfg.update(config) Operator.__init__(self, self_address, home_dir, key_storage, \ is_init_node, node_name, cur_cfg) self.status = DS_INITIALIZE self.ranges_table = HashRangesTable() self.save_path = os.path.join(home_dir, 'dht_range') if not os.path.exists(self.save_path): os.mkdir(self.save_path) self.__usr_md_cache = MetadataCache() self.__split_requests_cache = [] self.__dht_range = FSMappedDHTRange.discovery_range(self.save_path) self.ranges_table.append(self.__dht_range.get_start(), self.__dht_range.get_end(), self.self_address) self.__start_dht_try_count = 0 self.__init_dht_thread = None self.__check_hash_table_thread = CheckLocalHashTableThread(self) self.__check_hash_table_thread.setName('%s-CheckLocalHashTableThread'%self.node_name) self.__check_hash_table_thread.start() self.__monitor_dht_ranges = MonitorDHTRanges(self) self.__monitor_dht_ranges.setName('%s-MonitorDHTRanges'%self.node_name) self.__monitor_dht_ranges.start() self.status = DS_PREINIT
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, is_init_node=False, node_name='unknown', config={}): cur_cfg = {} cur_cfg.update(DEFAULT_MONITOR_CONFIG) cur_cfg.update(config) Operator.__init__(self, self_address, home_dir, key_storage, is_init_node, node_name, cur_cfg) self.__db_conn_str = None self.__db_api = None self.check_database() if key_storage: cert = key_storage.get_node_cert() ckey = key_storage.get_node_cert_key() else: cert = ckey = None client = FriClient(bool(cert), cert, ckey) self.__collect_up_nodes_stat_thread = CollectNodeStatisticsThread(self, client, UP) self.__collect_up_nodes_stat_thread.setName('%s-UP-CollectNodeStatisticsThread'%self.node_name) self.__collect_up_nodes_stat_thread.start() self.__collect_dn_nodes_stat_thread = CollectNodeStatisticsThread(self, client, DOWN) self.__collect_dn_nodes_stat_thread.setName('%s-DN-CollectNodeStatisticsThread'%self.node_name) self.__collect_dn_nodes_stat_thread.start() self.__discovery_topology_thrd = DiscoverTopologyThread(self) self.__discovery_topology_thrd.setName('%s-DiscoverTopologyThread'%self.node_name) self.__discovery_topology_thrd.start()
def test02_ssl_bad_cert(self): keystorage = FileBasedKeyStorage(VALID_STORAGE, PASSWD) inv_keystorage = FileBasedKeyStorage(INVALID_STORAGE, PASSWD) server1 = server2 = None try: operator = Operator('127.0.0.1:1986', key_storage=keystorage) operator.neighbours = ['127.0.0.1:1987'] operator.register_operation('ECHO', EchoOperation) server1 = FriServer('0.0.0.0', 1986, operator, 10, 'node_1', keystorage) ret = server1.start() self.assertEqual(ret, True) operator = Operator('127.0.0.1:1987', key_storage=inv_keystorage) operator.neighbours = ['127.0.0.1:1986'] operator.register_operation('ECHO', EchoOperation) server2 = FriServer('0.0.0.0', 1987, operator, 10, 'node_2', inv_keystorage) ret = server2.start() self.assertEqual(ret, True) packet = { 'message_id': 323232, 'method': 'ECHO', 'sync': False, 'sender': '127.0.0.1:1987', 'parameters': {'message': 'test message'}} packet_obj = FabnetPacketRequest(**packet) rcode, rmsg = operator.call_node('127.0.0.1:1986', packet_obj) self.assertEqual(rcode, 1, rmsg) time.sleep(.1) finally: if server1: server1.stop() if server2: server2.stop()
def stop(self): self.status = DS_DESTROYING for range_obj in self.ranges_table.iter_table(): if range_obj.node_address == self.self_address: self._move_range(range_obj) break Operator.stop(self) self.__check_hash_table_thread.stop() self.__monitor_dht_ranges.stop() self.__check_hash_table_thread.join() self.__monitor_dht_ranges.join()
def test_manage_neighbours(self): operator = Operator('127.0.0.1:1986') manage_neighbours = ManageNeighbour(operator, FriClient(), '127.0.0.1:1986', '/tmp', None) manage_neighbours._init_operation = _init_operation_mock threads = [] for i in xrange(10): tst = TestServerThread(manage_neighbours) threads.append(tst) for thread in threads: thread.start() for thread in threads: thread.join() operator.stop()
def get_statistic(self): stat = Operator.get_statistic(self) dht_range = self.get_dht_range() stat['status'] = self.status stat['range_start'] = '%040x'% dht_range.get_start() stat['range_end'] = '%040x'% dht_range.get_end() stat['range_size'] = dht_range.get_range_size() stat['replicas_size'] = dht_range.get_replicas_size() stat['free_size'] = dht_range.get_free_size() return stat
def on_statisic_request(self): stat = Operator.on_statisic_request(self) dht_range = self.get_dht_range() dht_i = {} dht_i['status'] = self.status dht_i['range_start'] = '%040x'% dht_range.get_start() dht_i['range_end'] = '%040x'% dht_range.get_end() dht_i['range_size'] = dht_range.get_range_size() dht_i['replicas_size'] = dht_range.get_replicas_size() dht_i['free_size'] = dht_range.get_free_size() dht_i['free_size_percents'] = dht_range.get_free_size_percents() stat['DHTInfo'] = dht_i return stat
def test_node(self): try: server = None address = '127.0.0.1:1987' operator = Operator(address) operator.register_operation('NodeStatistic', NodeStatisticOperation) server = FriServer('0.0.0.0', 1987, operator, server_name='node_test') ret = server.start() self.assertEqual(ret, True) time.sleep(.5) packet = { 'message_id': 323232, 'method': 'NodeStatistic', 'sender': '', 'parameters': {}, 'sync': True} packet_obj = FabnetPacketRequest(**packet) client = FriClient() ret_packet = client.call_sync('127.0.0.1:1987', packet_obj) self.assertEqual(isinstance(ret_packet, FabnetPacketResponse), True) self.assertEqual(ret_packet.ret_code, 0, ret_packet.ret_message) self.assertTrue(int(ret_packet.ret_parameters['workers_count']) > 2) self.assertEqual(int(ret_packet.ret_parameters['uppers_balance']), -1) self.assertEqual(int(ret_packet.ret_parameters['superiors_balance']), -1) self.assertTrue(int(ret_packet.ret_parameters['threads']) > 6) self.assertTrue(int(ret_packet.ret_parameters['memory']) > 1000) self.assertEqual(len(ret_packet.ret_parameters['methods_stat']), 1) self.assertEqual(ret_packet.ret_parameters['methods_stat']['NodeStatistic'], 1) print ret_packet.ret_parameters time.sleep(.2) except Exception, err: print 'ERROR: %s'%err raise err
def on_statisic_request(self): stat = Operator.on_statisic_request(self) dht_range = self.get_dht_range() dht_i = {} dht_i['status'] = self.status dht_i['range_start'] = '%040x'% dht_range.get_start() dht_i['range_end'] = '%040x'% dht_range.get_end() #FIXME! make me in separate thread! dht_i['range_size'] = dht_range.get_data_size(FSMappedDHTRange.DBCT_MASTER) dht_i['replicas_size'] = dht_range.get_data_size(FSMappedDHTRange.DBCT_REPLICA) dht_i['metadata_size'] = dht_range.get_data_size(FSMappedDHTRange.DBCT_MD_MASTER) \ + dht_range.get_data_size(FSMappedDHTRange.DBCT_MD_REPLICA) dht_i['free_size'] = dht_range.get_free_size() dht_i['free_size_percents'] = dht_range.get_free_size_percents() stat['DHTInfo'] = dht_i return stat
def run(self): address = '127.0.0.1:%s'%self.port operator = Operator(address) self.operator = operator operator.register_operation('ManageNeighbour', ManageNeighbour) operator.register_operation('DiscoveryOperation', DiscoveryOperation) operator.register_operation('TopologyCognition', TopologyCognition) server = FriServer('0.0.0.0', self.port, operator, server_name='node_%s'%self.port) ret = server.start() if not ret: return self.stopped = False while not self.stopped: time.sleep(0.1) server.stop()
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, is_init_node=False, node_name='unknown', config={}): cur_cfg = {} cur_cfg.update(DEFAULT_MONITOR_CONFIG) cur_cfg.update(config) Operator.__init__(self, self_address, home_dir, key_storage, is_init_node, node_name, cur_cfg) self.__db_conn_str = None self.__db_api = None self.check_database() client = FriClient(key_storage) db_conn_str = Config.get('db_conn_str') self.__dbm = MgmtDatabaseManager(db_conn_str) ca_host = Config.get('mgmt_ca_host', None) ca_port = int(Config.get('mgmt_ca_port', 8080)) self.mgmt_api = mgmt_api = ManagementEngineAPI(self.__dbm, self_address, init_scheduler=True, \ ca_host=ca_host, ca_port=ca_port) BaseMgmtCLIHandler.mgmtManagementAPI = mgmt_api host = Config.get('mgmt_cli_host', '0.0.0.0') cli_port = int(Config.get('mgmt_cli_port', '23')) logger.info('Starting CLI at %s:%s'%(host, cli_port)) cli_server = TelnetServer((host, cli_port), BaseMgmtCLIHandler) #if key_storage: # cert, key = self.get_ssl_keycert('cli_api') # cli_server.socket = ssl.wrap_socket(cli_server.socket, certfile=cert, keyfile=key, server_side=True) self.__cli_api_thrd = ExternalAPIThread(cli_server, host, cli_port) self.__cli_api_thrd.setName('%s-CLIAPIThread'%self.node_name) self.__cli_api_thrd.start() RESTHandler.setup_mgmt_api(mgmt_api) host = Config.get('mgmt_rest_host', '0.0.0.0') rest_port = int(Config.get('mgmt_rest_port', '8080')) logger.info('Starting REST at %s:%s'%(host, rest_port)) rest_server = BaseHTTPServer.HTTPServer((host, rest_port), RESTHandler) if key_storage: cert, key = self.get_ssl_keycert('rest_api') rest_server.socket = ssl.wrap_socket(rest_server.socket, certfile=cert, keyfile=key, server_side=True) self.__rest_api_thrd = ExternalAPIThread(rest_server, host, rest_port) self.__rest_api_thrd.setName('%s-RESTAPIThread'%self.node_name) self.__rest_api_thrd.start() if not self.__cli_api_thrd.started(): self.__rest_api_thrd.stop() raise Exception('CLI API does not started!') if not self.__rest_api_thrd.started(): self.__cli_api_thrd.stop() raise Exception('CLI API does not started!') self.__collect_up_nodes_stat_thread = CollectNodeStatisticsThread(self, client, STATUS_UP) self.__collect_up_nodes_stat_thread.setName('%s-UP-CollectNodeStatisticsThread'%self.node_name) self.__collect_up_nodes_stat_thread.start() self.__collect_dn_nodes_stat_thread = CollectNodeStatisticsThread(self, client, STATUS_DOWN) self.__collect_dn_nodes_stat_thread.setName('%s-DN-CollectNodeStatisticsThread'%self.node_name) self.__collect_dn_nodes_stat_thread.start() self.__discovery_topology_thrd = DiscoverTopologyThread(self) self.__discovery_topology_thrd.setName('%s-DiscoverTopologyThread'%self.node_name) self.__discovery_topology_thrd.start()
def test_big_data_test(self): keystorage = FileBasedKeyStorage(VALID_STORAGE, PASSWD) try: operator = Operator("127.0.0.1:1986", key_storage=keystorage) operator.neighbours = ["127.0.0.1:1987"] operator.register_operation("ECHO", EchoOperation) server1 = FriServer("0.0.0.0", 1986, operator, 10, "node_1", keystorage) ret = server1.start() self.assertEqual(ret, True) operator = Operator("127.0.0.1:1987", key_storage=keystorage) operator.neighbours = ["127.0.0.1:1986"] operator.register_operation("ECHO", EchoOperation) server2 = FriServer("0.0.0.0", 1987, operator, 10, "node_2", keystorage) ret = server2.start() self.assertEqual(ret, True) print "Init big data..." data = "0123456789" * 900000 print "Done" data += "" packet = { "message_id": 323232, "method": "ECHO", "sync": False, "sender": "127.0.0.1:1987", "binary_data": data, } t0 = datetime.now() packet_obj = FabnetPacketRequest(**packet) rcode, rmsg = operator.call_node("127.0.0.1:1986", packet_obj) self.assertEqual(rcode, 0, rmsg) operator.wait_response(323232, 20) print "Echo big data time: %s" % (datetime.now() - t0) time.sleep(1) rcv_data = open("/tmp/big_message.out").read() self.assertEqual(len(rcv_data), len(data)) os.remove("/tmp/big_message.out") finally: if server1: server1.stop() if server2: server2.stop()
def __init__(self, self_address, home_dir='/tmp/', key_storage=None, \ is_init_node=False, node_name='unknown', config={}): Operator.__init__(self, self_address, home_dir, key_storage, \ is_init_node, node_name, config)
def __test_server(self, keystorage=None): server1 = server2 = None try: operator = Operator('127.0.0.1:1986', key_storage=keystorage) operator.neighbours = ['127.0.0.1:1987'] operator.register_operation('ECHO', EchoOperation) server1 = FriServer('0.0.0.0', 1986, operator, 10, 'node_1', keystorage) ret = server1.start() self.assertEqual(ret, True) operator = Operator('127.0.0.1:1987', key_storage=keystorage) operator.neighbours = ['127.0.0.1:1986'] operator.register_operation('ECHO', EchoOperation) server2 = FriServer('0.0.0.0', 1987, operator, 10, 'node_2', keystorage) ret = server2.start() self.assertEqual(ret, True) packet = { 'message_id': 323232, 'method': 'ECHO', 'sync': False, 'sender': '127.0.0.1:1987', 'parameters': {'message': 'test message'}} if keystorage: packet['session_id'] = keystorage.get_node_cert_key() packet_obj = FabnetPacketRequest(**packet) rcode, rmsg = operator.call_node('127.0.0.1:1986', packet_obj) self.assertEqual(rcode, 0, rmsg) operator.wait_response(323232, 1) self.assertEqual(os.path.exists('/tmp/server1.out'), True) self.assertEqual(os.path.exists('/tmp/server2.out'), True) request = json.loads(open('/tmp/server1.out').read()) response = json.loads(open('/tmp/server2.out').read()) self.assertEqual(request, packet) good_resp = {'from_node': '127.0.0.1:1986','message_id': 323232, 'ret_code': 0, 'ret_message': 'ok', 'ret_parameters': {'message': 'test message'}} self.assertEqual(response, good_resp) finally: if server1: server1.stop() if server2: server2.stop()
def test_discovery_operation(self): server1 = server2 = None try: operator = Operator('127.0.0.1:1986') operator.register_operation('ManageNeighbour', ManageNeighbour) operator.register_operation('DiscoveryOperation', DiscoveryOperation) operator.register_operation('TopologyCognition', TopologyCognition) server1 = FriServer('0.0.0.0', 1986, operator, server_name='node_1') ret = server1.start() self.assertEqual(ret, True) operator1 = Operator('127.0.0.1:1987') operator1.register_operation('ManageNeighbour', ManageNeighbour) operator1.register_operation('DiscoveryOperation', DiscoveryOperation) operator1.register_operation('TopologyCognition', TopologyCognition) server2 = FriServer('0.0.0.0', 1987, operator1, server_name='node_2') ret = server2.start() self.assertEqual(ret, True) packet = { 'message_id': 323232, 'method': 'DiscoveryOperation', 'sender': '127.0.0.1:1987', 'parameters': {}} packet_obj = FabnetPacketRequest(**packet) rcode, rmsg = operator1.call_node('127.0.0.1:1986', packet_obj) self.assertEqual(rcode, 0, rmsg) time.sleep(1) self.assertEqual(operator.upper_neighbours, ['127.0.0.1:1987']) self.assertEqual(operator.superior_neighbours, ['127.0.0.1:1987']) self.assertEqual(operator1.upper_neighbours, ['127.0.0.1:1986']) self.assertEqual(operator1.superior_neighbours, ['127.0.0.1:1986']) finally: if server1: server1.stop() if server2: server2.stop()
def test_discovery_operation(self): server1 = server2 = None try: operator = Operator("127.0.0.1:1986") operator.register_operation("ManageNeighbour", ManageNeighbour) operator.register_operation("DiscoveryOperation", DiscoveryOperation) operator.register_operation("TopologyCognition", TopologyCognition) server1 = FriServer("0.0.0.0", 1986, operator, server_name="node_1") ret = server1.start() self.assertEqual(ret, True) operator1 = Operator("127.0.0.1:1987") operator1.register_operation("ManageNeighbour", ManageNeighbour) operator1.register_operation("DiscoveryOperation", DiscoveryOperation) operator1.register_operation("TopologyCognition", TopologyCognition) server2 = FriServer("0.0.0.0", 1987, operator1, server_name="node_2") ret = server2.start() self.assertEqual(ret, True) packet = { "message_id": 323232, "method": "DiscoveryOperation", "sender": "127.0.0.1:1987", "parameters": {}, } packet_obj = FabnetPacketRequest(**packet) rcode, rmsg = operator1.call_node("127.0.0.1:1986", packet_obj) self.assertEqual(rcode, 0, rmsg) operator1.wait_response(323232, 1) server1.stop() time.sleep(3) server1 = None self.assertEqual(operator.upper_neighbours, ["127.0.0.1:1987"]) self.assertEqual(operator.superior_neighbours, ["127.0.0.1:1987"]) self.assertEqual(operator1.upper_neighbours, []) self.assertEqual(operator1.superior_neighbours, []) finally: if server1: server1.stop() if server2: server2.stop()