def create_network(ip_addr, hdds_size): addresses = [] processes = [] start_port = 1990 for i in xrange(len(hdds_size)): node_name = 'node%02i'%i destroy_fake_hdd(node_name, '/dev/loop%i'%i) homedir = make_fake_hdd(node_name, hdds_size[i], '/dev/loop%i'%i) if not addresses: n_node = 'init-fabnet' else: n_node = random.choice(addresses) address = '%s:%s'%(ip_addr, start_port) addresses.append(address) start_port += 1 logger.warning('{SNP} STARTING NODE %s'%address) p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, node_name, homedir, 'DHT', '--nodaemon']) processes.append(p) wait_node(address) logger.warning('{SNP} NODE %s IS STARTED'%address) print 'Network is started!' return addresses, processes
def start(self, neighbour): address = '%s:%s' % (self.hostname, self.port) if not neighbour: is_init_node = True else: is_init_node = False operator = OPERATOR(address, self.home_dir, self.keystore, is_init_node, self.node_name) for (op_name, op_class) in OPERATIONS_MAP.items(): operator.register_operation(op_name, op_class) self.server = FriServer(self.hostname, self.port, operator, \ server_name=self.node_name, \ keystorage=self.keystore) started = self.server.start() if not started: return started if is_init_node: return True packet = FabnetPacketRequest(method='DiscoveryOperation', sender=address) rcode, rmsg = self.server.operator.call_node(neighbour, packet) if rcode: logger.warning('Neighbour %s does not respond!'%neighbour) return True
def process(self, packet): """In this method should be implemented logic of processing reuqest packet from sender node @param packet - object of FabnetPacketRequest class @return object of FabnetPacketResponse or None for disabling packet response to sender """ key = packet.parameters.get('key', None) replica_count = packet.parameters.get('replica_count', None) if key is None: return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Key is not found in request packet!') if replica_count is None: return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Replica count should be passed to GetKeysInfo operation') self._validate_key(key) keys = KeyUtils.get_all_keys(key, replica_count) is_replica = False ret_keys = [] for key in keys: long_key = self._validate_key(key) range_obj = self.operator.ranges_table.find(long_key) if not range_obj: logger.warning('[GetKeysInfoOperation] Internal error: No hash range found for key=%s!'%key) else: ret_keys.append((key, is_replica, range_obj.node_address)) is_replica = True return FabnetPacketResponse(ret_parameters={'keys_info': ret_keys})
def create_monitor(neigbour): os.system('rm -rf %s'%monitoring_home) os.system('mkdir %s'%monitoring_home) address = '%s:%s'%('127.0.0.1', 1989) logger.warning('{SNP} STARTING MONITORING NODE %s'%address) mon_p = subprocess.Popen(['/usr/bin/python', FABNET_NODE_BIN, address, neigbour, 'monitor', monitoring_home, 'TestMonitor', '--nodaemon'], env=ENVIRON) logger.warning('{SNP} PROCESS STARTED') time.sleep(1) return mon_p
def create_net(self, nodes_count): global PROCESSES global ADDRESSES for unuse in range(nodes_count): if not ADDRESSES: n_node = 'init-fabnet' i = 1900 else: n_node = random.choice(ADDRESSES) i = int(ADDRESSES[-1].split(':')[-1])+1 self._wait_node(n_node) address = '127.0.0.1:%s'%i ADDRESSES.append(address) home = '/tmp/node_%s'%i if os.path.exists(home): shutil.rmtree(home) os.mkdir(home) os.system('cp fabnet_core/tests/cert/test_certs.ca %s/'%home) logger.warning('{SNP} STARTING NODE %s'%address) if n_node == 'init-fabnet': ntype = 'MGMT' Config.load(os.path.join(home, 'fabnet.conf')) Config.update_config({'db_engine': 'mongodb', \ 'db_conn_str': "mongodb://127.0.0.1/%s"%MONITOR_DB,\ 'COLLECT_NODES_STAT_TIMEOUT': 1, \ 'mgmt_cli_port': 2323, 'AUTH_KEY_CHANGE_PERIOD': 2, \ 'mgmt_rest_port': 9923}) print open(os.path.join(home, 'fabnet.conf')).read() else: ntype = 'Base' args = ['/usr/bin/python', './fabnet_core/bin/fabnet-node', address, n_node, 'NODE%.02i'%i, home, ntype, \ KS_PATH, '--input-pwd', '--nodaemon'] if DEBUG: args.append('--debug') print ' '.join(args) #p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\ p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stdout,\ env={'FABNET_PLUGINS_CONF': 'tests/plugins.yaml', 'PYTHONPATH': os.path.abspath('.')}) p.stdin.write(KS_PASSWD+'\n') logger.warning('{SNP} PROCESS STARTED') time.sleep(1) PROCESSES.append(p) #if len(ADDRESSES) > 2: # self._check_stat(address) for address in ADDRESSES: self._check_stat(address) time.sleep(1.5) print 'NETWORK STARTED'
def get_section(cls, section): plugins_config = cls.__get_plugins_config_file() if not os.path.exists(plugins_config): logger.warning('Plugins configuration does not found in %s'%plugins_config) return {} operators = {} data = yaml.load(open(plugins_config).read()) return data.get(section, {})
def create_net(self, nodes_count): global PROCESSES global ADDRESSES for unuse in range(nodes_count): if not ADDRESSES: n_node = "init-fabnet" i = 1900 else: n_node = random.choice(ADDRESSES) i = int(ADDRESSES[-1].split(":")[-1]) + 1 self._wait_node(n_node) address = "127.0.0.1:%s" % i ADDRESSES.append(address) home = "/tmp/node_%s" % i if os.path.exists(home): shutil.rmtree(home) os.mkdir(home) logger.warning("{SNP} STARTING NODE %s" % address) if n_node == "init-fabnet": ntype = "Monitor" Config.load(os.path.join(home, "node_config")) Config.update_config( {"db_engine": "postgresql", "db_conn_str": "dbname=%s user=postgres" % MONITOR_DB}, "Monitor" ) else: ntype = "DHT" args = [ "/usr/bin/python", "./fabnet/bin/fabnet-node", address, n_node, "%.02i" % i, home, ntype, "--nodaemon", ] if DEBUG: args.append("--debug") p = subprocess.Popen(args) logger.warning("{SNP} PROCESS STARTED") time.sleep(1) PROCESSES.append(p) # if len(ADDRESSES) > 2: # self._check_stat(address) for address in ADDRESSES: self._check_stat(address) time.sleep(1.5) print "NETWORK STARTED"
def create_monitor(neigbour): os.system('rm -rf %s'%monitoring_home) os.system('mkdir %s'%monitoring_home) Config.load(os.path.join(monitoring_home, 'node_config')) Config.update_config({'db_engine': 'postgresql', \ 'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB}, 'Monitor') address = '%s:%s'%('127.0.0.1', 1989) logger.warning('{SNP} STARTING MONITORING NODE %s'%address) mon_p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, neigbour, 'monitor', monitoring_home, 'Monitor', '--nodaemon']) logger.warning('{SNP} PROCESS STARTED') time.sleep(1) return mon_p
def reboot_nodes(processes, addresses, timeout=None): ret_processes = [] try: for i, address in enumerate(addresses): node_name = 'node%02i'%i homedir = '/tmp/mnt_%s'%node_name proc = processes[i] #stop node logger.warning('{SNP} STOPPING NODE %s'%address) proc.send_signal(signal.SIGINT) proc.wait() logger.warning('{SNP} NODE %s IS STOPPED'%address) if timeout: time.sleep(timeout) #start node while True: n_node = random.choice(addresses) if n_node != address: break logger.warning('{SNP} STARTING NODE %s'%address) p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, node_name, homedir, 'DHT', '--nodaemon']) ret_processes.append(p) wait_node(address) logger.warning('{SNP} NODE %s IS STARTED'%address) except Exception, err: destroy_network(ret_processes) raise err
def get_operators(cls): operators = cls.get_section('operators') for node_type, op_info in operators.items(): obj_path = op_info['object_path'] version = op_info.get('version', 'unknown') cls.__versions[node_type.lower()] = version parts = obj_path.split('.') obj = parts[-1] module = '.'.join(parts[:-1]) try: exec('from %s import %s'%(module, obj)) except Exception, err: logger.warning('Plugin %s loading error. Details: %s'%(obj_path, err)) continue operators[node_type.lower()] = eval(obj)
def create_net(self, nodes_count): PROCESSES = [] addresses = [] try: for i in range(1900, 1900+nodes_count): address = '127.0.0.1:%s'%i if not addresses: n_node = 'fake:9999' else: n_node = random.choice(addresses) addresses.append(address) home = '/tmp/node_%s'%i if os.path.exists(home): shutil.rmtree(home) os.mkdir(home) logger.warning('{SNP} STARTING NODE %s'%address) p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, '%.02i'%i, home, 'DHT']) logger.warning('{SNP} PROCESS STARTED') time.sleep(.5) PROCESSES.append(p) if len(addresses) > 3: self._check_stat(address) p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address]) node_i = address.split(':')[-1] self._wait_topology(node_i, len(addresses)) os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-orig.tgf'%(node_i, len(addresses))) for address in addresses: print 'Collecting topology from %s ...'%address p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address, '{"need_rebalance": 1}']) node_i = address.split(':')[-1] self._wait_topology(node_i, nodes_count) os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-balanced.tgf'%(node_i, nodes_count,)) print 'TOPOLOGY REBALANCED' return del_count = nodes_count/3 for i in xrange(del_count): process = random.choice(PROCESSES) idx = PROCESSES.index(process) del_address = addresses[idx] print 'STOPPING %s'%del_address del PROCESSES[idx] del addresses[idx] process.send_signal(signal.SIGINT) process.wait() time.sleep(80) address = random.choice(addresses) print 'Collecting topology from %s ...'%address p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address]) time.sleep(1) node_i = address.split(':')[-1] self._wait_topology(node_i, nodes_count-del_count) os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-%s.tgf'% \ (node_i, nodes_count, del_count)) finally: for process in PROCESSES: process.send_signal(signal.SIGINT) print 'SENDED SIGNALS' for process in PROCESSES: process.wait() print 'STOPPED'