Ejemplo n.º 1
0
def create_network(ip_addr, hdds_size):
    addresses = []
    processes = []

    start_port = 1990
    for i in xrange(len(hdds_size)):
        node_name = 'node%02i'%i
        destroy_fake_hdd(node_name, '/dev/loop%i'%i)
        homedir = make_fake_hdd(node_name, hdds_size[i], '/dev/loop%i'%i)

        if not addresses:
            n_node = 'init-fabnet'
        else:
            n_node = random.choice(addresses)

        address = '%s:%s'%(ip_addr, start_port)
        addresses.append(address)
        start_port += 1

        logger.warning('{SNP} STARTING NODE %s'%address)
        p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, node_name, homedir, 'DHT', '--nodaemon'])
        processes.append(p)
        wait_node(address)
        logger.warning('{SNP} NODE %s IS STARTED'%address)

    print 'Network is started!'
    return addresses, processes
Ejemplo n.º 2
0
    def start(self, neighbour):
        address = '%s:%s' % (self.hostname, self.port)
        if not neighbour:
            is_init_node = True
        else:
            is_init_node = False

        operator = OPERATOR(address, self.home_dir, self.keystore, is_init_node, self.node_name)

        for (op_name, op_class) in OPERATIONS_MAP.items():
            operator.register_operation(op_name, op_class)

        self.server = FriServer(self.hostname, self.port, operator, \
                                    server_name=self.node_name, \
                                    keystorage=self.keystore)

        started = self.server.start()
        if not started:
            return started

        if is_init_node:
            return True

        packet = FabnetPacketRequest(method='DiscoveryOperation', sender=address)

        rcode, rmsg = self.server.operator.call_node(neighbour, packet)
        if rcode:
            logger.warning('Neighbour %s does not respond!'%neighbour)

        return True
Ejemplo n.º 3
0
    def process(self, packet):
        """In this method should be implemented logic of processing
        reuqest packet from sender node

        @param packet - object of FabnetPacketRequest class
        @return object of FabnetPacketResponse
                or None for disabling packet response to sender
        """
        key = packet.parameters.get('key', None)
        replica_count = packet.parameters.get('replica_count', None)
        if key is None:
            return FabnetPacketResponse(ret_code=RC_ERROR,
                    ret_message='Key is not found in request packet!')

        if replica_count is None:
            return FabnetPacketResponse(ret_code=RC_ERROR,
                    ret_message='Replica count should be passed to GetKeysInfo operation')

        self._validate_key(key)
        keys = KeyUtils.get_all_keys(key, replica_count)

        is_replica = False
        ret_keys = []
        for key in keys:
            long_key = self._validate_key(key)
            range_obj = self.operator.ranges_table.find(long_key)
            if not range_obj:
                logger.warning('[GetKeysInfoOperation] Internal error: No hash range found for key=%s!'%key)
            else:
                ret_keys.append((key, is_replica, range_obj.node_address))
            is_replica = True

        return FabnetPacketResponse(ret_parameters={'keys_info': ret_keys})
Ejemplo n.º 4
0
def create_monitor(neigbour):
    os.system('rm -rf %s'%monitoring_home)
    os.system('mkdir %s'%monitoring_home)
    address = '%s:%s'%('127.0.0.1', 1989)
    logger.warning('{SNP} STARTING MONITORING NODE %s'%address)
    mon_p = subprocess.Popen(['/usr/bin/python', FABNET_NODE_BIN, address, neigbour, 'monitor', monitoring_home, 'TestMonitor', '--nodaemon'], env=ENVIRON)
    logger.warning('{SNP} PROCESS STARTED')
    time.sleep(1)
    return mon_p
Ejemplo n.º 5
0
    def create_net(self, nodes_count):
        global PROCESSES
        global ADDRESSES

        for unuse in range(nodes_count):
            if not ADDRESSES:
                n_node = 'init-fabnet'
                i = 1900
            else:
                n_node = random.choice(ADDRESSES)
                i = int(ADDRESSES[-1].split(':')[-1])+1
                self._wait_node(n_node)

            address = '127.0.0.1:%s'%i
            ADDRESSES.append(address)

            home = '/tmp/node_%s'%i
            if os.path.exists(home):
                shutil.rmtree(home)
            os.mkdir(home)
            os.system('cp fabnet_core/tests/cert/test_certs.ca %s/'%home)

            logger.warning('{SNP} STARTING NODE %s'%address)
            if n_node == 'init-fabnet':
                ntype = 'MGMT'
                Config.load(os.path.join(home, 'fabnet.conf'))
                Config.update_config({'db_engine': 'mongodb', \
                        'db_conn_str': "mongodb://127.0.0.1/%s"%MONITOR_DB,\
                        'COLLECT_NODES_STAT_TIMEOUT': 1, \
                        'mgmt_cli_port': 2323,
                        'AUTH_KEY_CHANGE_PERIOD': 2, \
                        'mgmt_rest_port': 9923})
                print open(os.path.join(home, 'fabnet.conf')).read()
            else:
                ntype = 'Base'

            args = ['/usr/bin/python', './fabnet_core/bin/fabnet-node', address, n_node, 'NODE%.02i'%i, home, ntype, \
                    KS_PATH, '--input-pwd', '--nodaemon']
            if DEBUG:
                args.append('--debug')
            print ' '.join(args)
            #p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,\
            p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=sys.stdout,\
                    env={'FABNET_PLUGINS_CONF': 'tests/plugins.yaml', 'PYTHONPATH': os.path.abspath('.')})
            p.stdin.write(KS_PASSWD+'\n')
            logger.warning('{SNP} PROCESS STARTED')
            time.sleep(1)

            PROCESSES.append(p)
            #if len(ADDRESSES) > 2:
            #    self._check_stat(address)

        for address in ADDRESSES:
            self._check_stat(address)

        time.sleep(1.5)
        print 'NETWORK STARTED'
Ejemplo n.º 6
0
    def get_section(cls, section):
        plugins_config = cls.__get_plugins_config_file()
        if not os.path.exists(plugins_config):
            logger.warning('Plugins configuration does not found in %s'%plugins_config)
            return {}

        operators = {}
        data = yaml.load(open(plugins_config).read())
        return data.get(section, {})
Ejemplo n.º 7
0
    def create_net(self, nodes_count):
        global PROCESSES
        global ADDRESSES

        for unuse in range(nodes_count):
            if not ADDRESSES:
                n_node = "init-fabnet"
                i = 1900
            else:
                n_node = random.choice(ADDRESSES)
                i = int(ADDRESSES[-1].split(":")[-1]) + 1
                self._wait_node(n_node)

            address = "127.0.0.1:%s" % i
            ADDRESSES.append(address)

            home = "/tmp/node_%s" % i
            if os.path.exists(home):
                shutil.rmtree(home)
            os.mkdir(home)

            logger.warning("{SNP} STARTING NODE %s" % address)
            if n_node == "init-fabnet":
                ntype = "Monitor"
                Config.load(os.path.join(home, "node_config"))
                Config.update_config(
                    {"db_engine": "postgresql", "db_conn_str": "dbname=%s user=postgres" % MONITOR_DB}, "Monitor"
                )
            else:
                ntype = "DHT"
            args = [
                "/usr/bin/python",
                "./fabnet/bin/fabnet-node",
                address,
                n_node,
                "%.02i" % i,
                home,
                ntype,
                "--nodaemon",
            ]
            if DEBUG:
                args.append("--debug")
            p = subprocess.Popen(args)
            logger.warning("{SNP} PROCESS STARTED")
            time.sleep(1)

            PROCESSES.append(p)
            # if len(ADDRESSES) > 2:
            #    self._check_stat(address)

        for address in ADDRESSES:
            self._check_stat(address)

        time.sleep(1.5)
        print "NETWORK STARTED"
Ejemplo n.º 8
0
def create_monitor(neigbour):
    os.system('rm -rf %s'%monitoring_home)
    os.system('mkdir %s'%monitoring_home)
    Config.load(os.path.join(monitoring_home, 'node_config'))
    Config.update_config({'db_engine': 'postgresql', \
        'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB}, 'Monitor')
    address = '%s:%s'%('127.0.0.1', 1989)
    logger.warning('{SNP} STARTING MONITORING NODE %s'%address)
    mon_p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, neigbour, 'monitor', monitoring_home, 'Monitor', '--nodaemon'])
    logger.warning('{SNP} PROCESS STARTED')
    time.sleep(1)
    return mon_p
Ejemplo n.º 9
0
def reboot_nodes(processes, addresses, timeout=None):
    ret_processes = []
    try:
        for i, address in enumerate(addresses):
            node_name = 'node%02i'%i
            homedir = '/tmp/mnt_%s'%node_name
            proc = processes[i]

            #stop node
            logger.warning('{SNP} STOPPING NODE %s'%address)
            proc.send_signal(signal.SIGINT)
            proc.wait()
            logger.warning('{SNP} NODE %s IS STOPPED'%address)

            if timeout:
                time.sleep(timeout)

            #start node
            while True:
                n_node = random.choice(addresses)
                if n_node != address:
                    break
            logger.warning('{SNP} STARTING NODE %s'%address)
            p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, node_name, homedir, 'DHT', '--nodaemon'])
            ret_processes.append(p)
            wait_node(address)
            logger.warning('{SNP} NODE %s IS STARTED'%address)
    except Exception, err:
        destroy_network(ret_processes)
        raise err
Ejemplo n.º 10
0
    def get_operators(cls):
        operators = cls.get_section('operators')

        for node_type, op_info in operators.items():
            obj_path = op_info['object_path']
            version = op_info.get('version', 'unknown')
            cls.__versions[node_type.lower()] = version
            parts = obj_path.split('.')
            obj = parts[-1]
            module = '.'.join(parts[:-1])

            try:
                exec('from %s import %s'%(module, obj))
            except Exception, err:
                logger.warning('Plugin %s loading error. Details: %s'%(obj_path, err))
                continue

            operators[node_type.lower()] = eval(obj)
Ejemplo n.º 11
0
    def create_net(self, nodes_count):
        PROCESSES = []
        addresses = []
        try:
            for i in range(1900, 1900+nodes_count):
                address = '127.0.0.1:%s'%i
                if not addresses:
                    n_node = 'fake:9999'
                else:
                    n_node = random.choice(addresses)

                addresses.append(address)

                home = '/tmp/node_%s'%i
                if os.path.exists(home):
                    shutil.rmtree(home)
                os.mkdir(home)
                logger.warning('{SNP} STARTING NODE %s'%address)
                p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fabnet-node', address, n_node, '%.02i'%i, home, 'DHT'])
                logger.warning('{SNP} PROCESS STARTED')
                time.sleep(.5)

                PROCESSES.append(p)
                if len(addresses) > 3:
                    self._check_stat(address)


            p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address])
            node_i = address.split(':')[-1]
            self._wait_topology(node_i, len(addresses))
            os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-orig.tgf'%(node_i, len(addresses)))


            for address in addresses:
                print 'Collecting topology from %s ...'%address
                p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address, '{"need_rebalance": 1}'])

                node_i = address.split(':')[-1]
                self._wait_topology(node_i, nodes_count)

                os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-balanced.tgf'%(node_i, nodes_count,))

            print 'TOPOLOGY REBALANCED'

            return

            del_count = nodes_count/3
            for i in xrange(del_count):
                process = random.choice(PROCESSES)
                idx = PROCESSES.index(process)
                del_address = addresses[idx]
                print 'STOPPING %s'%del_address
                del PROCESSES[idx]
                del addresses[idx]
                process.send_signal(signal.SIGINT)
                process.wait()

            time.sleep(80)

            address = random.choice(addresses)
            print 'Collecting topology from %s ...'%address
            p = subprocess.Popen(['/usr/bin/python', './fabnet/bin/fri-caller', 'TopologyCognition', address])
            time.sleep(1)
            node_i = address.split(':')[-1]
            self._wait_topology(node_i, nodes_count-del_count)
            os.system('python ./tests/topology_to_tgf /tmp/node_%s/fabnet_topology.db /tmp/fabnet_topology.%s-%s.tgf'% \
                        (node_i, nodes_count, del_count))

        finally:
            for process in PROCESSES:
                process.send_signal(signal.SIGINT)

            print 'SENDED SIGNALS'
            for process in PROCESSES:
                process.wait()

            print 'STOPPED'