示例#1
0
    def _wait_node(self, node):
        idx = None
        for i, addr in enumerate(ADDRESSES):
            if addr == node:
                idx = i
        if idx is None:
            raise Exception('address %s does not found!'%node)

        proc = PROCESSES[idx]

        key_storage = init_keystore(KS_PATH, KS_PASSWD)
        client = FriClient(key_storage)
        while True:
            if not os.path.exists('/proc/%s'%proc.pid):
                raise Exception('Node process for %s does not found!')
            for line in open("/proc/%d/status" % proc.pid).readlines():
                if line.startswith("State:"):
                    status = line.split(":",1)[1].strip().split(' ')[0]
            if status == 'Z':
                raise Exception('Node died at %s'%node)

            packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
            ret_packet = client.call_sync(node, packet_obj)
            if ret_packet.ret_code:
                print ret_packet.ret_message
                print 'Node does not init FRI server yet. Waiting it...'
                time.sleep(.5)
                continue
            break
示例#2
0
    def get_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, client_ks=None, user_id_hash=''):
        params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash}
        req = FabnetPacketRequest(method='GetDataBlock', parameters=params)

        client = FriClient(client_ks)
        resp = client.call_sync('127.0.0.1:%s'%self.port, req)
        return resp
示例#3
0
    def test_node_upgrade(self):
        server_proc = None
        try:
            os.system('rm -rf /tmp/fabnet_node_code')
            os.system('rm -rf /tmp/upgrade_node.log')
            os.system('git clone %s /tmp/fabnet_node_code/'%ORIG_REPO)
            os.chdir('/tmp/fabnet_node_code/')
            #os.system('git reset --hard f797284a29fcbaa08d8ff68d07e5e6e352a95fe3')
            os.system('find . -name *.pyc -delete')

            address = '127.0.0.1:1987'
            args = ['/usr/bin/python', '/tmp/fabnet_node_code/fabnet/bin/fabnet-node', address, 'init-fabnet', 'test_upgr_node', '/tmp', 'DHT']
            args.append('--nodaemon')
            server_proc = subprocess.Popen(args)
            time.sleep(2)

            os.system("echo '{\"DHT\": 100500}' > UPGRADE_VERSION")
            packet_obj = FabnetPacketRequest(method='UpgradeNode', parameters={'origin_repo_url': 'bad_url'}, sync=True)
            client = FriClient()
            ret = client.call_sync('127.0.0.1:1987', packet_obj)
            self.assertEqual(ret.ret_code, 0)

            print 'upgrading /tmp/fabnet_node_code from %s'%ORIG_REPO
            packet_obj = FabnetPacketRequest(method='UpgradeNode', parameters={'origin_repo_url': ORIG_REPO}, sync=True)
            client = FriClient()
            ret = client.call_sync('127.0.0.1:1987', packet_obj)
            self.assertEqual(ret.ret_code, 0)
            self.assertTrue(os.path.exists('/tmp/upgrade_node.log'))
            self.assertTrue('successfully' in open('/tmp/upgrade_node.log').read())
        finally:
            os.system('rm -rf /tmp/fabnet_node_code')
            if server_proc:
                server_proc.send_signal(signal.SIGINT)
                server_proc.wait()
示例#4
0
def get_data_block(addresses, key):
    client = FriClient()
    packet_obj = FabnetPacketRequest(method='GetKeysInfo', \
            parameters={'key': key, 'replica_count': 2})

    nodeaddr = random.choice(addresses)
    ret_packet = client.call_sync(nodeaddr, packet_obj)
    if ret_packet.ret_code != 0:
        raise Exception('GetKeysInfo failed on %s: %s'%(nodeaddr, ret_packet.ret_message))

    for key, dbct, nodeaddr in ret_packet.ret_parameters['keys_info']:
        params = {'key': key, 'dbct': dbct}
        packet_obj = FabnetPacketRequest(method='GetDataBlock', parameters=params)
        ret_packet = client.call_sync(nodeaddr, packet_obj)
        if ret_packet.ret_code != 0:
            continue

        data = ret_packet.binary_data.data()
        if hashlib.sha1(data).hexdigest() != ret_packet.ret_parameters['checksum']:
            ret_packet.ret_message = 'Data block checksum failed!'
            continue

        return data
        
    raise Exception('GetDataBlock failed on %s: %s'%(nodeaddr, ret_packet.ret_message))
示例#5
0
    def activate_on_dht(self, user_name, dht_endpoint):
        conn = self.__ca_call('/get_certificate_info', {'cn': user_name})
        try:
            resp = conn.getresponse()
            if resp.status != 200:
                raise Exception('CA service error! Get cert info: [%s %s] %s'%\
                        (resp.status, resp.reason, resp.read()))
            data = resp.read()
        finally:
            conn.close()

        data = json.loads(data)
        ###data['cert_term']
        #cert_add_info
        #cert_term
        #status
        fri_client = FriClient(self.ks)
        add_info = json.loads(data['cert_add_info'])

        params = {'user_id_hash': hashlib.sha1(user_name).hexdigest(), 'storage_size': add_info['storage_capacity']}

        packet_obj = FabnetPacketRequest(method='UpdateUserProfile', parameters=params)
        ret_packet = fri_client.call_sync(dht_endpoint, packet_obj) 
        if ret_packet.ret_code != 0:
            raise Exception(ret_packet.ret_message)
示例#6
0
    def test_node(self):
        try:
            server = None
            address = "127.0.0.1:1987"
            server = Node("127.0.0.1", 1987, "/tmp", "test_node", ks_path=None, ks_passwd=None, node_type="BASE")

            server.start(None)
            time.sleep(0.5)

            fri_client = FriClient()
            params = {}
            packet_obj = FabnetPacketRequest(method="UpdateNodeConfig", parameters=params, sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertNotEqual(resp.ret_code, 0)

            params = {"config": {"TEST": {"TEST_CONFIG": "str", "TEST_INT": 234}}}
            packet_obj = FabnetPacketRequest(method="UpdateNodeConfig", parameters=params, sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertEqual(resp.ret_code, 0, resp.ret_message)

            packet_obj = FabnetPacketRequest(method="GetNodeConfig", sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertEqual(resp.ret_code, 0, resp.ret_message)
            self.assertTrue("TEST" in resp.ret_parameters, resp.ret_parameters)

            self.assertEqual(resp.ret_parameters["TEST"]["TEST_CONFIG"], "str")
            self.assertEqual(resp.ret_parameters["TEST"]["TEST_INT"], 234)

            time.sleep(0.2)
        except Exception, err:
            print "ERROR: %s" % err
            raise err
示例#7
0
    def delete(self, key, session_id=None):
        client = FriClient(session_id=session_id or self.SESSION_ID)
        params = {'key': key, 'replica_count': 2}
        packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
        return ret_packet
    def test_node(self):
        try:
            server = None
            address = '127.0.0.1:1987'
            server = Node('127.0.0.1', 1987, '/tmp', 'test_node',
                    ks_path=None, ks_passwd=None, node_type='BASE')

            server.start(None)
            time.sleep(.5)

            fri_client = FriClient()
            params = {}
            packet_obj = FabnetPacketRequest(method='UpdateNodeConfig', parameters=params, sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertNotEqual(resp.ret_code, 0)

            params = {'config': {'TEST_CONFIG': 'str', 'TEST_INT': 234}}
            packet_obj = FabnetPacketRequest(method='UpdateNodeConfig', parameters=params, sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertEqual(resp.ret_code, 0, resp.ret_message)

            packet_obj = FabnetPacketRequest(method='GetNodeConfig', sync=True)
            resp = fri_client.call_sync(address, packet_obj)
            self.assertEqual(resp.ret_code, 0, resp.ret_message)

            self.assertEqual(resp.ret_parameters['test_config'], 'str')
            self.assertEqual(resp.ret_parameters['test_int'], 234)

            time.sleep(.2)
        except Exception, err:
            print 'ERROR: %s'%err
            raise err
示例#9
0
def get_maximum_free_space(addresses, perc=0):
    max_perc = 0
    full_cnt = 0
    dht_info_lst = []
    client = FriClient()
    for address in addresses:
        packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
        for i in xrange(3):
            ret_packet = client.call_sync(address, packet_obj)
            if ret_packet.ret_code == 0:
                break
        else:
            raise Exception('ERROR! NodeStatistic failed on %s with message: %s'%(address, ret_packet.ret_message))

        dht_info = ret_packet.ret_parameters['DHTInfo']

        if max_perc < dht_info['free_size_percents']:
            max_perc = dht_info['free_size_percents']

        if dht_info['free_size_percents'] < perc:
            full_cnt += 1

        dht_info_lst.append((address, dht_info))

    return max_perc, full_cnt, dht_info_lst
示例#10
0
def print_ranges(addresses, out_streem):
    client = FriClient()
    out_streem.write('\nRANGES SIZES:\n')
    ranges = {}
    for address in addresses:
        packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
        ret_packet = client.call_sync(address, packet_obj)
        if ret_packet.ret_code:
            raise Exception('NodeStatistic failed on %s: %s'%(address, ret_packet.ret_message))

        start = ret_packet.ret_parameters['DHTInfo']['range_start']
        end = ret_packet.ret_parameters['DHTInfo']['range_end']
        range_size = ret_packet.ret_parameters['DHTInfo']['range_size']
        replicas_size = ret_packet.ret_parameters['DHTInfo']['replicas_size']
        ranges[address] = (start, end, range_size, replicas_size)

    h_ranges = HashRangesTable()
    for address, (start, end, _, _) in ranges.items():
        h_ranges.append(long(start,16), long(end,16), address)

    for h_range in h_ranges.iter_table():
        start = h_range.start
        end = h_range.end
        address = h_range.node_address
        len_r = end - start
        out_streem.write('On node %s: {%040x-%040x}[%040x] = %s KB (%s KB)\n'%(address, start, end, len_r,\
                 ranges[address][2]/1024, ranges[address][3]/1024))
    out_streem.flush()

    end = -1
    for h_range in h_ranges.iter_table():
        if end+1 != h_range.start:
            raise Exception('Distributed range table is not full!')
        end = h_range.end
    return ranges
示例#11
0
    def get_object_info(self, obj_path, client_ks, req_user_info=True):
        params = {'obj_path': obj_path, 'req_user_info': req_user_info}
        req = FabnetPacketRequest(method='GetObjectInfo', parameters=params)

        client = FriClient(client_ks)
        resp = client.call_sync('127.0.0.1:%s'%self.port, req)
        return resp
示例#12
0
    def delete(self, key, client_ks=None, replica_count=2):
        client = FriClient(client_ks)
        params = {'key': key, 'replica_count': replica_count}
        packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
        return ret_packet
示例#13
0
    def test01_dht_restore_after_network_fail(self):
        server = server1 = None
        try:
            home1 = '/tmp/node_1986_home'
            home2 = '/tmp/node_1987_home'
            if os.path.exists(home1):
                shutil.rmtree(home1)
            os.mkdir(home1)
            if os.path.exists(home2):
                shutil.rmtree(home2)
            os.mkdir(home2)

            server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99})
            server.start()
            time.sleep(1)
            server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986',  config={'MAX_USED_SIZE_PERCENTS':99})
            server1.start()
            time.sleep(.2)
            self.__wait_oper_status(server1, DS_NORMALWORK)

            node86_stat = server.get_stat()
            node87_stat = server1.get_stat()

            self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
            self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
            self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
            self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)

            packet_obj = FabnetPacketRequest(method='TopologyCognition', sender=None)
            fri_client = FriClient()
            fri_client.call('127.0.0.1:1987', packet_obj)
            time.sleep(2)

            os.system('sudo /sbin/iptables -A INPUT -p tcp --destination-port 1986 -j DROP')
            os.system('sudo /sbin/iptables -A OUTPUT -p tcp --destination-port 1986 -j DROP')

            time.sleep(10)
            node87_stat = server1.get_stat()
            self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), 0L)
            self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)

            os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
            os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')

            time.sleep(10)
            node86_stat = server.get_stat()
            node87_stat = server1.get_stat()

            self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
            self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
            self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
            self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
        finally:
            os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
            os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')
            if server:
                server.stop()
            if server1:
                server1.stop()
示例#14
0
    def get_stat(self):
        packet_obj = FabnetPacketRequest(method='NodeStatistic')

        client = FriClient(self.ks)
        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
        if ret_packet.ret_code != 0:
            raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message)
        return ret_packet.ret_parameters
示例#15
0
    def get_keys_info(self, key):
        client = FriClient(session_id=self.SESSION_ID)
        packet = FabnetPacketRequest(method='GetKeysInfo', \
                parameters={'key': key, 'replica_count': 2}, sync=True)

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
        keys_info = ret_packet.ret_parameters['keys_info']
        return keys_info
示例#16
0
            def call_method():
                cert = ks.get_node_cert()
                ckey = ks.get_node_cert_key()

                fri_client = FriClient(bool(cert), cert, ckey)
                for i in xrange(1000):
                    resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='HelloFabregas'))
                    self.assertEqual(resp.ret_code, 0, resp.ret_message)
示例#17
0
    def test_node(self):
        try:
            server = None
            address = '127.0.0.1:1987'

            node = Node('127.0.0.1', 1987, '/tmp', 'node_stat_test',
                        ks_path=None, ks_passwd=None, node_type='BASE')
            node.start(None)
            server = node
            time.sleep(1)

            packet = {  'method': 'NodeStatistic',
                        'sender': '',
                        'parameters': {'reset_op_stat': True},
                        'sync': True}
            packet_obj = FabnetPacketRequest(**packet)

            client = FriClient()
            ret_packet = client.call_sync('127.0.0.1:1987', packet_obj)
            time.sleep(1.5)
            packet['parameters']['base_info'] = True
            packet_obj = FabnetPacketRequest(**packet)
            ret_packet = client.call_sync('127.0.0.1:1987', packet_obj)

            self.assertEqual(isinstance(ret_packet, FabnetPacketResponse), True)
            self.assertEqual(ret_packet.ret_code, 0, ret_packet.ret_message)
            print json.dumps(ret_packet.ret_parameters)
            self.assertEqual(ret_packet.ret_parameters['BaseInfo']['node_name'], 'node_stat_test')
            self.assertEqual(ret_packet.ret_parameters['BaseInfo']['home_dir'], '/tmp')
            self.assertEqual(ret_packet.ret_parameters['BaseInfo']['node_types'], ['Base'])
            self.assertEqual(int(ret_packet.ret_parameters['NeighboursInfo']['uppers_balance']), -1)
            self.assertEqual(int(ret_packet.ret_parameters['NeighboursInfo']['superiors_balance']), -1)
            self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_5']) >= 0)
            self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_10']) >= 0)
            self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_15']) >= 0)
            self.assertTrue(len(ret_packet.ret_parameters['SystemInfo']['fabnet_version'])>0)
            self.assertTrue(len(ret_packet.ret_parameters['SystemInfo']['uptime']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['FriAgentWMStat']['workers']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['FriAgentWMStat']['busy']) == 0)
            self.assertTrue(float(ret_packet.ret_parameters['OperatorWorkerWMStat']['workers']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['OperatorWorkerWMStat']['busy']) >= 1)
            self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorWMStat']['workers']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorWMStat']['busy']) == 0)
            self.assertTrue(float(ret_packet.ret_parameters['FriServerProcStat']['threads']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['FriServerProcStat']['memory']) > 1000)
            self.assertTrue(float(ret_packet.ret_parameters['OperatorProcStat']['threads']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['OperatorProcStat']['memory']) > 1000)
            self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorProcStat']['threads']) > 0)
            self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorProcStat']['memory']) > 1000)

            self.assertTrue(ret_packet.ret_parameters['OperationsProcTime']['NodeStatistic'] > 0)
            self.assertEqual(ret_packet.ret_parameters['OperationsProcTime']['TopologyCognition'], 0)

            time.sleep(.2)
        finally:
            if server:
                server.stop()
        def call_methods():
            fri_client = FriClient()
            resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='HelloFabregas'))
            self.assertEqual(resp.ret_code, 0, resp.ret_message)
            self.assertEqual(resp.ret_message, 'Hello, dear friend!')

            resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='SomeMethod'))
            self.assertNotEqual(resp.ret_code, 0, resp.ret_message)
            self.assertEqual(resp.ret_message, 'Unknown operation "SomeMethod"')
            time.sleep(1)
示例#19
0
 def _wait_node(self, node):
     client = FriClient()
     while True:
         packet_obj = FabnetPacketRequest(method="NodeStatistic", sync=True)
         ret_packet = client.call_sync(node, packet_obj)
         if ret_packet.ret_code:
             print "Node does not init FRI server yet. Waiting it..."
             time.sleep(0.5)
             continue
         break
示例#20
0
    def get_keys_info(self, key, replica_count=2):
        client = FriClient(self.ks)
        packet = FabnetPacketRequest(method='GetKeysInfo', \
                parameters={'key': key, 'replica_count': replica_count}, sync=True)

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
        if ret_packet.ret_code != 0:
            raise Exception('GetKeysInfo error: %s'%ret_packet.ret_message)
        keys_info = ret_packet.ret_parameters['keys_info']
        return keys_info
示例#21
0
    def stop(self):
        try:
            address = '%s:%s'%(self.hostname, self.port)
            params = {'event_type': ET_INFO, 'event_topic': 'NodeDown', \
                        'event_message': 'Goodbye, fabnet :(', 'event_provider': address}
            packet = FabnetPacketRequest(method='NotifyOperation', parameters=params, sender=None, multicast=True)

            fri_client = FriClient(self.keystore)
            fri_client.call(address, packet)
        except Exception, err:
            logger.warning('Cant send notification to network. Details: %s'%err)
示例#22
0
def put_data_to_network(addresses, stat_buffer, perc=10):
    client = FriClient()
    rs = 2**160
    print '-'*100
    for address in addresses:
        packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
        ret_packet = client.call_sync(address, packet_obj)
        if ret_packet.ret_code:
            raise Exception('ERROR! NodeStatistic failed on %s with message: %s'%(address, ret_packet.ret_message))

        dht_info = ret_packet.ret_parameters['DHTInfo']
        start = dht_info['range_start']
        end = dht_info['range_end']
        len_r = long(end, 16) - long(start, 16)
        stat_buffer.write('%s, %s, %s, %s\n'%(address, len_r, len_r*100./rs, dht_info['free_size_percents']))
        print 'On node %s: {%s-%s}[%x]'%(address, start, end, len_r)
        print '           %s KB / %s KB (%s perc. free)'% (dht_info['range_size']/1024, \
                        dht_info['replicas_size']/1024, dht_info['free_size_percents'])
    stat_buffer.write('\n')
    print '-'*100


    print 'PUT DATA...'
    while True:
        for i in xrange(50):
            data = Random.new().read(1024*1024)
            checksum =  hashlib.sha1(data).hexdigest()

            params = {'checksum': checksum, 'wait_writes_count': 3}
            packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=RamBasedBinaryData(data), sync=True)

            ret_packet = client.call_sync(random.choice(addresses), packet_obj)
            if ret_packet.ret_code != 0:
                print 'ERROR! Cant put data block to network! Details: %s'%ret_packet.ret_message


        max_perc, full_cnt, dht_info_lst = get_maximum_free_space(addresses, perc)
        if full_cnt:
            print '='*100
            for address, dht_info in dht_info_lst:
                start = dht_info['range_start']
                end = dht_info['range_end']
                len_r = long(end, 16) - long(start, 16)
                stat_buffer.write('%s, %s, %s, %s\n'%(address, len_r, len_r*100./rs, dht_info['free_size_percents']))

                print 'On node %s: {%s-%s}[%x]'%(address, start, end, len_r)
                print '           %s KB / %s KB (%s perc. free)'% (dht_info['range_size']/1024, \
                                dht_info['replicas_size']/1024, dht_info['free_size_percents'])
            stat_buffer.write('\n')
            print '='*100

            if max_perc > 10:
                return False
            return True
示例#23
0
 def get_data_block(self, key, is_replica=False, user_id=0):
     params = {'key': key, 'is_replica':is_replica}
     req = FabnetPacketRequest(method='GetDataBlock',\
                         sync=True, parameters=params)
     client = FriClient(session_id=user_id or self.SESSION_ID)
     resp = client.call_sync('127.0.0.1:%s'%self.port, req)
     if resp.ret_code == constants.RC_NO_DATA:
         return None
     if resp.ret_code != 0:
         raise Exception('GetDataBlock operation failed on %s. Details: %s'%('127.0.0.1:%s'%self.port, resp.ret_message))
     return resp.binary_data
示例#24
0
    def put_data_block(self, data, key, dbct=FSMappedDHTRange.DBCT_MASTER, user_id_hash='', \
                careful_save=False, init_block=False, stored_unixtime=None):

        params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash, 'init_block': init_block,\
                'carefully_save': careful_save, 'stored_unixtime': stored_unixtime}

        req = FabnetPacketRequest(method='PutDataBlock',\
                            binary_data=data, parameters=params)

        client = FriClient(self.ks)
        resp = client.call_sync('127.0.0.1:%s'%self.port, req)
        return resp
示例#25
0
def wait_node(node):
    client = FriClient()
    while True:
        packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
        ret_packet = client.call_sync(node, packet_obj)
        if ret_packet.ret_code:
            print 'Node %s does not init FRI server yet. Waiting it...'%node
            time.sleep(.5)
            continue
        if ret_packet.ret_parameters['DHTInfo']['status'] != 'normwork':
            print 'Node %s does not init as DHT member yet. Waiting...'%node
            time.sleep(.5)
            continue
        break
示例#26
0
    def put_data_block(self, data, key, is_replica=False, npkey=False, user_id=0):
        checksum = hashlib.sha1(data).hexdigest()
        params = {'key': key, 'checksum': checksum, 'is_replica':is_replica, \
                'carefully_save': True, 'replica_count': 2, 'user_id': user_id}

        if not npkey:
            params['primary_key'] = key

        req = FabnetPacketRequest(method='PutDataBlock',\
                            binary_data=data, sync=True,
                            parameters=params)
        client = FriClient()
        resp = client.call_sync('127.0.0.1:%s'%self.port, req)
        return resp
示例#27
0
    def put(self, data, wait_writes=3):
        client = FriClient(session_id=self.SESSION_ID)
        checksum = hashlib.sha1(data).hexdigest()

        params = {'checksum': checksum, 'wait_writes_count': wait_writes, 'replica_count':2}
        data = RamBasedBinaryData(data, 20)
        packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True)
        #print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
        #print '========SENDED DATA BLOCK'
        if ret_packet.ret_code != 0:
            raise Exception('put data failed: %s'%ret_packet.ret_message)
        return ret_packet.ret_parameters['key']
示例#28
0
    def put_object_part(self, obj_path, data, seek=0, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2):
        if not client_ks:
            client_ks = self.ks
        client = FriClient(client_ks)

        params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \
                'init_block': init_block, 'key': key, 'obj_path': obj_path, 'seek': seek}
        data = RamBasedBinaryData(data, 20)
        packet_obj = FabnetPacketRequest(method='PutObjectPart', parameters=params, binary_data=data, sync=True)
        #print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())

        ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
        #print '========SENDED DATA BLOCK'
        return ret_packet
示例#29
0
def get_data_blocks(addresses, keys):
    client = FriClient()

    for key in keys:
        params = {'key': key, 'replica_count':2}
        packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True)
        nodeaddr = random.choice(addresses)

        ret_packet = client.call_sync(nodeaddr, packet_obj)
        if ret_packet.ret_code != 0:
            raise Exception('ClientGetData failed on %s: %s'%(nodeaddr, ret_packet.ret_message))

        data = ret_packet.binary_data.data()
        if hashlib.sha1(data).hexdigest() != ret_packet.ret_parameters['checksum']:
            raise Exception('Data block checksum failed!')
示例#30
0
    def test_topology_cognition(self):
        servers = []
        addresses = []
        #os.system('rm /tmp/fabnet_topology.db')
        try:
            for i in range(1900, 1900+NODES_COUNT):
                address = '127.0.0.1:%s'%i
                if not addresses:
                    neighbour = None
                else:
                    neighbour = random.choice(addresses)

                server = TestServerThread(i, neighbour)
                server.start()
                servers.append(server)
                addresses.append(address)
                time.sleep(1.5)

            time.sleep(1)

            packet = {  'method': 'TopologyCognition',
                        'sender': None,
                        'parameters': {}}
            packet_obj = FabnetPacketRequest(**packet)
            fri_client = FriClient()
            addr = random.choice(addresses)
            fri_client.call(addr, packet_obj)

            time.sleep(1)
            operator = OperatorClient('node%s'%addr.split(':')[-1])
            home_dir = operator.get_home_dir()

            db = SafeJsonFile(os.path.join(home_dir, 'fabnet_topology.db'))
            nodes = db.read()
            print 'NODES LIST: %s'%str(nodes)

            for i in range(1900, 1900+NODES_COUNT):
                address = '127.0.0.1:%s'%i
                self.assertTrue(nodes.has_key(address))
                self.assertTrue(len(nodes[address]['uppers']) >= 2)
                self.assertTrue(len(nodes[address]['superiors']) >= 2)
                self.assertEqual(nodes[address]['node_name'], 'node%s'%i)
        finally:
            for server in servers:
                if server:
                    server.stop()
                    server.join()
                    time.sleep(1)
    def test_manage_neighbours(self):
        operator = Operator('127.0.0.1:1986')

        manage_neighbours = ManageNeighbour(operator, FriClient(),
                                            '127.0.0.1:1986', '/tmp', None)

        manage_neighbours._init_operation = _init_operation_mock

        threads = []
        for i in xrange(10):
            tst = TestServerThread(manage_neighbours)
            threads.append(tst)

        for thread in threads:
            thread.start()

        for thread in threads:
            thread.join()

        operator.stop()