def _wait_node(self, node):
idx = None
for i, addr in enumerate(ADDRESSES):
if addr == node:
idx = i
if idx is None:
raise Exception('address %s does not found!'%node)
proc = PROCESSES[idx]
key_storage = init_keystore(KS_PATH, KS_PASSWD)
client = FriClient(key_storage)
while True:
if not os.path.exists('/proc/%s'%proc.pid):
raise Exception('Node process for %s does not found!')
for line in open("/proc/%d/status" % proc.pid).readlines():
if line.startswith("State:"):
status = line.split(":",1)[1].strip().split(' ')[0]
if status == 'Z':
raise Exception('Node died at %s'%node)
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(node, packet_obj)
if ret_packet.ret_code:
print ret_packet.ret_message
print 'Node does not init FRI server yet. Waiting it...'
time.sleep(.5)
continue
break
def get_data_block(self, key, dbct=FSMappedDHTRange.DBCT_MASTER, client_ks=None, user_id_hash=''):
params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash}
req = FabnetPacketRequest(method='GetDataBlock', parameters=params)
client = FriClient(client_ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def test_node_upgrade(self):
server_proc = None
try:
os.system('rm -rf /tmp/fabnet_node_code')
os.system('rm -rf /tmp/upgrade_node.log')
os.system('git clone %s /tmp/fabnet_node_code/'%ORIG_REPO)
os.chdir('/tmp/fabnet_node_code/')
#os.system('git reset --hard f797284a29fcbaa08d8ff68d07e5e6e352a95fe3')
os.system('find . -name *.pyc -delete')
address = '127.0.0.1:1987'
args = ['/usr/bin/python', '/tmp/fabnet_node_code/fabnet/bin/fabnet-node', address, 'init-fabnet', 'test_upgr_node', '/tmp', 'DHT']
args.append('--nodaemon')
server_proc = subprocess.Popen(args)
time.sleep(2)
os.system("echo '{\"DHT\": 100500}' > UPGRADE_VERSION")
packet_obj = FabnetPacketRequest(method='UpgradeNode', parameters={'origin_repo_url': 'bad_url'}, sync=True)
client = FriClient()
ret = client.call_sync('127.0.0.1:1987', packet_obj)
self.assertEqual(ret.ret_code, 0)
print 'upgrading /tmp/fabnet_node_code from %s'%ORIG_REPO
packet_obj = FabnetPacketRequest(method='UpgradeNode', parameters={'origin_repo_url': ORIG_REPO}, sync=True)
client = FriClient()
ret = client.call_sync('127.0.0.1:1987', packet_obj)
self.assertEqual(ret.ret_code, 0)
self.assertTrue(os.path.exists('/tmp/upgrade_node.log'))
self.assertTrue('successfully' in open('/tmp/upgrade_node.log').read())
finally:
os.system('rm -rf /tmp/fabnet_node_code')
if server_proc:
server_proc.send_signal(signal.SIGINT)
server_proc.wait()
def get_data_block(addresses, key):
client = FriClient()
packet_obj = FabnetPacketRequest(method='GetKeysInfo', \
parameters={'key': key, 'replica_count': 2})
nodeaddr = random.choice(addresses)
ret_packet = client.call_sync(nodeaddr, packet_obj)
if ret_packet.ret_code != 0:
raise Exception('GetKeysInfo failed on %s: %s'%(nodeaddr, ret_packet.ret_message))
for key, dbct, nodeaddr in ret_packet.ret_parameters['keys_info']:
params = {'key': key, 'dbct': dbct}
packet_obj = FabnetPacketRequest(method='GetDataBlock', parameters=params)
ret_packet = client.call_sync(nodeaddr, packet_obj)
if ret_packet.ret_code != 0:
continue
data = ret_packet.binary_data.data()
if hashlib.sha1(data).hexdigest() != ret_packet.ret_parameters['checksum']:
ret_packet.ret_message = 'Data block checksum failed!'
continue
return data
raise Exception('GetDataBlock failed on %s: %s'%(nodeaddr, ret_packet.ret_message))
def activate_on_dht(self, user_name, dht_endpoint):
conn = self.__ca_call('/get_certificate_info', {'cn': user_name})
try:
resp = conn.getresponse()
if resp.status != 200:
raise Exception('CA service error! Get cert info: [%s %s] %s'%\
(resp.status, resp.reason, resp.read()))
data = resp.read()
finally:
conn.close()
data = json.loads(data)
###data['cert_term']
#cert_add_info
#cert_term
#status
fri_client = FriClient(self.ks)
add_info = json.loads(data['cert_add_info'])
params = {'user_id_hash': hashlib.sha1(user_name).hexdigest(), 'storage_size': add_info['storage_capacity']}
packet_obj = FabnetPacketRequest(method='UpdateUserProfile', parameters=params)
ret_packet = fri_client.call_sync(dht_endpoint, packet_obj)
if ret_packet.ret_code != 0:
raise Exception(ret_packet.ret_message)
def test_node(self):
try:
server = None
address = "127.0.0.1:1987"
server = Node("127.0.0.1", 1987, "/tmp", "test_node", ks_path=None, ks_passwd=None, node_type="BASE")
server.start(None)
time.sleep(0.5)
fri_client = FriClient()
params = {}
packet_obj = FabnetPacketRequest(method="UpdateNodeConfig", parameters=params, sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertNotEqual(resp.ret_code, 0)
params = {"config": {"TEST": {"TEST_CONFIG": "str", "TEST_INT": 234}}}
packet_obj = FabnetPacketRequest(method="UpdateNodeConfig", parameters=params, sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertEqual(resp.ret_code, 0, resp.ret_message)
packet_obj = FabnetPacketRequest(method="GetNodeConfig", sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertTrue("TEST" in resp.ret_parameters, resp.ret_parameters)
self.assertEqual(resp.ret_parameters["TEST"]["TEST_CONFIG"], "str")
self.assertEqual(resp.ret_parameters["TEST"]["TEST_INT"], 234)
time.sleep(0.2)
except Exception, err:
print "ERROR: %s" % err
raise err
def delete(self, key, session_id=None):
client = FriClient(session_id=session_id or self.SESSION_ID)
params = {'key': key, 'replica_count': 2}
packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def test_node(self):
try:
server = None
address = '127.0.0.1:1987'
server = Node('127.0.0.1', 1987, '/tmp', 'test_node',
ks_path=None, ks_passwd=None, node_type='BASE')
server.start(None)
time.sleep(.5)
fri_client = FriClient()
params = {}
packet_obj = FabnetPacketRequest(method='UpdateNodeConfig', parameters=params, sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertNotEqual(resp.ret_code, 0)
params = {'config': {'TEST_CONFIG': 'str', 'TEST_INT': 234}}
packet_obj = FabnetPacketRequest(method='UpdateNodeConfig', parameters=params, sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertEqual(resp.ret_code, 0, resp.ret_message)
packet_obj = FabnetPacketRequest(method='GetNodeConfig', sync=True)
resp = fri_client.call_sync(address, packet_obj)
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.ret_parameters['test_config'], 'str')
self.assertEqual(resp.ret_parameters['test_int'], 234)
time.sleep(.2)
except Exception, err:
print 'ERROR: %s'%err
raise err
def get_maximum_free_space(addresses, perc=0):
max_perc = 0
full_cnt = 0
dht_info_lst = []
client = FriClient()
for address in addresses:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
for i in xrange(3):
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code == 0:
break
else:
raise Exception('ERROR! NodeStatistic failed on %s with message: %s'%(address, ret_packet.ret_message))
dht_info = ret_packet.ret_parameters['DHTInfo']
if max_perc < dht_info['free_size_percents']:
max_perc = dht_info['free_size_percents']
if dht_info['free_size_percents'] < perc:
full_cnt += 1
dht_info_lst.append((address, dht_info))
return max_perc, full_cnt, dht_info_lst
def print_ranges(addresses, out_streem):
client = FriClient()
out_streem.write('\nRANGES SIZES:\n')
ranges = {}
for address in addresses:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
raise Exception('NodeStatistic failed on %s: %s'%(address, ret_packet.ret_message))
start = ret_packet.ret_parameters['DHTInfo']['range_start']
end = ret_packet.ret_parameters['DHTInfo']['range_end']
range_size = ret_packet.ret_parameters['DHTInfo']['range_size']
replicas_size = ret_packet.ret_parameters['DHTInfo']['replicas_size']
ranges[address] = (start, end, range_size, replicas_size)
h_ranges = HashRangesTable()
for address, (start, end, _, _) in ranges.items():
h_ranges.append(long(start,16), long(end,16), address)
for h_range in h_ranges.iter_table():
start = h_range.start
end = h_range.end
address = h_range.node_address
len_r = end - start
out_streem.write('On node %s: {%040x-%040x}[%040x] = %s KB (%s KB)\n'%(address, start, end, len_r,\
ranges[address][2]/1024, ranges[address][3]/1024))
out_streem.flush()
end = -1
for h_range in h_ranges.iter_table():
if end+1 != h_range.start:
raise Exception('Distributed range table is not full!')
end = h_range.end
return ranges
def get_object_info(self, obj_path, client_ks, req_user_info=True):
params = {'obj_path': obj_path, 'req_user_info': req_user_info}
req = FabnetPacketRequest(method='GetObjectInfo', parameters=params)
client = FriClient(client_ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def delete(self, key, client_ks=None, replica_count=2):
client = FriClient(client_ks)
params = {'key': key, 'replica_count': replica_count}
packet_obj = FabnetPacketRequest(method='ClientDeleteData', parameters=params, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
return ret_packet
def test01_dht_restore_after_network_fail(self):
server = server1 = None
try:
home1 = '/tmp/node_1986_home'
home2 = '/tmp/node_1987_home'
if os.path.exists(home1):
shutil.rmtree(home1)
os.mkdir(home1)
if os.path.exists(home2):
shutil.rmtree(home2)
os.mkdir(home2)
server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99})
server.start()
time.sleep(1)
server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986', config={'MAX_USED_SIZE_PERCENTS':99})
server1.start()
time.sleep(.2)
self.__wait_oper_status(server1, DS_NORMALWORK)
node86_stat = server.get_stat()
node87_stat = server1.get_stat()
self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
packet_obj = FabnetPacketRequest(method='TopologyCognition', sender=None)
fri_client = FriClient()
fri_client.call('127.0.0.1:1987', packet_obj)
time.sleep(2)
os.system('sudo /sbin/iptables -A INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -A OUTPUT -p tcp --destination-port 1986 -j DROP')
time.sleep(10)
node87_stat = server1.get_stat()
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')
time.sleep(10)
node86_stat = server.get_stat()
node87_stat = server1.get_stat()
self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
finally:
os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')
if server:
server.stop()
if server1:
server1.stop()
def get_stat(self):
packet_obj = FabnetPacketRequest(method='NodeStatistic')
client = FriClient(self.ks)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
if ret_packet.ret_code != 0:
raise Exception('cant get node statistic. details: %s'%ret_packet.ret_message)
return ret_packet.ret_parameters
def get_keys_info(self, key):
client = FriClient(session_id=self.SESSION_ID)
packet = FabnetPacketRequest(method='GetKeysInfo', \
parameters={'key': key, 'replica_count': 2}, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
keys_info = ret_packet.ret_parameters['keys_info']
return keys_info
def call_method():
cert = ks.get_node_cert()
ckey = ks.get_node_cert_key()
fri_client = FriClient(bool(cert), cert, ckey)
for i in xrange(1000):
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='HelloFabregas'))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
def test_node(self):
try:
server = None
address = '127.0.0.1:1987'
node = Node('127.0.0.1', 1987, '/tmp', 'node_stat_test',
ks_path=None, ks_passwd=None, node_type='BASE')
node.start(None)
server = node
time.sleep(1)
packet = { 'method': 'NodeStatistic',
'sender': '',
'parameters': {'reset_op_stat': True},
'sync': True}
packet_obj = FabnetPacketRequest(**packet)
client = FriClient()
ret_packet = client.call_sync('127.0.0.1:1987', packet_obj)
time.sleep(1.5)
packet['parameters']['base_info'] = True
packet_obj = FabnetPacketRequest(**packet)
ret_packet = client.call_sync('127.0.0.1:1987', packet_obj)
self.assertEqual(isinstance(ret_packet, FabnetPacketResponse), True)
self.assertEqual(ret_packet.ret_code, 0, ret_packet.ret_message)
print json.dumps(ret_packet.ret_parameters)
self.assertEqual(ret_packet.ret_parameters['BaseInfo']['node_name'], 'node_stat_test')
self.assertEqual(ret_packet.ret_parameters['BaseInfo']['home_dir'], '/tmp')
self.assertEqual(ret_packet.ret_parameters['BaseInfo']['node_types'], ['Base'])
self.assertEqual(int(ret_packet.ret_parameters['NeighboursInfo']['uppers_balance']), -1)
self.assertEqual(int(ret_packet.ret_parameters['NeighboursInfo']['superiors_balance']), -1)
self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_5']) >= 0)
self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_10']) >= 0)
self.assertTrue(float(ret_packet.ret_parameters['SystemInfo']['loadavg_15']) >= 0)
self.assertTrue(len(ret_packet.ret_parameters['SystemInfo']['fabnet_version'])>0)
self.assertTrue(len(ret_packet.ret_parameters['SystemInfo']['uptime']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['FriAgentWMStat']['workers']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['FriAgentWMStat']['busy']) == 0)
self.assertTrue(float(ret_packet.ret_parameters['OperatorWorkerWMStat']['workers']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['OperatorWorkerWMStat']['busy']) >= 1)
self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorWMStat']['workers']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorWMStat']['busy']) == 0)
self.assertTrue(float(ret_packet.ret_parameters['FriServerProcStat']['threads']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['FriServerProcStat']['memory']) > 1000)
self.assertTrue(float(ret_packet.ret_parameters['OperatorProcStat']['threads']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['OperatorProcStat']['memory']) > 1000)
self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorProcStat']['threads']) > 0)
self.assertTrue(float(ret_packet.ret_parameters['OperationsProcessorProcStat']['memory']) > 1000)
self.assertTrue(ret_packet.ret_parameters['OperationsProcTime']['NodeStatistic'] > 0)
self.assertEqual(ret_packet.ret_parameters['OperationsProcTime']['TopologyCognition'], 0)
time.sleep(.2)
finally:
if server:
server.stop()
def call_methods():
fri_client = FriClient()
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='HelloFabregas'))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.ret_message, 'Hello, dear friend!')
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='SomeMethod'))
self.assertNotEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.ret_message, 'Unknown operation "SomeMethod"')
time.sleep(1)
def _wait_node(self, node):
client = FriClient()
while True:
packet_obj = FabnetPacketRequest(method="NodeStatistic", sync=True)
ret_packet = client.call_sync(node, packet_obj)
if ret_packet.ret_code:
print "Node does not init FRI server yet. Waiting it..."
time.sleep(0.5)
continue
break
def get_keys_info(self, key, replica_count=2):
client = FriClient(self.ks)
packet = FabnetPacketRequest(method='GetKeysInfo', \
parameters={'key': key, 'replica_count': replica_count}, sync=True)
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet)
if ret_packet.ret_code != 0:
raise Exception('GetKeysInfo error: %s'%ret_packet.ret_message)
keys_info = ret_packet.ret_parameters['keys_info']
return keys_info
def stop(self):
try:
address = '%s:%s'%(self.hostname, self.port)
params = {'event_type': ET_INFO, 'event_topic': 'NodeDown', \
'event_message': 'Goodbye, fabnet :(', 'event_provider': address}
packet = FabnetPacketRequest(method='NotifyOperation', parameters=params, sender=None, multicast=True)
fri_client = FriClient(self.keystore)
fri_client.call(address, packet)
except Exception, err:
logger.warning('Cant send notification to network. Details: %s'%err)
def put_data_to_network(addresses, stat_buffer, perc=10):
client = FriClient()
rs = 2**160
print '-'*100
for address in addresses:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
if ret_packet.ret_code:
raise Exception('ERROR! NodeStatistic failed on %s with message: %s'%(address, ret_packet.ret_message))
dht_info = ret_packet.ret_parameters['DHTInfo']
start = dht_info['range_start']
end = dht_info['range_end']
len_r = long(end, 16) - long(start, 16)
stat_buffer.write('%s, %s, %s, %s\n'%(address, len_r, len_r*100./rs, dht_info['free_size_percents']))
print 'On node %s: {%s-%s}[%x]'%(address, start, end, len_r)
print ' %s KB / %s KB (%s perc. free)'% (dht_info['range_size']/1024, \
dht_info['replicas_size']/1024, dht_info['free_size_percents'])
stat_buffer.write('\n')
print '-'*100
print 'PUT DATA...'
while True:
for i in xrange(50):
data = Random.new().read(1024*1024)
checksum = hashlib.sha1(data).hexdigest()
params = {'checksum': checksum, 'wait_writes_count': 3}
packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=RamBasedBinaryData(data), sync=True)
ret_packet = client.call_sync(random.choice(addresses), packet_obj)
if ret_packet.ret_code != 0:
print 'ERROR! Cant put data block to network! Details: %s'%ret_packet.ret_message
max_perc, full_cnt, dht_info_lst = get_maximum_free_space(addresses, perc)
if full_cnt:
print '='*100
for address, dht_info in dht_info_lst:
start = dht_info['range_start']
end = dht_info['range_end']
len_r = long(end, 16) - long(start, 16)
stat_buffer.write('%s, %s, %s, %s\n'%(address, len_r, len_r*100./rs, dht_info['free_size_percents']))
print 'On node %s: {%s-%s}[%x]'%(address, start, end, len_r)
print ' %s KB / %s KB (%s perc. free)'% (dht_info['range_size']/1024, \
dht_info['replicas_size']/1024, dht_info['free_size_percents'])
stat_buffer.write('\n')
print '='*100
if max_perc > 10:
return False
return True
def get_data_block(self, key, is_replica=False, user_id=0):
params = {'key': key, 'is_replica':is_replica}
req = FabnetPacketRequest(method='GetDataBlock',\
sync=True, parameters=params)
client = FriClient(session_id=user_id or self.SESSION_ID)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
if resp.ret_code == constants.RC_NO_DATA:
return None
if resp.ret_code != 0:
raise Exception('GetDataBlock operation failed on %s. Details: %s'%('127.0.0.1:%s'%self.port, resp.ret_message))
return resp.binary_data
def put_data_block(self, data, key, dbct=FSMappedDHTRange.DBCT_MASTER, user_id_hash='', \
careful_save=False, init_block=False, stored_unixtime=None):
params = {'key': key, 'dbct': dbct, 'user_id_hash': user_id_hash, 'init_block': init_block,\
'carefully_save': careful_save, 'stored_unixtime': stored_unixtime}
req = FabnetPacketRequest(method='PutDataBlock',\
binary_data=data, parameters=params)
client = FriClient(self.ks)
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def wait_node(node):
client = FriClient()
while True:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(node, packet_obj)
if ret_packet.ret_code:
print 'Node %s does not init FRI server yet. Waiting it...'%node
time.sleep(.5)
continue
if ret_packet.ret_parameters['DHTInfo']['status'] != 'normwork':
print 'Node %s does not init as DHT member yet. Waiting...'%node
time.sleep(.5)
continue
break
def put_data_block(self, data, key, is_replica=False, npkey=False, user_id=0):
checksum = hashlib.sha1(data).hexdigest()
params = {'key': key, 'checksum': checksum, 'is_replica':is_replica, \
'carefully_save': True, 'replica_count': 2, 'user_id': user_id}
if not npkey:
params['primary_key'] = key
req = FabnetPacketRequest(method='PutDataBlock',\
binary_data=data, sync=True,
parameters=params)
client = FriClient()
resp = client.call_sync('127.0.0.1:%s'%self.port, req)
return resp
def put(self, data, wait_writes=3):
client = FriClient(session_id=self.SESSION_ID)
checksum = hashlib.sha1(data).hexdigest()
params = {'checksum': checksum, 'wait_writes_count': wait_writes, 'replica_count':2}
data = RamBasedBinaryData(data, 20)
packet_obj = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True)
#print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
#print '========SENDED DATA BLOCK'
if ret_packet.ret_code != 0:
raise Exception('put data failed: %s'%ret_packet.ret_message)
return ret_packet.ret_parameters['key']
def put_object_part(self, obj_path, data, seek=0, wait_writes=3, init_block=True, client_ks=None, key=None, replica_count=2):
if not client_ks:
client_ks = self.ks
client = FriClient(client_ks)
params = {'wait_writes_count': wait_writes, 'replica_count': replica_count, \
'init_block': init_block, 'key': key, 'obj_path': obj_path, 'seek': seek}
data = RamBasedBinaryData(data, 20)
packet_obj = FabnetPacketRequest(method='PutObjectPart', parameters=params, binary_data=data, sync=True)
#print '========SENDING DATA BLOCK %s (%s chunks)'%(packet_obj, data.chunks_count())
ret_packet = client.call_sync('127.0.0.1:%s'%self.port, packet_obj)
#print '========SENDED DATA BLOCK'
return ret_packet
def get_data_blocks(addresses, keys):
client = FriClient()
for key in keys:
params = {'key': key, 'replica_count':2}
packet_obj = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True)
nodeaddr = random.choice(addresses)
ret_packet = client.call_sync(nodeaddr, packet_obj)
if ret_packet.ret_code != 0:
raise Exception('ClientGetData failed on %s: %s'%(nodeaddr, ret_packet.ret_message))
data = ret_packet.binary_data.data()
if hashlib.sha1(data).hexdigest() != ret_packet.ret_parameters['checksum']:
raise Exception('Data block checksum failed!')
def test_topology_cognition(self):
servers = []
addresses = []
#os.system('rm /tmp/fabnet_topology.db')
try:
for i in range(1900, 1900+NODES_COUNT):
address = '127.0.0.1:%s'%i
if not addresses:
neighbour = None
else:
neighbour = random.choice(addresses)
server = TestServerThread(i, neighbour)
server.start()
servers.append(server)
addresses.append(address)
time.sleep(1.5)
time.sleep(1)
packet = { 'method': 'TopologyCognition',
'sender': None,
'parameters': {}}
packet_obj = FabnetPacketRequest(**packet)
fri_client = FriClient()
addr = random.choice(addresses)
fri_client.call(addr, packet_obj)
time.sleep(1)
operator = OperatorClient('node%s'%addr.split(':')[-1])
home_dir = operator.get_home_dir()
db = SafeJsonFile(os.path.join(home_dir, 'fabnet_topology.db'))
nodes = db.read()
print 'NODES LIST: %s'%str(nodes)
for i in range(1900, 1900+NODES_COUNT):
address = '127.0.0.1:%s'%i
self.assertTrue(nodes.has_key(address))
self.assertTrue(len(nodes[address]['uppers']) >= 2)
self.assertTrue(len(nodes[address]['superiors']) >= 2)
self.assertEqual(nodes[address]['node_name'], 'node%s'%i)
finally:
for server in servers:
if server:
server.stop()
server.join()
time.sleep(1)
def test_manage_neighbours(self):
operator = Operator('127.0.0.1:1986')
manage_neighbours = ManageNeighbour(operator, FriClient(),
'127.0.0.1:1986', '/tmp', None)
manage_neighbours._init_operation = _init_operation_mock
threads = []
for i in xrange(10):
tst = TestServerThread(manage_neighbours)
threads.append(tst)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
operator.stop()