def test01_dht_restore_after_network_fail(self):
server = server1 = None
try:
home1 = '/tmp/node_1986_home'
home2 = '/tmp/node_1987_home'
if os.path.exists(home1):
shutil.rmtree(home1)
os.mkdir(home1)
if os.path.exists(home2):
shutil.rmtree(home2)
os.mkdir(home2)
server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99})
server.start()
time.sleep(1)
server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986', config={'MAX_USED_SIZE_PERCENTS':99})
server1.start()
time.sleep(.2)
self.__wait_oper_status(server1, DS_NORMALWORK)
node86_stat = server.get_stat()
node87_stat = server1.get_stat()
self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
packet_obj = FabnetPacketRequest(method='TopologyCognition', sender=None)
fri_client = FriClient()
fri_client.call('127.0.0.1:1987', packet_obj)
time.sleep(2)
os.system('sudo /sbin/iptables -A INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -A OUTPUT -p tcp --destination-port 1986 -j DROP')
time.sleep(10)
node87_stat = server1.get_stat()
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')
time.sleep(10)
node86_stat = server.get_stat()
node87_stat = server1.get_stat()
self.assertEqual(long(node86_stat['DHTInfo']['range_start'], 16), 0L)
self.assertEqual(long(node86_stat['DHTInfo']['range_end'], 16), MAX_HASH/2)
self.assertEqual(long(node87_stat['DHTInfo']['range_start'], 16), MAX_HASH/2+1)
self.assertEqual(long(node87_stat['DHTInfo']['range_end'], 16), MAX_HASH)
finally:
os.system('sudo /sbin/iptables -D INPUT -p tcp --destination-port 1986 -j DROP')
os.system('sudo /sbin/iptables -D OUTPUT -p tcp --destination-port 1986 -j DROP')
if server:
server.stop()
if server1:
server1.stop()
def stop(self):
try:
address = '%s:%s'%(self.hostname, self.port)
params = {'event_type': ET_INFO, 'event_topic': 'NodeDown', \
'event_message': 'Goodbye, fabnet :(', 'event_provider': address}
packet = FabnetPacketRequest(method='NotifyOperation', parameters=params, sender=None, multicast=True)
fri_client = FriClient(self.keystore)
fri_client.call(address, packet)
except Exception, err:
logger.warning('Cant send notification to network. Details: %s'%err)
def test_topology_cognition(self):
servers = []
addresses = []
#os.system('rm /tmp/fabnet_topology.db')
try:
for i in range(1900, 1900+NODES_COUNT):
address = '127.0.0.1:%s'%i
if not addresses:
neighbour = None
else:
neighbour = random.choice(addresses)
server = TestServerThread(i, neighbour)
server.start()
servers.append(server)
addresses.append(address)
time.sleep(1.5)
time.sleep(1)
packet = { 'method': 'TopologyCognition',
'sender': None,
'parameters': {}}
packet_obj = FabnetPacketRequest(**packet)
fri_client = FriClient()
addr = random.choice(addresses)
fri_client.call(addr, packet_obj)
time.sleep(1)
operator = OperatorClient('node%s'%addr.split(':')[-1])
home_dir = operator.get_home_dir()
db = SafeJsonFile(os.path.join(home_dir, 'fabnet_topology.db'))
nodes = db.read()
print 'NODES LIST: %s'%str(nodes)
for i in range(1900, 1900+NODES_COUNT):
address = '127.0.0.1:%s'%i
self.assertTrue(nodes.has_key(address))
self.assertTrue(len(nodes[address]['uppers']) >= 2)
self.assertTrue(len(nodes[address]['superiors']) >= 2)
self.assertEqual(nodes[address]['node_name'], 'node%s'%i)
finally:
for server in servers:
if server:
server.stop()
server.join()
time.sleep(1)
def call_repair_data(address, out_streem, expect_res, invalid_node=None):
dbconn = DBConnection("dbname=%s user=postgres"%MONITOR_DB)
dbconn.execute("DELETE FROM notification")
client = FriClient()
params = {}
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
free_size = dht_info['free_size_percents']
if invalid_node:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(invalid_node, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
start = dht_info['range_start']
end = dht_info['range_end']
params = {'check_range_start': start, 'check_range_end': end}
t0 = datetime.now()
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params)
rcode, rmsg = client.call(address, packet_obj)
if rcode != 0:
raise Exception('RepairDataBlocks does not started. Details: %s'%rmsg)
cnt = 0
try_cnt = 0
while cnt != expect_res:
if try_cnt == 100:
print 'Notifications count: %s, but expected: %s'%(cnt, expect_res)
try_cnt = 0
try_cnt += 1
try:
cnt = dbconn.select_one("SELECT count(*) FROM notification WHERE notify_topic='RepairDataBlocks'")
except Exception, err:
print 'DB ERROR: %s'%err
time.sleep(.2)
def __defferent_calls(self, key_storage=None):
fri_server, operator_proc = self.__start_node('testnode00', key_storage)
try:
if key_storage:
cert = key_storage.get_node_cert()
ckey = key_storage.get_node_cert_key()
else:
cert = ckey = None
fri_client = FriClient(bool(cert), cert, ckey)
#sync call without binary
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo', \
parameters={'message': 'hello, fabregas!'}))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.ret_parameters['message'], 'hello, fabregas!')
#sync call without binary
resp = fri_client.call('127.0.0.1:6666', FabnetPacketRequest(method='echo', \
parameters={'message': 'hello, fabregas!'}))
time.sleep(1)
EchoOperation.check_resp('hello, fabregas!')
#sync call with binary unchanked
data = '0123456789'*900000
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo', \
parameters={'message': 'hello, fabregas!'}, \
binary_data=RamBasedBinaryData(data)))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.binary_data.data(), data)
self.assertEqual(resp.ret_parameters['message'], 'hello, fabregas!')
#sync call with binary chanked
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo', \
parameters={'message': 'hello, fabregas!'}, \
binary_data=RamBasedBinaryData(data, 900000)))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.binary_data.data(), data)
self.assertEqual(resp.ret_parameters['message'], 'hello, fabregas!')
#async call with binary
resp = fri_client.call('127.0.0.1:6666', FabnetPacketRequest(method='echo', \
parameters={'message': 'hello, fabregas!'}, \
binary_data=RamBasedBinaryData(data, 900000)))
time.sleep(1)
EchoOperation.check_resp('hello, fabregas!', data)
#unauth cal
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='unauth'))
if key_storage:
self.assertEqual(resp.ret_code, RC_PERMISSION_DENIED, resp.ret_message)
else:
self.assertEqual(resp.ret_code, 0, resp.ret_message)
#EchoWithCallOperation
#sync call without binary
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo_with_call', \
parameters={'message': 'hello, fabregas!', 'is_sync_call':True},\
binary_data=RamBasedBinaryData(data, 900000)))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
self.assertEqual(resp.ret_parameters['message'], 'hello, fabregas!')
self.assertEqual(resp.binary_data.data(), data)
#sync call without binary (async in operation)
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo_with_call', \
parameters={'message': 'hello, fabregas!', 'is_sync_call':False}))
self.assertEqual(resp.ret_code, 0, resp.ret_message)
time.sleep(1)
EchoOperation.check_resp('hello, fabregas!')
#sync call with binary (async in operation)
resp = fri_client.call_sync('127.0.0.1:6666', FabnetPacketRequest(method='echo_with_call', \
parameters={'message': 'hello, fabregas!', 'is_sync_call':False},\
binary_data=RamBasedBinaryData(data, 900000)))
for i in xrange(10):
if os.path.exists('/tmp/callback.log'):
break
time.sleep(1)
time.sleep(1)
self.assertEqual(resp.ret_code, 0, resp.ret_message)
EchoOperation.check_resp('hello, fabregas!', data)
finally:
fri_server.stop()
operator_proc.stop()
operator_proc.join()
def call_repair_data(address, out_streem, expect_res, invalid_node=None):
if os.path.exists(NDB):
os.remove(NDB)
client = FriClient()
params = {}
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
free_size = dht_info['free_size_percents']
if invalid_node:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(invalid_node, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
start = dht_info['range_start']
end = dht_info['range_end']
params = {'check_range_start': start, 'check_range_end': end}
t0 = datetime.now()
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params)
rcode, rmsg = client.call(address, packet_obj)
if rcode != 0:
raise Exception('RepairDataBlocks does not started. Details: %s'%rmsg)
cnt = 0
try_cnt = 0
while cnt != expect_res:
if try_cnt == 100:
print 'Notifications count: %s, but expected: %s'%(cnt, expect_res)
try_cnt = 0
try_cnt += 1
db = SafeJsonFile(NDB)
data = db.read()
events = data.get('notifications', [])
events = filter(lambda e: e['event_topic']=='RepairDataBlocks', events)
cnt = len(events)
if cnt == 2:
break
time.sleep(.2)
dt = datetime.now() - t0
db = SafeJsonFile(NDB)
data = db.read()
events = data.get('notifications', [])
events = filter(lambda e: e['event_topic']=='RepairDataBlocks', events)
for event in events:
out_streem.write('[%s][%s][%s] %s\n'%(event['event_provider'], event['notify_dt'], event['event_type'], event['event_message']))
#packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
#client = FriClient()
#ret_packet = client.call_sync(event[0], packet_obj)
#out_streem.write('[%s] stat: %s\n'%(event[0], ret_packet.ret_parameters['methods_stat']))
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
post_free_size = dht_info['free_size_percents']
if post_free_size < free_size:
out_streem.write('INVALID RESTORED RANGE FREE SIZE. BEFORE=%s, AFTER=%s\n'%(free_size, post_free_size))
out_streem.write('Process time: %s\n'%dt)
def test05_repair_data(self):
server = server1 = None
try:
home1 = '/tmp/node_1986_home'
home2 = '/tmp/node_1987_home'
monitor_home = '/tmp/node_monitor_home'
if os.path.exists(home1):
shutil.rmtree(home1)
os.mkdir(home1)
if os.path.exists(home2):
shutil.rmtree(home2)
os.mkdir(home2)
if os.path.exists(monitor_home):
shutil.rmtree(monitor_home)
os.mkdir(monitor_home)
server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99})
server.start()
time.sleep(1)
server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986', config={'MAX_USED_SIZE_PERCENTS': 99})
server1.start()
time.sleep(1)
monitor = TestServerThread(1990, monitor_home, is_monitor=True, neighbour='127.0.0.1:1986', \
config={'db_engine': 'postgresql', 'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB})
monitor.start()
time.sleep(1.5)
self.__wait_oper_status(server, DS_NORMALWORK)
self.__wait_oper_status(server1, DS_NORMALWORK)
data = 'Hello, fabregas!'*10
data_key = server.put(data)
data = 'This is replica data!'*10
data_key2 = server1.put(data)
conn = DBConnection("dbname=%s user=postgres"%MONITOR_DB)
conn.execute('DELETE FROM notification')
time.sleep(.2)
client = FriClient()
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={})
rcode, rmsg = client.call('127.0.0.1:1986', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(1.5)
events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification')
conn.execute('DELETE FROM notification')
stat = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=0, failed_repair_foreign_blocks=0'
self.assertEqual(len(events), 2)
event86 = event87 = None
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event86[0], ET_INFO)
self.assertEqual(event86[1], '127.0.0.1:1986', events)
cnt86 = len(os.listdir(server.get_range_dir())) + len(os.listdir(server.get_replicas_dir()))
self.assertTrue(stat%cnt86 in event86[2], event86[2])
self.assertEqual(event87[0], ET_INFO)
self.assertEqual(event87[1], '127.0.0.1:1987')
cnt87 = len(os.listdir(server1.get_range_dir())) + len(os.listdir(server1.get_replicas_dir()))
self.assertTrue(stat%cnt87 in event87[2], event87[2])
node86_stat = server.get_stat()
server.stop()
server.join()
server = None
time.sleep(1)
params = {'check_range_start': node86_stat['DHTInfo']['range_start'], 'check_range_end': node86_stat['DHTInfo']['range_end']}
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params)
rcode, rmsg = client.call('127.0.0.1:1987', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(2)
events = conn.select("SELECT notify_type, node_address, notify_msg FROM notification WHERE notify_topic='RepairDataBlocks'")
conn.execute('DELETE FROM notification')
self.assertEqual(len(events), 1, events)
event86 = event87 = None
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event87[0], ET_INFO)
self.assertEqual(event87[1], '127.0.0.1:1987')
stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0'
self.assertTrue(stat_rep%(cnt87, cnt86) in event87[2], event87[2])
open(os.path.join(server1.get_range_dir(), data_key), 'wr').write('wrong data')
open(os.path.join(server1.get_range_dir(), data_key2), 'ar').write('wrong data')
time.sleep(.2)
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={})
rcode, rmsg = client.call('127.0.0.1:1987', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(2)
events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification')
conn.execute('DELETE FROM notification')
self.assertEqual(len(events), 1)
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event87[1], '127.0.0.1:1987')
stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=%i, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0'
self.assertTrue(stat_rep%(cnt87+cnt86, 1, 2) in event87[2], event87[2])
conn.close()
finally:
if server:
server.stop()
if server1:
server1.stop()
if monitor:
monitor.stop()