def test01_monitor(self):
qeury_notify = "SELECT notify_type, node_address, notify_msg FROM notification WHERE notify_topic=%s"
qeury_nodes = "SELECT node_address, node_name, status, superiors, uppers, statistic, last_check FROM nodes_info"
conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB)
events = conn.select(qeury_notify, ("NodeUp",))
self.assertEqual(len(events), 3, events)
nodes_info = conn.select(qeury_nodes)
self.assertEqual(len(nodes_info), 3, nodes_info)
self.assertTrue(len(nodes_info[0][0]) > 0, nodes_info[0][0])
self.assertTrue(len(nodes_info[0][1]) == 0, nodes_info[0][1])
self.assertTrue(nodes_info[0][2] == 1, nodes_info[0][2])
self.assertTrue(nodes_info[0][3] == None, nodes_info[0][3])
self.assertTrue(nodes_info[0][4] == None, nodes_info[0][4])
self.assertTrue(nodes_info[0][5] == None, nodes_info[0][5])
self.assertTrue(nodes_info[0][6] == None, nodes_info[0][6])
p = subprocess.Popen(
["/usr/bin/python", "./fabnet/bin/fri-caller", "TopologyCognition", ADDRESSES[0], "{}", "async"]
)
time.sleep(2)
nodes_info = conn.select(qeury_nodes)
self.assertEqual(len(nodes_info), 4, nodes_info)
nodes_info = conn.select(qeury_nodes + " WHERE node_address=%s", (ADDRESSES[0],))
self.assertEqual(len(nodes_info), 1, nodes_info)
self.assertEqual(nodes_info[0][0], ADDRESSES[0])
self.assertEqual(nodes_info[0][1], "1900")
self.assertTrue(nodes_info[0][2] == 1, nodes_info[0][2])
self.assertTrue(len(nodes_info[0][3]) > 0, nodes_info[0][3])
self.assertTrue(len(nodes_info[0][4]) > 0, nodes_info[0][4])
self.assertTrue(nodes_info[0][5] == None, nodes_info[0][5])
self.assertTrue(nodes_info[0][6] == None, nodes_info[0][6])
def then_see_collected_stats_for_all_nodes(step):
conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB)
qeury_nodes = "SELECT node_address, node_name, status, superiors, uppers, statistic, last_check FROM nodes_info"
nodes = conn.select(qeury_nodes)
conn.close()
nodes_count = len(world.processes) + 1
if len(nodes) != nodes_count:
raise Exception("Expected %i nodes in nodes_info table. But %i occured!" % (nodes_count, len(nodes)))
for node in nodes:
for field in node:
if not field:
raise Exception("Invalid node info found: %s" % str(node))
def call_repair_data(address, out_streem, expect_res, invalid_node=None):
dbconn = DBConnection("dbname=%s user=postgres"%MONITOR_DB)
dbconn.execute("DELETE FROM notification")
client = FriClient()
params = {}
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(address, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
free_size = dht_info['free_size_percents']
if invalid_node:
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = client.call_sync(invalid_node, packet_obj)
dht_info = ret_packet.ret_parameters['DHTInfo']
start = dht_info['range_start']
end = dht_info['range_end']
params = {'check_range_start': start, 'check_range_end': end}
t0 = datetime.now()
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params)
rcode, rmsg = client.call(address, packet_obj)
if rcode != 0:
raise Exception('RepairDataBlocks does not started. Details: %s'%rmsg)
cnt = 0
try_cnt = 0
while cnt != expect_res:
if try_cnt == 100:
print 'Notifications count: %s, but expected: %s'%(cnt, expect_res)
try_cnt = 0
try_cnt += 1
try:
cnt = dbconn.select_one("SELECT count(*) FROM notification WHERE notify_topic='RepairDataBlocks'")
except Exception, err:
print 'DB ERROR: %s'%err
time.sleep(.2)
def __init__(self, connect_string):
self._conn = DBConnection(connect_string)
class PostgresDBAPI(AbstractDBAPI):
def __init__(self, connect_string):
self._conn = DBConnection(connect_string)
def get_nodes_list(self, status):
try:
return self._conn.select_col("SELECT node_address FROM nodes_info WHERE status=%s", (status,))
except DBEmptyResult:
return []
def change_node_status(self, nodeaddr, status):
rows = self._conn.select("SELECT id FROM nodes_info WHERE node_address=%s", (nodeaddr, ))
if not rows:
self._conn.execute("INSERT INTO nodes_info (node_address, node_name, status) VALUES (%s, %s, %s)", (nodeaddr, '', status))
else:
self._conn.execute("UPDATE nodes_info SET status=%s WHERE id=%s", (status, rows[0][0]))
def update_node_info(self, nodeaddr, node_name, home_dir, node_type, superior_neighbours, upper_neighbours):
superiors = ','.join(superior_neighbours)
uppers = ','.join(upper_neighbours)
rows = self._conn.select("SELECT id, node_name, home_dir, node_type, status, superiors, uppers \
FROM nodes_info WHERE node_address=%s", (nodeaddr, ))
if not rows:
self._conn.execute("INSERT INTO nodes_info (node_address, node_name, home_dir, node_type, status, superiors, uppers) \
VALUES (%s, %s, %s, %s, %s, %s, %s)", (nodeaddr, node_name, home_dir, node_type, UP, superiors, uppers))
else:
if rows[0][1:] == (node_name, home_dir, node_type, UP, superiors, uppers):
return
self._conn.execute("UPDATE nodes_info \
SET node_name=%s, home_dir=%s, node_type=%s, status=%s, superiors=%s, uppers=%s \
WHERE id=%s", \
(node_name, home_dir, node_type, UP, superiors, uppers, rows[0][0]))
def update_node_stat(self, nodeaddr, stat):
self._conn.execute("UPDATE nodes_info SET status=%s, statistic=%s, last_check=%s \
WHERE node_address=%s", (UP, stat, datetime.now(), nodeaddr))
def notification(self, notify_provider, notify_type, notify_topic, message, date):
self._conn.execute("INSERT INTO notification (node_address, notify_type, notify_topic, \
notify_msg, notify_dt) VALUES (%s, %s, %s, %s, %s)", \
(notify_provider, notify_type, notify_topic, message, date))
def close(self):
self._conn.close()
def test00_initnet(self):
os.system("dropdb -U postgres %s" % MONITOR_DB)
os.system("createdb -U postgres %s" % MONITOR_DB)
conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB)
conn.select("select 1")
try:
notification_tbl = """CREATE TABLE notification (
id serial PRIMARY KEY,
node_address varchar(512) NOT NULL,
notify_type varchar(64) NOT NULL,
notify_topic varchar(512),
notify_msg text,
notify_dt timestamp
)"""
nodes_info_tbl = """CREATE TABLE nodes_info (
id serial PRIMARY KEY,
node_address varchar(512) UNIQUE NOT NULL,
node_name varchar(128) NOT NULL,
node_type varchar(128),
home_dir varchar(1024),
status integer NOT NULL DEFAULT 0,
superiors text,
uppers text,
statistic text,
last_check timestamp
)"""
try:
conn.select("SELECT id FROM nodes_info WHERE id=1")
except DBOperationalException:
conn.execute(nodes_info_tbl)
try:
conn.select("SELECT id FROM notification WHERE id=1")
except DBOperationalException:
conn.execute(notification_tbl)
except Exception, err:
raise err
def and_clear_monitoring_stat(step):
conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB)
conn.execute("DELETE FROM notification")
conn.execute("DELETE FROM nodes_info")
def test05_repair_data(self):
server = server1 = None
try:
home1 = '/tmp/node_1986_home'
home2 = '/tmp/node_1987_home'
monitor_home = '/tmp/node_monitor_home'
if os.path.exists(home1):
shutil.rmtree(home1)
os.mkdir(home1)
if os.path.exists(home2):
shutil.rmtree(home2)
os.mkdir(home2)
if os.path.exists(monitor_home):
shutil.rmtree(monitor_home)
os.mkdir(monitor_home)
server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99})
server.start()
time.sleep(1)
server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986', config={'MAX_USED_SIZE_PERCENTS': 99})
server1.start()
time.sleep(1)
monitor = TestServerThread(1990, monitor_home, is_monitor=True, neighbour='127.0.0.1:1986', \
config={'db_engine': 'postgresql', 'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB})
monitor.start()
time.sleep(1.5)
self.__wait_oper_status(server, DS_NORMALWORK)
self.__wait_oper_status(server1, DS_NORMALWORK)
data = 'Hello, fabregas!'*10
data_key = server.put(data)
data = 'This is replica data!'*10
data_key2 = server1.put(data)
conn = DBConnection("dbname=%s user=postgres"%MONITOR_DB)
conn.execute('DELETE FROM notification')
time.sleep(.2)
client = FriClient()
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={})
rcode, rmsg = client.call('127.0.0.1:1986', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(1.5)
events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification')
conn.execute('DELETE FROM notification')
stat = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=0, failed_repair_foreign_blocks=0'
self.assertEqual(len(events), 2)
event86 = event87 = None
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event86[0], ET_INFO)
self.assertEqual(event86[1], '127.0.0.1:1986', events)
cnt86 = len(os.listdir(server.get_range_dir())) + len(os.listdir(server.get_replicas_dir()))
self.assertTrue(stat%cnt86 in event86[2], event86[2])
self.assertEqual(event87[0], ET_INFO)
self.assertEqual(event87[1], '127.0.0.1:1987')
cnt87 = len(os.listdir(server1.get_range_dir())) + len(os.listdir(server1.get_replicas_dir()))
self.assertTrue(stat%cnt87 in event87[2], event87[2])
node86_stat = server.get_stat()
server.stop()
server.join()
server = None
time.sleep(1)
params = {'check_range_start': node86_stat['DHTInfo']['range_start'], 'check_range_end': node86_stat['DHTInfo']['range_end']}
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params)
rcode, rmsg = client.call('127.0.0.1:1987', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(2)
events = conn.select("SELECT notify_type, node_address, notify_msg FROM notification WHERE notify_topic='RepairDataBlocks'")
conn.execute('DELETE FROM notification')
self.assertEqual(len(events), 1, events)
event86 = event87 = None
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event87[0], ET_INFO)
self.assertEqual(event87[1], '127.0.0.1:1987')
stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0'
self.assertTrue(stat_rep%(cnt87, cnt86) in event87[2], event87[2])
open(os.path.join(server1.get_range_dir(), data_key), 'wr').write('wrong data')
open(os.path.join(server1.get_range_dir(), data_key2), 'ar').write('wrong data')
time.sleep(.2)
packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={})
rcode, rmsg = client.call('127.0.0.1:1987', packet_obj)
self.assertEqual(rcode, 0, rmsg)
time.sleep(2)
events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification')
conn.execute('DELETE FROM notification')
self.assertEqual(len(events), 1)
for event in events:
if event[1] == '127.0.0.1:1986':
event86 = event
elif event[1] == '127.0.0.1:1987':
event87 = event
self.assertEqual(event87[1], '127.0.0.1:1987')
stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=%i, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0'
self.assertTrue(stat_rep%(cnt87+cnt86, 1, 2) in event87[2], event87[2])
conn.close()
finally:
if server:
server.stop()
if server1:
server1.stop()
if monitor:
monitor.stop()
