def test00_initnet(self): os.system("dropdb -U postgres %s" % MONITOR_DB) os.system("createdb -U postgres %s" % MONITOR_DB) conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB) conn.select("select 1") try: notification_tbl = """CREATE TABLE notification ( id serial PRIMARY KEY, node_address varchar(512) NOT NULL, notify_type varchar(64) NOT NULL, notify_topic varchar(512), notify_msg text, notify_dt timestamp )""" nodes_info_tbl = """CREATE TABLE nodes_info ( id serial PRIMARY KEY, node_address varchar(512) UNIQUE NOT NULL, node_name varchar(128) NOT NULL, node_type varchar(128), home_dir varchar(1024), status integer NOT NULL DEFAULT 0, superiors text, uppers text, statistic text, last_check timestamp )""" try: conn.select("SELECT id FROM nodes_info WHERE id=1") except DBOperationalException: conn.execute(nodes_info_tbl) try: conn.select("SELECT id FROM notification WHERE id=1") except DBOperationalException: conn.execute(notification_tbl) except Exception, err: raise err
def call_repair_data(address, out_streem, expect_res, invalid_node=None): dbconn = DBConnection("dbname=%s user=postgres"%MONITOR_DB) dbconn.execute("DELETE FROM notification") client = FriClient() params = {} packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = client.call_sync(address, packet_obj) dht_info = ret_packet.ret_parameters['DHTInfo'] free_size = dht_info['free_size_percents'] if invalid_node: packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = client.call_sync(invalid_node, packet_obj) dht_info = ret_packet.ret_parameters['DHTInfo'] start = dht_info['range_start'] end = dht_info['range_end'] params = {'check_range_start': start, 'check_range_end': end} t0 = datetime.now() packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params) rcode, rmsg = client.call(address, packet_obj) if rcode != 0: raise Exception('RepairDataBlocks does not started. Details: %s'%rmsg) cnt = 0 try_cnt = 0 while cnt != expect_res: if try_cnt == 100: print 'Notifications count: %s, but expected: %s'%(cnt, expect_res) try_cnt = 0 try_cnt += 1 try: cnt = dbconn.select_one("SELECT count(*) FROM notification WHERE notify_topic='RepairDataBlocks'") except Exception, err: print 'DB ERROR: %s'%err time.sleep(.2)
class PostgresDBAPI(AbstractDBAPI): def __init__(self, connect_string): self._conn = DBConnection(connect_string) def get_nodes_list(self, status): try: return self._conn.select_col("SELECT node_address FROM nodes_info WHERE status=%s", (status,)) except DBEmptyResult: return [] def change_node_status(self, nodeaddr, status): rows = self._conn.select("SELECT id FROM nodes_info WHERE node_address=%s", (nodeaddr, )) if not rows: self._conn.execute("INSERT INTO nodes_info (node_address, node_name, status) VALUES (%s, %s, %s)", (nodeaddr, '', status)) else: self._conn.execute("UPDATE nodes_info SET status=%s WHERE id=%s", (status, rows[0][0])) def update_node_info(self, nodeaddr, node_name, home_dir, node_type, superior_neighbours, upper_neighbours): superiors = ','.join(superior_neighbours) uppers = ','.join(upper_neighbours) rows = self._conn.select("SELECT id, node_name, home_dir, node_type, status, superiors, uppers \ FROM nodes_info WHERE node_address=%s", (nodeaddr, )) if not rows: self._conn.execute("INSERT INTO nodes_info (node_address, node_name, home_dir, node_type, status, superiors, uppers) \ VALUES (%s, %s, %s, %s, %s, %s, %s)", (nodeaddr, node_name, home_dir, node_type, UP, superiors, uppers)) else: if rows[0][1:] == (node_name, home_dir, node_type, UP, superiors, uppers): return self._conn.execute("UPDATE nodes_info \ SET node_name=%s, home_dir=%s, node_type=%s, status=%s, superiors=%s, uppers=%s \ WHERE id=%s", \ (node_name, home_dir, node_type, UP, superiors, uppers, rows[0][0])) def update_node_stat(self, nodeaddr, stat): self._conn.execute("UPDATE nodes_info SET status=%s, statistic=%s, last_check=%s \ WHERE node_address=%s", (UP, stat, datetime.now(), nodeaddr)) def notification(self, notify_provider, notify_type, notify_topic, message, date): self._conn.execute("INSERT INTO notification (node_address, notify_type, notify_topic, \ notify_msg, notify_dt) VALUES (%s, %s, %s, %s, %s)", \ (notify_provider, notify_type, notify_topic, message, date)) def close(self): self._conn.close()
def and_clear_monitoring_stat(step): conn = DBConnection("dbname=%s user=postgres" % MONITOR_DB) conn.execute("DELETE FROM notification") conn.execute("DELETE FROM nodes_info")
def test05_repair_data(self): server = server1 = None try: home1 = '/tmp/node_1986_home' home2 = '/tmp/node_1987_home' monitor_home = '/tmp/node_monitor_home' if os.path.exists(home1): shutil.rmtree(home1) os.mkdir(home1) if os.path.exists(home2): shutil.rmtree(home2) os.mkdir(home2) if os.path.exists(monitor_home): shutil.rmtree(monitor_home) os.mkdir(monitor_home) server = TestServerThread(1986, home1, config={'MAX_USED_SIZE_PERCENTS': 99}) server.start() time.sleep(1) server1 = TestServerThread(1987, home2, neighbour='127.0.0.1:1986', config={'MAX_USED_SIZE_PERCENTS': 99}) server1.start() time.sleep(1) monitor = TestServerThread(1990, monitor_home, is_monitor=True, neighbour='127.0.0.1:1986', \ config={'db_engine': 'postgresql', 'db_conn_str': "dbname=%s user=postgres"%MONITOR_DB}) monitor.start() time.sleep(1.5) self.__wait_oper_status(server, DS_NORMALWORK) self.__wait_oper_status(server1, DS_NORMALWORK) data = 'Hello, fabregas!'*10 data_key = server.put(data) data = 'This is replica data!'*10 data_key2 = server1.put(data) conn = DBConnection("dbname=%s user=postgres"%MONITOR_DB) conn.execute('DELETE FROM notification') time.sleep(.2) client = FriClient() packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={}) rcode, rmsg = client.call('127.0.0.1:1986', packet_obj) self.assertEqual(rcode, 0, rmsg) time.sleep(1.5) events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification') conn.execute('DELETE FROM notification') stat = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=0, failed_repair_foreign_blocks=0' self.assertEqual(len(events), 2) event86 = event87 = None for event in events: if event[1] == '127.0.0.1:1986': event86 = event elif event[1] == '127.0.0.1:1987': event87 = event self.assertEqual(event86[0], ET_INFO) self.assertEqual(event86[1], '127.0.0.1:1986', events) cnt86 = len(os.listdir(server.get_range_dir())) + len(os.listdir(server.get_replicas_dir())) self.assertTrue(stat%cnt86 in event86[2], event86[2]) self.assertEqual(event87[0], ET_INFO) self.assertEqual(event87[1], '127.0.0.1:1987') cnt87 = len(os.listdir(server1.get_range_dir())) + len(os.listdir(server1.get_replicas_dir())) self.assertTrue(stat%cnt87 in event87[2], event87[2]) node86_stat = server.get_stat() server.stop() server.join() server = None time.sleep(1) params = {'check_range_start': node86_stat['DHTInfo']['range_start'], 'check_range_end': node86_stat['DHTInfo']['range_end']} packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters=params) rcode, rmsg = client.call('127.0.0.1:1987', packet_obj) self.assertEqual(rcode, 0, rmsg) time.sleep(2) events = conn.select("SELECT notify_type, node_address, notify_msg FROM notification WHERE notify_topic='RepairDataBlocks'") conn.execute('DELETE FROM notification') self.assertEqual(len(events), 1, events) event86 = event87 = None for event in events: if event[1] == '127.0.0.1:1986': event86 = event elif event[1] == '127.0.0.1:1987': event87 = event self.assertEqual(event87[0], ET_INFO) self.assertEqual(event87[1], '127.0.0.1:1987') stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=0, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0' self.assertTrue(stat_rep%(cnt87, cnt86) in event87[2], event87[2]) open(os.path.join(server1.get_range_dir(), data_key), 'wr').write('wrong data') open(os.path.join(server1.get_range_dir(), data_key2), 'ar').write('wrong data') time.sleep(.2) packet_obj = FabnetPacketRequest(method='RepairDataBlocks', is_multicast=True, parameters={}) rcode, rmsg = client.call('127.0.0.1:1987', packet_obj) self.assertEqual(rcode, 0, rmsg) time.sleep(2) events = conn.select('SELECT notify_type, node_address, notify_msg FROM notification') conn.execute('DELETE FROM notification') self.assertEqual(len(events), 1) for event in events: if event[1] == '127.0.0.1:1986': event86 = event elif event[1] == '127.0.0.1:1987': event87 = event self.assertEqual(event87[1], '127.0.0.1:1987') stat_rep = 'processed_local_blocks=%i, invalid_local_blocks=%i, repaired_foreign_blocks=%i, failed_repair_foreign_blocks=0' self.assertTrue(stat_rep%(cnt87+cnt86, 1, 2) in event87[2], event87[2]) conn.close() finally: if server: server.stop() if server1: server1.stop() if monitor: monitor.stop()